@syncular/server 0.32.1 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob-handlers.js +3 -3
- package/dist/bucket-blob-store.d.ts +77 -0
- package/dist/bucket-blob-store.js +266 -0
- package/dist/bucket-segment-store.d.ts +77 -0
- package/dist/bucket-segment-store.js +445 -0
- package/dist/context.d.ts +7 -0
- package/dist/d1-storage.js +2 -2
- package/dist/handler.js +4 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/object-bucket.d.ts +78 -0
- package/dist/object-bucket.js +18 -0
- package/dist/operations-realtime.js +3 -0
- package/dist/operations.d.ts +20 -3
- package/dist/operations.js +23 -3
- package/dist/realtime.d.ts +11 -0
- package/dist/realtime.js +25 -58
- package/dist/relational-rows.d.ts +6 -3
- package/dist/relational-rows.js +8 -9
- package/dist/s3-blob-store.d.ts +9 -64
- package/dist/s3-blob-store.js +25 -359
- package/dist/s3-object-bucket.d.ts +39 -0
- package/dist/s3-object-bucket.js +164 -0
- package/dist/s3-segment-store.d.ts +9 -61
- package/dist/s3-segment-store.js +25 -503
- package/dist/sqlite-storage.js +3 -8
- package/package.json +2 -2
- package/src/blob-handlers.ts +4 -4
- package/src/bucket-blob-store.ts +351 -0
- package/src/bucket-segment-store.ts +582 -0
- package/src/context.ts +10 -0
- package/src/d1-storage.ts +3 -3
- package/src/handler.ts +11 -0
- package/src/index.ts +4 -0
- package/src/object-bucket.ts +83 -0
- package/src/operations-realtime.ts +7 -0
- package/src/operations.ts +52 -5
- package/src/realtime.ts +29 -58
- package/src/relational-rows.ts +8 -11
- package/src/s3-blob-store.ts +25 -466
- package/src/s3-object-bucket.ts +233 -0
- package/src/s3-segment-store.ts +25 -646
- package/src/sqlite-storage.ts +3 -10
package/src/s3-segment-store.ts
CHANGED
|
@@ -1,72 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* S3-compatible segment store (SPEC.md §5.1 cache semantics, §5.4
|
|
3
|
-
* delegated presign) — AWS S3, Cloudflare R2, MinIO. Zero
|
|
4
|
-
* hand-rolled SigV4 over `fetch` (sigv4.ts).
|
|
3
|
+
* delegated presign) — AWS S3, Cloudflare R2's S3 endpoint, MinIO. Zero
|
|
4
|
+
* dependencies: hand-rolled SigV4 over `fetch` (sigv4.ts).
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* directly and the client's §5.1 hash check passes).
|
|
12
|
-
*
|
|
13
|
-
* {keyPrefix}rec/sha256/{hex}.json
|
|
14
|
-
* The `SegmentRecord` (§5.1). It is the one MUTABLE object of the two:
|
|
15
|
-
* the scope-digest set is a union that every publication extends, so
|
|
16
|
-
* the merge is a read-modify-write. Keeping the record in the body (not
|
|
17
|
-
* in the bytes object's user metadata) is what makes that write
|
|
18
|
-
* conditional — a body ETag changes when the union changes, while the
|
|
19
|
-
* bytes ETag is a content hash and is identical for identical bytes. A
|
|
20
|
-
* `412` re-reads and merges again, so no concurrent publication can drop
|
|
21
|
-
* another one's digest. `get` therefore reads two objects.
|
|
22
|
-
*
|
|
23
|
-
* {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
|
|
24
|
-
* Whole-table reuse pointer (§5.3): written only when
|
|
25
|
-
* `rowCursor === null`, body = the record JSON. The reuse key is the
|
|
26
|
-
* canonical JSON array `[partition, table, schemaVersion, mediaType,
|
|
27
|
-
* scopeDigest, asOfCommitSeq]`, so `find` is one GET (plus a HEAD to
|
|
28
|
-
* confirm the segment object itself still exists — lifecycle GC may
|
|
29
|
-
* remove objects independently of pointers).
|
|
30
|
-
*
|
|
31
|
-
* Records written before the split kept the record in the bytes object's
|
|
32
|
-
* user metadata `x-amz-meta-syncular-record`; `get` still reads that shape
|
|
33
|
-
* and the next `put` materializes the record object from it, so an upgrade
|
|
34
|
-
* loses no cache entry.
|
|
35
|
-
*
|
|
36
|
-
* TTL mapping: expiry is **store-side and authoritative** — `expiresAtMs`
|
|
37
|
-
* (put-time + `ttlMs`, default 24 h) is recorded in the record and the
|
|
38
|
-
* pointer; `get` returns expired records so §5.5 can answer
|
|
39
|
-
* `sync.segment_expired`, and `find` filters them out itself. S3 lifecycle
|
|
40
|
-
* expiration is garbage collection only: configure it comfortably ABOVE
|
|
41
|
-
* `ttlMs` (e.g. 2 days for the 24 h default) so clients normally see the
|
|
42
|
-
* precise, retryable `sync.segment_expired` and hit `sync.not_found` only
|
|
43
|
-
* long after. Never set lifecycle below `ttlMs`.
|
|
6
|
+
* The store logic (key layout, record CAS, reuse pointers, stats
|
|
7
|
+
* accumulator) is `BucketSegmentStore`, shared with `R2SegmentStore` in
|
|
8
|
+
* `@syncular/server-workers`; this class adds the S3 REST transport
|
|
9
|
+
* (`S3ObjectBucket`) and the SigV4 presign methods, which a bucket binding
|
|
10
|
+
* cannot mint.
|
|
44
11
|
*/
|
|
45
|
-
import type {
|
|
46
|
-
SegmentFindKey,
|
|
47
|
-
SegmentMetadata,
|
|
48
|
-
SegmentPublication,
|
|
49
|
-
SegmentRecord,
|
|
50
|
-
SegmentStore,
|
|
51
|
-
SegmentStoreStats,
|
|
52
|
-
SegmentStream,
|
|
53
|
-
} from './segment-store';
|
|
54
12
|
import {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
segmentIdFor,
|
|
60
|
-
selectSegmentPublication,
|
|
61
|
-
} from './segment-store';
|
|
13
|
+
BucketSegmentStore,
|
|
14
|
+
type BucketSegmentStoreOptions,
|
|
15
|
+
} from './bucket-segment-store';
|
|
16
|
+
import { S3ObjectBucket } from './s3-object-bucket';
|
|
62
17
|
import type { DelegatedPresignConfig, SegmentUrlIssue } from './signed-url';
|
|
63
|
-
import {
|
|
64
|
-
EMPTY_PAYLOAD_SHA256,
|
|
65
|
-
presignUrl,
|
|
66
|
-
type SigV4Credentials,
|
|
67
|
-
sha256Hex,
|
|
68
|
-
signRequest,
|
|
69
|
-
} from './sigv4';
|
|
18
|
+
import { presignUrl, type SigV4Credentials } from './sigv4';
|
|
70
19
|
|
|
71
20
|
export interface S3SegmentStoreConfig {
|
|
72
21
|
/**
|
|
@@ -90,223 +39,22 @@ export interface S3SegmentStoreConfig {
|
|
|
90
39
|
readonly ttlMs?: number;
|
|
91
40
|
}
|
|
92
41
|
|
|
93
|
-
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
94
42
|
const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
95
|
-
const RECORD_META_HEADER = 'x-amz-meta-syncular-record';
|
|
96
|
-
|
|
97
|
-
/**
|
|
98
|
-
* Stats accumulator (§ "S3 stats" in the README). The store keeps a small
|
|
99
|
-
* counter object under a fixed key, updated read-modify-write on `put`. It
|
|
100
|
-
* exists because a LIST-free store (every lookup is a GET/HEAD) has no cheap
|
|
101
|
-
* way to count objects on demand — so `stats()` reads the accumulator instead
|
|
102
|
-
* of enumerating the bucket.
|
|
103
|
-
*
|
|
104
|
-
* The write is guarded by an ETag compare-and-swap (`If-Match`, or
|
|
105
|
-
* `If-None-Match: *` to create): the loop re-reads and retries on a `412`
|
|
106
|
-
* precondition failure, so concurrent writers do not silently clobber each
|
|
107
|
-
* other's increment. The counters are still APPROXIMATE — a crash between the
|
|
108
|
-
* segment PUT and the accumulator CAS, or a writer against a backend that
|
|
109
|
-
* ignores conditional headers, can drift them — so the admin surface labels
|
|
110
|
-
* S3 stats `approximate: true`. They are a health gauge, not an invoice.
|
|
111
|
-
*/
|
|
112
|
-
const STATS_KEY_SUFFIX = 'stats/segments.json';
|
|
113
|
-
const CAS_ATTEMPTS = 16;
|
|
114
|
-
|
|
115
|
-
interface StatsAccumulator {
|
|
116
|
-
count: number;
|
|
117
|
-
bytes: number;
|
|
118
|
-
rowsSegments: number;
|
|
119
|
-
sqliteSegments: number;
|
|
120
|
-
}
|
|
121
43
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
function parseStatsJson(json: string): StatsAccumulator {
|
|
127
|
-
const parsed = JSON.parse(json) as Partial<StatsAccumulator>;
|
|
128
|
-
return {
|
|
129
|
-
count: Number(parsed.count ?? 0),
|
|
130
|
-
bytes: Number(parsed.bytes ?? 0),
|
|
131
|
-
rowsSegments: Number(parsed.rowsSegments ?? 0),
|
|
132
|
-
sqliteSegments: Number(parsed.sqliteSegments ?? 0),
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
// Runtime-neutral base64url: `Buffer` is not present on
|
|
137
|
-
// Cloudflare Workers without `nodejs_compat`, so a legacy object-metadata
|
|
138
|
-
// record header is decoded with `atob`, available in every runtime.
|
|
139
|
-
function base64urlToUtf8(value: string): string {
|
|
140
|
-
const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/'));
|
|
141
|
-
const bytes = new Uint8Array(binary.length);
|
|
142
|
-
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
143
|
-
return new TextDecoder().decode(bytes);
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
function recordToJson(record: SegmentRecord): string {
|
|
147
|
-
return JSON.stringify(record);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
function isMediaType(value: unknown): value is 'rows' | 'sqlite' {
|
|
151
|
-
return value === 'rows' || value === 'sqlite';
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/** One publication inside a stored record (or a legacy materialized one). */
|
|
155
|
-
function parsePublication(value: unknown, source: string): SegmentPublication {
|
|
156
|
-
if (typeof value !== 'object' || value === null) {
|
|
157
|
-
throw new Error(`S3SegmentStore: corrupt publication in ${source}`);
|
|
158
|
-
}
|
|
159
|
-
const p = value as Record<string, unknown>;
|
|
160
|
-
const strings = ['partition', 'logEpoch', 'table', 'scopeDigest'] as const;
|
|
161
|
-
const numbers = [
|
|
162
|
-
'schemaVersion',
|
|
163
|
-
'asOfCommitSeq',
|
|
164
|
-
'rowCount',
|
|
165
|
-
'byteLength',
|
|
166
|
-
'createdAtMs',
|
|
167
|
-
'expiresAtMs',
|
|
168
|
-
] as const;
|
|
169
|
-
for (const field of strings) {
|
|
170
|
-
if (typeof p[field] !== 'string') {
|
|
171
|
-
throw new Error(
|
|
172
|
-
`S3SegmentStore: bad publication field ${field} in ${source}`,
|
|
173
|
-
);
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
for (const field of numbers) {
|
|
177
|
-
if (typeof p[field] !== 'number') {
|
|
178
|
-
throw new Error(
|
|
179
|
-
`S3SegmentStore: bad publication field ${field} in ${source}`,
|
|
180
|
-
);
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
if (!isMediaType(p.mediaType)) {
|
|
184
|
-
throw new Error(`S3SegmentStore: bad publication mediaType in ${source}`);
|
|
185
|
-
}
|
|
186
|
-
const cursorOk = (v: unknown): v is string | null =>
|
|
187
|
-
v === null || typeof v === 'string';
|
|
188
|
-
if (!cursorOk(p.rowCursor) || !cursorOk(p.nextRowCursor)) {
|
|
189
|
-
throw new Error(`S3SegmentStore: bad publication cursor in ${source}`);
|
|
190
|
-
}
|
|
191
|
-
return {
|
|
192
|
-
partition: p.partition as string,
|
|
193
|
-
logEpoch: p.logEpoch as string,
|
|
194
|
-
table: p.table as string,
|
|
195
|
-
schemaVersion: p.schemaVersion as number,
|
|
196
|
-
mediaType: p.mediaType,
|
|
197
|
-
scopeDigest: p.scopeDigest as string,
|
|
198
|
-
asOfCommitSeq: p.asOfCommitSeq as number,
|
|
199
|
-
rowCount: p.rowCount as number,
|
|
200
|
-
rowCursor: p.rowCursor,
|
|
201
|
-
nextRowCursor: p.nextRowCursor,
|
|
202
|
-
byteLength: p.byteLength as number,
|
|
203
|
-
createdAtMs: p.createdAtMs as number,
|
|
204
|
-
expiresAtMs: p.expiresAtMs as number,
|
|
205
|
-
};
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
function parseRecordJson(json: string, source: string): SegmentRecord {
|
|
209
|
-
let parsed: unknown;
|
|
210
|
-
try {
|
|
211
|
-
parsed = JSON.parse(json);
|
|
212
|
-
} catch {
|
|
213
|
-
throw new Error(`S3SegmentStore: corrupt record JSON in ${source}`);
|
|
214
|
-
}
|
|
215
|
-
if (typeof parsed !== 'object' || parsed === null) {
|
|
216
|
-
throw new Error(`S3SegmentStore: corrupt record in ${source}`);
|
|
217
|
-
}
|
|
218
|
-
const r = parsed as Record<string, unknown>;
|
|
219
|
-
const strings = [
|
|
220
|
-
'segmentId',
|
|
221
|
-
'partition',
|
|
222
|
-
'logEpoch',
|
|
223
|
-
'table',
|
|
224
|
-
'scopeDigest',
|
|
225
|
-
] as const;
|
|
226
|
-
const numbers = [
|
|
227
|
-
'schemaVersion',
|
|
228
|
-
'asOfCommitSeq',
|
|
229
|
-
'rowCount',
|
|
230
|
-
'byteLength',
|
|
231
|
-
'createdAtMs',
|
|
232
|
-
'expiresAtMs',
|
|
233
|
-
] as const;
|
|
234
|
-
for (const field of strings) {
|
|
235
|
-
if (typeof r[field] !== 'string') {
|
|
236
|
-
throw new Error(`S3SegmentStore: bad record field ${field} in ${source}`);
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
for (const field of numbers) {
|
|
240
|
-
if (typeof r[field] !== 'number') {
|
|
241
|
-
throw new Error(`S3SegmentStore: bad record field ${field} in ${source}`);
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
if (!isMediaType(r.mediaType)) {
|
|
245
|
-
throw new Error(`S3SegmentStore: bad record mediaType in ${source}`);
|
|
246
|
-
}
|
|
247
|
-
const cursorOk = (v: unknown): v is string | null =>
|
|
248
|
-
v === null || typeof v === 'string';
|
|
249
|
-
if (!cursorOk(r.rowCursor) || !cursorOk(r.nextRowCursor)) {
|
|
250
|
-
throw new Error(`S3SegmentStore: bad record cursor in ${source}`);
|
|
251
|
-
}
|
|
252
|
-
const scopeDigest = r.scopeDigest as string;
|
|
253
|
-
// Records written before §5.1 recorded the digest set carry only the
|
|
254
|
-
// primary digest; the object is a cache entry with a TTL, so reading it
|
|
255
|
-
// back as a one-digest record is correct, not a degraded path.
|
|
256
|
-
const rawDigests = r.scopeDigests;
|
|
257
|
-
if (
|
|
258
|
-
rawDigests !== undefined &&
|
|
259
|
-
(!Array.isArray(rawDigests) ||
|
|
260
|
-
rawDigests.some((digest) => typeof digest !== 'string'))
|
|
261
|
-
) {
|
|
262
|
-
throw new Error(`S3SegmentStore: bad record scopeDigests in ${source}`);
|
|
263
|
-
}
|
|
264
|
-
const rawPublications = r.publications;
|
|
265
|
-
if (rawPublications !== undefined && !Array.isArray(rawPublications)) {
|
|
266
|
-
throw new Error(`S3SegmentStore: bad record publications in ${source}`);
|
|
267
|
-
}
|
|
268
|
-
const top = {
|
|
269
|
-
segmentId: r.segmentId as string,
|
|
270
|
-
partition: r.partition as string,
|
|
271
|
-
logEpoch: r.logEpoch as string,
|
|
272
|
-
table: r.table as string,
|
|
273
|
-
schemaVersion: r.schemaVersion as number,
|
|
274
|
-
mediaType: r.mediaType,
|
|
275
|
-
scopeDigest,
|
|
276
|
-
scopeDigests: [
|
|
277
|
-
scopeDigest,
|
|
278
|
-
...((rawDigests ?? []) as string[]).filter(
|
|
279
|
-
(digest) => digest !== scopeDigest,
|
|
280
|
-
),
|
|
281
|
-
],
|
|
282
|
-
asOfCommitSeq: r.asOfCommitSeq as number,
|
|
283
|
-
rowCount: r.rowCount as number,
|
|
284
|
-
rowCursor: r.rowCursor,
|
|
285
|
-
nextRowCursor: r.nextRowCursor,
|
|
286
|
-
byteLength: r.byteLength as number,
|
|
287
|
-
createdAtMs: r.createdAtMs as number,
|
|
288
|
-
expiresAtMs: r.expiresAtMs as number,
|
|
289
|
-
};
|
|
290
|
-
return {
|
|
291
|
-
...top,
|
|
292
|
-
// A record from before per-publication provenance, or a legacy record
|
|
293
|
-
// whose union came from object metadata, materializes its publications
|
|
294
|
-
// from the merged top-level context — the old single-record read.
|
|
295
|
-
publications:
|
|
296
|
-
rawPublications === undefined
|
|
297
|
-
? publicationsForRecord(top)
|
|
298
|
-
: rawPublications.map((value) => parsePublication(value, source)),
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
export class S3SegmentStore implements SegmentStore {
|
|
303
|
-
readonly #config: S3SegmentStoreConfig;
|
|
44
|
+
export class S3SegmentStore extends BucketSegmentStore {
|
|
45
|
+
readonly #bucket: S3ObjectBucket;
|
|
46
|
+
readonly #region: string;
|
|
304
47
|
readonly #credentials: SigV4Credentials;
|
|
305
|
-
readonly #ttlMs: number;
|
|
306
|
-
readonly #prefix: string;
|
|
307
48
|
|
|
308
49
|
constructor(config: S3SegmentStoreConfig) {
|
|
309
|
-
|
|
50
|
+
const options: BucketSegmentStoreOptions = {
|
|
51
|
+
keyPrefix: config.keyPrefix,
|
|
52
|
+
ttlMs: config.ttlMs,
|
|
53
|
+
};
|
|
54
|
+
const bucket = new S3ObjectBucket(config);
|
|
55
|
+
super(bucket, options);
|
|
56
|
+
this.#bucket = bucket;
|
|
57
|
+
this.#region = config.region;
|
|
310
58
|
this.#credentials = {
|
|
311
59
|
accessKeyId: config.accessKeyId,
|
|
312
60
|
secretAccessKey: config.secretAccessKey,
|
|
@@ -314,375 +62,6 @@ export class S3SegmentStore implements SegmentStore {
|
|
|
314
62
|
? { sessionToken: config.sessionToken }
|
|
315
63
|
: {}),
|
|
316
64
|
};
|
|
317
|
-
this.#ttlMs = config.ttlMs ?? DEFAULT_TTL_MS;
|
|
318
|
-
this.#prefix = config.keyPrefix ?? '';
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
/** `{keyPrefix}seg/sha256/{hex}` — the §5.4 "key embeds the segmentId". */
|
|
322
|
-
objectKeyFor(segmentId: string): string {
|
|
323
|
-
return `${this.#prefix}seg/${segmentId.replace(':', '/')}`;
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
/** `{keyPrefix}rec/sha256/{hex}.json` — the §5.1 record (§5.1 union). */
|
|
327
|
-
#recordKeyFor(segmentId: string): string {
|
|
328
|
-
return `${this.#prefix}rec/${segmentId.replace(':', '/')}.json`;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
async #findKeyFor(key: SegmentFindKey): Promise<string> {
|
|
332
|
-
const canonical = JSON.stringify([
|
|
333
|
-
key.partition,
|
|
334
|
-
key.logEpoch,
|
|
335
|
-
key.table,
|
|
336
|
-
key.schemaVersion,
|
|
337
|
-
key.mediaType,
|
|
338
|
-
key.scopeDigest,
|
|
339
|
-
key.asOfCommitSeq,
|
|
340
|
-
]);
|
|
341
|
-
return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
#statsKey(): string {
|
|
345
|
-
return `${this.#prefix}${STATS_KEY_SUFFIX}`;
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
#urlFor(key: string): URL {
|
|
349
|
-
const endpoint = this.#config.endpoint.replace(/\/+$/, '');
|
|
350
|
-
return new URL(`${endpoint}/${this.#config.bucket}/${key}`);
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
async #request(
|
|
354
|
-
method: 'GET' | 'HEAD' | 'PUT' | 'DELETE',
|
|
355
|
-
key: string,
|
|
356
|
-
options?: {
|
|
357
|
-
readonly body?: Uint8Array;
|
|
358
|
-
readonly headers?: Readonly<Record<string, string>>;
|
|
359
|
-
/** Return a `412` response instead of throwing (conditional writes). */
|
|
360
|
-
readonly allow412?: boolean;
|
|
361
|
-
},
|
|
362
|
-
): Promise<Response | undefined> {
|
|
363
|
-
const url = this.#urlFor(key);
|
|
364
|
-
const body = options?.body;
|
|
365
|
-
const headers = await signRequest({
|
|
366
|
-
method,
|
|
367
|
-
url,
|
|
368
|
-
region: this.#config.region,
|
|
369
|
-
credentials: this.#credentials,
|
|
370
|
-
nowMs: Date.now(),
|
|
371
|
-
payloadHash:
|
|
372
|
-
body === undefined ? EMPTY_PAYLOAD_SHA256 : await sha256Hex(body),
|
|
373
|
-
...(options?.headers !== undefined ? { headers: options.headers } : {}),
|
|
374
|
-
});
|
|
375
|
-
const response = await fetch(url, {
|
|
376
|
-
method,
|
|
377
|
-
headers,
|
|
378
|
-
...(body !== undefined ? { body: body as BodyInit } : {}),
|
|
379
|
-
});
|
|
380
|
-
if (response.status === 404) {
|
|
381
|
-
await response.arrayBuffer();
|
|
382
|
-
return undefined;
|
|
383
|
-
}
|
|
384
|
-
if (options?.allow412 === true && response.status === 412) return response;
|
|
385
|
-
if (!response.ok) {
|
|
386
|
-
const detail = (await response.text()).slice(0, 300);
|
|
387
|
-
throw new Error(
|
|
388
|
-
`S3SegmentStore: ${method} ${key} failed with ${response.status}: ${detail}`,
|
|
389
|
-
);
|
|
390
|
-
}
|
|
391
|
-
return response;
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
/** Read the stats accumulator + its ETag (for the CAS write), or none. */
|
|
395
|
-
async #readStats(): Promise<{
|
|
396
|
-
stats: StatsAccumulator;
|
|
397
|
-
etag: string | undefined;
|
|
398
|
-
}> {
|
|
399
|
-
const response = await this.#request('GET', this.#statsKey());
|
|
400
|
-
if (response === undefined) {
|
|
401
|
-
return { stats: emptyStats(), etag: undefined };
|
|
402
|
-
}
|
|
403
|
-
const etag = response.headers.get('etag') ?? undefined;
|
|
404
|
-
const stats = parseStatsJson(await response.text());
|
|
405
|
-
return { stats, etag };
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
/**
|
|
409
|
-
* Conditionally write the stats accumulator. `etag` present ⇒ `If-Match`
|
|
410
|
-
* (update the object we read); absent ⇒ `If-None-Match: *` (create only).
|
|
411
|
-
* Returns `false` on a `412` precondition failure so the caller retries.
|
|
412
|
-
*/
|
|
413
|
-
async #writeStatsCas(
|
|
414
|
-
stats: StatsAccumulator,
|
|
415
|
-
etag: string | undefined,
|
|
416
|
-
): Promise<boolean> {
|
|
417
|
-
const url = this.#urlFor(this.#statsKey());
|
|
418
|
-
const body = new TextEncoder().encode(JSON.stringify(stats));
|
|
419
|
-
const conditional: Record<string, string> =
|
|
420
|
-
etag === undefined ? { 'if-none-match': '*' } : { 'if-match': etag };
|
|
421
|
-
const headers = await signRequest({
|
|
422
|
-
method: 'PUT',
|
|
423
|
-
url,
|
|
424
|
-
region: this.#config.region,
|
|
425
|
-
credentials: this.#credentials,
|
|
426
|
-
nowMs: Date.now(),
|
|
427
|
-
payloadHash: await sha256Hex(body),
|
|
428
|
-
headers: { 'content-type': 'application/json', ...conditional },
|
|
429
|
-
});
|
|
430
|
-
const response = await fetch(url, {
|
|
431
|
-
method: 'PUT',
|
|
432
|
-
headers,
|
|
433
|
-
body: body as BodyInit,
|
|
434
|
-
});
|
|
435
|
-
if (response.status === 412) {
|
|
436
|
-
await response.arrayBuffer();
|
|
437
|
-
return false;
|
|
438
|
-
}
|
|
439
|
-
if (!response.ok) {
|
|
440
|
-
const detail = (await response.text()).slice(0, 300);
|
|
441
|
-
throw new Error(
|
|
442
|
-
`S3SegmentStore: stats PUT failed with ${response.status}: ${detail}`,
|
|
443
|
-
);
|
|
444
|
-
}
|
|
445
|
-
await response.arrayBuffer();
|
|
446
|
-
return true;
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
/**
|
|
450
|
-
* Fold `delta` into the stats accumulator under an ETag compare-and-swap
|
|
451
|
-
* retry loop. Best-effort: exhausting the retry budget leaves the counters
|
|
452
|
-
* drifted (they are documented APPROXIMATE) rather than failing the `put`
|
|
453
|
-
* that already stored the bytes.
|
|
454
|
-
*/
|
|
455
|
-
async #bumpStats(delta: StatsAccumulator): Promise<void> {
|
|
456
|
-
for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
|
|
457
|
-
const { stats, etag } = await this.#readStats();
|
|
458
|
-
const next: StatsAccumulator = {
|
|
459
|
-
count: stats.count + delta.count,
|
|
460
|
-
bytes: stats.bytes + delta.bytes,
|
|
461
|
-
rowsSegments: stats.rowsSegments + delta.rowsSegments,
|
|
462
|
-
sqliteSegments: stats.sqliteSegments + delta.sqliteSegments,
|
|
463
|
-
};
|
|
464
|
-
if (await this.#writeStatsCas(next, etag)) return;
|
|
465
|
-
}
|
|
466
|
-
// Lost the CAS race `CAS_ATTEMPTS` times running — leave the
|
|
467
|
-
// counters as-is. `stats()` stays APPROXIMATE by contract.
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
async put(
|
|
471
|
-
metadata: SegmentMetadata,
|
|
472
|
-
bytes: Uint8Array,
|
|
473
|
-
nowMs: number,
|
|
474
|
-
): Promise<SegmentRecord> {
|
|
475
|
-
const segmentId = await segmentIdFor(bytes);
|
|
476
|
-
await this.#putBytes(segmentId, bytes);
|
|
477
|
-
for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
|
|
478
|
-
// The id IS the content address, so an existing record means a second
|
|
479
|
-
// scope published the same bytes: read it back and merge the digest set
|
|
480
|
-
// (§5.1).
|
|
481
|
-
const preexisting = await this.#readRecord(segmentId);
|
|
482
|
-
const record = mergeSegmentRecord(
|
|
483
|
-
preexisting?.record,
|
|
484
|
-
metadata,
|
|
485
|
-
segmentId,
|
|
486
|
-
bytes.length,
|
|
487
|
-
nowMs,
|
|
488
|
-
this.#ttlMs,
|
|
489
|
-
);
|
|
490
|
-
const recordJson = recordToJson(record);
|
|
491
|
-
// The digest set is monotone (§5.1) and this is a read-modify-write: a
|
|
492
|
-
// concurrent publisher of the same bytes computed its own union from the
|
|
493
|
-
// same pre-state, so an unconditional PUT would drop one of the digests
|
|
494
|
-
// and make that scope's download `sync.forbidden`. The write is
|
|
495
|
-
// conditional on the record body that was read — the record ETag is the
|
|
496
|
-
// body hash, so it changes whenever the union does — and a `412`
|
|
497
|
-
// re-reads and merges again. In-process dedupe cannot cover this: two
|
|
498
|
-
// scopes whose rows are byte-identical are two builds, and two server
|
|
499
|
-
// instances share one bucket.
|
|
500
|
-
const conditional: Record<string, string> =
|
|
501
|
-
preexisting?.etag === undefined
|
|
502
|
-
? { 'if-none-match': '*' }
|
|
503
|
-
: { 'if-match': preexisting.etag };
|
|
504
|
-
const response = await this.#request(
|
|
505
|
-
'PUT',
|
|
506
|
-
this.#recordKeyFor(segmentId),
|
|
507
|
-
{
|
|
508
|
-
body: new TextEncoder().encode(recordJson),
|
|
509
|
-
headers: { 'content-type': 'application/json', ...conditional },
|
|
510
|
-
allow412: true,
|
|
511
|
-
},
|
|
512
|
-
);
|
|
513
|
-
if (response !== undefined && response.status === 412) {
|
|
514
|
-
await response.arrayBuffer();
|
|
515
|
-
continue;
|
|
516
|
-
}
|
|
517
|
-
await response?.arrayBuffer();
|
|
518
|
-
if (preexisting === undefined) {
|
|
519
|
-
await this.#bumpStats({
|
|
520
|
-
count: 1,
|
|
521
|
-
bytes: bytes.length,
|
|
522
|
-
rowsSegments: metadata.mediaType === 'rows' ? 1 : 0,
|
|
523
|
-
sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
|
|
524
|
-
});
|
|
525
|
-
}
|
|
526
|
-
if (metadata.rowCursor === null) {
|
|
527
|
-
const pointer = await this.#request(
|
|
528
|
-
'PUT',
|
|
529
|
-
await this.#findKeyFor(metadata),
|
|
530
|
-
{
|
|
531
|
-
body: new TextEncoder().encode(recordJson),
|
|
532
|
-
headers: { 'content-type': 'application/json' },
|
|
533
|
-
},
|
|
534
|
-
);
|
|
535
|
-
await pointer?.arrayBuffer();
|
|
536
|
-
}
|
|
537
|
-
return record;
|
|
538
|
-
}
|
|
539
|
-
throw new Error(
|
|
540
|
-
`S3SegmentStore: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`,
|
|
541
|
-
);
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
/**
|
|
545
|
-
* Write the immutable bytes object. Create-only, so a second publisher of
|
|
546
|
-
* identical bytes is a no-op and a second publisher of DIFFERENT bytes under
|
|
547
|
-
* the same content address is the hash collision §5.1 requires to fail
|
|
548
|
-
* loudly instead of overwriting.
|
|
549
|
-
*/
|
|
550
|
-
async #putBytes(segmentId: string, bytes: Uint8Array): Promise<void> {
|
|
551
|
-
const key = this.objectKeyFor(segmentId);
|
|
552
|
-
const response = await this.#request('PUT', key, {
|
|
553
|
-
body: bytes,
|
|
554
|
-
headers: {
|
|
555
|
-
'content-type': 'application/octet-stream',
|
|
556
|
-
'if-none-match': '*',
|
|
557
|
-
},
|
|
558
|
-
allow412: true,
|
|
559
|
-
});
|
|
560
|
-
if (response === undefined || response.status !== 412) {
|
|
561
|
-
await response?.arrayBuffer();
|
|
562
|
-
return;
|
|
563
|
-
}
|
|
564
|
-
await response.arrayBuffer();
|
|
565
|
-
const current = await this.#request('GET', key);
|
|
566
|
-
if (current === undefined) return;
|
|
567
|
-
const stored = new Uint8Array(await current.arrayBuffer());
|
|
568
|
-
if (!segmentBytesEqual(stored, bytes)) {
|
|
569
|
-
throw new Error(
|
|
570
|
-
`S3SegmentStore: content-address collision at ${segmentId} (§5.1)`,
|
|
571
|
-
);
|
|
572
|
-
}
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
/**
|
|
576
|
-
* Read the record object plus the ETag of the body that was read, so the
|
|
577
|
-
* union write can be made conditional on exactly that pre-state (§5.1).
|
|
578
|
-
*
|
|
579
|
-
* A record written before the record/bytes split rides in the bytes object's
|
|
580
|
-
* user metadata; it is read here and its union is materialized into a record
|
|
581
|
-
* object by the next `put`. That shape carries no usable ETag for the union
|
|
582
|
-
* (the bytes ETag is a content hash), which is exactly why it was replaced.
|
|
583
|
-
* A bytes object with neither a record object nor the legacy metadata is an
|
|
584
|
-
* entry whose record is missing — a cache miss, not corruption, so `get`
|
|
585
|
-
* reports it as absent and the client re-pulls.
|
|
586
|
-
*/
|
|
587
|
-
async #readRecord(
|
|
588
|
-
segmentId: string,
|
|
589
|
-
): Promise<{ record: SegmentRecord; etag: string | undefined } | undefined> {
|
|
590
|
-
if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
|
|
591
|
-
const response = await this.#request('GET', this.#recordKeyFor(segmentId));
|
|
592
|
-
if (response !== undefined) {
|
|
593
|
-
return {
|
|
594
|
-
record: parseRecordJson(await response.text(), 'segment record'),
|
|
595
|
-
etag: response.headers.get('etag') ?? undefined,
|
|
596
|
-
};
|
|
597
|
-
}
|
|
598
|
-
const legacy = await this.#request('GET', this.objectKeyFor(segmentId));
|
|
599
|
-
if (legacy === undefined) return undefined;
|
|
600
|
-
const meta = legacy.headers.get(RECORD_META_HEADER);
|
|
601
|
-
await legacy.arrayBuffer();
|
|
602
|
-
if (meta === null) return undefined;
|
|
603
|
-
return {
|
|
604
|
-
record: parseRecordJson(base64urlToUtf8(meta), segmentId),
|
|
605
|
-
etag: undefined,
|
|
606
|
-
};
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
async get(
|
|
610
|
-
segmentId: string,
|
|
611
|
-
): Promise<{ record: SegmentRecord; bytes: Uint8Array } | undefined> {
|
|
612
|
-
if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
|
|
613
|
-
const record = await this.#readRecord(segmentId);
|
|
614
|
-
if (record === undefined) return undefined;
|
|
615
|
-
const response = await this.#request('GET', this.objectKeyFor(segmentId));
|
|
616
|
-
if (response === undefined) return undefined;
|
|
617
|
-
return {
|
|
618
|
-
record: record.record,
|
|
619
|
-
bytes: new Uint8Array(await response.arrayBuffer()),
|
|
620
|
-
};
|
|
621
|
-
}
|
|
622
|
-
|
|
623
|
-
/** The record plus the object GET's body, never buffered (see `get`). */
|
|
624
|
-
async open(segmentId: string): Promise<SegmentStream | undefined> {
|
|
625
|
-
if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
|
|
626
|
-
const record = await this.#readRecord(segmentId);
|
|
627
|
-
if (record === undefined) return undefined;
|
|
628
|
-
const response = await this.#request('GET', this.objectKeyFor(segmentId));
|
|
629
|
-
if (response === undefined || response.body === null) return undefined;
|
|
630
|
-
return {
|
|
631
|
-
record: record.record,
|
|
632
|
-
body: response.body,
|
|
633
|
-
byteLength: record.record.byteLength,
|
|
634
|
-
};
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
async find(
|
|
638
|
-
key: SegmentFindKey,
|
|
639
|
-
nowMs: number,
|
|
640
|
-
): Promise<SegmentRecord | undefined> {
|
|
641
|
-
const response = await this.#request('GET', await this.#findKeyFor(key));
|
|
642
|
-
if (response === undefined) return undefined;
|
|
643
|
-
const record = parseRecordJson(await response.text(), 'reuse pointer');
|
|
644
|
-
// The pointer is keyed by the full context, but the record body is the
|
|
645
|
-
// whole merged entry: select the publication the key names, never the
|
|
646
|
-
// top-level fields or the digest union (§5.1).
|
|
647
|
-
const publication = selectSegmentPublication(record, {
|
|
648
|
-
partition: key.partition,
|
|
649
|
-
logEpoch: key.logEpoch,
|
|
650
|
-
table: key.table,
|
|
651
|
-
schemaVersion: key.schemaVersion,
|
|
652
|
-
mediaType: key.mediaType,
|
|
653
|
-
scopeDigest: key.scopeDigest,
|
|
654
|
-
asOfCommitSeq: key.asOfCommitSeq,
|
|
655
|
-
rowCursor: null,
|
|
656
|
-
nowMs,
|
|
657
|
-
});
|
|
658
|
-
if (publication === undefined) return undefined;
|
|
659
|
-
// Lifecycle GC may have removed the object while the pointer survived.
|
|
660
|
-
const head = await this.#request(
|
|
661
|
-
'HEAD',
|
|
662
|
-
this.objectKeyFor(record.segmentId),
|
|
663
|
-
);
|
|
664
|
-
if (head === undefined) return undefined;
|
|
665
|
-
await head.arrayBuffer();
|
|
666
|
-
return publicationRecord(record, publication);
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
/**
|
|
670
|
-
* Store-wide counters from the pointer-object accumulator (never a LIST).
|
|
671
|
-
* Marked `approximate: true`: the counters are maintained by a best-effort
|
|
672
|
-
* ETag-CAS read-modify-write on `put`, so a crash between the segment PUT
|
|
673
|
-
* and the accumulator write, or lifecycle GC that deletes objects the
|
|
674
|
-
* accumulator still counts, can drift them. Exact within a single writer;
|
|
675
|
-
* a health gauge under concurrency. See the README "S3 stats".
|
|
676
|
-
*/
|
|
677
|
-
async stats(): Promise<SegmentStoreStats> {
|
|
678
|
-
const { stats } = await this.#readStats();
|
|
679
|
-
return {
|
|
680
|
-
count: stats.count,
|
|
681
|
-
bytes: stats.bytes,
|
|
682
|
-
rowsSegments: stats.rowsSegments,
|
|
683
|
-
sqliteSegments: stats.sqliteSegments,
|
|
684
|
-
approximate: true,
|
|
685
|
-
};
|
|
686
65
|
}
|
|
687
66
|
|
|
688
67
|
/**
|
|
@@ -702,8 +81,8 @@ export class S3SegmentStore implements SegmentStore {
|
|
|
702
81
|
const ttlSeconds = options?.ttlSeconds ?? 900;
|
|
703
82
|
const nowMs = options?.nowMs ?? Date.now();
|
|
704
83
|
const url = await presignUrl({
|
|
705
|
-
url: this.#urlFor(this.objectKeyFor(segmentId)),
|
|
706
|
-
region: this.#
|
|
84
|
+
url: this.#bucket.urlFor(this.objectKeyFor(segmentId)),
|
|
85
|
+
region: this.#region,
|
|
707
86
|
credentials: this.#credentials,
|
|
708
87
|
nowMs,
|
|
709
88
|
expiresSeconds: ttlSeconds,
|