@syncular/server 0.32.1 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/blob-handlers.js +3 -3
  2. package/dist/bucket-blob-store.d.ts +77 -0
  3. package/dist/bucket-blob-store.js +266 -0
  4. package/dist/bucket-segment-store.d.ts +77 -0
  5. package/dist/bucket-segment-store.js +445 -0
  6. package/dist/context.d.ts +7 -0
  7. package/dist/d1-storage.js +2 -2
  8. package/dist/handler.js +4 -0
  9. package/dist/index.d.ts +4 -0
  10. package/dist/index.js +4 -0
  11. package/dist/object-bucket.d.ts +78 -0
  12. package/dist/object-bucket.js +18 -0
  13. package/dist/operations-realtime.js +3 -0
  14. package/dist/operations.d.ts +20 -3
  15. package/dist/operations.js +23 -3
  16. package/dist/realtime.d.ts +11 -0
  17. package/dist/realtime.js +25 -58
  18. package/dist/relational-rows.d.ts +6 -3
  19. package/dist/relational-rows.js +8 -9
  20. package/dist/s3-blob-store.d.ts +9 -64
  21. package/dist/s3-blob-store.js +25 -359
  22. package/dist/s3-object-bucket.d.ts +39 -0
  23. package/dist/s3-object-bucket.js +164 -0
  24. package/dist/s3-segment-store.d.ts +9 -61
  25. package/dist/s3-segment-store.js +25 -503
  26. package/dist/sqlite-storage.js +3 -8
  27. package/package.json +2 -2
  28. package/src/blob-handlers.ts +4 -4
  29. package/src/bucket-blob-store.ts +351 -0
  30. package/src/bucket-segment-store.ts +582 -0
  31. package/src/context.ts +10 -0
  32. package/src/d1-storage.ts +3 -3
  33. package/src/handler.ts +11 -0
  34. package/src/index.ts +4 -0
  35. package/src/object-bucket.ts +83 -0
  36. package/src/operations-realtime.ts +7 -0
  37. package/src/operations.ts +52 -5
  38. package/src/realtime.ts +29 -58
  39. package/src/relational-rows.ts +8 -11
  40. package/src/s3-blob-store.ts +25 -466
  41. package/src/s3-object-bucket.ts +233 -0
  42. package/src/s3-segment-store.ts +25 -646
  43. package/src/sqlite-storage.ts +3 -10
@@ -1,72 +1,21 @@
1
1
  /**
2
2
  * S3-compatible segment store (SPEC.md §5.1 cache semantics, §5.4
3
- * delegated presign) — AWS S3, Cloudflare R2, MinIO. Zero dependencies:
4
- * hand-rolled SigV4 over `fetch` (sigv4.ts).
3
+ * delegated presign) — AWS S3, Cloudflare R2's S3 endpoint, MinIO. Zero
4
+ * dependencies: hand-rolled SigV4 over `fetch` (sigv4.ts).
5
5
  *
6
- * Key layout (deterministic — every lookup is a GET/HEAD, never a LIST):
7
- *
8
- * {keyPrefix}seg/sha256/{hex}
9
- * The segment bytes, verbatim and immutable (the object body MUST be
10
- * exactly the content-addressed bytes so presigned GETs serve them
11
- * directly and the client's §5.1 hash check passes).
12
- *
13
- * {keyPrefix}rec/sha256/{hex}.json
14
- * The `SegmentRecord` (§5.1). It is the one MUTABLE object of the two:
15
- * the scope-digest set is a union that every publication extends, so
16
- * the merge is a read-modify-write. Keeping the record in the body (not
17
- * in the bytes object's user metadata) is what makes that write
18
- * conditional — a body ETag changes when the union changes, while the
19
- * bytes ETag is a content hash and is identical for identical bytes. A
20
- * `412` re-reads and merges again, so no concurrent publication can drop
21
- * another one's digest. `get` therefore reads two objects.
22
- *
23
- * {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
24
- * Whole-table reuse pointer (§5.3): written only when
25
- * `rowCursor === null`, body = the record JSON. The reuse key is the
26
- * canonical JSON array `[partition, table, schemaVersion, mediaType,
27
- * scopeDigest, asOfCommitSeq]`, so `find` is one GET (plus a HEAD to
28
- * confirm the segment object itself still exists — lifecycle GC may
29
- * remove objects independently of pointers).
30
- *
31
- * Records written before the split kept the record in the bytes object's
32
- * user metadata `x-amz-meta-syncular-record`; `get` still reads that shape
33
- * and the next `put` materializes the record object from it, so an upgrade
34
- * loses no cache entry.
35
- *
36
- * TTL mapping: expiry is **store-side and authoritative** — `expiresAtMs`
37
- * (put-time + `ttlMs`, default 24 h) is recorded in the record and the
38
- * pointer; `get` returns expired records so §5.5 can answer
39
- * `sync.segment_expired`, and `find` filters them out itself. S3 lifecycle
40
- * expiration is garbage collection only: configure it comfortably ABOVE
41
- * `ttlMs` (e.g. 2 days for the 24 h default) so clients normally see the
42
- * precise, retryable `sync.segment_expired` and hit `sync.not_found` only
43
- * long after. Never set lifecycle below `ttlMs`.
6
+ * The store logic (key layout, record CAS, reuse pointers, stats
7
+ * accumulator) is `BucketSegmentStore`, shared with `R2SegmentStore` in
8
+ * `@syncular/server-workers`; this class adds the S3 REST transport
9
+ * (`S3ObjectBucket`) and the SigV4 presign methods, which a bucket binding
10
+ * cannot mint.
44
11
  */
45
- import type {
46
- SegmentFindKey,
47
- SegmentMetadata,
48
- SegmentPublication,
49
- SegmentRecord,
50
- SegmentStore,
51
- SegmentStoreStats,
52
- SegmentStream,
53
- } from './segment-store';
54
12
  import {
55
- mergeSegmentRecord,
56
- publicationRecord,
57
- publicationsForRecord,
58
- segmentBytesEqual,
59
- segmentIdFor,
60
- selectSegmentPublication,
61
- } from './segment-store';
13
+ BucketSegmentStore,
14
+ type BucketSegmentStoreOptions,
15
+ } from './bucket-segment-store';
16
+ import { S3ObjectBucket } from './s3-object-bucket';
62
17
  import type { DelegatedPresignConfig, SegmentUrlIssue } from './signed-url';
63
- import {
64
- EMPTY_PAYLOAD_SHA256,
65
- presignUrl,
66
- type SigV4Credentials,
67
- sha256Hex,
68
- signRequest,
69
- } from './sigv4';
18
+ import { presignUrl, type SigV4Credentials } from './sigv4';
70
19
 
71
20
  export interface S3SegmentStoreConfig {
72
21
  /**
@@ -90,223 +39,22 @@ export interface S3SegmentStoreConfig {
90
39
  readonly ttlMs?: number;
91
40
  }
92
41
 
93
- const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
94
42
  const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
95
- const RECORD_META_HEADER = 'x-amz-meta-syncular-record';
96
-
97
- /**
98
- * Stats accumulator (§ "S3 stats" in the README). The store keeps a small
99
- * counter object under a fixed key, updated read-modify-write on `put`. It
100
- * exists because a LIST-free store (every lookup is a GET/HEAD) has no cheap
101
- * way to count objects on demand — so `stats()` reads the accumulator instead
102
- * of enumerating the bucket.
103
- *
104
- * The write is guarded by an ETag compare-and-swap (`If-Match`, or
105
- * `If-None-Match: *` to create): the loop re-reads and retries on a `412`
106
- * precondition failure, so concurrent writers do not silently clobber each
107
- * other's increment. The counters are still APPROXIMATE — a crash between the
108
- * segment PUT and the accumulator CAS, or a writer against a backend that
109
- * ignores conditional headers, can drift them — so the admin surface labels
110
- * S3 stats `approximate: true`. They are a health gauge, not an invoice.
111
- */
112
- const STATS_KEY_SUFFIX = 'stats/segments.json';
113
- const CAS_ATTEMPTS = 16;
114
-
115
- interface StatsAccumulator {
116
- count: number;
117
- bytes: number;
118
- rowsSegments: number;
119
- sqliteSegments: number;
120
- }
121
43
 
122
- function emptyStats(): StatsAccumulator {
123
- return { count: 0, bytes: 0, rowsSegments: 0, sqliteSegments: 0 };
124
- }
125
-
126
- function parseStatsJson(json: string): StatsAccumulator {
127
- const parsed = JSON.parse(json) as Partial<StatsAccumulator>;
128
- return {
129
- count: Number(parsed.count ?? 0),
130
- bytes: Number(parsed.bytes ?? 0),
131
- rowsSegments: Number(parsed.rowsSegments ?? 0),
132
- sqliteSegments: Number(parsed.sqliteSegments ?? 0),
133
- };
134
- }
135
-
136
- // Runtime-neutral base64url: `Buffer` is not present on
137
- // Cloudflare Workers without `nodejs_compat`, so a legacy object-metadata
138
- // record header is decoded with `atob`, available in every runtime.
139
- function base64urlToUtf8(value: string): string {
140
- const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/'));
141
- const bytes = new Uint8Array(binary.length);
142
- for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
143
- return new TextDecoder().decode(bytes);
144
- }
145
-
146
- function recordToJson(record: SegmentRecord): string {
147
- return JSON.stringify(record);
148
- }
149
-
150
- function isMediaType(value: unknown): value is 'rows' | 'sqlite' {
151
- return value === 'rows' || value === 'sqlite';
152
- }
153
-
154
- /** One publication inside a stored record (or a legacy materialized one). */
155
- function parsePublication(value: unknown, source: string): SegmentPublication {
156
- if (typeof value !== 'object' || value === null) {
157
- throw new Error(`S3SegmentStore: corrupt publication in ${source}`);
158
- }
159
- const p = value as Record<string, unknown>;
160
- const strings = ['partition', 'logEpoch', 'table', 'scopeDigest'] as const;
161
- const numbers = [
162
- 'schemaVersion',
163
- 'asOfCommitSeq',
164
- 'rowCount',
165
- 'byteLength',
166
- 'createdAtMs',
167
- 'expiresAtMs',
168
- ] as const;
169
- for (const field of strings) {
170
- if (typeof p[field] !== 'string') {
171
- throw new Error(
172
- `S3SegmentStore: bad publication field ${field} in ${source}`,
173
- );
174
- }
175
- }
176
- for (const field of numbers) {
177
- if (typeof p[field] !== 'number') {
178
- throw new Error(
179
- `S3SegmentStore: bad publication field ${field} in ${source}`,
180
- );
181
- }
182
- }
183
- if (!isMediaType(p.mediaType)) {
184
- throw new Error(`S3SegmentStore: bad publication mediaType in ${source}`);
185
- }
186
- const cursorOk = (v: unknown): v is string | null =>
187
- v === null || typeof v === 'string';
188
- if (!cursorOk(p.rowCursor) || !cursorOk(p.nextRowCursor)) {
189
- throw new Error(`S3SegmentStore: bad publication cursor in ${source}`);
190
- }
191
- return {
192
- partition: p.partition as string,
193
- logEpoch: p.logEpoch as string,
194
- table: p.table as string,
195
- schemaVersion: p.schemaVersion as number,
196
- mediaType: p.mediaType,
197
- scopeDigest: p.scopeDigest as string,
198
- asOfCommitSeq: p.asOfCommitSeq as number,
199
- rowCount: p.rowCount as number,
200
- rowCursor: p.rowCursor,
201
- nextRowCursor: p.nextRowCursor,
202
- byteLength: p.byteLength as number,
203
- createdAtMs: p.createdAtMs as number,
204
- expiresAtMs: p.expiresAtMs as number,
205
- };
206
- }
207
-
208
- function parseRecordJson(json: string, source: string): SegmentRecord {
209
- let parsed: unknown;
210
- try {
211
- parsed = JSON.parse(json);
212
- } catch {
213
- throw new Error(`S3SegmentStore: corrupt record JSON in ${source}`);
214
- }
215
- if (typeof parsed !== 'object' || parsed === null) {
216
- throw new Error(`S3SegmentStore: corrupt record in ${source}`);
217
- }
218
- const r = parsed as Record<string, unknown>;
219
- const strings = [
220
- 'segmentId',
221
- 'partition',
222
- 'logEpoch',
223
- 'table',
224
- 'scopeDigest',
225
- ] as const;
226
- const numbers = [
227
- 'schemaVersion',
228
- 'asOfCommitSeq',
229
- 'rowCount',
230
- 'byteLength',
231
- 'createdAtMs',
232
- 'expiresAtMs',
233
- ] as const;
234
- for (const field of strings) {
235
- if (typeof r[field] !== 'string') {
236
- throw new Error(`S3SegmentStore: bad record field ${field} in ${source}`);
237
- }
238
- }
239
- for (const field of numbers) {
240
- if (typeof r[field] !== 'number') {
241
- throw new Error(`S3SegmentStore: bad record field ${field} in ${source}`);
242
- }
243
- }
244
- if (!isMediaType(r.mediaType)) {
245
- throw new Error(`S3SegmentStore: bad record mediaType in ${source}`);
246
- }
247
- const cursorOk = (v: unknown): v is string | null =>
248
- v === null || typeof v === 'string';
249
- if (!cursorOk(r.rowCursor) || !cursorOk(r.nextRowCursor)) {
250
- throw new Error(`S3SegmentStore: bad record cursor in ${source}`);
251
- }
252
- const scopeDigest = r.scopeDigest as string;
253
- // Records written before §5.1 recorded the digest set carry only the
254
- // primary digest; the object is a cache entry with a TTL, so reading it
255
- // back as a one-digest record is correct, not a degraded path.
256
- const rawDigests = r.scopeDigests;
257
- if (
258
- rawDigests !== undefined &&
259
- (!Array.isArray(rawDigests) ||
260
- rawDigests.some((digest) => typeof digest !== 'string'))
261
- ) {
262
- throw new Error(`S3SegmentStore: bad record scopeDigests in ${source}`);
263
- }
264
- const rawPublications = r.publications;
265
- if (rawPublications !== undefined && !Array.isArray(rawPublications)) {
266
- throw new Error(`S3SegmentStore: bad record publications in ${source}`);
267
- }
268
- const top = {
269
- segmentId: r.segmentId as string,
270
- partition: r.partition as string,
271
- logEpoch: r.logEpoch as string,
272
- table: r.table as string,
273
- schemaVersion: r.schemaVersion as number,
274
- mediaType: r.mediaType,
275
- scopeDigest,
276
- scopeDigests: [
277
- scopeDigest,
278
- ...((rawDigests ?? []) as string[]).filter(
279
- (digest) => digest !== scopeDigest,
280
- ),
281
- ],
282
- asOfCommitSeq: r.asOfCommitSeq as number,
283
- rowCount: r.rowCount as number,
284
- rowCursor: r.rowCursor,
285
- nextRowCursor: r.nextRowCursor,
286
- byteLength: r.byteLength as number,
287
- createdAtMs: r.createdAtMs as number,
288
- expiresAtMs: r.expiresAtMs as number,
289
- };
290
- return {
291
- ...top,
292
- // A record from before per-publication provenance, or a legacy record
293
- // whose union came from object metadata, materializes its publications
294
- // from the merged top-level context — the old single-record read.
295
- publications:
296
- rawPublications === undefined
297
- ? publicationsForRecord(top)
298
- : rawPublications.map((value) => parsePublication(value, source)),
299
- };
300
- }
301
-
302
- export class S3SegmentStore implements SegmentStore {
303
- readonly #config: S3SegmentStoreConfig;
44
+ export class S3SegmentStore extends BucketSegmentStore {
45
+ readonly #bucket: S3ObjectBucket;
46
+ readonly #region: string;
304
47
  readonly #credentials: SigV4Credentials;
305
- readonly #ttlMs: number;
306
- readonly #prefix: string;
307
48
 
308
49
  constructor(config: S3SegmentStoreConfig) {
309
- this.#config = config;
50
+ const options: BucketSegmentStoreOptions = {
51
+ keyPrefix: config.keyPrefix,
52
+ ttlMs: config.ttlMs,
53
+ };
54
+ const bucket = new S3ObjectBucket(config);
55
+ super(bucket, options);
56
+ this.#bucket = bucket;
57
+ this.#region = config.region;
310
58
  this.#credentials = {
311
59
  accessKeyId: config.accessKeyId,
312
60
  secretAccessKey: config.secretAccessKey,
@@ -314,375 +62,6 @@ export class S3SegmentStore implements SegmentStore {
314
62
  ? { sessionToken: config.sessionToken }
315
63
  : {}),
316
64
  };
317
- this.#ttlMs = config.ttlMs ?? DEFAULT_TTL_MS;
318
- this.#prefix = config.keyPrefix ?? '';
319
- }
320
-
321
- /** `{keyPrefix}seg/sha256/{hex}` — the §5.4 "key embeds the segmentId". */
322
- objectKeyFor(segmentId: string): string {
323
- return `${this.#prefix}seg/${segmentId.replace(':', '/')}`;
324
- }
325
-
326
- /** `{keyPrefix}rec/sha256/{hex}.json` — the §5.1 record (§5.1 union). */
327
- #recordKeyFor(segmentId: string): string {
328
- return `${this.#prefix}rec/${segmentId.replace(':', '/')}.json`;
329
- }
330
-
331
- async #findKeyFor(key: SegmentFindKey): Promise<string> {
332
- const canonical = JSON.stringify([
333
- key.partition,
334
- key.logEpoch,
335
- key.table,
336
- key.schemaVersion,
337
- key.mediaType,
338
- key.scopeDigest,
339
- key.asOfCommitSeq,
340
- ]);
341
- return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
342
- }
343
-
344
- #statsKey(): string {
345
- return `${this.#prefix}${STATS_KEY_SUFFIX}`;
346
- }
347
-
348
- #urlFor(key: string): URL {
349
- const endpoint = this.#config.endpoint.replace(/\/+$/, '');
350
- return new URL(`${endpoint}/${this.#config.bucket}/${key}`);
351
- }
352
-
353
- async #request(
354
- method: 'GET' | 'HEAD' | 'PUT' | 'DELETE',
355
- key: string,
356
- options?: {
357
- readonly body?: Uint8Array;
358
- readonly headers?: Readonly<Record<string, string>>;
359
- /** Return a `412` response instead of throwing (conditional writes). */
360
- readonly allow412?: boolean;
361
- },
362
- ): Promise<Response | undefined> {
363
- const url = this.#urlFor(key);
364
- const body = options?.body;
365
- const headers = await signRequest({
366
- method,
367
- url,
368
- region: this.#config.region,
369
- credentials: this.#credentials,
370
- nowMs: Date.now(),
371
- payloadHash:
372
- body === undefined ? EMPTY_PAYLOAD_SHA256 : await sha256Hex(body),
373
- ...(options?.headers !== undefined ? { headers: options.headers } : {}),
374
- });
375
- const response = await fetch(url, {
376
- method,
377
- headers,
378
- ...(body !== undefined ? { body: body as BodyInit } : {}),
379
- });
380
- if (response.status === 404) {
381
- await response.arrayBuffer();
382
- return undefined;
383
- }
384
- if (options?.allow412 === true && response.status === 412) return response;
385
- if (!response.ok) {
386
- const detail = (await response.text()).slice(0, 300);
387
- throw new Error(
388
- `S3SegmentStore: ${method} ${key} failed with ${response.status}: ${detail}`,
389
- );
390
- }
391
- return response;
392
- }
393
-
394
- /** Read the stats accumulator + its ETag (for the CAS write), or none. */
395
- async #readStats(): Promise<{
396
- stats: StatsAccumulator;
397
- etag: string | undefined;
398
- }> {
399
- const response = await this.#request('GET', this.#statsKey());
400
- if (response === undefined) {
401
- return { stats: emptyStats(), etag: undefined };
402
- }
403
- const etag = response.headers.get('etag') ?? undefined;
404
- const stats = parseStatsJson(await response.text());
405
- return { stats, etag };
406
- }
407
-
408
- /**
409
- * Conditionally write the stats accumulator. `etag` present ⇒ `If-Match`
410
- * (update the object we read); absent ⇒ `If-None-Match: *` (create only).
411
- * Returns `false` on a `412` precondition failure so the caller retries.
412
- */
413
- async #writeStatsCas(
414
- stats: StatsAccumulator,
415
- etag: string | undefined,
416
- ): Promise<boolean> {
417
- const url = this.#urlFor(this.#statsKey());
418
- const body = new TextEncoder().encode(JSON.stringify(stats));
419
- const conditional: Record<string, string> =
420
- etag === undefined ? { 'if-none-match': '*' } : { 'if-match': etag };
421
- const headers = await signRequest({
422
- method: 'PUT',
423
- url,
424
- region: this.#config.region,
425
- credentials: this.#credentials,
426
- nowMs: Date.now(),
427
- payloadHash: await sha256Hex(body),
428
- headers: { 'content-type': 'application/json', ...conditional },
429
- });
430
- const response = await fetch(url, {
431
- method: 'PUT',
432
- headers,
433
- body: body as BodyInit,
434
- });
435
- if (response.status === 412) {
436
- await response.arrayBuffer();
437
- return false;
438
- }
439
- if (!response.ok) {
440
- const detail = (await response.text()).slice(0, 300);
441
- throw new Error(
442
- `S3SegmentStore: stats PUT failed with ${response.status}: ${detail}`,
443
- );
444
- }
445
- await response.arrayBuffer();
446
- return true;
447
- }
448
-
449
- /**
450
- * Fold `delta` into the stats accumulator under an ETag compare-and-swap
451
- * retry loop. Best-effort: exhausting the retry budget leaves the counters
452
- * drifted (they are documented APPROXIMATE) rather than failing the `put`
453
- * that already stored the bytes.
454
- */
455
- async #bumpStats(delta: StatsAccumulator): Promise<void> {
456
- for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
457
- const { stats, etag } = await this.#readStats();
458
- const next: StatsAccumulator = {
459
- count: stats.count + delta.count,
460
- bytes: stats.bytes + delta.bytes,
461
- rowsSegments: stats.rowsSegments + delta.rowsSegments,
462
- sqliteSegments: stats.sqliteSegments + delta.sqliteSegments,
463
- };
464
- if (await this.#writeStatsCas(next, etag)) return;
465
- }
466
- // Lost the CAS race `CAS_ATTEMPTS` times running — leave the
467
- // counters as-is. `stats()` stays APPROXIMATE by contract.
468
- }
469
-
470
- async put(
471
- metadata: SegmentMetadata,
472
- bytes: Uint8Array,
473
- nowMs: number,
474
- ): Promise<SegmentRecord> {
475
- const segmentId = await segmentIdFor(bytes);
476
- await this.#putBytes(segmentId, bytes);
477
- for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
478
- // The id IS the content address, so an existing record means a second
479
- // scope published the same bytes: read it back and merge the digest set
480
- // (§5.1).
481
- const preexisting = await this.#readRecord(segmentId);
482
- const record = mergeSegmentRecord(
483
- preexisting?.record,
484
- metadata,
485
- segmentId,
486
- bytes.length,
487
- nowMs,
488
- this.#ttlMs,
489
- );
490
- const recordJson = recordToJson(record);
491
- // The digest set is monotone (§5.1) and this is a read-modify-write: a
492
- // concurrent publisher of the same bytes computed its own union from the
493
- // same pre-state, so an unconditional PUT would drop one of the digests
494
- // and make that scope's download `sync.forbidden`. The write is
495
- // conditional on the record body that was read — the record ETag is the
496
- // body hash, so it changes whenever the union does — and a `412`
497
- // re-reads and merges again. In-process dedupe cannot cover this: two
498
- // scopes whose rows are byte-identical are two builds, and two server
499
- // instances share one bucket.
500
- const conditional: Record<string, string> =
501
- preexisting?.etag === undefined
502
- ? { 'if-none-match': '*' }
503
- : { 'if-match': preexisting.etag };
504
- const response = await this.#request(
505
- 'PUT',
506
- this.#recordKeyFor(segmentId),
507
- {
508
- body: new TextEncoder().encode(recordJson),
509
- headers: { 'content-type': 'application/json', ...conditional },
510
- allow412: true,
511
- },
512
- );
513
- if (response !== undefined && response.status === 412) {
514
- await response.arrayBuffer();
515
- continue;
516
- }
517
- await response?.arrayBuffer();
518
- if (preexisting === undefined) {
519
- await this.#bumpStats({
520
- count: 1,
521
- bytes: bytes.length,
522
- rowsSegments: metadata.mediaType === 'rows' ? 1 : 0,
523
- sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
524
- });
525
- }
526
- if (metadata.rowCursor === null) {
527
- const pointer = await this.#request(
528
- 'PUT',
529
- await this.#findKeyFor(metadata),
530
- {
531
- body: new TextEncoder().encode(recordJson),
532
- headers: { 'content-type': 'application/json' },
533
- },
534
- );
535
- await pointer?.arrayBuffer();
536
- }
537
- return record;
538
- }
539
- throw new Error(
540
- `S3SegmentStore: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`,
541
- );
542
- }
543
-
544
- /**
545
- * Write the immutable bytes object. Create-only, so a second publisher of
546
- * identical bytes is a no-op and a second publisher of DIFFERENT bytes under
547
- * the same content address is the hash collision §5.1 requires to fail
548
- * loudly instead of overwriting.
549
- */
550
- async #putBytes(segmentId: string, bytes: Uint8Array): Promise<void> {
551
- const key = this.objectKeyFor(segmentId);
552
- const response = await this.#request('PUT', key, {
553
- body: bytes,
554
- headers: {
555
- 'content-type': 'application/octet-stream',
556
- 'if-none-match': '*',
557
- },
558
- allow412: true,
559
- });
560
- if (response === undefined || response.status !== 412) {
561
- await response?.arrayBuffer();
562
- return;
563
- }
564
- await response.arrayBuffer();
565
- const current = await this.#request('GET', key);
566
- if (current === undefined) return;
567
- const stored = new Uint8Array(await current.arrayBuffer());
568
- if (!segmentBytesEqual(stored, bytes)) {
569
- throw new Error(
570
- `S3SegmentStore: content-address collision at ${segmentId} (§5.1)`,
571
- );
572
- }
573
- }
574
-
575
- /**
576
- * Read the record object plus the ETag of the body that was read, so the
577
- * union write can be made conditional on exactly that pre-state (§5.1).
578
- *
579
- * A record written before the record/bytes split rides in the bytes object's
580
- * user metadata; it is read here and its union is materialized into a record
581
- * object by the next `put`. That shape carries no usable ETag for the union
582
- * (the bytes ETag is a content hash), which is exactly why it was replaced.
583
- * A bytes object with neither a record object nor the legacy metadata is an
584
- * entry whose record is missing — a cache miss, not corruption, so `get`
585
- * reports it as absent and the client re-pulls.
586
- */
587
- async #readRecord(
588
- segmentId: string,
589
- ): Promise<{ record: SegmentRecord; etag: string | undefined } | undefined> {
590
- if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
591
- const response = await this.#request('GET', this.#recordKeyFor(segmentId));
592
- if (response !== undefined) {
593
- return {
594
- record: parseRecordJson(await response.text(), 'segment record'),
595
- etag: response.headers.get('etag') ?? undefined,
596
- };
597
- }
598
- const legacy = await this.#request('GET', this.objectKeyFor(segmentId));
599
- if (legacy === undefined) return undefined;
600
- const meta = legacy.headers.get(RECORD_META_HEADER);
601
- await legacy.arrayBuffer();
602
- if (meta === null) return undefined;
603
- return {
604
- record: parseRecordJson(base64urlToUtf8(meta), segmentId),
605
- etag: undefined,
606
- };
607
- }
608
-
609
- async get(
610
- segmentId: string,
611
- ): Promise<{ record: SegmentRecord; bytes: Uint8Array } | undefined> {
612
- if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
613
- const record = await this.#readRecord(segmentId);
614
- if (record === undefined) return undefined;
615
- const response = await this.#request('GET', this.objectKeyFor(segmentId));
616
- if (response === undefined) return undefined;
617
- return {
618
- record: record.record,
619
- bytes: new Uint8Array(await response.arrayBuffer()),
620
- };
621
- }
622
-
623
- /** The record plus the object GET's body, never buffered (see `get`). */
624
- async open(segmentId: string): Promise<SegmentStream | undefined> {
625
- if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
626
- const record = await this.#readRecord(segmentId);
627
- if (record === undefined) return undefined;
628
- const response = await this.#request('GET', this.objectKeyFor(segmentId));
629
- if (response === undefined || response.body === null) return undefined;
630
- return {
631
- record: record.record,
632
- body: response.body,
633
- byteLength: record.record.byteLength,
634
- };
635
- }
636
-
637
- async find(
638
- key: SegmentFindKey,
639
- nowMs: number,
640
- ): Promise<SegmentRecord | undefined> {
641
- const response = await this.#request('GET', await this.#findKeyFor(key));
642
- if (response === undefined) return undefined;
643
- const record = parseRecordJson(await response.text(), 'reuse pointer');
644
- // The pointer is keyed by the full context, but the record body is the
645
- // whole merged entry: select the publication the key names, never the
646
- // top-level fields or the digest union (§5.1).
647
- const publication = selectSegmentPublication(record, {
648
- partition: key.partition,
649
- logEpoch: key.logEpoch,
650
- table: key.table,
651
- schemaVersion: key.schemaVersion,
652
- mediaType: key.mediaType,
653
- scopeDigest: key.scopeDigest,
654
- asOfCommitSeq: key.asOfCommitSeq,
655
- rowCursor: null,
656
- nowMs,
657
- });
658
- if (publication === undefined) return undefined;
659
- // Lifecycle GC may have removed the object while the pointer survived.
660
- const head = await this.#request(
661
- 'HEAD',
662
- this.objectKeyFor(record.segmentId),
663
- );
664
- if (head === undefined) return undefined;
665
- await head.arrayBuffer();
666
- return publicationRecord(record, publication);
667
- }
668
-
669
- /**
670
- * Store-wide counters from the pointer-object accumulator (never a LIST).
671
- * Marked `approximate: true`: the counters are maintained by a best-effort
672
- * ETag-CAS read-modify-write on `put`, so a crash between the segment PUT
673
- * and the accumulator write, or lifecycle GC that deletes objects the
674
- * accumulator still counts, can drift them. Exact within a single writer;
675
- * a health gauge under concurrency. See the README "S3 stats".
676
- */
677
- async stats(): Promise<SegmentStoreStats> {
678
- const { stats } = await this.#readStats();
679
- return {
680
- count: stats.count,
681
- bytes: stats.bytes,
682
- rowsSegments: stats.rowsSegments,
683
- sqliteSegments: stats.sqliteSegments,
684
- approximate: true,
685
- };
686
65
  }
687
66
 
688
67
  /**
@@ -702,8 +81,8 @@ export class S3SegmentStore implements SegmentStore {
702
81
  const ttlSeconds = options?.ttlSeconds ?? 900;
703
82
  const nowMs = options?.nowMs ?? Date.now();
704
83
  const url = await presignUrl({
705
- url: this.#urlFor(this.objectKeyFor(segmentId)),
706
- region: this.#config.region,
84
+ url: this.#bucket.urlFor(this.objectKeyFor(segmentId)),
85
+ region: this.#region,
707
86
  credentials: this.#credentials,
708
87
  nowMs,
709
88
  expiresSeconds: ttlSeconds,