@syncular/server 0.58.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +49 -36
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +46 -40
  4. package/dist/blob-handlers.d.ts +25 -31
  5. package/dist/blob-handlers.js +63 -41
  6. package/dist/blob-store.d.ts +42 -24
  7. package/dist/blob-store.js +0 -0
  8. package/dist/bucket-blob-store.d.ts +12 -8
  9. package/dist/bucket-blob-store.js +54 -45
  10. package/dist/bucket-segment-store.js +18 -16
  11. package/dist/content-encoding.d.ts +5 -2
  12. package/dist/content-encoding.js +42 -13
  13. package/dist/context.d.ts +19 -5
  14. package/dist/context.js +19 -1
  15. package/dist/d1-storage.d.ts +14 -98
  16. package/dist/d1-storage.js +249 -1119
  17. package/dist/errors.js +16 -0
  18. package/dist/events.d.ts +36 -18
  19. package/dist/frame-bytes.js +24 -6
  20. package/dist/handler.js +503 -208
  21. package/dist/local-authority.d.ts +2 -0
  22. package/dist/local-authority.js +24 -12
  23. package/dist/object-bucket.d.ts +2 -0
  24. package/dist/operations-realtime.js +69 -50
  25. package/dist/operations.d.ts +5 -2
  26. package/dist/operations.js +20 -29
  27. package/dist/pg-executor-pglite.d.ts +5 -6
  28. package/dist/pg-executor-pglite.js +22 -14
  29. package/dist/pg-executor.d.ts +4 -1
  30. package/dist/postgres-storage.d.ts +24 -19
  31. package/dist/postgres-storage.js +420 -262
  32. package/dist/pull.d.ts +70 -47
  33. package/dist/pull.js +270 -301
  34. package/dist/push.d.ts +12 -3
  35. package/dist/push.js +361 -173
  36. package/dist/realtime.d.ts +85 -39
  37. package/dist/realtime.js +355 -219
  38. package/dist/relational-rows.d.ts +21 -66
  39. package/dist/relational-rows.js +28 -109
  40. package/dist/s3-blob-store.d.ts +11 -12
  41. package/dist/s3-blob-store.js +20 -13
  42. package/dist/s3-object-bucket.d.ts +8 -1
  43. package/dist/s3-object-bucket.js +16 -3
  44. package/dist/schema.d.ts +14 -0
  45. package/dist/schema.js +23 -0
  46. package/dist/scopes.d.ts +6 -2
  47. package/dist/scopes.js +22 -4
  48. package/dist/seed.js +10 -16
  49. package/dist/segment-download.d.ts +4 -1
  50. package/dist/segment-download.js +22 -6
  51. package/dist/signed-url.d.ts +2 -2
  52. package/dist/signed-url.js +9 -2
  53. package/dist/sigv4.d.ts +8 -2
  54. package/dist/sigv4.js +8 -4
  55. package/dist/sqlite-blob-store.d.ts +1 -1
  56. package/dist/sqlite-blob-store.js +23 -22
  57. package/dist/sqlite-dialect.d.ts +164 -84
  58. package/dist/sqlite-dialect.js +1145 -150
  59. package/dist/sqlite-lease-store.js +13 -18
  60. package/dist/sqlite-storage.d.ts +9 -62
  61. package/dist/sqlite-storage.js +303 -1123
  62. package/dist/storage-query.d.ts +30 -1
  63. package/dist/storage-query.js +68 -0
  64. package/dist/storage.d.ts +185 -63
  65. package/dist/storage.js +15 -0
  66. package/package.json +2 -2
  67. package/src/admin.ts +99 -57
  68. package/src/blob-handlers.ts +91 -72
  69. package/src/blob-store.ts +0 -0
  70. package/src/bucket-blob-store.ts +75 -48
  71. package/src/bucket-segment-store.ts +20 -15
  72. package/src/content-encoding.ts +48 -16
  73. package/src/context.ts +33 -6
  74. package/src/d1-storage.ts +350 -1700
  75. package/src/errors.ts +16 -0
  76. package/src/events.ts +37 -18
  77. package/src/frame-bytes.ts +26 -6
  78. package/src/handler.ts +654 -261
  79. package/src/local-authority.ts +27 -14
  80. package/src/object-bucket.ts +2 -0
  81. package/src/operations-realtime.ts +85 -61
  82. package/src/operations.ts +44 -42
  83. package/src/pg-executor-pglite.ts +27 -23
  84. package/src/pg-executor.ts +5 -1
  85. package/src/postgres-storage.ts +557 -354
  86. package/src/pull.ts +406 -440
  87. package/src/push.ts +466 -220
  88. package/src/realtime.ts +447 -290
  89. package/src/relational-rows.ts +43 -121
  90. package/src/s3-blob-store.ts +23 -13
  91. package/src/s3-object-bucket.ts +20 -2
  92. package/src/schema.ts +31 -0
  93. package/src/scopes.ts +23 -3
  94. package/src/seed.ts +13 -20
  95. package/src/segment-download.ts +33 -7
  96. package/src/signed-url.ts +10 -4
  97. package/src/sigv4.ts +17 -4
  98. package/src/sqlite-blob-store.ts +27 -26
  99. package/src/sqlite-dialect.ts +1851 -199
  100. package/src/sqlite-lease-store.ts +12 -23
  101. package/src/sqlite-storage.ts +390 -1821
  102. package/src/storage-query.ts +102 -0
  103. package/src/storage.ts +215 -67
@@ -7,41 +7,67 @@
7
7
  * adapters map these onto `PUT/GET <mount>/blobs/{blobId}`.
8
8
  */
9
9
  import { blobIdFor, isBlobId } from './blob-store.js';
10
+ import { decodeRequestBody } from './content-encoding.js';
10
11
  import { clockOf, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
11
12
  import { SyncError, syncError } from './errors.js';
12
13
  import { emitEvent } from './events.js';
13
14
  import { compileSchema } from './schema.js';
14
15
  import { authorizeWrite } from './scopes.js';
16
+ import { bytesStream } from './segment-store.js';
15
17
  import { issueBlobUploadUrl, issueBlobUrl } from './signed-url.js';
16
18
  const DEFAULT_MAX_BLOB_BYTES = 64 * 1024 * 1024;
17
- /** `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. */
18
- export async function handleBlobUpload(ctx, request) {
19
+ /**
20
+ * `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
21
+ * size cap applies to the declared `Content-Length` before any byte is read
22
+ * and again to the counted decoded bytes while reading, so an oversized body
23
+ * is never buffered whole.
24
+ */
25
+ export async function handleBlobUpload(ctx, blobId, request) {
19
26
  await touchAuthenticatedPartition(ctx);
20
27
  const store = ctx.blobs;
21
28
  if (store === undefined) {
22
29
  throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
23
30
  }
24
- if (!isBlobId(request.blobId)) {
31
+ if (!isBlobId(blobId)) {
25
32
  throw syncError('blob.hash_mismatch', 'blobId path is not "sha256:<64 hex>" (§5.9.1)');
26
33
  }
27
34
  const maxBytes = ctx.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
28
- if (request.bytes.length > maxBytes) {
29
- throw syncError('blob.too_large', `blob exceeds the ${maxBytes}-byte cap (§5.9.3)`);
35
+ const tooLarge = syncError('blob.too_large', 'blob exceeds the server size cap (§5.9.3)', JSON.stringify({ max: maxBytes }));
36
+ if (Number(request.headers.get('content-length')) > maxBytes)
37
+ throw tooLarge;
38
+ let bytes;
39
+ try {
40
+ bytes = await decodeRequestBody(request, maxBytes);
41
+ }
42
+ catch (error) {
43
+ throw error instanceof SyncError && error.code === 'sync.limit_exceeded'
44
+ ? tooLarge
45
+ : error;
30
46
  }
31
- const computed = await blobIdFor(request.bytes);
32
- if (computed !== request.blobId) {
47
+ if (bytes === undefined) {
48
+ throw syncError('sync.invalid_request', 'unsupported content encoding');
49
+ }
50
+ if ((await blobIdFor(bytes)) !== blobId) {
33
51
  throw syncError('blob.hash_mismatch', 'uploaded bytes do not hash to the blobId path (§5.9.3)');
34
52
  }
53
+ const contentType = request.headers
54
+ .get('content-type')
55
+ ?.split(';')[0]
56
+ ?.trim();
35
57
  const clock = clockOf(ctx);
36
- await store.put(ctx.partition, request.blobId, request.bytes, clock(), request.mediaType);
58
+ await store.put(ctx.partition, blobId, bytes, clock(), contentType === undefined ||
59
+ contentType === '' ||
60
+ contentType === 'application/octet-stream'
61
+ ? undefined
62
+ : contentType);
37
63
  if (ctx.events !== undefined) {
38
64
  emitEvent(ctx.events, {
39
65
  type: 'blob.uploaded',
40
66
  atMs: clock(),
41
67
  partition: ctx.partition,
42
68
  actorId: ctx.actorId,
43
- blobId: request.blobId,
44
- bytes: request.bytes.length,
69
+ blobId,
70
+ bytes: bytes.length,
45
71
  });
46
72
  }
47
73
  }
@@ -68,7 +94,7 @@ export async function handleBlobDownload(ctx, blobId) {
68
94
  outcome: 'ok',
69
95
  // Presigned delivery carries no inline bytes (§5.9.5 always-issue); the
70
96
  // byte count is unknown to the sync server, which exited the egress path.
71
- bytes: result.bytes?.length ?? 0,
97
+ bytes: 'byteLength' in result ? result.byteLength : 0,
72
98
  durationMs: clock() - startedAtMs,
73
99
  });
74
100
  return result;
@@ -95,21 +121,10 @@ async function downloadBlob(ctx, blobId) {
95
121
  if (!isBlobId(blobId)) {
96
122
  throw syncError('blob.not_found', 'malformed blobId (§5.9.1)');
97
123
  }
98
- // §5.9.5 always-issue: when presign is configured the server exits the
99
- // egress path — it only needs to know the object EXISTS (a HEAD/`has`), not
100
- // the bytes. Inline delivery loads the bytes; presigned delivery does not.
101
- const presignConfigured = ctx.blobSignedUrls !== undefined;
102
- let entry;
103
- if (presignConfigured) {
104
- if (!(await store.has(ctx.partition, blobId))) {
105
- throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
106
- }
107
- }
108
- else {
109
- entry = await store.get(ctx.partition, blobId);
110
- if (entry === undefined) {
111
- throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
112
- }
124
+ // Existence first (a HEAD), authorization second, bytes last: no blob byte
125
+ // is read for an actor the reference index does not authorize (§5.9.5).
126
+ if (!(await store.has(ctx.partition, blobId))) {
127
+ throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
113
128
  }
114
129
  // §5.9.5 authorization: resolve once, then test referencing rows.
115
130
  const listRows = ctx.storage.listRowsReferencingBlob;
@@ -177,24 +192,33 @@ async function downloadBlob(ctx, blobId) {
177
192
  headers: {
178
193
  ETag: `"${blobId}"`,
179
194
  'Cache-Control': 'private, max-age=0',
180
- Vary: 'Authorization, Accept-Encoding',
195
+ Vary: 'Authorization',
181
196
  },
182
197
  url: issued.url,
183
198
  urlExpiresAtMs: issued.urlExpiresAtMs,
184
199
  };
185
200
  }
186
- // Inline delivery (no presign store configured): the bytes were loaded.
187
- const bytes = entry?.bytes;
188
- if (bytes === undefined) {
189
- // Unreachable: presign not configured ⇒ `entry` was loaded above.
190
- throw syncError('blob.not_found', 'blob bytes unavailable (§5.9.5)');
201
+ // Inline delivery streams the stored bytes verbatim (identity encoding,
202
+ // §5.9.5). A store without `open` serves its `get` bytes as one chunk.
203
+ const opened = store.open !== undefined
204
+ ? await store.open(ctx.partition, blobId)
205
+ : await store.get(ctx.partition, blobId).then((entry) => entry === undefined
206
+ ? undefined
207
+ : {
208
+ record: entry.record,
209
+ body: bytesStream(entry.bytes),
210
+ byteLength: entry.bytes.length,
211
+ });
212
+ if (opened === undefined) {
213
+ throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
191
214
  }
192
215
  // §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
193
216
  // pointed at the route renders them. A JSON type stays octet-stream: a
194
217
  // JSON body is reserved for the signed-URL envelope.
195
- const mediaType = entry?.record.mediaType;
218
+ const mediaType = opened.record.mediaType;
196
219
  return {
197
- bytes,
220
+ body: opened.body,
221
+ byteLength: opened.byteLength,
198
222
  headers: {
199
223
  'Content-Type': mediaType === undefined ||
200
224
  mediaType.toLowerCase().includes('application/json')
@@ -204,7 +228,7 @@ async function downloadBlob(ctx, blobId) {
204
228
  'Content-Security-Policy': "sandbox; default-src 'none'",
205
229
  ETag: `"${blobId}"`,
206
230
  'Cache-Control': 'private, max-age=0',
207
- Vary: 'Authorization, Accept-Encoding',
231
+ Vary: 'Authorization',
208
232
  },
209
233
  };
210
234
  }
@@ -213,11 +237,9 @@ async function downloadBlob(ctx, blobId) {
213
237
  * auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
214
238
  * so any authenticated actor may obtain a grant within the size cap (§5.9.3).
215
239
  *
216
- * The size cap is enforced HERE, up front, against the declared `byteLength`
217
- * (the object-store hop cannot re-check the streamed byte count). Integrity is
218
- * NOT checked here: the object store places bytes at the content-addressed key,
219
- * and the address is verified at reference time (§5.9.6 push existence) and on
220
- * every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
240
+ * The size cap is enforced HERE, up front, against the declared `byteLength`;
241
+ * the presigned PUT binds that length and the content address, so the object
242
+ * store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
221
243
  * `present` marker if the blob already exists (idempotent §5.9.3), or an empty
222
244
  * result when no presigned-upload store is configured (client streams direct).
223
245
  */
@@ -235,7 +257,7 @@ export async function handleBlobUploadGrant(ctx, request) {
235
257
  request.byteLength < 0 ||
236
258
  request.byteLength > maxBytes) {
237
259
  // §5.9.3: cap enforced at grant issuance, before any URL is minted.
238
- throw syncError('blob.too_large', `blob byteLength exceeds the ${maxBytes}-byte cap (§5.9.3)`);
260
+ throw syncError('blob.too_large', 'blob byteLength exceeds the server size cap (§5.9.3)', JSON.stringify({ max: maxBytes }));
239
261
  }
240
262
  // §5.9.3 no presigned-upload store: report no grant; the client streams
241
263
  // through the direct PUT endpoint (capability, not fallback).
@@ -17,8 +17,15 @@ export interface BlobRecord {
17
17
  readonly byteLength: number;
18
18
  /** Advisory MIME type persisted from the upload, if any (§5.9.5). */
19
19
  readonly mediaType?: string;
20
+ /** Time of the latest upload; the orphan sweep's grace clock (§5.9.2). */
20
21
  readonly createdAtMs: number;
21
22
  }
23
+ /** `BlobStore.open`: a stored blob with its bytes as a stream. */
24
+ export interface BlobStream {
25
+ readonly record: BlobRecord;
26
+ readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
27
+ readonly byteLength: number;
28
+ }
22
29
  /**
23
30
  * Coarse blob-store counters for the admin/console read surface (work item
24
31
  * §2.5), partition-scoped. Optional. `count`/`bytes` are total stored
@@ -36,10 +43,12 @@ export interface BlobStoreStats {
36
43
  }
37
44
  export interface BlobStore {
38
45
  /**
39
- * Store blob bytes under `blobId` (idempotent — same id ⇒ no-op success,
40
- * §5.9.3). A re-upload that declares a media type records it on a blob
41
- * stored without one; a recorded media type never changes. The caller has
42
- * already verified the content address matches.
46
+ * Store blob bytes under `blobId` (idempotent: same id ⇒ success, §5.9.3).
47
+ * A re-upload moves `createdAtMs` forward to `nowMs`, so a blob that a
48
+ * second client is about to reference gets a fresh sweep grace period. A
49
+ * re-upload that declares a media type records it on a blob stored without
50
+ * one; a recorded media type never changes. The caller has already
51
+ * verified the content address matches.
43
52
  */
44
53
  put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
45
54
  /** Presence check for the push existence rule (§5.9.6) and download. */
@@ -49,12 +58,20 @@ export interface BlobStore {
49
58
  bytes: Uint8Array;
50
59
  } | undefined>;
51
60
  /**
52
- * §5.9.2 orphan sweep: delete blobs uploaded before `olderThanMs` that
53
- * no live row references. The set of referenced blobIds is supplied by
54
- * the storage reference index — the store never scans rows itself.
55
- * Returns the deleted blobIds.
61
+ * Optional streaming read: the record plus the bytes as a stream, so the
62
+ * inline download route relays a blob without buffering it (the
63
+ * `SegmentStore.open` shape). Stores without it are served through `get`.
64
+ */
65
+ open?(partition: string, blobId: string): Promise<BlobStream | undefined>;
66
+ /**
67
+ * §5.9.2 orphan sweep: delete blobs last uploaded before `olderThanMs`
68
+ * that no live row references. `isReferenced` asks the storage reference
69
+ * index about one blob; the store calls it for each old candidate right
70
+ * before that candidate's delete, so a reference that lands during the
71
+ * sweep keeps its blob. The store never scans rows itself. Returns the
72
+ * deleted blobIds.
56
73
  */
57
- sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
74
+ sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
58
75
  /**
59
76
  * §4.6.1 erasure: delete one blob whose last reference is gone. Deleting a
60
77
  * missing blob is a no-op.
@@ -74,7 +91,7 @@ export declare class MemoryBlobStore implements BlobStore {
74
91
  record: BlobRecord;
75
92
  bytes: Uint8Array;
76
93
  } | undefined>;
77
- sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
94
+ sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
78
95
  delete(partition: string, blobId: string): Promise<void>;
79
96
  stats(partition: string): Promise<BlobStoreStats>;
80
97
  }
@@ -95,25 +112,26 @@ export interface SweepOrphanBlobsOptions {
95
112
  export interface SweepOrphanBlobsResult {
96
113
  readonly partition: string;
97
114
  readonly swept: string[];
115
+ /** Candidates past the grace period that a live row still references. */
98
116
  readonly referencedCount: number;
99
117
  }
100
118
  /**
101
- * §5.9.2 orphan sweep — the ready-made GC helper. Reads the live keep-set
102
- * (`storage.listReferencedBlobIds`) and hands it, with the grace cutoff, to
103
- * `blobStore.sweepOrphans`, which deletes only blobs BOTH unreferenced AND
104
- * older than the grace period. Returns the swept ids and emits one
105
- * `blob.swept` ops event.
119
+ * §5.9.2 orphan sweep — the ready-made GC helper. Hands the grace cutoff and
120
+ * a reference check to `blobStore.sweepOrphans`, which deletes only blobs
121
+ * older than the grace period that no row references at the moment of the
122
+ * delete. The check consults the keep-set read when the sweep started and,
123
+ * for a blob outside it, reads the reference index again right before that
124
+ * blob's delete, so a push that lands during the sweep keeps its blob.
125
+ * Returns the swept ids and emits one `blob.swept` ops event.
106
126
  *
107
- * The upload-before-reference race, stated honestly: a client uploads bytes
108
- * before pushing the row that references them (§5.9.2), so a just-uploaded
109
- * blob is unreferenced until its push arrives. The grace period is the ONLY
110
- * thing standing between that fresh blob and deletion — keep it far above any
111
- * push latency (the 24 h default is deliberately generous). The store's
112
- * `createdAtMs` (object metadata) is the upload time the cutoff compares
113
- * against.
127
+ * A client uploads bytes before pushing the row that references them
128
+ * (§5.9.2), so a just-uploaded blob is unreferenced until its push arrives.
129
+ * The grace period protects that window: keep it far above any push latency
130
+ * (the 24 h default). The store's upload time (`createdAtMs` metadata, or the
131
+ * backend's last-modified time for an object uploaded through a presigned
132
+ * PUT) is what the cutoff compares against; a re-upload refreshes it.
114
133
  *
115
134
  * Requires `storage.listReferencedBlobIds` (the §5.9.4 reference index). A
116
- * storage backend without it cannot be swept safely — this throws rather
117
- * than sweeping against an empty keep-set (which would delete everything).
135
+ * storage backend without it cannot be swept safely, so this throws.
118
136
  */
119
137
  export declare function sweepOrphanBlobs(storage: ServerStorage, blobStore: BlobStore, partition: string, options?: SweepOrphanBlobsOptions): Promise<SweepOrphanBlobsResult>;
Binary file
@@ -29,10 +29,11 @@
29
29
  *
30
30
  * The orphan sweep is the ONLY operation that LISTs: `sweepOrphans` pages a
31
31
  * prefix listing over the `blob/` prefix (an admin/GC path, off the hot path)
32
- * to find candidates, reads each object's `createdAtMs` from its metadata for
33
- * the grace check, and DELETEs the unreferenced-and-old ones.
32
+ * to find candidates, reads each object's upload time from its metadata (or
33
+ * the backend's last-modified time for an object a presigned PUT wrote), and
34
+ * DELETEs the old ones the reference index does not name at that moment.
34
35
  */
35
- import type { BlobRecord, BlobStore, BlobStoreStats } from './blob-store.js';
36
+ import type { BlobRecord, BlobStore, BlobStoreStats, BlobStream } from './blob-store.js';
36
37
  import type { ObjectBucket } from './object-bucket.js';
37
38
  export interface BucketBlobStoreOptions {
38
39
  /** Key namespace inside the bucket, e.g. `syncular/`. Default: none. */
@@ -57,14 +58,17 @@ export declare class BucketBlobStore implements BlobStore {
57
58
  record: BlobRecord;
58
59
  bytes: Uint8Array;
59
60
  } | undefined>;
61
+ /** The record plus the object GET's body, never buffered (see `get`). */
62
+ open(partition: string, blobId: string): Promise<BlobStream | undefined>;
60
63
  /**
61
64
  * §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
62
- * {partition}/` prefix, HEAD each object for its `createdAtMs`, and DELETE
63
- * the ones both older than `olderThanMs` AND absent from
64
- * `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
65
- * referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
65
+ * {partition}/` prefix, HEAD each object for its upload time (the metadata
66
+ * `createdAtMs`, else the backend's last-modified time; an object with
67
+ * neither is skipped), and DELETE the ones older than `olderThanMs` that
68
+ * `isReferenced` denies right before the delete. Returns the deleted
69
+ * blobIds. Off the hot path — a host-scheduled GC job.
66
70
  */
67
- sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
71
+ sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
68
72
  /**
69
73
  * Store-wide counters from the pointer-object accumulator (never a LIST on
70
74
  * the read path). `approximate: true` for the same reasons the segment
@@ -138,38 +138,26 @@ export class BucketBlobStore {
138
138
  };
139
139
  const objectKey = this.objectKeyFor(partition, blobId);
140
140
  // Idempotent re-put (same content-address ⇒ same key): count each distinct
141
- // blob once, and preserve the FIRST createdAtMs so the sweep's grace clock
142
- // is not reset by a re-upload (§5.9.2 upload age).
141
+ // blob once. A re-upload rewrites the object so its metadata carries the
142
+ // re-upload time (the sweep's grace clock, §5.9.2) and keeps a recorded
143
+ // media type.
143
144
  const preexisting = await this.#bucket.head(objectKey);
144
145
  const storedMeta = preexisting?.metadata[BLOB_META_KEY];
145
146
  const existing = storedMeta === undefined
146
147
  ? undefined
147
148
  : parseMetaJson(base64urlToUtf8(storedMeta), blobId, this.#label);
148
- if (existing !== undefined &&
149
- (existing.mediaType !== undefined || mediaType === undefined)) {
150
- return {
151
- blobId,
152
- partition,
153
- byteLength: existing.byteLength,
154
- ...(existing.mediaType !== undefined
155
- ? { mediaType: existing.mediaType }
156
- : {}),
157
- createdAtMs: existing.createdAtMs,
158
- };
159
- }
160
- // A re-upload that names the media type an earlier upload left out
161
- // rewrites the object under the same precondition, keeping its age.
149
+ const storedType = existing?.mediaType ?? mediaType;
162
150
  const meta = {
163
151
  byteLength: bytes.length,
164
- ...(mediaType !== undefined ? { mediaType } : {}),
165
- createdAtMs: existing?.createdAtMs ?? nowMs,
152
+ ...(storedType !== undefined ? { mediaType: storedType } : {}),
153
+ createdAtMs: Math.max(existing?.createdAtMs ?? nowMs, nowMs),
166
154
  };
167
155
  // Conditional write: of two concurrent uploads of the same blob exactly
168
- // one lands, so the first createdAtMs survives and the blob counts once.
169
- // The loser re-reads and returns the winner's record.
156
+ // one lands per pre-state and the blob counts once. The loser re-reads
157
+ // and writes over the winner.
170
158
  const written = await this.#bucket.put(objectKey, {
171
159
  body: bytes,
172
- contentType: mediaType ?? 'application/octet-stream',
160
+ contentType: storedType ?? 'application/octet-stream',
173
161
  metadata: { [BLOB_META_KEY]: utf8ToBase64url(metaToJson(meta)) },
174
162
  ...(preexisting === undefined
175
163
  ? { createOnly: true }
@@ -179,7 +167,11 @@ export class BucketBlobStore {
179
167
  return this.put(partition, blobId, bytes, nowMs, mediaType);
180
168
  if (preexisting === undefined)
181
169
  await this.#bumpStats({ count: 1, bytes: bytes.length });
182
- return { ...record, createdAtMs: meta.createdAtMs };
170
+ return {
171
+ ...record,
172
+ ...(storedType !== undefined ? { mediaType: storedType } : {}),
173
+ createdAtMs: meta.createdAtMs,
174
+ };
183
175
  }
184
176
  async delete(partition, blobId) {
185
177
  await this.#bucket.delete(this.objectKeyFor(partition, blobId));
@@ -191,35 +183,51 @@ export class BucketBlobStore {
191
183
  return head !== undefined;
192
184
  }
193
185
  async get(partition, blobId) {
194
- if (!BLOB_ID_PATTERN.test(blobId))
195
- return undefined;
196
- const object = await this.#bucket.get(this.objectKeyFor(partition, blobId));
186
+ const object = await this.#object(partition, blobId);
197
187
  if (object === undefined)
198
188
  return undefined;
199
189
  const bytes = new Uint8Array(await object.arrayBuffer());
190
+ return {
191
+ record: this.#record(partition, blobId, object, bytes.length),
192
+ bytes,
193
+ };
194
+ }
195
+ /** The record plus the object GET's body, never buffered (see `get`). */
196
+ async open(partition, blobId) {
197
+ const object = await this.#object(partition, blobId);
198
+ if (object === undefined || object.body === null)
199
+ return undefined;
200
+ const record = this.#record(partition, blobId, object, object.size ?? 0);
201
+ return { record, body: object.body, byteLength: record.byteLength };
202
+ }
203
+ async #object(partition, blobId) {
204
+ if (!BLOB_ID_PATTERN.test(blobId))
205
+ return undefined;
206
+ return this.#bucket.get(this.objectKeyFor(partition, blobId));
207
+ }
208
+ /** An object without Syncular metadata was written by a presigned PUT. */
209
+ #record(partition, blobId, object, byteLength) {
200
210
  const rawMeta = object.metadata[BLOB_META_KEY];
201
211
  const meta = rawMeta !== undefined
202
212
  ? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
203
- : { byteLength: bytes.length, createdAtMs: 0 };
213
+ : { byteLength, createdAtMs: 0 };
204
214
  return {
205
- record: {
206
- blobId,
207
- partition,
208
- byteLength: meta.byteLength,
209
- ...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
210
- createdAtMs: meta.createdAtMs,
211
- },
212
- bytes,
215
+ blobId,
216
+ partition,
217
+ byteLength: meta.byteLength,
218
+ ...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
219
+ createdAtMs: meta.createdAtMs,
213
220
  };
214
221
  }
215
222
  /**
216
223
  * §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
217
- * {partition}/` prefix, HEAD each object for its `createdAtMs`, and DELETE
218
- * the ones both older than `olderThanMs` AND absent from
219
- * `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
220
- * referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
224
+ * {partition}/` prefix, HEAD each object for its upload time (the metadata
225
+ * `createdAtMs`, else the backend's last-modified time; an object with
226
+ * neither is skipped), and DELETE the ones older than `olderThanMs` that
227
+ * `isReferenced` denies right before the delete. Returns the deleted
228
+ * blobIds. Off the hot path — a host-scheduled GC job.
221
229
  */
222
- async sweepOrphans(partition, olderThanMs, referencedBlobIds) {
230
+ async sweepOrphans(partition, olderThanMs, isReferenced) {
223
231
  const listPrefix = `${this.#prefix}blob/${partition}/`;
224
232
  const swept = [];
225
233
  let deletedBytes = 0;
@@ -235,20 +243,21 @@ export class BucketBlobStore {
235
243
  const blobId = `sha256:${rest.slice('sha256/'.length)}`;
236
244
  if (!BLOB_ID_PATTERN.test(blobId))
237
245
  continue;
238
- if (referencedBlobIds.has(blobId))
239
- continue;
240
246
  const head = await this.#bucket.head(key);
241
247
  if (head === undefined)
242
248
  continue;
243
249
  const rawMeta = head.metadata[BLOB_META_KEY];
244
- const meta = rawMeta !== undefined
250
+ const uploadedAtMs = rawMeta !== undefined
245
251
  ? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
246
- : { byteLength: head.size ?? 0, createdAtMs: 0 };
247
- if (meta.createdAtMs >= olderThanMs)
252
+ .createdAtMs
253
+ : head.lastModifiedMs;
254
+ if (uploadedAtMs === undefined || uploadedAtMs >= olderThanMs)
255
+ continue;
256
+ if (await isReferenced(blobId))
248
257
  continue;
249
258
  await this.#bucket.delete(key);
250
259
  swept.push(blobId);
251
- deletedBytes += meta.byteLength;
260
+ deletedBytes += head.size ?? 0;
252
261
  }
253
262
  } while (cursor !== undefined);
254
263
  if (swept.length > 0) {
@@ -1,4 +1,4 @@
1
- import { mergeSegmentRecord, publicationRecord, publicationsForRecord, segmentBytesEqual, segmentIdFor, selectSegmentPublication, } from './segment-store.js';
1
+ import { mergeSegmentRecord, publicationRecord, publicationsForRecord, segmentIdFor, selectSegmentPublication, } from './segment-store.js';
2
2
  import { sha256Hex } from './sigv4.js';
3
3
  const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
4
4
  const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
@@ -311,25 +311,27 @@ export class BucketSegmentStore {
311
311
  throw new Error(`${this.#label}: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`);
312
312
  }
313
313
  /**
314
- * Write the immutable bytes object. Create-only, so a second publisher of
315
- * identical bytes is a no-op and a second publisher of DIFFERENT bytes under
316
- * the same content address is the hash collision §5.1 requires to fail
317
- * loudly instead of overwriting.
314
+ * Write the immutable bytes object unless it is stored already. A HEAD
315
+ * comes first, so republishing a segment (every bootstrap pull of an
316
+ * unchanged window) moves no bytes. The write is create-only, so two first
317
+ * publishers cannot overwrite each other. A stored object under the same
318
+ * content address with a different size is the hash collision §5.1
319
+ * requires to fail loudly; equal size under one SHA-256 address is the same
320
+ * content.
318
321
  */
319
322
  async #putBytes(segmentId, bytes) {
320
323
  const key = this.objectKeyFor(segmentId);
321
- const written = await this.#bucket.put(key, {
322
- body: bytes,
323
- contentType: 'application/octet-stream',
324
- createOnly: true,
325
- });
326
- if (written)
327
- return;
328
- const current = await this.#bucket.get(key);
329
- if (current === undefined)
324
+ const head = await this.#bucket.head(key);
325
+ if (head === undefined &&
326
+ (await this.#bucket.put(key, {
327
+ body: bytes,
328
+ contentType: 'application/octet-stream',
329
+ createOnly: true,
330
+ }))) {
330
331
  return;
331
- const stored = new Uint8Array(await current.arrayBuffer());
332
- if (!segmentBytesEqual(stored, bytes)) {
332
+ }
333
+ const stored = head ?? (await this.#bucket.head(key));
334
+ if (stored !== undefined && stored.size !== bytes.length) {
333
335
  throw new Error(`${this.#label}: content-address collision at ${segmentId} (§5.1)`);
334
336
  }
335
337
  }
@@ -40,6 +40,9 @@ export declare function encodeStream(body: ReadableStream<Uint8Array<ArrayBuffer
40
40
  * `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
41
41
  * servers MUST accept it). Returns `undefined` for any other declared
42
42
  * encoding — the adapter answers 415 rather than parse bytes it cannot read.
43
- * A gzip body that does not decompress is a `sync.invalid_request`.
43
+ * A gzip body that does not decompress is a `sync.invalid_request`. One read
44
+ * loop counts the decoded bytes of identity and gzip bodies alike and stops
45
+ * with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
46
+ * small compressed body never inflates past the cap.
44
47
  */
45
- export declare function decodeRequestBody(request: Request): Promise<Uint8Array | undefined>;
48
+ export declare function decodeRequestBody(request: Request, maxBytes: number): Promise<Uint8Array | undefined>;
@@ -1,7 +1,7 @@
1
1
  /**
2
- * §1.3 transport compression for every buffered HTTP body the adapters
3
- * serve — `<mount>/sync`, `<mount>/operations`, blob GETs and segment
4
- * downloads: negotiate `Content-Encoding` from the request's
2
+ * §1.3 transport compression for the HTTP bodies the adapters serve —
3
+ * `<mount>/sync`, `<mount>/operations` and segment downloads (blob downloads
4
+ * stream identity, §5.9.5): negotiate `Content-Encoding` from the request's
5
5
  * `Accept-Encoding` (zstd preferred, gzip fallback, identity when the client
6
6
  * offers neither), feature-detecting the runtime's codecs, zero
7
7
  * dependencies. Bun supplies zstd + gzip natively; every other runtime
@@ -121,26 +121,55 @@ export function encodeStream(body, acceptEncoding) {
121
121
  * `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
122
122
  * servers MUST accept it). Returns `undefined` for any other declared
123
123
  * encoding — the adapter answers 415 rather than parse bytes it cannot read.
124
- * A gzip body that does not decompress is a `sync.invalid_request`.
124
+ * A gzip body that does not decompress is a `sync.invalid_request`. One read
125
+ * loop counts the decoded bytes of identity and gzip bodies alike and stops
126
+ * with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
127
+ * small compressed body never inflates past the cap.
125
128
  */
126
- export async function decodeRequestBody(request) {
129
+ export async function decodeRequestBody(request, maxBytes) {
127
130
  const encoding = request.headers
128
131
  .get('content-encoding')
129
132
  ?.trim()
130
133
  .toLowerCase();
131
- if (encoding === undefined || encoding === '' || encoding === 'identity') {
132
- return new Uint8Array(await request.arrayBuffer());
133
- }
134
- if (encoding !== 'gzip' || typeof DecompressionStream !== 'function') {
134
+ const identity = encoding === undefined || encoding === '' || encoding === 'identity';
135
+ if (!identity &&
136
+ (encoding !== 'gzip' || typeof DecompressionStream !== 'function')) {
135
137
  return undefined;
136
138
  }
137
139
  if (request.body === null) {
140
+ if (identity)
141
+ return new Uint8Array(0);
138
142
  throw syncError('sync.invalid_request', 'malformed gzip request body');
139
143
  }
140
- try {
141
- return new Uint8Array(await new Response(request.body.pipeThrough(new DecompressionStream('gzip'))).arrayBuffer());
144
+ const reader = (identity
145
+ ? request.body
146
+ : request.body.pipeThrough(new DecompressionStream('gzip'))).getReader();
147
+ const chunks = [];
148
+ let total = 0;
149
+ for (;;) {
150
+ let step;
151
+ try {
152
+ step = await reader.read();
153
+ }
154
+ catch (error) {
155
+ if (identity)
156
+ throw error;
157
+ throw syncError('sync.invalid_request', 'malformed gzip request body');
158
+ }
159
+ if (step.done)
160
+ break;
161
+ total += step.value.length;
162
+ if (total > maxBytes) {
163
+ await reader.cancel().catch(() => { });
164
+ throw syncError('sync.limit_exceeded', 'request body exceeds the server limit (§1.7)', JSON.stringify({ limit: 'maxRequestBytes', max: maxBytes }));
165
+ }
166
+ chunks.push(step.value);
142
167
  }
143
- catch {
144
- throw syncError('sync.invalid_request', 'malformed gzip request body');
168
+ const bytes = new Uint8Array(total);
169
+ let offset = 0;
170
+ for (const chunk of chunks) {
171
+ bytes.set(chunk, offset);
172
+ offset += chunk.length;
145
173
  }
174
+ return bytes;
146
175
  }