@syncular/server 0.57.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +31 -22
  2. package/dist/admin.js +8 -4
  3. package/dist/blob-handlers.d.ts +25 -31
  4. package/dist/blob-handlers.js +63 -41
  5. package/dist/blob-store.d.ts +42 -24
  6. package/dist/blob-store.js +0 -0
  7. package/dist/bucket-blob-store.d.ts +12 -8
  8. package/dist/bucket-blob-store.js +54 -45
  9. package/dist/bucket-segment-store.js +18 -16
  10. package/dist/content-encoding.d.ts +5 -2
  11. package/dist/content-encoding.js +42 -13
  12. package/dist/context.d.ts +16 -0
  13. package/dist/context.js +21 -1
  14. package/dist/d1-storage.d.ts +14 -98
  15. package/dist/d1-storage.js +223 -1111
  16. package/dist/errors.js +8 -0
  17. package/dist/events.d.ts +1 -1
  18. package/dist/handler.js +135 -103
  19. package/dist/local-authority.d.ts +2 -0
  20. package/dist/local-authority.js +17 -3
  21. package/dist/object-bucket.d.ts +2 -0
  22. package/dist/operations-realtime.js +69 -50
  23. package/dist/pg-executor-pglite.d.ts +5 -6
  24. package/dist/pg-executor-pglite.js +22 -14
  25. package/dist/pg-executor.d.ts +4 -1
  26. package/dist/postgres-storage.d.ts +1 -1
  27. package/dist/postgres-storage.js +182 -105
  28. package/dist/pull.d.ts +28 -20
  29. package/dist/pull.js +85 -87
  30. package/dist/push.js +272 -147
  31. package/dist/realtime.d.ts +68 -28
  32. package/dist/realtime.js +297 -136
  33. package/dist/relational-rows.d.ts +27 -15
  34. package/dist/relational-rows.js +38 -22
  35. package/dist/s3-blob-store.d.ts +11 -12
  36. package/dist/s3-blob-store.js +20 -13
  37. package/dist/s3-object-bucket.d.ts +8 -1
  38. package/dist/s3-object-bucket.js +16 -3
  39. package/dist/schema.d.ts +14 -0
  40. package/dist/schema.js +23 -0
  41. package/dist/scopes.d.ts +6 -2
  42. package/dist/scopes.js +22 -4
  43. package/dist/seed.js +1 -2
  44. package/dist/segment-download.js +18 -4
  45. package/dist/signed-url.d.ts +2 -2
  46. package/dist/signed-url.js +9 -2
  47. package/dist/sigv4.d.ts +8 -2
  48. package/dist/sigv4.js +8 -4
  49. package/dist/sqlite-blob-store.d.ts +1 -1
  50. package/dist/sqlite-blob-store.js +23 -22
  51. package/dist/sqlite-dialect.d.ts +150 -47
  52. package/dist/sqlite-dialect.js +987 -88
  53. package/dist/sqlite-lease-store.js +13 -18
  54. package/dist/sqlite-storage.d.ts +9 -62
  55. package/dist/sqlite-storage.js +293 -1117
  56. package/dist/storage.d.ts +32 -12
  57. package/package.json +2 -2
  58. package/src/admin.ts +16 -2
  59. package/src/blob-handlers.ts +91 -72
  60. package/src/blob-store.ts +0 -0
  61. package/src/bucket-blob-store.ts +75 -48
  62. package/src/bucket-segment-store.ts +20 -15
  63. package/src/content-encoding.ts +48 -16
  64. package/src/context.ts +32 -1
  65. package/src/d1-storage.ts +326 -1692
  66. package/src/errors.ts +8 -0
  67. package/src/events.ts +1 -1
  68. package/src/handler.ts +178 -127
  69. package/src/local-authority.ts +17 -3
  70. package/src/object-bucket.ts +2 -0
  71. package/src/operations-realtime.ts +85 -61
  72. package/src/pg-executor-pglite.ts +27 -23
  73. package/src/pg-executor.ts +5 -1
  74. package/src/postgres-storage.ts +231 -138
  75. package/src/pull.ts +133 -147
  76. package/src/push.ts +356 -205
  77. package/src/realtime.ts +380 -179
  78. package/src/relational-rows.ts +54 -27
  79. package/src/s3-blob-store.ts +23 -13
  80. package/src/s3-object-bucket.ts +20 -2
  81. package/src/schema.ts +31 -0
  82. package/src/scopes.ts +23 -3
  83. package/src/seed.ts +1 -2
  84. package/src/segment-download.ts +25 -4
  85. package/src/signed-url.ts +10 -4
  86. package/src/sigv4.ts +17 -4
  87. package/src/sqlite-blob-store.ts +27 -26
  88. package/src/sqlite-dialect.ts +1607 -114
  89. package/src/sqlite-lease-store.ts +12 -23
  90. package/src/sqlite-storage.ts +360 -1811
  91. package/src/storage.ts +33 -12
package/dist/storage.d.ts CHANGED
@@ -128,10 +128,25 @@ export interface NewChange {
128
128
  readonly op: 'upsert' | 'delete';
129
129
  /** Present for upsert (§2.2). */
130
130
  readonly rowVersion?: number;
131
+ /**
132
+ * Present for delete: the deleted row's last `server_version`, recorded on
133
+ * its tombstone (§2.2). Never reaches the wire.
134
+ */
135
+ readonly deletedVersion?: number;
131
136
  readonly scopes: Record<string, string>;
132
137
  /** Present for upsert. */
133
138
  readonly payload?: Uint8Array;
134
139
  }
140
+ /** §2.2 delete-precedence tombstone of a row deleted within the horizon. */
141
+ export interface Tombstone {
142
+ readonly commitSeq: number;
143
+ /**
144
+ * The deleted row's last `server_version` and stored scopes. Both are
145
+ * absent on a tombstone written before servers recorded them.
146
+ */
147
+ readonly serverVersion?: number;
148
+ readonly scopes?: Record<string, string>;
149
+ }
135
150
  export interface NewCommit {
136
151
  readonly clientId: string;
137
152
  readonly clientCommitId: string;
@@ -453,17 +468,17 @@ export interface StorageTransaction {
453
468
  readonly rowIds: readonly string[];
454
469
  }[]): Promise<void>;
455
470
  /**
456
- * §5 delete-precedence tombstone read: the `commit_seq` of the newest
457
- * applied delete of this row still within the pruning horizon, or
458
- * `undefined` when no tombstone exists. Consulted only when an upsert's
459
- * stored row is absent. Required: in-tree adapters implement it on the
460
- * transaction's own connection with read-your-own-writes semantics.
471
+ * §5 delete-precedence tombstone read: the newest applied delete of this
472
+ * row still within the pruning horizon, or `undefined` when no tombstone
473
+ * exists. Consulted only when an upsert's stored row is absent. Required:
474
+ * in-tree adapters implement it on the transaction's own connection with
475
+ * read-your-own-writes semantics.
461
476
  */
462
- getTombstoneSeq(table: string, rowId: string): Promise<number | undefined>;
477
+ getTombstone(table: string, rowId: string): Promise<Tombstone | undefined>;
463
478
  /**
464
- * Remove the row's tombstone because the row was recreated in this
465
- * transaction (§5: an explicit insert recreates). Called on every insert
466
- * over an absent row; a no-op when no tombstone exists.
479
+ * Remove the row's tombstone because the row is live again in this
480
+ * transaction: an insert over its tombstone (§6.2) or a §2.2 scope move.
481
+ * A no-op when no tombstone exists.
467
482
  */
468
483
  clearTombstone(table: string, rowId: string): Promise<void>;
469
484
  /**
@@ -763,7 +778,10 @@ export interface ServerStorage {
763
778
  /** Advance an existing client's ACK cursor and timestamp atomically,
764
779
  * preserving actor, wire version, and subscriptions. Missing records stay absent. */
765
780
  updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
766
- /** Minimum cursor with updatedAtMs >= cutoff; null when none are active. */
781
+ /**
782
+ * Minimum non-negative cursor with updatedAtMs >= cutoff (§4.6); null when
783
+ * none are active. A bootstrapping client (cursor < 0) holds no cursor.
784
+ */
767
785
  getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
768
786
  /** Cursor records for client listings and administrative counts. */
769
787
  listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
@@ -791,9 +809,11 @@ export interface ServerStorage {
791
809
  * `eraseScopeValueHistory`: under the partition write lock, removes up to
792
810
  * `limit` upsert change records of `table` at or below `throughSeq` whose
793
811
  * scopes bind `variable` to `value`, then up to `limit` change scope index
794
- * entries of that binding whose commit holds no such change any more.
812
+ * entries of that binding whose commit holds no such change any more,
813
+ * then strips `serverRow` from up to `limit` stored conflict results of
814
+ * `table` whose conflict authorization scopes bind `variable` to `value`.
795
815
  * Delete change records stay: clients behind the erasure still need them.
796
- * Returns the number of records removed.
816
+ * Returns the number of records removed or stripped.
797
817
  */
798
818
  listScopeValueRows?(partition: string, query: ScopeValueQuery): Promise<string[]>;
799
819
  eraseScopeValueHistory?(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.57.0",
3
+ "version": "0.59.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.57.0"
71
+ "@syncular/core": "0.59.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
package/src/admin.ts CHANGED
@@ -535,6 +535,7 @@ const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
535
535
 
536
536
  /** One known client's subscriptions on the row's table and its verdict. */
537
537
  function traceVerdict(
538
+ declared: ReadonlySet<string>,
538
539
  record: ClientRecord,
539
540
  row: AdminRowInspection,
540
541
  lastChange: RowChangeRecord | undefined,
@@ -565,7 +566,10 @@ function traceVerdict(
565
566
  mismatches,
566
567
  };
567
568
  if (allowed === undefined || allowed === 'outage') return traced;
568
- const effective = computeEffective(sub.scopes, { ok: true, allowed });
569
+ const effective = computeEffective(declared, sub.scopes, {
570
+ ok: true,
571
+ allowed,
572
+ });
569
573
  return {
570
574
  ...traced,
571
575
  effectiveMatch:
@@ -906,7 +910,14 @@ export class SyncularAdmin {
906
910
  ...base,
907
911
  client: (await this.#clientView(partition))(record),
908
912
  ...(allowed !== undefined ? { allowed } : {}),
909
- ...traceVerdict(record, row, lastChange, horizonSeq, allowed),
913
+ ...traceVerdict(
914
+ this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set(),
915
+ record,
916
+ row,
917
+ lastChange,
918
+ horizonSeq,
919
+ allowed,
920
+ ),
910
921
  };
911
922
  }
912
923
 
@@ -929,6 +940,8 @@ export class SyncularAdmin {
929
940
  const lastChange = row.history[0];
930
941
  const horizonSeq = await this.#storage.getHorizonSeq(partition);
931
942
  const view = await this.#clientView(partition);
943
+ const declared =
944
+ this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set();
932
945
  const allowedByActor = new Map<string, ScopeMap | 'outage' | undefined>();
933
946
  const clients: AdminRowDeliveryClient[] = [];
934
947
  for (const record of await list(partition)) {
@@ -941,6 +954,7 @@ export class SyncularAdmin {
941
954
  clients.push({
942
955
  client: view(record),
943
956
  verdict: traceVerdict(
957
+ declared,
944
958
  record,
945
959
  row,
946
960
  lastChange,
@@ -6,7 +6,8 @@
6
6
  * `blobId` is never a bearer capability. Framework-free; the hono/runtime
7
7
  * adapters map these onto `PUT/GET <mount>/blobs/{blobId}`.
8
8
  */
9
- import { type BlobRecord, blobIdFor, isBlobId } from './blob-store';
9
+ import { blobIdFor, isBlobId } from './blob-store';
10
+ import { decodeRequestBody } from './content-encoding';
10
11
  import type { SyncRequestContext } from './context';
11
12
  import {
12
13
  clockOf,
@@ -17,36 +18,29 @@ import { SyncError, syncError } from './errors';
17
18
  import { emitEvent } from './events';
18
19
  import { compileSchema } from './schema';
19
20
  import { authorizeWrite, type ResolvedScopes } from './scopes';
21
+ import { bytesStream } from './segment-store';
20
22
  import { issueBlobUploadUrl, issueBlobUrl } from './signed-url';
21
23
 
22
24
  const DEFAULT_MAX_BLOB_BYTES = 64 * 1024 * 1024;
23
25
 
24
- export interface BlobUploadRequest {
25
- readonly blobId: string;
26
- readonly bytes: Uint8Array;
27
- /** Advisory MIME type from the upload `Content-Type`, if any. */
28
- readonly mediaType?: string;
29
- }
30
-
31
- export interface BlobDownloadResult {
32
- /**
33
- * The blob bytes, when served inline. **Absent** when the host configured
34
- * `blobSignedUrls` and a `url` was issued: the sync server exits the egress
35
- * path and the client fetches `url` directly (§5.9.5 always-issue).
36
- */
37
- readonly bytes?: Uint8Array;
38
- readonly headers: Record<string, string>;
39
- /**
40
- * §5.9.5 delegated presign (always-issue): a provider-signed GET URL for
41
- * the bytes, issued only after the row-derived authorization check passed.
42
- * Present iff the host configured `blobSignedUrls`; when present, `bytes` is
43
- * absent and the client MUST fetch the URL directly (no host auth), verify
44
- * the content address, and on failure re-request this endpoint — never fall
45
- * through (§5.9.5 recovery rule).
46
- */
47
- readonly url?: string;
48
- readonly urlExpiresAtMs?: number;
49
- }
26
+ /**
27
+ * A §5.9.5 download: the stored bytes as a stream with their length (inline
28
+ * delivery), or, when the host configured `blobSignedUrls`, a provider-signed
29
+ * GET URL issued after the row-derived authorization check. The client
30
+ * fetches a URL directly (no host auth), verifies the content address, and on
31
+ * failure re-requests this endpoint (§5.9.5 recovery rule).
32
+ */
33
+ export type BlobDownloadResult =
34
+ | {
35
+ readonly headers: Record<string, string>;
36
+ readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
37
+ readonly byteLength: number;
38
+ }
39
+ | {
40
+ readonly headers: Record<string, string>;
41
+ readonly url: string;
42
+ readonly urlExpiresAtMs: number;
43
+ };
50
44
 
51
45
  /** `POST <mount>/blobs/{blobId}/upload-grant` request body (§5.9.3). */
52
46
  export interface BlobUploadGrantRequest {
@@ -71,43 +65,67 @@ export interface BlobUploadGrantResult {
71
65
  readonly present?: boolean;
72
66
  }
73
67
 
74
- /** `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. */
68
+ /**
69
+ * `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
70
+ * size cap applies to the declared `Content-Length` before any byte is read
71
+ * and again to the counted decoded bytes while reading, so an oversized body
72
+ * is never buffered whole.
73
+ */
75
74
  export async function handleBlobUpload(
76
75
  ctx: SyncRequestContext,
77
- request: BlobUploadRequest,
76
+ blobId: string,
77
+ request: Request,
78
78
  ): Promise<void> {
79
79
  await touchAuthenticatedPartition(ctx);
80
80
  const store = ctx.blobs;
81
81
  if (store === undefined) {
82
82
  throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
83
83
  }
84
- if (!isBlobId(request.blobId)) {
84
+ if (!isBlobId(blobId)) {
85
85
  throw syncError(
86
86
  'blob.hash_mismatch',
87
87
  'blobId path is not "sha256:<64 hex>" (§5.9.1)',
88
88
  );
89
89
  }
90
90
  const maxBytes = ctx.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
91
- if (request.bytes.length > maxBytes) {
92
- throw syncError(
93
- 'blob.too_large',
94
- `blob exceeds the ${maxBytes}-byte cap (§5.9.3)`,
95
- );
91
+ const tooLarge = syncError(
92
+ 'blob.too_large',
93
+ 'blob exceeds the server size cap (§5.9.3)',
94
+ JSON.stringify({ max: maxBytes }),
95
+ );
96
+ if (Number(request.headers.get('content-length')) > maxBytes) throw tooLarge;
97
+ let bytes: Uint8Array | undefined;
98
+ try {
99
+ bytes = await decodeRequestBody(request, maxBytes);
100
+ } catch (error) {
101
+ throw error instanceof SyncError && error.code === 'sync.limit_exceeded'
102
+ ? tooLarge
103
+ : error;
96
104
  }
97
- const computed = await blobIdFor(request.bytes);
98
- if (computed !== request.blobId) {
105
+ if (bytes === undefined) {
106
+ throw syncError('sync.invalid_request', 'unsupported content encoding');
107
+ }
108
+ if ((await blobIdFor(bytes)) !== blobId) {
99
109
  throw syncError(
100
110
  'blob.hash_mismatch',
101
111
  'uploaded bytes do not hash to the blobId path (§5.9.3)',
102
112
  );
103
113
  }
114
+ const contentType = request.headers
115
+ .get('content-type')
116
+ ?.split(';')[0]
117
+ ?.trim();
104
118
  const clock = clockOf(ctx);
105
119
  await store.put(
106
120
  ctx.partition,
107
- request.blobId,
108
- request.bytes,
121
+ blobId,
122
+ bytes,
109
123
  clock(),
110
- request.mediaType,
124
+ contentType === undefined ||
125
+ contentType === '' ||
126
+ contentType === 'application/octet-stream'
127
+ ? undefined
128
+ : contentType,
111
129
  );
112
130
  if (ctx.events !== undefined) {
113
131
  emitEvent(ctx.events, {
@@ -115,8 +133,8 @@ export async function handleBlobUpload(
115
133
  atMs: clock(),
116
134
  partition: ctx.partition,
117
135
  actorId: ctx.actorId,
118
- blobId: request.blobId,
119
- bytes: request.bytes.length,
136
+ blobId,
137
+ bytes: bytes.length,
120
138
  });
121
139
  }
122
140
  }
@@ -146,7 +164,7 @@ export async function handleBlobDownload(
146
164
  outcome: 'ok',
147
165
  // Presigned delivery carries no inline bytes (§5.9.5 always-issue); the
148
166
  // byte count is unknown to the sync server, which exited the egress path.
149
- bytes: result.bytes?.length ?? 0,
167
+ bytes: 'byteLength' in result ? result.byteLength : 0,
150
168
  durationMs: clock() - startedAtMs,
151
169
  });
152
170
  return result;
@@ -176,20 +194,10 @@ async function downloadBlob(
176
194
  if (!isBlobId(blobId)) {
177
195
  throw syncError('blob.not_found', 'malformed blobId (§5.9.1)');
178
196
  }
179
- // §5.9.5 always-issue: when presign is configured the server exits the
180
- // egress path — it only needs to know the object EXISTS (a HEAD/`has`), not
181
- // the bytes. Inline delivery loads the bytes; presigned delivery does not.
182
- const presignConfigured = ctx.blobSignedUrls !== undefined;
183
- let entry: { record: BlobRecord; bytes: Uint8Array } | undefined;
184
- if (presignConfigured) {
185
- if (!(await store.has(ctx.partition, blobId))) {
186
- throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
187
- }
188
- } else {
189
- entry = await store.get(ctx.partition, blobId);
190
- if (entry === undefined) {
191
- throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
192
- }
197
+ // Existence first (a HEAD), authorization second, bytes last: no blob byte
198
+ // is read for an actor the reference index does not authorize (§5.9.5).
199
+ if (!(await store.has(ctx.partition, blobId))) {
200
+ throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
193
201
  }
194
202
 
195
203
  // §5.9.5 authorization: resolve once, then test referencing rows.
@@ -264,25 +272,37 @@ async function downloadBlob(
264
272
  headers: {
265
273
  ETag: `"${blobId}"`,
266
274
  'Cache-Control': 'private, max-age=0',
267
- Vary: 'Authorization, Accept-Encoding',
275
+ Vary: 'Authorization',
268
276
  },
269
277
  url: issued.url,
270
278
  urlExpiresAtMs: issued.urlExpiresAtMs,
271
279
  };
272
280
  }
273
281
 
274
- // Inline delivery (no presign store configured): the bytes were loaded.
275
- const bytes = entry?.bytes;
276
- if (bytes === undefined) {
277
- // Unreachable: presign not configured ⇒ `entry` was loaded above.
278
- throw syncError('blob.not_found', 'blob bytes unavailable (§5.9.5)');
282
+ // Inline delivery streams the stored bytes verbatim (identity encoding,
283
+ // §5.9.5). A store without `open` serves its `get` bytes as one chunk.
284
+ const opened =
285
+ store.open !== undefined
286
+ ? await store.open(ctx.partition, blobId)
287
+ : await store.get(ctx.partition, blobId).then((entry) =>
288
+ entry === undefined
289
+ ? undefined
290
+ : {
291
+ record: entry.record,
292
+ body: bytesStream(entry.bytes),
293
+ byteLength: entry.bytes.length,
294
+ },
295
+ );
296
+ if (opened === undefined) {
297
+ throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
279
298
  }
280
299
  // §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
281
300
  // pointed at the route renders them. A JSON type stays octet-stream: a
282
301
  // JSON body is reserved for the signed-URL envelope.
283
- const mediaType = entry?.record.mediaType;
302
+ const mediaType = opened.record.mediaType;
284
303
  return {
285
- bytes,
304
+ body: opened.body,
305
+ byteLength: opened.byteLength,
286
306
  headers: {
287
307
  'Content-Type':
288
308
  mediaType === undefined ||
@@ -293,7 +313,7 @@ async function downloadBlob(
293
313
  'Content-Security-Policy': "sandbox; default-src 'none'",
294
314
  ETag: `"${blobId}"`,
295
315
  'Cache-Control': 'private, max-age=0',
296
- Vary: 'Authorization, Accept-Encoding',
316
+ Vary: 'Authorization',
297
317
  },
298
318
  };
299
319
  }
@@ -303,11 +323,9 @@ async function downloadBlob(
303
323
  * auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
304
324
  * so any authenticated actor may obtain a grant within the size cap (§5.9.3).
305
325
  *
306
- * The size cap is enforced HERE, up front, against the declared `byteLength`
307
- * (the object-store hop cannot re-check the streamed byte count). Integrity is
308
- * NOT checked here: the object store places bytes at the content-addressed key,
309
- * and the address is verified at reference time (§5.9.6 push existence) and on
310
- * every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
326
+ * The size cap is enforced HERE, up front, against the declared `byteLength`;
327
+ * the presigned PUT binds that length and the content address, so the object
328
+ * store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
311
329
  * `present` marker if the blob already exists (idempotent §5.9.3), or an empty
312
330
  * result when no presigned-upload store is configured (client streams direct).
313
331
  */
@@ -335,7 +353,8 @@ export async function handleBlobUploadGrant(
335
353
  // §5.9.3: cap enforced at grant issuance, before any URL is minted.
336
354
  throw syncError(
337
355
  'blob.too_large',
338
- `blob byteLength exceeds the ${maxBytes}-byte cap (§5.9.3)`,
356
+ 'blob byteLength exceeds the server size cap (§5.9.3)',
357
+ JSON.stringify({ max: maxBytes }),
339
358
  );
340
359
  }
341
360
  // §5.9.3 no presigned-upload store: report no grant; the client streams
package/src/blob-store.ts CHANGED
Binary file
@@ -29,10 +29,17 @@
29
29
  *
30
30
  * The orphan sweep is the ONLY operation that LISTs: `sweepOrphans` pages a
31
31
  * prefix listing over the `blob/` prefix (an admin/GC path, off the hot path)
32
- * to find candidates, reads each object's `createdAtMs` from its metadata for
33
- * the grace check, and DELETEs the unreferenced-and-old ones.
32
+ * to find candidates, reads each object's upload time from its metadata (or
33
+ * the backend's last-modified time for an object a presigned PUT wrote), and
34
+ * DELETEs the old ones the reference index does not name at that moment.
34
35
  */
35
- import type { BlobRecord, BlobStore, BlobStoreStats } from './blob-store';
36
+ import type {
37
+ BlobRecord,
38
+ BlobStore,
39
+ BlobStoreStats,
40
+ BlobStream,
41
+ } from './blob-store';
42
+ import type { BucketObject } from './object-bucket';
36
43
  import type { ObjectBucket } from './object-bucket';
37
44
 
38
45
  export interface BucketBlobStoreOptions {
@@ -215,41 +222,27 @@ export class BucketBlobStore implements BlobStore {
215
222
  };
216
223
  const objectKey = this.objectKeyFor(partition, blobId);
217
224
  // Idempotent re-put (same content-address ⇒ same key): count each distinct
218
- // blob once, and preserve the FIRST createdAtMs so the sweep's grace clock
219
- // is not reset by a re-upload (§5.9.2 upload age).
225
+ // blob once. A re-upload rewrites the object so its metadata carries the
226
+ // re-upload time (the sweep's grace clock, §5.9.2) and keeps a recorded
227
+ // media type.
220
228
  const preexisting = await this.#bucket.head(objectKey);
221
229
  const storedMeta = preexisting?.metadata[BLOB_META_KEY];
222
230
  const existing =
223
231
  storedMeta === undefined
224
232
  ? undefined
225
233
  : parseMetaJson(base64urlToUtf8(storedMeta), blobId, this.#label);
226
- if (
227
- existing !== undefined &&
228
- (existing.mediaType !== undefined || mediaType === undefined)
229
- ) {
230
- return {
231
- blobId,
232
- partition,
233
- byteLength: existing.byteLength,
234
- ...(existing.mediaType !== undefined
235
- ? { mediaType: existing.mediaType }
236
- : {}),
237
- createdAtMs: existing.createdAtMs,
238
- };
239
- }
240
- // A re-upload that names the media type an earlier upload left out
241
- // rewrites the object under the same precondition, keeping its age.
234
+ const storedType = existing?.mediaType ?? mediaType;
242
235
  const meta: BlobMeta = {
243
236
  byteLength: bytes.length,
244
- ...(mediaType !== undefined ? { mediaType } : {}),
245
- createdAtMs: existing?.createdAtMs ?? nowMs,
237
+ ...(storedType !== undefined ? { mediaType: storedType } : {}),
238
+ createdAtMs: Math.max(existing?.createdAtMs ?? nowMs, nowMs),
246
239
  };
247
240
  // Conditional write: of two concurrent uploads of the same blob exactly
248
- // one lands, so the first createdAtMs survives and the blob counts once.
249
- // The loser re-reads and returns the winner's record.
241
+ // one lands per pre-state and the blob counts once. The loser re-reads
242
+ // and writes over the winner.
250
243
  const written = await this.#bucket.put(objectKey, {
251
244
  body: bytes,
252
- contentType: mediaType ?? 'application/octet-stream',
245
+ contentType: storedType ?? 'application/octet-stream',
253
246
  metadata: { [BLOB_META_KEY]: utf8ToBase64url(metaToJson(meta)) },
254
247
  ...(preexisting === undefined
255
248
  ? { createOnly: true }
@@ -258,7 +251,11 @@ export class BucketBlobStore implements BlobStore {
258
251
  if (!written) return this.put(partition, blobId, bytes, nowMs, mediaType);
259
252
  if (preexisting === undefined)
260
253
  await this.#bumpStats({ count: 1, bytes: bytes.length });
261
- return { ...record, createdAtMs: meta.createdAtMs };
254
+ return {
255
+ ...record,
256
+ ...(storedType !== undefined ? { mediaType: storedType } : {}),
257
+ createdAtMs: meta.createdAtMs,
258
+ };
262
259
  }
263
260
 
264
261
  async delete(partition: string, blobId: string): Promise<void> {
@@ -275,38 +272,67 @@ export class BucketBlobStore implements BlobStore {
275
272
  partition: string,
276
273
  blobId: string,
277
274
  ): Promise<{ record: BlobRecord; bytes: Uint8Array } | undefined> {
278
- if (!BLOB_ID_PATTERN.test(blobId)) return undefined;
279
- const object = await this.#bucket.get(this.objectKeyFor(partition, blobId));
275
+ const object = await this.#object(partition, blobId);
280
276
  if (object === undefined) return undefined;
281
277
  const bytes = new Uint8Array(await object.arrayBuffer());
278
+ return {
279
+ record: this.#record(partition, blobId, object, bytes.length),
280
+ bytes,
281
+ };
282
+ }
283
+
284
+ /** The record plus the object GET's body, never buffered (see `get`). */
285
+ async open(
286
+ partition: string,
287
+ blobId: string,
288
+ ): Promise<BlobStream | undefined> {
289
+ const object = await this.#object(partition, blobId);
290
+ if (object === undefined || object.body === null) return undefined;
291
+ const record = this.#record(partition, blobId, object, object.size ?? 0);
292
+ return { record, body: object.body, byteLength: record.byteLength };
293
+ }
294
+
295
+ async #object(
296
+ partition: string,
297
+ blobId: string,
298
+ ): Promise<BucketObject | undefined> {
299
+ if (!BLOB_ID_PATTERN.test(blobId)) return undefined;
300
+ return this.#bucket.get(this.objectKeyFor(partition, blobId));
301
+ }
302
+
303
+ /** An object without Syncular metadata was written by a presigned PUT. */
304
+ #record(
305
+ partition: string,
306
+ blobId: string,
307
+ object: BucketObject,
308
+ byteLength: number,
309
+ ): BlobRecord {
282
310
  const rawMeta = object.metadata[BLOB_META_KEY];
283
311
  const meta =
284
312
  rawMeta !== undefined
285
313
  ? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
286
- : { byteLength: bytes.length, createdAtMs: 0 };
314
+ : { byteLength, createdAtMs: 0 };
287
315
  return {
288
- record: {
289
- blobId,
290
- partition,
291
- byteLength: meta.byteLength,
292
- ...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
293
- createdAtMs: meta.createdAtMs,
294
- },
295
- bytes,
316
+ blobId,
317
+ partition,
318
+ byteLength: meta.byteLength,
319
+ ...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
320
+ createdAtMs: meta.createdAtMs,
296
321
  };
297
322
  }
298
323
 
299
324
  /**
300
325
  * §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
301
- * {partition}/` prefix, HEAD each object for its `createdAtMs`, and DELETE
302
- * the ones both older than `olderThanMs` AND absent from
303
- * `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
304
- * referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
326
+ * {partition}/` prefix, HEAD each object for its upload time (the metadata
327
+ * `createdAtMs`, else the backend's last-modified time; an object with
328
+ * neither is skipped), and DELETE the ones older than `olderThanMs` that
329
+ * `isReferenced` denies right before the delete. Returns the deleted
330
+ * blobIds. Off the hot path — a host-scheduled GC job.
305
331
  */
306
332
  async sweepOrphans(
307
333
  partition: string,
308
334
  olderThanMs: number,
309
- referencedBlobIds: ReadonlySet<string>,
335
+ isReferenced: (blobId: string) => Promise<boolean>,
310
336
  ): Promise<string[]> {
311
337
  const listPrefix = `${this.#prefix}blob/${partition}/`;
312
338
  const swept: string[] = [];
@@ -321,18 +347,19 @@ export class BucketBlobStore implements BlobStore {
321
347
  if (!rest.startsWith('sha256/')) continue;
322
348
  const blobId = `sha256:${rest.slice('sha256/'.length)}`;
323
349
  if (!BLOB_ID_PATTERN.test(blobId)) continue;
324
- if (referencedBlobIds.has(blobId)) continue;
325
350
  const head = await this.#bucket.head(key);
326
351
  if (head === undefined) continue;
327
352
  const rawMeta = head.metadata[BLOB_META_KEY];
328
- const meta =
353
+ const uploadedAtMs =
329
354
  rawMeta !== undefined
330
355
  ? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
331
- : { byteLength: head.size ?? 0, createdAtMs: 0 };
332
- if (meta.createdAtMs >= olderThanMs) continue;
356
+ .createdAtMs
357
+ : head.lastModifiedMs;
358
+ if (uploadedAtMs === undefined || uploadedAtMs >= olderThanMs) continue;
359
+ if (await isReferenced(blobId)) continue;
333
360
  await this.#bucket.delete(key);
334
361
  swept.push(blobId);
335
- deletedBytes += meta.byteLength;
362
+ deletedBytes += head.size ?? 0;
336
363
  }
337
364
  } while (cursor !== undefined);
338
365
  if (swept.length > 0) {