@syncular/server 0.57.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +31 -22
  2. package/dist/admin.js +8 -4
  3. package/dist/blob-handlers.d.ts +25 -31
  4. package/dist/blob-handlers.js +63 -41
  5. package/dist/blob-store.d.ts +42 -24
  6. package/dist/blob-store.js +0 -0
  7. package/dist/bucket-blob-store.d.ts +12 -8
  8. package/dist/bucket-blob-store.js +54 -45
  9. package/dist/bucket-segment-store.js +18 -16
  10. package/dist/content-encoding.d.ts +5 -2
  11. package/dist/content-encoding.js +42 -13
  12. package/dist/context.d.ts +16 -0
  13. package/dist/context.js +21 -1
  14. package/dist/d1-storage.d.ts +14 -98
  15. package/dist/d1-storage.js +223 -1111
  16. package/dist/errors.js +8 -0
  17. package/dist/events.d.ts +1 -1
  18. package/dist/handler.js +135 -103
  19. package/dist/local-authority.d.ts +2 -0
  20. package/dist/local-authority.js +17 -3
  21. package/dist/object-bucket.d.ts +2 -0
  22. package/dist/operations-realtime.js +69 -50
  23. package/dist/pg-executor-pglite.d.ts +5 -6
  24. package/dist/pg-executor-pglite.js +22 -14
  25. package/dist/pg-executor.d.ts +4 -1
  26. package/dist/postgres-storage.d.ts +1 -1
  27. package/dist/postgres-storage.js +182 -105
  28. package/dist/pull.d.ts +28 -20
  29. package/dist/pull.js +85 -87
  30. package/dist/push.js +272 -147
  31. package/dist/realtime.d.ts +68 -28
  32. package/dist/realtime.js +297 -136
  33. package/dist/relational-rows.d.ts +27 -15
  34. package/dist/relational-rows.js +38 -22
  35. package/dist/s3-blob-store.d.ts +11 -12
  36. package/dist/s3-blob-store.js +20 -13
  37. package/dist/s3-object-bucket.d.ts +8 -1
  38. package/dist/s3-object-bucket.js +16 -3
  39. package/dist/schema.d.ts +14 -0
  40. package/dist/schema.js +23 -0
  41. package/dist/scopes.d.ts +6 -2
  42. package/dist/scopes.js +22 -4
  43. package/dist/seed.js +1 -2
  44. package/dist/segment-download.js +18 -4
  45. package/dist/signed-url.d.ts +2 -2
  46. package/dist/signed-url.js +9 -2
  47. package/dist/sigv4.d.ts +8 -2
  48. package/dist/sigv4.js +8 -4
  49. package/dist/sqlite-blob-store.d.ts +1 -1
  50. package/dist/sqlite-blob-store.js +23 -22
  51. package/dist/sqlite-dialect.d.ts +150 -47
  52. package/dist/sqlite-dialect.js +987 -88
  53. package/dist/sqlite-lease-store.js +13 -18
  54. package/dist/sqlite-storage.d.ts +9 -62
  55. package/dist/sqlite-storage.js +293 -1117
  56. package/dist/storage.d.ts +32 -12
  57. package/package.json +2 -2
  58. package/src/admin.ts +16 -2
  59. package/src/blob-handlers.ts +91 -72
  60. package/src/blob-store.ts +0 -0
  61. package/src/bucket-blob-store.ts +75 -48
  62. package/src/bucket-segment-store.ts +20 -15
  63. package/src/content-encoding.ts +48 -16
  64. package/src/context.ts +32 -1
  65. package/src/d1-storage.ts +326 -1692
  66. package/src/errors.ts +8 -0
  67. package/src/events.ts +1 -1
  68. package/src/handler.ts +178 -127
  69. package/src/local-authority.ts +17 -3
  70. package/src/object-bucket.ts +2 -0
  71. package/src/operations-realtime.ts +85 -61
  72. package/src/pg-executor-pglite.ts +27 -23
  73. package/src/pg-executor.ts +5 -1
  74. package/src/postgres-storage.ts +231 -138
  75. package/src/pull.ts +133 -147
  76. package/src/push.ts +356 -205
  77. package/src/realtime.ts +380 -179
  78. package/src/relational-rows.ts +54 -27
  79. package/src/s3-blob-store.ts +23 -13
  80. package/src/s3-object-bucket.ts +20 -2
  81. package/src/schema.ts +31 -0
  82. package/src/scopes.ts +23 -3
  83. package/src/seed.ts +1 -2
  84. package/src/segment-download.ts +25 -4
  85. package/src/signed-url.ts +10 -4
  86. package/src/sigv4.ts +17 -4
  87. package/src/sqlite-blob-store.ts +27 -26
  88. package/src/sqlite-dialect.ts +1607 -114
  89. package/src/sqlite-lease-store.ts +12 -23
  90. package/src/sqlite-storage.ts +360 -1811
  91. package/src/storage.ts +33 -12
package/README.md CHANGED
@@ -357,7 +357,7 @@ revision (`catalog-v1` to `catalog-v2`) and rerun it; do not delete the database
357
357
  or unrelated rows. This does not apply to application commands. After an
358
358
  unknown command outcome, reuse the original idempotency key because changing it
359
359
  can execute the operation twice. The full inspection and recovery recipe is in
360
- the public [server guide](https://syncular.dev/guide-server/#seeding-data).
360
+ the public [server guide](https://syncular.dev/server-operations/#seeding-data).
361
361
 
362
362
  One extra rule applies when the seed actor changes. A Syncular `clientId` is
363
363
  bound to its first actor inside a partition. Keep the client ID stable for seed
@@ -412,7 +412,7 @@ scope variable, named-query obligation, subscription descriptor, or client
412
412
  authority. Never expose table/index/value selection through a client-controlled
413
413
  route. Custom adapters may omit the additive method; authoritative commands
414
414
  must check for it and fail closed. See the public
415
- [storage lookup guide](https://syncular.dev/server-storage/#choosing-the-right-row-lookup)
415
+ [storage lookup guide](https://syncular.dev/server-storage-reference/#advanced-row-lookups-for-trusted-server-code)
416
416
  for a user-scoped key-grant table revoked through a Workspace index and for the
417
417
  atomic reverse-index/queue fallback required by ordered or derived lookups.
418
418
 
@@ -583,7 +583,7 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
583
583
  | `reaction.prune_completed` | One bounded terminal-reaction retention pass finished | `completedBeforeMs`, `deadLetterBeforeMs`, `limit`, `removedCompleted`, `removedDeadLetter`, `mayHaveMore` |
584
584
  | `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `subscriptions[]`: `{id, table, status, mode` (`bootstrap` \| `incremental` \| `none`)`, fromCursor, nextCursor, commits, changes, segments[]}`; each segment: `{mediaType` (`rows` \| `sqlite`)`, delivery` (`inline` \| `ref`)`, origin` (`built` \| `reused`)`, bytes, rows}` |
585
585
  | `segment.downloaded` | Every direct segment download (§5.5), success or failure | `segmentId`, `outcome` (`ok` \| `error`), `errorCode?`, `mediaType?`, `bytes?`, `durationMs` |
586
- | `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (keep-set size), `graceMs` |
586
+ | `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (old candidates still referenced), `graceMs` |
587
587
  | `realtime.opened` | A socket registered with the hub and got `hello` (§8.1) | `sessionId`, `clientId`, `registrations`, `cursor`, `latestSeq` |
588
588
  | `realtime.closed` | A session left the hub (once per session) | `sessionId`, `durationMs` |
589
589
  | `realtime.delta` | A delta message pushed over the socket (§8.2) | `sessionId`, `commitSeq`, `bytes`, `changes` |
@@ -959,12 +959,12 @@ and mints a fresh URL) — never a fall-through.
959
959
  `{ byteLength, mediaType? }`) mints a single presigned PUT; the client PUTs
960
960
  bytes straight to the object store. **Upload authz is host-authentication only**
961
961
  (any authenticated actor may obtain a grant within the size cap) — uploading
962
- bytes is not a scope-bearing act, because the content address discloses nothing
963
- and integrity is enforced at *reference time* (the §5.9.6 push existence check
964
- + every download's content-address verify), never by a store-side hash
965
- recompute. The **size cap is enforced up front** against the declared
966
- `byteLength`, before any URL is minted (the object-store hop cannot re-check the
967
- streamed byte count). An already-present blob returns `{ present: true }` (skip
962
+ bytes is not a scope-bearing act, because the content address discloses
963
+ nothing. The grant signs `content-length` (the declared `byteLength`) and
964
+ `x-amz-checksum-sha256` (the `blobId` digest as base64), and the client sends
965
+ that checksum header, so the object store refuses a PUT of another size or
966
+ other bytes. The **size cap is enforced up front** against the declared
967
+ `byteLength`, before any URL is minted. An already-present blob returns `{ present: true }` (skip
968
968
  the PUT, idempotent §5.9.3). A **single PUT only** — never a multipart or chunk
969
969
  protocol; resumable upload, when it lands, is provider multipart behind this
970
970
  same grant. Absent config ⇒ the client streams through the direct
@@ -1006,8 +1006,11 @@ Nothing reclaims blobs automatically — the host schedules the sweep, the blob
1006
1006
  analogue of `pruneCommitLog`. `sweepOrphanBlobs(storage, blobStore, partition,
1007
1007
  { graceMs })` reads the live keep-set from the §5.9.4 reference index
1008
1008
  (`storage.listReferencedBlobIds`) and deletes every blob that is **both**
1009
- unreferenced **and** older than the grace period. It emits one `blob.swept`
1010
- ops event (`{ swept, referenced, graceMs }`) and returns the deleted ids.
1009
+ unreferenced **and** older than the grace period. Before each delete it reads
1010
+ the index again, so a reference that lands during the sweep keeps its blob. It
1011
+ emits one `blob.swept` ops event (`{ swept, referenced, graceMs }`, where
1012
+ `referenced` counts old candidates a row still references) and returns the
1013
+ deleted ids.
1011
1014
 
1012
1015
  ```ts
1013
1016
  import { sweepOrphanBlobs } from '@syncular/server';
@@ -1028,8 +1031,10 @@ arrived. So the grace period must comfortably exceed any sane upload→push
1028
1031
  latency — the **default is 24 h**, deliberately far above any push window.
1029
1032
  Lower it only if you fully understand your clients' outbox latency; there is
1030
1033
  no upside to a tight grace and a real data-loss risk. The sweep compares
1031
- against the blob's **upload time** (`createdAtMs` from object metadata), and an
1032
- idempotent re-upload does **not** reset that clock.
1034
+ against the blob's **latest upload time**: `createdAtMs` from object metadata,
1035
+ or the object's last-modified time for an object a presigned PUT wrote. An
1036
+ object with neither is skipped, and an idempotent re-upload rewrites the
1037
+ object with the re-upload time, which restarts its grace period.
1033
1038
 
1034
1039
  `sweepOrphanBlobs` **requires** `storage.listReferencedBlobIds` (the §5.9.4
1035
1040
  reference index — SQLite, D1, and Postgres all implement it). Against a
@@ -1058,11 +1063,10 @@ row-derived authorization check passes** (`handleBlobDownload` resolves the
1058
1063
  actor's scopes and tests the referencing rows first; a `blobId` is never a
1059
1064
  bearer capability minted from the id alone). The signed object key embeds the
1060
1065
  `blobId`, TTL SHOULD be ≤ 15 min (default 900 s), and the URL is a short-lived
1061
- grant to exactly those immutable bytes. The issued `url`/`urlExpiresAtMs` ride
1062
- additively on `BlobDownloadResult` alongside the bytes; the mounted §5.9.5
1063
- direct-download endpoint stays the default serving path. Client consumption of
1064
- the presigned URL (following it instead of streaming bytes through the sync
1065
- server) is a later rung — the server-side issuance ships now.
1066
+ grant to exactly those immutable bytes. When `blobSignedUrls` is set, the
1067
+ download answers `{ url, urlExpiresAtMs }` and no bytes, and the reference
1068
+ clients fetch the URL and verify the content address (see "Presigned blobs,
1069
+ end to end" above).
1066
1070
 
1067
1071
  ### Native HMAC vs delegated presign (§5.4)
1068
1072
 
@@ -1235,7 +1239,7 @@ function bunSqlExecutor(sql: import('bun').SQL): PgExecutor {
1235
1239
  const over = (h: any): PgQueryable => ({
1236
1240
  async query(text, params) {
1237
1241
  const rows = await h.unsafe(text, params ? [...params] : []);
1238
- return { rows, rowCount: rows.length };
1242
+ return { rows };
1239
1243
  },
1240
1244
  });
1241
1245
  return {
@@ -1283,6 +1287,11 @@ function pgPoolExecutor(pool: Pool): PgExecutor {
1283
1287
  }
1284
1288
  ```
1285
1289
 
1290
+ **Result shape.** `query` returns `{ rows }`. The storage never reads an
1291
+ affected-row count: every write whose outcome matters ends in `RETURNING`
1292
+ and counts the returned rows, because drivers disagree on the count of a
1293
+ statement without result rows.
1294
+
1286
1295
  **Type-parser note.** `commit_seq`/`server_version` are `int8`. Drivers
1287
1296
  decode `int8` differently (node-postgres → `string`, Bun.sql → `bigint`,
1288
1297
  pglite → `number`); the storage layer coerces every sequence read through
@@ -1367,9 +1376,9 @@ SYNCULAR_PG_URL=postgres://user:pass@localhost:5432/db bun run bench
1367
1376
  ```
1368
1377
 
1369
1378
  Custom storage adapters must implement
1370
- `getActiveClientCursorFloor(partition, cutoffMs)`. Return the minimum cursor
1371
- whose `updatedAtMs >= cutoffMs`, or `null` when no client qualifies. Preserve
1372
- negative bootstrap cursors. Pruning and admin horizon status use this scalar
1379
+ `getActiveClientCursorFloor(partition, cutoffMs)`. Return the minimum
1380
+ non-negative cursor whose `updatedAtMs >= cutoffMs`, or `null` when no client
1381
+ qualifies. A bootstrapping client at cursor -1 holds no cursor. Pruning and admin horizon status use this scalar
1373
1382
  aggregate; `listClientCursors` remains the explicit listing interface.
1374
1383
 
1375
1384
  SQLite image builders now return `Promise<Uint8Array>` and receive
package/dist/admin.js CHANGED
@@ -55,7 +55,7 @@ const VERDICT_ORDER = [
55
55
  'delivered',
56
56
  ];
57
57
  /** One known client's subscriptions on the row's table and its verdict. */
58
- function traceVerdict(record, row, lastChange, horizonSeq, allowed) {
58
+ function traceVerdict(declared, record, row, lastChange, horizonSeq, allowed) {
59
59
  const rowScopes = row.scopes ?? {};
60
60
  const subscriptions = record.subscriptions
61
61
  .filter((sub) => sub.table === row.table)
@@ -78,7 +78,10 @@ function traceVerdict(record, row, lastChange, horizonSeq, allowed) {
78
78
  };
79
79
  if (allowed === undefined || allowed === 'outage')
80
80
  return traced;
81
- const effective = computeEffective(sub.scopes, { ok: true, allowed });
81
+ const effective = computeEffective(declared, sub.scopes, {
82
+ ok: true,
83
+ allowed,
84
+ });
82
85
  return {
83
86
  ...traced,
84
87
  effectiveMatch: effective.status === 'active' &&
@@ -349,7 +352,7 @@ export class SyncularAdmin {
349
352
  ...base,
350
353
  client: (await this.#clientView(partition))(record),
351
354
  ...(allowed !== undefined ? { allowed } : {}),
352
- ...traceVerdict(record, row, lastChange, horizonSeq, allowed),
355
+ ...traceVerdict(this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set(), record, row, lastChange, horizonSeq, allowed),
353
356
  };
354
357
  }
355
358
  /**
@@ -364,6 +367,7 @@ export class SyncularAdmin {
364
367
  const lastChange = row.history[0];
365
368
  const horizonSeq = await this.#storage.getHorizonSeq(partition);
366
369
  const view = await this.#clientView(partition);
370
+ const declared = this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set();
367
371
  const allowedByActor = new Map();
368
372
  const clients = [];
369
373
  for (const record of await list(partition)) {
@@ -372,7 +376,7 @@ export class SyncularAdmin {
372
376
  }
373
377
  clients.push({
374
378
  client: view(record),
375
- verdict: traceVerdict(record, row, lastChange, horizonSeq, allowedByActor.get(record.actorId)).verdict,
379
+ verdict: traceVerdict(declared, record, row, lastChange, horizonSeq, allowedByActor.get(record.actorId)).verdict,
376
380
  });
377
381
  }
378
382
  clients.sort((a, b) => VERDICT_ORDER.indexOf(a.verdict) - VERDICT_ORDER.indexOf(b.verdict) ||
@@ -1,29 +1,20 @@
1
1
  import type { SyncRequestContext } from './context.js';
2
- export interface BlobUploadRequest {
3
- readonly blobId: string;
4
- readonly bytes: Uint8Array;
5
- /** Advisory MIME type from the upload `Content-Type`, if any. */
6
- readonly mediaType?: string;
7
- }
8
- export interface BlobDownloadResult {
9
- /**
10
- * The blob bytes, when served inline. **Absent** when the host configured
11
- * `blobSignedUrls` and a `url` was issued: the sync server exits the egress
12
- * path and the client fetches `url` directly (§5.9.5 always-issue).
13
- */
14
- readonly bytes?: Uint8Array;
2
+ /**
3
+ * A §5.9.5 download: the stored bytes as a stream with their length (inline
4
+ * delivery), or, when the host configured `blobSignedUrls`, a provider-signed
5
+ * GET URL issued after the row-derived authorization check. The client
6
+ * fetches a URL directly (no host auth), verifies the content address, and on
7
+ * failure re-requests this endpoint (§5.9.5 recovery rule).
8
+ */
9
+ export type BlobDownloadResult = {
15
10
  readonly headers: Record<string, string>;
16
- /**
17
- * §5.9.5 delegated presign (always-issue): a provider-signed GET URL for
18
- * the bytes, issued only after the row-derived authorization check passed.
19
- * Present iff the host configured `blobSignedUrls`; when present, `bytes` is
20
- * absent and the client MUST fetch the URL directly (no host auth), verify
21
- * the content address, and on failure re-request this endpoint — never fall
22
- * through (§5.9.5 recovery rule).
23
- */
24
- readonly url?: string;
25
- readonly urlExpiresAtMs?: number;
26
- }
11
+ readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
12
+ readonly byteLength: number;
13
+ } | {
14
+ readonly headers: Record<string, string>;
15
+ readonly url: string;
16
+ readonly urlExpiresAtMs: number;
17
+ };
27
18
  /** `POST <mount>/blobs/{blobId}/upload-grant` request body (§5.9.3). */
28
19
  export interface BlobUploadGrantRequest {
29
20
  readonly blobId: string;
@@ -45,8 +36,13 @@ export interface BlobUploadGrantResult {
45
36
  readonly urlExpiresAtMs?: number;
46
37
  readonly present?: boolean;
47
38
  }
48
- /** `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. */
49
- export declare function handleBlobUpload(ctx: SyncRequestContext, request: BlobUploadRequest): Promise<void>;
39
+ /**
40
+ * `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
41
+ * size cap applies to the declared `Content-Length` before any byte is read
42
+ * and again to the counted decoded bytes while reading, so an oversized body
43
+ * is never buffered whole.
44
+ */
45
+ export declare function handleBlobUpload(ctx: SyncRequestContext, blobId: string, request: Request): Promise<void>;
50
46
  /**
51
47
  * `GET <mount>/blobs/{blobId}` (§5.9.5). Authorization derives from the
52
48
  * referencing rows: the actor may download iff at least one row in the
@@ -58,11 +54,9 @@ export declare function handleBlobDownload(ctx: SyncRequestContext, blobId: stri
58
54
  * auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
59
55
  * so any authenticated actor may obtain a grant within the size cap (§5.9.3).
60
56
  *
61
- * The size cap is enforced HERE, up front, against the declared `byteLength`
62
- * (the object-store hop cannot re-check the streamed byte count). Integrity is
63
- * NOT checked here: the object store places bytes at the content-addressed key,
64
- * and the address is verified at reference time (§5.9.6 push existence) and on
65
- * every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
57
+ * The size cap is enforced HERE, up front, against the declared `byteLength`;
58
+ * the presigned PUT binds that length and the content address, so the object
59
+ * store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
66
60
  * `present` marker if the blob already exists (idempotent §5.9.3), or an empty
67
61
  * result when no presigned-upload store is configured (client streams direct).
68
62
  */
@@ -7,41 +7,67 @@
7
7
  * adapters map these onto `PUT/GET <mount>/blobs/{blobId}`.
8
8
  */
9
9
  import { blobIdFor, isBlobId } from './blob-store.js';
10
+ import { decodeRequestBody } from './content-encoding.js';
10
11
  import { clockOf, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
11
12
  import { SyncError, syncError } from './errors.js';
12
13
  import { emitEvent } from './events.js';
13
14
  import { compileSchema } from './schema.js';
14
15
  import { authorizeWrite } from './scopes.js';
16
+ import { bytesStream } from './segment-store.js';
15
17
  import { issueBlobUploadUrl, issueBlobUrl } from './signed-url.js';
16
18
  const DEFAULT_MAX_BLOB_BYTES = 64 * 1024 * 1024;
17
- /** `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. */
18
- export async function handleBlobUpload(ctx, request) {
19
+ /**
20
+ * `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
21
+ * size cap applies to the declared `Content-Length` before any byte is read
22
+ * and again to the counted decoded bytes while reading, so an oversized body
23
+ * is never buffered whole.
24
+ */
25
+ export async function handleBlobUpload(ctx, blobId, request) {
19
26
  await touchAuthenticatedPartition(ctx);
20
27
  const store = ctx.blobs;
21
28
  if (store === undefined) {
22
29
  throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
23
30
  }
24
- if (!isBlobId(request.blobId)) {
31
+ if (!isBlobId(blobId)) {
25
32
  throw syncError('blob.hash_mismatch', 'blobId path is not "sha256:<64 hex>" (§5.9.1)');
26
33
  }
27
34
  const maxBytes = ctx.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
28
- if (request.bytes.length > maxBytes) {
29
- throw syncError('blob.too_large', `blob exceeds the ${maxBytes}-byte cap (§5.9.3)`);
35
+ const tooLarge = syncError('blob.too_large', 'blob exceeds the server size cap (§5.9.3)', JSON.stringify({ max: maxBytes }));
36
+ if (Number(request.headers.get('content-length')) > maxBytes)
37
+ throw tooLarge;
38
+ let bytes;
39
+ try {
40
+ bytes = await decodeRequestBody(request, maxBytes);
41
+ }
42
+ catch (error) {
43
+ throw error instanceof SyncError && error.code === 'sync.limit_exceeded'
44
+ ? tooLarge
45
+ : error;
30
46
  }
31
- const computed = await blobIdFor(request.bytes);
32
- if (computed !== request.blobId) {
47
+ if (bytes === undefined) {
48
+ throw syncError('sync.invalid_request', 'unsupported content encoding');
49
+ }
50
+ if ((await blobIdFor(bytes)) !== blobId) {
33
51
  throw syncError('blob.hash_mismatch', 'uploaded bytes do not hash to the blobId path (§5.9.3)');
34
52
  }
53
+ const contentType = request.headers
54
+ .get('content-type')
55
+ ?.split(';')[0]
56
+ ?.trim();
35
57
  const clock = clockOf(ctx);
36
- await store.put(ctx.partition, request.blobId, request.bytes, clock(), request.mediaType);
58
+ await store.put(ctx.partition, blobId, bytes, clock(), contentType === undefined ||
59
+ contentType === '' ||
60
+ contentType === 'application/octet-stream'
61
+ ? undefined
62
+ : contentType);
37
63
  if (ctx.events !== undefined) {
38
64
  emitEvent(ctx.events, {
39
65
  type: 'blob.uploaded',
40
66
  atMs: clock(),
41
67
  partition: ctx.partition,
42
68
  actorId: ctx.actorId,
43
- blobId: request.blobId,
44
- bytes: request.bytes.length,
69
+ blobId,
70
+ bytes: bytes.length,
45
71
  });
46
72
  }
47
73
  }
@@ -68,7 +94,7 @@ export async function handleBlobDownload(ctx, blobId) {
68
94
  outcome: 'ok',
69
95
  // Presigned delivery carries no inline bytes (§5.9.5 always-issue); the
70
96
  // byte count is unknown to the sync server, which exited the egress path.
71
- bytes: result.bytes?.length ?? 0,
97
+ bytes: 'byteLength' in result ? result.byteLength : 0,
72
98
  durationMs: clock() - startedAtMs,
73
99
  });
74
100
  return result;
@@ -95,21 +121,10 @@ async function downloadBlob(ctx, blobId) {
95
121
  if (!isBlobId(blobId)) {
96
122
  throw syncError('blob.not_found', 'malformed blobId (§5.9.1)');
97
123
  }
98
- // §5.9.5 always-issue: when presign is configured the server exits the
99
- // egress path — it only needs to know the object EXISTS (a HEAD/`has`), not
100
- // the bytes. Inline delivery loads the bytes; presigned delivery does not.
101
- const presignConfigured = ctx.blobSignedUrls !== undefined;
102
- let entry;
103
- if (presignConfigured) {
104
- if (!(await store.has(ctx.partition, blobId))) {
105
- throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
106
- }
107
- }
108
- else {
109
- entry = await store.get(ctx.partition, blobId);
110
- if (entry === undefined) {
111
- throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
112
- }
124
+ // Existence first (a HEAD), authorization second, bytes last: no blob byte
125
+ // is read for an actor the reference index does not authorize (§5.9.5).
126
+ if (!(await store.has(ctx.partition, blobId))) {
127
+ throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
113
128
  }
114
129
  // §5.9.5 authorization: resolve once, then test referencing rows.
115
130
  const listRows = ctx.storage.listRowsReferencingBlob;
@@ -177,24 +192,33 @@ async function downloadBlob(ctx, blobId) {
177
192
  headers: {
178
193
  ETag: `"${blobId}"`,
179
194
  'Cache-Control': 'private, max-age=0',
180
- Vary: 'Authorization, Accept-Encoding',
195
+ Vary: 'Authorization',
181
196
  },
182
197
  url: issued.url,
183
198
  urlExpiresAtMs: issued.urlExpiresAtMs,
184
199
  };
185
200
  }
186
- // Inline delivery (no presign store configured): the bytes were loaded.
187
- const bytes = entry?.bytes;
188
- if (bytes === undefined) {
189
- // Unreachable: presign not configured ⇒ `entry` was loaded above.
190
- throw syncError('blob.not_found', 'blob bytes unavailable (§5.9.5)');
201
+ // Inline delivery streams the stored bytes verbatim (identity encoding,
202
+ // §5.9.5). A store without `open` serves its `get` bytes as one chunk.
203
+ const opened = store.open !== undefined
204
+ ? await store.open(ctx.partition, blobId)
205
+ : await store.get(ctx.partition, blobId).then((entry) => entry === undefined
206
+ ? undefined
207
+ : {
208
+ record: entry.record,
209
+ body: bytesStream(entry.bytes),
210
+ byteLength: entry.bytes.length,
211
+ });
212
+ if (opened === undefined) {
213
+ throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
191
214
  }
192
215
  // §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
193
216
  // pointed at the route renders them. A JSON type stays octet-stream: a
194
217
  // JSON body is reserved for the signed-URL envelope.
195
- const mediaType = entry?.record.mediaType;
218
+ const mediaType = opened.record.mediaType;
196
219
  return {
197
- bytes,
220
+ body: opened.body,
221
+ byteLength: opened.byteLength,
198
222
  headers: {
199
223
  'Content-Type': mediaType === undefined ||
200
224
  mediaType.toLowerCase().includes('application/json')
@@ -204,7 +228,7 @@ async function downloadBlob(ctx, blobId) {
204
228
  'Content-Security-Policy': "sandbox; default-src 'none'",
205
229
  ETag: `"${blobId}"`,
206
230
  'Cache-Control': 'private, max-age=0',
207
- Vary: 'Authorization, Accept-Encoding',
231
+ Vary: 'Authorization',
208
232
  },
209
233
  };
210
234
  }
@@ -213,11 +237,9 @@ async function downloadBlob(ctx, blobId) {
213
237
  * auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
214
238
  * so any authenticated actor may obtain a grant within the size cap (§5.9.3).
215
239
  *
216
- * The size cap is enforced HERE, up front, against the declared `byteLength`
217
- * (the object-store hop cannot re-check the streamed byte count). Integrity is
218
- * NOT checked here: the object store places bytes at the content-addressed key,
219
- * and the address is verified at reference time (§5.9.6 push existence) and on
220
- * every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
240
+ * The size cap is enforced HERE, up front, against the declared `byteLength`;
241
+ * the presigned PUT binds that length and the content address, so the object
242
+ * store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
221
243
  * `present` marker if the blob already exists (idempotent §5.9.3), or an empty
222
244
  * result when no presigned-upload store is configured (client streams direct).
223
245
  */
@@ -235,7 +257,7 @@ export async function handleBlobUploadGrant(ctx, request) {
235
257
  request.byteLength < 0 ||
236
258
  request.byteLength > maxBytes) {
237
259
  // §5.9.3: cap enforced at grant issuance, before any URL is minted.
238
- throw syncError('blob.too_large', `blob byteLength exceeds the ${maxBytes}-byte cap (§5.9.3)`);
260
+ throw syncError('blob.too_large', 'blob byteLength exceeds the server size cap (§5.9.3)', JSON.stringify({ max: maxBytes }));
239
261
  }
240
262
  // §5.9.3 no presigned-upload store: report no grant; the client streams
241
263
  // through the direct PUT endpoint (capability, not fallback).
@@ -17,8 +17,15 @@ export interface BlobRecord {
17
17
  readonly byteLength: number;
18
18
  /** Advisory MIME type persisted from the upload, if any (§5.9.5). */
19
19
  readonly mediaType?: string;
20
+ /** Time of the latest upload; the orphan sweep's grace clock (§5.9.2). */
20
21
  readonly createdAtMs: number;
21
22
  }
23
+ /** `BlobStore.open`: a stored blob with its bytes as a stream. */
24
+ export interface BlobStream {
25
+ readonly record: BlobRecord;
26
+ readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
27
+ readonly byteLength: number;
28
+ }
22
29
  /**
23
30
  * Coarse blob-store counters for the admin/console read surface (work item
24
31
  * §2.5), partition-scoped. Optional. `count`/`bytes` are total stored
@@ -36,10 +43,12 @@ export interface BlobStoreStats {
36
43
  }
37
44
  export interface BlobStore {
38
45
  /**
39
- * Store blob bytes under `blobId` (idempotent — same id ⇒ no-op success,
40
- * §5.9.3). A re-upload that declares a media type records it on a blob
41
- * stored without one; a recorded media type never changes. The caller has
42
- * already verified the content address matches.
46
+ * Store blob bytes under `blobId` (idempotent: same id ⇒ success, §5.9.3).
47
+ * A re-upload moves `createdAtMs` forward to `nowMs`, so a blob that a
48
+ * second client is about to reference gets a fresh sweep grace period. A
49
+ * re-upload that declares a media type records it on a blob stored without
50
+ * one; a recorded media type never changes. The caller has already
51
+ * verified the content address matches.
43
52
  */
44
53
  put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
45
54
  /** Presence check for the push existence rule (§5.9.6) and download. */
@@ -49,12 +58,20 @@ export interface BlobStore {
49
58
  bytes: Uint8Array;
50
59
  } | undefined>;
51
60
  /**
52
- * §5.9.2 orphan sweep: delete blobs uploaded before `olderThanMs` that
53
- * no live row references. The set of referenced blobIds is supplied by
54
- * the storage reference index — the store never scans rows itself.
55
- * Returns the deleted blobIds.
61
+ * Optional streaming read: the record plus the bytes as a stream, so the
62
+ * inline download route relays a blob without buffering it (the
63
+ * `SegmentStore.open` shape). Stores without it are served through `get`.
64
+ */
65
+ open?(partition: string, blobId: string): Promise<BlobStream | undefined>;
66
+ /**
67
+ * §5.9.2 orphan sweep: delete blobs last uploaded before `olderThanMs`
68
+ * that no live row references. `isReferenced` asks the storage reference
69
+ * index about one blob; the store calls it for each old candidate right
70
+ * before that candidate's delete, so a reference that lands during the
71
+ * sweep keeps its blob. The store never scans rows itself. Returns the
72
+ * deleted blobIds.
56
73
  */
57
- sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
74
+ sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
58
75
  /**
59
76
  * §4.6.1 erasure: delete one blob whose last reference is gone. Deleting a
60
77
  * missing blob is a no-op.
@@ -74,7 +91,7 @@ export declare class MemoryBlobStore implements BlobStore {
74
91
  record: BlobRecord;
75
92
  bytes: Uint8Array;
76
93
  } | undefined>;
77
- sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
94
+ sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
78
95
  delete(partition: string, blobId: string): Promise<void>;
79
96
  stats(partition: string): Promise<BlobStoreStats>;
80
97
  }
@@ -95,25 +112,26 @@ export interface SweepOrphanBlobsOptions {
95
112
  export interface SweepOrphanBlobsResult {
96
113
  readonly partition: string;
97
114
  readonly swept: string[];
115
+ /** Candidates past the grace period that a live row still references. */
98
116
  readonly referencedCount: number;
99
117
  }
100
118
  /**
101
- * §5.9.2 orphan sweep — the ready-made GC helper. Reads the live keep-set
102
- * (`storage.listReferencedBlobIds`) and hands it, with the grace cutoff, to
103
- * `blobStore.sweepOrphans`, which deletes only blobs BOTH unreferenced AND
104
- * older than the grace period. Returns the swept ids and emits one
105
- * `blob.swept` ops event.
119
+ * §5.9.2 orphan sweep — the ready-made GC helper. Hands the grace cutoff and
120
+ * a reference check to `blobStore.sweepOrphans`, which deletes only blobs
121
+ * older than the grace period that no row references at the moment of the
122
+ * delete. The check consults the keep-set read when the sweep started and,
123
+ * for a blob outside it, reads the reference index again right before that
124
+ * blob's delete, so a push that lands during the sweep keeps its blob.
125
+ * Returns the swept ids and emits one `blob.swept` ops event.
106
126
  *
107
- * The upload-before-reference race, stated honestly: a client uploads bytes
108
- * before pushing the row that references them (§5.9.2), so a just-uploaded
109
- * blob is unreferenced until its push arrives. The grace period is the ONLY
110
- * thing standing between that fresh blob and deletion — keep it far above any
111
- * push latency (the 24 h default is deliberately generous). The store's
112
- * `createdAtMs` (object metadata) is the upload time the cutoff compares
113
- * against.
127
+ * A client uploads bytes before pushing the row that references them
128
+ * (§5.9.2), so a just-uploaded blob is unreferenced until its push arrives.
129
+ * The grace period protects that window: keep it far above any push latency
130
+ * (the 24 h default). The store's upload time (`createdAtMs` metadata, or the
131
+ * backend's last-modified time for an object uploaded through a presigned
132
+ * PUT) is what the cutoff compares against; a re-upload refreshes it.
114
133
  *
115
134
  * Requires `storage.listReferencedBlobIds` (the §5.9.4 reference index). A
116
- * storage backend without it cannot be swept safely — this throws rather
117
- * than sweeping against an empty keep-set (which would delete everything).
135
+ * storage backend without it cannot be swept safely, so this throws.
118
136
  */
119
137
  export declare function sweepOrphanBlobs(storage: ServerStorage, blobStore: BlobStore, partition: string, options?: SweepOrphanBlobsOptions): Promise<SweepOrphanBlobsResult>;
Binary file
@@ -29,10 +29,11 @@
29
29
  *
30
30
  * The orphan sweep is the ONLY operation that LISTs: `sweepOrphans` pages a
31
31
  * prefix listing over the `blob/` prefix (an admin/GC path, off the hot path)
32
- * to find candidates, reads each object's `createdAtMs` from its metadata for
33
- * the grace check, and DELETEs the unreferenced-and-old ones.
32
+ * to find candidates, reads each object's upload time from its metadata (or
33
+ * the backend's last-modified time for an object a presigned PUT wrote), and
34
+ * DELETEs the old ones the reference index does not name at that moment.
34
35
  */
35
- import type { BlobRecord, BlobStore, BlobStoreStats } from './blob-store.js';
36
+ import type { BlobRecord, BlobStore, BlobStoreStats, BlobStream } from './blob-store.js';
36
37
  import type { ObjectBucket } from './object-bucket.js';
37
38
  export interface BucketBlobStoreOptions {
38
39
  /** Key namespace inside the bucket, e.g. `syncular/`. Default: none. */
@@ -57,14 +58,17 @@ export declare class BucketBlobStore implements BlobStore {
57
58
  record: BlobRecord;
58
59
  bytes: Uint8Array;
59
60
  } | undefined>;
61
+ /** The record plus the object GET's body, never buffered (see `get`). */
62
+ open(partition: string, blobId: string): Promise<BlobStream | undefined>;
60
63
  /**
61
64
  * §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
62
- * {partition}/` prefix, HEAD each object for its `createdAtMs`, and DELETE
63
- * the ones both older than `olderThanMs` AND absent from
64
- * `referencedBlobIds`. Returns the deleted blobIds. Never deletes a
65
- * referenced blob (§5.9.2). Off the hot path — a host-scheduled GC job.
65
+ * {partition}/` prefix, HEAD each object for its upload time (the metadata
66
+ * `createdAtMs`, else the backend's last-modified time; an object with
67
+ * neither is skipped), and DELETE the ones older than `olderThanMs` that
68
+ * `isReferenced` denies right before the delete. Returns the deleted
69
+ * blobIds. Off the hot path — a host-scheduled GC job.
66
70
  */
67
- sweepOrphans(partition: string, olderThanMs: number, referencedBlobIds: ReadonlySet<string>): Promise<string[]>;
71
+ sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
68
72
  /**
69
73
  * Store-wide counters from the pointer-object accumulator (never a LIST on
70
74
  * the read path). `approximate: true` for the same reasons the segment