@syncular/server 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/README.md
CHANGED
|
@@ -357,7 +357,7 @@ revision (`catalog-v1` to `catalog-v2`) and rerun it; do not delete the database
|
|
|
357
357
|
or unrelated rows. This does not apply to application commands. After an
|
|
358
358
|
unknown command outcome, reuse the original idempotency key because changing it
|
|
359
359
|
can execute the operation twice. The full inspection and recovery recipe is in
|
|
360
|
-
the public [server guide](https://syncular.dev/
|
|
360
|
+
the public [server guide](https://syncular.dev/server-operations/#seeding-data).
|
|
361
361
|
|
|
362
362
|
One extra rule applies when the seed actor changes. A Syncular `clientId` is
|
|
363
363
|
bound to its first actor inside a partition. Keep the client ID stable for seed
|
|
@@ -412,7 +412,7 @@ scope variable, named-query obligation, subscription descriptor, or client
|
|
|
412
412
|
authority. Never expose table/index/value selection through a client-controlled
|
|
413
413
|
route. Custom adapters may omit the additive method; authoritative commands
|
|
414
414
|
must check for it and fail closed. See the public
|
|
415
|
-
[storage lookup guide](https://syncular.dev/server-storage/#
|
|
415
|
+
[storage lookup guide](https://syncular.dev/server-storage-reference/#advanced-row-lookups-for-trusted-server-code)
|
|
416
416
|
for a user-scoped key-grant table revoked through a Workspace index and for the
|
|
417
417
|
atomic reverse-index/queue fallback required by ordered or derived lookups.
|
|
418
418
|
|
|
@@ -583,7 +583,7 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
|
|
|
583
583
|
| `reaction.prune_completed` | One bounded terminal-reaction retention pass finished | `completedBeforeMs`, `deadLetterBeforeMs`, `limit`, `removedCompleted`, `removedDeadLetter`, `mayHaveMore` |
|
|
584
584
|
| `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `subscriptions[]`: `{id, table, status, mode` (`bootstrap` \| `incremental` \| `none`)`, fromCursor, nextCursor, commits, changes, segments[]}`; each segment: `{mediaType` (`rows` \| `sqlite`)`, delivery` (`inline` \| `ref`)`, origin` (`built` \| `reused`)`, bytes, rows}` |
|
|
585
585
|
| `segment.downloaded` | Every direct segment download (§5.5), success or failure | `segmentId`, `outcome` (`ok` \| `error`), `errorCode?`, `mediaType?`, `bytes?`, `durationMs` |
|
|
586
|
-
| `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (
|
|
586
|
+
| `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (old candidates still referenced), `graceMs` |
|
|
587
587
|
| `realtime.opened` | A socket registered with the hub and got `hello` (§8.1) | `sessionId`, `clientId`, `registrations`, `cursor`, `latestSeq` |
|
|
588
588
|
| `realtime.closed` | A session left the hub (once per session) | `sessionId`, `durationMs` |
|
|
589
589
|
| `realtime.delta` | A delta message pushed over the socket (§8.2) | `sessionId`, `commitSeq`, `bytes`, `changes` |
|
|
@@ -959,12 +959,12 @@ and mints a fresh URL) — never a fall-through.
|
|
|
959
959
|
`{ byteLength, mediaType? }`) mints a single presigned PUT; the client PUTs
|
|
960
960
|
bytes straight to the object store. **Upload authz is host-authentication only**
|
|
961
961
|
(any authenticated actor may obtain a grant within the size cap) — uploading
|
|
962
|
-
bytes is not a scope-bearing act, because the content address discloses
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
962
|
+
bytes is not a scope-bearing act, because the content address discloses
|
|
963
|
+
nothing. The grant signs `content-length` (the declared `byteLength`) and
|
|
964
|
+
`x-amz-checksum-sha256` (the `blobId` digest as base64), and the client sends
|
|
965
|
+
that checksum header, so the object store refuses a PUT of another size or
|
|
966
|
+
other bytes. The **size cap is enforced up front** against the declared
|
|
967
|
+
`byteLength`, before any URL is minted. An already-present blob returns `{ present: true }` (skip
|
|
968
968
|
the PUT, idempotent §5.9.3). A **single PUT only** — never a multipart or chunk
|
|
969
969
|
protocol; resumable upload, when it lands, is provider multipart behind this
|
|
970
970
|
same grant. Absent config ⇒ the client streams through the direct
|
|
@@ -1006,8 +1006,11 @@ Nothing reclaims blobs automatically — the host schedules the sweep, the blob
|
|
|
1006
1006
|
analogue of `pruneCommitLog`. `sweepOrphanBlobs(storage, blobStore, partition,
|
|
1007
1007
|
{ graceMs })` reads the live keep-set from the §5.9.4 reference index
|
|
1008
1008
|
(`storage.listReferencedBlobIds`) and deletes every blob that is **both**
|
|
1009
|
-
unreferenced **and** older than the grace period.
|
|
1010
|
-
|
|
1009
|
+
unreferenced **and** older than the grace period. Before each delete it reads
|
|
1010
|
+
the index again, so a reference that lands during the sweep keeps its blob. It
|
|
1011
|
+
emits one `blob.swept` ops event (`{ swept, referenced, graceMs }`, where
|
|
1012
|
+
`referenced` counts old candidates a row still references) and returns the
|
|
1013
|
+
deleted ids.
|
|
1011
1014
|
|
|
1012
1015
|
```ts
|
|
1013
1016
|
import { sweepOrphanBlobs } from '@syncular/server';
|
|
@@ -1028,8 +1031,10 @@ arrived. So the grace period must comfortably exceed any sane upload→push
|
|
|
1028
1031
|
latency — the **default is 24 h**, deliberately far above any push window.
|
|
1029
1032
|
Lower it only if you fully understand your clients' outbox latency; there is
|
|
1030
1033
|
no upside to a tight grace and a real data-loss risk. The sweep compares
|
|
1031
|
-
against the blob's **upload time
|
|
1032
|
-
|
|
1034
|
+
against the blob's **latest upload time**: `createdAtMs` from object metadata,
|
|
1035
|
+
or the object's last-modified time for an object a presigned PUT wrote. An
|
|
1036
|
+
object with neither is skipped, and an idempotent re-upload rewrites the
|
|
1037
|
+
object with the re-upload time, which restarts its grace period.
|
|
1033
1038
|
|
|
1034
1039
|
`sweepOrphanBlobs` **requires** `storage.listReferencedBlobIds` (the §5.9.4
|
|
1035
1040
|
reference index — SQLite, D1, and Postgres all implement it). Against a
|
|
@@ -1058,11 +1063,10 @@ row-derived authorization check passes** (`handleBlobDownload` resolves the
|
|
|
1058
1063
|
actor's scopes and tests the referencing rows first; a `blobId` is never a
|
|
1059
1064
|
bearer capability minted from the id alone). The signed object key embeds the
|
|
1060
1065
|
`blobId`, TTL SHOULD be ≤ 15 min (default 900 s), and the URL is a short-lived
|
|
1061
|
-
grant to exactly those immutable bytes.
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
server) is a later rung — the server-side issuance ships now.
|
|
1066
|
+
grant to exactly those immutable bytes. When `blobSignedUrls` is set, the
|
|
1067
|
+
download answers `{ url, urlExpiresAtMs }` and no bytes, and the reference
|
|
1068
|
+
clients fetch the URL and verify the content address (see "Presigned blobs,
|
|
1069
|
+
end to end" above).
|
|
1066
1070
|
|
|
1067
1071
|
### Native HMAC vs delegated presign (§5.4)
|
|
1068
1072
|
|
|
@@ -1235,7 +1239,7 @@ function bunSqlExecutor(sql: import('bun').SQL): PgExecutor {
|
|
|
1235
1239
|
const over = (h: any): PgQueryable => ({
|
|
1236
1240
|
async query(text, params) {
|
|
1237
1241
|
const rows = await h.unsafe(text, params ? [...params] : []);
|
|
1238
|
-
return { rows
|
|
1242
|
+
return { rows };
|
|
1239
1243
|
},
|
|
1240
1244
|
});
|
|
1241
1245
|
return {
|
|
@@ -1283,6 +1287,11 @@ function pgPoolExecutor(pool: Pool): PgExecutor {
|
|
|
1283
1287
|
}
|
|
1284
1288
|
```
|
|
1285
1289
|
|
|
1290
|
+
**Result shape.** `query` returns `{ rows }`. The storage never reads an
|
|
1291
|
+
affected-row count: every write whose outcome matters ends in `RETURNING`
|
|
1292
|
+
and counts the returned rows, because drivers disagree on the count of a
|
|
1293
|
+
statement without result rows.
|
|
1294
|
+
|
|
1286
1295
|
**Type-parser note.** `commit_seq`/`server_version` are `int8`. Drivers
|
|
1287
1296
|
decode `int8` differently (node-postgres → `string`, Bun.sql → `bigint`,
|
|
1288
1297
|
pglite → `number`); the storage layer coerces every sequence read through
|
|
@@ -1367,9 +1376,9 @@ SYNCULAR_PG_URL=postgres://user:pass@localhost:5432/db bun run bench
|
|
|
1367
1376
|
```
|
|
1368
1377
|
|
|
1369
1378
|
Custom storage adapters must implement
|
|
1370
|
-
`getActiveClientCursorFloor(partition, cutoffMs)`. Return the minimum
|
|
1371
|
-
whose `updatedAtMs >= cutoffMs`, or `null` when no client
|
|
1372
|
-
|
|
1379
|
+
`getActiveClientCursorFloor(partition, cutoffMs)`. Return the minimum
|
|
1380
|
+
non-negative cursor whose `updatedAtMs >= cutoffMs`, or `null` when no client
|
|
1381
|
+
qualifies. A bootstrapping client at cursor -1 holds no cursor. Pruning and admin horizon status use this scalar
|
|
1373
1382
|
aggregate; `listClientCursors` remains the explicit listing interface.
|
|
1374
1383
|
|
|
1375
1384
|
SQLite image builders now return `Promise<Uint8Array>` and receive
|
package/dist/admin.js
CHANGED
|
@@ -55,7 +55,7 @@ const VERDICT_ORDER = [
|
|
|
55
55
|
'delivered',
|
|
56
56
|
];
|
|
57
57
|
/** One known client's subscriptions on the row's table and its verdict. */
|
|
58
|
-
function traceVerdict(record, row, lastChange, horizonSeq, allowed) {
|
|
58
|
+
function traceVerdict(declared, record, row, lastChange, horizonSeq, allowed) {
|
|
59
59
|
const rowScopes = row.scopes ?? {};
|
|
60
60
|
const subscriptions = record.subscriptions
|
|
61
61
|
.filter((sub) => sub.table === row.table)
|
|
@@ -78,7 +78,10 @@ function traceVerdict(record, row, lastChange, horizonSeq, allowed) {
|
|
|
78
78
|
};
|
|
79
79
|
if (allowed === undefined || allowed === 'outage')
|
|
80
80
|
return traced;
|
|
81
|
-
const effective = computeEffective(sub.scopes, {
|
|
81
|
+
const effective = computeEffective(declared, sub.scopes, {
|
|
82
|
+
ok: true,
|
|
83
|
+
allowed,
|
|
84
|
+
});
|
|
82
85
|
return {
|
|
83
86
|
...traced,
|
|
84
87
|
effectiveMatch: effective.status === 'active' &&
|
|
@@ -349,7 +352,7 @@ export class SyncularAdmin {
|
|
|
349
352
|
...base,
|
|
350
353
|
client: (await this.#clientView(partition))(record),
|
|
351
354
|
...(allowed !== undefined ? { allowed } : {}),
|
|
352
|
-
...traceVerdict(record, row, lastChange, horizonSeq, allowed),
|
|
355
|
+
...traceVerdict(this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set(), record, row, lastChange, horizonSeq, allowed),
|
|
353
356
|
};
|
|
354
357
|
}
|
|
355
358
|
/**
|
|
@@ -364,6 +367,7 @@ export class SyncularAdmin {
|
|
|
364
367
|
const lastChange = row.history[0];
|
|
365
368
|
const horizonSeq = await this.#storage.getHorizonSeq(partition);
|
|
366
369
|
const view = await this.#clientView(partition);
|
|
370
|
+
const declared = this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set();
|
|
367
371
|
const allowedByActor = new Map();
|
|
368
372
|
const clients = [];
|
|
369
373
|
for (const record of await list(partition)) {
|
|
@@ -372,7 +376,7 @@ export class SyncularAdmin {
|
|
|
372
376
|
}
|
|
373
377
|
clients.push({
|
|
374
378
|
client: view(record),
|
|
375
|
-
verdict: traceVerdict(record, row, lastChange, horizonSeq, allowedByActor.get(record.actorId)).verdict,
|
|
379
|
+
verdict: traceVerdict(declared, record, row, lastChange, horizonSeq, allowedByActor.get(record.actorId)).verdict,
|
|
376
380
|
});
|
|
377
381
|
}
|
|
378
382
|
clients.sort((a, b) => VERDICT_ORDER.indexOf(a.verdict) - VERDICT_ORDER.indexOf(b.verdict) ||
|
package/dist/blob-handlers.d.ts
CHANGED
|
@@ -1,29 +1,20 @@
|
|
|
1
1
|
import type { SyncRequestContext } from './context.js';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
* The blob bytes, when served inline. **Absent** when the host configured
|
|
11
|
-
* `blobSignedUrls` and a `url` was issued: the sync server exits the egress
|
|
12
|
-
* path and the client fetches `url` directly (§5.9.5 always-issue).
|
|
13
|
-
*/
|
|
14
|
-
readonly bytes?: Uint8Array;
|
|
2
|
+
/**
|
|
3
|
+
* A §5.9.5 download: the stored bytes as a stream with their length (inline
|
|
4
|
+
* delivery), or, when the host configured `blobSignedUrls`, a provider-signed
|
|
5
|
+
* GET URL issued after the row-derived authorization check. The client
|
|
6
|
+
* fetches a URL directly (no host auth), verifies the content address, and on
|
|
7
|
+
* failure re-requests this endpoint (§5.9.5 recovery rule).
|
|
8
|
+
*/
|
|
9
|
+
export type BlobDownloadResult = {
|
|
15
10
|
readonly headers: Record<string, string>;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
*/
|
|
24
|
-
readonly url?: string;
|
|
25
|
-
readonly urlExpiresAtMs?: number;
|
|
26
|
-
}
|
|
11
|
+
readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
12
|
+
readonly byteLength: number;
|
|
13
|
+
} | {
|
|
14
|
+
readonly headers: Record<string, string>;
|
|
15
|
+
readonly url: string;
|
|
16
|
+
readonly urlExpiresAtMs: number;
|
|
17
|
+
};
|
|
27
18
|
/** `POST <mount>/blobs/{blobId}/upload-grant` request body (§5.9.3). */
|
|
28
19
|
export interface BlobUploadGrantRequest {
|
|
29
20
|
readonly blobId: string;
|
|
@@ -45,8 +36,13 @@ export interface BlobUploadGrantResult {
|
|
|
45
36
|
readonly urlExpiresAtMs?: number;
|
|
46
37
|
readonly present?: boolean;
|
|
47
38
|
}
|
|
48
|
-
/**
|
|
49
|
-
|
|
39
|
+
/**
|
|
40
|
+
* `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
|
|
41
|
+
* size cap applies to the declared `Content-Length` before any byte is read
|
|
42
|
+
* and again to the counted decoded bytes while reading, so an oversized body
|
|
43
|
+
* is never buffered whole.
|
|
44
|
+
*/
|
|
45
|
+
export declare function handleBlobUpload(ctx: SyncRequestContext, blobId: string, request: Request): Promise<void>;
|
|
50
46
|
/**
|
|
51
47
|
* `GET <mount>/blobs/{blobId}` (§5.9.5). Authorization derives from the
|
|
52
48
|
* referencing rows: the actor may download iff at least one row in the
|
|
@@ -58,11 +54,9 @@ export declare function handleBlobDownload(ctx: SyncRequestContext, blobId: stri
|
|
|
58
54
|
* auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
|
|
59
55
|
* so any authenticated actor may obtain a grant within the size cap (§5.9.3).
|
|
60
56
|
*
|
|
61
|
-
* The size cap is enforced HERE, up front, against the declared `byteLength
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
* and the address is verified at reference time (§5.9.6 push existence) and on
|
|
65
|
-
* every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
|
|
57
|
+
* The size cap is enforced HERE, up front, against the declared `byteLength`;
|
|
58
|
+
* the presigned PUT binds that length and the content address, so the object
|
|
59
|
+
* store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
|
|
66
60
|
* `present` marker if the blob already exists (idempotent §5.9.3), or an empty
|
|
67
61
|
* result when no presigned-upload store is configured (client streams direct).
|
|
68
62
|
*/
|
package/dist/blob-handlers.js
CHANGED
|
@@ -7,41 +7,67 @@
|
|
|
7
7
|
* adapters map these onto `PUT/GET <mount>/blobs/{blobId}`.
|
|
8
8
|
*/
|
|
9
9
|
import { blobIdFor, isBlobId } from './blob-store.js';
|
|
10
|
+
import { decodeRequestBody } from './content-encoding.js';
|
|
10
11
|
import { clockOf, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
11
12
|
import { SyncError, syncError } from './errors.js';
|
|
12
13
|
import { emitEvent } from './events.js';
|
|
13
14
|
import { compileSchema } from './schema.js';
|
|
14
15
|
import { authorizeWrite } from './scopes.js';
|
|
16
|
+
import { bytesStream } from './segment-store.js';
|
|
15
17
|
import { issueBlobUploadUrl, issueBlobUrl } from './signed-url.js';
|
|
16
18
|
const DEFAULT_MAX_BLOB_BYTES = 64 * 1024 * 1024;
|
|
17
|
-
/**
|
|
18
|
-
|
|
19
|
+
/**
|
|
20
|
+
* `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
|
|
21
|
+
* size cap applies to the declared `Content-Length` before any byte is read
|
|
22
|
+
* and again to the counted decoded bytes while reading, so an oversized body
|
|
23
|
+
* is never buffered whole.
|
|
24
|
+
*/
|
|
25
|
+
export async function handleBlobUpload(ctx, blobId, request) {
|
|
19
26
|
await touchAuthenticatedPartition(ctx);
|
|
20
27
|
const store = ctx.blobs;
|
|
21
28
|
if (store === undefined) {
|
|
22
29
|
throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
|
|
23
30
|
}
|
|
24
|
-
if (!isBlobId(
|
|
31
|
+
if (!isBlobId(blobId)) {
|
|
25
32
|
throw syncError('blob.hash_mismatch', 'blobId path is not "sha256:<64 hex>" (§5.9.1)');
|
|
26
33
|
}
|
|
27
34
|
const maxBytes = ctx.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
|
|
28
|
-
|
|
29
|
-
|
|
35
|
+
const tooLarge = syncError('blob.too_large', 'blob exceeds the server size cap (§5.9.3)', JSON.stringify({ max: maxBytes }));
|
|
36
|
+
if (Number(request.headers.get('content-length')) > maxBytes)
|
|
37
|
+
throw tooLarge;
|
|
38
|
+
let bytes;
|
|
39
|
+
try {
|
|
40
|
+
bytes = await decodeRequestBody(request, maxBytes);
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
throw error instanceof SyncError && error.code === 'sync.limit_exceeded'
|
|
44
|
+
? tooLarge
|
|
45
|
+
: error;
|
|
30
46
|
}
|
|
31
|
-
|
|
32
|
-
|
|
47
|
+
if (bytes === undefined) {
|
|
48
|
+
throw syncError('sync.invalid_request', 'unsupported content encoding');
|
|
49
|
+
}
|
|
50
|
+
if ((await blobIdFor(bytes)) !== blobId) {
|
|
33
51
|
throw syncError('blob.hash_mismatch', 'uploaded bytes do not hash to the blobId path (§5.9.3)');
|
|
34
52
|
}
|
|
53
|
+
const contentType = request.headers
|
|
54
|
+
.get('content-type')
|
|
55
|
+
?.split(';')[0]
|
|
56
|
+
?.trim();
|
|
35
57
|
const clock = clockOf(ctx);
|
|
36
|
-
await store.put(ctx.partition,
|
|
58
|
+
await store.put(ctx.partition, blobId, bytes, clock(), contentType === undefined ||
|
|
59
|
+
contentType === '' ||
|
|
60
|
+
contentType === 'application/octet-stream'
|
|
61
|
+
? undefined
|
|
62
|
+
: contentType);
|
|
37
63
|
if (ctx.events !== undefined) {
|
|
38
64
|
emitEvent(ctx.events, {
|
|
39
65
|
type: 'blob.uploaded',
|
|
40
66
|
atMs: clock(),
|
|
41
67
|
partition: ctx.partition,
|
|
42
68
|
actorId: ctx.actorId,
|
|
43
|
-
blobId
|
|
44
|
-
bytes:
|
|
69
|
+
blobId,
|
|
70
|
+
bytes: bytes.length,
|
|
45
71
|
});
|
|
46
72
|
}
|
|
47
73
|
}
|
|
@@ -68,7 +94,7 @@ export async function handleBlobDownload(ctx, blobId) {
|
|
|
68
94
|
outcome: 'ok',
|
|
69
95
|
// Presigned delivery carries no inline bytes (§5.9.5 always-issue); the
|
|
70
96
|
// byte count is unknown to the sync server, which exited the egress path.
|
|
71
|
-
bytes: result.
|
|
97
|
+
bytes: 'byteLength' in result ? result.byteLength : 0,
|
|
72
98
|
durationMs: clock() - startedAtMs,
|
|
73
99
|
});
|
|
74
100
|
return result;
|
|
@@ -95,21 +121,10 @@ async function downloadBlob(ctx, blobId) {
|
|
|
95
121
|
if (!isBlobId(blobId)) {
|
|
96
122
|
throw syncError('blob.not_found', 'malformed blobId (§5.9.1)');
|
|
97
123
|
}
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
let entry;
|
|
103
|
-
if (presignConfigured) {
|
|
104
|
-
if (!(await store.has(ctx.partition, blobId))) {
|
|
105
|
-
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
else {
|
|
109
|
-
entry = await store.get(ctx.partition, blobId);
|
|
110
|
-
if (entry === undefined) {
|
|
111
|
-
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
112
|
-
}
|
|
124
|
+
// Existence first (a HEAD), authorization second, bytes last: no blob byte
|
|
125
|
+
// is read for an actor the reference index does not authorize (§5.9.5).
|
|
126
|
+
if (!(await store.has(ctx.partition, blobId))) {
|
|
127
|
+
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
113
128
|
}
|
|
114
129
|
// §5.9.5 authorization: resolve once, then test referencing rows.
|
|
115
130
|
const listRows = ctx.storage.listRowsReferencingBlob;
|
|
@@ -177,24 +192,33 @@ async function downloadBlob(ctx, blobId) {
|
|
|
177
192
|
headers: {
|
|
178
193
|
ETag: `"${blobId}"`,
|
|
179
194
|
'Cache-Control': 'private, max-age=0',
|
|
180
|
-
Vary: 'Authorization
|
|
195
|
+
Vary: 'Authorization',
|
|
181
196
|
},
|
|
182
197
|
url: issued.url,
|
|
183
198
|
urlExpiresAtMs: issued.urlExpiresAtMs,
|
|
184
199
|
};
|
|
185
200
|
}
|
|
186
|
-
// Inline delivery
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
201
|
+
// Inline delivery streams the stored bytes verbatim (identity encoding,
|
|
202
|
+
// §5.9.5). A store without `open` serves its `get` bytes as one chunk.
|
|
203
|
+
const opened = store.open !== undefined
|
|
204
|
+
? await store.open(ctx.partition, blobId)
|
|
205
|
+
: await store.get(ctx.partition, blobId).then((entry) => entry === undefined
|
|
206
|
+
? undefined
|
|
207
|
+
: {
|
|
208
|
+
record: entry.record,
|
|
209
|
+
body: bytesStream(entry.bytes),
|
|
210
|
+
byteLength: entry.bytes.length,
|
|
211
|
+
});
|
|
212
|
+
if (opened === undefined) {
|
|
213
|
+
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
191
214
|
}
|
|
192
215
|
// §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
|
|
193
216
|
// pointed at the route renders them. A JSON type stays octet-stream: a
|
|
194
217
|
// JSON body is reserved for the signed-URL envelope.
|
|
195
|
-
const mediaType =
|
|
218
|
+
const mediaType = opened.record.mediaType;
|
|
196
219
|
return {
|
|
197
|
-
|
|
220
|
+
body: opened.body,
|
|
221
|
+
byteLength: opened.byteLength,
|
|
198
222
|
headers: {
|
|
199
223
|
'Content-Type': mediaType === undefined ||
|
|
200
224
|
mediaType.toLowerCase().includes('application/json')
|
|
@@ -204,7 +228,7 @@ async function downloadBlob(ctx, blobId) {
|
|
|
204
228
|
'Content-Security-Policy': "sandbox; default-src 'none'",
|
|
205
229
|
ETag: `"${blobId}"`,
|
|
206
230
|
'Cache-Control': 'private, max-age=0',
|
|
207
|
-
Vary: 'Authorization
|
|
231
|
+
Vary: 'Authorization',
|
|
208
232
|
},
|
|
209
233
|
};
|
|
210
234
|
}
|
|
@@ -213,11 +237,9 @@ async function downloadBlob(ctx, blobId) {
|
|
|
213
237
|
* auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
|
|
214
238
|
* so any authenticated actor may obtain a grant within the size cap (§5.9.3).
|
|
215
239
|
*
|
|
216
|
-
* The size cap is enforced HERE, up front, against the declared `byteLength
|
|
217
|
-
*
|
|
218
|
-
*
|
|
219
|
-
* and the address is verified at reference time (§5.9.6 push existence) and on
|
|
220
|
-
* every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
|
|
240
|
+
* The size cap is enforced HERE, up front, against the declared `byteLength`;
|
|
241
|
+
* the presigned PUT binds that length and the content address, so the object
|
|
242
|
+
* store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
|
|
221
243
|
* `present` marker if the blob already exists (idempotent §5.9.3), or an empty
|
|
222
244
|
* result when no presigned-upload store is configured (client streams direct).
|
|
223
245
|
*/
|
|
@@ -235,7 +257,7 @@ export async function handleBlobUploadGrant(ctx, request) {
|
|
|
235
257
|
request.byteLength < 0 ||
|
|
236
258
|
request.byteLength > maxBytes) {
|
|
237
259
|
// §5.9.3: cap enforced at grant issuance, before any URL is minted.
|
|
238
|
-
throw syncError('blob.too_large',
|
|
260
|
+
throw syncError('blob.too_large', 'blob byteLength exceeds the server size cap (§5.9.3)', JSON.stringify({ max: maxBytes }));
|
|
239
261
|
}
|
|
240
262
|
// §5.9.3 no presigned-upload store: report no grant; the client streams
|
|
241
263
|
// through the direct PUT endpoint (capability, not fallback).
|
package/dist/blob-store.d.ts
CHANGED
|
@@ -17,8 +17,15 @@ export interface BlobRecord {
|
|
|
17
17
|
readonly byteLength: number;
|
|
18
18
|
/** Advisory MIME type persisted from the upload, if any (§5.9.5). */
|
|
19
19
|
readonly mediaType?: string;
|
|
20
|
+
/** Time of the latest upload; the orphan sweep's grace clock (§5.9.2). */
|
|
20
21
|
readonly createdAtMs: number;
|
|
21
22
|
}
|
|
23
|
+
/** `BlobStore.open`: a stored blob with its bytes as a stream. */
|
|
24
|
+
export interface BlobStream {
|
|
25
|
+
readonly record: BlobRecord;
|
|
26
|
+
readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
27
|
+
readonly byteLength: number;
|
|
28
|
+
}
|
|
22
29
|
/**
|
|
23
30
|
* Coarse blob-store counters for the admin/console read surface (work item
|
|
24
31
|
* §2.5), partition-scoped. Optional. `count`/`bytes` are total stored
|
|
@@ -36,10 +43,12 @@ export interface BlobStoreStats {
|
|
|
36
43
|
}
|
|
37
44
|
export interface BlobStore {
|
|
38
45
|
/**
|
|
39
|
-
* Store blob bytes under `blobId` (idempotent
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
46
|
+
* Store blob bytes under `blobId` (idempotent: same id ⇒ success, §5.9.3).
|
|
47
|
+
* A re-upload moves `createdAtMs` forward to `nowMs`, so a blob that a
|
|
48
|
+
* second client is about to reference gets a fresh sweep grace period. A
|
|
49
|
+
* re-upload that declares a media type records it on a blob stored without
|
|
50
|
+
* one; a recorded media type never changes. The caller has already
|
|
51
|
+
* verified the content address matches.
|
|
43
52
|
*/
|
|
44
53
|
put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
|
|
45
54
|
/** Presence check for the push existence rule (§5.9.6) and download. */
|
|
@@ -49,12 +58,20 @@ export interface BlobStore {
|
|
|
49
58
|
bytes: Uint8Array;
|
|
50
59
|
} | undefined>;
|
|
51
60
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
|
|
61
|
+
* Optional streaming read: the record plus the bytes as a stream, so the
|
|
62
|
+
* inline download route relays a blob without buffering it (the
|
|
63
|
+
* `SegmentStore.open` shape). Stores without it are served through `get`.
|
|
64
|
+
*/
|
|
65
|
+
open?(partition: string, blobId: string): Promise<BlobStream | undefined>;
|
|
66
|
+
/**
|
|
67
|
+
* §5.9.2 orphan sweep: delete blobs last uploaded before `olderThanMs`
|
|
68
|
+
* that no live row references. `isReferenced` asks the storage reference
|
|
69
|
+
* index about one blob; the store calls it for each old candidate right
|
|
70
|
+
* before that candidate's delete, so a reference that lands during the
|
|
71
|
+
* sweep keeps its blob. The store never scans rows itself. Returns the
|
|
72
|
+
* deleted blobIds.
|
|
56
73
|
*/
|
|
57
|
-
sweepOrphans(partition: string, olderThanMs: number,
|
|
74
|
+
sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
|
|
58
75
|
/**
|
|
59
76
|
* §4.6.1 erasure: delete one blob whose last reference is gone. Deleting a
|
|
60
77
|
* missing blob is a no-op.
|
|
@@ -74,7 +91,7 @@ export declare class MemoryBlobStore implements BlobStore {
|
|
|
74
91
|
record: BlobRecord;
|
|
75
92
|
bytes: Uint8Array;
|
|
76
93
|
} | undefined>;
|
|
77
|
-
sweepOrphans(partition: string, olderThanMs: number,
|
|
94
|
+
sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
|
|
78
95
|
delete(partition: string, blobId: string): Promise<void>;
|
|
79
96
|
stats(partition: string): Promise<BlobStoreStats>;
|
|
80
97
|
}
|
|
@@ -95,25 +112,26 @@ export interface SweepOrphanBlobsOptions {
|
|
|
95
112
|
export interface SweepOrphanBlobsResult {
|
|
96
113
|
readonly partition: string;
|
|
97
114
|
readonly swept: string[];
|
|
115
|
+
/** Candidates past the grace period that a live row still references. */
|
|
98
116
|
readonly referencedCount: number;
|
|
99
117
|
}
|
|
100
118
|
/**
|
|
101
|
-
* §5.9.2 orphan sweep — the ready-made GC helper.
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
119
|
+
* §5.9.2 orphan sweep — the ready-made GC helper. Hands the grace cutoff and
|
|
120
|
+
* a reference check to `blobStore.sweepOrphans`, which deletes only blobs
|
|
121
|
+
* older than the grace period that no row references at the moment of the
|
|
122
|
+
* delete. The check consults the keep-set read when the sweep started and,
|
|
123
|
+
* for a blob outside it, reads the reference index again right before that
|
|
124
|
+
* blob's delete, so a push that lands during the sweep keeps its blob.
|
|
125
|
+
* Returns the swept ids and emits one `blob.swept` ops event.
|
|
106
126
|
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
* against.
|
|
127
|
+
* A client uploads bytes before pushing the row that references them
|
|
128
|
+
* (§5.9.2), so a just-uploaded blob is unreferenced until its push arrives.
|
|
129
|
+
* The grace period protects that window: keep it far above any push latency
|
|
130
|
+
* (the 24 h default). The store's upload time (`createdAtMs` metadata, or the
|
|
131
|
+
* backend's last-modified time for an object uploaded through a presigned
|
|
132
|
+
* PUT) is what the cutoff compares against; a re-upload refreshes it.
|
|
114
133
|
*
|
|
115
134
|
* Requires `storage.listReferencedBlobIds` (the §5.9.4 reference index). A
|
|
116
|
-
* storage backend without it cannot be swept safely
|
|
117
|
-
* than sweeping against an empty keep-set (which would delete everything).
|
|
135
|
+
* storage backend without it cannot be swept safely, so this throws.
|
|
118
136
|
*/
|
|
119
137
|
export declare function sweepOrphanBlobs(storage: ServerStorage, blobStore: BlobStore, partition: string, options?: SweepOrphanBlobsOptions): Promise<SweepOrphanBlobsResult>;
|
package/dist/blob-store.js
CHANGED
|
Binary file
|
|
@@ -29,10 +29,11 @@
|
|
|
29
29
|
*
|
|
30
30
|
* The orphan sweep is the ONLY operation that LISTs: `sweepOrphans` pages a
|
|
31
31
|
* prefix listing over the `blob/` prefix (an admin/GC path, off the hot path)
|
|
32
|
-
* to find candidates, reads each object's
|
|
33
|
-
* the
|
|
32
|
+
* to find candidates, reads each object's upload time from its metadata (or
|
|
33
|
+
* the backend's last-modified time for an object a presigned PUT wrote), and
|
|
34
|
+
* DELETEs the old ones the reference index does not name at that moment.
|
|
34
35
|
*/
|
|
35
|
-
import type { BlobRecord, BlobStore, BlobStoreStats } from './blob-store.js';
|
|
36
|
+
import type { BlobRecord, BlobStore, BlobStoreStats, BlobStream } from './blob-store.js';
|
|
36
37
|
import type { ObjectBucket } from './object-bucket.js';
|
|
37
38
|
export interface BucketBlobStoreOptions {
|
|
38
39
|
/** Key namespace inside the bucket, e.g. `syncular/`. Default: none. */
|
|
@@ -57,14 +58,17 @@ export declare class BucketBlobStore implements BlobStore {
|
|
|
57
58
|
record: BlobRecord;
|
|
58
59
|
bytes: Uint8Array;
|
|
59
60
|
} | undefined>;
|
|
61
|
+
/** The record plus the object GET's body, never buffered (see `get`). */
|
|
62
|
+
open(partition: string, blobId: string): Promise<BlobStream | undefined>;
|
|
60
63
|
/**
|
|
61
64
|
* §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
|
|
62
|
-
* {partition}/` prefix, HEAD each object for its
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
65
|
+
* {partition}/` prefix, HEAD each object for its upload time (the metadata
|
|
66
|
+
* `createdAtMs`, else the backend's last-modified time; an object with
|
|
67
|
+
* neither is skipped), and DELETE the ones older than `olderThanMs` that
|
|
68
|
+
* `isReferenced` denies right before the delete. Returns the deleted
|
|
69
|
+
* blobIds. Off the hot path — a host-scheduled GC job.
|
|
66
70
|
*/
|
|
67
|
-
sweepOrphans(partition: string, olderThanMs: number,
|
|
71
|
+
sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
|
|
68
72
|
/**
|
|
69
73
|
* Store-wide counters from the pointer-object accumulator (never a LIST on
|
|
70
74
|
* the read path). `approximate: true` for the same reasons the segment
|