@syncular/server 0.57.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/dist/storage.d.ts
CHANGED
|
@@ -128,10 +128,25 @@ export interface NewChange {
|
|
|
128
128
|
readonly op: 'upsert' | 'delete';
|
|
129
129
|
/** Present for upsert (§2.2). */
|
|
130
130
|
readonly rowVersion?: number;
|
|
131
|
+
/**
|
|
132
|
+
* Present for delete: the deleted row's last `server_version`, recorded on
|
|
133
|
+
* its tombstone (§2.2). Never reaches the wire.
|
|
134
|
+
*/
|
|
135
|
+
readonly deletedVersion?: number;
|
|
131
136
|
readonly scopes: Record<string, string>;
|
|
132
137
|
/** Present for upsert. */
|
|
133
138
|
readonly payload?: Uint8Array;
|
|
134
139
|
}
|
|
140
|
+
/** §2.2 delete-precedence tombstone of a row deleted within the horizon. */
|
|
141
|
+
export interface Tombstone {
|
|
142
|
+
readonly commitSeq: number;
|
|
143
|
+
/**
|
|
144
|
+
* The deleted row's last `server_version` and stored scopes. Both are
|
|
145
|
+
* absent on a tombstone written before servers recorded them.
|
|
146
|
+
*/
|
|
147
|
+
readonly serverVersion?: number;
|
|
148
|
+
readonly scopes?: Record<string, string>;
|
|
149
|
+
}
|
|
135
150
|
export interface NewCommit {
|
|
136
151
|
readonly clientId: string;
|
|
137
152
|
readonly clientCommitId: string;
|
|
@@ -453,17 +468,17 @@ export interface StorageTransaction {
|
|
|
453
468
|
readonly rowIds: readonly string[];
|
|
454
469
|
}[]): Promise<void>;
|
|
455
470
|
/**
|
|
456
|
-
* §5 delete-precedence tombstone read: the
|
|
457
|
-
*
|
|
458
|
-
*
|
|
459
|
-
*
|
|
460
|
-
*
|
|
471
|
+
* §5 delete-precedence tombstone read: the newest applied delete of this
|
|
472
|
+
* row still within the pruning horizon, or `undefined` when no tombstone
|
|
473
|
+
* exists. Consulted only when an upsert's stored row is absent. Required:
|
|
474
|
+
* in-tree adapters implement it on the transaction's own connection with
|
|
475
|
+
* read-your-own-writes semantics.
|
|
461
476
|
*/
|
|
462
|
-
|
|
477
|
+
getTombstone(table: string, rowId: string): Promise<Tombstone | undefined>;
|
|
463
478
|
/**
|
|
464
|
-
* Remove the row's tombstone because the row
|
|
465
|
-
* transaction
|
|
466
|
-
*
|
|
479
|
+
* Remove the row's tombstone because the row is live again in this
|
|
480
|
+
* transaction: an insert over its tombstone (§6.2) or a §2.2 scope move.
|
|
481
|
+
* A no-op when no tombstone exists.
|
|
467
482
|
*/
|
|
468
483
|
clearTombstone(table: string, rowId: string): Promise<void>;
|
|
469
484
|
/**
|
|
@@ -763,7 +778,10 @@ export interface ServerStorage {
|
|
|
763
778
|
/** Advance an existing client's ACK cursor and timestamp atomically,
|
|
764
779
|
* preserving actor, wire version, and subscriptions. Missing records stay absent. */
|
|
765
780
|
updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
766
|
-
/**
|
|
781
|
+
/**
|
|
782
|
+
* Minimum non-negative cursor with updatedAtMs >= cutoff (§4.6); null when
|
|
783
|
+
* none are active. A bootstrapping client (cursor < 0) holds no cursor.
|
|
784
|
+
*/
|
|
767
785
|
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
|
768
786
|
/** Cursor records for client listings and administrative counts. */
|
|
769
787
|
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
@@ -791,9 +809,11 @@ export interface ServerStorage {
|
|
|
791
809
|
* `eraseScopeValueHistory`: under the partition write lock, removes up to
|
|
792
810
|
* `limit` upsert change records of `table` at or below `throughSeq` whose
|
|
793
811
|
* scopes bind `variable` to `value`, then up to `limit` change scope index
|
|
794
|
-
* entries of that binding whose commit holds no such change any more
|
|
812
|
+
* entries of that binding whose commit holds no such change any more,
|
|
813
|
+
* then strips `serverRow` from up to `limit` stored conflict results of
|
|
814
|
+
* `table` whose conflict authorization scopes bind `variable` to `value`.
|
|
795
815
|
* Delete change records stay: clients behind the erasure still need them.
|
|
796
|
-
* Returns the number of records removed.
|
|
816
|
+
* Returns the number of records removed or stripped.
|
|
797
817
|
*/
|
|
798
818
|
listScopeValueRows?(partition: string, query: ScopeValueQuery): Promise<string[]>;
|
|
799
819
|
eraseScopeValueHistory?(partition: string, query: ScopeValueHistoryQuery): Promise<number>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.59.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.59.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/admin.ts
CHANGED
|
@@ -535,6 +535,7 @@ const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
|
|
|
535
535
|
|
|
536
536
|
/** One known client's subscriptions on the row's table and its verdict. */
|
|
537
537
|
function traceVerdict(
|
|
538
|
+
declared: ReadonlySet<string>,
|
|
538
539
|
record: ClientRecord,
|
|
539
540
|
row: AdminRowInspection,
|
|
540
541
|
lastChange: RowChangeRecord | undefined,
|
|
@@ -565,7 +566,10 @@ function traceVerdict(
|
|
|
565
566
|
mismatches,
|
|
566
567
|
};
|
|
567
568
|
if (allowed === undefined || allowed === 'outage') return traced;
|
|
568
|
-
const effective = computeEffective(sub.scopes, {
|
|
569
|
+
const effective = computeEffective(declared, sub.scopes, {
|
|
570
|
+
ok: true,
|
|
571
|
+
allowed,
|
|
572
|
+
});
|
|
569
573
|
return {
|
|
570
574
|
...traced,
|
|
571
575
|
effectiveMatch:
|
|
@@ -906,7 +910,14 @@ export class SyncularAdmin {
|
|
|
906
910
|
...base,
|
|
907
911
|
client: (await this.#clientView(partition))(record),
|
|
908
912
|
...(allowed !== undefined ? { allowed } : {}),
|
|
909
|
-
...traceVerdict(
|
|
913
|
+
...traceVerdict(
|
|
914
|
+
this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set(),
|
|
915
|
+
record,
|
|
916
|
+
row,
|
|
917
|
+
lastChange,
|
|
918
|
+
horizonSeq,
|
|
919
|
+
allowed,
|
|
920
|
+
),
|
|
910
921
|
};
|
|
911
922
|
}
|
|
912
923
|
|
|
@@ -929,6 +940,8 @@ export class SyncularAdmin {
|
|
|
929
940
|
const lastChange = row.history[0];
|
|
930
941
|
const horizonSeq = await this.#storage.getHorizonSeq(partition);
|
|
931
942
|
const view = await this.#clientView(partition);
|
|
943
|
+
const declared =
|
|
944
|
+
this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set();
|
|
932
945
|
const allowedByActor = new Map<string, ScopeMap | 'outage' | undefined>();
|
|
933
946
|
const clients: AdminRowDeliveryClient[] = [];
|
|
934
947
|
for (const record of await list(partition)) {
|
|
@@ -941,6 +954,7 @@ export class SyncularAdmin {
|
|
|
941
954
|
clients.push({
|
|
942
955
|
client: view(record),
|
|
943
956
|
verdict: traceVerdict(
|
|
957
|
+
declared,
|
|
944
958
|
record,
|
|
945
959
|
row,
|
|
946
960
|
lastChange,
|
package/src/blob-handlers.ts
CHANGED
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* `blobId` is never a bearer capability. Framework-free; the hono/runtime
|
|
7
7
|
* adapters map these onto `PUT/GET <mount>/blobs/{blobId}`.
|
|
8
8
|
*/
|
|
9
|
-
import {
|
|
9
|
+
import { blobIdFor, isBlobId } from './blob-store';
|
|
10
|
+
import { decodeRequestBody } from './content-encoding';
|
|
10
11
|
import type { SyncRequestContext } from './context';
|
|
11
12
|
import {
|
|
12
13
|
clockOf,
|
|
@@ -17,36 +18,29 @@ import { SyncError, syncError } from './errors';
|
|
|
17
18
|
import { emitEvent } from './events';
|
|
18
19
|
import { compileSchema } from './schema';
|
|
19
20
|
import { authorizeWrite, type ResolvedScopes } from './scopes';
|
|
21
|
+
import { bytesStream } from './segment-store';
|
|
20
22
|
import { issueBlobUploadUrl, issueBlobUrl } from './signed-url';
|
|
21
23
|
|
|
22
24
|
const DEFAULT_MAX_BLOB_BYTES = 64 * 1024 * 1024;
|
|
23
25
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
export
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
* Present iff the host configured `blobSignedUrls`; when present, `bytes` is
|
|
43
|
-
* absent and the client MUST fetch the URL directly (no host auth), verify
|
|
44
|
-
* the content address, and on failure re-request this endpoint — never fall
|
|
45
|
-
* through (§5.9.5 recovery rule).
|
|
46
|
-
*/
|
|
47
|
-
readonly url?: string;
|
|
48
|
-
readonly urlExpiresAtMs?: number;
|
|
49
|
-
}
|
|
26
|
+
/**
|
|
27
|
+
* A §5.9.5 download: the stored bytes as a stream with their length (inline
|
|
28
|
+
* delivery), or, when the host configured `blobSignedUrls`, a provider-signed
|
|
29
|
+
* GET URL issued after the row-derived authorization check. The client
|
|
30
|
+
* fetches a URL directly (no host auth), verifies the content address, and on
|
|
31
|
+
* failure re-requests this endpoint (§5.9.5 recovery rule).
|
|
32
|
+
*/
|
|
33
|
+
export type BlobDownloadResult =
|
|
34
|
+
| {
|
|
35
|
+
readonly headers: Record<string, string>;
|
|
36
|
+
readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
37
|
+
readonly byteLength: number;
|
|
38
|
+
}
|
|
39
|
+
| {
|
|
40
|
+
readonly headers: Record<string, string>;
|
|
41
|
+
readonly url: string;
|
|
42
|
+
readonly urlExpiresAtMs: number;
|
|
43
|
+
};
|
|
50
44
|
|
|
51
45
|
/** `POST <mount>/blobs/{blobId}/upload-grant` request body (§5.9.3). */
|
|
52
46
|
export interface BlobUploadGrantRequest {
|
|
@@ -71,43 +65,67 @@ export interface BlobUploadGrantResult {
|
|
|
71
65
|
readonly present?: boolean;
|
|
72
66
|
}
|
|
73
67
|
|
|
74
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
|
|
70
|
+
* size cap applies to the declared `Content-Length` before any byte is read
|
|
71
|
+
* and again to the counted decoded bytes while reading, so an oversized body
|
|
72
|
+
* is never buffered whole.
|
|
73
|
+
*/
|
|
75
74
|
export async function handleBlobUpload(
|
|
76
75
|
ctx: SyncRequestContext,
|
|
77
|
-
|
|
76
|
+
blobId: string,
|
|
77
|
+
request: Request,
|
|
78
78
|
): Promise<void> {
|
|
79
79
|
await touchAuthenticatedPartition(ctx);
|
|
80
80
|
const store = ctx.blobs;
|
|
81
81
|
if (store === undefined) {
|
|
82
82
|
throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
|
|
83
83
|
}
|
|
84
|
-
if (!isBlobId(
|
|
84
|
+
if (!isBlobId(blobId)) {
|
|
85
85
|
throw syncError(
|
|
86
86
|
'blob.hash_mismatch',
|
|
87
87
|
'blobId path is not "sha256:<64 hex>" (§5.9.1)',
|
|
88
88
|
);
|
|
89
89
|
}
|
|
90
90
|
const maxBytes = ctx.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
const tooLarge = syncError(
|
|
92
|
+
'blob.too_large',
|
|
93
|
+
'blob exceeds the server size cap (§5.9.3)',
|
|
94
|
+
JSON.stringify({ max: maxBytes }),
|
|
95
|
+
);
|
|
96
|
+
if (Number(request.headers.get('content-length')) > maxBytes) throw tooLarge;
|
|
97
|
+
let bytes: Uint8Array | undefined;
|
|
98
|
+
try {
|
|
99
|
+
bytes = await decodeRequestBody(request, maxBytes);
|
|
100
|
+
} catch (error) {
|
|
101
|
+
throw error instanceof SyncError && error.code === 'sync.limit_exceeded'
|
|
102
|
+
? tooLarge
|
|
103
|
+
: error;
|
|
96
104
|
}
|
|
97
|
-
|
|
98
|
-
|
|
105
|
+
if (bytes === undefined) {
|
|
106
|
+
throw syncError('sync.invalid_request', 'unsupported content encoding');
|
|
107
|
+
}
|
|
108
|
+
if ((await blobIdFor(bytes)) !== blobId) {
|
|
99
109
|
throw syncError(
|
|
100
110
|
'blob.hash_mismatch',
|
|
101
111
|
'uploaded bytes do not hash to the blobId path (§5.9.3)',
|
|
102
112
|
);
|
|
103
113
|
}
|
|
114
|
+
const contentType = request.headers
|
|
115
|
+
.get('content-type')
|
|
116
|
+
?.split(';')[0]
|
|
117
|
+
?.trim();
|
|
104
118
|
const clock = clockOf(ctx);
|
|
105
119
|
await store.put(
|
|
106
120
|
ctx.partition,
|
|
107
|
-
|
|
108
|
-
|
|
121
|
+
blobId,
|
|
122
|
+
bytes,
|
|
109
123
|
clock(),
|
|
110
|
-
|
|
124
|
+
contentType === undefined ||
|
|
125
|
+
contentType === '' ||
|
|
126
|
+
contentType === 'application/octet-stream'
|
|
127
|
+
? undefined
|
|
128
|
+
: contentType,
|
|
111
129
|
);
|
|
112
130
|
if (ctx.events !== undefined) {
|
|
113
131
|
emitEvent(ctx.events, {
|
|
@@ -115,8 +133,8 @@ export async function handleBlobUpload(
|
|
|
115
133
|
atMs: clock(),
|
|
116
134
|
partition: ctx.partition,
|
|
117
135
|
actorId: ctx.actorId,
|
|
118
|
-
blobId
|
|
119
|
-
bytes:
|
|
136
|
+
blobId,
|
|
137
|
+
bytes: bytes.length,
|
|
120
138
|
});
|
|
121
139
|
}
|
|
122
140
|
}
|
|
@@ -146,7 +164,7 @@ export async function handleBlobDownload(
|
|
|
146
164
|
outcome: 'ok',
|
|
147
165
|
// Presigned delivery carries no inline bytes (§5.9.5 always-issue); the
|
|
148
166
|
// byte count is unknown to the sync server, which exited the egress path.
|
|
149
|
-
bytes: result.
|
|
167
|
+
bytes: 'byteLength' in result ? result.byteLength : 0,
|
|
150
168
|
durationMs: clock() - startedAtMs,
|
|
151
169
|
});
|
|
152
170
|
return result;
|
|
@@ -176,20 +194,10 @@ async function downloadBlob(
|
|
|
176
194
|
if (!isBlobId(blobId)) {
|
|
177
195
|
throw syncError('blob.not_found', 'malformed blobId (§5.9.1)');
|
|
178
196
|
}
|
|
179
|
-
//
|
|
180
|
-
//
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
let entry: { record: BlobRecord; bytes: Uint8Array } | undefined;
|
|
184
|
-
if (presignConfigured) {
|
|
185
|
-
if (!(await store.has(ctx.partition, blobId))) {
|
|
186
|
-
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
187
|
-
}
|
|
188
|
-
} else {
|
|
189
|
-
entry = await store.get(ctx.partition, blobId);
|
|
190
|
-
if (entry === undefined) {
|
|
191
|
-
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
192
|
-
}
|
|
197
|
+
// Existence first (a HEAD), authorization second, bytes last: no blob byte
|
|
198
|
+
// is read for an actor the reference index does not authorize (§5.9.5).
|
|
199
|
+
if (!(await store.has(ctx.partition, blobId))) {
|
|
200
|
+
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
193
201
|
}
|
|
194
202
|
|
|
195
203
|
// §5.9.5 authorization: resolve once, then test referencing rows.
|
|
@@ -264,25 +272,37 @@ async function downloadBlob(
|
|
|
264
272
|
headers: {
|
|
265
273
|
ETag: `"${blobId}"`,
|
|
266
274
|
'Cache-Control': 'private, max-age=0',
|
|
267
|
-
Vary: 'Authorization
|
|
275
|
+
Vary: 'Authorization',
|
|
268
276
|
},
|
|
269
277
|
url: issued.url,
|
|
270
278
|
urlExpiresAtMs: issued.urlExpiresAtMs,
|
|
271
279
|
};
|
|
272
280
|
}
|
|
273
281
|
|
|
274
|
-
// Inline delivery
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
282
|
+
// Inline delivery streams the stored bytes verbatim (identity encoding,
|
|
283
|
+
// §5.9.5). A store without `open` serves its `get` bytes as one chunk.
|
|
284
|
+
const opened =
|
|
285
|
+
store.open !== undefined
|
|
286
|
+
? await store.open(ctx.partition, blobId)
|
|
287
|
+
: await store.get(ctx.partition, blobId).then((entry) =>
|
|
288
|
+
entry === undefined
|
|
289
|
+
? undefined
|
|
290
|
+
: {
|
|
291
|
+
record: entry.record,
|
|
292
|
+
body: bytesStream(entry.bytes),
|
|
293
|
+
byteLength: entry.bytes.length,
|
|
294
|
+
},
|
|
295
|
+
);
|
|
296
|
+
if (opened === undefined) {
|
|
297
|
+
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
279
298
|
}
|
|
280
299
|
// §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
|
|
281
300
|
// pointed at the route renders them. A JSON type stays octet-stream: a
|
|
282
301
|
// JSON body is reserved for the signed-URL envelope.
|
|
283
|
-
const mediaType =
|
|
302
|
+
const mediaType = opened.record.mediaType;
|
|
284
303
|
return {
|
|
285
|
-
|
|
304
|
+
body: opened.body,
|
|
305
|
+
byteLength: opened.byteLength,
|
|
286
306
|
headers: {
|
|
287
307
|
'Content-Type':
|
|
288
308
|
mediaType === undefined ||
|
|
@@ -293,7 +313,7 @@ async function downloadBlob(
|
|
|
293
313
|
'Content-Security-Policy': "sandbox; default-src 'none'",
|
|
294
314
|
ETag: `"${blobId}"`,
|
|
295
315
|
'Cache-Control': 'private, max-age=0',
|
|
296
|
-
Vary: 'Authorization
|
|
316
|
+
Vary: 'Authorization',
|
|
297
317
|
},
|
|
298
318
|
};
|
|
299
319
|
}
|
|
@@ -303,11 +323,9 @@ async function downloadBlob(
|
|
|
303
323
|
* auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
|
|
304
324
|
* so any authenticated actor may obtain a grant within the size cap (§5.9.3).
|
|
305
325
|
*
|
|
306
|
-
* The size cap is enforced HERE, up front, against the declared `byteLength
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
* and the address is verified at reference time (§5.9.6 push existence) and on
|
|
310
|
-
* every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
|
|
326
|
+
* The size cap is enforced HERE, up front, against the declared `byteLength`;
|
|
327
|
+
* the presigned PUT binds that length and the content address, so the object
|
|
328
|
+
* store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
|
|
311
329
|
* `present` marker if the blob already exists (idempotent §5.9.3), or an empty
|
|
312
330
|
* result when no presigned-upload store is configured (client streams direct).
|
|
313
331
|
*/
|
|
@@ -335,7 +353,8 @@ export async function handleBlobUploadGrant(
|
|
|
335
353
|
// §5.9.3: cap enforced at grant issuance, before any URL is minted.
|
|
336
354
|
throw syncError(
|
|
337
355
|
'blob.too_large',
|
|
338
|
-
|
|
356
|
+
'blob byteLength exceeds the server size cap (§5.9.3)',
|
|
357
|
+
JSON.stringify({ max: maxBytes }),
|
|
339
358
|
);
|
|
340
359
|
}
|
|
341
360
|
// §5.9.3 no presigned-upload store: report no grant; the client streams
|
package/src/blob-store.ts
CHANGED
|
Binary file
|
package/src/bucket-blob-store.ts
CHANGED
|
@@ -29,10 +29,17 @@
|
|
|
29
29
|
*
|
|
30
30
|
* The orphan sweep is the ONLY operation that LISTs: `sweepOrphans` pages a
|
|
31
31
|
* prefix listing over the `blob/` prefix (an admin/GC path, off the hot path)
|
|
32
|
-
* to find candidates, reads each object's
|
|
33
|
-
* the
|
|
32
|
+
* to find candidates, reads each object's upload time from its metadata (or
|
|
33
|
+
* the backend's last-modified time for an object a presigned PUT wrote), and
|
|
34
|
+
* DELETEs the old ones the reference index does not name at that moment.
|
|
34
35
|
*/
|
|
35
|
-
import type {
|
|
36
|
+
import type {
|
|
37
|
+
BlobRecord,
|
|
38
|
+
BlobStore,
|
|
39
|
+
BlobStoreStats,
|
|
40
|
+
BlobStream,
|
|
41
|
+
} from './blob-store';
|
|
42
|
+
import type { BucketObject } from './object-bucket';
|
|
36
43
|
import type { ObjectBucket } from './object-bucket';
|
|
37
44
|
|
|
38
45
|
export interface BucketBlobStoreOptions {
|
|
@@ -215,41 +222,27 @@ export class BucketBlobStore implements BlobStore {
|
|
|
215
222
|
};
|
|
216
223
|
const objectKey = this.objectKeyFor(partition, blobId);
|
|
217
224
|
// Idempotent re-put (same content-address ⇒ same key): count each distinct
|
|
218
|
-
// blob once
|
|
219
|
-
//
|
|
225
|
+
// blob once. A re-upload rewrites the object so its metadata carries the
|
|
226
|
+
// re-upload time (the sweep's grace clock, §5.9.2) and keeps a recorded
|
|
227
|
+
// media type.
|
|
220
228
|
const preexisting = await this.#bucket.head(objectKey);
|
|
221
229
|
const storedMeta = preexisting?.metadata[BLOB_META_KEY];
|
|
222
230
|
const existing =
|
|
223
231
|
storedMeta === undefined
|
|
224
232
|
? undefined
|
|
225
233
|
: parseMetaJson(base64urlToUtf8(storedMeta), blobId, this.#label);
|
|
226
|
-
|
|
227
|
-
existing !== undefined &&
|
|
228
|
-
(existing.mediaType !== undefined || mediaType === undefined)
|
|
229
|
-
) {
|
|
230
|
-
return {
|
|
231
|
-
blobId,
|
|
232
|
-
partition,
|
|
233
|
-
byteLength: existing.byteLength,
|
|
234
|
-
...(existing.mediaType !== undefined
|
|
235
|
-
? { mediaType: existing.mediaType }
|
|
236
|
-
: {}),
|
|
237
|
-
createdAtMs: existing.createdAtMs,
|
|
238
|
-
};
|
|
239
|
-
}
|
|
240
|
-
// A re-upload that names the media type an earlier upload left out
|
|
241
|
-
// rewrites the object under the same precondition, keeping its age.
|
|
234
|
+
const storedType = existing?.mediaType ?? mediaType;
|
|
242
235
|
const meta: BlobMeta = {
|
|
243
236
|
byteLength: bytes.length,
|
|
244
|
-
...(
|
|
245
|
-
createdAtMs: existing?.createdAtMs ?? nowMs,
|
|
237
|
+
...(storedType !== undefined ? { mediaType: storedType } : {}),
|
|
238
|
+
createdAtMs: Math.max(existing?.createdAtMs ?? nowMs, nowMs),
|
|
246
239
|
};
|
|
247
240
|
// Conditional write: of two concurrent uploads of the same blob exactly
|
|
248
|
-
// one lands
|
|
249
|
-
//
|
|
241
|
+
// one lands per pre-state and the blob counts once. The loser re-reads
|
|
242
|
+
// and writes over the winner.
|
|
250
243
|
const written = await this.#bucket.put(objectKey, {
|
|
251
244
|
body: bytes,
|
|
252
|
-
contentType:
|
|
245
|
+
contentType: storedType ?? 'application/octet-stream',
|
|
253
246
|
metadata: { [BLOB_META_KEY]: utf8ToBase64url(metaToJson(meta)) },
|
|
254
247
|
...(preexisting === undefined
|
|
255
248
|
? { createOnly: true }
|
|
@@ -258,7 +251,11 @@ export class BucketBlobStore implements BlobStore {
|
|
|
258
251
|
if (!written) return this.put(partition, blobId, bytes, nowMs, mediaType);
|
|
259
252
|
if (preexisting === undefined)
|
|
260
253
|
await this.#bumpStats({ count: 1, bytes: bytes.length });
|
|
261
|
-
return {
|
|
254
|
+
return {
|
|
255
|
+
...record,
|
|
256
|
+
...(storedType !== undefined ? { mediaType: storedType } : {}),
|
|
257
|
+
createdAtMs: meta.createdAtMs,
|
|
258
|
+
};
|
|
262
259
|
}
|
|
263
260
|
|
|
264
261
|
async delete(partition: string, blobId: string): Promise<void> {
|
|
@@ -275,38 +272,67 @@ export class BucketBlobStore implements BlobStore {
|
|
|
275
272
|
partition: string,
|
|
276
273
|
blobId: string,
|
|
277
274
|
): Promise<{ record: BlobRecord; bytes: Uint8Array } | undefined> {
|
|
278
|
-
|
|
279
|
-
const object = await this.#bucket.get(this.objectKeyFor(partition, blobId));
|
|
275
|
+
const object = await this.#object(partition, blobId);
|
|
280
276
|
if (object === undefined) return undefined;
|
|
281
277
|
const bytes = new Uint8Array(await object.arrayBuffer());
|
|
278
|
+
return {
|
|
279
|
+
record: this.#record(partition, blobId, object, bytes.length),
|
|
280
|
+
bytes,
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** The record plus the object GET's body, never buffered (see `get`). */
|
|
285
|
+
async open(
|
|
286
|
+
partition: string,
|
|
287
|
+
blobId: string,
|
|
288
|
+
): Promise<BlobStream | undefined> {
|
|
289
|
+
const object = await this.#object(partition, blobId);
|
|
290
|
+
if (object === undefined || object.body === null) return undefined;
|
|
291
|
+
const record = this.#record(partition, blobId, object, object.size ?? 0);
|
|
292
|
+
return { record, body: object.body, byteLength: record.byteLength };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
async #object(
|
|
296
|
+
partition: string,
|
|
297
|
+
blobId: string,
|
|
298
|
+
): Promise<BucketObject | undefined> {
|
|
299
|
+
if (!BLOB_ID_PATTERN.test(blobId)) return undefined;
|
|
300
|
+
return this.#bucket.get(this.objectKeyFor(partition, blobId));
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** An object without Syncular metadata was written by a presigned PUT. */
|
|
304
|
+
#record(
|
|
305
|
+
partition: string,
|
|
306
|
+
blobId: string,
|
|
307
|
+
object: BucketObject,
|
|
308
|
+
byteLength: number,
|
|
309
|
+
): BlobRecord {
|
|
282
310
|
const rawMeta = object.metadata[BLOB_META_KEY];
|
|
283
311
|
const meta =
|
|
284
312
|
rawMeta !== undefined
|
|
285
313
|
? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
|
|
286
|
-
: { byteLength
|
|
314
|
+
: { byteLength, createdAtMs: 0 };
|
|
287
315
|
return {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
createdAtMs: meta.createdAtMs,
|
|
294
|
-
},
|
|
295
|
-
bytes,
|
|
316
|
+
blobId,
|
|
317
|
+
partition,
|
|
318
|
+
byteLength: meta.byteLength,
|
|
319
|
+
...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
|
|
320
|
+
createdAtMs: meta.createdAtMs,
|
|
296
321
|
};
|
|
297
322
|
}
|
|
298
323
|
|
|
299
324
|
/**
|
|
300
325
|
* §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
|
|
301
|
-
* {partition}/` prefix, HEAD each object for its
|
|
302
|
-
*
|
|
303
|
-
*
|
|
304
|
-
*
|
|
326
|
+
* {partition}/` prefix, HEAD each object for its upload time (the metadata
|
|
327
|
+
* `createdAtMs`, else the backend's last-modified time; an object with
|
|
328
|
+
* neither is skipped), and DELETE the ones older than `olderThanMs` that
|
|
329
|
+
* `isReferenced` denies right before the delete. Returns the deleted
|
|
330
|
+
* blobIds. Off the hot path — a host-scheduled GC job.
|
|
305
331
|
*/
|
|
306
332
|
async sweepOrphans(
|
|
307
333
|
partition: string,
|
|
308
334
|
olderThanMs: number,
|
|
309
|
-
|
|
335
|
+
isReferenced: (blobId: string) => Promise<boolean>,
|
|
310
336
|
): Promise<string[]> {
|
|
311
337
|
const listPrefix = `${this.#prefix}blob/${partition}/`;
|
|
312
338
|
const swept: string[] = [];
|
|
@@ -321,18 +347,19 @@ export class BucketBlobStore implements BlobStore {
|
|
|
321
347
|
if (!rest.startsWith('sha256/')) continue;
|
|
322
348
|
const blobId = `sha256:${rest.slice('sha256/'.length)}`;
|
|
323
349
|
if (!BLOB_ID_PATTERN.test(blobId)) continue;
|
|
324
|
-
if (referencedBlobIds.has(blobId)) continue;
|
|
325
350
|
const head = await this.#bucket.head(key);
|
|
326
351
|
if (head === undefined) continue;
|
|
327
352
|
const rawMeta = head.metadata[BLOB_META_KEY];
|
|
328
|
-
const
|
|
353
|
+
const uploadedAtMs =
|
|
329
354
|
rawMeta !== undefined
|
|
330
355
|
? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
|
|
331
|
-
|
|
332
|
-
|
|
356
|
+
.createdAtMs
|
|
357
|
+
: head.lastModifiedMs;
|
|
358
|
+
if (uploadedAtMs === undefined || uploadedAtMs >= olderThanMs) continue;
|
|
359
|
+
if (await isReferenced(blobId)) continue;
|
|
333
360
|
await this.#bucket.delete(key);
|
|
334
361
|
swept.push(blobId);
|
|
335
|
-
deletedBytes +=
|
|
362
|
+
deletedBytes += head.size ?? 0;
|
|
336
363
|
}
|
|
337
364
|
} while (cursor !== undefined);
|
|
338
365
|
if (swept.length > 0) {
|