@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/dist/blob-handlers.js
CHANGED
|
@@ -7,41 +7,67 @@
|
|
|
7
7
|
* adapters map these onto `PUT/GET <mount>/blobs/{blobId}`.
|
|
8
8
|
*/
|
|
9
9
|
import { blobIdFor, isBlobId } from './blob-store.js';
|
|
10
|
+
import { decodeRequestBody } from './content-encoding.js';
|
|
10
11
|
import { clockOf, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
11
12
|
import { SyncError, syncError } from './errors.js';
|
|
12
13
|
import { emitEvent } from './events.js';
|
|
13
14
|
import { compileSchema } from './schema.js';
|
|
14
15
|
import { authorizeWrite } from './scopes.js';
|
|
16
|
+
import { bytesStream } from './segment-store.js';
|
|
15
17
|
import { issueBlobUploadUrl, issueBlobUrl } from './signed-url.js';
|
|
16
18
|
const DEFAULT_MAX_BLOB_BYTES = 64 * 1024 * 1024;
|
|
17
|
-
/**
|
|
18
|
-
|
|
19
|
+
/**
|
|
20
|
+
* `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
|
|
21
|
+
* size cap applies to the declared `Content-Length` before any byte is read
|
|
22
|
+
* and again to the counted decoded bytes while reading, so an oversized body
|
|
23
|
+
* is never buffered whole.
|
|
24
|
+
*/
|
|
25
|
+
export async function handleBlobUpload(ctx, blobId, request) {
|
|
19
26
|
await touchAuthenticatedPartition(ctx);
|
|
20
27
|
const store = ctx.blobs;
|
|
21
28
|
if (store === undefined) {
|
|
22
29
|
throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
|
|
23
30
|
}
|
|
24
|
-
if (!isBlobId(
|
|
31
|
+
if (!isBlobId(blobId)) {
|
|
25
32
|
throw syncError('blob.hash_mismatch', 'blobId path is not "sha256:<64 hex>" (§5.9.1)');
|
|
26
33
|
}
|
|
27
34
|
const maxBytes = ctx.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
|
|
28
|
-
|
|
29
|
-
|
|
35
|
+
const tooLarge = syncError('blob.too_large', 'blob exceeds the server size cap (§5.9.3)', JSON.stringify({ max: maxBytes }));
|
|
36
|
+
if (Number(request.headers.get('content-length')) > maxBytes)
|
|
37
|
+
throw tooLarge;
|
|
38
|
+
let bytes;
|
|
39
|
+
try {
|
|
40
|
+
bytes = await decodeRequestBody(request, maxBytes);
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
throw error instanceof SyncError && error.code === 'sync.limit_exceeded'
|
|
44
|
+
? tooLarge
|
|
45
|
+
: error;
|
|
30
46
|
}
|
|
31
|
-
|
|
32
|
-
|
|
47
|
+
if (bytes === undefined) {
|
|
48
|
+
throw syncError('sync.invalid_request', 'unsupported content encoding');
|
|
49
|
+
}
|
|
50
|
+
if ((await blobIdFor(bytes)) !== blobId) {
|
|
33
51
|
throw syncError('blob.hash_mismatch', 'uploaded bytes do not hash to the blobId path (§5.9.3)');
|
|
34
52
|
}
|
|
53
|
+
const contentType = request.headers
|
|
54
|
+
.get('content-type')
|
|
55
|
+
?.split(';')[0]
|
|
56
|
+
?.trim();
|
|
35
57
|
const clock = clockOf(ctx);
|
|
36
|
-
await store.put(ctx.partition,
|
|
58
|
+
await store.put(ctx.partition, blobId, bytes, clock(), contentType === undefined ||
|
|
59
|
+
contentType === '' ||
|
|
60
|
+
contentType === 'application/octet-stream'
|
|
61
|
+
? undefined
|
|
62
|
+
: contentType);
|
|
37
63
|
if (ctx.events !== undefined) {
|
|
38
64
|
emitEvent(ctx.events, {
|
|
39
65
|
type: 'blob.uploaded',
|
|
40
66
|
atMs: clock(),
|
|
41
67
|
partition: ctx.partition,
|
|
42
68
|
actorId: ctx.actorId,
|
|
43
|
-
blobId
|
|
44
|
-
bytes:
|
|
69
|
+
blobId,
|
|
70
|
+
bytes: bytes.length,
|
|
45
71
|
});
|
|
46
72
|
}
|
|
47
73
|
}
|
|
@@ -68,7 +94,7 @@ export async function handleBlobDownload(ctx, blobId) {
|
|
|
68
94
|
outcome: 'ok',
|
|
69
95
|
// Presigned delivery carries no inline bytes (§5.9.5 always-issue); the
|
|
70
96
|
// byte count is unknown to the sync server, which exited the egress path.
|
|
71
|
-
bytes: result.
|
|
97
|
+
bytes: 'byteLength' in result ? result.byteLength : 0,
|
|
72
98
|
durationMs: clock() - startedAtMs,
|
|
73
99
|
});
|
|
74
100
|
return result;
|
|
@@ -95,21 +121,10 @@ async function downloadBlob(ctx, blobId) {
|
|
|
95
121
|
if (!isBlobId(blobId)) {
|
|
96
122
|
throw syncError('blob.not_found', 'malformed blobId (§5.9.1)');
|
|
97
123
|
}
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
let entry;
|
|
103
|
-
if (presignConfigured) {
|
|
104
|
-
if (!(await store.has(ctx.partition, blobId))) {
|
|
105
|
-
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
else {
|
|
109
|
-
entry = await store.get(ctx.partition, blobId);
|
|
110
|
-
if (entry === undefined) {
|
|
111
|
-
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
112
|
-
}
|
|
124
|
+
// Existence first (a HEAD), authorization second, bytes last: no blob byte
|
|
125
|
+
// is read for an actor the reference index does not authorize (§5.9.5).
|
|
126
|
+
if (!(await store.has(ctx.partition, blobId))) {
|
|
127
|
+
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
113
128
|
}
|
|
114
129
|
// §5.9.5 authorization: resolve once, then test referencing rows.
|
|
115
130
|
const listRows = ctx.storage.listRowsReferencingBlob;
|
|
@@ -177,24 +192,33 @@ async function downloadBlob(ctx, blobId) {
|
|
|
177
192
|
headers: {
|
|
178
193
|
ETag: `"${blobId}"`,
|
|
179
194
|
'Cache-Control': 'private, max-age=0',
|
|
180
|
-
Vary: 'Authorization
|
|
195
|
+
Vary: 'Authorization',
|
|
181
196
|
},
|
|
182
197
|
url: issued.url,
|
|
183
198
|
urlExpiresAtMs: issued.urlExpiresAtMs,
|
|
184
199
|
};
|
|
185
200
|
}
|
|
186
|
-
// Inline delivery
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
201
|
+
// Inline delivery streams the stored bytes verbatim (identity encoding,
|
|
202
|
+
// §5.9.5). A store without `open` serves its `get` bytes as one chunk.
|
|
203
|
+
const opened = store.open !== undefined
|
|
204
|
+
? await store.open(ctx.partition, blobId)
|
|
205
|
+
: await store.get(ctx.partition, blobId).then((entry) => entry === undefined
|
|
206
|
+
? undefined
|
|
207
|
+
: {
|
|
208
|
+
record: entry.record,
|
|
209
|
+
body: bytesStream(entry.bytes),
|
|
210
|
+
byteLength: entry.bytes.length,
|
|
211
|
+
});
|
|
212
|
+
if (opened === undefined) {
|
|
213
|
+
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
191
214
|
}
|
|
192
215
|
// §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
|
|
193
216
|
// pointed at the route renders them. A JSON type stays octet-stream: a
|
|
194
217
|
// JSON body is reserved for the signed-URL envelope.
|
|
195
|
-
const mediaType =
|
|
218
|
+
const mediaType = opened.record.mediaType;
|
|
196
219
|
return {
|
|
197
|
-
|
|
220
|
+
body: opened.body,
|
|
221
|
+
byteLength: opened.byteLength,
|
|
198
222
|
headers: {
|
|
199
223
|
'Content-Type': mediaType === undefined ||
|
|
200
224
|
mediaType.toLowerCase().includes('application/json')
|
|
@@ -204,7 +228,7 @@ async function downloadBlob(ctx, blobId) {
|
|
|
204
228
|
'Content-Security-Policy': "sandbox; default-src 'none'",
|
|
205
229
|
ETag: `"${blobId}"`,
|
|
206
230
|
'Cache-Control': 'private, max-age=0',
|
|
207
|
-
Vary: 'Authorization
|
|
231
|
+
Vary: 'Authorization',
|
|
208
232
|
},
|
|
209
233
|
};
|
|
210
234
|
}
|
|
@@ -213,11 +237,9 @@ async function downloadBlob(ctx, blobId) {
|
|
|
213
237
|
* auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
|
|
214
238
|
* so any authenticated actor may obtain a grant within the size cap (§5.9.3).
|
|
215
239
|
*
|
|
216
|
-
* The size cap is enforced HERE, up front, against the declared `byteLength
|
|
217
|
-
*
|
|
218
|
-
*
|
|
219
|
-
* and the address is verified at reference time (§5.9.6 push existence) and on
|
|
220
|
-
* every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
|
|
240
|
+
* The size cap is enforced HERE, up front, against the declared `byteLength`;
|
|
241
|
+
* the presigned PUT binds that length and the content address, so the object
|
|
242
|
+
* store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
|
|
221
243
|
* `present` marker if the blob already exists (idempotent §5.9.3), or an empty
|
|
222
244
|
* result when no presigned-upload store is configured (client streams direct).
|
|
223
245
|
*/
|
|
@@ -235,7 +257,7 @@ export async function handleBlobUploadGrant(ctx, request) {
|
|
|
235
257
|
request.byteLength < 0 ||
|
|
236
258
|
request.byteLength > maxBytes) {
|
|
237
259
|
// §5.9.3: cap enforced at grant issuance, before any URL is minted.
|
|
238
|
-
throw syncError('blob.too_large',
|
|
260
|
+
throw syncError('blob.too_large', 'blob byteLength exceeds the server size cap (§5.9.3)', JSON.stringify({ max: maxBytes }));
|
|
239
261
|
}
|
|
240
262
|
// §5.9.3 no presigned-upload store: report no grant; the client streams
|
|
241
263
|
// through the direct PUT endpoint (capability, not fallback).
|
package/dist/blob-store.d.ts
CHANGED
|
@@ -17,8 +17,15 @@ export interface BlobRecord {
|
|
|
17
17
|
readonly byteLength: number;
|
|
18
18
|
/** Advisory MIME type persisted from the upload, if any (§5.9.5). */
|
|
19
19
|
readonly mediaType?: string;
|
|
20
|
+
/** Time of the latest upload; the orphan sweep's grace clock (§5.9.2). */
|
|
20
21
|
readonly createdAtMs: number;
|
|
21
22
|
}
|
|
23
|
+
/** `BlobStore.open`: a stored blob with its bytes as a stream. */
|
|
24
|
+
export interface BlobStream {
|
|
25
|
+
readonly record: BlobRecord;
|
|
26
|
+
readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
27
|
+
readonly byteLength: number;
|
|
28
|
+
}
|
|
22
29
|
/**
|
|
23
30
|
* Coarse blob-store counters for the admin/console read surface (work item
|
|
24
31
|
* §2.5), partition-scoped. Optional. `count`/`bytes` are total stored
|
|
@@ -36,10 +43,12 @@ export interface BlobStoreStats {
|
|
|
36
43
|
}
|
|
37
44
|
export interface BlobStore {
|
|
38
45
|
/**
|
|
39
|
-
* Store blob bytes under `blobId` (idempotent
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
46
|
+
* Store blob bytes under `blobId` (idempotent: same id ⇒ success, §5.9.3).
|
|
47
|
+
* A re-upload moves `createdAtMs` forward to `nowMs`, so a blob that a
|
|
48
|
+
* second client is about to reference gets a fresh sweep grace period. A
|
|
49
|
+
* re-upload that declares a media type records it on a blob stored without
|
|
50
|
+
* one; a recorded media type never changes. The caller has already
|
|
51
|
+
* verified the content address matches.
|
|
43
52
|
*/
|
|
44
53
|
put(partition: string, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): Promise<BlobRecord>;
|
|
45
54
|
/** Presence check for the push existence rule (§5.9.6) and download. */
|
|
@@ -49,12 +58,20 @@ export interface BlobStore {
|
|
|
49
58
|
bytes: Uint8Array;
|
|
50
59
|
} | undefined>;
|
|
51
60
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
|
|
61
|
+
* Optional streaming read: the record plus the bytes as a stream, so the
|
|
62
|
+
* inline download route relays a blob without buffering it (the
|
|
63
|
+
* `SegmentStore.open` shape). Stores without it are served through `get`.
|
|
64
|
+
*/
|
|
65
|
+
open?(partition: string, blobId: string): Promise<BlobStream | undefined>;
|
|
66
|
+
/**
|
|
67
|
+
* §5.9.2 orphan sweep: delete blobs last uploaded before `olderThanMs`
|
|
68
|
+
* that no live row references. `isReferenced` asks the storage reference
|
|
69
|
+
* index about one blob; the store calls it for each old candidate right
|
|
70
|
+
* before that candidate's delete, so a reference that lands during the
|
|
71
|
+
* sweep keeps its blob. The store never scans rows itself. Returns the
|
|
72
|
+
* deleted blobIds.
|
|
56
73
|
*/
|
|
57
|
-
sweepOrphans(partition: string, olderThanMs: number,
|
|
74
|
+
sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
|
|
58
75
|
/**
|
|
59
76
|
* §4.6.1 erasure: delete one blob whose last reference is gone. Deleting a
|
|
60
77
|
* missing blob is a no-op.
|
|
@@ -74,7 +91,7 @@ export declare class MemoryBlobStore implements BlobStore {
|
|
|
74
91
|
record: BlobRecord;
|
|
75
92
|
bytes: Uint8Array;
|
|
76
93
|
} | undefined>;
|
|
77
|
-
sweepOrphans(partition: string, olderThanMs: number,
|
|
94
|
+
sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
|
|
78
95
|
delete(partition: string, blobId: string): Promise<void>;
|
|
79
96
|
stats(partition: string): Promise<BlobStoreStats>;
|
|
80
97
|
}
|
|
@@ -95,25 +112,26 @@ export interface SweepOrphanBlobsOptions {
|
|
|
95
112
|
export interface SweepOrphanBlobsResult {
|
|
96
113
|
readonly partition: string;
|
|
97
114
|
readonly swept: string[];
|
|
115
|
+
/** Candidates past the grace period that a live row still references. */
|
|
98
116
|
readonly referencedCount: number;
|
|
99
117
|
}
|
|
100
118
|
/**
|
|
101
|
-
* §5.9.2 orphan sweep — the ready-made GC helper.
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
119
|
+
* §5.9.2 orphan sweep — the ready-made GC helper. Hands the grace cutoff and
|
|
120
|
+
* a reference check to `blobStore.sweepOrphans`, which deletes only blobs
|
|
121
|
+
* older than the grace period that no row references at the moment of the
|
|
122
|
+
* delete. The check consults the keep-set read when the sweep started and,
|
|
123
|
+
* for a blob outside it, reads the reference index again right before that
|
|
124
|
+
* blob's delete, so a push that lands during the sweep keeps its blob.
|
|
125
|
+
* Returns the swept ids and emits one `blob.swept` ops event.
|
|
106
126
|
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
* against.
|
|
127
|
+
* A client uploads bytes before pushing the row that references them
|
|
128
|
+
* (§5.9.2), so a just-uploaded blob is unreferenced until its push arrives.
|
|
129
|
+
* The grace period protects that window: keep it far above any push latency
|
|
130
|
+
* (the 24 h default). The store's upload time (`createdAtMs` metadata, or the
|
|
131
|
+
* backend's last-modified time for an object uploaded through a presigned
|
|
132
|
+
* PUT) is what the cutoff compares against; a re-upload refreshes it.
|
|
114
133
|
*
|
|
115
134
|
* Requires `storage.listReferencedBlobIds` (the §5.9.4 reference index). A
|
|
116
|
-
* storage backend without it cannot be swept safely
|
|
117
|
-
* than sweeping against an empty keep-set (which would delete everything).
|
|
135
|
+
* storage backend without it cannot be swept safely, so this throws.
|
|
118
136
|
*/
|
|
119
137
|
export declare function sweepOrphanBlobs(storage: ServerStorage, blobStore: BlobStore, partition: string, options?: SweepOrphanBlobsOptions): Promise<SweepOrphanBlobsResult>;
|
package/dist/blob-store.js
CHANGED
|
Binary file
|
|
@@ -29,10 +29,11 @@
|
|
|
29
29
|
*
|
|
30
30
|
* The orphan sweep is the ONLY operation that LISTs: `sweepOrphans` pages a
|
|
31
31
|
* prefix listing over the `blob/` prefix (an admin/GC path, off the hot path)
|
|
32
|
-
* to find candidates, reads each object's
|
|
33
|
-
* the
|
|
32
|
+
* to find candidates, reads each object's upload time from its metadata (or
|
|
33
|
+
* the backend's last-modified time for an object a presigned PUT wrote), and
|
|
34
|
+
* DELETEs the old ones the reference index does not name at that moment.
|
|
34
35
|
*/
|
|
35
|
-
import type { BlobRecord, BlobStore, BlobStoreStats } from './blob-store.js';
|
|
36
|
+
import type { BlobRecord, BlobStore, BlobStoreStats, BlobStream } from './blob-store.js';
|
|
36
37
|
import type { ObjectBucket } from './object-bucket.js';
|
|
37
38
|
export interface BucketBlobStoreOptions {
|
|
38
39
|
/** Key namespace inside the bucket, e.g. `syncular/`. Default: none. */
|
|
@@ -57,14 +58,17 @@ export declare class BucketBlobStore implements BlobStore {
|
|
|
57
58
|
record: BlobRecord;
|
|
58
59
|
bytes: Uint8Array;
|
|
59
60
|
} | undefined>;
|
|
61
|
+
/** The record plus the object GET's body, never buffered (see `get`). */
|
|
62
|
+
open(partition: string, blobId: string): Promise<BlobStream | undefined>;
|
|
60
63
|
/**
|
|
61
64
|
* §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
|
|
62
|
-
* {partition}/` prefix, HEAD each object for its
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
65
|
+
* {partition}/` prefix, HEAD each object for its upload time (the metadata
|
|
66
|
+
* `createdAtMs`, else the backend's last-modified time; an object with
|
|
67
|
+
* neither is skipped), and DELETE the ones older than `olderThanMs` that
|
|
68
|
+
* `isReferenced` denies right before the delete. Returns the deleted
|
|
69
|
+
* blobIds. Off the hot path — a host-scheduled GC job.
|
|
66
70
|
*/
|
|
67
|
-
sweepOrphans(partition: string, olderThanMs: number,
|
|
71
|
+
sweepOrphans(partition: string, olderThanMs: number, isReferenced: (blobId: string) => Promise<boolean>): Promise<string[]>;
|
|
68
72
|
/**
|
|
69
73
|
* Store-wide counters from the pointer-object accumulator (never a LIST on
|
|
70
74
|
* the read path). `approximate: true` for the same reasons the segment
|
|
@@ -138,38 +138,26 @@ export class BucketBlobStore {
|
|
|
138
138
|
};
|
|
139
139
|
const objectKey = this.objectKeyFor(partition, blobId);
|
|
140
140
|
// Idempotent re-put (same content-address ⇒ same key): count each distinct
|
|
141
|
-
// blob once
|
|
142
|
-
//
|
|
141
|
+
// blob once. A re-upload rewrites the object so its metadata carries the
|
|
142
|
+
// re-upload time (the sweep's grace clock, §5.9.2) and keeps a recorded
|
|
143
|
+
// media type.
|
|
143
144
|
const preexisting = await this.#bucket.head(objectKey);
|
|
144
145
|
const storedMeta = preexisting?.metadata[BLOB_META_KEY];
|
|
145
146
|
const existing = storedMeta === undefined
|
|
146
147
|
? undefined
|
|
147
148
|
: parseMetaJson(base64urlToUtf8(storedMeta), blobId, this.#label);
|
|
148
|
-
|
|
149
|
-
(existing.mediaType !== undefined || mediaType === undefined)) {
|
|
150
|
-
return {
|
|
151
|
-
blobId,
|
|
152
|
-
partition,
|
|
153
|
-
byteLength: existing.byteLength,
|
|
154
|
-
...(existing.mediaType !== undefined
|
|
155
|
-
? { mediaType: existing.mediaType }
|
|
156
|
-
: {}),
|
|
157
|
-
createdAtMs: existing.createdAtMs,
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
// A re-upload that names the media type an earlier upload left out
|
|
161
|
-
// rewrites the object under the same precondition, keeping its age.
|
|
149
|
+
const storedType = existing?.mediaType ?? mediaType;
|
|
162
150
|
const meta = {
|
|
163
151
|
byteLength: bytes.length,
|
|
164
|
-
...(
|
|
165
|
-
createdAtMs: existing?.createdAtMs ?? nowMs,
|
|
152
|
+
...(storedType !== undefined ? { mediaType: storedType } : {}),
|
|
153
|
+
createdAtMs: Math.max(existing?.createdAtMs ?? nowMs, nowMs),
|
|
166
154
|
};
|
|
167
155
|
// Conditional write: of two concurrent uploads of the same blob exactly
|
|
168
|
-
// one lands
|
|
169
|
-
//
|
|
156
|
+
// one lands per pre-state and the blob counts once. The loser re-reads
|
|
157
|
+
// and writes over the winner.
|
|
170
158
|
const written = await this.#bucket.put(objectKey, {
|
|
171
159
|
body: bytes,
|
|
172
|
-
contentType:
|
|
160
|
+
contentType: storedType ?? 'application/octet-stream',
|
|
173
161
|
metadata: { [BLOB_META_KEY]: utf8ToBase64url(metaToJson(meta)) },
|
|
174
162
|
...(preexisting === undefined
|
|
175
163
|
? { createOnly: true }
|
|
@@ -179,7 +167,11 @@ export class BucketBlobStore {
|
|
|
179
167
|
return this.put(partition, blobId, bytes, nowMs, mediaType);
|
|
180
168
|
if (preexisting === undefined)
|
|
181
169
|
await this.#bumpStats({ count: 1, bytes: bytes.length });
|
|
182
|
-
return {
|
|
170
|
+
return {
|
|
171
|
+
...record,
|
|
172
|
+
...(storedType !== undefined ? { mediaType: storedType } : {}),
|
|
173
|
+
createdAtMs: meta.createdAtMs,
|
|
174
|
+
};
|
|
183
175
|
}
|
|
184
176
|
async delete(partition, blobId) {
|
|
185
177
|
await this.#bucket.delete(this.objectKeyFor(partition, blobId));
|
|
@@ -191,35 +183,51 @@ export class BucketBlobStore {
|
|
|
191
183
|
return head !== undefined;
|
|
192
184
|
}
|
|
193
185
|
async get(partition, blobId) {
|
|
194
|
-
|
|
195
|
-
return undefined;
|
|
196
|
-
const object = await this.#bucket.get(this.objectKeyFor(partition, blobId));
|
|
186
|
+
const object = await this.#object(partition, blobId);
|
|
197
187
|
if (object === undefined)
|
|
198
188
|
return undefined;
|
|
199
189
|
const bytes = new Uint8Array(await object.arrayBuffer());
|
|
190
|
+
return {
|
|
191
|
+
record: this.#record(partition, blobId, object, bytes.length),
|
|
192
|
+
bytes,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
/** The record plus the object GET's body, never buffered (see `get`). */
|
|
196
|
+
async open(partition, blobId) {
|
|
197
|
+
const object = await this.#object(partition, blobId);
|
|
198
|
+
if (object === undefined || object.body === null)
|
|
199
|
+
return undefined;
|
|
200
|
+
const record = this.#record(partition, blobId, object, object.size ?? 0);
|
|
201
|
+
return { record, body: object.body, byteLength: record.byteLength };
|
|
202
|
+
}
|
|
203
|
+
async #object(partition, blobId) {
|
|
204
|
+
if (!BLOB_ID_PATTERN.test(blobId))
|
|
205
|
+
return undefined;
|
|
206
|
+
return this.#bucket.get(this.objectKeyFor(partition, blobId));
|
|
207
|
+
}
|
|
208
|
+
/** An object without Syncular metadata was written by a presigned PUT. */
|
|
209
|
+
#record(partition, blobId, object, byteLength) {
|
|
200
210
|
const rawMeta = object.metadata[BLOB_META_KEY];
|
|
201
211
|
const meta = rawMeta !== undefined
|
|
202
212
|
? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
|
|
203
|
-
: { byteLength
|
|
213
|
+
: { byteLength, createdAtMs: 0 };
|
|
204
214
|
return {
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
createdAtMs: meta.createdAtMs,
|
|
211
|
-
},
|
|
212
|
-
bytes,
|
|
215
|
+
blobId,
|
|
216
|
+
partition,
|
|
217
|
+
byteLength: meta.byteLength,
|
|
218
|
+
...(meta.mediaType !== undefined ? { mediaType: meta.mediaType } : {}),
|
|
219
|
+
createdAtMs: meta.createdAtMs,
|
|
213
220
|
};
|
|
214
221
|
}
|
|
215
222
|
/**
|
|
216
223
|
* §5.9.2 orphan sweep — the ONLY LISTing operation. Page the `{prefix}blob/
|
|
217
|
-
* {partition}/` prefix, HEAD each object for its
|
|
218
|
-
*
|
|
219
|
-
*
|
|
220
|
-
*
|
|
224
|
+
* {partition}/` prefix, HEAD each object for its upload time (the metadata
|
|
225
|
+
* `createdAtMs`, else the backend's last-modified time; an object with
|
|
226
|
+
* neither is skipped), and DELETE the ones older than `olderThanMs` that
|
|
227
|
+
* `isReferenced` denies right before the delete. Returns the deleted
|
|
228
|
+
* blobIds. Off the hot path — a host-scheduled GC job.
|
|
221
229
|
*/
|
|
222
|
-
async sweepOrphans(partition, olderThanMs,
|
|
230
|
+
async sweepOrphans(partition, olderThanMs, isReferenced) {
|
|
223
231
|
const listPrefix = `${this.#prefix}blob/${partition}/`;
|
|
224
232
|
const swept = [];
|
|
225
233
|
let deletedBytes = 0;
|
|
@@ -235,20 +243,21 @@ export class BucketBlobStore {
|
|
|
235
243
|
const blobId = `sha256:${rest.slice('sha256/'.length)}`;
|
|
236
244
|
if (!BLOB_ID_PATTERN.test(blobId))
|
|
237
245
|
continue;
|
|
238
|
-
if (referencedBlobIds.has(blobId))
|
|
239
|
-
continue;
|
|
240
246
|
const head = await this.#bucket.head(key);
|
|
241
247
|
if (head === undefined)
|
|
242
248
|
continue;
|
|
243
249
|
const rawMeta = head.metadata[BLOB_META_KEY];
|
|
244
|
-
const
|
|
250
|
+
const uploadedAtMs = rawMeta !== undefined
|
|
245
251
|
? parseMetaJson(base64urlToUtf8(rawMeta), blobId, this.#label)
|
|
246
|
-
|
|
247
|
-
|
|
252
|
+
.createdAtMs
|
|
253
|
+
: head.lastModifiedMs;
|
|
254
|
+
if (uploadedAtMs === undefined || uploadedAtMs >= olderThanMs)
|
|
255
|
+
continue;
|
|
256
|
+
if (await isReferenced(blobId))
|
|
248
257
|
continue;
|
|
249
258
|
await this.#bucket.delete(key);
|
|
250
259
|
swept.push(blobId);
|
|
251
|
-
deletedBytes +=
|
|
260
|
+
deletedBytes += head.size ?? 0;
|
|
252
261
|
}
|
|
253
262
|
} while (cursor !== undefined);
|
|
254
263
|
if (swept.length > 0) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { mergeSegmentRecord, publicationRecord, publicationsForRecord,
|
|
1
|
+
import { mergeSegmentRecord, publicationRecord, publicationsForRecord, segmentIdFor, selectSegmentPublication, } from './segment-store.js';
|
|
2
2
|
import { sha256Hex } from './sigv4.js';
|
|
3
3
|
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
4
4
|
const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
@@ -311,25 +311,27 @@ export class BucketSegmentStore {
|
|
|
311
311
|
throw new Error(`${this.#label}: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`);
|
|
312
312
|
}
|
|
313
313
|
/**
|
|
314
|
-
* Write the immutable bytes object
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
*
|
|
314
|
+
* Write the immutable bytes object unless it is stored already. A HEAD
|
|
315
|
+
* comes first, so republishing a segment (every bootstrap pull of an
|
|
316
|
+
* unchanged window) moves no bytes. The write is create-only, so two first
|
|
317
|
+
* publishers cannot overwrite each other. A stored object under the same
|
|
318
|
+
* content address with a different size is the hash collision §5.1
|
|
319
|
+
* requires to fail loudly; equal size under one SHA-256 address is the same
|
|
320
|
+
* content.
|
|
318
321
|
*/
|
|
319
322
|
async #putBytes(segmentId, bytes) {
|
|
320
323
|
const key = this.objectKeyFor(segmentId);
|
|
321
|
-
const
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
const current = await this.#bucket.get(key);
|
|
329
|
-
if (current === undefined)
|
|
324
|
+
const head = await this.#bucket.head(key);
|
|
325
|
+
if (head === undefined &&
|
|
326
|
+
(await this.#bucket.put(key, {
|
|
327
|
+
body: bytes,
|
|
328
|
+
contentType: 'application/octet-stream',
|
|
329
|
+
createOnly: true,
|
|
330
|
+
}))) {
|
|
330
331
|
return;
|
|
331
|
-
|
|
332
|
-
|
|
332
|
+
}
|
|
333
|
+
const stored = head ?? (await this.#bucket.head(key));
|
|
334
|
+
if (stored !== undefined && stored.size !== bytes.length) {
|
|
333
335
|
throw new Error(`${this.#label}: content-address collision at ${segmentId} (§5.1)`);
|
|
334
336
|
}
|
|
335
337
|
}
|
|
@@ -40,6 +40,9 @@ export declare function encodeStream(body: ReadableStream<Uint8Array<ArrayBuffer
|
|
|
40
40
|
* `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
|
|
41
41
|
* servers MUST accept it). Returns `undefined` for any other declared
|
|
42
42
|
* encoding — the adapter answers 415 rather than parse bytes it cannot read.
|
|
43
|
-
* A gzip body that does not decompress is a `sync.invalid_request`.
|
|
43
|
+
* A gzip body that does not decompress is a `sync.invalid_request`. One read
|
|
44
|
+
* loop counts the decoded bytes of identity and gzip bodies alike and stops
|
|
45
|
+
* with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
|
|
46
|
+
* small compressed body never inflates past the cap.
|
|
44
47
|
*/
|
|
45
|
-
export declare function decodeRequestBody(request: Request): Promise<Uint8Array | undefined>;
|
|
48
|
+
export declare function decodeRequestBody(request: Request, maxBytes: number): Promise<Uint8Array | undefined>;
|
package/dist/content-encoding.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* §1.3 transport compression for
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* §1.3 transport compression for the HTTP bodies the adapters serve —
|
|
3
|
+
* `<mount>/sync`, `<mount>/operations` and segment downloads (blob downloads
|
|
4
|
+
* stream identity, §5.9.5): negotiate `Content-Encoding` from the request's
|
|
5
5
|
* `Accept-Encoding` (zstd preferred, gzip fallback, identity when the client
|
|
6
6
|
* offers neither), feature-detecting the runtime's codecs, zero
|
|
7
7
|
* dependencies. Bun supplies zstd + gzip natively; every other runtime
|
|
@@ -121,26 +121,55 @@ export function encodeStream(body, acceptEncoding) {
|
|
|
121
121
|
* `Content-Encoding: gzip` (§1.3: clients MAY compress a request body and
|
|
122
122
|
* servers MUST accept it). Returns `undefined` for any other declared
|
|
123
123
|
* encoding — the adapter answers 415 rather than parse bytes it cannot read.
|
|
124
|
-
* A gzip body that does not decompress is a `sync.invalid_request`.
|
|
124
|
+
* A gzip body that does not decompress is a `sync.invalid_request`. One read
|
|
125
|
+
* loop counts the decoded bytes of identity and gzip bodies alike and stops
|
|
126
|
+
* with `sync.limit_exceeded` past `maxBytes` (§1.7 `maxRequestBytes`), so a
|
|
127
|
+
* small compressed body never inflates past the cap.
|
|
125
128
|
*/
|
|
126
|
-
export async function decodeRequestBody(request) {
|
|
129
|
+
export async function decodeRequestBody(request, maxBytes) {
|
|
127
130
|
const encoding = request.headers
|
|
128
131
|
.get('content-encoding')
|
|
129
132
|
?.trim()
|
|
130
133
|
.toLowerCase();
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
if (encoding !== 'gzip' || typeof DecompressionStream !== 'function') {
|
|
134
|
+
const identity = encoding === undefined || encoding === '' || encoding === 'identity';
|
|
135
|
+
if (!identity &&
|
|
136
|
+
(encoding !== 'gzip' || typeof DecompressionStream !== 'function')) {
|
|
135
137
|
return undefined;
|
|
136
138
|
}
|
|
137
139
|
if (request.body === null) {
|
|
140
|
+
if (identity)
|
|
141
|
+
return new Uint8Array(0);
|
|
138
142
|
throw syncError('sync.invalid_request', 'malformed gzip request body');
|
|
139
143
|
}
|
|
140
|
-
|
|
141
|
-
|
|
144
|
+
const reader = (identity
|
|
145
|
+
? request.body
|
|
146
|
+
: request.body.pipeThrough(new DecompressionStream('gzip'))).getReader();
|
|
147
|
+
const chunks = [];
|
|
148
|
+
let total = 0;
|
|
149
|
+
for (;;) {
|
|
150
|
+
let step;
|
|
151
|
+
try {
|
|
152
|
+
step = await reader.read();
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
if (identity)
|
|
156
|
+
throw error;
|
|
157
|
+
throw syncError('sync.invalid_request', 'malformed gzip request body');
|
|
158
|
+
}
|
|
159
|
+
if (step.done)
|
|
160
|
+
break;
|
|
161
|
+
total += step.value.length;
|
|
162
|
+
if (total > maxBytes) {
|
|
163
|
+
await reader.cancel().catch(() => { });
|
|
164
|
+
throw syncError('sync.limit_exceeded', 'request body exceeds the server limit (§1.7)', JSON.stringify({ limit: 'maxRequestBytes', max: maxBytes }));
|
|
165
|
+
}
|
|
166
|
+
chunks.push(step.value);
|
|
142
167
|
}
|
|
143
|
-
|
|
144
|
-
|
|
168
|
+
const bytes = new Uint8Array(total);
|
|
169
|
+
let offset = 0;
|
|
170
|
+
for (const chunk of chunks) {
|
|
171
|
+
bytes.set(chunk, offset);
|
|
172
|
+
offset += chunk.length;
|
|
145
173
|
}
|
|
174
|
+
return bytes;
|
|
146
175
|
}
|