@syncular/server 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -6
- package/dist/admin.d.ts +3 -1
- package/dist/admin.js +7 -1
- package/dist/blob-handlers.js +1 -1
- package/dist/context.d.ts +8 -1
- package/dist/d1-storage.d.ts +11 -3
- package/dist/d1-storage.js +84 -2
- package/dist/errors.js +10 -0
- package/dist/handler.js +50 -4
- package/dist/operations.js +3 -3
- package/dist/pg-executor.d.ts +11 -1
- package/dist/postgres-storage.d.ts +20 -5
- package/dist/postgres-storage.js +345 -9
- package/dist/pull.js +9 -5
- package/dist/push.js +18 -0
- package/dist/readiness.d.ts +9 -2
- package/dist/readiness.js +27 -2
- package/dist/realtime.d.ts +40 -1
- package/dist/realtime.js +116 -19
- package/dist/relational-rows.d.ts +20 -0
- package/dist/relational-rows.js +29 -0
- package/dist/s3-segment-store.d.ts +20 -8
- package/dist/s3-segment-store.js +216 -68
- package/dist/schema.js +11 -0
- package/dist/segment-download.d.ts +1 -1
- package/dist/segment-download.js +50 -17
- package/dist/segment-store.d.ts +86 -0
- package/dist/segment-store.js +156 -17
- package/dist/signed-url.d.ts +7 -1
- package/dist/signed-url.js +10 -1
- package/dist/sqlite-dialect.d.ts +7 -1
- package/dist/sqlite-dialect.js +23 -0
- package/dist/sqlite-segment-store.js +153 -42
- package/dist/sqlite-storage.d.ts +11 -3
- package/dist/sqlite-storage.js +346 -34
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +4 -0
- package/dist/storage.d.ts +152 -2
- package/dist/storage.js +40 -1
- package/dist/validate.d.ts +20 -1
- package/package.json +2 -2
- package/src/admin.ts +13 -1
- package/src/blob-handlers.ts +1 -1
- package/src/context.ts +8 -0
- package/src/d1-storage.ts +143 -1
- package/src/errors.ts +10 -0
- package/src/handler.ts +62 -4
- package/src/operations.ts +17 -13
- package/src/pg-executor.ts +11 -1
- package/src/postgres-storage.ts +520 -5
- package/src/pull.ts +9 -1
- package/src/push.ts +18 -0
- package/src/readiness.ts +39 -4
- package/src/realtime.ts +117 -18
- package/src/relational-rows.ts +36 -0
- package/src/s3-segment-store.ts +275 -84
- package/src/schema.ts +15 -0
- package/src/segment-download.ts +79 -28
- package/src/segment-store.ts +250 -18
- package/src/signed-url.ts +12 -2
- package/src/sqlite-dialect.ts +23 -0
- package/src/sqlite-segment-store.ts +245 -65
- package/src/sqlite-storage.ts +494 -36
- package/src/storage-errors.ts +13 -1
- package/src/storage.ts +243 -1
- package/src/validate.ts +20 -1
package/dist/s3-segment-store.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { segmentIdFor } from './segment-store.js';
|
|
1
|
+
import { mergeSegmentRecord, publicationRecord, publicationsForRecord, segmentBytesEqual, segmentIdFor, selectSegmentPublication, } from './segment-store.js';
|
|
2
2
|
import { EMPTY_PAYLOAD_SHA256, presignUrl, sha256Hex, signRequest, } from './sigv4.js';
|
|
3
3
|
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
4
4
|
const SEGMENT_ID_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
@@ -19,7 +19,7 @@ const RECORD_META_HEADER = 'x-amz-meta-syncular-record';
|
|
|
19
19
|
* S3 stats `approximate: true`. They are a health gauge, not an invoice.
|
|
20
20
|
*/
|
|
21
21
|
const STATS_KEY_SUFFIX = 'stats/segments.json';
|
|
22
|
-
const
|
|
22
|
+
const CAS_ATTEMPTS = 16;
|
|
23
23
|
function emptyStats() {
|
|
24
24
|
return { count: 0, bytes: 0, rowsSegments: 0, sqliteSegments: 0 };
|
|
25
25
|
}
|
|
@@ -33,18 +33,8 @@ function parseStatsJson(json) {
|
|
|
33
33
|
};
|
|
34
34
|
}
|
|
35
35
|
// Runtime-neutral base64url: `Buffer` is not present on
|
|
36
|
-
// Cloudflare Workers without `nodejs_compat`, so
|
|
37
|
-
// header is
|
|
38
|
-
function utf8ToBase64url(text) {
|
|
39
|
-
let binary = '';
|
|
40
|
-
for (const byte of new TextEncoder().encode(text)) {
|
|
41
|
-
binary += String.fromCharCode(byte);
|
|
42
|
-
}
|
|
43
|
-
return btoa(binary)
|
|
44
|
-
.replaceAll('+', '-')
|
|
45
|
-
.replaceAll('/', '_')
|
|
46
|
-
.replace(/=+$/, '');
|
|
47
|
-
}
|
|
36
|
+
// Cloudflare Workers without `nodejs_compat`, so a legacy object-metadata
|
|
37
|
+
// record header is decoded with `atob`, available in every runtime.
|
|
48
38
|
function base64urlToUtf8(value) {
|
|
49
39
|
const binary = atob(value.replaceAll('-', '+').replaceAll('_', '/'));
|
|
50
40
|
const bytes = new Uint8Array(binary.length);
|
|
@@ -58,6 +48,54 @@ function recordToJson(record) {
|
|
|
58
48
|
function isMediaType(value) {
|
|
59
49
|
return value === 'rows' || value === 'sqlite';
|
|
60
50
|
}
|
|
51
|
+
/** One publication inside a stored record (or a legacy materialized one). */
|
|
52
|
+
function parsePublication(value, source) {
|
|
53
|
+
if (typeof value !== 'object' || value === null) {
|
|
54
|
+
throw new Error(`S3SegmentStore: corrupt publication in ${source}`);
|
|
55
|
+
}
|
|
56
|
+
const p = value;
|
|
57
|
+
const strings = ['partition', 'logEpoch', 'table', 'scopeDigest'];
|
|
58
|
+
const numbers = [
|
|
59
|
+
'schemaVersion',
|
|
60
|
+
'asOfCommitSeq',
|
|
61
|
+
'rowCount',
|
|
62
|
+
'byteLength',
|
|
63
|
+
'createdAtMs',
|
|
64
|
+
'expiresAtMs',
|
|
65
|
+
];
|
|
66
|
+
for (const field of strings) {
|
|
67
|
+
if (typeof p[field] !== 'string') {
|
|
68
|
+
throw new Error(`S3SegmentStore: bad publication field ${field} in ${source}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
for (const field of numbers) {
|
|
72
|
+
if (typeof p[field] !== 'number') {
|
|
73
|
+
throw new Error(`S3SegmentStore: bad publication field ${field} in ${source}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (!isMediaType(p.mediaType)) {
|
|
77
|
+
throw new Error(`S3SegmentStore: bad publication mediaType in ${source}`);
|
|
78
|
+
}
|
|
79
|
+
const cursorOk = (v) => v === null || typeof v === 'string';
|
|
80
|
+
if (!cursorOk(p.rowCursor) || !cursorOk(p.nextRowCursor)) {
|
|
81
|
+
throw new Error(`S3SegmentStore: bad publication cursor in ${source}`);
|
|
82
|
+
}
|
|
83
|
+
return {
|
|
84
|
+
partition: p.partition,
|
|
85
|
+
logEpoch: p.logEpoch,
|
|
86
|
+
table: p.table,
|
|
87
|
+
schemaVersion: p.schemaVersion,
|
|
88
|
+
mediaType: p.mediaType,
|
|
89
|
+
scopeDigest: p.scopeDigest,
|
|
90
|
+
asOfCommitSeq: p.asOfCommitSeq,
|
|
91
|
+
rowCount: p.rowCount,
|
|
92
|
+
rowCursor: p.rowCursor,
|
|
93
|
+
nextRowCursor: p.nextRowCursor,
|
|
94
|
+
byteLength: p.byteLength,
|
|
95
|
+
createdAtMs: p.createdAtMs,
|
|
96
|
+
expiresAtMs: p.expiresAtMs,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
61
99
|
function parseRecordJson(json, source) {
|
|
62
100
|
let parsed;
|
|
63
101
|
try {
|
|
@@ -102,14 +140,32 @@ function parseRecordJson(json, source) {
|
|
|
102
140
|
if (!cursorOk(r.rowCursor) || !cursorOk(r.nextRowCursor)) {
|
|
103
141
|
throw new Error(`S3SegmentStore: bad record cursor in ${source}`);
|
|
104
142
|
}
|
|
105
|
-
|
|
143
|
+
const scopeDigest = r.scopeDigest;
|
|
144
|
+
// Records written before §5.1 recorded the digest set carry only the
|
|
145
|
+
// primary digest; the object is a cache entry with a TTL, so reading it
|
|
146
|
+
// back as a one-digest record is correct, not a degraded path.
|
|
147
|
+
const rawDigests = r.scopeDigests;
|
|
148
|
+
if (rawDigests !== undefined &&
|
|
149
|
+
(!Array.isArray(rawDigests) ||
|
|
150
|
+
rawDigests.some((digest) => typeof digest !== 'string'))) {
|
|
151
|
+
throw new Error(`S3SegmentStore: bad record scopeDigests in ${source}`);
|
|
152
|
+
}
|
|
153
|
+
const rawPublications = r.publications;
|
|
154
|
+
if (rawPublications !== undefined && !Array.isArray(rawPublications)) {
|
|
155
|
+
throw new Error(`S3SegmentStore: bad record publications in ${source}`);
|
|
156
|
+
}
|
|
157
|
+
const top = {
|
|
106
158
|
segmentId: r.segmentId,
|
|
107
159
|
partition: r.partition,
|
|
108
160
|
logEpoch: r.logEpoch,
|
|
109
161
|
table: r.table,
|
|
110
162
|
schemaVersion: r.schemaVersion,
|
|
111
163
|
mediaType: r.mediaType,
|
|
112
|
-
scopeDigest
|
|
164
|
+
scopeDigest,
|
|
165
|
+
scopeDigests: [
|
|
166
|
+
scopeDigest,
|
|
167
|
+
...(rawDigests ?? []).filter((digest) => digest !== scopeDigest),
|
|
168
|
+
],
|
|
113
169
|
asOfCommitSeq: r.asOfCommitSeq,
|
|
114
170
|
rowCount: r.rowCount,
|
|
115
171
|
rowCursor: r.rowCursor,
|
|
@@ -118,6 +174,15 @@ function parseRecordJson(json, source) {
|
|
|
118
174
|
createdAtMs: r.createdAtMs,
|
|
119
175
|
expiresAtMs: r.expiresAtMs,
|
|
120
176
|
};
|
|
177
|
+
return {
|
|
178
|
+
...top,
|
|
179
|
+
// A record from before per-publication provenance, or a legacy record
|
|
180
|
+
// whose union came from object metadata, materializes its publications
|
|
181
|
+
// from the merged top-level context — the old single-record read.
|
|
182
|
+
publications: rawPublications === undefined
|
|
183
|
+
? publicationsForRecord(top)
|
|
184
|
+
: rawPublications.map((value) => parsePublication(value, source)),
|
|
185
|
+
};
|
|
121
186
|
}
|
|
122
187
|
export class S3SegmentStore {
|
|
123
188
|
#config;
|
|
@@ -140,6 +205,10 @@ export class S3SegmentStore {
|
|
|
140
205
|
objectKeyFor(segmentId) {
|
|
141
206
|
return `${this.#prefix}seg/${segmentId.replace(':', '/')}`;
|
|
142
207
|
}
|
|
208
|
+
/** `{keyPrefix}rec/sha256/{hex}.json` — the §5.1 record (§5.1 union). */
|
|
209
|
+
#recordKeyFor(segmentId) {
|
|
210
|
+
return `${this.#prefix}rec/${segmentId.replace(':', '/')}.json`;
|
|
211
|
+
}
|
|
143
212
|
async #findKeyFor(key) {
|
|
144
213
|
const canonical = JSON.stringify([
|
|
145
214
|
key.partition,
|
|
@@ -180,6 +249,8 @@ export class S3SegmentStore {
|
|
|
180
249
|
await response.arrayBuffer();
|
|
181
250
|
return undefined;
|
|
182
251
|
}
|
|
252
|
+
if (options?.allow412 === true && response.status === 412)
|
|
253
|
+
return response;
|
|
183
254
|
if (!response.ok) {
|
|
184
255
|
const detail = (await response.text()).slice(0, 300);
|
|
185
256
|
throw new Error(`S3SegmentStore: ${method} ${key} failed with ${response.status}: ${detail}`);
|
|
@@ -237,7 +308,7 @@ export class S3SegmentStore {
|
|
|
237
308
|
* that already stored the bytes.
|
|
238
309
|
*/
|
|
239
310
|
async #bumpStats(delta) {
|
|
240
|
-
for (let attempt = 0; attempt <
|
|
311
|
+
for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
|
|
241
312
|
const { stats, etag } = await this.#readStats();
|
|
242
313
|
const next = {
|
|
243
314
|
count: stats.count + delta.count,
|
|
@@ -248,87 +319,164 @@ export class S3SegmentStore {
|
|
|
248
319
|
if (await this.#writeStatsCas(next, etag))
|
|
249
320
|
return;
|
|
250
321
|
}
|
|
251
|
-
// Lost the CAS race `
|
|
322
|
+
// Lost the CAS race `CAS_ATTEMPTS` times running — leave the
|
|
252
323
|
// counters as-is. `stats()` stays APPROXIMATE by contract.
|
|
253
324
|
}
|
|
254
325
|
async put(metadata, bytes, nowMs) {
|
|
255
326
|
const segmentId = await segmentIdFor(bytes);
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
327
|
+
await this.#putBytes(segmentId, bytes);
|
|
328
|
+
for (let attempt = 0; attempt < CAS_ATTEMPTS; attempt++) {
|
|
329
|
+
// The id IS the content address, so an existing record means a second
|
|
330
|
+
// scope published the same bytes: read it back and merge the digest set
|
|
331
|
+
// (§5.1).
|
|
332
|
+
const preexisting = await this.#readRecord(segmentId);
|
|
333
|
+
const record = mergeSegmentRecord(preexisting?.record, metadata, segmentId, bytes.length, nowMs, this.#ttlMs);
|
|
334
|
+
const recordJson = recordToJson(record);
|
|
335
|
+
// The digest set is monotone (§5.1) and this is a read-modify-write: a
|
|
336
|
+
// concurrent publisher of the same bytes computed its own union from the
|
|
337
|
+
// same pre-state, so an unconditional PUT would drop one of the digests
|
|
338
|
+
// and make that scope's download `sync.forbidden`. The write is
|
|
339
|
+
// conditional on the record body that was read — the record ETag is the
|
|
340
|
+
// body hash, so it changes whenever the union does — and a `412`
|
|
341
|
+
// re-reads and merges again. In-process dedupe cannot cover this: two
|
|
342
|
+
// scopes whose rows are byte-identical are two builds, and two server
|
|
343
|
+
// instances share one bucket.
|
|
344
|
+
const conditional = preexisting?.etag === undefined
|
|
345
|
+
? { 'if-none-match': '*' }
|
|
346
|
+
: { 'if-match': preexisting.etag };
|
|
347
|
+
const response = await this.#request('PUT', this.#recordKeyFor(segmentId), {
|
|
348
|
+
body: new TextEncoder().encode(recordJson),
|
|
349
|
+
headers: { 'content-type': 'application/json', ...conditional },
|
|
350
|
+
allow412: true,
|
|
351
|
+
});
|
|
352
|
+
if (response !== undefined && response.status === 412) {
|
|
353
|
+
await response.arrayBuffer();
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
await response?.arrayBuffer();
|
|
357
|
+
if (preexisting === undefined) {
|
|
358
|
+
await this.#bumpStats({
|
|
359
|
+
count: 1,
|
|
360
|
+
bytes: bytes.length,
|
|
361
|
+
rowsSegments: metadata.mediaType === 'rows' ? 1 : 0,
|
|
362
|
+
sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
if (metadata.rowCursor === null) {
|
|
366
|
+
const pointer = await this.#request('PUT', await this.#findKeyFor(metadata), {
|
|
367
|
+
body: new TextEncoder().encode(recordJson),
|
|
368
|
+
headers: { 'content-type': 'application/json' },
|
|
369
|
+
});
|
|
370
|
+
await pointer?.arrayBuffer();
|
|
371
|
+
}
|
|
372
|
+
return record;
|
|
373
|
+
}
|
|
374
|
+
throw new Error(`S3SegmentStore: ${CAS_ATTEMPTS} conditional writes in a row lost the race for ${segmentId} — the stored digest set would have dropped a scope`);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Write the immutable bytes object. Create-only, so a second publisher of
|
|
378
|
+
* identical bytes is a no-op and a second publisher of DIFFERENT bytes under
|
|
379
|
+
* the same content address is the hash collision §5.1 requires to fail
|
|
380
|
+
* loudly instead of overwriting.
|
|
381
|
+
*/
|
|
382
|
+
async #putBytes(segmentId, bytes) {
|
|
383
|
+
const key = this.objectKeyFor(segmentId);
|
|
384
|
+
const response = await this.#request('PUT', key, {
|
|
271
385
|
body: bytes,
|
|
272
386
|
headers: {
|
|
273
387
|
'content-type': 'application/octet-stream',
|
|
274
|
-
|
|
388
|
+
'if-none-match': '*',
|
|
275
389
|
},
|
|
390
|
+
allow412: true,
|
|
276
391
|
});
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
count: 1,
|
|
281
|
-
bytes: bytes.length,
|
|
282
|
-
rowsSegments: metadata.mediaType === 'rows' ? 1 : 0,
|
|
283
|
-
sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
|
|
284
|
-
});
|
|
392
|
+
if (response === undefined || response.status !== 412) {
|
|
393
|
+
await response?.arrayBuffer();
|
|
394
|
+
return;
|
|
285
395
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
396
|
+
await response.arrayBuffer();
|
|
397
|
+
const current = await this.#request('GET', key);
|
|
398
|
+
if (current === undefined)
|
|
399
|
+
return;
|
|
400
|
+
const stored = new Uint8Array(await current.arrayBuffer());
|
|
401
|
+
if (!segmentBytesEqual(stored, bytes)) {
|
|
402
|
+
throw new Error(`S3SegmentStore: content-address collision at ${segmentId} (§5.1)`);
|
|
292
403
|
}
|
|
293
|
-
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* Read the record object plus the ETag of the body that was read, so the
|
|
407
|
+
* union write can be made conditional on exactly that pre-state (§5.1).
|
|
408
|
+
*
|
|
409
|
+
* A record written before the record/bytes split rides in the bytes object's
|
|
410
|
+
* user metadata; it is read here and its union is materialized into a record
|
|
411
|
+
* object by the next `put`. That shape carries no usable ETag for the union
|
|
412
|
+
* (the bytes ETag is a content hash), which is exactly why it was replaced.
|
|
413
|
+
* A bytes object with neither a record object nor the legacy metadata is an
|
|
414
|
+
* entry whose record is missing — a cache miss, not corruption, so `get`
|
|
415
|
+
* reports it as absent and the client re-pulls.
|
|
416
|
+
*/
|
|
417
|
+
async #readRecord(segmentId) {
|
|
418
|
+
if (!SEGMENT_ID_PATTERN.test(segmentId))
|
|
419
|
+
return undefined;
|
|
420
|
+
const response = await this.#request('GET', this.#recordKeyFor(segmentId));
|
|
421
|
+
if (response !== undefined) {
|
|
422
|
+
return {
|
|
423
|
+
record: parseRecordJson(await response.text(), 'segment record'),
|
|
424
|
+
etag: response.headers.get('etag') ?? undefined,
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
const legacy = await this.#request('GET', this.objectKeyFor(segmentId));
|
|
428
|
+
if (legacy === undefined)
|
|
429
|
+
return undefined;
|
|
430
|
+
const meta = legacy.headers.get(RECORD_META_HEADER);
|
|
431
|
+
await legacy.arrayBuffer();
|
|
432
|
+
if (meta === null)
|
|
433
|
+
return undefined;
|
|
434
|
+
return {
|
|
435
|
+
record: parseRecordJson(base64urlToUtf8(meta), segmentId),
|
|
436
|
+
etag: undefined,
|
|
437
|
+
};
|
|
294
438
|
}
|
|
295
439
|
async get(segmentId) {
|
|
296
440
|
if (!SEGMENT_ID_PATTERN.test(segmentId))
|
|
297
441
|
return undefined;
|
|
442
|
+
const record = await this.#readRecord(segmentId);
|
|
443
|
+
if (record === undefined)
|
|
444
|
+
return undefined;
|
|
298
445
|
const response = await this.#request('GET', this.objectKeyFor(segmentId));
|
|
299
446
|
if (response === undefined)
|
|
300
447
|
return undefined;
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
}
|
|
306
|
-
const record = parseRecordJson(base64urlToUtf8(meta), segmentId);
|
|
307
|
-
return { record, bytes };
|
|
448
|
+
return {
|
|
449
|
+
record: record.record,
|
|
450
|
+
bytes: new Uint8Array(await response.arrayBuffer()),
|
|
451
|
+
};
|
|
308
452
|
}
|
|
309
453
|
async find(key, nowMs) {
|
|
310
454
|
const response = await this.#request('GET', await this.#findKeyFor(key));
|
|
311
455
|
if (response === undefined)
|
|
312
456
|
return undefined;
|
|
313
457
|
const record = parseRecordJson(await response.text(), 'reuse pointer');
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
458
|
+
// The pointer is keyed by the full context, but the record body is the
|
|
459
|
+
// whole merged entry: select the publication the key names, never the
|
|
460
|
+
// top-level fields or the digest union (§5.1).
|
|
461
|
+
const publication = selectSegmentPublication(record, {
|
|
462
|
+
partition: key.partition,
|
|
463
|
+
logEpoch: key.logEpoch,
|
|
464
|
+
table: key.table,
|
|
465
|
+
schemaVersion: key.schemaVersion,
|
|
466
|
+
mediaType: key.mediaType,
|
|
467
|
+
scopeDigest: key.scopeDigest,
|
|
468
|
+
asOfCommitSeq: key.asOfCommitSeq,
|
|
469
|
+
rowCursor: null,
|
|
470
|
+
nowMs,
|
|
471
|
+
});
|
|
472
|
+
if (publication === undefined)
|
|
325
473
|
return undefined;
|
|
326
474
|
// Lifecycle GC may have removed the object while the pointer survived.
|
|
327
475
|
const head = await this.#request('HEAD', this.objectKeyFor(record.segmentId));
|
|
328
476
|
if (head === undefined)
|
|
329
477
|
return undefined;
|
|
330
478
|
await head.arrayBuffer();
|
|
331
|
-
return record;
|
|
479
|
+
return publicationRecord(record, publication);
|
|
332
480
|
}
|
|
333
481
|
/**
|
|
334
482
|
* Store-wide counters from the pointer-object accumulator (never a LIST).
|
package/dist/schema.js
CHANGED
|
@@ -72,6 +72,17 @@ export function compileSchema(schema) {
|
|
|
72
72
|
if (primaryKeyIndex === undefined) {
|
|
73
73
|
throw new Error(`table ${table.name}: primary key ${JSON.stringify(table.primaryKey)} is not a column`);
|
|
74
74
|
}
|
|
75
|
+
// §2.4 primary-key eligibility: a primary key MUST render to a `rowId`
|
|
76
|
+
// string every implementation produces identically and local storage
|
|
77
|
+
// resolves without deferring to a SQLite build.
|
|
78
|
+
const primaryKeyColumn = table.columns[primaryKeyIndex];
|
|
79
|
+
if (primaryKeyColumn !== undefined &&
|
|
80
|
+
primaryKeyColumn.type !== 'string' &&
|
|
81
|
+
primaryKeyColumn.type !== 'integer' &&
|
|
82
|
+
primaryKeyColumn.type !== 'boolean' &&
|
|
83
|
+
primaryKeyColumn.type !== 'json') {
|
|
84
|
+
throw new Error(`table ${table.name}: primary key ${JSON.stringify(table.primaryKey)} has an ineligible column type (§2.4); a primary key must be string, integer, boolean, or json`);
|
|
85
|
+
}
|
|
75
86
|
if (table.scopes.length === 0) {
|
|
76
87
|
throw new Error(`table ${table.name}: every synced table declares at least one scope pattern (§3.1)`);
|
|
77
88
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { SyncRequestContext } from './context.js';
|
|
2
|
-
import type
|
|
2
|
+
import { type SegmentRecord } from './segment-store.js';
|
|
3
3
|
export interface SegmentDownloadRequest {
|
|
4
4
|
readonly segmentId: string;
|
|
5
5
|
/** The `X-Syncular-Scopes` header: canonical JSON of the requested map. */
|
package/dist/segment-download.js
CHANGED
|
@@ -3,6 +3,7 @@ import { SyncError, syncError } from './errors.js';
|
|
|
3
3
|
import { emitEvent } from './events.js';
|
|
4
4
|
import { compileSchema } from './schema.js';
|
|
5
5
|
import { computeEffective, scopeDigest } from './scopes.js';
|
|
6
|
+
import { publicationRecord } from './segment-store.js';
|
|
6
7
|
function parseScopesHeader(header) {
|
|
7
8
|
let parsed;
|
|
8
9
|
try {
|
|
@@ -62,25 +63,18 @@ export async function handleSegmentDownload(ctx, request) {
|
|
|
62
63
|
async function downloadSegment(ctx, request) {
|
|
63
64
|
const registry = await touchAuthenticatedPartition(ctx);
|
|
64
65
|
const entry = await ctx.segments.get(request.segmentId);
|
|
65
|
-
if (entry === undefined
|
|
66
|
-
entry.record.partition !== ctx.partition ||
|
|
67
|
-
entry.record.logEpoch !== registry.logEpoch) {
|
|
66
|
+
if (entry === undefined) {
|
|
68
67
|
throw syncError('sync.not_found', 'unknown segment (§5.5)');
|
|
69
68
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
if (
|
|
76
|
-
throw syncError('sync.not_found', 'segment
|
|
69
|
+
// Select the caller's own publication context. A publication under a
|
|
70
|
+
// different partition or a stale log epoch is not this caller's grant, and
|
|
71
|
+
// the entry's existence must not leak across that boundary (§5.5).
|
|
72
|
+
const inContext = entry.record.publications.filter((publication) => publication.partition === ctx.partition &&
|
|
73
|
+
publication.logEpoch === registry.logEpoch);
|
|
74
|
+
if (inContext.length === 0) {
|
|
75
|
+
throw syncError('sync.not_found', 'unknown segment (§5.5)');
|
|
77
76
|
}
|
|
78
77
|
const requested = parseScopesHeader(request.scopesHeader);
|
|
79
|
-
for (const [key, values] of Object.entries(requested)) {
|
|
80
|
-
if (!table.declaredVariables.has(key) || values.includes('*')) {
|
|
81
|
-
throw syncError('sync.invalid_subscription', 'invalid requested scopes (§3.2)');
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
78
|
let resolved;
|
|
85
79
|
try {
|
|
86
80
|
const allowed = await ctx.resolveScopes({
|
|
@@ -111,11 +105,50 @@ async function downloadSegment(ctx, request) {
|
|
|
111
105
|
throw syncError('sync.forbidden', 'segment scopes not held (§5.5)');
|
|
112
106
|
}
|
|
113
107
|
const digest = await scopeDigest(outcome.effective);
|
|
114
|
-
|
|
108
|
+
// The digest is the grant, but only inside the caller's partition and live
|
|
109
|
+
// epoch: identical bytes published elsewhere never authorize this caller.
|
|
110
|
+
const forDigest = inContext.filter((publication) => publication.scopeDigest === digest);
|
|
111
|
+
if (forDigest.length === 0) {
|
|
115
112
|
throw syncError('sync.forbidden', 'scope digest mismatch (§3.5, §5.5)');
|
|
116
113
|
}
|
|
114
|
+
// TTL is per publication: an unrelated refresh of another publication must
|
|
115
|
+
// not extend this grant, and an expired old pin must not shadow a live one.
|
|
116
|
+
const now = clockOf(ctx)();
|
|
117
|
+
const live = forDigest.filter((publication) => publication.expiresAtMs > now);
|
|
118
|
+
if (live.length === 0) {
|
|
119
|
+
throw syncError('sync.segment_expired', 'segment TTL elapsed — re-pull to mint fresh descriptors (§5.1)');
|
|
120
|
+
}
|
|
121
|
+
// Choose among the live candidates that the compiled schema serves and that
|
|
122
|
+
// declare every requested scope. A malformed or custom record whose newer
|
|
123
|
+
// publication names another table must not fail the download while an
|
|
124
|
+
// earlier valid publication of the same bytes exists.
|
|
125
|
+
const schema = compileSchema(ctx.schema);
|
|
126
|
+
const candidates = live.filter((publication) => {
|
|
127
|
+
const table = schema.tables.get(publication.table);
|
|
128
|
+
return (table !== undefined &&
|
|
129
|
+
Object.entries(requested).every(([key, values]) => table.declaredVariables.has(key) && !values.includes('*')));
|
|
130
|
+
});
|
|
131
|
+
if (candidates.length === 0) {
|
|
132
|
+
const served = live.some((publication) => schema.tables.has(publication.table));
|
|
133
|
+
if (!served) {
|
|
134
|
+
throw syncError('sync.not_found', 'segment table no longer served');
|
|
135
|
+
}
|
|
136
|
+
throw syncError('sync.invalid_subscription', 'invalid requested scopes (§3.2)');
|
|
137
|
+
}
|
|
138
|
+
const publication = candidates[candidates.length - 1];
|
|
139
|
+
// RFC 0007: a descriptor minted before this partition declared a backfill
|
|
140
|
+
// checkpoint is not a complete window once the projection becomes
|
|
141
|
+
// checkpointed. SPEC §9 mints segments per schema version, so refuse and
|
|
142
|
+
// let the client re-pull for a fresh descriptor.
|
|
143
|
+
const checkpoints = await ctx.storage.readCheckpoints(ctx.partition);
|
|
144
|
+
if (checkpoints.some((checkpoint) => checkpoint.schemaVersion > publication.schemaVersion)) {
|
|
145
|
+
throw syncError('sync.segment_expired', 'segment predates the partition backfill checkpoint — re-pull (§5.1)');
|
|
146
|
+
}
|
|
117
147
|
return {
|
|
118
|
-
record
|
|
148
|
+
// The returned record is projected onto the selected publication, so its
|
|
149
|
+
// `scopeDigests`/`publications` compatibility view cannot union a grant
|
|
150
|
+
// from another partition (§5.1).
|
|
151
|
+
record: publicationRecord(entry.record, publication),
|
|
119
152
|
bytes: entry.bytes,
|
|
120
153
|
headers: {
|
|
121
154
|
'Content-Type': 'application/octet-stream',
|
package/dist/segment-store.d.ts
CHANGED
|
@@ -12,12 +12,90 @@ export interface SegmentMetadata {
|
|
|
12
12
|
readonly rowCursor: string | null;
|
|
13
13
|
readonly nextRowCursor: string | null;
|
|
14
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* One publication of a content address (§5.1): the full descriptor context
|
|
17
|
+
* that authorized the bytes plus that publication's own cache lifetime.
|
|
18
|
+
* Identical bytes have one content address, so one stored entry carries one
|
|
19
|
+
* publication per distinct (partition, logEpoch, table, schemaVersion,
|
|
20
|
+
* mediaType, scopeDigest, asOfCommitSeq, cursors) context it was published
|
|
21
|
+
* under. Authorization selects a matching publication; it never unions
|
|
22
|
+
* digests across a different partition or epoch.
|
|
23
|
+
*/
|
|
24
|
+
export interface SegmentPublication extends SegmentMetadata {
|
|
25
|
+
readonly byteLength: number;
|
|
26
|
+
readonly createdAtMs: number;
|
|
27
|
+
readonly expiresAtMs: number;
|
|
28
|
+
}
|
|
15
29
|
export interface SegmentRecord extends SegmentMetadata {
|
|
16
30
|
readonly segmentId: string;
|
|
31
|
+
/**
|
|
32
|
+
* Every scope digest this content was published under (§3.5), primary
|
|
33
|
+
* first. A compatibility view over `publications`; download and `find`
|
|
34
|
+
* authorize on a matching publication, not on this union.
|
|
35
|
+
*/
|
|
36
|
+
readonly scopeDigests: readonly string[];
|
|
37
|
+
/** Every publication of this content address, oldest first (§5.1). */
|
|
38
|
+
readonly publications: readonly SegmentPublication[];
|
|
17
39
|
readonly byteLength: number;
|
|
18
40
|
readonly createdAtMs: number;
|
|
19
41
|
readonly expiresAtMs: number;
|
|
20
42
|
}
|
|
43
|
+
/** Canonical identity of one publication; two equal keys are one grant. */
|
|
44
|
+
export declare function publicationKey(metadata: SegmentMetadata): string;
|
|
45
|
+
/**
|
|
46
|
+
* Materialize publications for a record written before provenance existed
|
|
47
|
+
* (in-flight 0.21 objects, §5.1). Only the merged top-level context is
|
|
48
|
+
* known, so each recorded digest is one publication under it: that is the
|
|
49
|
+
* old single-record read behavior preserved verbatim.
|
|
50
|
+
*/
|
|
51
|
+
export declare function publicationsForRecord(record: SegmentMetadata & {
|
|
52
|
+
readonly scopeDigests: readonly string[];
|
|
53
|
+
readonly byteLength: number;
|
|
54
|
+
readonly createdAtMs: number;
|
|
55
|
+
readonly expiresAtMs: number;
|
|
56
|
+
}): SegmentPublication[];
|
|
57
|
+
/**
|
|
58
|
+
* The newest publication matching a context, or undefined (§5.5). Optional
|
|
59
|
+
* `nowMs` skips publications whose own TTL has elapsed, so an expired
|
|
60
|
+
* `asOf`/schema publication never shadows a newer live one.
|
|
61
|
+
*/
|
|
62
|
+
export interface SegmentPublicationSelector {
|
|
63
|
+
readonly partition: string;
|
|
64
|
+
readonly logEpoch: string;
|
|
65
|
+
readonly scopeDigest: string;
|
|
66
|
+
readonly table?: string;
|
|
67
|
+
readonly schemaVersion?: number;
|
|
68
|
+
readonly mediaType?: 'rows' | 'sqlite';
|
|
69
|
+
readonly asOfCommitSeq?: number;
|
|
70
|
+
readonly rowCursor?: string | null;
|
|
71
|
+
readonly nowMs?: number;
|
|
72
|
+
}
|
|
73
|
+
export declare function selectSegmentPublication(record: SegmentRecord, context: SegmentPublicationSelector): SegmentPublication | undefined;
|
|
74
|
+
/**
|
|
75
|
+
* Project a record onto one publication. The compatibility view is narrowed
|
|
76
|
+
* to that grant: `scopeDigests` and `publications` hold only the chosen
|
|
77
|
+
* publication, so a caller that authorizes from them cannot union a grant
|
|
78
|
+
* from a different partition or epoch.
|
|
79
|
+
*/
|
|
80
|
+
export declare function publicationRecord(record: SegmentRecord, publication: SegmentPublication): SegmentRecord;
|
|
81
|
+
/** Byte equality for the content-address collision check (§5.1). */
|
|
82
|
+
export declare function segmentBytesEqual(a: Uint8Array, b: Uint8Array): boolean;
|
|
83
|
+
/**
|
|
84
|
+
* Merge one publication into the stored entry for the same content address
|
|
85
|
+
* (§5.1). Identical bytes are one entry: the record takes the incoming
|
|
86
|
+
* publication's metadata (a later publisher's descriptor, scope digest, and
|
|
87
|
+
* cursors stay consistent with what it just published), the scope digests
|
|
88
|
+
* union with the incoming digest first (the record's primary `scopeDigest`),
|
|
89
|
+
* `createdAtMs` keeps the earliest sighting, and `expiresAtMs` takes the
|
|
90
|
+
* later expiry. A publication with the incoming context merges into that
|
|
91
|
+
* publication's own lifetime; a publication with a new context is appended,
|
|
92
|
+
* so no context's grant is overwritten or extended by another. Publications
|
|
93
|
+
* whose own TTL has elapsed at `nowMs` are pruned from the stored entry —
|
|
94
|
+
* SPEC §5.5 lets the cache forget expired grants, and pruning on put keeps
|
|
95
|
+
* the publication set bounded by live contexts instead of by every context
|
|
96
|
+
* ever seen. Callers MUST have proven the bytes identical first.
|
|
97
|
+
*/
|
|
98
|
+
export declare function mergeSegmentRecord(existing: SegmentRecord | undefined, metadata: SegmentMetadata, segmentId: string, byteLength: number, nowMs: number, ttlMs: number): SegmentRecord;
|
|
21
99
|
/**
|
|
22
100
|
* The §5.3 reuse key: sqlite images are not byte-deterministic, so
|
|
23
101
|
* cross-client dedup is a metadata lookup, not hash convergence — one
|
|
@@ -63,6 +141,14 @@ export interface SegmentStore {
|
|
|
63
141
|
* Unexpired record for the §5.3 reuse key (whole-table segments only:
|
|
64
142
|
* `rowCursor` null), or undefined. Servers MUST reuse instead of
|
|
65
143
|
* rebuilding sqlite images while one exists (§5.3).
|
|
144
|
+
*
|
|
145
|
+
* The digest matches when it is one of the record's `scopeDigests`
|
|
146
|
+
* (§5.1). That cannot hand a sqlite image to the wrong digest: a sqlite
|
|
147
|
+
* image embeds its `scopeDigest` in its bytes, and a merge requires byte
|
|
148
|
+
* equality, so two images published under different digests are never
|
|
149
|
+
* byte-identical and never share one entry. Only rows segments merge
|
|
150
|
+
* across digests, and their bytes carry no digest for a reader to compare
|
|
151
|
+
* against the descriptor.
|
|
66
152
|
*/
|
|
67
153
|
find(key: SegmentFindKey, nowMs: number): Promise<SegmentRecord | undefined>;
|
|
68
154
|
/** Admin/console counters: ADDITIVE, optional. */
|