@syncular/server 0.74.0 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.d.ts +6 -3
- package/dist/admin.js +10 -7
- package/dist/blob-handlers.js +2 -0
- package/dist/bucket-segment-store.d.ts +4 -3
- package/dist/bucket-segment-store.js +4 -2
- package/dist/context.d.ts +29 -1
- package/dist/context.js +23 -2
- package/dist/handler.js +7 -2
- package/dist/index.d.ts +1 -1
- package/dist/lease-store.d.ts +10 -3
- package/dist/lease-store.js +0 -0
- package/dist/operations.js +2 -0
- package/dist/pull.d.ts +9 -4
- package/dist/pull.js +129 -76
- package/dist/realtime.d.ts +14 -1
- package/dist/realtime.js +14 -3
- package/dist/segment-download.js +1 -0
- package/dist/segment-store.d.ts +6 -4
- package/dist/segment-store.js +4 -2
- package/dist/signed-url.d.ts +15 -0
- package/dist/signed-url.js +7 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +18 -3
- package/dist/sqlite-image.js +27 -5
- package/dist/sqlite-lease-store.d.ts +1 -1
- package/dist/sqlite-lease-store.js +19 -6
- package/dist/sqlite-node.js +3 -2
- package/dist/sqlite-segment-store.js +7 -7
- package/package.json +2 -2
- package/src/admin.ts +11 -5
- package/src/blob-handlers.ts +2 -0
- package/src/bucket-segment-store.ts +8 -5
- package/src/context.ts +38 -2
- package/src/handler.ts +9 -1
- package/src/index.ts +1 -0
- package/src/lease-store.ts +0 -0
- package/src/operations.ts +2 -0
- package/src/pull.ts +200 -94
- package/src/realtime.ts +26 -12
- package/src/segment-download.ts +1 -0
- package/src/segment-store.ts +10 -6
- package/src/signed-url.ts +22 -2
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +42 -7
- package/src/sqlite-lease-store.ts +22 -4
- package/src/sqlite-node.ts +3 -2
- package/src/sqlite-segment-store.ts +29 -7
package/dist/pull.js
CHANGED
|
@@ -144,7 +144,9 @@ function parseBootstrapToken(raw, tables, maxSeq) {
|
|
|
144
144
|
parsed.tableIndex === undefined ||
|
|
145
145
|
parsed.tableIndex < 0 ||
|
|
146
146
|
parsed.tableIndex >= tables.length ||
|
|
147
|
-
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
|
|
147
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
|
|
148
|
+
(parsed.image !== undefined &&
|
|
149
|
+
(parsed.image !== true || parsed.rowCursor === null))) {
|
|
148
150
|
return undefined;
|
|
149
151
|
}
|
|
150
152
|
return {
|
|
@@ -152,6 +154,7 @@ function parseBootstrapToken(raw, tables, maxSeq) {
|
|
|
152
154
|
tables: parsed.tables,
|
|
153
155
|
tableIndex: parsed.tableIndex,
|
|
154
156
|
rowCursor: parsed.rowCursor ?? null,
|
|
157
|
+
...(parsed.image === true ? { image: true } : {}),
|
|
155
158
|
};
|
|
156
159
|
}
|
|
157
160
|
catch {
|
|
@@ -176,6 +179,7 @@ async function signedUrlFields(ctx, limits, segmentId, digest, now) {
|
|
|
176
179
|
partition: ctx.partition,
|
|
177
180
|
scopeDigest: digest,
|
|
178
181
|
nowMs: now,
|
|
182
|
+
...(ctx.context !== undefined ? { context: ctx.context } : {}),
|
|
179
183
|
});
|
|
180
184
|
}
|
|
181
185
|
function segmentRefFrame(record, scopeDigest, extra) {
|
|
@@ -192,8 +196,12 @@ function segmentRefFrame(record, scopeDigest, extra) {
|
|
|
192
196
|
// under different scopes (§5.1), and the record's primary digest is not
|
|
193
197
|
// necessarily the caller's (§5.5).
|
|
194
198
|
scopeDigest,
|
|
195
|
-
|
|
196
|
-
|
|
199
|
+
// §5.3: an image descriptor carries no cursors; the resume token holds
|
|
200
|
+
// the position of a table's next range.
|
|
201
|
+
...(record.mediaType === 'rows' && record.rowCursor !== null
|
|
202
|
+
? { rowCursor: record.rowCursor }
|
|
203
|
+
: {}),
|
|
204
|
+
...(record.mediaType === 'rows' && record.nextRowCursor !== null
|
|
197
205
|
? { nextRowCursor: record.nextRowCursor }
|
|
198
206
|
: {}),
|
|
199
207
|
...(extra.url !== undefined ? { url: extra.url } : {}),
|
|
@@ -203,26 +211,51 @@ function segmentRefFrame(record, scopeDigest, extra) {
|
|
|
203
211
|
};
|
|
204
212
|
}
|
|
205
213
|
/**
|
|
206
|
-
* §5.3: the sqlite-image lane
|
|
207
|
-
*
|
|
214
|
+
* §5.3: the sqlite-image lane, chosen at the start of a table or continued
|
|
215
|
+
* after a range image (a table on the rows lane never switches to images),
|
|
208
216
|
* and only when the client advertised bit 2. §5.11: a table with any
|
|
209
217
|
* encrypted column is image-INELIGIBLE — an image copies ciphertext
|
|
210
218
|
* wholesale with no per-row decrypt pass, so it MUST be served on the rows
|
|
211
219
|
* lane (which decrypts per row on the client).
|
|
212
220
|
*/
|
|
213
|
-
function sqliteLaneEligible(limits, table, startRowCursor) {
|
|
221
|
+
function sqliteLaneEligible(limits, table, startRowCursor, imageLane) {
|
|
214
222
|
return ((limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
215
|
-
startRowCursor === null &&
|
|
223
|
+
(startRowCursor === null || imageLane) &&
|
|
216
224
|
table.encryptedColumnIndices.length === 0);
|
|
217
225
|
}
|
|
218
|
-
|
|
226
|
+
/**
|
|
227
|
+
* §5.3 reuse lookup for the range image that starts after `rowCursor`. Only
|
|
228
|
+
* a position with more than one rows page left can match a stored image, so
|
|
229
|
+
* a smaller remainder skips the lookup. A stored image past the bound (built
|
|
230
|
+
* under a larger one) does not match unless it holds a single row.
|
|
231
|
+
*/
|
|
232
|
+
async function findImage(ctx, schema, limits, pair, scopePin, rowCursor, probe, logEpoch) {
|
|
233
|
+
if (probe.length <= limits.limitSnapshotRows)
|
|
234
|
+
return undefined;
|
|
235
|
+
const stored = await ctx.segments.find({
|
|
236
|
+
partition: ctx.partition,
|
|
237
|
+
logEpoch,
|
|
238
|
+
table: pair.table.name,
|
|
239
|
+
schemaVersion: schema.version,
|
|
240
|
+
mediaType: 'sqlite',
|
|
241
|
+
scopeDigest: await scopeDigest(pair.effective),
|
|
242
|
+
asOfCommitSeq: scopePin,
|
|
243
|
+
rowCursor,
|
|
244
|
+
}, clockOf(ctx)());
|
|
245
|
+
return stored !== undefined &&
|
|
246
|
+
(stored.byteLength <= limitsOf(ctx).maxImageSegmentBytes ||
|
|
247
|
+
stored.rowCount === 1)
|
|
248
|
+
? stored
|
|
249
|
+
: undefined;
|
|
250
|
+
}
|
|
251
|
+
async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, imageLane, maxSeq, horizonSeq, logEpoch) {
|
|
219
252
|
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
220
253
|
table: pair.table.name,
|
|
221
254
|
scopeFilter: pair.effective,
|
|
222
255
|
afterRowId: startRowCursor,
|
|
223
256
|
limit: limits.limitSnapshotRows + 1,
|
|
224
257
|
});
|
|
225
|
-
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
258
|
+
if (!sqliteLaneEligible(limits, pair.table, startRowCursor, imageLane)) {
|
|
226
259
|
return { firstPage: await scan };
|
|
227
260
|
}
|
|
228
261
|
// §4.7: a sqlite-lane table pins at the newest commit carrying all of the
|
|
@@ -237,30 +270,17 @@ async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, m
|
|
|
237
270
|
}),
|
|
238
271
|
]);
|
|
239
272
|
const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
240
|
-
|
|
241
|
-
// rows page, so the probe decides before the reuse lookup; a remote `find`
|
|
242
|
-
// on a small table would be a wasted round trip.
|
|
243
|
-
const stored = firstPage.length <= limits.limitSnapshotRows
|
|
244
|
-
? undefined
|
|
245
|
-
: await ctx.segments.find({
|
|
246
|
-
partition: ctx.partition,
|
|
247
|
-
logEpoch,
|
|
248
|
-
table: pair.table.name,
|
|
249
|
-
schemaVersion: schema.version,
|
|
250
|
-
mediaType: 'sqlite',
|
|
251
|
-
scopeDigest: await scopeDigest(pair.effective),
|
|
252
|
-
asOfCommitSeq: scopePin,
|
|
253
|
-
}, clockOf(ctx)());
|
|
273
|
+
const stored = await findImage(ctx, schema, limits, pair, scopePin, startRowCursor, firstPage, logEpoch);
|
|
254
274
|
return { firstPage, image: { scopePin, stored } };
|
|
255
275
|
}
|
|
256
276
|
/**
|
|
257
|
-
* §5.3 sqlite-image lane:
|
|
258
|
-
*
|
|
259
|
-
* schemaVersion, scope digest, scope
|
|
260
|
-
*
|
|
261
|
-
* takes over).
|
|
277
|
+
* §5.3 sqlite-image lane: one range image starting after `rowCursor`, only
|
|
278
|
+
* when more than one rows page remains there. Reuses an unexpired stored
|
|
279
|
+
* image for the same (partition, table, schemaVersion, scope digest, scope
|
|
280
|
+
* pin, range start) instead of rebuilding — the bootstrap-storm rule.
|
|
281
|
+
* Returns undefined when no image is served (the rows lane takes over).
|
|
262
282
|
*/
|
|
263
|
-
async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest, trace, logEpoch, existing, probe) {
|
|
283
|
+
async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, rowCursor, digest, trace, logEpoch, existing, probe) {
|
|
264
284
|
const { storage, segments, partition } = ctx;
|
|
265
285
|
const now = clockOf(ctx)();
|
|
266
286
|
if (existing !== undefined) {
|
|
@@ -272,15 +292,15 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
|
|
|
272
292
|
rows: existing.rowCount,
|
|
273
293
|
});
|
|
274
294
|
yield segmentRefFrame(existing, digest, await signedUrlFields(ctx, limits, existing.segmentId, digest, now));
|
|
275
|
-
return
|
|
295
|
+
return existing;
|
|
276
296
|
}
|
|
277
297
|
if (probe.length <= limits.limitSnapshotRows)
|
|
278
|
-
return
|
|
298
|
+
return undefined;
|
|
279
299
|
// §5.3: hosts opt into image construction with a runtime-specific builder.
|
|
280
300
|
// Without one, matching stored images remain usable and cold pulls use rows.
|
|
281
301
|
const buildImage = ctx.sqliteImageBuilder;
|
|
282
302
|
if (buildImage === undefined)
|
|
283
|
-
return
|
|
303
|
+
return undefined;
|
|
284
304
|
let stores = imageBuilds.get(storage);
|
|
285
305
|
if (stores === undefined) {
|
|
286
306
|
stores = new WeakMap();
|
|
@@ -299,6 +319,7 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
|
|
|
299
319
|
mediaType: 'sqlite',
|
|
300
320
|
scopeDigest: digest,
|
|
301
321
|
asOfCommitSeq: scopePin,
|
|
322
|
+
rowCursor,
|
|
302
323
|
};
|
|
303
324
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name);
|
|
304
325
|
const key = JSON.stringify(identity);
|
|
@@ -307,43 +328,55 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
|
|
|
307
328
|
if (building === undefined) {
|
|
308
329
|
building = (async () => {
|
|
309
330
|
// Another request can finish while this one's eligibility probe awaits.
|
|
310
|
-
const cached = await
|
|
331
|
+
const cached = await findImage(ctx, schema, limits, pair, scopePin, rowCursor, probe, logEpoch);
|
|
311
332
|
if (cached !== undefined)
|
|
312
333
|
return cached;
|
|
313
334
|
builtHere = true;
|
|
314
|
-
|
|
315
|
-
|
|
335
|
+
// The builder stops at the size bound inside the latest batch; these
|
|
336
|
+
// locate the last row it took and whether rows follow it.
|
|
337
|
+
let yielded = 0;
|
|
338
|
+
let batch = [];
|
|
339
|
+
let beforeBatch = rowCursor;
|
|
340
|
+
let exhausted = false;
|
|
341
|
+
const image = await buildImage({
|
|
316
342
|
table: pair.table,
|
|
317
343
|
schemaVersion: schema.version,
|
|
318
344
|
asOfCommitSeq: scopePin,
|
|
319
345
|
scopeDigest: digest,
|
|
346
|
+
maxBytes: limitsOf(ctx).maxImageSegmentBytes,
|
|
320
347
|
rowBatches: (async function* () {
|
|
321
|
-
|
|
322
|
-
yield probe.map((row) => ({
|
|
323
|
-
...row,
|
|
324
|
-
payload: projectRowPayload(current, pair.table, row.payload),
|
|
325
|
-
}));
|
|
326
|
-
let afterRowId = probe[probe.length - 1].rowId;
|
|
348
|
+
let rows = probe;
|
|
327
349
|
for (;;) {
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
afterRowId,
|
|
332
|
-
limit: 5_000,
|
|
333
|
-
});
|
|
334
|
-
rowCount += rows.length;
|
|
350
|
+
beforeBatch = batch[batch.length - 1]?.rowId ?? beforeBatch;
|
|
351
|
+
batch = rows;
|
|
352
|
+
yielded += rows.length;
|
|
335
353
|
yield rows.map((row) => ({
|
|
336
354
|
...row,
|
|
337
355
|
payload: projectRowPayload(current, pair.table, row.payload),
|
|
338
356
|
}));
|
|
339
357
|
const last = rows[rows.length - 1];
|
|
340
|
-
if (rows.length < 5_000
|
|
358
|
+
if (last === undefined || (rows !== probe && rows.length < 5_000))
|
|
341
359
|
break;
|
|
342
|
-
|
|
360
|
+
rows = await storage.scanRows(partition, {
|
|
361
|
+
table: pair.table.name,
|
|
362
|
+
scopeFilter: pair.effective,
|
|
363
|
+
afterRowId: last.rowId,
|
|
364
|
+
limit: 5_000,
|
|
365
|
+
});
|
|
343
366
|
}
|
|
367
|
+
exhausted = true;
|
|
344
368
|
})(),
|
|
345
369
|
});
|
|
346
|
-
|
|
370
|
+
const index = image.rowCount - (yielded - batch.length) - 1;
|
|
371
|
+
if (image.rowCount < 1 || image.rowCount > yielded || index < -1) {
|
|
372
|
+
throw new Error('sqliteImageBuilder consumed rows out of order');
|
|
373
|
+
}
|
|
374
|
+
const last = index < 0 ? beforeBatch : batch[index].rowId;
|
|
375
|
+
return segments.put({
|
|
376
|
+
...identity,
|
|
377
|
+
rowCount: image.rowCount,
|
|
378
|
+
nextRowCursor: image.rowCount < yielded || !exhausted ? last : null,
|
|
379
|
+
}, image.bytes, clockOf(ctx)());
|
|
347
380
|
})();
|
|
348
381
|
builds.set(key, building);
|
|
349
382
|
}
|
|
@@ -363,24 +396,22 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
|
|
|
363
396
|
rows: record.rowCount,
|
|
364
397
|
});
|
|
365
398
|
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, clockOf(ctx)()));
|
|
366
|
-
return
|
|
399
|
+
return record;
|
|
367
400
|
}
|
|
368
401
|
/**
|
|
369
|
-
* One table's segments within the page budget
|
|
370
|
-
*
|
|
402
|
+
* One table's segments within the page budget: range images while the table
|
|
403
|
+
* is on the image lane, then rows pages. Returns the pages it used, the row
|
|
404
|
+
* cursor to resume at (`null` once the table is exhausted) and whether that
|
|
405
|
+
* position is on the image lane.
|
|
371
406
|
*/
|
|
372
407
|
async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, start, pageBudget, trace, logEpoch) {
|
|
373
408
|
const { storage, segments, partition } = ctx;
|
|
374
409
|
const serverLimits = limitsOf(ctx);
|
|
375
410
|
const digest = await scopeDigest(pair.effective);
|
|
376
411
|
const now = clockOf(ctx)();
|
|
377
|
-
if (start.image !== undefined) {
|
|
378
|
-
const imaged = yield* sqliteImageSegment(ctx, schema, limits, pair, start.image.scopePin, digest, trace, logEpoch, start.image.stored, start.firstPage);
|
|
379
|
-
if (imaged)
|
|
380
|
-
return { pages: 1, rowCursor: null };
|
|
381
|
-
}
|
|
382
412
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name);
|
|
383
413
|
let rowCursor = startRowCursor;
|
|
414
|
+
let image = start.image;
|
|
384
415
|
for (let page = 0; page < pageBudget; page++) {
|
|
385
416
|
const scanned = page === 0
|
|
386
417
|
? start.firstPage
|
|
@@ -390,6 +421,20 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
|
|
|
390
421
|
afterRowId: rowCursor,
|
|
391
422
|
limit: limits.limitSnapshotRows + 1,
|
|
392
423
|
});
|
|
424
|
+
if (image !== undefined) {
|
|
425
|
+
// §5.3: every range of the table shares the scope pin computed at this
|
|
426
|
+
// section's table start.
|
|
427
|
+
const record = yield* sqliteImageSegment(ctx, schema, limits, pair, image.scopePin, rowCursor, digest, trace, logEpoch, page === 0
|
|
428
|
+
? image.stored
|
|
429
|
+
: await findImage(ctx, schema, limits, pair, image.scopePin, rowCursor, scanned, logEpoch), scanned);
|
|
430
|
+
if (record !== undefined) {
|
|
431
|
+
if (record.nextRowCursor === null)
|
|
432
|
+
return { pages: page + 1, rowCursor: null, image: false };
|
|
433
|
+
rowCursor = record.nextRowCursor;
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
image = undefined;
|
|
437
|
+
}
|
|
393
438
|
const pageRows = scanned.slice(0, limits.limitSnapshotRows);
|
|
394
439
|
const hasMore = scanned.length > limits.limitSnapshotRows;
|
|
395
440
|
// §5.2: every row record carries the row's current server_version.
|
|
@@ -442,10 +487,10 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
|
|
|
442
487
|
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
|
|
443
488
|
}
|
|
444
489
|
if (!hasMore)
|
|
445
|
-
return { pages: page + 1, rowCursor: null };
|
|
490
|
+
return { pages: page + 1, rowCursor: null, image: false };
|
|
446
491
|
rowCursor = nextRowCursor;
|
|
447
492
|
}
|
|
448
|
-
return { pages: pageBudget, rowCursor };
|
|
493
|
+
return { pages: pageBudget, rowCursor, image: image !== undefined };
|
|
449
494
|
}
|
|
450
495
|
/**
|
|
451
496
|
* Plan one unit's bootstrap and start the reads of every table it can reach
|
|
@@ -463,13 +508,14 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
|
|
|
463
508
|
: undefined;
|
|
464
509
|
const tableIndex = resume?.tableIndex ?? 0;
|
|
465
510
|
const rowCursor = resume?.rowCursor ?? null;
|
|
511
|
+
const image = resume?.image === true;
|
|
466
512
|
const starts = new Map();
|
|
467
513
|
for (let index = tableIndex; index < unit.tables.length && starts.size < limits.maxSnapshotPages; index++) {
|
|
468
514
|
const table = unit.tables[index];
|
|
469
515
|
const effective = unit.pairs.get(table.name);
|
|
470
516
|
if (effective === undefined)
|
|
471
517
|
continue;
|
|
472
|
-
const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, maxSeq, horizonSeq, logEpoch);
|
|
518
|
+
const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, index === tableIndex && image, maxSeq, horizonSeq, logEpoch);
|
|
473
519
|
// A section that stops early never awaits the later starts.
|
|
474
520
|
start.catch(() => undefined);
|
|
475
521
|
starts.set(index, start);
|
|
@@ -479,6 +525,7 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
|
|
|
479
525
|
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
480
526
|
tableIndex,
|
|
481
527
|
rowCursor,
|
|
528
|
+
image,
|
|
482
529
|
starts,
|
|
483
530
|
};
|
|
484
531
|
}
|
|
@@ -490,13 +537,14 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
|
|
|
490
537
|
*/
|
|
491
538
|
export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch) {
|
|
492
539
|
const { unit, pin } = plan;
|
|
493
|
-
let { tableIndex, rowCursor } = plan;
|
|
540
|
+
let { tableIndex, rowCursor, image } = plan;
|
|
494
541
|
let pages = 0;
|
|
495
542
|
for (;;) {
|
|
496
543
|
while (tableIndex < unit.tables.length &&
|
|
497
544
|
!unit.pairs.has(unit.tables[tableIndex].name)) {
|
|
498
545
|
tableIndex += 1;
|
|
499
546
|
rowCursor = null;
|
|
547
|
+
image = false;
|
|
500
548
|
}
|
|
501
549
|
if (tableIndex === unit.tables.length)
|
|
502
550
|
return undefined;
|
|
@@ -506,6 +554,7 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
|
|
|
506
554
|
tables: unit.tables.map((table) => table.name),
|
|
507
555
|
tableIndex,
|
|
508
556
|
rowCursor,
|
|
557
|
+
...(image ? { image: true } : {}),
|
|
509
558
|
};
|
|
510
559
|
return JSON.stringify(token);
|
|
511
560
|
}
|
|
@@ -513,12 +562,13 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
|
|
|
513
562
|
const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
|
|
514
563
|
pages += outcome.pages;
|
|
515
564
|
rowCursor = outcome.rowCursor;
|
|
565
|
+
image = outcome.image;
|
|
516
566
|
if (rowCursor === null)
|
|
517
567
|
tableIndex += 1;
|
|
518
568
|
}
|
|
519
569
|
}
|
|
520
570
|
/**
|
|
521
|
-
* Publish the §5.3
|
|
571
|
+
* Publish the §5.3 range images that a pull of (`scopes`, `table`) looks up,
|
|
522
572
|
* from a host with a SQLite engine (a Bun process with the production
|
|
523
573
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
524
574
|
* reuse key, the build), so the stored identity is exactly the one a
|
|
@@ -553,33 +603,36 @@ export async function publishSqliteImage(input) {
|
|
|
553
603
|
partition,
|
|
554
604
|
actorId: 'syncular.image-publisher',
|
|
555
605
|
};
|
|
556
|
-
// One-row pages make every
|
|
606
|
+
// One-row pages make every range above one row image-eligible (§5.3), and
|
|
607
|
+
// an unbounded page budget stores every range under one scope pin.
|
|
557
608
|
const limits = {
|
|
558
609
|
limitCommits: 1,
|
|
559
610
|
limitSnapshotRows: 1,
|
|
560
|
-
maxSnapshotPages:
|
|
611
|
+
maxSnapshotPages: Number.MAX_SAFE_INTEGER,
|
|
561
612
|
accept: ACCEPT_SQLITE,
|
|
562
613
|
};
|
|
563
614
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
564
615
|
const horizonSeq = await config.storage.getHorizonSeq(partition);
|
|
565
616
|
const trace = { segments: [] };
|
|
566
617
|
const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
|
|
567
|
-
|
|
618
|
+
const refs = [];
|
|
568
619
|
for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
|
|
569
620
|
if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
570
|
-
|
|
621
|
+
refs.push(frame);
|
|
571
622
|
}
|
|
572
623
|
}
|
|
573
|
-
const
|
|
574
|
-
if (
|
|
624
|
+
const first = refs[0];
|
|
625
|
+
if (first === undefined) {
|
|
575
626
|
throw new Error('publishSqliteImage: the scope holds at most one row, which pulls serve inline');
|
|
576
627
|
}
|
|
577
628
|
return {
|
|
578
|
-
|
|
579
|
-
asOfCommitSeq:
|
|
580
|
-
scopeDigest:
|
|
581
|
-
rowCount: ref.rowCount,
|
|
582
|
-
byteLength: ref.byteLength,
|
|
583
|
-
origin:
|
|
629
|
+
segmentIds: refs.map((ref) => ref.segmentId),
|
|
630
|
+
asOfCommitSeq: first.asOfCommitSeq,
|
|
631
|
+
scopeDigest: first.scopeDigest,
|
|
632
|
+
rowCount: refs.reduce((sum, ref) => sum + ref.rowCount, 0),
|
|
633
|
+
byteLength: refs.reduce((sum, ref) => sum + ref.byteLength, 0),
|
|
634
|
+
origin: trace.segments.every((segment) => segment.mediaType !== 'sqlite' || segment.origin === 'reused')
|
|
635
|
+
? 'reused'
|
|
636
|
+
: 'built',
|
|
584
637
|
};
|
|
585
638
|
}
|
package/dist/realtime.d.ts
CHANGED
|
@@ -58,6 +58,12 @@ export interface RealtimeConnectOptions {
|
|
|
58
58
|
readonly partition: string;
|
|
59
59
|
readonly actorId: string;
|
|
60
60
|
readonly clientId: string;
|
|
61
|
+
/**
|
|
62
|
+
* The authentication context the upgrade authenticated (§1.1, §8.1). The
|
|
63
|
+
* connection resolves scopes under it for its lifetime: registrations,
|
|
64
|
+
* presence grants and socket rounds.
|
|
65
|
+
*/
|
|
66
|
+
readonly context?: string;
|
|
61
67
|
/**
|
|
62
68
|
* Socket send: JSON control messages as text, tagged binary otherwise
|
|
63
69
|
* (§8.7). A host MAY return a promise that resolves when the socket
|
|
@@ -108,6 +114,7 @@ export declare class RealtimeSession {
|
|
|
108
114
|
readonly partition: string;
|
|
109
115
|
readonly actorId: string;
|
|
110
116
|
readonly clientId: string;
|
|
117
|
+
readonly context: string | undefined;
|
|
111
118
|
readonly logEpoch: string;
|
|
112
119
|
/** Response/delta layout selected by the most recent socket round. */
|
|
113
120
|
wireVersion: number;
|
|
@@ -234,7 +241,12 @@ export declare class RealtimeHub {
|
|
|
234
241
|
* Resolve the client record's live units into per-connection
|
|
235
242
|
* registrations (§8.1 at upgrade; §8.7 at socket-round end).
|
|
236
243
|
*/
|
|
237
|
-
loadRegistrations(
|
|
244
|
+
loadRegistrations(identity: {
|
|
245
|
+
readonly partition: string;
|
|
246
|
+
readonly actorId: string;
|
|
247
|
+
readonly clientId: string;
|
|
248
|
+
readonly context?: string | undefined;
|
|
249
|
+
}, record: ClientRecord | undefined): Promise<Registration[]>;
|
|
238
250
|
/**
|
|
239
251
|
* Build the per-round request context for a socket sync round (§8.7):
|
|
240
252
|
* the same shape the HTTP adapter builds, so the round drives the
|
|
@@ -243,6 +255,7 @@ export declare class RealtimeHub {
|
|
|
243
255
|
requestContextFor(identity: {
|
|
244
256
|
readonly partition: string;
|
|
245
257
|
readonly actorId: string;
|
|
258
|
+
readonly context?: string | undefined;
|
|
246
259
|
}): SyncRequestContext;
|
|
247
260
|
/** The host's error reporter (`SyncServerConfig.onError`), if any. */
|
|
248
261
|
get onError(): SyncularErrorHandler | undefined;
|
package/dist/realtime.js
CHANGED
|
@@ -88,6 +88,7 @@ export class RealtimeSession {
|
|
|
88
88
|
partition;
|
|
89
89
|
actorId;
|
|
90
90
|
clientId;
|
|
91
|
+
context;
|
|
91
92
|
logEpoch;
|
|
92
93
|
/** Response/delta layout selected by the most recent socket round. */
|
|
93
94
|
wireVersion;
|
|
@@ -148,6 +149,7 @@ export class RealtimeSession {
|
|
|
148
149
|
this.partition = options.partition;
|
|
149
150
|
this.actorId = options.actorId;
|
|
150
151
|
this.clientId = options.clientId;
|
|
152
|
+
this.context = options.context;
|
|
151
153
|
this.logEpoch = logEpoch;
|
|
152
154
|
this.wireVersion = wireVersion;
|
|
153
155
|
this.cursor = cursor;
|
|
@@ -623,7 +625,7 @@ export class RealtimeSession {
|
|
|
623
625
|
const generation = ++this.#registrationGeneration;
|
|
624
626
|
let registrations;
|
|
625
627
|
try {
|
|
626
|
-
registrations = await this.#hub.loadRegistrations(this
|
|
628
|
+
registrations = await this.#hub.loadRegistrations(this, await this.#storage.getClientRecord(this.partition, this.clientId));
|
|
627
629
|
}
|
|
628
630
|
catch {
|
|
629
631
|
if (!failClosed) {
|
|
@@ -920,13 +922,15 @@ export class RealtimeHub {
|
|
|
920
922
|
* Resolve the client record's live units into per-connection
|
|
921
923
|
* registrations (§8.1 at upgrade; §8.7 at socket-round end).
|
|
922
924
|
*/
|
|
923
|
-
async loadRegistrations(
|
|
925
|
+
async loadRegistrations(identity, record) {
|
|
926
|
+
const { partition, actorId, clientId, context } = identity;
|
|
924
927
|
let resolved;
|
|
925
928
|
try {
|
|
926
929
|
const allowed = await this.#config.resolveScopes({
|
|
927
930
|
partition,
|
|
928
931
|
actorId,
|
|
929
932
|
clientId,
|
|
933
|
+
...(context !== undefined ? { context } : {}),
|
|
930
934
|
});
|
|
931
935
|
// §7.3.3: delta-fanout registration needs live scopes — an outage
|
|
932
936
|
// registers nothing (the client's next socket round, §8.7, is
|
|
@@ -1000,6 +1004,7 @@ export class RealtimeHub {
|
|
|
1000
1004
|
segments,
|
|
1001
1005
|
partition: identity.partition,
|
|
1002
1006
|
actorId: identity.actorId,
|
|
1007
|
+
...(identity.context !== undefined ? { context: identity.context } : {}),
|
|
1003
1008
|
realtime: this,
|
|
1004
1009
|
};
|
|
1005
1010
|
}
|
|
@@ -1045,6 +1050,9 @@ export class RealtimeHub {
|
|
|
1045
1050
|
...this.#config,
|
|
1046
1051
|
partition: options.partition,
|
|
1047
1052
|
actorId: options.actorId,
|
|
1053
|
+
...(options.context !== undefined
|
|
1054
|
+
? { context: options.context }
|
|
1055
|
+
: {}),
|
|
1048
1056
|
});
|
|
1049
1057
|
shared = {
|
|
1050
1058
|
logEpoch,
|
|
@@ -1083,6 +1091,9 @@ export class RealtimeHub {
|
|
|
1083
1091
|
...this.#config,
|
|
1084
1092
|
partition: options.partition,
|
|
1085
1093
|
actorId: options.actorId,
|
|
1094
|
+
...(options.context !== undefined
|
|
1095
|
+
? { context: options.context }
|
|
1096
|
+
: {}),
|
|
1086
1097
|
})).logEpoch;
|
|
1087
1098
|
if (options.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
1088
1099
|
throw syncError('sync.invalid_client_id', 'clientId uses a reserved server-command namespace (§1.5)');
|
|
@@ -1091,7 +1102,7 @@ export class RealtimeHub {
|
|
|
1091
1102
|
if (record !== undefined && record.actorId !== options.actorId) {
|
|
1092
1103
|
throw syncError('sync.invalid_client_id', 'clientId is bound to a different actor in this partition (§1.5)');
|
|
1093
1104
|
}
|
|
1094
|
-
const registrations = await this.loadRegistrations(options
|
|
1105
|
+
const registrations = await this.loadRegistrations(options, record);
|
|
1095
1106
|
const latestSeq = restored?.latestSeq ?? (await storage.getMaxCommitSeq(options.partition));
|
|
1096
1107
|
const cursor = record?.cursor ?? -1;
|
|
1097
1108
|
const session = new RealtimeSession(this, options, registrations, cursor, latestSeq, clock, this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES, storage, logEpoch, record !== undefined && isSupportedProtocolWireVersion(record.wireVersion)
|
package/dist/segment-download.js
CHANGED
|
@@ -143,6 +143,7 @@ async function authorizePublication(ctx, request, logEpoch, stored) {
|
|
|
143
143
|
const allowed = await ctx.resolveScopes({
|
|
144
144
|
partition: ctx.partition,
|
|
145
145
|
actorId: ctx.actorId,
|
|
146
|
+
...(ctx.context !== undefined ? { context: ctx.context } : {}),
|
|
146
147
|
});
|
|
147
148
|
// §7.3.3: leases authorize sync rounds, never downloads — an outage
|
|
148
149
|
// signal here denies (re-authorization needs live scopes, §5.5).
|
package/dist/segment-store.d.ts
CHANGED
|
@@ -99,7 +99,8 @@ export declare function mergeSegmentRecord(existing: SegmentRecord | undefined,
|
|
|
99
99
|
/**
|
|
100
100
|
* The §5.3 reuse key: sqlite images are not byte-deterministic, so
|
|
101
101
|
* cross-client dedup is a metadata lookup, not hash convergence — one
|
|
102
|
-
* stored image per (partition, table, schemaVersion, scope digest, pin
|
|
102
|
+
* stored image per (partition, table, schemaVersion, scope digest, pin, range
|
|
103
|
+
* start).
|
|
103
104
|
*/
|
|
104
105
|
export interface SegmentFindKey {
|
|
105
106
|
readonly partition: string;
|
|
@@ -109,6 +110,8 @@ export interface SegmentFindKey {
|
|
|
109
110
|
readonly mediaType: 'rows' | 'sqlite';
|
|
110
111
|
readonly scopeDigest: string;
|
|
111
112
|
readonly asOfCommitSeq: number;
|
|
113
|
+
/** The range start: the row id the image follows, `null` for the first. */
|
|
114
|
+
readonly rowCursor: string | null;
|
|
112
115
|
}
|
|
113
116
|
/**
|
|
114
117
|
* Coarse store-level counters for the admin/console read surface. Optional:
|
|
@@ -146,9 +149,8 @@ export interface SegmentStore {
|
|
|
146
149
|
bytes: Uint8Array;
|
|
147
150
|
} | undefined>;
|
|
148
151
|
/**
|
|
149
|
-
* Unexpired record for the §5.3 reuse key
|
|
150
|
-
*
|
|
151
|
-
* rebuilding sqlite images while one exists (§5.3).
|
|
152
|
+
* Unexpired record for the §5.3 reuse key, or undefined. Servers MUST
|
|
153
|
+
* reuse instead of rebuilding sqlite images while one exists (§5.3).
|
|
152
154
|
*
|
|
153
155
|
* The digest matches when it is one of the record's `scopeDigests`
|
|
154
156
|
* (§5.1). That cannot hand a sqlite image to the wrong digest: a sqlite
|
package/dist/segment-store.js
CHANGED
|
@@ -188,7 +188,9 @@ export class MemorySegmentStore {
|
|
|
188
188
|
};
|
|
189
189
|
}
|
|
190
190
|
async find(key, nowMs) {
|
|
191
|
-
|
|
191
|
+
// The newest publication wins: a rebuild supersedes an image stored
|
|
192
|
+
// under the same key that the server no longer serves (§5.3 bound).
|
|
193
|
+
for (const { record } of [...this.#entries.values()].reverse()) {
|
|
192
194
|
const publication = selectSegmentPublication(record, {
|
|
193
195
|
partition: key.partition,
|
|
194
196
|
logEpoch: key.logEpoch,
|
|
@@ -197,7 +199,7 @@ export class MemorySegmentStore {
|
|
|
197
199
|
mediaType: key.mediaType,
|
|
198
200
|
scopeDigest: key.scopeDigest,
|
|
199
201
|
asOfCommitSeq: key.asOfCommitSeq,
|
|
200
|
-
rowCursor:
|
|
202
|
+
rowCursor: key.rowCursor,
|
|
201
203
|
nowMs,
|
|
202
204
|
});
|
|
203
205
|
if (publication !== undefined) {
|
package/dist/signed-url.d.ts
CHANGED
|
@@ -23,6 +23,8 @@ export interface SegmentPresignArgs {
|
|
|
23
23
|
readonly nowMs: number;
|
|
24
24
|
/** Resolved TTL (config value or the 900 s default) — already applied. */
|
|
25
25
|
readonly ttlSeconds: number;
|
|
26
|
+
/** The authentication context (§1.1) the pull resolved scopes under. */
|
|
27
|
+
readonly context?: string;
|
|
26
28
|
}
|
|
27
29
|
/**
|
|
28
30
|
* Delegated presign (§5.4): the provider signs the URL. The callback MUST
|
|
@@ -49,6 +51,7 @@ export declare function issueSegmentUrl(config: SegmentUrlConfig, args: {
|
|
|
49
51
|
readonly partition: string;
|
|
50
52
|
readonly scopeDigest: string;
|
|
51
53
|
readonly nowMs: number;
|
|
54
|
+
readonly context?: string;
|
|
52
55
|
}): Promise<SegmentUrlIssue>;
|
|
53
56
|
/**
|
|
54
57
|
* Delegated presign for BLOB downloads (§5.9.5). The blob twin of
|
|
@@ -66,6 +69,8 @@ export interface BlobPresignArgs {
|
|
|
66
69
|
readonly nowMs: number;
|
|
67
70
|
/** Resolved TTL (config value or the 900 s default) — already applied. */
|
|
68
71
|
readonly ttlSeconds: number;
|
|
72
|
+
/** The authentication context (§1.1) the download was authorized under. */
|
|
73
|
+
readonly context?: string;
|
|
69
74
|
}
|
|
70
75
|
export interface BlobPresignConfig {
|
|
71
76
|
readonly presign: (args: BlobPresignArgs) => SegmentUrlIssue | Promise<SegmentUrlIssue>;
|
|
@@ -81,6 +86,7 @@ export declare function issueBlobUrl(config: BlobPresignConfig, args: {
|
|
|
81
86
|
readonly partition: string;
|
|
82
87
|
readonly blobId: string;
|
|
83
88
|
readonly nowMs: number;
|
|
89
|
+
readonly context?: string;
|
|
84
90
|
}): Promise<SegmentUrlIssue>;
|
|
85
91
|
/**
|
|
86
92
|
* Delegated presign for BLOB UPLOADS (§5.9.3 direct-to-storage). The upload
|
|
@@ -124,6 +130,8 @@ export interface SegmentTokenClaims {
|
|
|
124
130
|
readonly aud: string;
|
|
125
131
|
/** Unix seconds — the sole non-millisecond timestamp in the spec. */
|
|
126
132
|
readonly exp: number;
|
|
133
|
+
/** The issuing pull's authentication context (§1.1); absent without one. */
|
|
134
|
+
readonly ctx?: string;
|
|
127
135
|
}
|
|
128
136
|
export declare function signSegmentToken(key: string | Uint8Array, claims: SegmentTokenClaims): Promise<string>;
|
|
129
137
|
/**
|
|
@@ -142,4 +150,11 @@ export declare function verifySegmentToken(key: string | Uint8Array, token: stri
|
|
|
142
150
|
readonly audience: string;
|
|
143
151
|
readonly nowMs: number;
|
|
144
152
|
readonly skewSeconds?: number;
|
|
153
|
+
/**
|
|
154
|
+
* The authentication context (§1.1) of a download that authenticated.
|
|
155
|
+
* When set, a token minted under another context, or without one, is
|
|
156
|
+
* refused. A headerless signed download (§5.4) passes none: the
|
|
157
|
+
* MAC-bound `seg`, `sd` and `ctx` claims are the grant.
|
|
158
|
+
*/
|
|
159
|
+
readonly context?: string;
|
|
145
160
|
}): Promise<SegmentTokenClaims>;
|