@syncular/server 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/dist/pull.js
CHANGED
|
@@ -29,16 +29,27 @@ export function clampPullLimits(header) {
|
|
|
29
29
|
accept: header.accept,
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
|
-
|
|
32
|
+
/**
|
|
33
|
+
* §4.7: a usable resume token for `table`, or `undefined`. A token whose pin
|
|
34
|
+
* is not a commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`) or
|
|
35
|
+
* whose position does not name one of its tables is unusable.
|
|
36
|
+
*/
|
|
37
|
+
function parseBootstrapToken(raw, table, maxSeq) {
|
|
33
38
|
if (raw === undefined)
|
|
34
39
|
return undefined;
|
|
35
40
|
try {
|
|
36
41
|
const parsed = JSON.parse(raw);
|
|
37
|
-
if (
|
|
42
|
+
if (!Number.isSafeInteger(parsed.asOfCommitSeq) ||
|
|
43
|
+
parsed.asOfCommitSeq === undefined ||
|
|
44
|
+
parsed.asOfCommitSeq < 0 ||
|
|
45
|
+
parsed.asOfCommitSeq > maxSeq ||
|
|
38
46
|
!Array.isArray(parsed.tables) ||
|
|
39
|
-
|
|
40
|
-
(parsed.
|
|
41
|
-
parsed.
|
|
47
|
+
parsed.tables[0] !== table ||
|
|
48
|
+
!Number.isSafeInteger(parsed.tableIndex) ||
|
|
49
|
+
parsed.tableIndex === undefined ||
|
|
50
|
+
parsed.tableIndex < 0 ||
|
|
51
|
+
parsed.tableIndex >= parsed.tables.length ||
|
|
52
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')) {
|
|
42
53
|
return undefined;
|
|
43
54
|
}
|
|
44
55
|
return {
|
|
@@ -323,19 +334,27 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
|
|
|
323
334
|
}
|
|
324
335
|
return { complete: false, rowCursor };
|
|
325
336
|
}
|
|
326
|
-
/**
|
|
337
|
+
/**
|
|
338
|
+
* Decide the section of every plan and run its pre-frame storage reads. The
|
|
339
|
+
* reads of all plans start before any is awaited, so a storage that batches
|
|
340
|
+
* concurrent page reads (PostgreSQL, D1) answers the first pages of every
|
|
341
|
+
* plan in one statement per table, and the horizon re-check runs once.
|
|
342
|
+
*/
|
|
327
343
|
export async function prepareSections(ctx, schema, limits, plans, maxSeq, horizonSeq, logEpoch) {
|
|
328
|
-
const
|
|
344
|
+
const prepared = await Promise.all(plans.map(async (plan) => {
|
|
329
345
|
const sub = plan.frame;
|
|
330
346
|
if (plan.status === 'revoked')
|
|
331
|
-
return {};
|
|
332
|
-
|
|
333
|
-
|
|
347
|
+
return { kind: 'revoked' };
|
|
348
|
+
// §4.7: a present resume token keeps the subscription bootstrapping,
|
|
349
|
+
// whether or not it is usable.
|
|
350
|
+
if (sub.bootstrapState === undefined &&
|
|
334
351
|
sub.cursor >= 0 &&
|
|
335
|
-
sub.cursor >= horizonSeq &&
|
|
336
352
|
sub.cursor <= maxSeq) {
|
|
353
|
+
if (sub.cursor < horizonSeq)
|
|
354
|
+
return { kind: 'reset' };
|
|
337
355
|
return {
|
|
338
|
-
|
|
356
|
+
kind: 'incremental',
|
|
357
|
+
commits: await ctx.storage.readCommitWindow(ctx.partition, {
|
|
339
358
|
table: sub.table,
|
|
340
359
|
scopeFilter: plan.effective,
|
|
341
360
|
afterSeq: sub.cursor,
|
|
@@ -344,77 +363,67 @@ export async function prepareSections(ctx, schema, limits, plans, maxSeq, horizo
|
|
|
344
363
|
}),
|
|
345
364
|
};
|
|
346
365
|
}
|
|
347
|
-
// §4.
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
}
|
|
351
|
-
// Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
|
|
366
|
+
// §4.7: resume at the token's pin unless the token is unusable or the
|
|
367
|
+
// pin fell behind the horizon; then restart from a fresh pin.
|
|
368
|
+
const token = parseBootstrapToken(sub.bootstrapState, sub.table, maxSeq);
|
|
352
369
|
const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
353
370
|
? token
|
|
354
371
|
: undefined;
|
|
355
372
|
const startCursor = resume?.rowCursor ?? null;
|
|
356
|
-
const
|
|
373
|
+
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
357
374
|
table: plan.table.name,
|
|
358
375
|
scopeFilter: plan.effective,
|
|
359
376
|
afterRowId: startCursor,
|
|
360
377
|
limit: limits.limitSnapshotRows + 1,
|
|
361
378
|
});
|
|
362
|
-
if (!sqliteLaneEligible(limits, plan, startCursor))
|
|
363
|
-
return { firstPage };
|
|
364
|
-
// §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
|
|
365
|
-
// scope (never below the horizon, never above maxCommitSeq), so a stored
|
|
366
|
-
// image stays the reuse key's match until this scope changes (§5.3).
|
|
367
|
-
const pinned = resume !== undefined
|
|
368
|
-
? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
|
|
369
|
-
: ctx.storage
|
|
370
|
-
.latestScopedChangeSeq(ctx.partition, {
|
|
371
|
-
table: plan.table.name,
|
|
372
|
-
scopeFilter: plan.effective,
|
|
373
|
-
})
|
|
374
|
-
.then((latest) => {
|
|
375
|
-
const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
376
|
-
return { asOf: pin, pin };
|
|
377
|
-
});
|
|
378
|
-
const lane = pinned.then(async ({ asOf, pin }) => {
|
|
379
|
-
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
380
|
-
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
381
|
-
// `find` on a small table would be a wasted round trip.
|
|
382
|
-
const probe = await firstPage;
|
|
383
|
-
if (probe.length <= limits.limitSnapshotRows) {
|
|
384
|
-
return { pin, image: undefined };
|
|
385
|
-
}
|
|
379
|
+
if (!sqliteLaneEligible(limits, plan, startCursor)) {
|
|
386
380
|
return {
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
schemaVersion: schema.version,
|
|
393
|
-
mediaType: 'sqlite',
|
|
394
|
-
scopeDigest: await scopeDigest(plan.effective),
|
|
395
|
-
asOfCommitSeq: asOf,
|
|
396
|
-
}, clockOf(ctx)()),
|
|
381
|
+
kind: 'bootstrap',
|
|
382
|
+
asOf: resume?.asOfCommitSeq ?? maxSeq,
|
|
383
|
+
startCursor,
|
|
384
|
+
firstPage: await scan,
|
|
385
|
+
image: undefined,
|
|
397
386
|
};
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
387
|
+
}
|
|
388
|
+
// §4.7: a sqlite-lane bootstrap starts at the table start and pins at
|
|
389
|
+
// the newest change in its scope (never below the horizon, never above
|
|
390
|
+
// maxCommitSeq), so a stored image stays the reuse key's match until
|
|
391
|
+
// this scope changes (§5.3). The server computes the pin; a client
|
|
392
|
+
// token never supplies the reuse key.
|
|
393
|
+
const [firstPage, latest] = await Promise.all([
|
|
394
|
+
scan,
|
|
395
|
+
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
396
|
+
table: plan.table.name,
|
|
397
|
+
scopeFilter: plan.effective,
|
|
398
|
+
}),
|
|
399
|
+
]);
|
|
400
|
+
const asOf = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
401
|
+
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
402
|
+
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
403
|
+
// `find` on a small table would be a wasted round trip.
|
|
404
|
+
const image = firstPage.length <= limits.limitSnapshotRows
|
|
405
|
+
? undefined
|
|
406
|
+
: await ctx.segments.find({
|
|
407
|
+
partition: ctx.partition,
|
|
408
|
+
logEpoch,
|
|
409
|
+
table: plan.table.name,
|
|
410
|
+
schemaVersion: schema.version,
|
|
411
|
+
mediaType: 'sqlite',
|
|
412
|
+
scopeDigest: await scopeDigest(plan.effective),
|
|
413
|
+
asOfCommitSeq: asOf,
|
|
414
|
+
}, clockOf(ctx)());
|
|
415
|
+
return { kind: 'bootstrap', asOf, startCursor, firstPage, image };
|
|
416
|
+
}));
|
|
406
417
|
// Validate continuity before committing to an active section. A prune
|
|
407
418
|
// during a paged read can otherwise make an incomplete window look empty.
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
image: read.lane?.image,
|
|
417
|
-
}));
|
|
419
|
+
if (!prepared.some((section) => section.kind === 'incremental')) {
|
|
420
|
+
return prepared;
|
|
421
|
+
}
|
|
422
|
+
const windowHorizon = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
423
|
+
return prepared.map((section, index) => section.kind === 'incremental' &&
|
|
424
|
+
(plans[index]?.frame.cursor ?? -1) < windowHorizon
|
|
425
|
+
? { kind: 'reset' }
|
|
426
|
+
: section);
|
|
418
427
|
}
|
|
419
428
|
/**
|
|
420
429
|
* Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
|
|
@@ -435,7 +444,7 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
|
|
|
435
444
|
if (logEpoch === undefined || logEpoch.length === 0) {
|
|
436
445
|
throw new Error('subscriptionSection requires a non-empty log epoch');
|
|
437
446
|
}
|
|
438
|
-
if (
|
|
447
|
+
if (prepared.kind === 'revoked') {
|
|
439
448
|
yield {
|
|
440
449
|
type: 'SUB_START',
|
|
441
450
|
id: sub.id,
|
|
@@ -447,11 +456,9 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
|
|
|
447
456
|
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
448
457
|
return { nextCursor: sub.cursor, active: false };
|
|
449
458
|
}
|
|
450
|
-
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
451
|
-
const { commits, horizonSeq } = prepared;
|
|
452
459
|
// §4.6: a cursor behind the horizon (and not resuming a bootstrap)
|
|
453
460
|
// cannot compute deltas — answer `reset` and echo the cursor.
|
|
454
|
-
if (
|
|
461
|
+
if (prepared.kind === 'reset') {
|
|
455
462
|
yield {
|
|
456
463
|
type: 'SUB_START',
|
|
457
464
|
id: sub.id,
|
|
@@ -463,18 +470,8 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
|
|
|
463
470
|
yield { type: 'SUB_END', nextCursor: sub.cursor };
|
|
464
471
|
return { nextCursor: sub.cursor, active: false };
|
|
465
472
|
}
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
// §4.7: resume at the pinned point unless the pin fell behind the
|
|
469
|
-
// horizon (or the token is unusable) — then restart from a fresh pin.
|
|
470
|
-
const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
471
|
-
? token
|
|
472
|
-
: undefined;
|
|
473
|
-
const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
|
|
474
|
-
const startCursor = resume?.rowCursor ?? null;
|
|
475
|
-
if (prepared.firstPage === undefined) {
|
|
476
|
-
throw new Error('prepareSections did not read the bootstrap page');
|
|
477
|
-
}
|
|
473
|
+
if (prepared.kind === 'bootstrap') {
|
|
474
|
+
const { asOf } = prepared;
|
|
478
475
|
yield {
|
|
479
476
|
type: 'SUB_START',
|
|
480
477
|
id: sub.id,
|
|
@@ -483,7 +480,7 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
|
|
|
483
480
|
effectiveScopes: plan.effective,
|
|
484
481
|
bootstrap: true,
|
|
485
482
|
};
|
|
486
|
-
const outcome = yield* bootstrapSegments(ctx, schema, limits, plan, asOf, startCursor, trace, logEpoch, prepared.firstPage, prepared.image);
|
|
483
|
+
const outcome = yield* bootstrapSegments(ctx, schema, limits, plan, asOf, prepared.startCursor, trace, logEpoch, prepared.firstPage, prepared.image);
|
|
487
484
|
if (outcome.complete) {
|
|
488
485
|
yield { type: 'SUB_END', nextCursor: asOf };
|
|
489
486
|
}
|
|
@@ -515,6 +512,7 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
|
|
|
515
512
|
let delivered = 0;
|
|
516
513
|
let deliveredCommits = 0;
|
|
517
514
|
let lastDeliveredSeq = sub.cursor;
|
|
515
|
+
const { commits } = prepared;
|
|
518
516
|
for (const commit of commits) {
|
|
519
517
|
if (delivered > 0 &&
|
|
520
518
|
delivered + commit.changes.length > limits.limitCommits) {
|