@syncular/server 0.58.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +31 -22
  2. package/dist/admin.js +8 -4
  3. package/dist/blob-handlers.d.ts +25 -31
  4. package/dist/blob-handlers.js +63 -41
  5. package/dist/blob-store.d.ts +42 -24
  6. package/dist/blob-store.js +0 -0
  7. package/dist/bucket-blob-store.d.ts +12 -8
  8. package/dist/bucket-blob-store.js +54 -45
  9. package/dist/bucket-segment-store.js +18 -16
  10. package/dist/content-encoding.d.ts +5 -2
  11. package/dist/content-encoding.js +42 -13
  12. package/dist/context.d.ts +16 -0
  13. package/dist/context.js +21 -1
  14. package/dist/d1-storage.d.ts +14 -98
  15. package/dist/d1-storage.js +223 -1111
  16. package/dist/errors.js +8 -0
  17. package/dist/events.d.ts +1 -1
  18. package/dist/handler.js +135 -103
  19. package/dist/local-authority.d.ts +2 -0
  20. package/dist/local-authority.js +17 -3
  21. package/dist/object-bucket.d.ts +2 -0
  22. package/dist/operations-realtime.js +69 -50
  23. package/dist/pg-executor-pglite.d.ts +5 -6
  24. package/dist/pg-executor-pglite.js +22 -14
  25. package/dist/pg-executor.d.ts +4 -1
  26. package/dist/postgres-storage.d.ts +1 -1
  27. package/dist/postgres-storage.js +182 -105
  28. package/dist/pull.d.ts +28 -20
  29. package/dist/pull.js +85 -87
  30. package/dist/push.js +272 -147
  31. package/dist/realtime.d.ts +68 -28
  32. package/dist/realtime.js +297 -136
  33. package/dist/relational-rows.d.ts +27 -15
  34. package/dist/relational-rows.js +38 -22
  35. package/dist/s3-blob-store.d.ts +11 -12
  36. package/dist/s3-blob-store.js +20 -13
  37. package/dist/s3-object-bucket.d.ts +8 -1
  38. package/dist/s3-object-bucket.js +16 -3
  39. package/dist/schema.d.ts +14 -0
  40. package/dist/schema.js +23 -0
  41. package/dist/scopes.d.ts +6 -2
  42. package/dist/scopes.js +22 -4
  43. package/dist/seed.js +1 -2
  44. package/dist/segment-download.js +18 -4
  45. package/dist/signed-url.d.ts +2 -2
  46. package/dist/signed-url.js +9 -2
  47. package/dist/sigv4.d.ts +8 -2
  48. package/dist/sigv4.js +8 -4
  49. package/dist/sqlite-blob-store.d.ts +1 -1
  50. package/dist/sqlite-blob-store.js +23 -22
  51. package/dist/sqlite-dialect.d.ts +150 -47
  52. package/dist/sqlite-dialect.js +987 -88
  53. package/dist/sqlite-lease-store.js +13 -18
  54. package/dist/sqlite-storage.d.ts +9 -62
  55. package/dist/sqlite-storage.js +293 -1117
  56. package/dist/storage.d.ts +32 -12
  57. package/package.json +2 -2
  58. package/src/admin.ts +16 -2
  59. package/src/blob-handlers.ts +91 -72
  60. package/src/blob-store.ts +0 -0
  61. package/src/bucket-blob-store.ts +75 -48
  62. package/src/bucket-segment-store.ts +20 -15
  63. package/src/content-encoding.ts +48 -16
  64. package/src/context.ts +32 -1
  65. package/src/d1-storage.ts +326 -1692
  66. package/src/errors.ts +8 -0
  67. package/src/events.ts +1 -1
  68. package/src/handler.ts +178 -127
  69. package/src/local-authority.ts +17 -3
  70. package/src/object-bucket.ts +2 -0
  71. package/src/operations-realtime.ts +85 -61
  72. package/src/pg-executor-pglite.ts +27 -23
  73. package/src/pg-executor.ts +5 -1
  74. package/src/postgres-storage.ts +231 -138
  75. package/src/pull.ts +133 -147
  76. package/src/push.ts +356 -205
  77. package/src/realtime.ts +380 -179
  78. package/src/relational-rows.ts +54 -27
  79. package/src/s3-blob-store.ts +23 -13
  80. package/src/s3-object-bucket.ts +20 -2
  81. package/src/schema.ts +31 -0
  82. package/src/scopes.ts +23 -3
  83. package/src/seed.ts +1 -2
  84. package/src/segment-download.ts +25 -4
  85. package/src/signed-url.ts +10 -4
  86. package/src/sigv4.ts +17 -4
  87. package/src/sqlite-blob-store.ts +27 -26
  88. package/src/sqlite-dialect.ts +1607 -114
  89. package/src/sqlite-lease-store.ts +12 -23
  90. package/src/sqlite-storage.ts +360 -1811
  91. package/src/storage.ts +33 -12
package/dist/pull.js CHANGED
@@ -29,16 +29,27 @@ export function clampPullLimits(header) {
29
29
  accept: header.accept,
30
30
  };
31
31
  }
32
- function parseBootstrapToken(raw, table) {
32
+ /**
33
+ * §4.7: a usable resume token for `table`, or `undefined`. A token whose pin
34
+ * is not a commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`) or
35
+ * whose position does not name one of its tables is unusable.
36
+ */
37
+ function parseBootstrapToken(raw, table, maxSeq) {
33
38
  if (raw === undefined)
34
39
  return undefined;
35
40
  try {
36
41
  const parsed = JSON.parse(raw);
37
- if (typeof parsed.asOfCommitSeq !== 'number' ||
42
+ if (!Number.isSafeInteger(parsed.asOfCommitSeq) ||
43
+ parsed.asOfCommitSeq === undefined ||
44
+ parsed.asOfCommitSeq < 0 ||
45
+ parsed.asOfCommitSeq > maxSeq ||
38
46
  !Array.isArray(parsed.tables) ||
39
- typeof parsed.tableIndex !== 'number' ||
40
- (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
41
- parsed.tables[0] !== table) {
47
+ parsed.tables[0] !== table ||
48
+ !Number.isSafeInteger(parsed.tableIndex) ||
49
+ parsed.tableIndex === undefined ||
50
+ parsed.tableIndex < 0 ||
51
+ parsed.tableIndex >= parsed.tables.length ||
52
+ (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')) {
42
53
  return undefined;
43
54
  }
44
55
  return {
@@ -323,19 +334,27 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
323
334
  }
324
335
  return { complete: false, rowCursor };
325
336
  }
326
- /** Run the pre-frame storage reads of every subscription concurrently. */
337
+ /**
338
+ * Decide the section of every plan and run its pre-frame storage reads. The
339
+ * reads of all plans start before any is awaited, so a storage that batches
340
+ * concurrent page reads (PostgreSQL, D1) answers the first pages of every
341
+ * plan in one statement per table, and the horizon re-check runs once.
342
+ */
327
343
  export async function prepareSections(ctx, schema, limits, plans, maxSeq, horizonSeq, logEpoch) {
328
- const reads = plans.map((plan) => {
344
+ const prepared = await Promise.all(plans.map(async (plan) => {
329
345
  const sub = plan.frame;
330
346
  if (plan.status === 'revoked')
331
- return {};
332
- const token = parseBootstrapToken(sub.bootstrapState, sub.table);
333
- if (token === undefined &&
347
+ return { kind: 'revoked' };
348
+ // §4.7: a present resume token keeps the subscription bootstrapping,
349
+ // whether or not it is usable.
350
+ if (sub.bootstrapState === undefined &&
334
351
  sub.cursor >= 0 &&
335
- sub.cursor >= horizonSeq &&
336
352
  sub.cursor <= maxSeq) {
353
+ if (sub.cursor < horizonSeq)
354
+ return { kind: 'reset' };
337
355
  return {
338
- commits: ctx.storage.readCommitWindow(ctx.partition, {
356
+ kind: 'incremental',
357
+ commits: await ctx.storage.readCommitWindow(ctx.partition, {
339
358
  table: sub.table,
340
359
  scopeFilter: plan.effective,
341
360
  afterSeq: sub.cursor,
@@ -344,77 +363,67 @@ export async function prepareSections(ctx, schema, limits, plans, maxSeq, horizo
344
363
  }),
345
364
  };
346
365
  }
347
- // §4.6 reset: no reads.
348
- if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
349
- return {};
350
- }
351
- // Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
366
+ // §4.7: resume at the token's pin unless the token is unusable or the
367
+ // pin fell behind the horizon; then restart from a fresh pin.
368
+ const token = parseBootstrapToken(sub.bootstrapState, sub.table, maxSeq);
352
369
  const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
353
370
  ? token
354
371
  : undefined;
355
372
  const startCursor = resume?.rowCursor ?? null;
356
- const firstPage = ctx.storage.scanRows(ctx.partition, {
373
+ const scan = ctx.storage.scanRows(ctx.partition, {
357
374
  table: plan.table.name,
358
375
  scopeFilter: plan.effective,
359
376
  afterRowId: startCursor,
360
377
  limit: limits.limitSnapshotRows + 1,
361
378
  });
362
- if (!sqliteLaneEligible(limits, plan, startCursor))
363
- return { firstPage };
364
- // §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
365
- // scope (never below the horizon, never above maxCommitSeq), so a stored
366
- // image stays the reuse key's match until this scope changes (§5.3).
367
- const pinned = resume !== undefined
368
- ? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
369
- : ctx.storage
370
- .latestScopedChangeSeq(ctx.partition, {
371
- table: plan.table.name,
372
- scopeFilter: plan.effective,
373
- })
374
- .then((latest) => {
375
- const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
376
- return { asOf: pin, pin };
377
- });
378
- const lane = pinned.then(async ({ asOf, pin }) => {
379
- // §5.3 eligibility: a stored image only matches a snapshot larger than
380
- // one rows page, so the probe decides before the reuse lookup; a remote
381
- // `find` on a small table would be a wasted round trip.
382
- const probe = await firstPage;
383
- if (probe.length <= limits.limitSnapshotRows) {
384
- return { pin, image: undefined };
385
- }
379
+ if (!sqliteLaneEligible(limits, plan, startCursor)) {
386
380
  return {
387
- pin,
388
- image: await ctx.segments.find({
389
- partition: ctx.partition,
390
- logEpoch,
391
- table: plan.table.name,
392
- schemaVersion: schema.version,
393
- mediaType: 'sqlite',
394
- scopeDigest: await scopeDigest(plan.effective),
395
- asOfCommitSeq: asOf,
396
- }, clockOf(ctx)()),
381
+ kind: 'bootstrap',
382
+ asOf: resume?.asOfCommitSeq ?? maxSeq,
383
+ startCursor,
384
+ firstPage: await scan,
385
+ image: undefined,
397
386
  };
398
- });
399
- return { firstPage, lane };
400
- });
401
- const settled = await Promise.all(reads.map(async (read) => ({
402
- commits: read.commits === undefined ? undefined : await read.commits,
403
- firstPage: await read.firstPage,
404
- lane: await read.lane,
405
- })));
387
+ }
388
+ // §4.7: a sqlite-lane bootstrap starts at the table start and pins at
389
+ // the newest change in its scope (never below the horizon, never above
390
+ // maxCommitSeq), so a stored image stays the reuse key's match until
391
+ // this scope changes (§5.3). The server computes the pin; a client
392
+ // token never supplies the reuse key.
393
+ const [firstPage, latest] = await Promise.all([
394
+ scan,
395
+ ctx.storage.latestScopedChangeSeq(ctx.partition, {
396
+ table: plan.table.name,
397
+ scopeFilter: plan.effective,
398
+ }),
399
+ ]);
400
+ const asOf = Math.min(maxSeq, Math.max(latest, horizonSeq));
401
+ // §5.3 eligibility: a stored image only matches a snapshot larger than
402
+ // one rows page, so the probe decides before the reuse lookup; a remote
403
+ // `find` on a small table would be a wasted round trip.
404
+ const image = firstPage.length <= limits.limitSnapshotRows
405
+ ? undefined
406
+ : await ctx.segments.find({
407
+ partition: ctx.partition,
408
+ logEpoch,
409
+ table: plan.table.name,
410
+ schemaVersion: schema.version,
411
+ mediaType: 'sqlite',
412
+ scopeDigest: await scopeDigest(plan.effective),
413
+ asOfCommitSeq: asOf,
414
+ }, clockOf(ctx)());
415
+ return { kind: 'bootstrap', asOf, startCursor, firstPage, image };
416
+ }));
406
417
  // Validate continuity before committing to an active section. A prune
407
418
  // during a paged read can otherwise make an incomplete window look empty.
408
- const windowHorizon = settled.some((read) => read.commits !== undefined)
409
- ? Math.max(horizonSeq, await ctx.storage.getHorizonSeq(ctx.partition))
410
- : horizonSeq;
411
- return settled.map((read) => ({
412
- commits: read.commits ?? [],
413
- horizonSeq: read.commits === undefined ? horizonSeq : windowHorizon,
414
- firstPage: read.firstPage,
415
- pin: read.lane?.pin,
416
- image: read.lane?.image,
417
- }));
419
+ if (!prepared.some((section) => section.kind === 'incremental')) {
420
+ return prepared;
421
+ }
422
+ const windowHorizon = await ctx.storage.getHorizonSeq(ctx.partition);
423
+ return prepared.map((section, index) => section.kind === 'incremental' &&
424
+ (plans[index]?.frame.cursor ?? -1) < windowHorizon
425
+ ? { kind: 'reset' }
426
+ : section);
418
427
  }
419
428
  /**
420
429
  * Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
@@ -435,7 +444,7 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
435
444
  if (logEpoch === undefined || logEpoch.length === 0) {
436
445
  throw new Error('subscriptionSection requires a non-empty log epoch');
437
446
  }
438
- if (plan.status === 'revoked') {
447
+ if (prepared.kind === 'revoked') {
439
448
  yield {
440
449
  type: 'SUB_START',
441
450
  id: sub.id,
@@ -447,11 +456,9 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
447
456
  yield { type: 'SUB_END', nextCursor: sub.cursor };
448
457
  return { nextCursor: sub.cursor, active: false };
449
458
  }
450
- const token = parseBootstrapToken(sub.bootstrapState, sub.table);
451
- const { commits, horizonSeq } = prepared;
452
459
  // §4.6: a cursor behind the horizon (and not resuming a bootstrap)
453
460
  // cannot compute deltas — answer `reset` and echo the cursor.
454
- if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
461
+ if (prepared.kind === 'reset') {
455
462
  yield {
456
463
  type: 'SUB_START',
457
464
  id: sub.id,
@@ -463,18 +470,8 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
463
470
  yield { type: 'SUB_END', nextCursor: sub.cursor };
464
471
  return { nextCursor: sub.cursor, active: false };
465
472
  }
466
- const bootstrapping = token !== undefined || sub.cursor < 0 || sub.cursor > maxSeq;
467
- if (bootstrapping) {
468
- // §4.7: resume at the pinned point unless the pin fell behind the
469
- // horizon (or the token is unusable) — then restart from a fresh pin.
470
- const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
471
- ? token
472
- : undefined;
473
- const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
474
- const startCursor = resume?.rowCursor ?? null;
475
- if (prepared.firstPage === undefined) {
476
- throw new Error('prepareSections did not read the bootstrap page');
477
- }
473
+ if (prepared.kind === 'bootstrap') {
474
+ const { asOf } = prepared;
478
475
  yield {
479
476
  type: 'SUB_START',
480
477
  id: sub.id,
@@ -483,7 +480,7 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
483
480
  effectiveScopes: plan.effective,
484
481
  bootstrap: true,
485
482
  };
486
- const outcome = yield* bootstrapSegments(ctx, schema, limits, plan, asOf, startCursor, trace, logEpoch, prepared.firstPage, prepared.image);
483
+ const outcome = yield* bootstrapSegments(ctx, schema, limits, plan, asOf, prepared.startCursor, trace, logEpoch, prepared.firstPage, prepared.image);
487
484
  if (outcome.complete) {
488
485
  yield { type: 'SUB_END', nextCursor: asOf };
489
486
  }
@@ -515,6 +512,7 @@ export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepar
515
512
  let delivered = 0;
516
513
  let deliveredCommits = 0;
517
514
  let lastDeliveredSeq = sub.cursor;
515
+ const { commits } = prepared;
518
516
  for (const commit of commits) {
519
517
  if (delivered > 0 &&
520
518
  delivered + commit.changes.length > limits.limitCommits) {