@syncular/server 0.57.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +31 -22
  2. package/dist/admin.js +8 -4
  3. package/dist/blob-handlers.d.ts +25 -31
  4. package/dist/blob-handlers.js +63 -41
  5. package/dist/blob-store.d.ts +42 -24
  6. package/dist/blob-store.js +0 -0
  7. package/dist/bucket-blob-store.d.ts +12 -8
  8. package/dist/bucket-blob-store.js +54 -45
  9. package/dist/bucket-segment-store.js +18 -16
  10. package/dist/content-encoding.d.ts +5 -2
  11. package/dist/content-encoding.js +42 -13
  12. package/dist/context.d.ts +16 -0
  13. package/dist/context.js +21 -1
  14. package/dist/d1-storage.d.ts +14 -98
  15. package/dist/d1-storage.js +223 -1111
  16. package/dist/errors.js +8 -0
  17. package/dist/events.d.ts +1 -1
  18. package/dist/handler.js +135 -103
  19. package/dist/local-authority.d.ts +2 -0
  20. package/dist/local-authority.js +17 -3
  21. package/dist/object-bucket.d.ts +2 -0
  22. package/dist/operations-realtime.js +69 -50
  23. package/dist/pg-executor-pglite.d.ts +5 -6
  24. package/dist/pg-executor-pglite.js +22 -14
  25. package/dist/pg-executor.d.ts +4 -1
  26. package/dist/postgres-storage.d.ts +1 -1
  27. package/dist/postgres-storage.js +182 -105
  28. package/dist/pull.d.ts +28 -20
  29. package/dist/pull.js +85 -87
  30. package/dist/push.js +272 -147
  31. package/dist/realtime.d.ts +68 -28
  32. package/dist/realtime.js +297 -136
  33. package/dist/relational-rows.d.ts +27 -15
  34. package/dist/relational-rows.js +38 -22
  35. package/dist/s3-blob-store.d.ts +11 -12
  36. package/dist/s3-blob-store.js +20 -13
  37. package/dist/s3-object-bucket.d.ts +8 -1
  38. package/dist/s3-object-bucket.js +16 -3
  39. package/dist/schema.d.ts +14 -0
  40. package/dist/schema.js +23 -0
  41. package/dist/scopes.d.ts +6 -2
  42. package/dist/scopes.js +22 -4
  43. package/dist/seed.js +1 -2
  44. package/dist/segment-download.js +18 -4
  45. package/dist/signed-url.d.ts +2 -2
  46. package/dist/signed-url.js +9 -2
  47. package/dist/sigv4.d.ts +8 -2
  48. package/dist/sigv4.js +8 -4
  49. package/dist/sqlite-blob-store.d.ts +1 -1
  50. package/dist/sqlite-blob-store.js +23 -22
  51. package/dist/sqlite-dialect.d.ts +150 -47
  52. package/dist/sqlite-dialect.js +987 -88
  53. package/dist/sqlite-lease-store.js +13 -18
  54. package/dist/sqlite-storage.d.ts +9 -62
  55. package/dist/sqlite-storage.js +293 -1117
  56. package/dist/storage.d.ts +32 -12
  57. package/package.json +2 -2
  58. package/src/admin.ts +16 -2
  59. package/src/blob-handlers.ts +91 -72
  60. package/src/blob-store.ts +0 -0
  61. package/src/bucket-blob-store.ts +75 -48
  62. package/src/bucket-segment-store.ts +20 -15
  63. package/src/content-encoding.ts +48 -16
  64. package/src/context.ts +32 -1
  65. package/src/d1-storage.ts +326 -1692
  66. package/src/errors.ts +8 -0
  67. package/src/events.ts +1 -1
  68. package/src/handler.ts +178 -127
  69. package/src/local-authority.ts +17 -3
  70. package/src/object-bucket.ts +2 -0
  71. package/src/operations-realtime.ts +85 -61
  72. package/src/pg-executor-pglite.ts +27 -23
  73. package/src/pg-executor.ts +5 -1
  74. package/src/postgres-storage.ts +231 -138
  75. package/src/pull.ts +133 -147
  76. package/src/push.ts +356 -205
  77. package/src/realtime.ts +380 -179
  78. package/src/relational-rows.ts +54 -27
  79. package/src/s3-blob-store.ts +23 -13
  80. package/src/s3-object-bucket.ts +20 -2
  81. package/src/schema.ts +31 -0
  82. package/src/scopes.ts +23 -3
  83. package/src/seed.ts +1 -2
  84. package/src/segment-download.ts +25 -4
  85. package/src/signed-url.ts +10 -4
  86. package/src/sigv4.ts +17 -4
  87. package/src/sqlite-blob-store.ts +27 -26
  88. package/src/sqlite-dialect.ts +1607 -114
  89. package/src/sqlite-lease-store.ts +12 -23
  90. package/src/sqlite-storage.ts +360 -1811
  91. package/src/storage.ts +33 -12
package/src/pull.ts CHANGED
@@ -88,19 +88,31 @@ interface BootstrapToken {
88
88
  rowCursor: string | null;
89
89
  }
90
90
 
91
+ /**
92
+ * §4.7: a usable resume token for `table`, or `undefined`. A token whose pin
93
+ * is not a commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`) or
94
+ * whose position does not name one of its tables is unusable.
95
+ */
91
96
  function parseBootstrapToken(
92
97
  raw: string | undefined,
93
98
  table: string,
99
+ maxSeq: number,
94
100
  ): BootstrapToken | undefined {
95
101
  if (raw === undefined) return undefined;
96
102
  try {
97
103
  const parsed = JSON.parse(raw) as Partial<BootstrapToken>;
98
104
  if (
99
- typeof parsed.asOfCommitSeq !== 'number' ||
105
+ !Number.isSafeInteger(parsed.asOfCommitSeq) ||
106
+ parsed.asOfCommitSeq === undefined ||
107
+ parsed.asOfCommitSeq < 0 ||
108
+ parsed.asOfCommitSeq > maxSeq ||
100
109
  !Array.isArray(parsed.tables) ||
101
- typeof parsed.tableIndex !== 'number' ||
102
- (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
103
- parsed.tables[0] !== table
110
+ parsed.tables[0] !== table ||
111
+ !Number.isSafeInteger(parsed.tableIndex) ||
112
+ parsed.tableIndex === undefined ||
113
+ parsed.tableIndex < 0 ||
114
+ parsed.tableIndex >= parsed.tables.length ||
115
+ (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
104
116
  ) {
105
117
  return undefined;
106
118
  }
@@ -482,41 +494,35 @@ async function* bootstrapSegments(
482
494
  }
483
495
 
484
496
  /**
485
- * The storage reads a section needs before its first frame. A pull starts
486
- * them for every subscription before awaiting any, so a storage that batches
487
- * concurrent page reads (PostgreSQL, D1) answers the first pages of every
488
- * subscription in one statement, and the horizon re-check runs once per pull.
497
+ * One subscription's section decision (§4.5–§4.7) with the storage reads it
498
+ * needs before its first frame.
489
499
  */
490
- export interface PreparedSection {
491
- /** §4.5 window read for an incremental cursor; empty when none was read. */
492
- readonly commits: StoredCommit[];
493
- /** Horizon to judge the cursor against, re-read after any window read. */
494
- readonly horizonSeq: number;
495
- /**
496
- * A bootstrap's first snapshot page (`limitSnapshotRows + 1` rows from the
497
- * resume cursor), which is also the §5.3 image eligibility probe.
498
- */
499
- readonly firstPage: StoredRow[] | undefined;
500
- /** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
501
- readonly pin: number | undefined;
502
- /**
503
- * §5.3 stored image for a sqlite-lane bootstrap's reuse key. Read only when
504
- * the eligibility probe shows a snapshot larger than one rows page, since a
505
- * smaller table can never match a stored image.
506
- */
507
- readonly image: SegmentRecord | undefined;
508
- }
509
-
510
- interface PendingSection {
511
- readonly commits?: Promise<StoredCommit[]>;
512
- readonly firstPage?: Promise<StoredRow[]>;
513
- readonly lane?: Promise<{
514
- readonly pin: number | undefined;
515
- readonly image: SegmentRecord | undefined;
516
- }>;
517
- }
500
+ export type PreparedSection =
501
+ | { readonly kind: 'revoked' }
502
+ /** §4.6: the cursor is behind the horizon. */
503
+ | { readonly kind: 'reset' }
504
+ /** §4.5 window read for an in-range cursor. */
505
+ | { readonly kind: 'incremental'; readonly commits: StoredCommit[] }
506
+ | {
507
+ readonly kind: 'bootstrap';
508
+ /** §4.7 pin: a usable resume token's, the sqlite lane's scope pin, or `maxCommitSeq`. */
509
+ readonly asOf: number;
510
+ readonly startCursor: string | null;
511
+ /**
512
+ * The first snapshot page (`limitSnapshotRows + 1` rows from
513
+ * `startCursor`), which is also the §5.3 image eligibility probe.
514
+ */
515
+ readonly firstPage: StoredRow[];
516
+ /** §5.3 stored image under the reuse key, read only for an eligible probe. */
517
+ readonly image: SegmentRecord | undefined;
518
+ };
518
519
 
519
- /** Run the pre-frame storage reads of every subscription concurrently. */
520
+ /**
521
+ * Decide the section of every plan and run its pre-frame storage reads. The
522
+ * reads of all plans start before any is awaited, so a storage that batches
523
+ * concurrent page reads (PostgreSQL, D1) answers the first pages of every
524
+ * plan in one statement per table, and the horizon re-check runs once.
525
+ */
520
526
  export async function prepareSections(
521
527
  ctx: SyncRequestContext,
522
528
  schema: CompiledSchema,
@@ -526,103 +532,98 @@ export async function prepareSections(
526
532
  horizonSeq: number,
527
533
  logEpoch: string,
528
534
  ): Promise<PreparedSection[]> {
529
- const reads = plans.map((plan): PendingSection => {
530
- const sub = plan.frame;
531
- if (plan.status === 'revoked') return {};
532
- const token = parseBootstrapToken(sub.bootstrapState, sub.table);
533
- if (
534
- token === undefined &&
535
- sub.cursor >= 0 &&
536
- sub.cursor >= horizonSeq &&
537
- sub.cursor <= maxSeq
538
- ) {
539
- return {
540
- commits: ctx.storage.readCommitWindow(ctx.partition, {
541
- table: sub.table,
535
+ const prepared = await Promise.all(
536
+ plans.map(async (plan): Promise<PreparedSection> => {
537
+ const sub = plan.frame;
538
+ if (plan.status === 'revoked') return { kind: 'revoked' };
539
+ // §4.7: a present resume token keeps the subscription bootstrapping,
540
+ // whether or not it is usable.
541
+ if (
542
+ sub.bootstrapState === undefined &&
543
+ sub.cursor >= 0 &&
544
+ sub.cursor <= maxSeq
545
+ ) {
546
+ if (sub.cursor < horizonSeq) return { kind: 'reset' };
547
+ return {
548
+ kind: 'incremental',
549
+ commits: await ctx.storage.readCommitWindow(ctx.partition, {
550
+ table: sub.table,
551
+ scopeFilter: plan.effective,
552
+ afterSeq: sub.cursor,
553
+ throughSeq: maxSeq,
554
+ limitChanges: limits.limitCommits + 1,
555
+ }),
556
+ };
557
+ }
558
+ // §4.7: resume at the token's pin unless the token is unusable or the
559
+ // pin fell behind the horizon; then restart from a fresh pin.
560
+ const token = parseBootstrapToken(sub.bootstrapState, sub.table, maxSeq);
561
+ const resume =
562
+ token !== undefined && token.asOfCommitSeq >= horizonSeq
563
+ ? token
564
+ : undefined;
565
+ const startCursor = resume?.rowCursor ?? null;
566
+ const scan = ctx.storage.scanRows(ctx.partition, {
567
+ table: plan.table.name,
568
+ scopeFilter: plan.effective,
569
+ afterRowId: startCursor,
570
+ limit: limits.limitSnapshotRows + 1,
571
+ });
572
+ if (!sqliteLaneEligible(limits, plan, startCursor)) {
573
+ return {
574
+ kind: 'bootstrap',
575
+ asOf: resume?.asOfCommitSeq ?? maxSeq,
576
+ startCursor,
577
+ firstPage: await scan,
578
+ image: undefined,
579
+ };
580
+ }
581
+ // §4.7: a sqlite-lane bootstrap starts at the table start and pins at
582
+ // the newest change in its scope (never below the horizon, never above
583
+ // maxCommitSeq), so a stored image stays the reuse key's match until
584
+ // this scope changes (§5.3). The server computes the pin; a client
585
+ // token never supplies the reuse key.
586
+ const [firstPage, latest] = await Promise.all([
587
+ scan,
588
+ ctx.storage.latestScopedChangeSeq(ctx.partition, {
589
+ table: plan.table.name,
542
590
  scopeFilter: plan.effective,
543
- afterSeq: sub.cursor,
544
- throughSeq: maxSeq,
545
- limitChanges: limits.limitCommits + 1,
546
591
  }),
547
- };
548
- }
549
- // §4.6 reset: no reads.
550
- if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
551
- return {};
552
- }
553
- // Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
554
- const resume =
555
- token !== undefined && token.asOfCommitSeq >= horizonSeq
556
- ? token
557
- : undefined;
558
- const startCursor = resume?.rowCursor ?? null;
559
- const firstPage = ctx.storage.scanRows(ctx.partition, {
560
- table: plan.table.name,
561
- scopeFilter: plan.effective,
562
- afterRowId: startCursor,
563
- limit: limits.limitSnapshotRows + 1,
564
- });
565
- if (!sqliteLaneEligible(limits, plan, startCursor)) return { firstPage };
566
- // §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
567
- // scope (never below the horizon, never above maxCommitSeq), so a stored
568
- // image stays the reuse key's match until this scope changes (§5.3).
569
- const pinned =
570
- resume !== undefined
571
- ? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
572
- : ctx.storage
573
- .latestScopedChangeSeq(ctx.partition, {
574
- table: plan.table.name,
575
- scopeFilter: plan.effective,
576
- })
577
- .then((latest) => {
578
- const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
579
- return { asOf: pin, pin };
580
- });
581
- const lane = pinned.then(async ({ asOf, pin }) => {
592
+ ]);
593
+ const asOf = Math.min(maxSeq, Math.max(latest, horizonSeq));
582
594
  // §5.3 eligibility: a stored image only matches a snapshot larger than
583
595
  // one rows page, so the probe decides before the reuse lookup; a remote
584
596
  // `find` on a small table would be a wasted round trip.
585
- const probe = await firstPage;
586
- if (probe.length <= limits.limitSnapshotRows) {
587
- return { pin, image: undefined };
588
- }
589
- return {
590
- pin,
591
- image: await ctx.segments.find(
592
- {
593
- partition: ctx.partition,
594
- logEpoch,
595
- table: plan.table.name,
596
- schemaVersion: schema.version,
597
- mediaType: 'sqlite',
598
- scopeDigest: await scopeDigest(plan.effective),
599
- asOfCommitSeq: asOf,
600
- },
601
- clockOf(ctx)(),
602
- ),
603
- };
604
- });
605
- return { firstPage, lane };
606
- });
607
- const settled = await Promise.all(
608
- reads.map(async (read) => ({
609
- commits: read.commits === undefined ? undefined : await read.commits,
610
- firstPage: await read.firstPage,
611
- lane: await read.lane,
612
- })),
597
+ const image =
598
+ firstPage.length <= limits.limitSnapshotRows
599
+ ? undefined
600
+ : await ctx.segments.find(
601
+ {
602
+ partition: ctx.partition,
603
+ logEpoch,
604
+ table: plan.table.name,
605
+ schemaVersion: schema.version,
606
+ mediaType: 'sqlite',
607
+ scopeDigest: await scopeDigest(plan.effective),
608
+ asOfCommitSeq: asOf,
609
+ },
610
+ clockOf(ctx)(),
611
+ );
612
+ return { kind: 'bootstrap', asOf, startCursor, firstPage, image };
613
+ }),
613
614
  );
614
615
  // Validate continuity before committing to an active section. A prune
615
616
  // during a paged read can otherwise make an incomplete window look empty.
616
- const windowHorizon = settled.some((read) => read.commits !== undefined)
617
- ? Math.max(horizonSeq, await ctx.storage.getHorizonSeq(ctx.partition))
618
- : horizonSeq;
619
- return settled.map((read) => ({
620
- commits: read.commits ?? [],
621
- horizonSeq: read.commits === undefined ? horizonSeq : windowHorizon,
622
- firstPage: read.firstPage,
623
- pin: read.lane?.pin,
624
- image: read.lane?.image,
625
- }));
617
+ if (!prepared.some((section) => section.kind === 'incremental')) {
618
+ return prepared;
619
+ }
620
+ const windowHorizon = await ctx.storage.getHorizonSeq(ctx.partition);
621
+ return prepared.map((section, index) =>
622
+ section.kind === 'incremental' &&
623
+ (plans[index]?.frame.cursor ?? -1) < windowHorizon
624
+ ? { kind: 'reset' }
625
+ : section,
626
+ );
626
627
  }
627
628
 
628
629
  /**
@@ -680,7 +681,7 @@ export async function* preparedSection(
680
681
  throw new Error('subscriptionSection requires a non-empty log epoch');
681
682
  }
682
683
 
683
- if (plan.status === 'revoked') {
684
+ if (prepared.kind === 'revoked') {
684
685
  yield {
685
686
  type: 'SUB_START',
686
687
  id: sub.id,
@@ -693,12 +694,9 @@ export async function* preparedSection(
693
694
  return { nextCursor: sub.cursor, active: false };
694
695
  }
695
696
 
696
- const token = parseBootstrapToken(sub.bootstrapState, sub.table);
697
- const { commits, horizonSeq } = prepared;
698
-
699
697
  // §4.6: a cursor behind the horizon (and not resuming a bootstrap)
700
698
  // cannot compute deltas — answer `reset` and echo the cursor.
701
- if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
699
+ if (prepared.kind === 'reset') {
702
700
  yield {
703
701
  type: 'SUB_START',
704
702
  id: sub.id,
@@ -711,21 +709,8 @@ export async function* preparedSection(
711
709
  return { nextCursor: sub.cursor, active: false };
712
710
  }
713
711
 
714
- const bootstrapping =
715
- token !== undefined || sub.cursor < 0 || sub.cursor > maxSeq;
716
-
717
- if (bootstrapping) {
718
- // §4.7: resume at the pinned point unless the pin fell behind the
719
- // horizon (or the token is unusable) — then restart from a fresh pin.
720
- const resume =
721
- token !== undefined && token.asOfCommitSeq >= horizonSeq
722
- ? token
723
- : undefined;
724
- const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
725
- const startCursor = resume?.rowCursor ?? null;
726
- if (prepared.firstPage === undefined) {
727
- throw new Error('prepareSections did not read the bootstrap page');
728
- }
712
+ if (prepared.kind === 'bootstrap') {
713
+ const { asOf } = prepared;
729
714
  yield {
730
715
  type: 'SUB_START',
731
716
  id: sub.id,
@@ -740,7 +725,7 @@ export async function* preparedSection(
740
725
  limits,
741
726
  plan,
742
727
  asOf,
743
- startCursor,
728
+ prepared.startCursor,
744
729
  trace,
745
730
  logEpoch,
746
731
  prepared.firstPage,
@@ -777,6 +762,7 @@ export async function* preparedSection(
777
762
  let delivered = 0;
778
763
  let deliveredCommits = 0;
779
764
  let lastDeliveredSeq = sub.cursor;
765
+ const { commits } = prepared;
780
766
  for (const commit of commits) {
781
767
  if (
782
768
  delivered > 0 &&