@syncular/server 0.58.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +49 -36
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +46 -40
  4. package/dist/blob-handlers.d.ts +25 -31
  5. package/dist/blob-handlers.js +63 -41
  6. package/dist/blob-store.d.ts +42 -24
  7. package/dist/blob-store.js +0 -0
  8. package/dist/bucket-blob-store.d.ts +12 -8
  9. package/dist/bucket-blob-store.js +54 -45
  10. package/dist/bucket-segment-store.js +18 -16
  11. package/dist/content-encoding.d.ts +5 -2
  12. package/dist/content-encoding.js +42 -13
  13. package/dist/context.d.ts +19 -5
  14. package/dist/context.js +19 -1
  15. package/dist/d1-storage.d.ts +14 -98
  16. package/dist/d1-storage.js +249 -1119
  17. package/dist/errors.js +16 -0
  18. package/dist/events.d.ts +36 -18
  19. package/dist/frame-bytes.js +24 -6
  20. package/dist/handler.js +503 -208
  21. package/dist/local-authority.d.ts +2 -0
  22. package/dist/local-authority.js +24 -12
  23. package/dist/object-bucket.d.ts +2 -0
  24. package/dist/operations-realtime.js +69 -50
  25. package/dist/operations.d.ts +5 -2
  26. package/dist/operations.js +20 -29
  27. package/dist/pg-executor-pglite.d.ts +5 -6
  28. package/dist/pg-executor-pglite.js +22 -14
  29. package/dist/pg-executor.d.ts +4 -1
  30. package/dist/postgres-storage.d.ts +24 -19
  31. package/dist/postgres-storage.js +420 -262
  32. package/dist/pull.d.ts +70 -47
  33. package/dist/pull.js +270 -301
  34. package/dist/push.d.ts +12 -3
  35. package/dist/push.js +361 -173
  36. package/dist/realtime.d.ts +85 -39
  37. package/dist/realtime.js +355 -219
  38. package/dist/relational-rows.d.ts +21 -66
  39. package/dist/relational-rows.js +28 -109
  40. package/dist/s3-blob-store.d.ts +11 -12
  41. package/dist/s3-blob-store.js +20 -13
  42. package/dist/s3-object-bucket.d.ts +8 -1
  43. package/dist/s3-object-bucket.js +16 -3
  44. package/dist/schema.d.ts +14 -0
  45. package/dist/schema.js +23 -0
  46. package/dist/scopes.d.ts +6 -2
  47. package/dist/scopes.js +22 -4
  48. package/dist/seed.js +10 -16
  49. package/dist/segment-download.d.ts +4 -1
  50. package/dist/segment-download.js +22 -6
  51. package/dist/signed-url.d.ts +2 -2
  52. package/dist/signed-url.js +9 -2
  53. package/dist/sigv4.d.ts +8 -2
  54. package/dist/sigv4.js +8 -4
  55. package/dist/sqlite-blob-store.d.ts +1 -1
  56. package/dist/sqlite-blob-store.js +23 -22
  57. package/dist/sqlite-dialect.d.ts +164 -84
  58. package/dist/sqlite-dialect.js +1145 -150
  59. package/dist/sqlite-lease-store.js +13 -18
  60. package/dist/sqlite-storage.d.ts +9 -62
  61. package/dist/sqlite-storage.js +303 -1123
  62. package/dist/storage-query.d.ts +30 -1
  63. package/dist/storage-query.js +68 -0
  64. package/dist/storage.d.ts +185 -63
  65. package/dist/storage.js +15 -0
  66. package/package.json +2 -2
  67. package/src/admin.ts +99 -57
  68. package/src/blob-handlers.ts +91 -72
  69. package/src/blob-store.ts +0 -0
  70. package/src/bucket-blob-store.ts +75 -48
  71. package/src/bucket-segment-store.ts +20 -15
  72. package/src/content-encoding.ts +48 -16
  73. package/src/context.ts +33 -6
  74. package/src/d1-storage.ts +350 -1700
  75. package/src/errors.ts +16 -0
  76. package/src/events.ts +37 -18
  77. package/src/frame-bytes.ts +26 -6
  78. package/src/handler.ts +654 -261
  79. package/src/local-authority.ts +27 -14
  80. package/src/object-bucket.ts +2 -0
  81. package/src/operations-realtime.ts +85 -61
  82. package/src/operations.ts +44 -42
  83. package/src/pg-executor-pglite.ts +27 -23
  84. package/src/pg-executor.ts +5 -1
  85. package/src/postgres-storage.ts +557 -354
  86. package/src/pull.ts +406 -440
  87. package/src/push.ts +466 -220
  88. package/src/realtime.ts +447 -290
  89. package/src/relational-rows.ts +43 -121
  90. package/src/s3-blob-store.ts +23 -13
  91. package/src/s3-object-bucket.ts +20 -2
  92. package/src/schema.ts +31 -0
  93. package/src/scopes.ts +23 -3
  94. package/src/seed.ts +13 -20
  95. package/src/segment-download.ts +33 -7
  96. package/src/signed-url.ts +10 -4
  97. package/src/sigv4.ts +17 -4
  98. package/src/sqlite-blob-store.ts +27 -26
  99. package/src/sqlite-dialect.ts +1851 -199
  100. package/src/sqlite-lease-store.ts +12 -23
  101. package/src/sqlite-storage.ts +390 -1821
  102. package/src/storage-query.ts +102 -0
  103. package/src/storage.ts +215 -67
@@ -518,7 +518,7 @@ export function toSqlValue(
518
518
  column: RowColumn,
519
519
  value: RowValue,
520
520
  dialect: RelationalDialect,
521
- ): unknown {
521
+ ): RowValue {
522
522
  if (value === null) return null;
523
523
  if (column.type === 'boolean' && dialect === 'sqlite') {
524
524
  return value === true ? 1 : 0;
@@ -545,7 +545,7 @@ export function upsertValues(
545
545
  partition: string,
546
546
  row: StoredRow,
547
547
  dialect: RelationalDialect,
548
- ): unknown[] {
548
+ ): RowValue[] {
549
549
  // A non-materialized table skips the decode entirely — the upsert is a
550
550
  // six-column meta write (the old blob-store cost).
551
551
  const projection = table.materialize
@@ -609,17 +609,25 @@ export function selectRowSql(
609
609
 
610
610
  /**
611
611
  * SELECT many stored rows of one table by primary key in one statement.
612
- * Postgres binds the ids as one `text[]` (`row_id = ANY($2)`), so the
613
- * statement count does not grow with the id count. Params: [partition,
614
- * rowIds]; absent ids contribute no row.
612
+ * The ids bind as one parameter (Postgres `text[]` with `= ANY($2)`, SQLite
613
+ * a JSON array through `json_each`), so the statement count does not grow
614
+ * with the id count. Params: [partition, rowIds]; absent ids contribute no
615
+ * row.
615
616
  */
616
- export function selectRowsSql(table: CompiledTable): string {
617
- return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=ANY($2::text[])`;
617
+ export function selectRowsSql(
618
+ table: CompiledTable,
619
+ dialect: RelationalDialect,
620
+ ): string {
621
+ const [partition, rowIds] =
622
+ dialect === 'sqlite'
623
+ ? ['?', 'IN (SELECT value FROM json_each(?))']
624
+ : ['$1', '=ANY($2::text[])'];
625
+ return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${partition} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}${rowIds}`;
618
626
  }
619
627
 
620
628
  export interface IndexRowPageStatement {
621
629
  readonly sql: string;
622
- readonly params: readonly unknown[];
630
+ readonly params: readonly RowValue[];
623
631
  }
624
632
 
625
633
  /**
@@ -636,7 +644,7 @@ export function indexRowPageStatement(
636
644
  limit: number,
637
645
  dialect: RelationalDialect,
638
646
  ): IndexRowPageStatement {
639
- const params: unknown[] = [partition];
647
+ const params: RowValue[] = [partition];
640
648
  const placeholder = (): string =>
641
649
  dialect === 'sqlite' ? '?' : `$${params.length}`;
642
650
  const predicates = index.columns.map((columnName, valueIndex) => {
@@ -678,8 +686,11 @@ export function indexRowPageStatement(
678
686
  * Bind order:
679
687
  * - postgres: [partition, tbl, var, ...values, afterRowId, limit]
680
688
  * (the join reuses `$1` for the partition);
681
- * - sqlite: [partition, tbl, var, ...values, afterRowId, limit,
682
- * partition] (positional `?` — the partition binds again for the join).
689
+ * - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
690
+ * limit, partition] (positional `?`; the partition binds again for the
691
+ * join). The values expand through `json_each`, so the statement text and
692
+ * parameter count do not depend on `valueCount` and D1's 100-parameter
693
+ * cap is unreachable through scope values.
683
694
  */
684
695
  export function scanRowPageSql(
685
696
  table: CompiledTable,
@@ -687,14 +698,16 @@ export function scanRowPageSql(
687
698
  dialect: RelationalDialect,
688
699
  ): string {
689
700
  const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
690
- const values: string[] = [];
691
- for (let i = 0; i < valueCount; i++) values.push(p(4 + i));
701
+ const values =
702
+ dialect === 'sqlite'
703
+ ? 'SELECT value FROM json_each(?)'
704
+ : Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
692
705
  const after = p(4 + valueCount);
693
706
  const limit = p(5 + valueCount);
694
707
  const joinPartition = dialect === 'sqlite' ? '?' : '$1';
695
708
  return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
696
709
  FROM (SELECT DISTINCT row_id FROM sync_row_scopes
697
- WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values.join(',')})
710
+ WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
698
711
  AND row_id>${after}
699
712
  ORDER BY row_id LIMIT ${limit}) c
700
713
  LEFT JOIN ${quoteIdent(table.name)} r
@@ -702,59 +715,6 @@ export function scanRowPageSql(
702
715
  ORDER BY c.row_id`;
703
716
  }
704
717
 
705
- /**
706
- * One-round-trip page read for `readCommitWindow` (sync_* tables only; it
707
- * lives here beside `scanRowPageSql` because it is the same join-the-index
708
- * posture, dialect-parameterized the same way): candidates from the inverted
709
- * change-scope index (ordered + LIMITed at the covering `sync_change_scopes`
710
- * PK — exactly the old candidate query, so the index-first invariant of
711
- * `CommitWindowQuery` is unchanged) LEFT JOINed to the commit metadata and
712
- * to the commit's changes for the table, so a whole window page arrives in
713
- * ONE statement instead of 1 + 2×candidates round trips. This is the
714
- * incremental-pull hot path — it runs on every sync round; before the join a
715
- * 500-candidate catch-up window on Postgres paid ~1000 network round trips.
716
- *
717
- * LEFT JOIN (not INNER): a candidate whose commit vanished (scope-index
718
- * entry without a commit/changes row) must still reach the caller — its
719
- * `commit_seq` advances the window cursor — so it comes back with NULL
720
- * meta/change columns rather than disappearing from the page.
721
- *
722
- * Rows arrive ordered (commit_seq, idx): oldest-first commits, each commit's
723
- * change rows consecutive in `idx` order — the caller groups consecutive
724
- * rows and verifies the full multi-variable scope match in JS.
725
- *
726
- * Bind order:
727
- * - postgres: [partition, tbl, var, ...values, afterSeq, throughSeq,
728
- * limit] (the joins reuse `$1`/`$2`);
729
- * - sqlite: [partition, tbl, var, ...values, afterSeq, throughSeq,
730
- * limit, partition, partition, tbl] (positional `?` — partition/tbl bind
731
- * again for the joins).
732
- */
733
- export function commitWindowPageSql(
734
- valueCount: number,
735
- dialect: RelationalDialect,
736
- ): string {
737
- const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
738
- const values: string[] = [];
739
- for (let i = 0; i < valueCount; i++) values.push(p(4 + i));
740
- const after = p(4 + valueCount);
741
- const through = p(5 + valueCount);
742
- const limit = p(6 + valueCount);
743
- const joinPartition = dialect === 'sqlite' ? '?' : '$1';
744
- const joinTbl = dialect === 'sqlite' ? '?' : '$2';
745
- return `SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
746
- ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
747
- FROM (SELECT DISTINCT commit_seq FROM sync_change_scopes
748
- WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values.join(',')})
749
- AND commit_seq>${after} AND commit_seq<=${through}
750
- ORDER BY commit_seq LIMIT ${limit}) c
751
- LEFT JOIN sync_commits m
752
- ON m.partition=${joinPartition} AND m.commit_seq=c.commit_seq
753
- LEFT JOIN sync_changes ch
754
- ON ch.partition=${joinPartition} AND ch.commit_seq=c.commit_seq AND ch.tbl=${joinTbl}
755
- ORDER BY c.commit_seq, ch.idx`;
756
- }
757
-
758
718
  /**
759
719
  * PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
760
720
  * page of every subscription scanning any table in a pull. `$2` is a JSON
@@ -793,29 +753,6 @@ export function scanRowBatchSql(tables: readonly CompiledTable[]): string {
793
753
  ORDER BY sub, row_id`;
794
754
  }
795
755
 
796
- /**
797
- * PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
798
- * the window page of every subscription in a pull, across all tables. `$2`
799
- * is a JSON array of `{i, tbl, var, vals, after, through, lim}` records, each
800
- * running the same index-first candidate subquery through a LATERAL join.
801
- * Rows arrive ordered by (i, commit_seq, idx).
802
- *
803
- * Params: [partition, specs (JSON text)], the specs bound as text like
804
- * `scanRowBatchSql`.
805
- */
806
- export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
807
- ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
808
- FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after bigint, through bigint, lim int)
809
- CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
810
- WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
811
- AND commit_seq>s.after AND commit_seq<=s.through
812
- ORDER BY commit_seq LIMIT s.lim) c
813
- LEFT JOIN sync_commits m
814
- ON m.partition=$1 AND m.commit_seq=c.commit_seq
815
- LEFT JOIN sync_changes ch
816
- ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=s.tbl
817
- ORDER BY s.i, c.commit_seq, ch.idx`;
818
-
819
756
  /**
820
757
  * PostgreSQL push read-ahead across every table a commit targets, in one
821
758
  * statement: each requested row id with its stored row (NULL columns when
@@ -834,7 +771,7 @@ export function prefetchRowsBatchSql(
834
771
  const projection = tables
835
772
  .map((table) => {
836
773
  const quoted = quoteIdent(table.name);
837
- return `SELECT ids.tbl AS tbl, ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq
774
+ return `SELECT ids.tbl AS tbl, ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq, t.server_version AS tombstone_version, t.scopes AS tombstone_scopes
838
775
  FROM ids
839
776
  LEFT JOIN ${quoted} r
840
777
  ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${params.partition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=ids.id
@@ -852,38 +789,23 @@ export function prefetchRowsBatchSql(
852
789
  }
853
790
 
854
791
  /**
855
- * SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
856
- * one variable's values, as one index-only max per value (a `(partition,
857
- * tbl, var, value)` prefix), so the cost is independent of the table's
858
- * history. The values arrive as one JSON array, so the statement has no
859
- * compound SELECT and four bound parameters for any value count (D1 caps a
860
- * compound SELECT at 5 terms and a statement at 100 parameters).
861
- * Params: [partition, tbl, var, values as JSON array text].
862
- */
863
- export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commit_seq) FROM sync_change_scopes
864
- WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq
865
- FROM json_each(?) AS v`;
866
-
867
- /**
868
- * PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
869
- * subscriptions of a pull, across all tables. `$2` is a JSON array of
870
- * `{i, tbl, var, vals}` records. Params: [partition, specs (JSON text)].
792
+ * SELECT the rows that reference one blob with their stored scope maps, in
793
+ * one statement for any number of references: the by-blob index range
794
+ * (§5.9.4) supplies the candidates and a `CASE` on the candidate's table
795
+ * name looks each scope map up by primary key in that table's own relation.
796
+ * A reference whose table left the schema or whose row is gone has NULL
797
+ * `scopes`. Needs at least one table. Params: [partition, blobId].
871
798
  */
872
- export const LATEST_SCOPED_CHANGE_BATCH_SQL = `SELECT s.i AS sub,
873
- (SELECT max(latest.seq) FROM (
874
- SELECT (SELECT max(commit_seq) FROM sync_change_scopes
875
- WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value) AS seq
876
- FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq
877
- FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)
878
- ORDER BY s.i`;
879
-
880
- /** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
881
- export function selectRowScopesSql(
882
- table: CompiledTable,
799
+ export function rowsReferencingBlobSql(
800
+ tables: Iterable<CompiledTable>,
883
801
  dialect: RelationalDialect,
884
802
  ): string {
885
803
  const p = dialect === 'sqlite' ? ['?', '?'] : ['$1', '$2'];
886
- return `SELECT ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${p[0]} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${p[1]}`;
804
+ const branches = [...tables].map(
805
+ (table) =>
806
+ `WHEN '${table.name.replaceAll("'", "''")}' THEN (SELECT t.${quoteIdent(SYNC_SCOPES_COLUMN)} FROM ${quoteIdent(table.name)} t WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=r.partition AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=r.row_id)`,
807
+ );
808
+ return `SELECT r.tbl AS tbl, r.row_id AS row_id, CASE r.tbl ${branches.join(' ')} END AS scopes FROM sync_blob_refs r WHERE r.partition=${p[0]} AND r.blob_id=${p[1]}`;
887
809
  }
888
810
 
889
811
  /** DELETE one stored row. Params: [partition, rowId]. */
@@ -1124,8 +1046,8 @@ export function rewriteValues(
1124
1046
  rowId: string,
1125
1047
  payload: Uint8Array,
1126
1048
  dialect: RelationalDialect,
1127
- ): unknown[] {
1128
- const out: unknown[] = [];
1049
+ ): RowValue[] {
1050
+ const out: RowValue[] = [];
1129
1051
  if (table.materialize) {
1130
1052
  const values = decodeRow(table.columns, payload);
1131
1053
  for (let i = 0; i < table.columns.length; i++) {
@@ -97,19 +97,19 @@ export class S3BlobStore extends BucketBlobStore {
97
97
 
98
98
  /**
99
99
  * SigV4 presigned PUT for a blob object (§5.9.3 direct-to-storage upload).
100
- * The blob twin of `presignBlobGet`: the signed key embeds the `blobId`, so
101
- * the grant places bytes at exactly the content-addressed key. The store
102
- * does NOT recompute the SHA-256 (it is the object store, not the sync
103
- * server); integrity is enforced at REFERENCE time — the §5.9.6 push
104
- * existence check verifies the object exists (`has`) and every consumer
105
- * re-verifies the content address over the received bytes (§5.9.5/§5.1). A
106
- * client PUTting bytes that do not hash to `{blobId}` poisons only its own
107
- * upload; no honest reference resolves to it. Issued by the upload-grant
108
- * handler after host authentication; TTL SHOULD be ≤ 15 min (default 900).
100
+ * The signed key embeds the `blobId`, and the signature binds
101
+ * `content-length` to the granted `byteLength` and `x-amz-checksum-sha256`
102
+ * to the `blobId` digest (the same SHA-256, re-encoded from lowercase hex to
103
+ * the base64 of its 32 raw bytes). The object store therefore rejects a PUT
104
+ * of any other size (signature mismatch) or any other bytes (`BadDigest`),
105
+ * so a grant can only place the content-addressed bytes at their key.
106
+ * Issued by the upload-grant handler after host authentication and the
107
+ * size-cap check; TTL SHOULD be ≤ 15 min (default 900).
109
108
  */
110
109
  async presignBlobPut(
111
110
  partition: string,
112
111
  blobId: string,
112
+ byteLength: number,
113
113
  options?: { readonly ttlSeconds?: number; readonly nowMs?: number },
114
114
  ): Promise<SegmentUrlIssue> {
115
115
  if (!BLOB_ID_PATTERN.test(blobId)) {
@@ -117,6 +117,9 @@ export class S3BlobStore extends BucketBlobStore {
117
117
  }
118
118
  const ttlSeconds = options?.ttlSeconds ?? 900;
119
119
  const nowMs = options?.nowMs ?? Date.now();
120
+ const digest = (blobId.slice('sha256:'.length).match(/../g) ?? []).map(
121
+ (hex) => Number.parseInt(hex, 16),
122
+ );
120
123
  const url = await presignUrl({
121
124
  method: 'PUT',
122
125
  url: this.#bucket.urlFor(this.objectKeyFor(partition, blobId)),
@@ -124,6 +127,10 @@ export class S3BlobStore extends BucketBlobStore {
124
127
  credentials: this.#credentials,
125
128
  nowMs,
126
129
  expiresSeconds: ttlSeconds,
130
+ headers: {
131
+ 'content-length': String(byteLength),
132
+ 'x-amz-checksum-sha256': btoa(String.fromCharCode(...digest)),
133
+ },
127
134
  });
128
135
  return {
129
136
  url: url.toString(),
@@ -158,8 +165,8 @@ export function s3PresignedBlobUrls(
158
165
  * `blobUploadUrls: s3PresignedBlobUploads(blobStore, { ttlSeconds: 900 })`.
159
166
  * The upload-grant handler issues the presigned PUT only after host
160
167
  * authentication + the size-cap check; the client PUTs bytes straight to the
161
- * object store with no host auth (§5.9.3). Integrity stays the content-address
162
- * check at reference/download time — never a store-side hash recompute.
168
+ * object store with no host auth, and the store verifies the signed
169
+ * `content-length` and `x-amz-checksum-sha256` (§5.9.3).
163
170
  */
164
171
  export function s3PresignedBlobUploads(
165
172
  store: S3BlobStore,
@@ -169,7 +176,10 @@ export function s3PresignedBlobUploads(
169
176
  ...(options?.ttlSeconds !== undefined
170
177
  ? { ttlSeconds: options.ttlSeconds }
171
178
  : {}),
172
- presign: ({ partition, blobId, ttlSeconds, nowMs }) =>
173
- store.presignBlobPut(partition, blobId, { ttlSeconds, nowMs }),
179
+ presign: ({ partition, blobId, byteLength, ttlSeconds, nowMs }) =>
180
+ store.presignBlobPut(partition, blobId, byteLength, {
181
+ ttlSeconds,
182
+ nowMs,
183
+ }),
174
184
  };
175
185
  }
@@ -21,6 +21,7 @@ import {
21
21
  type SigV4Credentials,
22
22
  sha256Hex,
23
23
  signRequest,
24
+ uriEncode,
24
25
  } from './sigv4';
25
26
 
26
27
  export interface S3ObjectBucketConfig {
@@ -113,10 +114,23 @@ export class S3ObjectBucket implements ObjectBucket {
113
114
  };
114
115
  }
115
116
 
116
- /** The object URL the presign helpers sign (`{endpoint}/{bucket}/{key}`). */
117
+ /**
118
+ * The object URL the presign helpers sign (`{endpoint}/{bucket}/{key}`).
119
+ * Every `/`-separated key segment is URI-encoded the way SigV4
120
+ * canonicalizes an S3 path (RFC 3986 unreserved characters stay raw), so a
121
+ * partition id holding `#`, `?`, `%` or `:` stays inside its own key and the
122
+ * pathname is already the canonical path. A `.` or `..` segment has no URL
123
+ * form (the URL parser and S3 clients collapse it), so it is refused.
124
+ */
117
125
  urlFor(key: string): URL {
118
126
  const endpoint = this.#config.endpoint.replace(/\/+$/, '');
119
- return new URL(`${endpoint}/${this.#config.bucket}/${key}`);
127
+ const segments = key.split('/');
128
+ if (segments.some((segment) => segment === '.' || segment === '..')) {
129
+ throw new Error('S3ObjectBucket: object key has a dot segment');
130
+ }
131
+ return new URL(
132
+ `${endpoint}/${this.#config.bucket}/${segments.map((segment) => uriEncode(segment, true)).join('/')}`,
133
+ );
120
134
  }
121
135
 
122
136
  async #request(
@@ -183,9 +197,13 @@ export class S3ObjectBucket implements ObjectBucket {
183
197
  const response = await this.#request('HEAD', key);
184
198
  if (response === undefined) return undefined;
185
199
  await response.arrayBuffer();
200
+ const lastModified = Date.parse(
201
+ response.headers.get('last-modified') ?? '',
202
+ );
186
203
  return {
187
204
  etag: response.headers.get('etag') ?? undefined,
188
205
  size: sizeOf(response.headers),
206
+ lastModifiedMs: Number.isNaN(lastModified) ? undefined : lastModified,
189
207
  metadata: metadataOf(response.headers),
190
208
  };
191
209
  }
package/src/schema.ts CHANGED
@@ -161,6 +161,22 @@ export interface CompiledSchema {
161
161
 
162
162
  const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
163
163
 
164
+ /** A scope column has a type other than string or integer (§3.1). */
165
+ export class ScopeColumnTypeError extends Error {
166
+ readonly code = 'schema.scope_column_type';
167
+
168
+ constructor(
169
+ readonly details: {
170
+ readonly table: string;
171
+ readonly column: string;
172
+ readonly type: string;
173
+ },
174
+ ) {
175
+ super('a scope column must have type string or integer (§3.1)');
176
+ this.name = 'ScopeColumnTypeError';
177
+ }
178
+ }
179
+
164
180
  /**
165
181
  * Compile one `prefix:{variable}` scope pattern against the table's column
166
182
  * list, resolving the scope column's positional index for row extraction.
@@ -186,6 +202,14 @@ function compilePattern(
186
202
  `table ${table.name}: scope pattern ${JSON.stringify(pattern)} names unknown column ${JSON.stringify(column)}`,
187
203
  );
188
204
  }
205
+ const type = table.columns[index]?.type;
206
+ if (type !== 'string' && type !== 'integer') {
207
+ throw new ScopeColumnTypeError({
208
+ table: table.name,
209
+ column,
210
+ type: type ?? 'undefined',
211
+ });
212
+ }
189
213
  return { variable, prefix, column, columnIndex: index };
190
214
  }
191
215
 
@@ -409,6 +433,13 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
409
433
  `table ${child.name}: reference column ${JSON.stringify(reference.column)} names unknown table ${JSON.stringify(reference.parentTable)}`,
410
434
  );
411
435
  }
436
+ if (!child.materialize) {
437
+ // Enforcement reaches children through `scanRowsByIndex`, which reads
438
+ // the materialized columns.
439
+ throw new Error(
440
+ 'a declared reference requires a materialized child table (§6.11)',
441
+ );
442
+ }
412
443
  const childColumn = child.columns[reference.columnIndex];
413
444
  const parentKey = parent.columns[parent.primaryKeyIndex];
414
445
  if (childColumn === undefined || parentKey === undefined) {
package/src/scopes.ts CHANGED
@@ -22,22 +22,28 @@ export type EffectiveOutcome =
22
22
  | { readonly status: 'revoked' };
23
23
 
24
24
  /**
25
- * Effective = requested ∩ allowed per §3.2 rules 4–5. Inputs are assumed
25
+ * Effective = requested ∩ allowed per §3.2 rules 4–6. Inputs are assumed
26
26
  * key-validated (requested keys declared, no `'*'` requested values).
27
+ * `declared` is the table's declared variables: when a requested key passes
28
+ * through an allowed `'*'`, every declared variable the request omits and
29
+ * the actor holds a non-wildcard list for joins effective with that list
30
+ * (rule 4), so the wildcard never lifts the actor's other fences.
27
31
  */
28
32
  export function computeEffective(
33
+ declared: ReadonlySet<string>,
29
34
  requested: ScopeMap,
30
35
  resolved: ResolvedScopes,
31
36
  ): EffectiveOutcome {
32
37
  if (!resolved.ok) return { status: 'revoked' };
33
38
  const effective: ScopeMap = {};
34
- const requestedKeys = Object.keys(requested);
35
- for (const key of requestedKeys) {
39
+ let wildcard = false;
40
+ for (const key of Object.keys(requested)) {
36
41
  const requestedValues = requested[key] ?? [];
37
42
  const allowedValues = resolved.allowed[key];
38
43
  if (allowedValues === undefined) return { status: 'revoked' };
39
44
  let values: string[];
40
45
  if (allowedValues.includes('*')) {
46
+ wildcard = true;
41
47
  values = [...requestedValues];
42
48
  } else {
43
49
  values = requestedValues.filter((v) => allowedValues.includes(v));
@@ -46,6 +52,20 @@ export function computeEffective(
46
52
  effective[key] = values;
47
53
  }
48
54
  if (Object.keys(effective).length === 0) return { status: 'revoked' };
55
+ if (wildcard) {
56
+ for (const variable of declared) {
57
+ const allowedValues = resolved.allowed[variable];
58
+ if (
59
+ Object.hasOwn(requested, variable) ||
60
+ allowedValues === undefined ||
61
+ allowedValues.includes('*')
62
+ ) {
63
+ continue;
64
+ }
65
+ if (allowedValues.length === 0) return { status: 'revoked' };
66
+ effective[variable] = [...allowedValues];
67
+ }
68
+ }
49
69
  return { status: 'active', effective };
50
70
  }
51
71
 
package/src/seed.ts CHANGED
@@ -10,10 +10,11 @@
10
10
  * the seed replays the cached push result and writes nothing twice.
11
11
  */
12
12
  import {
13
- decodeMessage,
14
- encodeMessage,
13
+ decodeResponse,
14
+ encodeRequest,
15
15
  encodeSparseRow,
16
16
  PROTOCOL_WIRE_VERSION,
17
+ registrationDigest,
17
18
  type PushOperation,
18
19
  type PushResultFrame,
19
20
  type RequestFrame,
@@ -192,9 +193,9 @@ export async function seedMutations(
192
193
  partition: target.partition,
193
194
  actorId: target.actorId,
194
195
  };
195
- const acquisition = decodeMessage(
196
+ const acquisition = decodeResponse(
196
197
  await handleSyncRequest(
197
- encodeMessage({
198
+ encodeRequest({
198
199
  wireVersion: PROTOCOL_WIRE_VERSION,
199
200
  msgKind: 'request',
200
201
  frames: [
@@ -209,6 +210,9 @@ export async function seedMutations(
209
210
  limitSnapshotRows: 0,
210
211
  maxSnapshotPages: 0,
211
212
  accept: 0b0011,
213
+ cursor: -1,
214
+ registrationDigest: registrationDigest([]),
215
+ echoDigest: new Uint8Array(),
212
216
  },
213
217
  ],
214
218
  }),
@@ -216,11 +220,7 @@ export async function seedMutations(
216
220
  ),
217
221
  );
218
222
  const acquisitionHeader = acquisition.frames[0];
219
- if (
220
- acquisition.msgKind !== 'response' ||
221
- acquisitionHeader?.type !== 'RESP_HEADER' ||
222
- acquisitionHeader.logEpoch === undefined
223
- ) {
223
+ if (acquisitionHeader?.type !== 'RESP_HEADER') {
224
224
  throw new SyncError(
225
225
  'sync.invalid_request',
226
226
  'seedMutations: epoch acquisition returned an invalid response',
@@ -233,15 +233,8 @@ export async function seedMutations(
233
233
  schemaVersion: config.schema.version,
234
234
  logEpoch: acquisitionHeader.logEpoch,
235
235
  },
236
- { type: 'PUSH_COMMIT', clientCommitId, operations },
237
- // A pull that asks for nothing: the round exists for its push half.
238
- {
239
- type: 'PULL_HEADER',
240
- limitCommits: 0,
241
- limitSnapshotRows: 0,
242
- maxSnapshotPages: 0,
243
- accept: 0b0011,
244
- },
236
+ // §6.10: a database-less producer sends no outboxId and clientSeq 0.
237
+ { type: 'PUSH_COMMIT', clientCommitId, clientSeq: 0, operations },
245
238
  ];
246
239
  type SeedTerminalEvent = Extract<
247
240
  SyncularServerEvent,
@@ -260,7 +253,7 @@ export async function seedMutations(
260
253
  },
261
254
  };
262
255
  const response = await handleSyncRequest(
263
- encodeMessage({
256
+ encodeRequest({
264
257
  wireVersion: PROTOCOL_WIRE_VERSION,
265
258
  msgKind: 'request',
266
259
  frames,
@@ -275,7 +268,7 @@ export async function seedMutations(
275
268
  );
276
269
 
277
270
  // Fail loud: surface the first rejected/failed operation.
278
- const message = decodeMessage(response);
271
+ const message = decodeResponse(response);
279
272
  const result = message.frames.find(
280
273
  (frame): frame is PushResultFrame =>
281
274
  frame.type === 'PUSH_RESULT' && frame.clientCommitId === clientCommitId,
@@ -28,7 +28,10 @@ import {
28
28
 
29
29
  export interface SegmentDownloadRequest {
30
30
  readonly segmentId: string;
31
- /** The `X-Syncular-Scopes` header: canonical JSON of the requested map. */
31
+ /**
32
+ * The `X-Syncular-Scopes` header: canonical JSON of the owning unit's scope
33
+ * map, each value as a one-element list (§5.5).
34
+ */
32
35
  readonly scopesHeader: string;
33
36
  }
34
37
 
@@ -60,13 +63,15 @@ function parseScopesHeader(header: string): ScopeMap {
60
63
  }
61
64
  const scopes: ScopeMap = {};
62
65
  for (const [key, value] of Object.entries(parsed)) {
66
+ // §5.5: the owning unit's scope map, each value as a one-element list.
63
67
  if (
64
68
  !Array.isArray(value) ||
65
- value.some((v: unknown) => typeof v !== 'string')
69
+ value.length !== 1 ||
70
+ typeof value[0] !== 'string'
66
71
  ) {
67
72
  throw syncError(
68
73
  'sync.invalid_request',
69
- 'X-Syncular-Scopes values must be string lists (§0)',
74
+ 'X-Syncular-Scopes values must be one-element string lists (§5.5)',
70
75
  );
71
76
  }
72
77
  scopes[key] = value as string[];
@@ -263,15 +268,36 @@ async function authorizePublication(
263
268
  }
264
269
  resolved = { ok: false };
265
270
  }
266
- const outcome = computeEffective(requested, resolved);
267
- if (outcome.status !== 'active') {
271
+ // §3.2: the effective scopes, and so the digest, depend on the declared
272
+ // variables of the publication's table (the wildcard fence).
273
+ const digests = new Map<string, string | undefined>();
274
+ for (const publication of inContext) {
275
+ const key = `${publication.schemaVersion}\u0000${publication.table}`;
276
+ if (digests.has(key)) continue;
277
+ const table = schemaWindowOf(ctx.schema, ctx.schemaWindow)
278
+ .find((candidate) => candidate.version === publication.schemaVersion)
279
+ ?.tables.get(publication.table);
280
+ const outcome = computeEffective(
281
+ table?.declaredVariables ?? new Set(),
282
+ requested,
283
+ resolved,
284
+ );
285
+ digests.set(
286
+ key,
287
+ outcome.status === 'active'
288
+ ? await scopeDigest(outcome.effective)
289
+ : undefined,
290
+ );
291
+ }
292
+ if ([...digests.values()].every((digest) => digest === undefined)) {
268
293
  throw syncError('sync.forbidden', 'segment scopes not held (§5.5)');
269
294
  }
270
- const digest = await scopeDigest(outcome.effective);
271
295
  // The digest is the grant, but only inside the caller's partition and live
272
296
  // epoch: identical bytes published elsewhere never authorize this caller.
273
297
  const forDigest = inContext.filter(
274
- (publication) => publication.scopeDigest === digest,
298
+ (publication) =>
299
+ digests.get(`${publication.schemaVersion}\u0000${publication.table}`) ===
300
+ publication.scopeDigest,
275
301
  );
276
302
  if (forDigest.length === 0) {
277
303
  throw syncError('sync.forbidden', 'scope digest mismatch (§3.5, §5.5)');
package/src/signed-url.ts CHANGED
@@ -146,8 +146,8 @@ export async function issueBlobUrl(
146
146
  * twin of `BlobPresignConfig`: the provider signs a single PUT whose object
147
147
  * key embeds the `blobId`. Issuance happens ONLY after host authentication in
148
148
  * the upload-grant handler (uploading is host-auth-only, not scope-bearing —
149
- * §5.9.3). The `byteLength` rides through so a provider MAY bind
150
- * `Content-Length` (S3 conditional) to the granted size. Never a multipart or
149
+ * §5.9.3). The provider MUST bind the PUT to the granted `byteLength` and to
150
+ * the content address, so the object store rejects other bytes. Never a multipart or
151
151
  * chunk protocol — single PUT only, this rung and the next (§5.9.3 non-goal).
152
152
  * `S3BlobStore.presignBlobPut` (via `s3PresignedBlobUploads`) is the in-tree
153
153
  * implementation.
@@ -286,8 +286,14 @@ export async function verifySegmentToken(
286
286
  syncError('sync.forbidden', `segment token rejected: ${reason}`);
287
287
  const dot = token.indexOf('.');
288
288
  if (dot < 0) throw forbidden('malformed token');
289
- const payload = fromBase64url(token.slice(0, dot));
290
- const mac = fromBase64url(token.slice(dot + 1));
289
+ let payload: Uint8Array;
290
+ let mac: Uint8Array;
291
+ try {
292
+ payload = fromBase64url(token.slice(0, dot));
293
+ mac = fromBase64url(token.slice(dot + 1));
294
+ } catch {
295
+ throw forbidden('malformed token');
296
+ }
291
297
  const computed = await hmac(key, payload);
292
298
  if (!constantTimeEqual(mac, computed)) throw forbidden('bad MAC');
293
299
  let claims: SegmentTokenClaims;