@syncular/server 0.95.0 → 0.97.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +6 -5
  2. package/dist/context.d.ts +4 -16
  3. package/dist/context.js +0 -3
  4. package/dist/d1-storage.js +1 -1
  5. package/dist/errors.d.ts +11 -0
  6. package/dist/errors.js +36 -0
  7. package/dist/frame-bytes.js +1 -0
  8. package/dist/handler.d.ts +20 -2
  9. package/dist/handler.js +199 -59
  10. package/dist/postgres-storage.d.ts +3 -2
  11. package/dist/postgres-storage.js +18 -2
  12. package/dist/pull.d.ts +2 -2
  13. package/dist/pull.js +22 -16
  14. package/dist/push.js +41 -77
  15. package/dist/realtime.d.ts +5 -11
  16. package/dist/realtime.js +10 -23
  17. package/dist/relational-rows.d.ts +23 -13
  18. package/dist/relational-rows.js +49 -30
  19. package/dist/schema-window.js +2 -2
  20. package/dist/schema.d.ts +2 -2
  21. package/dist/schema.js +2 -2
  22. package/dist/segment-download.js +11 -4
  23. package/dist/sqlite-dialect.d.ts +3 -1
  24. package/dist/sqlite-dialect.js +37 -6
  25. package/dist/storage-query.d.ts +5 -2
  26. package/dist/storage-query.js +7 -4
  27. package/dist/storage.d.ts +12 -0
  28. package/dist/validate.d.ts +21 -39
  29. package/dist/validate.js +4 -3
  30. package/package.json +2 -2
  31. package/src/context.ts +4 -23
  32. package/src/d1-storage.ts +0 -1
  33. package/src/errors.ts +46 -0
  34. package/src/frame-bytes.ts +1 -0
  35. package/src/handler.ts +257 -73
  36. package/src/postgres-storage.ts +32 -6
  37. package/src/pull.ts +50 -35
  38. package/src/push.ts +51 -129
  39. package/src/realtime.ts +10 -22
  40. package/src/relational-rows.ts +51 -30
  41. package/src/schema-window.ts +2 -2
  42. package/src/schema.ts +4 -4
  43. package/src/segment-download.ts +19 -12
  44. package/src/sqlite-dialect.ts +47 -7
  45. package/src/storage-query.ts +7 -4
  46. package/src/storage.ts +13 -0
  47. package/src/validate.ts +21 -48
@@ -675,12 +675,22 @@ export function indexRowPageStatement(
675
675
 
676
676
  /**
677
677
  * One-round-trip page scan for `scanRows`: candidates from the inverted
678
- * scope index (ordered + LIMITed at the covering `sync_row_scopes` PK —
679
- * exactly the old candidate query, so the index-first posture is unchanged)
680
- * LEFT JOINed to the row table, so a whole page arrives in ONE statement
681
- * instead of one lookup per candidate. That per-candidate lookup was the
682
- * cold-bootstrap hot path: a 100k-row snapshot on Postgres paid ~100k
683
- * network round-trips (~10 s at 0.1 ms each) before this join.
678
+ * scope index LEFT JOINed to the row table, so a whole page arrives in ONE
679
+ * statement instead of one lookup per candidate. That per-candidate lookup
680
+ * was the cold-bootstrap hot path: a 100k-row snapshot on Postgres paid
681
+ * ~100k network round-trips (~10 s at 0.1 ms each) before this join.
682
+ *
683
+ * The candidates read at most `limit` entries of the covering
684
+ * `sync_row_scopes` PK per scope value. A bare `value IN (...)` (or
685
+ * `= ANY`) predicate gives up the PK's `row_id` order whenever the planner
686
+ * cannot prove a single value, and then every entry after the cursor is read
687
+ * and sorted before the LIMIT: a cost that grows with the entries left, so a
688
+ * bootstrap is quadratic. PostgreSQL reads one ordered, LIMITed range per
689
+ * value through a LATERAL join and merges them. SQLite has no LATERAL; it
690
+ * bounds the ranges from above instead: the smallest of the values' `limit`th
691
+ * entries after the cursor. That value alone has `limit` entries up to its
692
+ * bound, so the page ends there; a value with fewer entries has no bound, and
693
+ * when no value has one the empty blob `x''` sorts after every TEXT row id.
684
694
  *
685
695
  * LEFT JOIN (not INNER): a candidate whose row vanished (index entry
686
696
  * without a row) must still reach the caller — its `row_id` advances the
@@ -688,11 +698,9 @@ export function indexRowPageStatement(
688
698
  * disappearing from the page.
689
699
  *
690
700
  * Bind order:
691
- * - postgres: [partition, tbl, var, ...values, afterRowId, limit]
692
- * (the join reuses `$1` for the partition);
701
+ * - postgres: [partition, tbl, var, ...values, afterRowId, limit];
693
702
  * - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
694
- * limit, partition] (positional `?`; the partition binds again for the
695
- * join). The values expand through `json_each`, so the statement text and
703
+ * limit]. The values expand through `json_each`, so the statement text and
696
704
  * parameter count do not depend on `valueCount` and D1's 100-parameter
697
705
  * cap is unreachable through scope values.
698
706
  */
@@ -701,21 +709,26 @@ export function scanRowPageSql(
701
709
  valueCount: number,
702
710
  dialect: RelationalDialect,
703
711
  ): string {
704
- const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
705
- const values =
712
+ const after = dialect === 'sqlite' ? '?5' : `$${4 + valueCount}`;
713
+ const limit = dialect === 'sqlite' ? '?6' : `$${5 + valueCount}`;
714
+ const candidates =
706
715
  dialect === 'sqlite'
707
- ? 'SELECT value FROM json_each(?)'
708
- : Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
709
- const after = p(4 + valueCount);
710
- const limit = p(5 + valueCount);
711
- const joinPartition = dialect === 'sqlite' ? '?' : '$1';
716
+ ? `SELECT DISTINCT row_id FROM sync_row_scopes
717
+ WHERE partition=?1 AND tbl=?2 AND var=?3 AND value IN (SELECT value FROM json_each(?4))
718
+ AND row_id>?5
719
+ AND row_id<=coalesce((SELECT min((SELECT row_id FROM sync_row_scopes
720
+ WHERE partition=?1 AND tbl=?2 AND var=?3 AND value=v.value AND row_id>?5
721
+ ORDER BY row_id LIMIT 1 OFFSET ?6-1)) FROM json_each(?4) v), x'')
722
+ ORDER BY row_id LIMIT ?6`
723
+ : `SELECT DISTINCT x.row_id FROM (VALUES ${Array.from({ length: valueCount }, (_, i) => `($${4 + i})`).join(',')}) v(value)
724
+ CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
725
+ WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=v.value AND row_id>${after}
726
+ ORDER BY row_id LIMIT ${limit}) x
727
+ ORDER BY x.row_id LIMIT ${limit}`;
712
728
  return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
713
- FROM (SELECT DISTINCT row_id FROM sync_row_scopes
714
- WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
715
- AND row_id>${after}
716
- ORDER BY row_id LIMIT ${limit}) c
729
+ FROM (${candidates}) c
717
730
  LEFT JOIN ${quoteIdent(table.name)} r
718
- ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${joinPartition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
731
+ ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${dialect === 'sqlite' ? '?1' : '$1'} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
719
732
  ORDER BY c.row_id`;
720
733
  }
721
734
 
@@ -774,7 +787,7 @@ export function scanProbeSql(dialect: RelationalDialect): string {
774
787
  * PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
775
788
  * page of every subscription scanning any table in a pull. `$2` is a JSON
776
789
  * array of `{i, tbl, var, vals, after, lim}` records; each record runs the
777
- * same index-first candidate subquery through a LATERAL join, so every
790
+ * same per-value candidate ranges through a LATERAL join, so every
778
791
  * subscription keeps its own keyset cursor and LIMIT. The candidate side is
779
792
  * table-agnostic (a spec names its table), so the statement joins the row
780
793
  * table once per table present in the batch through a UNION ALL branch that
@@ -800,10 +813,11 @@ export function scanRowBatchSql(tables: readonly CompiledTable[]): string {
800
813
  return `WITH cand AS (
801
814
  SELECT s.i AS sub, c.row_id AS row_id, s.tbl AS tbl
802
815
  FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
803
- CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM sync_row_scopes
804
- WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
805
- AND row_id>s.after
806
- ORDER BY row_id LIMIT s.lim) c)
816
+ CROSS JOIN LATERAL (SELECT DISTINCT x.row_id FROM unnest(s.vals) v(value)
817
+ CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
818
+ WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value AND row_id>s.after
819
+ ORDER BY row_id LIMIT s.lim) x
820
+ ORDER BY x.row_id LIMIT s.lim) c)
807
821
  ${projection}
808
822
  ORDER BY sub, row_id`;
809
823
  }
@@ -1010,8 +1024,10 @@ export function assertStoredScopePatterns(
1010
1024
 
1011
1025
  /**
1012
1026
  * §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
1013
- * entries and tombstones, and raise every partition's horizon past the
1014
- * changes logged under the previous patterns. Empty when nothing re-keys.
1027
+ * entries and tombstones. The re-key takes a commit sequence number of its
1028
+ * own in every partition and records it as each re-keyed table's horizon
1029
+ * (§4.6), past the changes logged under the previous patterns. Empty when
1030
+ * nothing re-keys.
1015
1031
  */
1016
1032
  export function rekeyCleanupSql(tables: readonly string[]): string[] {
1017
1033
  if (tables.length === 0) return [];
@@ -1019,7 +1035,12 @@ export function rekeyCleanupSql(tables: readonly string[]): string[] {
1019
1035
  return [
1020
1036
  `DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
1021
1037
  `DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
1022
- 'UPDATE sync_partitions SET horizon_seq=max_commit_seq WHERE horizon_seq<max_commit_seq',
1038
+ 'UPDATE sync_partitions SET max_commit_seq=max_commit_seq+1',
1039
+ ...names.map(
1040
+ (name) => `INSERT INTO sync_table_horizons(partition, tbl, commit_seq)
1041
+ SELECT partition, ${name}, max_commit_seq FROM sync_partitions WHERE true
1042
+ ON CONFLICT(partition, tbl) DO UPDATE SET commit_seq=excluded.commit_seq`,
1043
+ ),
1023
1044
  ];
1024
1045
  }
1025
1046
 
@@ -31,7 +31,7 @@ export function schemaWindowOf(
31
31
  if (
32
32
  next === undefined ||
33
33
  table.columns.length > next.columns.length ||
34
- // §6.12: a version may mark an existing column server-owned; the flag
34
+ // §6.7: a version may mark an existing column server-owned; the flag
35
35
  // never reaches the wire.
36
36
  table.columns.some(
37
37
  (column, i) =>
@@ -42,7 +42,7 @@ export function schemaWindowOf(
42
42
  .slice(table.columns.length)
43
43
  .some((column) => !column.nullable) ||
44
44
  table.primaryKeyIndex !== next.primaryKeyIndex ||
45
- // §6.12: a pattern over an appended server-owned column is new to the
45
+ // §6.7: a pattern over an appended server-owned column is new to the
46
46
  // current version; an older client neither holds nor writes it.
47
47
  JSON.stringify(table.scopePatterns) !==
48
48
  JSON.stringify(
package/src/schema.ts CHANGED
@@ -154,8 +154,8 @@ export interface CompiledTable {
154
154
  * image copies ciphertext wholesale with no per-row decrypt pass, so an
155
155
  * encrypted table MUST be served via the rows lane. Empty ⇒ no restriction. */
156
156
  readonly encryptedColumnIndices: readonly number[];
157
- /** §6.12: column indices marked `serverOwned`. Empty ⇒ no server column
158
- * hook output is accepted for the table. */
157
+ /** §6.7: column indices marked `serverOwned`. Empty ⇒ no server column
158
+ * values are accepted from the table's validator. */
159
159
  readonly serverOwnedColumnIndices: readonly number[];
160
160
  }
161
161
 
@@ -404,7 +404,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
404
404
  if (column.type === 'blob_ref') blobRefColumnIndices.push(index);
405
405
  // §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
406
406
  if (column.encrypted === true) encryptedColumnIndices.push(index);
407
- // §6.12: the hook never changes who may write the row (owner set), and
407
+ // §6.7: a validator never changes who may write the row (owner set), and
408
408
  // a client's optimistic row holds NULL until the server row arrives.
409
409
  if (column.serverOwned === true) {
410
410
  if (
@@ -419,7 +419,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
419
419
  )
420
420
  ) {
421
421
  throw new Error(
422
- `table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.12)`,
422
+ `table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.7)`,
423
423
  );
424
424
  }
425
425
  serverOwnedColumnIndices.push(index);
@@ -15,7 +15,7 @@ import {
15
15
  RESOLVER_OUTAGE,
16
16
  touchAuthenticatedPartition,
17
17
  } from './context';
18
- import { SyncError, syncError } from './errors';
18
+ import { SyncError, segmentStoreFailure, syncError } from './errors';
19
19
  import { emitEvent } from './events';
20
20
  import { compileSchema } from './schema';
21
21
  import { schemaWindowOf } from './schema-window';
@@ -158,7 +158,9 @@ async function downloadSegment(
158
158
  request: SegmentDownloadRequest,
159
159
  ): Promise<SegmentDownloadResult> {
160
160
  const registry = await touchAuthenticatedPartition(ctx);
161
- const entry = await ctx.segments.get(request.segmentId);
161
+ const entry = await ctx.segments
162
+ .get(request.segmentId)
163
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'get'));
162
164
  if (entry === undefined) {
163
165
  throw syncError('sync.not_found', 'unknown segment (§5.5)');
164
166
  }
@@ -178,16 +180,21 @@ async function openSegment(
178
180
  const registry = await touchAuthenticatedPartition(ctx);
179
181
  const opened =
180
182
  ctx.segments.open !== undefined
181
- ? await ctx.segments.open(request.segmentId)
182
- : await ctx.segments.get(request.segmentId).then((entry) =>
183
- entry === undefined
184
- ? undefined
185
- : {
186
- record: entry.record,
187
- body: bytesStream(entry.bytes),
188
- byteLength: entry.bytes.length,
189
- },
190
- );
183
+ ? await ctx.segments
184
+ .open(request.segmentId)
185
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'open'))
186
+ : await ctx.segments
187
+ .get(request.segmentId)
188
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'get'))
189
+ .then((entry) =>
190
+ entry === undefined
191
+ ? undefined
192
+ : {
193
+ record: entry.record,
194
+ body: bytesStream(entry.bytes),
195
+ byteLength: entry.bytes.length,
196
+ },
197
+ );
191
198
  if (opened === undefined) {
192
199
  throw syncError('sync.not_found', 'unknown segment (§5.5)');
193
200
  }
@@ -46,8 +46,10 @@ import type { CompiledTable } from './schema';
46
46
  import { matchesEffective } from './scopes';
47
47
  import type { SqliteValue } from './sqlite-driver';
48
48
  import {
49
+ asJson,
49
50
  type ClientRowRecord,
50
51
  CLIENT_RECORD_COLUMNS,
52
+ type PartitionHorizons,
51
53
  PARTITION_TOUCH_INTERVAL_MS,
52
54
  pushResultFromJson,
53
55
  pushResultToJson,
@@ -257,6 +259,20 @@ const SQLITE_ADDED_COLUMNS = [
257
259
  ['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
258
260
  ] as const;
259
261
 
262
+ /**
263
+ * Core tables that `sqliteUpgradeDdl` creates, keyed by one of their columns,
264
+ * so a D1 database at an unchanged schema version gains them at its first
265
+ * open. `sync_table_horizons` holds the §4.6 table horizons.
266
+ */
267
+ const SQLITE_ADDED_TABLES = [
268
+ [
269
+ 'sync_table_horizons',
270
+ 'commit_seq',
271
+ // One line: D1's `exec` runs each line as a statement.
272
+ 'CREATE TABLE IF NOT EXISTS sync_table_horizons(partition TEXT NOT NULL, tbl TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(partition, tbl))',
273
+ ],
274
+ ] as const;
275
+
260
276
  /** Core-table columns wire version 4 deleted: the version 3 subscription list. */
261
277
  const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']] as const;
262
278
 
@@ -294,9 +310,11 @@ export const SQLITE_UPGRADE_PROBE: SqliteStatementSpec = {
294
310
  params: [
295
311
  JSON.stringify([
296
312
  ...new Set(
297
- [...SQLITE_ADDED_COLUMNS, ...SQLITE_DROPPED_COLUMNS].map(
298
- ([table]) => table,
299
- ),
313
+ [
314
+ ...SQLITE_ADDED_COLUMNS,
315
+ ...SQLITE_DROPPED_COLUMNS,
316
+ ...SQLITE_ADDED_TABLES,
317
+ ].map(([table]) => table),
300
318
  ),
301
319
  ]),
302
320
  JSON.stringify([
@@ -309,7 +327,7 @@ export const SQLITE_UPGRADE_PROBE: SqliteStatementSpec = {
309
327
  /**
310
328
  * The statements that bring the core tables of a database from an earlier
311
329
  * release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
312
- * columns, dropped columns, dropped indexes, then missing indexes. Empty for
330
+ * tables and columns, dropped columns, dropped indexes, then missing indexes. Empty for
313
331
  * a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
314
332
  * construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
315
333
  * and the first open of a database at the current schema version.
@@ -320,6 +338,9 @@ export function sqliteUpgradeDdl(
320
338
  const has = (table: string, name: string) =>
321
339
  present.some((row) => row.tbl === table && row.name === name);
322
340
  return [
341
+ ...SQLITE_ADDED_TABLES.filter(([table, name]) => !has(table, name)).map(
342
+ ([, , ddl]) => ddl,
343
+ ),
323
344
  ...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(
324
345
  ([table, name, definition]) =>
325
346
  `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`,
@@ -605,6 +626,20 @@ export abstract class SqliteFamilyStorage {
605
626
  return row?.horizon_seq ?? 0;
606
627
  }
607
628
 
629
+ async getHorizons(partition: string): Promise<PartitionHorizons> {
630
+ const [row] = await this.all<{ seq: number; tables: string }>(
631
+ `SELECT coalesce((SELECT horizon_seq FROM sync_partitions WHERE partition=?),0) AS seq,
632
+ (SELECT json_group_object(tbl, commit_seq) FROM sync_table_horizons WHERE partition=?) AS tables`,
633
+ [partition, partition],
634
+ );
635
+ return {
636
+ seq: row?.seq ?? 0,
637
+ tables: new Map(
638
+ Object.entries(asJson<Record<string, number>>(row?.tables ?? '{}')),
639
+ ),
640
+ };
641
+ }
642
+
608
643
  async setHorizonSeq(partition: string, seq: number): Promise<void> {
609
644
  await this.all(
610
645
  `INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
@@ -649,8 +684,14 @@ export abstract class SqliteFamilyStorage {
649
684
  AND ${epochGuard}`,
650
685
  params: [partition, partition, partition, query.logEpoch],
651
686
  },
652
- // §4.6: tombstones prune in the same pass as the commit log.
653
- ...['sync_commits', 'sync_changes', 'sync_tombstones'].map((table) => ({
687
+ // §4.6: tombstones and table horizons prune in the same pass as the
688
+ // commit log.
689
+ ...[
690
+ 'sync_commits',
691
+ 'sync_changes',
692
+ 'sync_tombstones',
693
+ 'sync_table_horizons',
694
+ ].map((table) => ({
654
695
  sql: `DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`,
655
696
  params: [partition, partition, partition, query.logEpoch],
656
697
  })),
@@ -1120,7 +1161,6 @@ export abstract class SqliteFamilyStorage {
1120
1161
  JSON.stringify(values),
1121
1162
  afterRowId,
1122
1163
  batchSize,
1123
- partition,
1124
1164
  ]);
1125
1165
  if (records.length === 0) break;
1126
1166
  for (const record of records) {
@@ -35,8 +35,11 @@ export interface ScanProbe {
35
35
  *
36
36
  * A single-variable filter needs no probe. Otherwise `count` counts each
37
37
  * variable's entries in one statement, up to `cap`. The cap starts at
38
- * max(64, limit) and grows fourfold, to at most 65,536, while the fewest
39
- * count is shared by several variables at the cap.
38
+ * max(64, limit) and grows fourfold while the fewest count is shared by
39
+ * several variables at the cap. A round runs only while every tied variable
40
+ * has at least `cap` entries, so the probe ends at the first cap above the
41
+ * smallest count `n`: at most 4n entries per variable in the last round, and
42
+ * the pick is the variable with the fewest entries whatever the starting cap.
40
43
  *
41
44
  * `chosen` memoizes the pick per (partition, table, filter), at most 1,024
42
45
  * entries, so the page loop of a bootstrap probes once. A stale pick only
@@ -78,7 +81,7 @@ export async function chooseScanVariable(
78
81
  const cached = chosen.get(key);
79
82
  if (cached !== undefined) return cached;
80
83
  let tied = variables;
81
- for (let cap = Math.max(64, query.limit); ; cap = Math.min(cap * 4, 65_536)) {
84
+ for (let cap = Math.max(64, query.limit); ; cap *= 4) {
82
85
  const counts = tied.map(() => 0);
83
86
  const probes = tied.map((variable, i) => ({
84
87
  i,
@@ -92,7 +95,7 @@ export async function chooseScanVariable(
92
95
  const fewest = Math.min(...counts);
93
96
  if (fewest === 0) return undefined;
94
97
  tied = tied.filter((_, i) => counts[i] === fewest);
95
- if (fewest < cap || tied.length === 1 || cap >= 65_536) break;
98
+ if (fewest < cap || tied.length === 1) break;
96
99
  }
97
100
  if (chosen.size >= 1_024) chosen.delete(chosen.keys().next().value!);
98
101
  chosen.set(key, tied[0]!);
package/src/storage.ts CHANGED
@@ -792,6 +792,17 @@ export function toClientRecord(record: ClientRowRecord): ClientRecord {
792
792
  };
793
793
  }
794
794
 
795
+ /** A partition's pruning horizon and its table horizons (§4.6). */
796
+ export interface PartitionHorizons {
797
+ /** `horizonSeq`. */
798
+ readonly seq: number;
799
+ /**
800
+ * Re-keyed table → the commit sequence number its re-key took. The changes
801
+ * of a table at or below it are never served.
802
+ */
803
+ readonly tables: ReadonlyMap<string, number>;
804
+ }
805
+
795
806
  /** One unit of the §4.5 scan set. */
796
807
  export interface LiveScanUnit {
797
808
  /** The unit's scope map; its first variable (keys sorted) is its index key. */
@@ -1313,6 +1324,8 @@ export interface ServerStorage {
1313
1324
 
1314
1325
  getMaxCommitSeq(partition: string): Promise<number>;
1315
1326
  getHorizonSeq(partition: string): Promise<number>;
1327
+ /** `horizonSeq` and the table horizons (§4.6) in one read. */
1328
+ getHorizons(partition: string): Promise<PartitionHorizons>;
1316
1329
  /** Monotonic within the current epoch; use atomic pruning for maintenance. */
1317
1330
  setHorizonSeq(partition: string, seq: number): Promise<void>;
1318
1331
  /**
package/src/validate.ts CHANGED
@@ -5,9 +5,10 @@
5
5
  * row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
6
6
  * commit transaction, once per operation. It is the seam for business
7
7
  * rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
8
- * "status ∈ {…}"). A throw (or rejected promise) rejects the whole commit
9
- * atomically (§6.4) with a host-defined code the client surfaces unchanged
10
- * in its rejection record (§6.3).
8
+ * "status ∈ {…}") and for the values of server-owned columns. A throw (or
9
+ * rejected promise) rejects the whole commit atomically (§6.4) with a
10
+ * host-defined code the client surfaces unchanged in its rejection record
11
+ * (§6.3).
11
12
  *
12
13
  * The feature is OFF by default (no `validators` on the config): the push
13
14
  * path pays only an `undefined` check per operation and builds no context
@@ -55,8 +56,9 @@ export type ValidateOpKind = 'upsert' | 'delete';
55
56
 
56
57
  /**
57
58
  * One push operation as the validator sees it (§6.7). For an `upsert`,
58
- * `row` is the post-merge, post-scope-strip row that will be written and
59
- * `stored` is the prior stored row (or `undefined` on insert). For a
59
+ * `row` is the post-merge, post-scope-strip row that will be written, with
60
+ * each server-owned column as stored (NULL on an insert), and `stored` is
61
+ * the prior stored row (or `undefined` on insert). For a
60
62
  * `delete`, `row` is `undefined` and `stored` is the row about to be
61
63
  * removed (a delete only reaches a validator when a stored row exists —
62
64
  * an absent-row delete is an idempotent no-op that never validates, §6.2).
@@ -188,7 +190,7 @@ export type CommitValidator = (
188
190
  * inside and the push never completes, and on a pool-backed executor (`pg`,
189
191
  * `Bun.sql`) it reads committed state without the staged writes, or waits
190
192
  * forever once push transactions hold every pool connection. An authority
191
- * read calls `queryAuthoritative` on this context instead.
193
+ * read goes through `read` instead.
192
194
  */
193
195
  export interface ValidateContext {
194
196
  /** Host-authenticated actor (§1.1) performing the write. */
@@ -198,17 +200,13 @@ export interface ValidateContext {
198
200
  /** The context `authenticate` returned for this request (§1.1); absent without one. */
199
201
  readonly context?: string;
200
202
  /**
201
- * Run a generated registered query on the push transaction (§6.7). The
202
- * request has the shape `ServerStorage.queryAuthoritative` takes; the
203
- * storage binds every relation to the commit's partition. SQLite and
204
- * PostgreSQL return the transaction's snapshot, including the operations
205
- * staged before this one; D1 fails with
203
+ * The §6.8 candidate-state reader of the push transaction: it sees every
204
+ * operation staged before this one. `read.queryAuthoritative` runs a
205
+ * generated registered query on that transaction; D1 fails with
206
206
  * `sync.storage.query_over_staged_writes` when the query reads a table the
207
207
  * commit has written.
208
208
  */
209
- readonly queryAuthoritative: (
210
- query: AuthoritativeQueryRequest,
211
- ) => Promise<AuthoritativeQueryResult>;
209
+ readonly read: CommitValidationReader;
212
210
  }
213
211
 
214
212
  /**
@@ -220,8 +218,15 @@ export interface ValidateContext {
220
218
  * `sync.constraint_violation` code (a validator SHOULD throw
221
219
  * `ValidationRejection` to control the code). MUST NOT mutate the row.
222
220
  *
221
+ * On an insert or update of a pushed commit or a registered command, the
222
+ * validator may return values for the table's server-owned columns; a column
223
+ * it omits keeps its value in `row`. A key that names no server-owned column,
224
+ * a value the column's type refuses, or a changed scope column on an update
225
+ * rejects the commit with `sync.constraint_violation`. The server ignores
226
+ * returned values on a delete, a rescope (§2.2) and a §6.11 cascade.
227
+ *
223
228
  * The hook runs inside the push transaction. An authority read uses
224
- * `ctx.queryAuthoritative`, which executes on that transaction; calling
229
+ * `ctx.read.queryAuthoritative`, which executes on that transaction; calling
225
230
  * `ServerStorage.queryAuthoritative` from here deadlocks on a
226
231
  * single-connection executor and reads committed state on a pool-backed one.
227
232
  * A rule over sibling operations belongs in the whole-commit
@@ -230,7 +235,7 @@ export interface ValidateContext {
230
235
  export type Validator = (
231
236
  op: ValidateOperation,
232
237
  ctx: ValidateContext,
233
- ) => void | Promise<void>;
238
+ ) => ValidateRow | void | Promise<ValidateRow | void>;
234
239
 
235
240
  /** Table name → validator (§6.7). Absent tables are unvalidated. */
236
241
  export type ValidatorRegistry = Readonly<Record<string, Validator>>;
@@ -315,35 +320,3 @@ export function toValidateRow(
315
320
  });
316
321
  return row;
317
322
  }
318
-
319
- /** §6.12: the accepted upsert a server column hook completes. `row` is the
320
- * merged row (present columns over stored values, crdt columns merged) with
321
- * every server-owned column as stored, or NULL on an insert. */
322
- export interface ServerColumnsInput {
323
- readonly table: string;
324
- readonly rowId: string;
325
- readonly row: ValidateRow;
326
- readonly stored: ValidateRow | undefined;
327
- }
328
-
329
- export interface ServerColumnsContext {
330
- readonly actorId: string;
331
- readonly partition: string;
332
- readonly context?: string;
333
- /** The §6.8 candidate-state reader of the push transaction. */
334
- readonly read: CommitValidationReader;
335
- }
336
-
337
- /**
338
- * §6.12: returns values for the table's server-owned columns; a column it
339
- * omits keeps its value in `row`. A key that names no server-owned column, a
340
- * value the column's type refuses, or a changed scope column on an update
341
- * rejects the commit with `sync.constraint_violation`, like any other throw
342
- * that is not a {@link ValidationRejection}.
343
- */
344
- export type ServerColumnsHook = (
345
- input: ServerColumnsInput,
346
- ctx: ServerColumnsContext,
347
- ) => ValidateRow | Promise<ValidateRow>;
348
-
349
- export type ServerColumnsRegistry = Readonly<Record<string, ServerColumnsHook>>;