@syncular/server 0.95.0 → 0.97.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +6 -5
  2. package/dist/context.d.ts +4 -16
  3. package/dist/context.js +0 -3
  4. package/dist/d1-storage.js +1 -1
  5. package/dist/errors.d.ts +11 -0
  6. package/dist/errors.js +36 -0
  7. package/dist/frame-bytes.js +1 -0
  8. package/dist/handler.d.ts +20 -2
  9. package/dist/handler.js +199 -59
  10. package/dist/postgres-storage.d.ts +3 -2
  11. package/dist/postgres-storage.js +18 -2
  12. package/dist/pull.d.ts +2 -2
  13. package/dist/pull.js +22 -16
  14. package/dist/push.js +41 -77
  15. package/dist/realtime.d.ts +5 -11
  16. package/dist/realtime.js +10 -23
  17. package/dist/relational-rows.d.ts +23 -13
  18. package/dist/relational-rows.js +49 -30
  19. package/dist/schema-window.js +2 -2
  20. package/dist/schema.d.ts +2 -2
  21. package/dist/schema.js +2 -2
  22. package/dist/segment-download.js +11 -4
  23. package/dist/sqlite-dialect.d.ts +3 -1
  24. package/dist/sqlite-dialect.js +37 -6
  25. package/dist/storage-query.d.ts +5 -2
  26. package/dist/storage-query.js +7 -4
  27. package/dist/storage.d.ts +12 -0
  28. package/dist/validate.d.ts +21 -39
  29. package/dist/validate.js +4 -3
  30. package/package.json +2 -2
  31. package/src/context.ts +4 -23
  32. package/src/d1-storage.ts +0 -1
  33. package/src/errors.ts +46 -0
  34. package/src/frame-bytes.ts +1 -0
  35. package/src/handler.ts +257 -73
  36. package/src/postgres-storage.ts +32 -6
  37. package/src/pull.ts +50 -35
  38. package/src/push.ts +51 -129
  39. package/src/realtime.ts +10 -22
  40. package/src/relational-rows.ts +51 -30
  41. package/src/schema-window.ts +2 -2
  42. package/src/schema.ts +4 -4
  43. package/src/segment-download.ts +19 -12
  44. package/src/sqlite-dialect.ts +47 -7
  45. package/src/storage-query.ts +7 -4
  46. package/src/storage.ts +13 -0
  47. package/src/validate.ts +21 -48
@@ -538,12 +538,22 @@ export function indexRowPageStatement(table, index, values, partition, afterRowI
538
538
  }
539
539
  /**
540
540
  * One-round-trip page scan for `scanRows`: candidates from the inverted
541
- * scope index (ordered + LIMITed at the covering `sync_row_scopes` PK —
542
- * exactly the old candidate query, so the index-first posture is unchanged)
543
- * LEFT JOINed to the row table, so a whole page arrives in ONE statement
544
- * instead of one lookup per candidate. That per-candidate lookup was the
545
- * cold-bootstrap hot path: a 100k-row snapshot on Postgres paid ~100k
546
- * network round-trips (~10 s at 0.1 ms each) before this join.
541
+ * scope index LEFT JOINed to the row table, so a whole page arrives in ONE
542
+ * statement instead of one lookup per candidate. That per-candidate lookup
543
+ * was the cold-bootstrap hot path: a 100k-row snapshot on Postgres paid
544
+ * ~100k network round-trips (~10 s at 0.1 ms each) before this join.
545
+ *
546
+ * The candidates read at most `limit` entries of the covering
547
+ * `sync_row_scopes` PK per scope value. A bare `value IN (...)` (or
548
+ * `= ANY`) predicate gives up the PK's `row_id` order whenever the planner
549
+ * cannot prove a single value, and then every entry after the cursor is read
550
+ * and sorted before the LIMIT: a cost that grows with the entries left, so a
551
+ * bootstrap is quadratic. PostgreSQL reads one ordered, LIMITed range per
552
+ * value through a LATERAL join and merges them. SQLite has no LATERAL; it
553
+ * bounds the ranges from above instead: the smallest of the values' `limit`th
554
+ * entries after the cursor. That value alone has `limit` entries up to its
555
+ * bound, so the page ends there; a value with fewer entries has no bound, and
556
+ * when no value has one the empty blob `x''` sorts after every TEXT row id.
547
557
  *
548
558
  * LEFT JOIN (not INNER): a candidate whose row vanished (index entry
549
559
  * without a row) must still reach the caller — its `row_id` advances the
@@ -551,29 +561,32 @@ export function indexRowPageStatement(table, index, values, partition, afterRowI
551
561
  * disappearing from the page.
552
562
  *
553
563
  * Bind order:
554
- * - postgres: [partition, tbl, var, ...values, afterRowId, limit]
555
- * (the join reuses `$1` for the partition);
564
+ * - postgres: [partition, tbl, var, ...values, afterRowId, limit];
556
565
  * - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
557
- * limit, partition] (positional `?`; the partition binds again for the
558
- * join). The values expand through `json_each`, so the statement text and
566
+ * limit]. The values expand through `json_each`, so the statement text and
559
567
  * parameter count do not depend on `valueCount` and D1's 100-parameter
560
568
  * cap is unreachable through scope values.
561
569
  */
562
570
  export function scanRowPageSql(table, valueCount, dialect) {
563
- const p = (n) => (dialect === 'sqlite' ? '?' : `$${n}`);
564
- const values = dialect === 'sqlite'
565
- ? 'SELECT value FROM json_each(?)'
566
- : Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
567
- const after = p(4 + valueCount);
568
- const limit = p(5 + valueCount);
569
- const joinPartition = dialect === 'sqlite' ? '?' : '$1';
571
+ const after = dialect === 'sqlite' ? '?5' : `$${4 + valueCount}`;
572
+ const limit = dialect === 'sqlite' ? '?6' : `$${5 + valueCount}`;
573
+ const candidates = dialect === 'sqlite'
574
+ ? `SELECT DISTINCT row_id FROM sync_row_scopes
575
+ WHERE partition=?1 AND tbl=?2 AND var=?3 AND value IN (SELECT value FROM json_each(?4))
576
+ AND row_id>?5
577
+ AND row_id<=coalesce((SELECT min((SELECT row_id FROM sync_row_scopes
578
+ WHERE partition=?1 AND tbl=?2 AND var=?3 AND value=v.value AND row_id>?5
579
+ ORDER BY row_id LIMIT 1 OFFSET ?6-1)) FROM json_each(?4) v), x'')
580
+ ORDER BY row_id LIMIT ?6`
581
+ : `SELECT DISTINCT x.row_id FROM (VALUES ${Array.from({ length: valueCount }, (_, i) => `($${4 + i})`).join(',')}) v(value)
582
+ CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
583
+ WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=v.value AND row_id>${after}
584
+ ORDER BY row_id LIMIT ${limit}) x
585
+ ORDER BY x.row_id LIMIT ${limit}`;
570
586
  return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
571
- FROM (SELECT DISTINCT row_id FROM sync_row_scopes
572
- WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
573
- AND row_id>${after}
574
- ORDER BY row_id LIMIT ${limit}) c
587
+ FROM (${candidates}) c
575
588
  LEFT JOIN ${quoteIdent(table.name)} r
576
- ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${joinPartition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
589
+ ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${dialect === 'sqlite' ? '?1' : '$1'} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
577
590
  ORDER BY c.row_id`;
578
591
  }
579
592
  /**
@@ -629,7 +642,7 @@ export function scanProbeSql(dialect) {
629
642
  * PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
630
643
  * page of every subscription scanning any table in a pull. `$2` is a JSON
631
644
  * array of `{i, tbl, var, vals, after, lim}` records; each record runs the
632
- * same index-first candidate subquery through a LATERAL join, so every
645
+ * same per-value candidate ranges through a LATERAL join, so every
633
646
  * subscription keeps its own keyset cursor and LIMIT. The candidate side is
634
647
  * table-agnostic (a spec names its table), so the statement joins the row
635
648
  * table once per table present in the batch through a UNION ALL branch that
@@ -655,10 +668,11 @@ export function scanRowBatchSql(tables) {
655
668
  return `WITH cand AS (
656
669
  SELECT s.i AS sub, c.row_id AS row_id, s.tbl AS tbl
657
670
  FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
658
- CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM sync_row_scopes
659
- WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
660
- AND row_id>s.after
661
- ORDER BY row_id LIMIT s.lim) c)
671
+ CROSS JOIN LATERAL (SELECT DISTINCT x.row_id FROM unnest(s.vals) v(value)
672
+ CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
673
+ WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value AND row_id>s.after
674
+ ORDER BY row_id LIMIT s.lim) x
675
+ ORDER BY x.row_id LIMIT s.lim) c)
662
676
  ${projection}
663
677
  ORDER BY sub, row_id`;
664
678
  }
@@ -816,8 +830,10 @@ export function assertStoredScopePatterns(schema, storedJson) {
816
830
  }
817
831
  /**
818
832
  * §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
819
- * entries and tombstones, and raise every partition's horizon past the
820
- * changes logged under the previous patterns. Empty when nothing re-keys.
833
+ * entries and tombstones. The re-key takes a commit sequence number of its
834
+ * own in every partition and records it as each re-keyed table's horizon
835
+ * (§4.6), past the changes logged under the previous patterns. Empty when
836
+ * nothing re-keys.
821
837
  */
822
838
  export function rekeyCleanupSql(tables) {
823
839
  if (tables.length === 0)
@@ -826,7 +842,10 @@ export function rekeyCleanupSql(tables) {
826
842
  return [
827
843
  `DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
828
844
  `DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
829
- 'UPDATE sync_partitions SET horizon_seq=max_commit_seq WHERE horizon_seq<max_commit_seq',
845
+ 'UPDATE sync_partitions SET max_commit_seq=max_commit_seq+1',
846
+ ...names.map((name) => `INSERT INTO sync_table_horizons(partition, tbl, commit_seq)
847
+ SELECT partition, ${name}, max_commit_seq FROM sync_partitions WHERE true
848
+ ON CONFLICT(partition, tbl) DO UPDATE SET commit_seq=excluded.commit_seq`),
830
849
  ];
831
850
  }
832
851
  /**
@@ -19,7 +19,7 @@ export function schemaWindowOf(current, configured) {
19
19
  const next = head.tables.get(table.name);
20
20
  if (next === undefined ||
21
21
  table.columns.length > next.columns.length ||
22
- // §6.12: a version may mark an existing column server-owned; the flag
22
+ // §6.7: a version may mark an existing column server-owned; the flag
23
23
  // never reaches the wire.
24
24
  table.columns.some((column, i) => JSON.stringify({ ...column, serverOwned: undefined }) !==
25
25
  JSON.stringify({ ...next.columns[i], serverOwned: undefined })) ||
@@ -27,7 +27,7 @@ export function schemaWindowOf(current, configured) {
27
27
  .slice(table.columns.length)
28
28
  .some((column) => !column.nullable) ||
29
29
  table.primaryKeyIndex !== next.primaryKeyIndex ||
30
- // §6.12: a pattern over an appended server-owned column is new to the
30
+ // §6.7: a pattern over an appended server-owned column is new to the
31
31
  // current version; an older client neither holds nor writes it.
32
32
  JSON.stringify(table.scopePatterns) !==
33
33
  JSON.stringify(next.scopePatterns.filter((pattern) => pattern.columnIndex < table.columns.length ||
package/dist/schema.d.ts CHANGED
@@ -141,8 +141,8 @@ export interface CompiledTable {
141
141
  * image copies ciphertext wholesale with no per-row decrypt pass, so an
142
142
  * encrypted table MUST be served via the rows lane. Empty ⇒ no restriction. */
143
143
  readonly encryptedColumnIndices: readonly number[];
144
- /** §6.12: column indices marked `serverOwned`. Empty ⇒ no server column
145
- * hook output is accepted for the table. */
144
+ /** §6.7: column indices marked `serverOwned`. Empty ⇒ no server column
145
+ * values are accepted from the table's validator. */
146
146
  readonly serverOwnedColumnIndices: readonly number[];
147
147
  }
148
148
  export interface CompiledSchema {
package/dist/schema.js CHANGED
@@ -174,7 +174,7 @@ export function compileSchema(schema) {
174
174
  // §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
175
175
  if (column.encrypted === true)
176
176
  encryptedColumnIndices.push(index);
177
- // §6.12: the hook never changes who may write the row (owner set), and
177
+ // §6.7: a validator never changes who may write the row (owner set), and
178
178
  // a client's optimistic row holds NULL until the server row arrives.
179
179
  if (column.serverOwned === true) {
180
180
  if (index === primaryKeyIndex ||
@@ -183,7 +183,7 @@ export function compileSchema(schema) {
183
183
  column.encrypted === true ||
184
184
  scopePatterns.some((pattern) => pattern.columnIndex === index &&
185
185
  writeScopeSets[0].includes(pattern.variable))) {
186
- throw new Error(`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.12)`);
186
+ throw new Error(`table ${table.name}: server-owned column ${JSON.stringify(column.name)} must be nullable and outside the primary key, the owner write set, crdt and encryption (§6.7)`);
187
187
  }
188
188
  serverOwnedColumnIndices.push(index);
189
189
  }
@@ -1,5 +1,5 @@
1
1
  import { clockOf, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
2
- import { SyncError, syncError } from './errors.js';
2
+ import { SyncError, segmentStoreFailure, syncError } from './errors.js';
3
3
  import { emitEvent } from './events.js';
4
4
  import { compileSchema } from './schema.js';
5
5
  import { schemaWindowOf } from './schema-window.js';
@@ -79,7 +79,9 @@ async function withDownloadEvent(ctx, request, run, bytesOf) {
79
79
  }
80
80
  async function downloadSegment(ctx, request) {
81
81
  const registry = await touchAuthenticatedPartition(ctx);
82
- const entry = await ctx.segments.get(request.segmentId);
82
+ const entry = await ctx.segments
83
+ .get(request.segmentId)
84
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'get'));
83
85
  if (entry === undefined) {
84
86
  throw syncError('sync.not_found', 'unknown segment (§5.5)');
85
87
  }
@@ -89,8 +91,13 @@ async function downloadSegment(ctx, request) {
89
91
  async function openSegment(ctx, request) {
90
92
  const registry = await touchAuthenticatedPartition(ctx);
91
93
  const opened = ctx.segments.open !== undefined
92
- ? await ctx.segments.open(request.segmentId)
93
- : await ctx.segments.get(request.segmentId).then((entry) => entry === undefined
94
+ ? await ctx.segments
95
+ .open(request.segmentId)
96
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'open'))
97
+ : await ctx.segments
98
+ .get(request.segmentId)
99
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'get'))
100
+ .then((entry) => entry === undefined
94
101
  ? undefined
95
102
  : {
96
103
  record: entry.record,
@@ -18,6 +18,7 @@ import type { UnitScopes } from '@syncular/core';
18
18
  import { type BoundAuthoritativeQuery } from './authoritative-query.js';
19
19
  import type { CompiledTable } from './schema.js';
20
20
  import type { SqliteValue } from './sqlite-driver.js';
21
+ import { type PartitionHorizons } from './storage.js';
21
22
  import type { AuthoritativeQueryRequest, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, CommitPruneQuery, CommitPruneResult, IndexRowScanQuery, LiveWindow, LiveWindowQuery, NewCommit, OutboxSequence, OutboxSettlement, NewReaction, PartitionRegistryEntry, PartitionTouch, PrunedReactionCounts, PushWatermark, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, StoredPushResult, StoredReaction, StoredRow, Tombstone } from './storage.js';
22
23
  /**
23
24
  * Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
@@ -44,7 +45,7 @@ export declare const SQLITE_UPGRADE_PROBE: SqliteStatementSpec;
44
45
  /**
45
46
  * The statements that bring the core tables of a database from an earlier
46
47
  * release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
47
- * columns, dropped columns, dropped indexes, then missing indexes. Empty for
48
+ * tables and columns, dropped columns, dropped indexes, then missing indexes. Empty for
48
49
  * a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
49
50
  * construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
50
51
  * and the first open of a database at the current schema version.
@@ -125,6 +126,7 @@ export declare abstract class SqliteFamilyStorage {
125
126
  getMaxCommitSeq(partition: string): Promise<number>;
126
127
  getPartitionLogEpoch(partition: string): Promise<string | undefined>;
127
128
  getHorizonSeq(partition: string): Promise<number>;
129
+ getHorizons(partition: string): Promise<PartitionHorizons>;
128
130
  setHorizonSeq(partition: string, seq: number): Promise<void>;
129
131
  pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
130
132
  getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
@@ -4,7 +4,7 @@ import { validateCommitPruneQuery } from './prune.js';
4
4
  import { StorageQueryError } from './storage-errors.js';
5
5
  import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, countRowsSql, scanProbeSql, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
6
6
  import { matchesEffective } from './scopes.js';
7
- import { CLIENT_RECORD_COLUMNS, PARTITION_TOUCH_INTERVAL_MS, pushResultFromJson, pushResultToJson, rowsLiveAfterDelete, toClientRecord, toStoredChange, toStoredReaction, toStoredRow, toTombstone, } from './storage.js';
7
+ import { asJson, CLIENT_RECORD_COLUMNS, PARTITION_TOUCH_INTERVAL_MS, pushResultFromJson, pushResultToJson, rowsLiveAfterDelete, toClientRecord, toStoredChange, toStoredReaction, toStoredRow, toTombstone, } from './storage.js';
8
8
  import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
9
9
  /**
10
10
  * Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
@@ -156,6 +156,19 @@ const SQLITE_ADDED_COLUMNS = [
156
156
  ['sync_push_results', 'client_seq', 'INTEGER'],
157
157
  ['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
158
158
  ];
159
+ /**
160
+ * Core tables that `sqliteUpgradeDdl` creates, keyed by one of their columns,
161
+ * so a D1 database at an unchanged schema version gains them at its first
162
+ * open. `sync_table_horizons` holds the §4.6 table horizons.
163
+ */
164
+ const SQLITE_ADDED_TABLES = [
165
+ [
166
+ 'sync_table_horizons',
167
+ 'commit_seq',
168
+ // One line: D1's `exec` runs each line as a statement.
169
+ 'CREATE TABLE IF NOT EXISTS sync_table_horizons(partition TEXT NOT NULL, tbl TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(partition, tbl))',
170
+ ],
171
+ ];
159
172
  /** Core-table columns wire version 4 deleted: the version 3 subscription list. */
160
173
  const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']];
161
174
  /**
@@ -189,7 +202,11 @@ export const SQLITE_UPGRADE_PROBE = {
189
202
  WHERE type='index' AND name IN (SELECT value FROM json_each(?))`,
190
203
  params: [
191
204
  JSON.stringify([
192
- ...new Set([...SQLITE_ADDED_COLUMNS, ...SQLITE_DROPPED_COLUMNS].map(([table]) => table)),
205
+ ...new Set([
206
+ ...SQLITE_ADDED_COLUMNS,
207
+ ...SQLITE_DROPPED_COLUMNS,
208
+ ...SQLITE_ADDED_TABLES,
209
+ ].map(([table]) => table)),
193
210
  ]),
194
211
  JSON.stringify([
195
212
  ...SQLITE_ADDED_INDEXES.map(([name]) => name),
@@ -200,7 +217,7 @@ export const SQLITE_UPGRADE_PROBE = {
200
217
  /**
201
218
  * The statements that bring the core tables of a database from an earlier
202
219
  * release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
203
- * columns, dropped columns, dropped indexes, then missing indexes. Empty for
220
+ * tables and columns, dropped columns, dropped indexes, then missing indexes. Empty for
204
221
  * a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
205
222
  * construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
206
223
  * and the first open of a database at the current schema version.
@@ -208,6 +225,7 @@ export const SQLITE_UPGRADE_PROBE = {
208
225
  export function sqliteUpgradeDdl(present) {
209
226
  const has = (table, name) => present.some((row) => row.tbl === table && row.name === name);
210
227
  return [
228
+ ...SQLITE_ADDED_TABLES.filter(([table, name]) => !has(table, name)).map(([, , ddl]) => ddl),
211
229
  ...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(([table, name, definition]) => `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`),
212
230
  // A database from before wire version 4 (and the delete floor) starts
213
231
  // the delete floor at the highest version its tombstones recorded
@@ -358,6 +376,14 @@ export class SqliteFamilyStorage {
358
376
  const [row] = await this.all('SELECT horizon_seq FROM sync_partitions WHERE partition=?', [partition]);
359
377
  return row?.horizon_seq ?? 0;
360
378
  }
379
+ async getHorizons(partition) {
380
+ const [row] = await this.all(`SELECT coalesce((SELECT horizon_seq FROM sync_partitions WHERE partition=?),0) AS seq,
381
+ (SELECT json_group_object(tbl, commit_seq) FROM sync_table_horizons WHERE partition=?) AS tables`, [partition, partition]);
382
+ return {
383
+ seq: row?.seq ?? 0,
384
+ tables: new Map(Object.entries(asJson(row?.tables ?? '{}'))),
385
+ };
386
+ }
361
387
  async setHorizonSeq(partition, seq) {
362
388
  await this.all(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
363
389
  ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`, [partition, seq]);
@@ -393,8 +419,14 @@ export class SqliteFamilyStorage {
393
419
  AND ${epochGuard}`,
394
420
  params: [partition, partition, partition, query.logEpoch],
395
421
  },
396
- // §4.6: tombstones prune in the same pass as the commit log.
397
- ...['sync_commits', 'sync_changes', 'sync_tombstones'].map((table) => ({
422
+ // §4.6: tombstones and table horizons prune in the same pass as the
423
+ // commit log.
424
+ ...[
425
+ 'sync_commits',
426
+ 'sync_changes',
427
+ 'sync_tombstones',
428
+ 'sync_table_horizons',
429
+ ].map((table) => ({
398
430
  sql: `DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`,
399
431
  params: [partition, partition, partition, query.logEpoch],
400
432
  })),
@@ -706,7 +738,6 @@ export class SqliteFamilyStorage {
706
738
  JSON.stringify(values),
707
739
  afterRowId,
708
740
  batchSize,
709
- partition,
710
741
  ]);
711
742
  if (records.length === 0)
712
743
  break;
@@ -20,8 +20,11 @@ export interface ScanProbe {
20
20
  *
21
21
  * A single-variable filter needs no probe. Otherwise `count` counts each
22
22
  * variable's entries in one statement, up to `cap`. The cap starts at
23
- * max(64, limit) and grows fourfold, to at most 65,536, while the fewest
24
- * count is shared by several variables at the cap.
23
+ * max(64, limit) and grows fourfold while the fewest count is shared by
24
+ * several variables at the cap. A round runs only while every tied variable
25
+ * has at least `cap` entries, so the probe ends at the first cap above the
26
+ * smallest count `n`: at most 4n entries per variable in the last round, and
27
+ * the pick is the variable with the fewest entries whatever the starting cap.
25
28
  *
26
29
  * `chosen` memoizes the pick per (partition, table, filter), at most 1,024
27
30
  * entries, so the page loop of a bootstrap probes once. A stale pick only
@@ -10,8 +10,11 @@ import { StorageQueryError } from './storage-errors.js';
10
10
  *
11
11
  * A single-variable filter needs no probe. Otherwise `count` counts each
12
12
  * variable's entries in one statement, up to `cap`. The cap starts at
13
- * max(64, limit) and grows fourfold, to at most 65,536, while the fewest
14
- * count is shared by several variables at the cap.
13
+ * max(64, limit) and grows fourfold while the fewest count is shared by
14
+ * several variables at the cap. A round runs only while every tied variable
15
+ * has at least `cap` entries, so the probe ends at the first cap above the
16
+ * smallest count `n`: at most 4n entries per variable in the last round, and
17
+ * the pick is the variable with the fewest entries whatever the starting cap.
15
18
  *
16
19
  * `chosen` memoizes the pick per (partition, table, filter), at most 1,024
17
20
  * entries, so the page loop of a bootstrap probes once. A stale pick only
@@ -42,7 +45,7 @@ export async function chooseScanVariable(partition, query, chosen, count) {
42
45
  if (cached !== undefined)
43
46
  return cached;
44
47
  let tied = variables;
45
- for (let cap = Math.max(64, query.limit);; cap = Math.min(cap * 4, 65_536)) {
48
+ for (let cap = Math.max(64, query.limit);; cap *= 4) {
46
49
  const counts = tied.map(() => 0);
47
50
  const probes = tied.map((variable, i) => ({
48
51
  i,
@@ -57,7 +60,7 @@ export async function chooseScanVariable(partition, query, chosen, count) {
57
60
  if (fewest === 0)
58
61
  return undefined;
59
62
  tied = tied.filter((_, i) => counts[i] === fewest);
60
- if (fewest < cap || tied.length === 1 || cap >= 65_536)
63
+ if (fewest < cap || tied.length === 1)
61
64
  break;
62
65
  }
63
66
  if (chosen.size >= 1_024)
package/dist/storage.d.ts CHANGED
@@ -440,6 +440,16 @@ export declare function toStoredChange(record: StoredChangeRecord): StoredChange
440
440
  export declare function toStoredReaction(record: StoredReactionRecord): StoredReaction;
441
441
  export declare const CLIENT_RECORD_COLUMNS = "client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures";
442
442
  export declare function toClientRecord(record: ClientRowRecord): ClientRecord;
443
+ /** A partition's pruning horizon and its table horizons (§4.6). */
444
+ export interface PartitionHorizons {
445
+ /** `horizonSeq`. */
446
+ readonly seq: number;
447
+ /**
448
+ * Re-keyed table → the commit sequence number its re-key took. The changes
449
+ * of a table at or below it are never served.
450
+ */
451
+ readonly tables: ReadonlyMap<string, number>;
452
+ }
443
453
  /** One unit of the §4.5 scan set. */
444
454
  export interface LiveScanUnit {
445
455
  /** The unit's scope map; its first variable (keys sorted) is its index key. */
@@ -866,6 +876,8 @@ export interface ServerStorage {
866
876
  begin(partition: string): Promise<StorageTransaction>;
867
877
  getMaxCommitSeq(partition: string): Promise<number>;
868
878
  getHorizonSeq(partition: string): Promise<number>;
879
+ /** `horizonSeq` and the table horizons (§4.6) in one read. */
880
+ getHorizons(partition: string): Promise<PartitionHorizons>;
869
881
  /** Monotonic within the current epoch; use atomic pruning for maintenance. */
870
882
  setHorizonSeq(partition: string, seq: number): Promise<void>;
871
883
  /**
@@ -5,9 +5,10 @@
5
5
  * row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
6
6
  * commit transaction, once per operation. It is the seam for business
7
7
  * rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
8
- * "status ∈ {…}"). A throw (or rejected promise) rejects the whole commit
9
- * atomically (§6.4) with a host-defined code the client surfaces unchanged
10
- * in its rejection record (§6.3).
8
+ * "status ∈ {…}") and for the values of server-owned columns. A throw (or
9
+ * rejected promise) rejects the whole commit atomically (§6.4) with a
10
+ * host-defined code the client surfaces unchanged in its rejection record
11
+ * (§6.3).
11
12
  *
12
13
  * The feature is OFF by default (no `validators` on the config): the push
13
14
  * path pays only an `undefined` check per operation and builds no context
@@ -36,8 +37,9 @@ export type ValidateRow = Readonly<Record<string, RowValue>>;
36
37
  export type ValidateOpKind = 'upsert' | 'delete';
37
38
  /**
38
39
  * One push operation as the validator sees it (§6.7). For an `upsert`,
39
- * `row` is the post-merge, post-scope-strip row that will be written and
40
- * `stored` is the prior stored row (or `undefined` on insert). For a
40
+ * `row` is the post-merge, post-scope-strip row that will be written, with
41
+ * each server-owned column as stored (NULL on an insert), and `stored` is
42
+ * the prior stored row (or `undefined` on insert). For a
41
43
  * `delete`, `row` is `undefined` and `stored` is the row about to be
42
44
  * removed (a delete only reaches a validator when a stored row exists —
43
45
  * an absent-row delete is an idempotent no-op that never validates, §6.2).
@@ -151,7 +153,7 @@ export type CommitValidator = (input: ValidateCommitInput) => void | Promise<voi
151
153
  * inside and the push never completes, and on a pool-backed executor (`pg`,
152
154
  * `Bun.sql`) it reads committed state without the staged writes, or waits
153
155
  * forever once push transactions hold every pool connection. An authority
154
- * read calls `queryAuthoritative` on this context instead.
156
+ * read goes through `read` instead.
155
157
  */
156
158
  export interface ValidateContext {
157
159
  /** Host-authenticated actor (§1.1) performing the write. */
@@ -161,15 +163,13 @@ export interface ValidateContext {
161
163
  /** The context `authenticate` returned for this request (§1.1); absent without one. */
162
164
  readonly context?: string;
163
165
  /**
164
- * Run a generated registered query on the push transaction (§6.7). The
165
- * request has the shape `ServerStorage.queryAuthoritative` takes; the
166
- * storage binds every relation to the commit's partition. SQLite and
167
- * PostgreSQL return the transaction's snapshot, including the operations
168
- * staged before this one; D1 fails with
166
+ * The §6.8 candidate-state reader of the push transaction: it sees every
167
+ * operation staged before this one. `read.queryAuthoritative` runs a
168
+ * generated registered query on that transaction; D1 fails with
169
169
  * `sync.storage.query_over_staged_writes` when the query reads a table the
170
170
  * commit has written.
171
171
  */
172
- readonly queryAuthoritative: (query: AuthoritativeQueryRequest) => Promise<AuthoritativeQueryResult>;
172
+ readonly read: CommitValidationReader;
173
173
  }
174
174
  /**
175
175
  * A per-table validation hook (§6.7). Called once per push operation on
@@ -180,14 +180,21 @@ export interface ValidateContext {
180
180
  * `sync.constraint_violation` code (a validator SHOULD throw
181
181
  * `ValidationRejection` to control the code). MUST NOT mutate the row.
182
182
  *
183
+ * On an insert or update of a pushed commit or a registered command, the
184
+ * validator may return values for the table's server-owned columns; a column
185
+ * it omits keeps its value in `row`. A key that names no server-owned column,
186
+ * a value the column's type refuses, or a changed scope column on an update
187
+ * rejects the commit with `sync.constraint_violation`. The server ignores
188
+ * returned values on a delete, a rescope (§2.2) and a §6.11 cascade.
189
+ *
183
190
  * The hook runs inside the push transaction. An authority read uses
184
- * `ctx.queryAuthoritative`, which executes on that transaction; calling
191
+ * `ctx.read.queryAuthoritative`, which executes on that transaction; calling
185
192
  * `ServerStorage.queryAuthoritative` from here deadlocks on a
186
193
  * single-connection executor and reads committed state on a pool-backed one.
187
194
  * A rule over sibling operations belongs in the whole-commit
188
195
  * `CommitValidator` (§6.8).
189
196
  */
190
- export type Validator = (op: ValidateOperation, ctx: ValidateContext) => void | Promise<void>;
197
+ export type Validator = (op: ValidateOperation, ctx: ValidateContext) => ValidateRow | void | Promise<ValidateRow | void>;
191
198
  /** Table name → validator (§6.7). Absent tables are unvalidated. */
192
199
  export type ValidatorRegistry = Readonly<Record<string, Validator>>;
193
200
  /**
@@ -222,28 +229,3 @@ export declare class CommitAuthorizationRejection extends SyncError {
222
229
  }
223
230
  /** Build the column-keyed row object a validator inspects (§6.7). */
224
231
  export declare function toValidateRow(columns: readonly RowColumn[], values: readonly RowValue[]): ValidateRow;
225
- /** §6.12: the accepted upsert a server column hook completes. `row` is the
226
- * merged row (present columns over stored values, crdt columns merged) with
227
- * every server-owned column as stored, or NULL on an insert. */
228
- export interface ServerColumnsInput {
229
- readonly table: string;
230
- readonly rowId: string;
231
- readonly row: ValidateRow;
232
- readonly stored: ValidateRow | undefined;
233
- }
234
- export interface ServerColumnsContext {
235
- readonly actorId: string;
236
- readonly partition: string;
237
- readonly context?: string;
238
- /** The §6.8 candidate-state reader of the push transaction. */
239
- readonly read: CommitValidationReader;
240
- }
241
- /**
242
- * §6.12: returns values for the table's server-owned columns; a column it
243
- * omits keeps its value in `row`. A key that names no server-owned column, a
244
- * value the column's type refuses, or a changed scope column on an update
245
- * rejects the commit with `sync.constraint_violation`, like any other throw
246
- * that is not a {@link ValidationRejection}.
247
- */
248
- export type ServerColumnsHook = (input: ServerColumnsInput, ctx: ServerColumnsContext) => ValidateRow | Promise<ValidateRow>;
249
- export type ServerColumnsRegistry = Readonly<Record<string, ServerColumnsHook>>;
package/dist/validate.js CHANGED
@@ -5,9 +5,10 @@
5
5
  * row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
6
6
  * commit transaction, once per operation. It is the seam for business
7
7
  * rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
8
- * "status ∈ {…}"). A throw (or rejected promise) rejects the whole commit
9
- * atomically (§6.4) with a host-defined code the client surfaces unchanged
10
- * in its rejection record (§6.3).
8
+ * "status ∈ {…}") and for the values of server-owned columns. A throw (or
9
+ * rejected promise) rejects the whole commit atomically (§6.4) with a
10
+ * host-defined code the client surfaces unchanged in its rejection record
11
+ * (§6.3).
11
12
  *
12
13
  * The feature is OFF by default (no `validators` on the config): the push
13
14
  * path pays only an `undefined` check per operation and builds no context
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.95.0",
3
+ "version": "0.97.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.95.0"
71
+ "@syncular/core": "0.97.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
package/src/context.ts CHANGED
@@ -25,11 +25,7 @@ import type {
25
25
  ServerStorage,
26
26
  StoredCommit,
27
27
  } from './storage';
28
- import type {
29
- CommitValidator,
30
- ServerColumnsRegistry,
31
- ValidatorRegistry,
32
- } from './validate';
28
+ import type { CommitValidator, ValidatorRegistry } from './validate';
33
29
 
34
30
  /** SSP2 body content type (§1.1). */
35
31
  export const SSP2_CONTENT_TYPE = 'application/vnd.syncular.sync.v2';
@@ -124,12 +120,6 @@ export interface ServerLimits {
124
120
  * bound gets an image of its own.
125
121
  */
126
122
  readonly maxImageSegmentBytes: number;
127
- /**
128
- * Segment puts one pull keeps in flight. The pull scans and encodes its
129
- * pages in order and stores up to this many at once, so it holds this
130
- * many encoded segments in memory.
131
- */
132
- readonly maxConcurrentSegmentPuts: number;
133
123
  }
134
124
 
135
125
  export const DEFAULT_LIMITS: ServerLimits = {
@@ -146,9 +136,6 @@ export const DEFAULT_LIMITS: ServerLimits = {
146
136
  maxLiveSectionBytes: 16 * 1024 * 1024,
147
137
  // §5.3 DECISION (2026-11-17): a client holds about three images at once.
148
138
  maxImageSegmentBytes: 32 * 1024 * 1024,
149
- // A Workers invocation holds six connections open at once, and an R2 put
150
- // uses one; past six the runtime queues the put.
151
- maxConcurrentSegmentPuts: 6,
152
139
  };
153
140
 
154
141
  /** Receives applied commits for realtime fanout (§8.2). */
@@ -200,19 +187,13 @@ export interface SyncServerConfig {
200
187
  * builds no validator context — zero cost. Present ⇒ each push operation
201
188
  * on a listed table runs its validator after decode + §3.4 scope
202
189
  * authorization, inside the commit transaction; a throw rejects the
203
- * whole commit atomically (§6.4) with the host code (§6.3). Kept a host
190
+ * whole commit atomically (§6.4) with the host code (§6.3). On an insert
191
+ * or update the validator may return values of the table's server-owned
192
+ * columns, which are stored and fanned out in the same commit. Kept a host
204
193
  * callback (like `resolveScopes`) — business rules live in the host
205
194
  * process, never on the wire.
206
195
  */
207
196
  readonly validators?: ValidatorRegistry;
208
- /**
209
- * §6.12 per-table server column hooks (`table` → hook). Each accepted
210
- * upsert on a listed table runs its hook inside the push transaction,
211
- * after §3.4 authorization and before §6.7 validation; the returned values
212
- * of the table's server-owned columns are stored and fanned out in the
213
- * same commit.
214
- */
215
- readonly serverColumns?: ServerColumnsRegistry;
216
197
  /**
217
198
  * §6.8 whole-commit validator. When present, the storage serializes this
218
199
  * partition before operation reads/writes, then invokes the callback once
package/src/d1-storage.ts CHANGED
@@ -443,7 +443,6 @@ class D1Transaction
443
443
  JSON.stringify(values),
444
444
  afterRowId,
445
445
  batchSize,
446
- this.partition,
447
446
  )
448
447
  .all<SqliteRowRecord & { payload: Uint8Array | null }>();
449
448
  if (records.length === 0) break;