@syncular/server 0.94.0 → 0.96.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@ import { validateCommitPruneQuery } from './prune.js';
4
4
  import { StorageQueryError } from './storage-errors.js';
5
5
  import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, countRowsSql, scanProbeSql, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
6
6
  import { matchesEffective } from './scopes.js';
7
- import { CLIENT_RECORD_COLUMNS, PARTITION_TOUCH_INTERVAL_MS, pushResultFromJson, pushResultToJson, rowsLiveAfterDelete, toClientRecord, toStoredChange, toStoredReaction, toStoredRow, toTombstone, } from './storage.js';
7
+ import { asJson, CLIENT_RECORD_COLUMNS, PARTITION_TOUCH_INTERVAL_MS, pushResultFromJson, pushResultToJson, rowsLiveAfterDelete, toClientRecord, toStoredChange, toStoredReaction, toStoredRow, toTombstone, } from './storage.js';
8
8
  import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
9
9
  /**
10
10
  * Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
@@ -156,6 +156,19 @@ const SQLITE_ADDED_COLUMNS = [
156
156
  ['sync_push_results', 'client_seq', 'INTEGER'],
157
157
  ['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
158
158
  ];
159
+ /**
160
+ * Core tables that `sqliteUpgradeDdl` creates, keyed by one of their columns,
161
+ * so a D1 database at an unchanged schema version gains them at its first
162
+ * open. `sync_table_horizons` holds the §4.6 table horizons.
163
+ */
164
+ const SQLITE_ADDED_TABLES = [
165
+ [
166
+ 'sync_table_horizons',
167
+ 'commit_seq',
168
+ // One line: D1's `exec` runs each line as a statement.
169
+ 'CREATE TABLE IF NOT EXISTS sync_table_horizons(partition TEXT NOT NULL, tbl TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(partition, tbl))',
170
+ ],
171
+ ];
159
172
  /** Core-table columns wire version 4 deleted: the version 3 subscription list. */
160
173
  const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']];
161
174
  /**
@@ -189,7 +202,11 @@ export const SQLITE_UPGRADE_PROBE = {
189
202
  WHERE type='index' AND name IN (SELECT value FROM json_each(?))`,
190
203
  params: [
191
204
  JSON.stringify([
192
- ...new Set([...SQLITE_ADDED_COLUMNS, ...SQLITE_DROPPED_COLUMNS].map(([table]) => table)),
205
+ ...new Set([
206
+ ...SQLITE_ADDED_COLUMNS,
207
+ ...SQLITE_DROPPED_COLUMNS,
208
+ ...SQLITE_ADDED_TABLES,
209
+ ].map(([table]) => table)),
193
210
  ]),
194
211
  JSON.stringify([
195
212
  ...SQLITE_ADDED_INDEXES.map(([name]) => name),
@@ -200,7 +217,7 @@ export const SQLITE_UPGRADE_PROBE = {
200
217
  /**
201
218
  * The statements that bring the core tables of a database from an earlier
202
219
  * release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
203
- * columns, dropped columns, dropped indexes, then missing indexes. Empty for
220
+ * tables and columns, dropped columns, dropped indexes, then missing indexes. Empty for
204
221
  * a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
205
222
  * construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
206
223
  * and the first open of a database at the current schema version.
@@ -208,6 +225,7 @@ export const SQLITE_UPGRADE_PROBE = {
208
225
  export function sqliteUpgradeDdl(present) {
209
226
  const has = (table, name) => present.some((row) => row.tbl === table && row.name === name);
210
227
  return [
228
+ ...SQLITE_ADDED_TABLES.filter(([table, name]) => !has(table, name)).map(([, , ddl]) => ddl),
211
229
  ...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(([table, name, definition]) => `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`),
212
230
  // A database from before wire version 4 (and the delete floor) starts
213
231
  // the delete floor at the highest version its tombstones recorded
@@ -358,6 +376,14 @@ export class SqliteFamilyStorage {
358
376
  const [row] = await this.all('SELECT horizon_seq FROM sync_partitions WHERE partition=?', [partition]);
359
377
  return row?.horizon_seq ?? 0;
360
378
  }
379
+ async getHorizons(partition) {
380
+ const [row] = await this.all(`SELECT coalesce((SELECT horizon_seq FROM sync_partitions WHERE partition=?),0) AS seq,
381
+ (SELECT json_group_object(tbl, commit_seq) FROM sync_table_horizons WHERE partition=?) AS tables`, [partition, partition]);
382
+ return {
383
+ seq: row?.seq ?? 0,
384
+ tables: new Map(Object.entries(asJson(row?.tables ?? '{}'))),
385
+ };
386
+ }
361
387
  async setHorizonSeq(partition, seq) {
362
388
  await this.all(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
363
389
  ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`, [partition, seq]);
@@ -393,8 +419,14 @@ export class SqliteFamilyStorage {
393
419
  AND ${epochGuard}`,
394
420
  params: [partition, partition, partition, query.logEpoch],
395
421
  },
396
- // §4.6: tombstones prune in the same pass as the commit log.
397
- ...['sync_commits', 'sync_changes', 'sync_tombstones'].map((table) => ({
422
+ // §4.6: tombstones and table horizons prune in the same pass as the
423
+ // commit log.
424
+ ...[
425
+ 'sync_commits',
426
+ 'sync_changes',
427
+ 'sync_tombstones',
428
+ 'sync_table_horizons',
429
+ ].map((table) => ({
398
430
  sql: `DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`,
399
431
  params: [partition, partition, partition, query.logEpoch],
400
432
  })),
package/dist/storage.d.ts CHANGED
@@ -440,6 +440,16 @@ export declare function toStoredChange(record: StoredChangeRecord): StoredChange
440
440
  export declare function toStoredReaction(record: StoredReactionRecord): StoredReaction;
441
441
  export declare const CLIENT_RECORD_COLUMNS = "client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures";
442
442
  export declare function toClientRecord(record: ClientRowRecord): ClientRecord;
443
+ /** A partition's pruning horizon and its table horizons (§4.6). */
444
+ export interface PartitionHorizons {
445
+ /** `horizonSeq`. */
446
+ readonly seq: number;
447
+ /**
448
+ * Re-keyed table → the commit sequence number its re-key took. The changes
449
+ * of a table at or below it are never served.
450
+ */
451
+ readonly tables: ReadonlyMap<string, number>;
452
+ }
443
453
  /** One unit of the §4.5 scan set. */
444
454
  export interface LiveScanUnit {
445
455
  /** The unit's scope map; its first variable (keys sorted) is its index key. */
@@ -866,6 +876,8 @@ export interface ServerStorage {
866
876
  begin(partition: string): Promise<StorageTransaction>;
867
877
  getMaxCommitSeq(partition: string): Promise<number>;
868
878
  getHorizonSeq(partition: string): Promise<number>;
879
+ /** `horizonSeq` and the table horizons (§4.6) in one read. */
880
+ getHorizons(partition: string): Promise<PartitionHorizons>;
869
881
  /** Monotonic within the current epoch; use atomic pruning for maintenance. */
870
882
  setHorizonSeq(partition: string, seq: number): Promise<void>;
871
883
  /**
@@ -5,9 +5,10 @@
5
5
  * row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
6
6
  * commit transaction, once per operation. It is the seam for business
7
7
  * rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
8
- * "status ∈ {…}"). A throw (or rejected promise) rejects the whole commit
9
- * atomically (§6.4) with a host-defined code the client surfaces unchanged
10
- * in its rejection record (§6.3).
8
+ * "status ∈ {…}") and for the values of server-owned columns. A throw (or
9
+ * rejected promise) rejects the whole commit atomically (§6.4) with a
10
+ * host-defined code the client surfaces unchanged in its rejection record
11
+ * (§6.3).
11
12
  *
12
13
  * The feature is OFF by default (no `validators` on the config): the push
13
14
  * path pays only an `undefined` check per operation and builds no context
@@ -36,8 +37,9 @@ export type ValidateRow = Readonly<Record<string, RowValue>>;
36
37
  export type ValidateOpKind = 'upsert' | 'delete';
37
38
  /**
38
39
  * One push operation as the validator sees it (§6.7). For an `upsert`,
39
- * `row` is the post-merge, post-scope-strip row that will be written and
40
- * `stored` is the prior stored row (or `undefined` on insert). For a
40
+ * `row` is the post-merge, post-scope-strip row that will be written, with
41
+ * each server-owned column as stored (NULL on an insert), and `stored` is
42
+ * the prior stored row (or `undefined` on insert). For a
41
43
  * `delete`, `row` is `undefined` and `stored` is the row about to be
42
44
  * removed (a delete only reaches a validator when a stored row exists —
43
45
  * an absent-row delete is an idempotent no-op that never validates, §6.2).
@@ -151,7 +153,7 @@ export type CommitValidator = (input: ValidateCommitInput) => void | Promise<voi
151
153
  * inside and the push never completes, and on a pool-backed executor (`pg`,
152
154
  * `Bun.sql`) it reads committed state without the staged writes, or waits
153
155
  * forever once push transactions hold every pool connection. An authority
154
- * read calls `queryAuthoritative` on this context instead.
156
+ * read goes through `read` instead.
155
157
  */
156
158
  export interface ValidateContext {
157
159
  /** Host-authenticated actor (§1.1) performing the write. */
@@ -161,15 +163,13 @@ export interface ValidateContext {
161
163
  /** The context `authenticate` returned for this request (§1.1); absent without one. */
162
164
  readonly context?: string;
163
165
  /**
164
- * Run a generated registered query on the push transaction (§6.7). The
165
- * request has the shape `ServerStorage.queryAuthoritative` takes; the
166
- * storage binds every relation to the commit's partition. SQLite and
167
- * PostgreSQL return the transaction's snapshot, including the operations
168
- * staged before this one; D1 fails with
166
+ * The §6.8 candidate-state reader of the push transaction: it sees every
167
+ * operation staged before this one. `read.queryAuthoritative` runs a
168
+ * generated registered query on that transaction; D1 fails with
169
169
  * `sync.storage.query_over_staged_writes` when the query reads a table the
170
170
  * commit has written.
171
171
  */
172
- readonly queryAuthoritative: (query: AuthoritativeQueryRequest) => Promise<AuthoritativeQueryResult>;
172
+ readonly read: CommitValidationReader;
173
173
  }
174
174
  /**
175
175
  * A per-table validation hook (§6.7). Called once per push operation on
@@ -180,14 +180,21 @@ export interface ValidateContext {
180
180
  * `sync.constraint_violation` code (a validator SHOULD throw
181
181
  * `ValidationRejection` to control the code). MUST NOT mutate the row.
182
182
  *
183
+ * On an insert or update of a pushed commit or a registered command, the
184
+ * validator may return values for the table's server-owned columns; a column
185
+ * it omits keeps its value in `row`. A key that names no server-owned column,
186
+ * a value the column's type refuses, or a changed scope column on an update
187
+ * rejects the commit with `sync.constraint_violation`. The server ignores
188
+ * returned values on a delete, a rescope (§2.2) and a §6.11 cascade.
189
+ *
183
190
  * The hook runs inside the push transaction. An authority read uses
184
- * `ctx.queryAuthoritative`, which executes on that transaction; calling
191
+ * `ctx.read.queryAuthoritative`, which executes on that transaction; calling
185
192
  * `ServerStorage.queryAuthoritative` from here deadlocks on a
186
193
  * single-connection executor and reads committed state on a pool-backed one.
187
194
  * A rule over sibling operations belongs in the whole-commit
188
195
  * `CommitValidator` (§6.8).
189
196
  */
190
- export type Validator = (op: ValidateOperation, ctx: ValidateContext) => void | Promise<void>;
197
+ export type Validator = (op: ValidateOperation, ctx: ValidateContext) => ValidateRow | void | Promise<ValidateRow | void>;
191
198
  /** Table name → validator (§6.7). Absent tables are unvalidated. */
192
199
  export type ValidatorRegistry = Readonly<Record<string, Validator>>;
193
200
  /**
@@ -222,28 +229,3 @@ export declare class CommitAuthorizationRejection extends SyncError {
222
229
  }
223
230
  /** Build the column-keyed row object a validator inspects (§6.7). */
224
231
  export declare function toValidateRow(columns: readonly RowColumn[], values: readonly RowValue[]): ValidateRow;
225
- /** §6.12: the accepted upsert a server column hook completes. `row` is the
226
- * merged row (present columns over stored values, crdt columns merged) with
227
- * every server-owned column as stored, or NULL on an insert. */
228
- export interface ServerColumnsInput {
229
- readonly table: string;
230
- readonly rowId: string;
231
- readonly row: ValidateRow;
232
- readonly stored: ValidateRow | undefined;
233
- }
234
- export interface ServerColumnsContext {
235
- readonly actorId: string;
236
- readonly partition: string;
237
- readonly context?: string;
238
- /** The §6.8 candidate-state reader of the push transaction. */
239
- readonly read: CommitValidationReader;
240
- }
241
- /**
242
- * §6.12: returns values for the table's server-owned columns; a column it
243
- * omits keeps its value in `row`. A key that names no server-owned column, a
244
- * value the column's type refuses, or a changed scope column on an update
245
- * rejects the commit with `sync.constraint_violation`, like any other throw
246
- * that is not a {@link ValidationRejection}.
247
- */
248
- export type ServerColumnsHook = (input: ServerColumnsInput, ctx: ServerColumnsContext) => ValidateRow | Promise<ValidateRow>;
249
- export type ServerColumnsRegistry = Readonly<Record<string, ServerColumnsHook>>;
package/dist/validate.js CHANGED
@@ -5,9 +5,10 @@
5
5
  * row-codec decode (§6.1) and the §3.4 scope authorization, INSIDE the
6
6
  * commit transaction, once per operation. It is the seam for business
7
7
  * rules that scopes cannot express ("title ≤ 200 chars", "amount ≥ 0",
8
- * "status ∈ {…}"). A throw (or rejected promise) rejects the whole commit
9
- * atomically (§6.4) with a host-defined code the client surfaces unchanged
10
- * in its rejection record (§6.3).
8
+ * "status ∈ {…}") and for the values of server-owned columns. A throw (or
9
+ * rejected promise) rejects the whole commit atomically (§6.4) with a
10
+ * host-defined code the client surfaces unchanged in its rejection record
11
+ * (§6.3).
11
12
  *
12
13
  * The feature is OFF by default (no `validators` on the config): the push
13
14
  * path pays only an `undefined` check per operation and builds no context
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.94.0",
3
+ "version": "0.96.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.94.0"
71
+ "@syncular/core": "0.96.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
package/src/context.ts CHANGED
@@ -25,11 +25,7 @@ import type {
25
25
  ServerStorage,
26
26
  StoredCommit,
27
27
  } from './storage';
28
- import type {
29
- CommitValidator,
30
- ServerColumnsRegistry,
31
- ValidatorRegistry,
32
- } from './validate';
28
+ import type { CommitValidator, ValidatorRegistry } from './validate';
33
29
 
34
30
  /** SSP2 body content type (§1.1). */
35
31
  export const SSP2_CONTENT_TYPE = 'application/vnd.syncular.sync.v2';
@@ -191,19 +187,13 @@ export interface SyncServerConfig {
191
187
  * builds no validator context — zero cost. Present ⇒ each push operation
192
188
  * on a listed table runs its validator after decode + §3.4 scope
193
189
  * authorization, inside the commit transaction; a throw rejects the
194
- * whole commit atomically (§6.4) with the host code (§6.3). Kept a host
190
+ * whole commit atomically (§6.4) with the host code (§6.3). On an insert
191
+ * or update the validator may return values of the table's server-owned
192
+ * columns, which are stored and fanned out in the same commit. Kept a host
195
193
  * callback (like `resolveScopes`) — business rules live in the host
196
194
  * process, never on the wire.
197
195
  */
198
196
  readonly validators?: ValidatorRegistry;
199
- /**
200
- * §6.12 per-table server column hooks (`table` → hook). Each accepted
201
- * upsert on a listed table runs its hook inside the push transaction,
202
- * after §3.4 authorization and before §6.7 validation; the returned values
203
- * of the table's server-owned columns are stored and fanned out in the
204
- * same commit.
205
- */
206
- readonly serverColumns?: ServerColumnsRegistry;
207
197
  /**
208
198
  * §6.8 whole-commit validator. When present, the storage serializes this
209
199
  * partition before operation reads/writes, then invokes the callback once
package/src/errors.ts CHANGED
@@ -234,6 +234,14 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
234
234
  recommendedAction: 'forceResync',
235
235
  httpStatus: 404,
236
236
  },
237
+ // §1.6: a response still building at the 300 s build deadline. In-band
238
+ // after its header frames, HTTP 503 before them.
239
+ 'sync.segment_build_timeout': {
240
+ category: 'internal',
241
+ retryable: true,
242
+ recommendedAction: 'retryLater',
243
+ httpStatus: 503,
244
+ },
237
245
  'sync.segment_expired': {
238
246
  category: 'not-found',
239
247
  retryable: true,
@@ -71,6 +71,7 @@ function wrapperFor(frame: ResponseFrame): {
71
71
  switch (frame.type) {
72
72
  case 'RESP_HEADER':
73
73
  return { frames: [frame], index: 0 };
74
+ case 'KEEPALIVE':
74
75
  case 'LEASE':
75
76
  // §7.3.2: LEASE rides immediately after RESP_HEADER.
76
77
  return { frames: [stubHeader(), frame], index: 1 };