@syncular/server 0.20.0 → 0.21.0

package/src/relational-rows.ts

@@ -3,7 +3,7 @@
  *
  * Every synced table is a REAL table in the server database — the app's
  * columns with proper type affinities, queryable with plain SQL/joins/BI —
- * plus five `_sync_*` meta columns:
+ * plus six `_sync_*` meta columns:
  *
  *   _sync_partition       TEXT      partition key (multi-partition servers
  *                                   share one database; same app PK in two
@@ -23,6 +23,10 @@
  *                                   so sync round-trips are byte-identical by
  *                                   construction. Typed columns are a
  *                                   queryable projection.
+ *   _sync_column_versions BLOB/BYTEA §2.2 per-column versions: LEB128
+ *                                   (ordinal, version) pairs for columns
+ *                                   above 1; NULL when none exceeds 1
+ *                                   (every fresh insert). Server-internal.
  *
  * PRIMARY KEY (_sync_partition, _sync_row_id).
  *
@@ -59,6 +63,67 @@ export const SYNC_ROW_ID_COLUMN = '_sync_row_id';
 export const SYNC_VERSION_COLUMN = '_sync_server_version';
 export const SYNC_SCOPES_COLUMN = '_sync_scopes';
 export const SYNC_PAYLOAD_COLUMN = '_sync_payload';
+export const SYNC_COLUMN_VERSIONS_COLUMN = '_sync_column_versions';
+
+/**
+ * §2.2 column-version storage: LEB128 `(ordinal, version)` varint pairs,
+ * ascending ordinal, for the columns whose version exceeds 1. An empty (or
+ * NULL) blob means no column exceeds 1. Server-internal: these bytes never
+ * ride the wire.
+ */
+export function encodeColumnVersions(
+  versions: ReadonlyMap<number, number>,
+): Uint8Array | undefined {
+  const bytes: number[] = [];
+  const varint = (value: number): void => {
+    let rest = value;
+    while (rest >= 0x80) {
+      bytes.push((rest & 0x7f) | 0x80);
+      rest = Math.floor(rest / 128);
+    }
+    bytes.push(rest);
+  };
+  for (const ordinal of [...versions.keys()].sort((a, b) => a - b)) {
+    const version = versions.get(ordinal) ?? 0;
+    if (version <= 1) continue;
+    varint(ordinal);
+    varint(version);
+  }
+  return bytes.length === 0 ? undefined : Uint8Array.from(bytes);
+}
+
+export function decodeColumnVersions(
+  blob: Uint8Array | undefined | null,
+): Map<number, number> {
+  const versions = new Map<number, number>();
+  if (blob === undefined || blob === null) return versions;
+  let offset = 0;
+  const varint = (): number => {
+    let value = 0;
+    let shift = 1;
+    for (;;) {
+      const byte = blob[offset];
+      if (byte === undefined) {
+        throw new Error('truncated _sync_column_versions blob');
+      }
+      offset++;
+      value += (byte & 0x7f) * shift;
+      if ((byte & 0x80) === 0) return value;
+      shift *= 128;
+    }
+  };
+  while (offset < blob.length) {
+    const ordinal = varint();
+    const version = varint();
+    if (version <= 1) {
+      throw new Error(
+        'corrupt _sync_column_versions blob: version at or below 1 is never stored',
+      );
+    }
+    versions.set(ordinal, version);
+  }
+  return versions;
+}
 
 /** SQL-standard identifier quoting (both dialects). */
 export function quoteIdent(name: string): string {
@@ -122,6 +187,7 @@ export function tableColumnNames(table: CompiledTable): string[] {
     SYNC_VERSION_COLUMN,
     SYNC_SCOPES_COLUMN,
     SYNC_PAYLOAD_COLUMN,
+    SYNC_COLUMN_VERSIONS_COLUMN,
   ];
 }
 
@@ -145,6 +211,7 @@ export function createTableDdl(
     `${quoteIdent(SYNC_VERSION_COLUMN)} ${versionType} NOT NULL`,
     `${quoteIdent(SYNC_SCOPES_COLUMN)} ${scopesType} NOT NULL`,
     `${quoteIdent(SYNC_PAYLOAD_COLUMN)} ${payloadType} NOT NULL`,
+    `${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} ${payloadType}`,
     `PRIMARY KEY (${quoteIdent(SYNC_PARTITION_COLUMN)}, ${quoteIdent(SYNC_ROW_ID_COLUMN)})`,
   ];
   return `CREATE TABLE IF NOT EXISTS ${quoteIdent(table.name)} (${defs.join(', ')})`;
@@ -153,14 +220,22 @@ export function createTableDdl(
 /**
  * ALTER TABLE ADD COLUMN statements for schema columns missing from
  * `existingColumns` (introspected). Added columns are nullable (header note).
+ * The `_sync_column_versions` meta column is added to pre-column-version
+ * tables too, materialized or not: it is storage-internal, nullable, and
+ * never part of the app-column migration subset.
  */
 export function addColumnDdl(
   table: CompiledTable,
   existingColumns: ReadonlySet<string>,
   dialect: RelationalDialect,
 ): string[] {
-  if (!table.materialize) return [];
   const out: string[] = [];
+  if (!existingColumns.has(SYNC_COLUMN_VERSIONS_COLUMN)) {
+    out.push(
+      `ALTER TABLE ${quoteIdent(table.name)} ADD COLUMN ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} ${dialect === 'postgres' ? 'BYTEA' : 'BLOB'}`,
+    );
+  }
+  if (!table.materialize) return out;
   for (const column of table.columns) {
     if (existingColumns.has(column.name)) continue;
     out.push(
@@ -267,7 +342,7 @@ export function upsertValues(
   dialect: RelationalDialect,
 ): unknown[] {
   // A non-materialized table skips the decode entirely — the upsert is a
-  // five-column meta write (the old blob-store cost).
+  // six-column meta write (the old blob-store cost).
   const projection = table.materialize
     ? decodeRow(table.columns, row.payload)
     : undefined;
@@ -282,6 +357,7 @@ export function upsertValues(
     row.serverVersion,
     JSON.stringify(row.scopes),
     row.payload,
+    row.columnVersions ?? null,
   ];
 }
 
@@ -318,7 +394,7 @@ export function selectRowSql(
   dialect: RelationalDialect,
 ): string {
   const p = dialect === 'sqlite' ? ['?', '?'] : ['$1', '$2'];
-  return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${p[0]} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${p[1]}`;
+  return `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${p[0]} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${p[1]}`;
 }
 
 export interface IndexRowPageStatement {
@@ -360,7 +436,7 @@ export function indexRowPageStatement(
   params.push(limit);
   const pageLimit = placeholder();
   return {
-    sql: `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${dialect === 'sqlite' ? '?' : '$1'} AND ${predicates.join(' AND ')} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}>${after} ORDER BY ${quoteIdent(SYNC_ROW_ID_COLUMN)} LIMIT ${pageLimit}`,
+    sql: `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${dialect === 'sqlite' ? '?' : '$1'} AND ${predicates.join(' AND ')} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}>${after} ORDER BY ${quoteIdent(SYNC_ROW_ID_COLUMN)} LIMIT ${pageLimit}`,
     params,
   };
 }
@@ -396,7 +472,7 @@ export function scanRowPageSql(
   const after = p(4 + valueCount);
   const limit = p(5 + valueCount);
   const joinPartition = dialect === 'sqlite' ? '?' : '$1';
-  return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload
+  return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
      FROM (SELECT DISTINCT row_id FROM sync_row_scopes
        WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values.join(',')})
          AND row_id>${after}

package/src/schema.ts

@@ -23,6 +23,34 @@ export interface IndexSchema {
   readonly unique?: boolean;
 }
 
+/** One declared reference (§6.11). Absent `onDelete` means `RESTRICT`. */
+export interface ReferenceSchema {
+  readonly column: string;
+  readonly parentTable: string;
+  readonly onDelete?: ReferenceOnDelete;
+}
+
+export type ReferenceOnDelete = 'RESTRICT' | 'CASCADE' | 'SET NULL';
+
+/** A compiled outgoing reference (§6.11). */
+export interface CompiledReference {
+  readonly column: string;
+  readonly columnIndex: number;
+  readonly parentTable: string;
+  readonly onDelete: ReferenceOnDelete;
+}
+
+/** A compiled INCOMING reference: rows of `table` whose `column` points at a
+ * row of the table that carries this entry. `index` is the declared
+ * single-column index the server probes through `scanRowsByIndex` (§6.8). */
+export interface CompiledIncomingReference {
+  readonly table: string;
+  readonly column: string;
+  readonly columnIndex: number;
+  readonly onDelete: ReferenceOnDelete;
+  readonly index: string;
+}
+
 export interface TableSchema {
   readonly name: string;
   /** Columns in schema-IR declaration order (the row-codec order, §2.4). */
@@ -33,6 +61,10 @@ export interface TableSchema {
   readonly scopes: readonly ScopePatternSpec[];
   /** User indexes (optional) — created on the server's relational tables. */
   readonly indexes?: readonly IndexSchema[];
+  /** Declared references (§6.11). Optional so pre-reference schemas stay
+   * valid; the reference index over each child column is declared in
+   * `indexes` (typegen emits it). */
+  readonly references?: readonly ReferenceSchema[];
   /**
    * Server-side column materialization
    * "optional materialization"). When `true` (the usual default) the server's
@@ -72,6 +104,10 @@ export interface CompiledTable {
   readonly scopePatterns: readonly CompiledScopePattern[];
   /** User indexes (validated: unique names, existing columns). */
   readonly indexes: readonly IndexSchema[];
+  /** Outgoing declared references (§6.11), resolved to column indices. */
+  readonly references: readonly CompiledReference[];
+  /** Incoming declared references: child rows to reach on a parent delete. */
+  readonly referencedBy: readonly CompiledIncomingReference[];
   /** Resolved materialization (see `TableSchema.materialize`). */
   readonly materialize: boolean;
   readonly columnIndex: ReadonlyMap<string, number>;
@@ -216,6 +252,21 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
     const blobRefColumnIndices: number[] = [];
     const crdtColumns: { index: number; crdtType: string }[] = [];
     const encryptedColumnIndices: number[] = [];
+    const references: CompiledReference[] = [];
+    for (const reference of table.references ?? []) {
+      const index = columnIndex.get(reference.column);
+      if (index === undefined) {
+        throw new Error(
+          `table ${table.name}: reference column ${JSON.stringify(reference.column)} is not a column`,
+        );
+      }
+      references.push({
+        column: reference.column,
+        columnIndex: index,
+        parentTable: reference.parentTable,
+        onDelete: reference.onDelete ?? 'RESTRICT',
+      });
+    }
     table.columns.forEach((column, index) => {
       if (column.type === 'blob_ref') blobRefColumnIndices.push(index);
       // §5.11: an encrypted column carries wire type `bytes` + `encrypted`.
@@ -253,6 +304,8 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
       primaryKeyIndex,
       scopePatterns,
       indexes,
+      references,
+      referencedBy: [],
       materialize,
       columnIndex,
       declaredVariables: variables,
@@ -261,6 +314,62 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
       encryptedColumnIndices,
     });
   }
+  // §6.11: resolve incoming references and enforce the declaration rules the
+  // generator enforces too, so a hand-written server schema fails loud.
+  for (const child of tables.values()) {
+    for (const reference of child.references) {
+      const parent = tables.get(reference.parentTable);
+      if (parent === undefined) {
+        throw new Error(
+          `table ${child.name}: reference column ${JSON.stringify(reference.column)} names unknown table ${JSON.stringify(reference.parentTable)}`,
+        );
+      }
+      const childColumn = child.columns[reference.columnIndex];
+      const parentKey = parent.columns[parent.primaryKeyIndex];
+      if (childColumn === undefined || parentKey === undefined) {
+        throw new Error(
+          `unreachable: ${child.name}.${reference.column} or a missing primary key on ${parent.name}`,
+        );
+      }
+      if (childColumn.type !== parentKey.type) {
+        throw new Error(
+          `table ${child.name}: reference column ${JSON.stringify(reference.column)} has type ${JSON.stringify(childColumn.type)} but ${parent.name}.${parentKey.name} has type ${JSON.stringify(parentKey.type)} — the types must match (§6.11)`,
+        );
+      }
+      if (reference.onDelete === 'SET NULL' && !childColumn.nullable) {
+        throw new Error(
+          `table ${child.name}: reference column ${JSON.stringify(reference.column)} is not nullable but declares ON DELETE SET NULL (§6.11)`,
+        );
+      }
+      const patterns = (compiled: CompiledTable): string =>
+        compiled.scopePatterns
+          .map((pattern) => `${pattern.prefix}:{${pattern.variable}}`)
+          .sort()
+          .join('\u0000');
+      if (patterns(child) !== patterns(parent)) {
+        throw new Error(
+          `table ${child.name}: reference column ${JSON.stringify(reference.column)} targets ${parent.name}, whose scope patterns differ — a cascade MUST NOT cross an authorization boundary (§6.11)`,
+        );
+      }
+      const index = child.indexes.find(
+        (candidate) =>
+          candidate.columns.length === 1 &&
+          candidate.columns[0] === reference.column,
+      );
+      if (index === undefined) {
+        throw new Error(
+          `table ${child.name}: reference column ${JSON.stringify(reference.column)} needs a declared single-column index so the server reaches children through scanRowsByIndex (§6.11)`,
+        );
+      }
+      (parent.referencedBy as CompiledIncomingReference[]).push({
+        table: child.name,
+        column: reference.column,
+        columnIndex: reference.columnIndex,
+        onDelete: reference.onDelete,
+        index: index.name,
+      });
+    }
+  }
   const compiled: CompiledSchema = {
     version: schema.version,
     tables,

package/src/seed.ts

@@ -12,7 +12,7 @@
 import {
   decodeMessage,
   encodeMessage,
-  encodeRow,
+  encodeSparseRow,
   PROTOCOL_WIRE_VERSION,
   type PushOperation,
   type PushResultFrame,
@@ -182,7 +182,7 @@ export async function seedMutations(
       table: mutation.table,
       rowId,
       op: 'upsert',
-      payload: encodeRow(table.columns, values),
+      payload: encodeSparseRow(table.columns, pkIndex, values),
     };
   });
 

package/src/sqlite-dialect.ts

@@ -114,6 +114,11 @@ CREATE TABLE IF NOT EXISTS sync_blob_refs(
 );
 CREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob
   ON sync_blob_refs(partition, blob_id);
+CREATE TABLE IF NOT EXISTS sync_tombstones(
+  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
+  commit_seq INTEGER NOT NULL,
+  PRIMARY KEY(partition, tbl, row_id)
+);
 `;
 
 /** Split the DDL into individual statements (D1 applies them one by one). */
@@ -151,6 +156,7 @@ interface SerializedResult {
   message?: string;
   serverVersion?: number;
   serverRow?: string;
+  conflictColumns?: string;
   retryable?: boolean;
   details?: import('@syncular/core').RejectionDetails;
 }
@@ -175,6 +181,7 @@ export function serializePushResult(result: StoredPushResult): string {
           message: record.message,
           serverVersion: record.serverVersion,
           serverRow: bytesToBase64(record.serverRow),
+          conflictColumns: bytesToBase64(record.conflictColumns),
         };
       }
       if (record.status === 'error') {
@@ -209,6 +216,7 @@ export function deserializePushResult(text: string): StoredPushResult {
         message: record.message ?? '',
         serverVersion: record.serverVersion ?? 0,
         serverRow: base64ToBytes(record.serverRow ?? ''),
+        conflictColumns: base64ToBytes(record.conflictColumns ?? ''),
       };
     }
     if (record.status === 'error') {
@@ -242,6 +250,7 @@ export interface SqliteRowRecord {
   server_version: number;
   scopes: string;
   payload: Uint8Array;
+  column_versions: Uint8Array | null;
 }
 
 export interface SqliteChangeRecord {
@@ -267,6 +276,9 @@ export function toStoredRow(record: SqliteRowRecord): StoredRow {
     serverVersion: record.server_version,
     scopes: JSON.parse(record.scopes) as Record<string, string>,
     payload: asUint8Array(record.payload),
+    ...(record.column_versions !== null
+      ? { columnVersions: asUint8Array(record.column_versions) }
+      : {}),
   };
 }
 

package/src/sqlite-storage.ts

@@ -163,6 +163,28 @@ class SqliteTransaction implements StorageTransaction {
     return this.#storage.getRow(this.#partition, table, rowId);
   }
 
+  async getTombstoneSeq(
+    table: string,
+    rowId: string,
+  ): Promise<number | undefined> {
+    this.#assertOpen();
+    const row = this.#storage.db
+      .query<{ commit_seq: number }, [string, string, string]>(
+        'SELECT commit_seq FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id=?',
+      )
+      .get(this.#partition, table, rowId);
+    return row?.commit_seq;
+  }
+
+  async clearTombstone(table: string, rowId: string): Promise<void> {
+    this.#assertOpen();
+    this.#storage.db
+      .query(
+        'DELETE FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id=?',
+      )
+      .run(this.#partition, table, rowId);
+  }
+
   getPushResult(
     clientId: string,
     clientCommitId: string,
@@ -302,6 +324,14 @@ class SqliteTransaction implements StorageTransaction {
         JSON.stringify(change.scopes),
         change.payload ?? null,
       );
+      if (change.op === 'delete') {
+        // §5 delete precedence: every applied delete leaves a tombstone,
+        // pruned with the commit log (§4.6).
+        db.query(
+          `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq) VALUES (?,?,?,?)
+           ON CONFLICT(partition, tbl, row_id) DO UPDATE SET commit_seq=excluded.commit_seq`,
+        ).run(p, change.table, change.rowId, commitSeq);
+      }
       for (const [variable, value] of Object.entries(change.scopes)) {
         db.query(
           'INSERT OR IGNORE INTO sync_change_scopes(partition, tbl, var, value, commit_seq) VALUES (?,?,?,?,?)',
@@ -833,6 +863,12 @@ export class SqliteServerStorage implements ServerStorage {
             'DELETE FROM sync_change_scopes WHERE partition=? AND commit_seq<=?',
           )
           .run(partition, horizonSeq);
+        // §4.6: tombstones prune in the same pass as the commit log.
+        this.db
+          .query(
+            'DELETE FROM sync_tombstones WHERE partition=? AND commit_seq<=?',
+          )
+          .run(partition, horizonSeq);
         this.db.exec('COMMIT');
         return {
           previousHorizonSeq,

package/src/storage.ts

@@ -40,6 +40,13 @@ export interface StoredRow {
   readonly scopes: Record<string, string>;
   /** Row-codec payload (§2.4) for the server's schema version. */
   readonly payload: Uint8Array;
+  /**
+   * §2.2 per-column versions: the `_sync_column_versions` blob (LEB128
+   * (ordinal, version) pairs for columns above 1). Absent when no column
+   * exceeds 1, and for rows written before column versions existed — a row
+   * without the blob carries every column at `serverVersion`.
+   */
+  readonly columnVersions?: Uint8Array;
 }
 
 export interface NewChange {
@@ -312,6 +319,20 @@ export interface AuthoritativeQueryResult {
  */
 export interface StorageTransaction {
   getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
+  /**
+   * §5 delete-precedence tombstone read: the `commit_seq` of the newest
+   * applied delete of this row still within the pruning horizon, or
+   * `undefined` when no tombstone exists. Consulted only when an upsert's
+   * stored row is absent. Required: in-tree adapters implement it on the
+   * transaction's own connection with read-your-own-writes semantics.
+   */
+  getTombstoneSeq(table: string, rowId: string): Promise<number | undefined>;
+  /**
+   * Remove the row's tombstone because the row was recreated in this
+   * transaction (§5: an explicit insert recreates). Called on every insert
+   * over an absent row; a no-op when no tombstone exists.
+   */
+  clearTombstone(table: string, rowId: string): Promise<void>;
   /**
    * Optional transaction-scoped idempotency lookup (§2.3) with the same
    * semantics as `ServerStorage.getPushResult`, including the