@syncular/server 0.36.0 → 0.38.0

package/src/d1-storage.ts

@@ -1,6 +1,7 @@
 import { validateCommitPruneQuery } from './prune';
 import { StorageQueryError } from './storage-errors';
 import type { CommitPruneQuery, CommitPruneResult } from './storage';
+import { PARTITION_TOUCH_INTERVAL_MS } from './storage';
 /**
  * Cloudflare D1 server storage for Workers deployments.
  *
@@ -77,6 +78,9 @@ import {
   type StoredColumnLayout,
   scanRowPageSql,
   schemaDdl,
+  SQLITE_TABLE_INDEXES_SQL,
+  type SqliteTableIndexRow,
+  sqliteIndexSignatures,
   selectRowScopesSql,
   selectRowSql,
   selectRowsForRewriteSql,
@@ -938,7 +942,7 @@ interface D1MigrationState {
   phase: 'core' | 'prepare' | 'ddl' | 'rewrite';
   offset: number;
   columns: Record<string, string[]>;
-  indexes: Record<string, string[]>;
+  indexes: Record<string, Record<string, string>>;
   ddl: string[];
   rewrites: Array<{ table: string; oldLayout?: readonly StoredColumnLayout[] }>;
   afterPartition: string;
@@ -1269,8 +1273,9 @@ export class D1ServerStorage implements ServerStorage {
             .prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
             .all<{ name: string }>();
           const indexes = await this.#db
-            .prepare(`PRAGMA index_list(${quoteIdent(table.name)})`)
-            .all<{ name: string; origin: string }>();
+            .prepare(SQLITE_TABLE_INDEXES_SQL)
+            .bind(table.name)
+            .all<SqliteTableIndexRow>();
           // Validate the old codec before any application DDL or row rewrite.
           rewritePlan(
             table,
@@ -1296,9 +1301,7 @@ export class D1ServerStorage implements ServerStorage {
                 },
                 indexes: {
                   ...state.indexes,
-                  [table.name]: indexes.results
-                    .filter((index) => index.origin === 'c')
-                    .map((index) => index.name),
+                  [table.name]: sqliteIndexSignatures(indexes.results),
                 },
               },
               [],
@@ -1314,9 +1317,9 @@ export class D1ServerStorage implements ServerStorage {
           ]),
         );
         const indexes = new Map(
-          Object.entries(state.indexes).map(([name, names]) => [
+          Object.entries(state.indexes).map(([name, signatures]) => [
             name,
-            new Set(names),
+            new Map(Object.entries(signatures)),
           ]),
         );
         const rewrites = tables.flatMap((table) => {
@@ -1581,27 +1584,38 @@ export class D1ServerStorage implements ServerStorage {
     if (initialLogEpoch.length === 0) {
       throw new Error('initial log epoch must be non-empty');
     }
-    await this.#db
-      .prepare(
-        `INSERT INTO sync_partition_registry(
-           partition, log_epoch, last_authenticated_at_ms
-         ) VALUES (?,?,?)
-         ON CONFLICT(partition) DO UPDATE SET
-           last_authenticated_at_ms=excluded.last_authenticated_at_ms`,
-      )
-      .bind(partition, initialLogEpoch, authenticatedAtMs)
-      .run();
-    const row = await this.#db
+    const select = this.#db
       .prepare(
         `SELECT log_epoch, epoch_required, last_authenticated_at_ms
            FROM sync_partition_registry WHERE partition=?`,
       )
-      .bind(partition)
-      .first<{
+      .bind(partition);
+    let row = await select.first<{
+      log_epoch: string;
+      epoch_required: number;
+      last_authenticated_at_ms: number;
+    }>();
+    if (
+      row === null ||
+      authenticatedAtMs - row.last_authenticated_at_ms >=
+        PARTITION_TOUCH_INTERVAL_MS
+    ) {
+      await this.#db
+        .prepare(
+          `INSERT INTO sync_partition_registry(
+             partition, log_epoch, last_authenticated_at_ms
+           ) VALUES (?,?,?)
+           ON CONFLICT(partition) DO UPDATE SET
+             last_authenticated_at_ms=excluded.last_authenticated_at_ms`,
+        )
+        .bind(partition, initialLogEpoch, authenticatedAtMs)
+        .run();
+      row = await select.first<{
         log_epoch: string;
         epoch_required: number;
         last_authenticated_at_ms: number;
       }>();
+    }
     if (row === null)
       throw new Error('partition registry write did not persist');
     return {

package/src/postgres-storage.ts

@@ -1,7 +1,7 @@
 import { validateCommitPruneQuery } from './prune';
 import { StorageQueryError } from './storage-errors';
 import type { CommitPruneQuery, CommitPruneResult } from './storage';
-import { serveGateRefusal } from './storage';
+import { PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal } from './storage';
 import { serveNotReadyError } from './readiness';
 /**
  * Postgres server storage: the production database path.
@@ -81,6 +81,7 @@ import {
   type StoredColumnLayout,
   scanRowBatchSql,
   scanRowPageSql,
+  indexSignature,
   schemaDdl,
   selectRowScopesSql,
   selectRowSql,
@@ -1554,7 +1555,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
         // the migration waits.
         await client.query('LOCK TABLE sync_partitions IN EXCLUSIVE MODE');
         const existing = new Map<string, ReadonlySet<string>>();
-        const existingIndexes = new Map<string, ReadonlySet<string>>();
+        const existingIndexes = new Map<string, ReadonlyMap<string, string>>();
         for (const table of schema.tables.values()) {
           const { rows } = await client.query<{ column_name: string }>(
             `SELECT column_name FROM information_schema.columns
@@ -1568,8 +1569,25 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
             // indexes (PRIMARY KEY, UNIQUE constraints) can only be removed
             // through their constraint, so DROP INDEX on them would abort
             // the migration transaction.
-            const indexes = await client.query<{ index_name: string }>(
-              `SELECT index_class.relname AS index_name
+            const indexes = await client.query<{
+              index_name: string;
+              is_unique: boolean;
+              is_partial: boolean;
+              columns: (string | null)[];
+            }>(
+              `SELECT index_class.relname AS index_name,
+                      index_meta.indisunique AS is_unique,
+                      index_meta.indpred IS NOT NULL AS is_partial,
+                      ARRAY(
+                        SELECT attribute.attname::text
+                          FROM unnest(index_meta.indkey::int2[])
+                               WITH ORDINALITY AS key(attnum, ord)
+                          LEFT JOIN pg_catalog.pg_attribute AS attribute
+                            ON attribute.attrelid = table_class.oid
+                           AND attribute.attnum = key.attnum
+                         WHERE key.ord <= index_meta.indnkeyatts
+                         ORDER BY key.ord
+                      ) AS columns
                  FROM pg_catalog.pg_class AS table_class
                  JOIN pg_catalog.pg_namespace AS namespace
                    ON namespace.oid = table_class.relnamespace
@@ -1589,7 +1607,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
             );
             existingIndexes.set(
               table.name,
-              new Set(indexes.rows.map((index) => index.index_name)),
+              new Map(
+                indexes.rows.map((index) => [
+                  index.index_name,
+                  indexSignature(
+                    index.columns,
+                    index.is_unique,
+                    index.is_partial,
+                  ),
+                ]),
+              ),
             );
           }
         }
@@ -1667,20 +1694,35 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
     if (initialLogEpoch.length === 0) {
       throw new Error('initial log epoch must be non-empty');
     }
-    const { rows } = await this.#exec.query<{
+    type Registry = {
       log_epoch: string;
       epoch_required: boolean;
       last_authenticated_at_ms: unknown;
-    }>(
-      `INSERT INTO sync_partition_registry(
-         partition, log_epoch, last_authenticated_at_ms
-       ) VALUES ($1,$2,$3)
-       ON CONFLICT(partition) DO UPDATE SET
-         last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms
-       RETURNING log_epoch, epoch_required, last_authenticated_at_ms`,
-      [partition, initialLogEpoch, authenticatedAtMs],
-    );
-    const row = rows[0];
+    };
+    let row = (
+      await this.#exec.query<Registry>(
+        `SELECT log_epoch, epoch_required, last_authenticated_at_ms
+           FROM sync_partition_registry WHERE partition=$1`,
+        [partition],
+      )
+    ).rows[0];
+    if (
+      row === undefined ||
+      authenticatedAtMs - asNumber(row.last_authenticated_at_ms) >=
+        PARTITION_TOUCH_INTERVAL_MS
+    ) {
+      row = (
+        await this.#exec.query<Registry>(
+          `INSERT INTO sync_partition_registry(
+             partition, log_epoch, last_authenticated_at_ms
+           ) VALUES ($1,$2,$3)
+           ON CONFLICT(partition) DO UPDATE SET
+             last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms
+           RETURNING log_epoch, epoch_required, last_authenticated_at_ms`,
+          [partition, initialLogEpoch, authenticatedAtMs],
+        )
+      ).rows[0];
+    }
     if (row === undefined)
       throw new Error('partition registry write did not persist');
     return {

package/src/relational-rows.ts

@@ -53,7 +53,12 @@ import {
   type RowColumn,
   type RowValue,
 } from '@syncular/core';
-import type { CompiledSchema, CompiledTable, IndexSchema } from './schema';
+import type {
+  CompiledSchema,
+  CompiledTable,
+  IndexSchema,
+  ServerIndexSchema,
+} from './schema';
 import type { StoredRow } from './storage';
 import { StorageQueryError } from './storage-errors';
 
@@ -356,8 +361,8 @@ export function addColumnDdl(
 
 /**
  * Ownership marker for server-created projection indexes. A version bump
- * rebuilds declared indexes by dropping every index Syncular owns and
- * re-creating the declared set; this prefix is what marks an index as
+ * drops every index Syncular owns whose definition changed or was removed
+ * and creates the missing ones; this prefix is what marks an index as
  * Syncular-owned, so operator-added tuning indexes survive bumps. Portable
  * identifier validation reserves the `sync_` prefix for the server storage
  * namespace, which keeps user and operator index names out of it.
@@ -388,22 +393,113 @@ export function physicalIndexName(declaredName: string): string {
 }
 
 /**
- * CREATE INDEX IF NOT EXISTS for the table's user-declared indexes. These use
- * the declared columns with a leading server partition column. Cross-table
+ * Physical name of one server-only index. The name carries a fingerprint of
+ * the definition, so a changed definition is a different index and the bump
+ * diff needs no catalog comparison of predicates.
+ */
+export function physicalServerIndexName(index: ServerIndexSchema): string {
+  const fingerprint = fnv1a64Hex(
+    JSON.stringify([index.columns, index.where ?? null]),
+  ).slice(0, 8);
+  const readable = `${SYNC_INDEX_PREFIX}${index.name}_${fingerprint}`;
+  if (new TextEncoder().encode(readable).length <= 63) return readable;
+  return `${SYNC_INDEX_PREFIX}${fnv1a64Hex(index.name)}_${fingerprint}`;
+}
+
+/**
+ * Canonical shape of one physical index, as read from the catalog and as
+ * declared: kind, then the key columns in order (`null` for an expression).
+ * Two indexes with one signature are interchangeable.
+ */
+export function indexSignature(
+  columns: readonly (string | null)[],
+  unique: boolean,
+  partial: boolean,
+): string {
+  return `${unique ? 'unique' : 'index'}${partial ? ' partial' : ''}(${columns.map((column) => column ?? '?').join(',')})`;
+}
+
+/**
+ * One statement that lists a table's free-standing SQLite indexes with their
+ * key columns in order (bind the table name). `origin='c'` excludes
+ * constraint-owned indexes, which only their constraint can remove. Feed the
+ * rows to {@link sqliteIndexSignatures}.
+ */
+export const SQLITE_TABLE_INDEXES_SQL = `SELECT il.name AS name, il."unique" AS is_unique, il.partial AS is_partial, ii.name AS column_name
+  FROM pragma_index_list(?) AS il
+  JOIN pragma_index_info(il.name) AS ii
+ WHERE il.origin='c'
+ ORDER BY il.name, ii.seqno`;
+
+export interface SqliteTableIndexRow {
+  readonly name: string;
+  readonly is_unique: number;
+  readonly is_partial: number;
+  readonly column_name: string | null;
+}
+
+/** Index name to {@link indexSignature} for the rows of one table. */
+export function sqliteIndexSignatures(
+  rows: readonly SqliteTableIndexRow[],
+): Record<string, string> {
+  const shapes = new Map<
+    string,
+    { unique: boolean; partial: boolean; columns: (string | null)[] }
+  >();
+  for (const row of rows) {
+    const shape = shapes.get(row.name) ?? {
+      unique: row.is_unique === 1,
+      partial: row.is_partial === 1,
+      columns: [],
+    };
+    shape.columns.push(row.column_name);
+    shapes.set(row.name, shape);
+  }
+  return Object.fromEntries(
+    [...shapes].map(([name, shape]) => [
+      name,
+      indexSignature(shape.columns, shape.unique, shape.partial),
+    ]),
+  );
+}
+
+interface DesiredIndex {
+  readonly name: string;
+  /** Absent when the physical name already fingerprints the definition. */
+  readonly signature: string | undefined;
+  readonly ddl: string;
+}
+
+/**
+ * The table's declared and server-only indexes. Declared indexes use the
+ * declared columns with a leading server partition column. Cross-table
  * index-name uniqueness is the user's schema concern, as it is client-side.
  * Server-side the physical name carries the {@link SYNC_INDEX_PREFIX}
  * ownership marker.
  */
-export function createIndexDdl(table: CompiledTable): string[] {
+function desiredIndexes(table: CompiledTable): DesiredIndex[] {
   // User indexes name app columns — nothing to index without the projection.
   if (!table.materialize) return [];
-  return table.indexes.map((index) => {
-    const unique = index.unique ? 'UNIQUE ' : '';
-    const columns = [SYNC_PARTITION_COLUMN, ...index.columns]
-      .map((column) => quoteIdent(column))
-      .join(', ');
-    return `CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(physicalIndexName(index.name))} ON ${quoteIdent(table.name)} (${columns})`;
-  });
+  return [
+    ...table.indexes.map((index) => {
+      const unique = index.unique === true;
+      const columns = [SYNC_PARTITION_COLUMN, ...index.columns];
+      const name = physicalIndexName(index.name);
+      return {
+        name,
+        signature: indexSignature(columns, unique, false),
+        ddl: `CREATE ${unique ? 'UNIQUE ' : ''}INDEX IF NOT EXISTS ${quoteIdent(name)} ON ${quoteIdent(table.name)} (${columns.map((column) => quoteIdent(column)).join(', ')})`,
+      };
+    }),
+    ...table.serverIndexes.map((index) => {
+      const name = physicalServerIndexName(index);
+      return {
+        name,
+        signature: undefined,
+        ddl: `CREATE INDEX IF NOT EXISTS ${quoteIdent(name)} ON ${quoteIdent(table.name)} (${index.columns.map((column) => quoteIdent(column)).join(', ')})${index.where === undefined ? '' : ` WHERE ${index.where}`}`,
+      };
+    }),
+  ];
 }
 
 /** Idempotent removal of one Syncular-owned relational projection index. */
@@ -1039,9 +1135,10 @@ export function rewritePlan(
 /**
  * Every DDL statement to bring a database from the introspected relational
  * projections to `schema`: CREATE TABLE for absent tables, ADD COLUMN for
- * missing columns, and rebuild declared secondary indexes. Rebuilding during
- * a version bump supports DROP INDEX and same-name index replacement without
- * requiring historical index definitions in the stored column-layout marker.
+ * missing columns, and the secondary-index diff. An existing Syncular-owned
+ * index stays when the declaration still wants its physical name with the
+ * same {@link indexSignature}; every other owned index is dropped and every
+ * missing one is created, so a bump rebuilds only changed or new indexes.
  *
  * The drop set is limited to Syncular-owned indexes: physical names carrying
  * {@link SYNC_INDEX_PREFIX}, plus bare declared names (databases whose
@@ -1053,17 +1150,30 @@ export function schemaDdl(
   schema: CompiledSchema,
   existingColumnsByTable: ReadonlyMap<string, ReadonlySet<string>>,
   dialect: RelationalDialect,
-  existingIndexesByTable: ReadonlyMap<string, ReadonlySet<string>> = new Map(),
+  existingIndexesByTable: ReadonlyMap<
+    string,
+    ReadonlyMap<string, string>
+  > = new Map(),
 ): string[] {
   const drops: string[] = [];
   const creates: string[] = [];
   for (const table of schema.tables.values()) {
     const existing = existingColumnsByTable.get(table.name);
+    const desired = desiredIndexes(table);
     if (existing === undefined) {
       creates.push(createTableDdl(table, dialect));
     } else {
       const declared = new Set(table.indexes.map((index) => index.name));
-      for (const indexName of existingIndexesByTable.get(table.name) ?? []) {
+      for (const [indexName, signature] of existingIndexesByTable.get(
+        table.name,
+      ) ?? []) {
+        const want = desired.find((index) => index.name === indexName);
+        if (
+          want !== undefined &&
+          (want.signature === undefined || want.signature === signature)
+        ) {
+          continue;
+        }
         if (
           indexName.startsWith(SYNC_INDEX_PREFIX) ||
           declared.has(indexName)
@@ -1073,7 +1183,7 @@ export function schemaDdl(
       }
       creates.push(...addColumnDdl(table, existing, dialect));
     }
-    creates.push(...createIndexDdl(table));
+    creates.push(...desired.map((index) => index.ddl));
   }
   // Every drop precedes every create: an index name moving between tables in
   // one bump must release the global (SQLite) index namespace before the

package/src/schema.ts

@@ -23,6 +23,22 @@ export interface IndexSchema {
   readonly unique?: boolean;
 }
 
+/**
+ * A server-only index on a table's relational projection. Unlike a declared
+ * {@link IndexSchema}, it never reaches a client, and its `columns` are the
+ * physical columns as written: Syncular does not prepend `_sync_partition`.
+ * Name `_sync_partition` and `_sync_row_id` explicitly where an index should
+ * lead with them. The index is never unique, because uniqueness is enforced
+ * per partition. `where` is the predicate of a partial index, in SQL that
+ * the server's dialect accepts. Creating or changing one requires a schema
+ * version bump, like every index.
+ */
+export interface ServerIndexSchema {
+  readonly name: string;
+  readonly columns: readonly string[];
+  readonly where?: string;
+}
+
 /** One declared reference (§6.11). Absent `onDelete` means `RESTRICT`. */
 export interface ReferenceSchema {
   readonly column: string;
@@ -61,6 +77,8 @@ export interface TableSchema {
   readonly scopes: readonly ScopePatternSpec[];
   /** User indexes (optional) — created on the server's relational tables. */
   readonly indexes?: readonly IndexSchema[];
+  /** Server-only indexes (optional); names share the namespace of `indexes`. */
+  readonly serverIndexes?: readonly ServerIndexSchema[];
   /** Declared references (§6.11). Optional so pre-reference schemas stay
    * valid; the reference index over each child column is declared in
    * `indexes` (typegen emits it). */
@@ -104,6 +122,8 @@ export interface CompiledTable {
   readonly scopePatterns: readonly CompiledScopePattern[];
   /** User indexes (validated: unique names, existing columns). */
   readonly indexes: readonly IndexSchema[];
+  /** Server-only indexes (validated like `indexes`). */
+  readonly serverIndexes: readonly ServerIndexSchema[];
   /** Outgoing declared references (§6.11), resolved to column indices. */
   readonly references: readonly CompiledReference[];
   /** Incoming declared references: child rows to reach on a parent delete. */
@@ -222,6 +242,40 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
         }
       }
     }
+    const serverIndexes = table.serverIndexes ?? [];
+    for (const index of serverIndexes) {
+      validatePortableRelationalIdentifier(
+        `table ${table.name}: server index`,
+        index.name,
+      );
+      if (indexNames.has(index.name)) {
+        throw new Error(
+          `table ${table.name}: duplicate index ${JSON.stringify(index.name)}`,
+        );
+      }
+      indexNames.add(index.name);
+      if (index.columns.length === 0) {
+        throw new Error(
+          `table ${table.name}: server index ${JSON.stringify(index.name)} must name at least one column`,
+        );
+      }
+      for (const column of index.columns) {
+        if (
+          !columnIndex.has(column) &&
+          column !== '_sync_partition' &&
+          column !== '_sync_row_id'
+        ) {
+          throw new Error(
+            `table ${table.name}: server index ${JSON.stringify(index.name)} names unknown column ${JSON.stringify(column)}`,
+          );
+        }
+      }
+      if (index.where !== undefined && index.where.trim().length === 0) {
+        throw new Error(
+          `table ${table.name}: server index ${JSON.stringify(index.name)} has an empty where predicate`,
+        );
+      }
+    }
     const primaryKeyIndex = columnIndex.get(table.primaryKey);
     if (primaryKeyIndex === undefined) {
       throw new Error(
@@ -319,6 +373,7 @@ export function compileSchema(schema: ServerSchema): CompiledSchema {
       primaryKeyIndex,
       scopePatterns,
       indexes,
+      serverIndexes,
       references,
       referencedBy: [],
       materialize,

package/src/sqlite-bun-driver.ts

@@ -22,7 +22,7 @@ export class BunSqliteDatabase implements SqliteDatabase {
   }
 
   run(sql: string, bindings: readonly SqliteValue[] = []): SqliteRunResult {
-    return this.native.run(sql, [...bindings]);
+    return this.native.query<unknown, SQLQueryBindings[]>(sql).run(...bindings);
   }
 
   query<Row, Params extends readonly SqliteValue[]>(

package/src/sqlite-node-driver.ts

@@ -1,4 +1,8 @@
-import { DatabaseSync, type SQLInputValue } from 'node:sqlite';
+import {
+  DatabaseSync,
+  type SQLInputValue,
+  type StatementSync,
+} from 'node:sqlite';
 import type {
   SqliteDatabase,
   SqliteRunResult,
@@ -16,6 +20,8 @@ function row<Row>(value: Record<string, unknown> | undefined): Row | null {
 
 export class NodeSqliteDatabase implements SqliteDatabase {
   readonly native: DatabaseSync;
+  /** Prepared statements by SQL text, as `bun:sqlite`'s `query` keeps them. */
+  readonly #statements = new Map<string, StatementSync>();
 
   constructor(path = ':memory:') {
     this.native = new DatabaseSync(path);
@@ -26,13 +32,22 @@ export class NodeSqliteDatabase implements SqliteDatabase {
   }
 
   run(sql: string, bindings: readonly SqliteValue[] = []): SqliteRunResult {
-    return this.native.prepare(sql).run(...bindings.map(bind));
+    return this.#prepare(sql).run(...bindings.map(bind));
+  }
+
+  #prepare(sql: string): StatementSync {
+    let statement = this.#statements.get(sql);
+    if (statement === undefined) {
+      statement = this.native.prepare(sql);
+      this.#statements.set(sql, statement);
+    }
+    return statement;
   }
 
   query<Row, Params extends readonly SqliteValue[]>(
     sql: string,
   ): SqliteStatement<Row, Params> {
-    const statement = this.native.prepare(sql);
+    const statement = this.#prepare(sql);
     return {
       run: (...params) => statement.run(...params.map(bind)),
       get: (...params) => row<Row>(statement.get(...params.map(bind))),