@syncular/client 0.14.0 → 0.15.0

package/README.md

@@ -3,6 +3,17 @@
 The TypeScript client protocol core (SPEC.md §§3–8, client side) plus its
 browser platform bindings.
 
+## Client-local FTS5 projections
+
+Generated schemas may attach `ftsIndexes` to a synced table (RFC 0005). The
+client materializes each as a contentful local FTS5 table with a private stable
+source identity and insert/update/delete triggers. Existing visible rows are
+bulk-indexed on first creation; schema reset recreates the projection. The FTS
+table is a read-only application query surface: it is never synced, subscribed,
+or mutated. A missing FTS5 build fails schema setup explicitly—Syncular does
+not substitute an unbounded `LIKE` scan. Indexed fields must be non-encrypted
+strings.
+
 ## Browser modes — there are exactly two
 
 **Persistent worker mode is THE mode** (REVISE Direction decision 2,

package/dist/schema.d.ts

@@ -17,6 +17,12 @@ export interface ClientIndexSpec {
     readonly columns: readonly string[];
     readonly unique: boolean;
 }
+/** One client-local contentful FTS5 projection (RFC 0005). */
+export interface ClientFtsIndexSpec {
+    readonly name: string;
+    readonly columns: readonly string[];
+    readonly tokenize: string;
+}
 export interface ClientTableSchema {
     readonly name: string;
     /** Columns in schema-IR declaration order (the row-codec order, §2.4). */
@@ -27,6 +33,8 @@ export interface ClientTableSchema {
     /** Local secondary indexes; absent in the generated schema when a table
      * declares none (typegen omits the key for index-free tables). */
     readonly indexes?: readonly ClientIndexSpec[];
+    /** Local FTS5 search projections; omitted when a table declares none. */
+    readonly ftsIndexes?: readonly ClientFtsIndexSpec[];
 }
 export interface ClientSchema {
     readonly version: number;
@@ -56,6 +64,8 @@ export interface CompiledClientTable {
     /** Local secondary indexes to create on the mirror table (declaration
      * order); empty when the table declares none. */
     readonly indexes: readonly ClientIndexSpec[];
+    /** Client-local FTS5 projections, in declaration order. */
+    readonly ftsIndexes: readonly ClientFtsIndexSpec[];
     /** §5.11: true when any column is `encrypted`. Drives the encrypt/decrypt
      * seam (skipped entirely when false) and the local-plaintext DDL. */
     readonly hasEncryptedColumns: boolean;

package/dist/schema.js

@@ -1,12 +1,25 @@
 import { ClientSyncError } from './errors.js';
 import { snakeToCamel } from './naming.js';
 const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
+const ALLOWED_FTS_TOKENIZERS = new Set([
+    'unicode61',
+    'unicode61 remove_diacritics 0',
+    'unicode61 remove_diacritics 1',
+    'unicode61 remove_diacritics 2',
+    'porter unicode61',
+    'trigram',
+]);
 export function compileClientSchema(schema) {
     const tables = new Map();
+    const schemaObjectNames = new Set();
     for (const table of schema.tables) {
         if (tables.has(table.name)) {
             throw new Error(`duplicate table ${JSON.stringify(table.name)}`);
         }
+        if (schemaObjectNames.has(table.name)) {
+            throw new Error(`table ${JSON.stringify(table.name)} conflicts with an index or FTS projection`);
+        }
+        schemaObjectNames.add(table.name);
         const columnIndex = new Map();
         table.columns.forEach((column, index) => {
             if (columnIndex.has(column.name)) {
@@ -59,12 +72,46 @@ export function compileClientSchema(schema) {
             columnIndexByCamel.delete(alias);
         const indexes = table.indexes ?? [];
         for (const index of indexes) {
+            if (schemaObjectNames.has(index.name)) {
+                throw new Error(`table ${table.name}: index ${JSON.stringify(index.name)} conflicts with another schema object`);
+            }
+            schemaObjectNames.add(index.name);
             for (const column of index.columns) {
                 if (!columnIndex.has(column)) {
                     throw new Error(`table ${table.name}: index ${JSON.stringify(index.name)} names unknown column ${JSON.stringify(column)}`);
                 }
             }
         }
+        const ftsIndexes = table.ftsIndexes ?? [];
+        for (const index of ftsIndexes) {
+            if (schemaObjectNames.has(index.name)) {
+                throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} conflicts with another schema object`);
+            }
+            schemaObjectNames.add(index.name);
+            if (index.columns.length === 0 || index.columns.length > 32) {
+                throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} needs between 1 and 32 columns`);
+            }
+            const seenColumns = new Set();
+            for (const columnName of index.columns) {
+                if (seenColumns.has(columnName)) {
+                    throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} repeats column ${JSON.stringify(columnName)}`);
+                }
+                seenColumns.add(columnName);
+                const column = table.columns.find((candidate) => candidate.name === columnName);
+                if (column === undefined) {
+                    throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} names unknown column ${JSON.stringify(columnName)}`);
+                }
+                if (localColumnType(column) !== 'string') {
+                    throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} column ${JSON.stringify(columnName)} must have string type`);
+                }
+                if (column.encrypted === true) {
+                    throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} cannot index encrypted column ${JSON.stringify(columnName)}`);
+                }
+            }
+            if (!ALLOWED_FTS_TOKENIZERS.has(index.tokenize)) {
+                throw new Error(`table ${table.name}: FTS projection ${JSON.stringify(index.name)} tokenizer ${JSON.stringify(index.tokenize)} is not allowlisted`);
+            }
+        }
         tables.set(table.name, {
             name: table.name,
             columns: table.columns,
@@ -75,6 +122,7 @@ export function compileClientSchema(schema) {
             scopeColumnByVariable,
             scopePrefixByVariable,
             indexes,
+            ftsIndexes,
             hasEncryptedColumns: table.columns.some((c) => c.encrypted === true),
         });
     }
@@ -167,6 +215,31 @@ function createSyncedTable(db, table) {
         db.exec(`CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(index.name)} ON ${quoteIdent(table.name)} (${cols})`);
     }
 }
+const FTS_SOURCE_ID_COLUMN = '_syncular_source_id';
+function createFtsProjection(db, table, index) {
+    const existed = db.query("SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?", [index.name]).length > 0;
+    const indexedColumns = index.columns.map(quoteIdent);
+    const tokenizer = index.tokenize.replaceAll("'", "''");
+    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${quoteIdent(index.name)} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED, ${indexedColumns.join(', ')}, tokenize='${tokenizer}')`);
+    const sourceId = `CAST(${quoteIdent(table.primaryKey)} AS TEXT)`;
+    const newSourceId = `CAST(new.${quoteIdent(table.primaryKey)} AS TEXT)`;
+    const oldSourceId = `CAST(old.${quoteIdent(table.primaryKey)} AS TEXT)`;
+    const projectionColumns = [
+        quoteIdent(FTS_SOURCE_ID_COLUMN),
+        ...indexedColumns,
+    ].join(', ');
+    const newValues = [
+        newSourceId,
+        ...index.columns.map((column) => `new.${quoteIdent(column)}`),
+    ].join(', ');
+    const deleteFor = (value) => `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} = ${value}`;
+    db.exec(`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`);
+    db.exec(`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_ad`)} AFTER DELETE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; END`);
+    db.exec(`CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`);
+    if (!existed) {
+        db.exec(`INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) SELECT ${sourceId}, ${indexedColumns.join(', ')} FROM ${quoteIdent(table.name)}`);
+    }
+}
 /**
  * Create the synced tables plus client bookkeeping tables (outbox,
  * subscription state, meta). Idempotent.
@@ -176,6 +249,11 @@ export function ensureLocalSchema(db, schema) {
         for (const table of schema.tables.values()) {
             createSyncedTable(db, table);
         }
+        for (const table of schema.tables.values()) {
+            for (const index of table.ftsIndexes) {
+                createFtsProjection(db, table, index);
+            }
+        }
         db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
       key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
         db.exec(`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`);
@@ -249,6 +327,14 @@ const RESERVED_TABLE_PREFIX = '_syncular_';
  * transaction and the subscription-state reset (state.ts).
  */
 export function dropAndRecreateSyncedTables(db, schema) {
+    // Drop virtual tables first. SQLite then removes their shadow tables as one
+    // unit, so the generic discovery below never tears an FTS5 projection apart.
+    const virtualTables = db.query(`SELECT name FROM sqlite_master WHERE type = 'table'
+       AND sql LIKE 'CREATE VIRTUAL TABLE%'
+       AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'`);
+    for (const row of virtualTables) {
+        db.exec(`DROP TABLE IF EXISTS ${quoteIdent(String(row.name))}`);
+    }
     const existing = db.query(`SELECT name FROM sqlite_master WHERE type = 'table'
        AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'
        AND name NOT LIKE 'sqlite_%'`);
@@ -258,6 +344,11 @@ export function dropAndRecreateSyncedTables(db, schema) {
     for (const table of schema.tables.values()) {
         createSyncedTable(db, table);
     }
+    for (const table of schema.tables.values()) {
+        for (const index of table.ftsIndexes) {
+            createFtsProjection(db, table, index);
+        }
+    }
 }
 // ---------------------------------------------------------------------------
 // Value conversion

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/client",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -81,7 +81,7 @@
   },
   "dependencies": {
     "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
-    "@syncular/core": "0.14.0"
+    "@syncular/core": "0.15.0"
   },
   "peerDependencies": {
     "better-sqlite3": ">=11"
@@ -92,7 +92,7 @@
     }
   },
   "devDependencies": {
-    "@syncular/server": "0.14.0",
+    "@syncular/server": "0.15.0",
     "@types/better-sqlite3": "^7.6.13",
     "better-sqlite3": "^12.11.1"
   }

package/src/schema.ts

@@ -19,6 +19,13 @@ export interface ClientIndexSpec {
   readonly unique: boolean;
 }
 
+/** One client-local contentful FTS5 projection (RFC 0005). */
+export interface ClientFtsIndexSpec {
+  readonly name: string;
+  readonly columns: readonly string[];
+  readonly tokenize: string;
+}
+
 export interface ClientTableSchema {
   readonly name: string;
   /** Columns in schema-IR declaration order (the row-codec order, §2.4). */
@@ -29,6 +36,8 @@ export interface ClientTableSchema {
   /** Local secondary indexes; absent in the generated schema when a table
    * declares none (typegen omits the key for index-free tables). */
   readonly indexes?: readonly ClientIndexSpec[];
+  /** Local FTS5 search projections; omitted when a table declares none. */
+  readonly ftsIndexes?: readonly ClientFtsIndexSpec[];
 }
 
 export interface ClientSchema {
@@ -60,6 +69,8 @@ export interface CompiledClientTable {
   /** Local secondary indexes to create on the mirror table (declaration
    * order); empty when the table declares none. */
   readonly indexes: readonly ClientIndexSpec[];
+  /** Client-local FTS5 projections, in declaration order. */
+  readonly ftsIndexes: readonly ClientFtsIndexSpec[];
   /** §5.11: true when any column is `encrypted`. Drives the encrypt/decrypt
    * seam (skipped entirely when false) and the local-plaintext DDL. */
   readonly hasEncryptedColumns: boolean;
@@ -71,15 +82,30 @@ export interface CompiledClientSchema {
 }
 
 const PATTERN_RE = /^([^{}]+):\{([^{}:]+)\}$/;
+const ALLOWED_FTS_TOKENIZERS = new Set([
+  'unicode61',
+  'unicode61 remove_diacritics 0',
+  'unicode61 remove_diacritics 1',
+  'unicode61 remove_diacritics 2',
+  'porter unicode61',
+  'trigram',
+]);
 
 export function compileClientSchema(
   schema: ClientSchema,
 ): CompiledClientSchema {
   const tables = new Map<string, CompiledClientTable>();
+  const schemaObjectNames = new Set<string>();
   for (const table of schema.tables) {
     if (tables.has(table.name)) {
       throw new Error(`duplicate table ${JSON.stringify(table.name)}`);
     }
+    if (schemaObjectNames.has(table.name)) {
+      throw new Error(
+        `table ${JSON.stringify(table.name)} conflicts with an index or FTS projection`,
+      );
+    }
+    schemaObjectNames.add(table.name);
     const columnIndex = new Map<string, number>();
     table.columns.forEach((column, index) => {
       if (columnIndex.has(column.name)) {
@@ -142,6 +168,12 @@ export function compileClientSchema(
     for (const alias of ambiguous) columnIndexByCamel.delete(alias);
     const indexes = table.indexes ?? [];
     for (const index of indexes) {
+      if (schemaObjectNames.has(index.name)) {
+        throw new Error(
+          `table ${table.name}: index ${JSON.stringify(index.name)} conflicts with another schema object`,
+        );
+      }
+      schemaObjectNames.add(index.name);
       for (const column of index.columns) {
         if (!columnIndex.has(column)) {
           throw new Error(
@@ -150,6 +182,52 @@ export function compileClientSchema(
         }
       }
     }
+    const ftsIndexes = table.ftsIndexes ?? [];
+    for (const index of ftsIndexes) {
+      if (schemaObjectNames.has(index.name)) {
+        throw new Error(
+          `table ${table.name}: FTS projection ${JSON.stringify(index.name)} conflicts with another schema object`,
+        );
+      }
+      schemaObjectNames.add(index.name);
+      if (index.columns.length === 0 || index.columns.length > 32) {
+        throw new Error(
+          `table ${table.name}: FTS projection ${JSON.stringify(index.name)} needs between 1 and 32 columns`,
+        );
+      }
+      const seenColumns = new Set<string>();
+      for (const columnName of index.columns) {
+        if (seenColumns.has(columnName)) {
+          throw new Error(
+            `table ${table.name}: FTS projection ${JSON.stringify(index.name)} repeats column ${JSON.stringify(columnName)}`,
+          );
+        }
+        seenColumns.add(columnName);
+        const column = table.columns.find(
+          (candidate) => candidate.name === columnName,
+        );
+        if (column === undefined) {
+          throw new Error(
+            `table ${table.name}: FTS projection ${JSON.stringify(index.name)} names unknown column ${JSON.stringify(columnName)}`,
+          );
+        }
+        if (localColumnType(column) !== 'string') {
+          throw new Error(
+            `table ${table.name}: FTS projection ${JSON.stringify(index.name)} column ${JSON.stringify(columnName)} must have string type`,
+          );
+        }
+        if (column.encrypted === true) {
+          throw new Error(
+            `table ${table.name}: FTS projection ${JSON.stringify(index.name)} cannot index encrypted column ${JSON.stringify(columnName)}`,
+          );
+        }
+      }
+      if (!ALLOWED_FTS_TOKENIZERS.has(index.tokenize)) {
+        throw new Error(
+          `table ${table.name}: FTS projection ${JSON.stringify(index.name)} tokenizer ${JSON.stringify(index.tokenize)} is not allowlisted`,
+        );
+      }
+    }
     tables.set(table.name, {
       name: table.name,
       columns: table.columns,
@@ -160,6 +238,7 @@ export function compileClientSchema(
       scopeColumnByVariable,
       scopePrefixByVariable,
       indexes,
+      ftsIndexes,
       hasEncryptedColumns: table.columns.some((c) => c.encrypted === true),
     });
   }
@@ -266,6 +345,55 @@ function createSyncedTable(
   }
 }
 
+const FTS_SOURCE_ID_COLUMN = '_syncular_source_id';
+
+function createFtsProjection(
+  db: ClientDatabase,
+  table: CompiledClientTable,
+  index: ClientFtsIndexSpec,
+): void {
+  const existed =
+    db.query(
+      "SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?",
+      [index.name],
+    ).length > 0;
+  const indexedColumns = index.columns.map(quoteIdent);
+  const tokenizer = index.tokenize.replaceAll("'", "''");
+  db.exec(
+    `CREATE VIRTUAL TABLE IF NOT EXISTS ${quoteIdent(index.name)} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED, ${indexedColumns.join(', ')}, tokenize='${tokenizer}')`,
+  );
+
+  const sourceId = `CAST(${quoteIdent(table.primaryKey)} AS TEXT)`;
+  const newSourceId = `CAST(new.${quoteIdent(table.primaryKey)} AS TEXT)`;
+  const oldSourceId = `CAST(old.${quoteIdent(table.primaryKey)} AS TEXT)`;
+  const projectionColumns = [
+    quoteIdent(FTS_SOURCE_ID_COLUMN),
+    ...indexedColumns,
+  ].join(', ');
+  const newValues = [
+    newSourceId,
+    ...index.columns.map((column) => `new.${quoteIdent(column)}`),
+  ].join(', ');
+  const deleteFor = (value: string) =>
+    `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} = ${value}`;
+
+  db.exec(
+    `CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`,
+  );
+  db.exec(
+    `CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_ad`)} AFTER DELETE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; END`,
+  );
+  db.exec(
+    `CREATE TRIGGER IF NOT EXISTS ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`,
+  );
+
+  if (!existed) {
+    db.exec(
+      `INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) SELECT ${sourceId}, ${indexedColumns.join(', ')} FROM ${quoteIdent(table.name)}`,
+    );
+  }
+}
+
 /**
  * Create the synced tables plus client bookkeeping tables (outbox,
  * subscription state, meta). Idempotent.
@@ -278,6 +406,11 @@ export function ensureLocalSchema(
     for (const table of schema.tables.values()) {
       createSyncedTable(db, table);
     }
+    for (const table of schema.tables.values()) {
+      for (const index of table.ftsIndexes) {
+        createFtsProjection(db, table, index);
+      }
+    }
     db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
       key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
     db.exec(
@@ -359,6 +492,16 @@ export function dropAndRecreateSyncedTables(
   db: ClientDatabase,
   schema: CompiledClientSchema,
 ): void {
+  // Drop virtual tables first. SQLite then removes their shadow tables as one
+  // unit, so the generic discovery below never tears an FTS5 projection apart.
+  const virtualTables = db.query(
+    `SELECT name FROM sqlite_master WHERE type = 'table'
+       AND sql LIKE 'CREATE VIRTUAL TABLE%'
+       AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'`,
+  );
+  for (const row of virtualTables) {
+    db.exec(`DROP TABLE IF EXISTS ${quoteIdent(String(row.name))}`);
+  }
   const existing = db.query(
     `SELECT name FROM sqlite_master WHERE type = 'table'
        AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'
@@ -370,6 +513,11 @@ export function dropAndRecreateSyncedTables(
   for (const table of schema.tables.values()) {
     createSyncedTable(db, table);
   }
+  for (const table of schema.tables.values()) {
+    for (const index of table.ftsIndexes) {
+      createFtsProjection(db, table, index);
+    }
+  }
 }
 
 // ---------------------------------------------------------------------------