@syncular/typegen 0.15.40 → 0.15.43

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/typegen",
-  "version": "0.15.40",
+  "version": "0.15.43",
   "description": "Syncular schema-to-TypeScript type generator and the `syncular` CLI",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -56,9 +56,9 @@
     "!dist/**/*.test.d.ts"
   ],
   "dependencies": {
-    "@syncular/core": "0.15.40"
+    "@syncular/core": "0.15.43"
   },
   "devDependencies": {
-    "@syncular/server": "0.15.40"
+    "@syncular/server": "0.15.43"
   }
 }

package/src/generate.ts

@@ -44,6 +44,7 @@ import {
 import {
   buildMigrationLock,
   LEGACY_MIGRATION_LOCK_FORMAT_VERSION,
+  lockedMigrationNames,
   MIGRATION_LOCK_FILENAME,
   readMigrationLock,
   serializeMigrationLock,
@@ -268,10 +269,13 @@ function buildSchemaVersions(
   return out;
 }
 
-/** Pure IR construction — the testable heart of the generator. */
+/** Pure IR construction — the testable heart of the generator. Migrations
+ * named in `lockedNames` replay as immutable deployed history: rules that
+ * only appended migrations must satisfy are skipped for them. */
 export function buildIr(
   manifest: Manifest,
   migrations: readonly MigrationInput[],
+  lockedNames?: ReadonlySet<string>,
 ): IrDocument {
   if (migrations.length === 0) {
     throw new TypegenError(MANIFEST_FILENAME, 'no migrations found');
@@ -284,6 +288,7 @@ export function buildIr(
       migration.sql,
       `${migration.name}/up.sql`,
       droppedTables,
+      { lockedHistory: lockedNames?.has(migration.name) === true },
     );
   }
   validateFinalSchemaIdentifiers(parsedTables);
@@ -600,21 +605,35 @@ export function upgradeMigrationHistory(manifestDir: string): GeneratedOutput {
   }
   // Validate the immutable prefix before replacing its representation.
   updateMigrationLock(locked, migrations);
-  buildIr(manifest, migrations);
+  // upgrade-lock converts the committed representation of history that is
+  // already locked. Extending the lock over appended migrations is its own
+  // reviewable act, so unlocked migrations stop the conversion here.
+  if (migrations.length > locked.migrations.length) {
+    const appended = migrations
+      .slice(locked.migrations.length)
+      .map((migration) => migration.name)
+      .join(', ');
+    throw new TypegenError(
+      MIGRATION_LOCK_FILENAME,
+      `migrations ${appended} are appended beyond the locked history — run generate to extend the lock over them, commit the extended lock, then rerun \`migrations upgrade-lock\``,
+    );
+  }
+  const lockedNames = lockedMigrationNames(locked);
+  buildIr(manifest, migrations, lockedNames);
   return {
     path: resolve(manifestDir, MIGRATION_LOCK_FILENAME),
-    content: serializeMigrationLock(buildMigrationLock(migrations)),
+    content: serializeMigrationLock(
+      buildMigrationLock(migrations, undefined, lockedNames),
+    ),
   };
 }
 
 /** Fast CI check for migration history without emitting/query analysis. */
 export function checkMigrationHistory(manifestDir: string): string[] {
   const { manifest, migrations } = loadProjectInputs(manifestDir);
-  const current = updateMigrationLock(
-    readMigrationLock(manifestDir),
-    migrations,
-  );
-  buildIr(manifest, migrations);
+  const locked = readMigrationLock(manifestDir);
+  const current = updateMigrationLock(locked, migrations);
+  buildIr(manifest, migrations, lockedMigrationNames(locked));
   const output = {
     path: resolve(manifestDir, MIGRATION_LOCK_FILENAME),
     content: serializeMigrationLock(current),
@@ -630,11 +649,9 @@ export function checkMigrationHistory(manifestDir: string): string[] {
 /** Read `syncular.json` + migrations under `manifestDir`; build outputs. */
 export function generate(manifestDir: string): GenerateResult {
   const { manifest, migrations } = loadProjectInputs(manifestDir);
-  const migrationLock = updateMigrationLock(
-    readMigrationLock(manifestDir),
-    migrations,
-  );
-  const ir = buildIr(manifest, migrations);
+  const locked = readMigrationLock(manifestDir);
+  const migrationLock = updateMigrationLock(locked, migrations);
+  const ir = buildIr(manifest, migrations, lockedMigrationNames(locked));
 
   // §5/§12 naming: the emitter targets this run generates (keyword hazards
   // are only real on targets that exist), and the per-table collision check

package/src/lsp.ts

@@ -7,6 +7,7 @@ import { formatSyql } from './fmt';
 import { buildIr, loadMigrations, loadQueries, makeQueryDb } from './generate';
 import type { IrDocument } from './ir';
 import { MANIFEST_FILENAME, parseManifest } from './manifest';
+import { lockedMigrationNames, readMigrationLock } from './migration-lock';
 import type { QueryDb, QueryNamingOptions } from './query';
 import { SyqlFrontendError, type SyqlSourceSpan } from './syql-lexer';
 import type { SyqlLoweredQuery } from './syql-lowering';
@@ -260,9 +261,18 @@ export class SyqlLanguageServer {
           readFileSync(resolve(manifestDir, MANIFEST_FILENAME), 'utf8'),
         ),
       );
+      // Tolerate locked history exactly like generation does; a project with
+      // no readable lock builds strictly.
+      let lockedNames: ReadonlySet<string> | undefined;
+      try {
+        lockedNames = lockedMigrationNames(readMigrationLock(manifestDir));
+      } catch {
+        lockedNames = undefined;
+      }
       const ir = buildIr(
         manifest,
         loadMigrations(resolve(manifestDir, manifest.migrations)),
+        lockedNames,
       );
       const { db } = makeQueryDb(ir);
       const targets: QueryNamingOptions['targets'][number][] = ['ts'];

package/src/migration-lock.ts

@@ -81,18 +81,29 @@ function snapshotColumn(column: IrColumn): MigrationLockColumn {
 function snapshotTables(
   tables: ReadonlyMap<string, ParsedTable>,
 ): MigrationLockTable[] {
-  return [...tables.values()]
-    .sort((a, b) => a.name.localeCompare(b.name))
-    .map((table) => ({
-      name: table.name,
-      primaryKey: table.primaryKey,
-      columns: table.columns.map(snapshotColumn),
-    }));
+  return (
+    [...tables.values()]
+      // Code-point order keeps the serialization byte-identical across locales.
+      .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+      .map((table) => ({
+        name: table.name,
+        primaryKey: table.primaryKey,
+        columns: table.columns.map(snapshotColumn),
+      }))
+  );
 }
 
-/** Replay a complete migration sequence with ephemeral diagnostic snapshots. */
+/** The names of every migration a committed lock has made immutable. */
+export function lockedMigrationNames(lock: MigrationLock): ReadonlySet<string> {
+  return new Set(lock.migrations.map((migration) => migration.name));
+}
+
+/** Replay a complete migration sequence with ephemeral diagnostic snapshots.
+ * Migrations named in `lockedNames` replay as deployed history; rules that
+ * only appended migrations must satisfy are skipped for them. */
 function buildMigrationHistory(
   migrations: readonly MigrationInput[],
+  lockedNames?: ReadonlySet<string>,
 ): CurrentMigrationHistory {
   const tables = new Map<string, ParsedTable>();
   const droppedTables = new Set<string>();
@@ -103,6 +114,7 @@ function buildMigrationHistory(
       migration.sql,
       `${migration.name}/up.sql`,
       droppedTables,
+      { lockedHistory: lockedNames?.has(migration.name) === true },
     );
     entries.push({
       name: migration.name,
@@ -145,8 +157,12 @@ export function buildMigrationLock(
   formatVersion:
     | typeof LEGACY_MIGRATION_LOCK_FORMAT_VERSION
     | typeof MIGRATION_LOCK_FORMAT_VERSION = MIGRATION_LOCK_FORMAT_VERSION,
+  lockedNames?: ReadonlySet<string>,
 ): MigrationLock {
-  return lockFromHistory(buildMigrationHistory(migrations), formatVersion);
+  return lockFromHistory(
+    buildMigrationHistory(migrations, lockedNames),
+    formatVersion,
+  );
 }
 
 /** Fixed-order, byte-deterministic serialization for clean code review. */
@@ -447,7 +463,10 @@ export function updateMigrationLock(
   locked: MigrationLock,
   migrations: readonly MigrationInput[],
 ): MigrationLock {
-  const current = buildMigrationHistory(migrations);
+  const current = buildMigrationHistory(
+    migrations,
+    lockedMigrationNames(locked),
+  );
   validateMigrationHistory(locked, current);
   return lockFromHistory(current, locked.formatVersion);
 }
@@ -457,5 +476,8 @@ export function validateMigrationLock(
   locked: MigrationLock,
   migrations: readonly MigrationInput[],
 ): void {
-  validateMigrationHistory(locked, buildMigrationHistory(migrations));
+  validateMigrationHistory(
+    locked,
+    buildMigrationHistory(migrations, lockedMigrationNames(locked)),
+  );
 }

package/src/query.ts

@@ -767,6 +767,11 @@ function hasCommaJoinedSchemaTable(sql: string, ir: IrDocument): boolean {
   ]);
   let depth = 0;
   let cursor = 0;
+  /** The previous significant token: an uppercased word or a single
+   * punctuation character. Distinguishes a table-source group — a `(` after
+   * FROM, JOIN, `,` or another `(` — from a function call or parenthesized
+   * expression, whose `(` follows an identifier or an operator. */
+  let previous: string | undefined;
   const nextIdentifier = (start: number): string | undefined => {
     let index = start;
     while (
@@ -781,9 +786,19 @@ function hasCommaJoinedSchemaTable(sql: string, ir: IrDocument): boolean {
 
   while (cursor < cleaned.length) {
     const char = cleaned[cursor] as string;
+    if (/\s/.test(char)) {
+      cursor += 1;
+      continue;
+    }
     if (char === '(') {
+      const opensTableSource =
+        previous === 'FROM' ||
+        previous === 'JOIN' ||
+        previous === ',' ||
+        previous === '(';
       const next = nextIdentifier(cursor + 1);
       if (
+        opensTableSource &&
         activeFromDepths.has(depth) &&
         next !== undefined &&
         known.has(next.toLowerCase())
@@ -791,18 +806,21 @@ function hasCommaJoinedSchemaTable(sql: string, ir: IrDocument): boolean {
         activeFromDepths.add(depth + 1);
       }
       depth += 1;
+      previous = '(';
       cursor += 1;
       continue;
     }
     if (char === ')') {
       activeFromDepths.delete(depth);
       depth = Math.max(0, depth - 1);
+      previous = ')';
       cursor += 1;
       continue;
     }
     if (char === ',' && activeFromDepths.has(depth)) {
       const next = nextIdentifier(cursor + 1);
       if (next !== undefined && known.has(next.toLowerCase())) return true;
+      previous = ',';
       cursor += 1;
       continue;
     }
@@ -817,9 +835,11 @@ function hasCommaJoinedSchemaTable(sql: string, ir: IrDocument): boolean {
       const word = cleaned.slice(cursor, end).toUpperCase();
       if (word === 'FROM') activeFromDepths.add(depth);
       else if (clauseEnders.has(word)) activeFromDepths.delete(depth);
+      previous = word;
       cursor = end;
       continue;
     }
+    previous = char;
     cursor += 1;
   }
   return false;

package/src/sql.ts

@@ -80,11 +80,32 @@ const DATA_MUTATION_GUIDANCE =
 const tableIdentifierSources = new WeakMap<ParsedTable, string>();
 const columnIdentifierSources = new WeakMap<IrColumn, string>();
 const indexIdentifierSources = new WeakMap<IrIndex, string>();
+const ftsIndexIdentifierSources = new WeakMap<IrFtsIndex, string>();
+
+/** Objects created while replaying the locked, immutable history prefix.
+ * Rules adopted while that prefix was already deployed (nullable-only ADD
+ * COLUMN, ASCII-portable identifiers) replay tolerantly for these objects;
+ * every migration beyond the locked prefix enforces them. */
+const lockedHistoryObjects = new WeakSet<object>();
+
+/** Options for replaying one migration's SQL. */
+export interface ApplyMigrationSqlOptions {
+  /** True while this migration is part of the locked, immutable history
+   * prefix (`syncular.migrations.lock.json`). Deployed migrations are
+   * immutable, so checks that would require editing them are skipped for the
+   * locked prefix and enforced for appended migrations. */
+  readonly lockedHistory?: boolean;
+}
+
+/** ASCII identifier contract shared with named-query analysis: query
+ * dependency tracking and reactivity resolve identifiers with this shape. */
+const PORTABLE_ASCII_IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
 
 function validateIdentifier(
   source: string,
   kind: string,
   identifier: string,
+  target: object,
 ): void {
   try {
     validatePortableRelationalIdentifier(kind, identifier);
@@ -94,25 +115,35 @@ function validateIdentifier(
       error instanceof Error ? error.message : 'invalid relational identifier',
     );
   }
+  if (lockedHistoryObjects.has(target)) return;
+  if (!PORTABLE_ASCII_IDENTIFIER_RE.test(identifier)) {
+    throw new TypegenError(
+      source,
+      `${kind} name ${JSON.stringify(identifier)} must be an ASCII identifier matching [A-Za-z_][A-Za-z0-9_]* — named-query analysis resolves ASCII identifiers, so a non-ASCII name silently loses dependency tracking and reactivity`,
+    );
+  }
 }
 
 /**
  * Validate the accumulated head schema after every migration has been
  * applied. This deliberately permits a locked historical migration to create
  * an invalid identifier when a later forward migration drops it; only final
- * generated objects must be portable.
+ * generated objects must be portable. Objects created by locked migrations
+ * are additionally exempt from the ASCII rule (deployed migrations are
+ * immutable, so the name can only be retired by a forward migration).
  */
 export function validateFinalSchemaIdentifiers(
   tables: ReadonlyMap<string, ParsedTable>,
 ): void {
   for (const table of tables.values()) {
     const tableSource = tableIdentifierSources.get(table) ?? 'migrations';
-    validateIdentifier(tableSource, 'table', table.name);
+    validateIdentifier(tableSource, 'table', table.name, table);
     for (const column of table.columns) {
       validateIdentifier(
         columnIdentifierSources.get(column) ?? tableSource,
         `table ${table.name}: column`,
         column.name,
+        column,
       );
     }
     for (const index of table.indexes) {
@@ -120,6 +151,15 @@ export function validateFinalSchemaIdentifiers(
         indexIdentifierSources.get(index) ?? tableSource,
         `table ${table.name}: index`,
         index.name,
+        index,
+      );
+    }
+    for (const ftsIndex of table.ftsIndexes) {
+      validateIdentifier(
+        ftsIndexIdentifierSources.get(ftsIndex) ?? tableSource,
+        `table ${table.name}: FTS virtual table`,
+        ftsIndex.name,
+        ftsIndex,
       );
     }
   }
@@ -359,22 +399,23 @@ function parseCreate(
   tables: Map<string, ParsedTable>,
   droppedTables: ReadonlySet<string>,
   source: string,
+  options: ApplyMigrationSqlOptions,
 ): void {
   if (cursor.eatWord('VIRTUAL')) {
-    parseCreateVirtualTable(cursor, tables);
+    parseCreateVirtualTable(cursor, tables, source, options);
     return;
   }
   if (cursor.eatWord('TABLE')) {
-    parseCreateTable(cursor, tables, droppedTables, source);
+    parseCreateTable(cursor, tables, droppedTables, source, options);
     return;
   }
   if (cursor.eatWord('UNIQUE')) {
     cursor.expectWord('INDEX', 'after CREATE UNIQUE');
-    parseCreateIndex(cursor, tables, true, source);
+    parseCreateIndex(cursor, tables, true, source, options);
     return;
   }
   if (cursor.eatWord('INDEX')) {
-    parseCreateIndex(cursor, tables, false, source);
+    parseCreateIndex(cursor, tables, false, source, options);
     return;
   }
   const token = cursor.peek();
@@ -398,6 +439,8 @@ const ALLOWED_FTS_TOKENIZERS = new Set([
 function parseCreateVirtualTable(
   cursor: Cursor,
   tables: Map<string, ParsedTable>,
+  source: string,
+  options: ApplyMigrationSqlOptions,
 ): void {
   cursor.expectWord('TABLE', 'after CREATE VIRTUAL');
   if (cursor.eatWord('IF')) {
@@ -510,7 +553,10 @@ function parseCreateVirtualTable(
       );
     }
   }
-  table.ftsIndexes.push({ name, columns, tokenize });
+  const ftsIndex = { name, columns, tokenize };
+  table.ftsIndexes.push(ftsIndex);
+  ftsIndexIdentifierSources.set(ftsIndex, source);
+  if (options.lockedHistory === true) lockedHistoryObjects.add(ftsIndex);
 }
 
 function parseCreateTable(
@@ -518,6 +564,7 @@ function parseCreateTable(
   tables: Map<string, ParsedTable>,
   droppedTables: ReadonlySet<string>,
   source: string,
+  options: ApplyMigrationSqlOptions,
 ): void {
   if (cursor.eatWord('IF')) {
     cursor.expectWord('NOT', 'after IF');
@@ -618,8 +665,10 @@ function parseCreateTable(
   };
   tables.set(name, table);
   tableIdentifierSources.set(table, source);
+  if (options.lockedHistory === true) lockedHistoryObjects.add(table);
   for (const column of finalColumns) {
     columnIdentifierSources.set(column, source);
+    if (options.lockedHistory === true) lockedHistoryObjects.add(column);
   }
 }
 
@@ -636,6 +685,7 @@ function parseCreateIndex(
   tables: Map<string, ParsedTable>,
   unique: boolean,
   source: string,
+  options: ApplyMigrationSqlOptions,
 ): void {
   // `INDEX` already matched by the dispatcher.
   if (cursor.eatWord('IF')) {
@@ -715,12 +765,14 @@ function parseCreateIndex(
   const index: IrIndex = { name, columns, unique };
   table.indexes.push(index);
   indexIdentifierSources.set(index, source);
+  if (options.lockedHistory === true) lockedHistoryObjects.add(index);
 }
 
 function parseAlterTable(
   cursor: Cursor,
   tables: Map<string, ParsedTable>,
   source: string,
+  options: ApplyMigrationSqlOptions,
 ): void {
   cursor.expectWord('TABLE', 'after ALTER');
   const name = cursor.identifier('a table name');
@@ -737,7 +789,9 @@ function parseAlterTable(
   cursor.eatWord('COLUMN');
   const def = parseColumnDef(cursor, false);
   cursor.expectEnd();
-  if (!def.column.nullable) {
+  // A locked migration replays as deployed; the nullable rule applies to
+  // migrations beyond the locked prefix, where the SQL can still change.
+  if (!def.column.nullable && options.lockedHistory !== true) {
     cursor.fail(
       `ALTER TABLE ${name}: added column ${JSON.stringify(def.column.name)} must be nullable — SQL defaults do not backfill Syncular row payloads; add the column nullable, backfill it through versioned server-authoritative writes, and enforce required values in application validation`,
     );
@@ -749,6 +803,7 @@ function parseAlterTable(
   }
   table.columns.push(def.column);
   columnIdentifierSources.set(def.column, source);
+  if (options.lockedHistory === true) lockedHistoryObjects.add(def.column);
 }
 
 /** `DROP TABLE [IF EXISTS] name`; table-name reuse remains unsupported. */
@@ -792,6 +847,17 @@ function parseDropIndex(
     table.indexes.splice(index, 1);
     return;
   }
+  for (const table of tables.values()) {
+    if (table.ftsIndexes.some((candidate) => candidate.name === name)) {
+      // `DROP INDEX IF EXISTS <fts>` matches SQLite: an FTS5 virtual table is
+      // no regular index, so IF EXISTS resolves to a tolerant no-op (this also
+      // keeps a locked migration carrying the statement replayable).
+      if (ifExists) return;
+      cursor.fail(
+        `DROP INDEX ${name}: ${name} is an FTS5 virtual table (CREATE VIRTUAL TABLE … USING fts5); the migration subset removes an FTS projection together with its owning content table (DROP TABLE ${table.name})`,
+      );
+    }
+  }
   if (!ifExists) {
     cursor.fail(`DROP INDEX ${name}: index does not exist at this point`);
   }
@@ -825,15 +891,16 @@ export function applyMigrationSql(
   sql: string,
   source: string,
   droppedTables: Set<string> = new Set<string>(),
+  options: ApplyMigrationSqlOptions = {},
 ): void {
   for (const tokens of tokenizeStatements(sql, source)) {
     const cursor = new Cursor(tokens, source);
     const head = cursor.next();
     const word = head.kind === 'word' ? head.text.toUpperCase() : '';
     if (word === 'CREATE') {
-      parseCreate(cursor, tables, droppedTables, source);
+      parseCreate(cursor, tables, droppedTables, source, options);
     } else if (word === 'ALTER') {
-      parseAlterTable(cursor, tables, source);
+      parseAlterTable(cursor, tables, source, options);
     } else if (word === 'DROP') {
       parseDrop(cursor, tables, droppedTables);
     } else {