latticesql 1.3.1 → 1.4.0

package/dist/cli.js

@@ -321,7 +321,15 @@ import { join as join3 } from "path";
 import { existsSync as existsSync3, readFileSync as readFileSync3 } from "fs";
 
 // src/render/writer.ts
-import { writeFileSync as writeFileSync2, mkdirSync as mkdirSync2, renameSync, copyFileSync, unlinkSync, existsSync as existsSync2, readFileSync as readFileSync2 } from "fs";
+import {
+  writeFileSync as writeFileSync2,
+  mkdirSync as mkdirSync2,
+  renameSync,
+  copyFileSync,
+  unlinkSync,
+  existsSync as existsSync2,
+  readFileSync as readFileSync2
+} from "fs";
 import { createHash } from "crypto";
 import { dirname as dirname3, join as join2 } from "path";
 import { tmpdir } from "os";
@@ -389,15 +397,21 @@ function writeManifest(outputDir, manifest) {
 
 // src/db/sqlite.ts
 import Database from "better-sqlite3";
+var DDL_RE = /^\s*(CREATE|ALTER|DROP|PRAGMA)\b/i;
+var DEFAULT_CACHE_MAX = 500;
 var SQLiteAdapter = class {
   _db = null;
   _path;
   _wal;
   _busyTimeout;
+  /** Cached prepared statements keyed by SQL string. */
+  _stmtCache = /* @__PURE__ */ new Map();
+  _cacheMax;
   constructor(path, options) {
     this._path = path;
     this._wal = options?.wal ?? true;
     this._busyTimeout = options?.busyTimeout ?? 5e3;
+    this._cacheMax = options?.cacheMax ?? DEFAULT_CACHE_MAX;
   }
   get db() {
     if (!this._db) throw new Error("SQLiteAdapter: not open \u2014 call open() first");
@@ -411,20 +425,46 @@ var SQLiteAdapter = class {
     }
   }
   close() {
+    this._stmtCache.clear();
     this._db?.close();
     this._db = null;
   }
+  /** Clear the prepared-statement cache. Call after DDL changes (schema/migrations). */
+  clearStatementCache() {
+    this._stmtCache.clear();
+  }
+  /**
+   * Return a cached prepared statement for the given SQL, or compile and cache a new one.
+   * DDL statements bypass the cache entirely.
+   */
+  _cachedPrepare(sql) {
+    if (DDL_RE.test(sql)) {
+      return this.db.prepare(sql);
+    }
+    let stmt = this._stmtCache.get(sql);
+    if (stmt) return stmt;
+    stmt = this.db.prepare(sql);
+    if (this._stmtCache.size >= this._cacheMax) {
+      this._stmtCache.clear();
+    }
+    this._stmtCache.set(sql, stmt);
+    return stmt;
+  }
   run(sql, params = []) {
-    this.db.prepare(sql).run(...params);
+    this._cachedPrepare(sql).run(...params);
   }
   get(sql, params = []) {
-    return this.db.prepare(sql).get(...params);
+    return this._cachedPrepare(sql).get(...params);
   }
   all(sql, params = []) {
-    return this.db.prepare(sql).all(...params);
+    return this._cachedPrepare(sql).all(...params);
+  }
+  /** Execute raw SQL that may contain multiple statements (e.g. migrations). */
+  exec(sql) {
+    this.db.exec(sql);
   }
   prepare(sql) {
-    const stmt = this.db.prepare(sql);
+    const stmt = this._cachedPrepare(sql);
     return {
       run: (...params) => stmt.run(...params),
       get: (...params) => stmt.get(...params),
@@ -518,19 +558,40 @@ var SchemaManager = class {
     });
   }
   /** Run explicit versioned migrations in order, idempotently */
-  applyMigrations(adapter, migrations) {
+  applyMigrations(adapter, migrations, validator) {
     const sorted = [...migrations].sort((a, b) => {
       const va = String(a.version);
       const vb = String(b.version);
       return va.localeCompare(vb, void 0, { numeric: true });
     });
+    if (validator) {
+      const errors = [];
+      for (const m of sorted) {
+        const versionStr = String(m.version);
+        const exists = adapter.get("SELECT 1 FROM __lattice_migrations WHERE version = ?", [
+          versionStr
+        ]);
+        if (!exists) {
+          const result = validator(m.sql);
+          if (!result.valid) {
+            errors.push(
+              `Migration ${versionStr}: ${(result.errors ?? ["invalid SQL"]).join("; ")}`
+            );
+          }
+        }
+      }
+      if (errors.length > 0) {
+        throw new Error(`Migration validation failed:
+${errors.join("\n")}`);
+      }
+    }
     for (const m of sorted) {
       const versionStr = String(m.version);
       const exists = adapter.get("SELECT 1 FROM __lattice_migrations WHERE version = ?", [
         versionStr
       ]);
       if (!exists) {
-        adapter.run(m.sql);
+        adapter.exec(m.sql);
         adapter.run("INSERT INTO __lattice_migrations (version, applied_at) VALUES (?, ?)", [
           versionStr,
           (/* @__PURE__ */ new Date()).toISOString()
@@ -868,6 +929,223 @@ function truncateContent(content, budget) {
   if (budget === void 0 || content.length <= budget) return content;
   return content.slice(0, budget) + "\n\n*[truncated \u2014 context budget exceeded]*";
 }
+var BATCH_CHUNK_SIZE = 500;
+function groupBy(rows, keyCol) {
+  const map = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const key = String(row[keyCol] ?? "");
+    let arr = map.get(key);
+    if (!arr) {
+      arr = [];
+      map.set(key, arr);
+    }
+    arr.push(row);
+  }
+  return map;
+}
+function buildBatchClauses(params, opts, tableAlias) {
+  let sql = "";
+  const prefix = tableAlias ? `${tableAlias}.` : "";
+  for (const f of effectiveFilters(opts)) {
+    if (!SAFE_COL_RE.test(f.col)) continue;
+    switch (f.op) {
+      case "eq":
+        sql += ` AND ${prefix}"${f.col}" = ?`;
+        params.push(f.val);
+        break;
+      case "ne":
+        sql += ` AND ${prefix}"${f.col}" != ?`;
+        params.push(f.val);
+        break;
+      case "gt":
+        sql += ` AND ${prefix}"${f.col}" > ?`;
+        params.push(f.val);
+        break;
+      case "gte":
+        sql += ` AND ${prefix}"${f.col}" >= ?`;
+        params.push(f.val);
+        break;
+      case "lt":
+        sql += ` AND ${prefix}"${f.col}" < ?`;
+        params.push(f.val);
+        break;
+      case "lte":
+        sql += ` AND ${prefix}"${f.col}" <= ?`;
+        params.push(f.val);
+        break;
+      case "like":
+        sql += ` AND ${prefix}"${f.col}" LIKE ?`;
+        params.push(f.val);
+        break;
+      case "in": {
+        const arr = f.val;
+        if (arr.length === 0) {
+          sql += " AND 0";
+        } else {
+          sql += ` AND ${prefix}"${f.col}" IN (${arr.map(() => "?").join(", ")})`;
+          params.push(...arr);
+        }
+        break;
+      }
+      case "isNull":
+        sql += ` AND ${prefix}"${f.col}" IS NULL`;
+        break;
+      case "isNotNull":
+        sql += ` AND ${prefix}"${f.col}" IS NOT NULL`;
+        break;
+    }
+  }
+  if (opts.orderBy) {
+    if (typeof opts.orderBy === "string") {
+      if (SAFE_COL_RE.test(opts.orderBy)) {
+        const dir = opts.orderDir === "desc" ? "DESC" : "ASC";
+        sql += ` ORDER BY ${prefix}"${opts.orderBy}" ${dir}`;
+      }
+    } else {
+      const clauses = opts.orderBy.filter((spec) => SAFE_COL_RE.test(spec.col)).map((spec) => `${prefix}"${spec.col}" ${spec.dir === "desc" ? "DESC" : "ASC"}`);
+      if (clauses.length > 0) {
+        sql += ` ORDER BY ${clauses.join(", ")}`;
+      }
+    }
+  }
+  return sql;
+}
+function batchQuery(adapter, buildSql, inValues, extraParams) {
+  if (inValues.length === 0) return [];
+  const allRows = [];
+  for (let i = 0; i < inValues.length; i += BATCH_CHUNK_SIZE) {
+    const chunk = inValues.slice(i, i + BATCH_CHUNK_SIZE);
+    const placeholders = chunk.map(() => "?").join(", ");
+    const sql = buildSql(placeholders);
+    allRows.push(...adapter.all(sql, [...chunk, ...extraParams]));
+  }
+  return allRows;
+}
+function batchPrefetchEntitySources(files, allEntityRows, entityPk, adapter, protection) {
+  const results = /* @__PURE__ */ new Map();
+  const unbatched = /* @__PURE__ */ new Set();
+  const allPkValues = allEntityRows.map((r) => r[entityPk]);
+  for (const [filename, spec] of Object.entries(files)) {
+    const source = spec.source;
+    if (source.type === "self") {
+      continue;
+    }
+    if (source.type === "custom" || source.type === "enriched") {
+      unbatched.add(filename);
+      continue;
+    }
+    if (source.type === "hasMany") {
+      if (protection?.protectedTables.has(source.table)) {
+        unbatched.add(filename);
+        continue;
+      }
+      const ref = source.references ?? entityPk;
+      const pkValues = allEntityRows.map((r) => r[ref]);
+      const extraParams = [];
+      const clauses = buildBatchClauses(extraParams, source);
+      const rows = batchQuery(
+        adapter,
+        (ph) => `SELECT * FROM "${source.table}" WHERE "${source.foreignKey}" IN (${ph})${clauses}`,
+        pkValues,
+        extraParams
+      );
+      const grouped = groupBy(rows, source.foreignKey);
+      if (source.limit !== void 0 && source.limit > 0) {
+        for (const [key, arr] of grouped) {
+          if (arr.length > source.limit) grouped.set(key, arr.slice(0, source.limit));
+        }
+      }
+      results.set(filename, grouped);
+      continue;
+    }
+    if (source.type === "manyToMany") {
+      if (protection?.protectedTables.has(source.remoteTable)) {
+        unbatched.add(filename);
+        continue;
+      }
+      const remotePk = source.references ?? "id";
+      const extraParams = [];
+      const clauses = buildBatchClauses(extraParams, source, "r");
+      let selectCols = "r.*";
+      if (source.junctionColumns?.length) {
+        const jCols = source.junctionColumns.map((jc) => {
+          if (typeof jc === "string") {
+            if (!SAFE_COL_RE.test(jc)) return null;
+            return `j."${jc}"`;
+          }
+          if (!SAFE_COL_RE.test(jc.col) || !SAFE_COL_RE.test(jc.as)) return null;
+          return `j."${jc.col}" AS "${jc.as}"`;
+        }).filter(Boolean);
+        if (jCols.length > 0) selectCols += ", " + jCols.join(", ");
+      }
+      const batchKeyCol = "__lattice_batch_key";
+      const rows = batchQuery(
+        adapter,
+        (ph) => `SELECT ${selectCols}, j."${source.localKey}" AS "${batchKeyCol}" FROM "${source.remoteTable}" r JOIN "${source.junctionTable}" j ON j."${source.remoteKey}" = r."${remotePk}" WHERE j."${source.localKey}" IN (${ph})${clauses}`,
+        allPkValues,
+        extraParams
+      );
+      const grouped = /* @__PURE__ */ new Map();
+      for (const row of rows) {
+        const key = String(row[batchKeyCol] ?? "");
+        delete row[batchKeyCol];
+        let arr = grouped.get(key);
+        if (!arr) {
+          arr = [];
+          grouped.set(key, arr);
+        }
+        arr.push(row);
+      }
+      if (source.limit !== void 0 && source.limit > 0) {
+        for (const [key, arr] of grouped) {
+          if (arr.length > source.limit) grouped.set(key, arr.slice(0, source.limit));
+        }
+      }
+      results.set(filename, grouped);
+      continue;
+    }
+    if (source.type === "belongsTo") {
+      if (protection?.protectedTables.has(source.table)) {
+        unbatched.add(filename);
+        continue;
+      }
+      const fkValues = [
+        ...new Set(allEntityRows.map((r) => r[source.foreignKey]).filter((v) => v != null))
+      ];
+      if (fkValues.length === 0) {
+        results.set(filename, /* @__PURE__ */ new Map());
+        continue;
+      }
+      const refCol = source.references ?? "id";
+      const extraParams = [];
+      const clauses = buildBatchClauses(extraParams, source);
+      const rows = batchQuery(
+        adapter,
+        (ph) => `SELECT * FROM "${source.table}" WHERE "${refCol}" IN (${ph})${clauses}`,
+        fkValues,
+        extraParams
+      );
+      const lookup = /* @__PURE__ */ new Map();
+      for (const row of rows) {
+        lookup.set(String(row[refCol] ?? ""), row);
+      }
+      const grouped = /* @__PURE__ */ new Map();
+      for (const entityRow of allEntityRows) {
+        const fkVal = entityRow[source.foreignKey];
+        const pkVal = String(entityRow[entityPk] ?? "");
+        if (fkVal == null) {
+          grouped.set(pkVal, []);
+        } else {
+          const related = lookup.get(String(fkVal));
+          grouped.set(pkVal, related ? [related] : []);
+        }
+      }
+      results.set(filename, grouped);
+      continue;
+    }
+  }
+  return { results, unbatched };
+}
 
 // src/render/markdown.ts
 function frontmatter(fields) {
@@ -1168,7 +1446,9 @@ var RenderEngine = class {
           }
         }
         if (!def.prioritizeBy) {
-          rows.sort((a, b) => (b._reward_total ?? 0) - (a._reward_total ?? 0));
+          rows.sort(
+            (a, b) => (b._reward_total ?? 0) - (a._reward_total ?? 0)
+          );
         }
       }
       if (def.enrich) {
@@ -1270,13 +1550,25 @@ var RenderEngine = class {
           counters.skipped++;
         }
       }
+      const protection = protectedTables.size > 0 ? { protectedTables, currentTable: table } : void 0;
+      const mergedFiles = {};
+      for (const [filename, spec] of Object.entries(def.files)) {
+        const mergeDefaults = def.sourceDefaults && spec.source.type !== "self" && spec.source.type !== "custom" && spec.source.type !== "enriched";
+        const source = mergeDefaults ? { ...def.sourceDefaults, ...spec.source } : spec.source;
+        mergedFiles[filename] = { source, limit: spec.budget };
+      }
+      const batch = batchPrefetchEntitySources(
+        mergedFiles,
+        allRows,
+        entityPk,
+        this._adapter,
+        protection
+      );
       for (const entityRow of allRows) {
         const rawSlug = def.slug(entityRow);
         const slug = rawSlug.replace(/[\u00A0\u2000-\u200B\u202F\u205F\u3000]/g, " ").replace(/[\x00-\x1F\x7F]/g, "");
         if (/[^a-zA-Z0-9.\-_ @(),#&'+:;!~\[\]]/.test(slug)) {
-          throw new Error(
-            `Invalid slug "${slug}": contains characters outside the allowed set`
-          );
+          throw new Error(`Invalid slug "${slug}": contains characters outside the allowed set`);
         }
         const entityDir = def.directory ? join5(outputDir, def.directory(entityRow)) : join5(outputDir, directoryRoot, slug);
         const resolvedDir = resolve3(entityDir);
@@ -1303,11 +1595,19 @@ var RenderEngine = class {
         }
         const renderedFiles = /* @__PURE__ */ new Map();
         const entityFileHashes = {};
-        const protection = protectedTables.size > 0 ? { protectedTables, currentTable: table } : void 0;
+        const entityPkVal = String(entityRow[entityPk] ?? "");
         for (const [filename, spec] of Object.entries(def.files)) {
-          const mergeDefaults = def.sourceDefaults && spec.source.type !== "self" && spec.source.type !== "custom" && spec.source.type !== "enriched";
-          const source = mergeDefaults ? { ...def.sourceDefaults, ...spec.source } : spec.source;
-          const rows = resolveEntitySource(source, entityRow, entityPk, this._adapter, protection);
+          let rows;
+          if (spec.source.type === "self") {
+            rows = [entityRow];
+          } else if (batch.unbatched.has(filename)) {
+            const source = mergedFiles[filename].source;
+            rows = resolveEntitySource(source, entityRow, entityPk, this._adapter, protection);
+          } else if (batch.results.has(filename)) {
+            rows = batch.results.get(filename).get(entityPkVal) ?? [];
+          } else {
+            rows = [];
+          }
           if (spec.omitIfEmpty && rows.length === 0) continue;
           const renderFn = compileEntityRender(spec.render);
           const content = truncateContent(renderFn(rows), spec.budget);
@@ -1787,10 +2087,10 @@ async function storeEmbedding(adapter, table, pk, row, config) {
   );
 }
 function removeEmbedding(adapter, table, pk) {
-  adapter.run(
-    `DELETE FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ? AND "row_pk" = ?`,
-    [table, pk]
-  );
+  adapter.run(`DELETE FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ? AND "row_pk" = ?`, [
+    table,
+    pk
+  ]);
 }
 function cosineSimilarity(a, b) {
   const len = Math.min(a.length, b.length);
@@ -1851,6 +2151,10 @@ var Lattice = class {
   _encryptionKeyRaw;
   /** Current task context string for relevance filtering. */
   _taskContext = "";
+  /** Per-table write version counter, incremented on every mutation. */
+  _tableWriteVersion = /* @__PURE__ */ new Map();
+  /** Snapshot of write versions at the last successful render. */
+  _lastRenderedVersions = /* @__PURE__ */ new Map();
   _auditHandlers = [];
   _renderHandlers = [];
   _writebackHandlers = [];
@@ -1944,8 +2248,9 @@ var Lattice = class {
     this._adapter.open();
     this._schema.applySchema(this._adapter);
     if (options.migrations?.length) {
-      this._schema.applyMigrations(this._adapter, options.migrations);
+      this._schema.applyMigrations(this._adapter, options.migrations, options.validateMigrationSQL);
     }
+    this._adapter.clearStatementCache();
     for (const tableName of this._schema.getTables().keys()) {
       const rows = this._adapter.all(`PRAGMA table_info("${tableName}")`);
       this._columnCache.set(tableName, new Set(rows.map((r) => r.name)));
@@ -1969,6 +2274,7 @@ var Lattice = class {
       return Promise.reject(new Error("Lattice: not initialized \u2014 call init() first"));
     }
     this._schema.applyMigrations(this._adapter, migrations);
+    this._adapter.clearStatementCache();
     for (const tableName of this._schema.getTables().keys()) {
       const rows = this._adapter.all(`PRAGMA table_info("${tableName}")`);
       this._columnCache.set(tableName, new Set(rows.map((r) => r.name)));
@@ -1993,6 +2299,20 @@ var Lattice = class {
     this._taskContext = context;
     return this;
   }
+  /**
+   * Mark one or all tables as dirty so the next render() will run a full cycle.
+   * Use this after writing directly via the `db` escape hatch.
+   */
+  markDirty(table) {
+    if (table) {
+      this._bumpWriteVersion(table);
+    } else {
+      for (const t of this._schema.getTables().keys()) {
+        this._bumpWriteVersion(t);
+      }
+    }
+    return this;
+  }
   /** Return the current task context string. */
   getTaskContext() {
     return this._taskContext;
@@ -2113,6 +2433,7 @@ var Lattice = class {
     const rawPk = rowWithPk[pkCol];
     const pkValue = rawPk != null ? String(rawPk) : "";
     this._sanitizer.emitAudit(table, "update", pkValue);
+    this._bumpWriteVersion(table);
     return Promise.resolve(pkValue);
   }
   upsertBy(table, col, val, row) {
@@ -2289,6 +2610,7 @@ var Lattice = class {
          AND (deleted_at IS NULL OR deleted_at = '')`,
         [sourceFile, ...currentKeys]
       );
+      this._bumpWriteVersion(table);
     }
     return Promise.resolve(count);
   }
@@ -2345,6 +2667,7 @@ var Lattice = class {
       `${verb} INTO "${junctionTable}" (${colNames}) VALUES (${placeholders})`,
       Object.values(filtered)
     );
+    this._bumpWriteVersion(junctionTable);
     return Promise.resolve();
   }
   /**
@@ -2360,6 +2683,7 @@ var Lattice = class {
       `DELETE FROM "${junctionTable}" WHERE ${where}`,
       entries.map(([, v]) => v)
     );
+    this._bumpWriteVersion(junctionTable);
     return Promise.resolve();
   }
   // -------------------------------------------------------------------------
@@ -2563,9 +2887,7 @@ var Lattice = class {
     if (notInit) return notInit;
     const def = this._schema.getTables().get(table);
     if (!def?.rewardTracking) {
-      return Promise.reject(
-        new Error(`Table "${table}" does not have rewardTracking enabled`)
-      );
+      return Promise.reject(new Error(`Table "${table}" does not have rewardTracking enabled`));
     }
     const vals = Object.values(scores);
     if (vals.length === 0) return Promise.resolve();
@@ -2575,6 +2897,7 @@ var Lattice = class {
       `UPDATE "${table}" SET "_reward_total" = ("_reward_total" * "_reward_count" + ?) / ("_reward_count" + 1), "_reward_count" = "_reward_count" + 1 WHERE ${clause}`,
       [avg, ...pkParams]
     );
+    this._bumpWriteVersion(table);
     return Promise.resolve();
   }
   // -------------------------------------------------------------------------
@@ -2594,9 +2917,7 @@ var Lattice = class {
     if (notInit) return notInit;
     const def = this._schema.getTables().get(table);
     if (!def?.embeddings) {
-      return Promise.reject(
-        new Error(`Table "${table}" does not have embeddings configured`)
-      );
+      return Promise.reject(new Error(`Table "${table}" does not have embeddings configured`));
     }
     const pkCol = this._schema.getPrimaryKey(table)[0] ?? "id";
     return searchByEmbedding(
@@ -2683,13 +3004,27 @@ var Lattice = class {
     const notInit = this._notInitError();
     if (notInit) return notInit;
     const result = await this._render.render(outputDir);
+    for (const [t, v] of this._tableWriteVersion) {
+      this._lastRenderedVersions.set(t, v);
+    }
     for (const h of this._renderHandlers) h(result);
     return result;
   }
+  /**
+   * Returns `true` when no table has been written to since the last render.
+   * Useful for consumers implementing their own polling/watch loops to skip
+   * redundant render cycles.
+   */
+  isDirty() {
+    return !this._isClean();
+  }
   async sync(outputDir) {
     const notInit = this._notInitError();
     if (notInit) return notInit;
     const renderResult = await this._render.render(outputDir);
+    for (const [t, v] of this._tableWriteVersion) {
+      this._lastRenderedVersions.set(t, v);
+    }
     for (const h of this._renderHandlers) h(renderResult);
     const writebackProcessed = await this._writeback.process();
     return { ...renderResult, writebackProcessed };
@@ -2850,8 +3185,21 @@ var Lattice = class {
     }
     return { clauses, params };
   }
+  /** Increment the write version for a table, marking it dirty for render. */
+  _bumpWriteVersion(table) {
+    this._tableWriteVersion.set(table, (this._tableWriteVersion.get(table) ?? 0) + 1);
+  }
+  /** True when no table has been written to since the last render. */
+  _isClean() {
+    if (this._lastRenderedVersions.size === 0) return false;
+    for (const [table, version] of this._tableWriteVersion) {
+      if (version !== (this._lastRenderedVersions.get(table) ?? -1)) return false;
+    }
+    return true;
+  }
   /** Returns a rejected Promise if not initialized; null if ready. */
   _fireWriteHooks(table, op, row, pk, changedColumns) {
+    this._bumpWriteVersion(table);
     for (const hook of this._writeHooks) {
       if (hook.table !== table) continue;
       if (!hook.on.includes(op)) continue;