latticesql 1.2.6 → 1.3.1

package/dist/index.js

@@ -6,7 +6,7 @@ import { join as join2 } from "path";
 import { existsSync as existsSync2, readFileSync as readFileSync2 } from "fs";
 
 // src/render/writer.ts
-import { writeFileSync, mkdirSync, renameSync, existsSync, readFileSync } from "fs";
+import { writeFileSync, mkdirSync, renameSync, copyFileSync, unlinkSync, existsSync, readFileSync } from "fs";
 import { createHash } from "crypto";
 import { dirname, join } from "path";
 import { tmpdir } from "os";
@@ -19,7 +19,16 @@ function atomicWrite(filePath, content) {
   if (currentHash === newHash) return false;
   const tmp = join(tmpdir(), `lattice-${randomBytes(8).toString("hex")}.tmp`);
   writeFileSync(tmp, content, "utf8");
-  renameSync(tmp, filePath);
+  try {
+    renameSync(tmp, filePath);
+  } catch (err) {
+    if (err.code === "EXDEV") {
+      copyFileSync(tmp, filePath);
+      unlinkSync(tmp);
+    } else {
+      throw err;
+    }
+  }
   return true;
 }
 function existingHash(filePath) {
@@ -329,7 +338,56 @@ var Sanitizer = class {
 
 // src/render/engine.ts
 import { join as join4, basename, isAbsolute, resolve } from "path";
-import { mkdirSync as mkdirSync2, existsSync as existsSync4, copyFileSync } from "fs";
+import { mkdirSync as mkdirSync2, existsSync as existsSync4, copyFileSync as copyFileSync2 } from "fs";
+
+// src/render/token-budget.ts
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function applyTokenBudget(rows, renderFn, budget, prioritizeBy) {
+  const fullContent = renderFn(rows);
+  if (estimateTokens(fullContent) <= budget) return fullContent;
+  if (rows.length === 0) return fullContent;
+  const prioritized = [...rows];
+  if (typeof prioritizeBy === "function") {
+    prioritized.sort(prioritizeBy);
+  } else if (typeof prioritizeBy === "string") {
+    const col = prioritizeBy;
+    prioritized.sort((a, b) => {
+      const va = a[col];
+      const vb = b[col];
+      if (va == null && vb == null) return 0;
+      if (va == null) return 1;
+      if (vb == null) return -1;
+      if (va < vb) return 1;
+      if (va > vb) return -1;
+      return 0;
+    });
+  }
+  let lo = 0;
+  let hi = prioritized.length;
+  let bestContent = "";
+  let bestCount = 0;
+  while (lo < hi) {
+    const mid = Math.ceil((lo + hi) / 2);
+    const content = renderFn(prioritized.slice(0, mid));
+    if (estimateTokens(content) <= budget) {
+      bestContent = content;
+      bestCount = mid;
+      lo = mid;
+      if (lo === hi) break;
+    } else {
+      hi = mid - 1;
+    }
+  }
+  if (bestCount === 0) {
+    bestContent = renderFn([]);
+  }
+  const tokens = estimateTokens(bestContent);
+  return bestContent + `
+
+[truncated: ${bestCount} of ${rows.length} rows rendered, ~${tokens} tokens]`;
+}
 
 // src/render/entity-query.ts
 var SAFE_COL_RE = /^[a-zA-Z0-9_]+$/;
@@ -664,7 +722,7 @@ ${tmpl.perRow.body(row)}
 
 // src/lifecycle/cleanup.ts
 import { join as join3 } from "path";
-import { existsSync as existsSync3, readdirSync, unlinkSync, rmdirSync, statSync } from "fs";
+import { existsSync as existsSync3, readdirSync, unlinkSync as unlinkSync2, rmdirSync, statSync } from "fs";
 function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, manifest, options = {}, newManifest) {
   const result = {
     directoriesRemoved: [],
@@ -706,7 +764,7 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
           if (globalProtected.has(filename)) continue;
           const filePath = join3(entityDir, filename);
           if (!existsSync3(filePath)) continue;
-          if (!options.dryRun) unlinkSync(filePath);
+          if (!options.dryRun) unlinkSync2(filePath);
           options.onOrphan?.(filePath, "file");
           result.filesRemoved.push(filePath);
         }
@@ -751,7 +809,7 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
           if (globalProtected.has(filename)) continue;
           const filePath = join3(entityDir, filename);
           if (!existsSync3(filePath)) continue;
-          if (!options.dryRun) unlinkSync(filePath);
+          if (!options.dryRun) unlinkSync2(filePath);
           options.onOrphan?.(filePath, "file");
           result.filesRemoved.push(filePath);
         }
@@ -765,9 +823,11 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
 var RenderEngine = class {
   _schema;
   _adapter;
-  constructor(schema, adapter) {
+  _getTaskContext;
+  constructor(schema, adapter, getTaskContext) {
     this._schema = schema;
     this._adapter = adapter;
+    this._getTaskContext = getTaskContext ?? (() => "");
   }
   async render(outputDir) {
     const start = Date.now();
@@ -775,8 +835,38 @@ var RenderEngine = class {
     const counters = { skipped: 0 };
     for (const [name, def] of this._schema.getTables()) {
       let rows = this._schema.queryTable(this._adapter, name);
+      if (def.relevanceFilter) {
+        const ctx = this._getTaskContext();
+        rows = rows.filter((row) => def.relevanceFilter(row, ctx));
+      }
       if (def.filter) rows = def.filter(rows);
-      const content = def.render(rows);
+      if (def.rewardTracking) {
+        if (def.pruneBelow !== void 0) {
+          const threshold = def.pruneBelow;
+          const toPrune = rows.filter(
+            (r) => r._reward_count > 0 && r._reward_total < threshold
+          );
+          if (toPrune.length > 0) {
+            for (const r of toPrune) {
+              const pkCol = this._schema.getPrimaryKey(name)[0] ?? "id";
+              this._adapter.run(
+                `UPDATE "${name}" SET deleted_at = datetime('now') WHERE "${pkCol}" = ?`,
+                [r[pkCol]]
+              );
+            }
+            rows = rows.filter(
+              (r) => r._reward_count === 0 || r._reward_total >= threshold
+            );
+          }
+        }
+        if (!def.prioritizeBy) {
+          rows.sort((a, b) => (b._reward_total ?? 0) - (a._reward_total ?? 0));
+        }
+      }
+      if (def.enrich) {
+        for (const fn of def.enrich) rows = fn(rows);
+      }
+      const content = def.tokenBudget ? applyTokenBudget(rows, def.render, def.tokenBudget, def.prioritizeBy) : def.render(rows);
       const filePath = join4(outputDir, def.outputFile);
       if (atomicWrite(filePath, content)) {
         filesWritten.push(filePath);
@@ -895,7 +985,7 @@ var RenderEngine = class {
               const destPath = join4(entityDir, basename(absPath));
               if (!existsSync4(destPath)) {
                 try {
-                  copyFileSync(absPath, destPath);
+                  copyFileSync2(absPath, destPath);
                   filesWritten.push(destPath);
                 } catch {
                 }
@@ -1236,6 +1326,14 @@ var WritebackPipeline = class {
           if (store.isSeen(filePath, key)) continue;
           store.markSeen(filePath, key);
         }
+        if (def.validate) {
+          const result = await def.validate(entry);
+          const threshold = def.rejectBelow ?? 0;
+          if (!result.pass || result.score < threshold) {
+            def.onReject?.(entry, result);
+            continue;
+          }
+        }
         await def.persist(entry, filePath);
         processed++;
       }
@@ -1642,6 +1740,73 @@ function resolveEncryptedColumns(encrypted, allColumns) {
   return new Set(allColumns.filter((c) => !SKIP_COLUMNS.has(c)));
 }
 
+// src/search/embeddings.ts
+var EMBEDDINGS_TABLE = "_lattice_embeddings";
+function ensureEmbeddingsTable(adapter) {
+  adapter.run(`CREATE TABLE IF NOT EXISTS "${EMBEDDINGS_TABLE}" (
+    "table_name" TEXT NOT NULL,
+    "row_pk"     TEXT NOT NULL,
+    "embedding"  TEXT NOT NULL,
+    PRIMARY KEY ("table_name", "row_pk")
+  )`);
+}
+async function storeEmbedding(adapter, table, pk, row, config) {
+  const text = config.fields.map((f) => {
+    const v = row[f];
+    return v == null ? "" : String(v);
+  }).filter((s) => s.length > 0).join(" ");
+  if (text.length === 0) return;
+  const vector = await config.embed(text);
+  adapter.run(
+    `INSERT OR REPLACE INTO "${EMBEDDINGS_TABLE}" ("table_name", "row_pk", "embedding") VALUES (?, ?, ?)`,
+    [table, pk, JSON.stringify(vector)]
+  );
+}
+function removeEmbedding(adapter, table, pk) {
+  adapter.run(
+    `DELETE FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ? AND "row_pk" = ?`,
+    [table, pk]
+  );
+}
+function cosineSimilarity(a, b) {
+  const len = Math.min(a.length, b.length);
+  let dot = 0;
+  let magA = 0;
+  let magB = 0;
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  return denom === 0 ? 0 : dot / denom;
+}
+async function searchByEmbedding(adapter, table, queryText, config, topK, minScore, pkColumn = "id") {
+  const queryVector = await config.embed(queryText);
+  const stored = adapter.all(
+    `SELECT "row_pk", "embedding" FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
+    [table]
+  );
+  const scored = [];
+  for (const entry of stored) {
+    const vec = JSON.parse(entry.embedding);
+    const score = cosineSimilarity(queryVector, vec);
+    if (score >= minScore) {
+      scored.push({ pk: entry.row_pk, score });
+    }
+  }
+  scored.sort((a, b) => b.score - a.score);
+  const topResults = scored.slice(0, topK);
+  const results = [];
+  for (const { pk, score } of topResults) {
+    const row = adapter.get(`SELECT * FROM "${table}" WHERE "${pkColumn}" = ?`, [pk]);
+    if (row) {
+      results.push({ row, score });
+    }
+  }
+  return results;
+}
+
 // src/lattice.ts
 var Lattice = class {
   _adapter;
@@ -1660,6 +1825,8 @@ var Lattice = class {
   _encryptedTableColumns = /* @__PURE__ */ new Map();
   /** Raw encryption key passphrase from constructor options. */
   _encryptionKeyRaw;
+  /** Current task context string for relevance filtering. */
+  _taskContext = "";
   _auditHandlers = [];
   _renderHandlers = [];
   _writebackHandlers = [];
@@ -1686,7 +1853,7 @@ var Lattice = class {
     this._adapter = new SQLiteAdapter(dbPath, adapterOpts);
     this._schema = new SchemaManager();
     this._sanitizer = new Sanitizer(options.security);
-    this._render = new RenderEngine(this._schema, this._adapter);
+    this._render = new RenderEngine(this._schema, this._adapter, () => this._taskContext);
     this._reverseSync = new ReverseSyncEngine(this._schema, this._adapter);
     this._loop = new SyncLoop(this._render);
     this._writeback = new WritebackPipeline();
@@ -1710,8 +1877,10 @@ var Lattice = class {
   // -------------------------------------------------------------------------
   define(table, def) {
     this._assertNotInit("define");
+    const columns = def.rewardTracking ? { ...def.columns, _reward_total: "REAL DEFAULT 0", _reward_count: "INTEGER DEFAULT 0" } : def.columns;
     const compiledDef = {
       ...def,
+      columns,
       render: def.render ? compileRender(
         def,
         table,
@@ -1757,6 +1926,10 @@ var Lattice = class {
       const rows = this._adapter.all(`PRAGMA table_info("${tableName}")`);
       this._columnCache.set(tableName, new Set(rows.map((r) => r.name)));
     }
+    const hasEmbeddings = [...this._schema.getTables().values()].some((d) => d.embeddings);
+    if (hasEmbeddings) {
+      ensureEmbeddingsTable(this._adapter);
+    }
     this._setupEncryption();
     this._initialized = true;
     return Promise.resolve();
@@ -1786,6 +1959,21 @@ var Lattice = class {
     this._initialized = false;
   }
   // -------------------------------------------------------------------------
+  // Task context (for relevance filtering)
+  // -------------------------------------------------------------------------
+  /**
+   * Set the current task context string. Tables with a `relevanceFilter`
+   * will use this value to filter rows before rendering.
+   */
+  setTaskContext(context) {
+    this._taskContext = context;
+    return this;
+  }
+  /** Return the current task context string. */
+  getTaskContext() {
+    return this._taskContext;
+  }
+  // -------------------------------------------------------------------------
   // Encryption helpers
   // -------------------------------------------------------------------------
   _setupEncryption() {
@@ -1860,6 +2048,7 @@ var Lattice = class {
     const pkValue = rawPk != null ? String(rawPk) : "";
     this._sanitizer.emitAudit(table, "insert", pkValue);
     this._fireWriteHooks(table, "insert", rowWithPk, pkValue);
+    this._syncEmbedding(table, "insert", rowWithPk, pkValue);
     return Promise.resolve(pkValue);
   }
   /**
@@ -1927,6 +2116,11 @@ var Lattice = class {
     const auditId = typeof id === "string" ? id : JSON.stringify(id);
     this._sanitizer.emitAudit(table, "update", auditId);
     this._fireWriteHooks(table, "update", sanitized, auditId, Object.keys(sanitized));
+    const def = this._schema.getTables().get(table);
+    if (def?.embeddings) {
+      const fullRow = this._adapter.get(`SELECT * FROM "${table}" WHERE ${clause}`, pkParams);
+      if (fullRow) this._syncEmbedding(table, "update", fullRow, auditId);
+    }
     return Promise.resolve();
   }
   /**
@@ -1948,6 +2142,7 @@ var Lattice = class {
     const auditId = typeof id === "string" ? id : JSON.stringify(id);
     this._sanitizer.emitAudit(table, "delete", auditId);
     this._fireWriteHooks(table, "delete", { id: auditId }, auditId);
+    this._syncEmbedding(table, "delete", {}, auditId);
     return Promise.resolve();
   }
   get(table, id) {
@@ -2331,6 +2526,65 @@ var Lattice = class {
     const ms = unit === "h" ? num * 36e5 : unit === "d" ? num * 864e5 : num * 6e4;
     return new Date(Date.now() - ms).toISOString();
   }
+  // -------------------------------------------------------------------------
+  // Reward tracking
+  // -------------------------------------------------------------------------
+  /**
+   * Update reward scores for a row. The total reward is recalculated as
+   * the running average across all reward calls. Requires `rewardTracking`
+   * on the table definition.
+   */
+  reward(table, id, scores) {
+    const notInit = this._notInitError();
+    if (notInit) return notInit;
+    const def = this._schema.getTables().get(table);
+    if (!def?.rewardTracking) {
+      return Promise.reject(
+        new Error(`Table "${table}" does not have rewardTracking enabled`)
+      );
+    }
+    const vals = Object.values(scores);
+    if (vals.length === 0) return Promise.resolve();
+    const avg = vals.reduce((a, b) => a + b, 0) / vals.length;
+    const { clause, params: pkParams } = this._pkWhere(table, id);
+    this._adapter.run(
+      `UPDATE "${table}" SET "_reward_total" = ("_reward_total" * "_reward_count" + ?) / ("_reward_count" + 1), "_reward_count" = "_reward_count" + 1 WHERE ${clause}`,
+      [avg, ...pkParams]
+    );
+    return Promise.resolve();
+  }
+  // -------------------------------------------------------------------------
+  // Semantic search
+  // -------------------------------------------------------------------------
+  /**
+   * Search for rows by semantic similarity. Requires `embeddings` config
+   * on the table definition.
+   *
+   * @param table  - Table to search
+   * @param query  - Natural-language query text
+   * @param opts   - Search options (topK, minScore)
+   * @returns Matching rows with similarity scores, sorted best-first.
+   */
+  async search(table, query, opts = {}) {
+    const notInit = this._notInitError();
+    if (notInit) return notInit;
+    const def = this._schema.getTables().get(table);
+    if (!def?.embeddings) {
+      return Promise.reject(
+        new Error(`Table "${table}" does not have embeddings configured`)
+      );
+    }
+    const pkCol = this._schema.getPrimaryKey(table)[0] ?? "id";
+    return searchByEmbedding(
+      this._adapter,
+      table,
+      query,
+      def.embeddings,
+      opts.topK ?? 10,
+      opts.minScore ?? 0,
+      pkCol
+    );
+  }
   query(table, opts = {}) {
     const notInit = this._notInitError();
     if (notInit) return notInit;
@@ -2589,6 +2843,23 @@ var Lattice = class {
       }
     }
   }
+  /**
+   * Update or remove the embedding for a row.
+   * No-op if the table doesn't have `embeddings` configured.
+   */
+  _syncEmbedding(table, op, row, pk) {
+    const def = this._schema.getTables().get(table);
+    if (!def?.embeddings) return;
+    if (op === "delete") {
+      removeEmbedding(this._adapter, table, pk);
+      return;
+    }
+    storeEmbedding(this._adapter, table, pk, row, def.embeddings).catch((err) => {
+      for (const h of this._errorHandlers) {
+        h(err instanceof Error ? err : new Error(String(err)));
+      }
+    });
+  }
   _notInitError() {
     if (!this._initialized) {
       return Promise.reject(
@@ -3129,6 +3400,7 @@ export {
   InMemoryStateStore,
   Lattice,
   READ_ONLY_HEADER,
+  applyTokenBudget,
   applyWriteEntry,
   autoUpdate,
   contentHash,
@@ -3138,6 +3410,7 @@ export {
   deriveKey,
   encrypt,
   entityFileNames,
+  estimateTokens,
   fixSchemaConflicts,
   frontmatter,
   generateEntryId,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latticesql",
-  "version": "1.2.6",
+  "version": "1.3.1",
   "description": "Persistent structured memory for AI agent systems — SQLite ↔ LLM context bridge",
   "type": "module",
   "main": "./dist/index.js",