latticesql 1.2.6 → 1.3.1

package/README.md

@@ -18,6 +18,8 @@ Every AI agent session starts cold — no memory of what happened yesterday, wha
 2. **Watches** for DB changes and re-renders automatically
 3. **Ingests** agent-written output back into the DB via the writeback pipeline
 4. **Manages** state with full CRUD, natural-key operations, seeding, and soft-delete
+5. **Optimizes** context with token budgets, relevance filtering, enrichment pipelines, and reward-scored memory
+6. **Searches** semantically via bring-your-own embeddings and cosine similarity
 
 Lattice has no opinions about your schema, your agents, or your file format. You define the tables. You control the rendering. Lattice runs the sync loop.
 
@@ -41,6 +43,9 @@ Lattice has no opinions about your schema, your agents, or your file format. You
   - [Render, sync, watch, and reconcile](#render-sync-watch-and-reconcile)
   - [Events](#events)
   - [Raw DB access](#raw-db-access)
+  - [Context optimization](#context-optimization-v13)
+  - [Semantic search](#semantic-search-v13)
+  - [Writeback validation](#writeback-validation-v13)
 - [Template rendering](#template-rendering)
   - [Built-in templates](#built-in-templates)
   - [Lifecycle hooks](#lifecycle-hooks)
@@ -1261,6 +1266,140 @@ const rows = db.db
 
 ---
 
+### Context optimization (v1.3+)
+
+Lattice provides several options on `TableDefinition` to optimize what gets rendered into context files.
+
+#### Token budget
+
+Limit the token count of rendered output. When content exceeds the budget, rows are pruned by priority:
+
+```typescript
+db.define('tickets', {
+  columns: { id: 'TEXT PRIMARY KEY', title: 'TEXT', updated_at: 'TEXT' },
+  render: (rows) => rows.map((r) => `- ${r.title}`).join('\n'),
+  outputFile: 'TICKETS.md',
+  tokenBudget: 4000,            // max estimated tokens (~4 chars/token)
+  prioritizeBy: 'updated_at',   // keep most recent rows when pruning
+});
+```
+
+A truncation footer is appended: `[truncated: 47 of 123 rows rendered, ~3800 tokens]`
+
+#### Relevance filtering
+
+Dynamically filter rows based on a task context string:
+
+```typescript
+db.define('knowledge', {
+  columns: { id: 'TEXT PRIMARY KEY', topic: 'TEXT', body: 'TEXT' },
+  render: (rows) => rows.map((r) => `## ${r.topic}\n${r.body}`).join('\n\n'),
+  outputFile: 'KNOWLEDGE.md',
+  relevanceFilter: (row, ctx) =>
+    ctx ? String(row.body).toLowerCase().includes(ctx.toLowerCase()) : true,
+});
+
+// Set the current task context — only matching rows are rendered
+db.setTaskContext('deployment');
+await db.render('./context');
+```
+
+#### Enrichment pipeline
+
+Transform rows between filtering and rendering — add computed fields, cluster, summarize:
+
+```typescript
+db.define('incidents', {
+  columns: { id: 'TEXT PRIMARY KEY', severity: 'TEXT', title: 'TEXT', created_at: 'TEXT' },
+  render: (rows) => JSON.stringify(rows, null, 2),
+  outputFile: 'incidents.json',
+  enrich: [
+    (rows) => rows.map((r) => ({
+      ...r,
+      _age_hours: Math.round((Date.now() - new Date(r.created_at as string).getTime()) / 3600000),
+    })),
+    (rows) => rows.length > 100 ? [{ _summary: `${rows.length} incidents` }] : rows,
+  ],
+});
+```
+
+#### Reward-scored memory
+
+Track which data is useful. High-reward rows are prioritized in rendering; low-scoring rows can be auto-pruned:
+
+```typescript
+db.define('tips', {
+  columns: { id: 'TEXT PRIMARY KEY', tip: 'TEXT', deleted_at: 'TEXT' },
+  render: (rows) => rows.map((r) => `- ${r.tip}`).join('\n'),
+  outputFile: 'TIPS.md',
+  rewardTracking: true,   // auto-adds _reward_total, _reward_count columns
+  pruneBelow: 0.3,         // soft-delete rows with reward < 0.3 (requires deleted_at column)
+});
+
+await db.init();
+const id = await db.insert('tips', { tip: 'Use batch inserts for bulk data' });
+
+// After the agent confirms this tip was useful:
+await db.reward('tips', id, { relevance: 0.9, accuracy: 1.0 });
+```
+
+### Semantic search (v1.3+)
+
+Enable embedding-based search on any table. Bring your own embedding function:
+
+```typescript
+db.define('docs', {
+  columns: { id: 'TEXT PRIMARY KEY', title: 'TEXT', body: 'TEXT' },
+  render: (rows) => rows.map((r) => `## ${r.title}\n${r.body}`).join('\n\n---\n\n'),
+  outputFile: 'DOCS.md',
+  embeddings: {
+    fields: ['title', 'body'],
+    embed: async (text) => {
+      const res = await openai.embeddings.create({ input: text, model: 'text-embedding-3-small' });
+      return res.data[0].embedding;
+    },
+  },
+});
+
+await db.init();
+await db.insert('docs', { title: 'Deploy guide', body: 'How to deploy to production...' });
+
+// Search by meaning, not keywords
+const results = await db.search('docs', 'ship to prod', { topK: 5, minScore: 0.7 });
+for (const { row, score } of results) {
+  console.log(`${score.toFixed(2)} — ${row.title}`);
+}
+```
+
+Embeddings are stored in a companion SQLite table and cosine similarity is computed in JS — no external vector database required.
+
+### Writeback validation (v1.3+)
+
+Validate agent-written data before persisting. Reject low-quality or hallucinated writes:
+
+```typescript
+db.defineWriteback({
+  file: './agent-output/*.md',
+  parse: (content, offset) => ({ entries: [content.slice(offset)], nextOffset: content.length }),
+  persist: async (entry) => { /* save to DB */ },
+  validate: async (entry) => {
+    const text = entry as string;
+    const hasRequiredFields = text.includes('## Title') && text.includes('## Body');
+    return {
+      pass: hasRequiredFields,
+      score: hasRequiredFields ? 0.9 : 0.1,
+      reason: hasRequiredFields ? undefined : 'Missing required sections',
+    };
+  },
+  rejectBelow: 0.5,
+  onReject: (entry, result) => {
+    console.warn(`Rejected write: ${result.reason} (score: ${result.score})`);
+  },
+});
+```
+
+---
+
 ## Template rendering
 
 ### Built-in templates

package/dist/cli.js

@@ -321,7 +321,7 @@ import { join as join3 } from "path";
 import { existsSync as existsSync3, readFileSync as readFileSync3 } from "fs";
 
 // src/render/writer.ts
-import { writeFileSync as writeFileSync2, mkdirSync as mkdirSync2, renameSync, existsSync as existsSync2, readFileSync as readFileSync2 } from "fs";
+import { writeFileSync as writeFileSync2, mkdirSync as mkdirSync2, renameSync, copyFileSync, unlinkSync, existsSync as existsSync2, readFileSync as readFileSync2 } from "fs";
 import { createHash } from "crypto";
 import { dirname as dirname3, join as join2 } from "path";
 import { tmpdir } from "os";
@@ -334,7 +334,16 @@ function atomicWrite(filePath, content) {
   if (currentHash === newHash) return false;
   const tmp = join2(tmpdir(), `lattice-${randomBytes(8).toString("hex")}.tmp`);
   writeFileSync2(tmp, content, "utf8");
-  renameSync(tmp, filePath);
+  try {
+    renameSync(tmp, filePath);
+  } catch (err) {
+    if (err.code === "EXDEV") {
+      copyFileSync(tmp, filePath);
+      unlinkSync(tmp);
+    } else {
+      throw err;
+    }
+  }
   return true;
 }
 function existingHash(filePath) {
@@ -644,7 +653,56 @@ var Sanitizer = class {
 
 // src/render/engine.ts
 import { join as join5, basename, isAbsolute, resolve as resolve3 } from "path";
-import { mkdirSync as mkdirSync3, existsSync as existsSync5, copyFileSync } from "fs";
+import { mkdirSync as mkdirSync3, existsSync as existsSync5, copyFileSync as copyFileSync2 } from "fs";
+
+// src/render/token-budget.ts
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function applyTokenBudget(rows, renderFn, budget, prioritizeBy) {
+  const fullContent = renderFn(rows);
+  if (estimateTokens(fullContent) <= budget) return fullContent;
+  if (rows.length === 0) return fullContent;
+  const prioritized = [...rows];
+  if (typeof prioritizeBy === "function") {
+    prioritized.sort(prioritizeBy);
+  } else if (typeof prioritizeBy === "string") {
+    const col = prioritizeBy;
+    prioritized.sort((a, b) => {
+      const va = a[col];
+      const vb = b[col];
+      if (va == null && vb == null) return 0;
+      if (va == null) return 1;
+      if (vb == null) return -1;
+      if (va < vb) return 1;
+      if (va > vb) return -1;
+      return 0;
+    });
+  }
+  let lo = 0;
+  let hi = prioritized.length;
+  let bestContent = "";
+  let bestCount = 0;
+  while (lo < hi) {
+    const mid = Math.ceil((lo + hi) / 2);
+    const content = renderFn(prioritized.slice(0, mid));
+    if (estimateTokens(content) <= budget) {
+      bestContent = content;
+      bestCount = mid;
+      lo = mid;
+      if (lo === hi) break;
+    } else {
+      hi = mid - 1;
+    }
+  }
+  if (bestCount === 0) {
+    bestContent = renderFn([]);
+  }
+  const tokens = estimateTokens(bestContent);
+  return bestContent + `
+
+[truncated: ${bestCount} of ${rows.length} rows rendered, ~${tokens} tokens]`;
+}
 
 // src/render/entity-query.ts
 var SAFE_COL_RE = /^[a-zA-Z0-9_]+$/;
@@ -972,7 +1030,7 @@ ${tmpl.perRow.body(row)}
 
 // src/lifecycle/cleanup.ts
 import { join as join4 } from "path";
-import { existsSync as existsSync4, readdirSync, unlinkSync, rmdirSync, statSync } from "fs";
+import { existsSync as existsSync4, readdirSync, unlinkSync as unlinkSync2, rmdirSync, statSync } from "fs";
 function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, manifest, options = {}, newManifest) {
   const result = {
     directoriesRemoved: [],
@@ -1014,7 +1072,7 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
           if (globalProtected.has(filename)) continue;
           const filePath = join4(entityDir, filename);
           if (!existsSync4(filePath)) continue;
-          if (!options.dryRun) unlinkSync(filePath);
+          if (!options.dryRun) unlinkSync2(filePath);
           options.onOrphan?.(filePath, "file");
           result.filesRemoved.push(filePath);
         }
@@ -1059,7 +1117,7 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
           if (globalProtected.has(filename)) continue;
           const filePath = join4(entityDir, filename);
           if (!existsSync4(filePath)) continue;
-          if (!options.dryRun) unlinkSync(filePath);
+          if (!options.dryRun) unlinkSync2(filePath);
           options.onOrphan?.(filePath, "file");
           result.filesRemoved.push(filePath);
         }
@@ -1073,9 +1131,11 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
 var RenderEngine = class {
   _schema;
   _adapter;
-  constructor(schema, adapter) {
+  _getTaskContext;
+  constructor(schema, adapter, getTaskContext) {
     this._schema = schema;
     this._adapter = adapter;
+    this._getTaskContext = getTaskContext ?? (() => "");
   }
   async render(outputDir) {
     const start = Date.now();
@@ -1083,8 +1143,38 @@ var RenderEngine = class {
     const counters = { skipped: 0 };
     for (const [name, def] of this._schema.getTables()) {
       let rows = this._schema.queryTable(this._adapter, name);
+      if (def.relevanceFilter) {
+        const ctx = this._getTaskContext();
+        rows = rows.filter((row) => def.relevanceFilter(row, ctx));
+      }
       if (def.filter) rows = def.filter(rows);
-      const content = def.render(rows);
+      if (def.rewardTracking) {
+        if (def.pruneBelow !== void 0) {
+          const threshold = def.pruneBelow;
+          const toPrune = rows.filter(
+            (r) => r._reward_count > 0 && r._reward_total < threshold
+          );
+          if (toPrune.length > 0) {
+            for (const r of toPrune) {
+              const pkCol = this._schema.getPrimaryKey(name)[0] ?? "id";
+              this._adapter.run(
+                `UPDATE "${name}" SET deleted_at = datetime('now') WHERE "${pkCol}" = ?`,
+                [r[pkCol]]
+              );
+            }
+            rows = rows.filter(
+              (r) => r._reward_count === 0 || r._reward_total >= threshold
+            );
+          }
+        }
+        if (!def.prioritizeBy) {
+          rows.sort((a, b) => (b._reward_total ?? 0) - (a._reward_total ?? 0));
+        }
+      }
+      if (def.enrich) {
+        for (const fn of def.enrich) rows = fn(rows);
+      }
+      const content = def.tokenBudget ? applyTokenBudget(rows, def.render, def.tokenBudget, def.prioritizeBy) : def.render(rows);
       const filePath = join5(outputDir, def.outputFile);
       if (atomicWrite(filePath, content)) {
         filesWritten.push(filePath);
@@ -1203,7 +1293,7 @@ var RenderEngine = class {
               const destPath = join5(entityDir, basename(absPath));
               if (!existsSync5(destPath)) {
                 try {
-                  copyFileSync(absPath, destPath);
+                  copyFileSync2(absPath, destPath);
                   filesWritten.push(destPath);
                 } catch {
                 }
@@ -1488,6 +1578,14 @@ var WritebackPipeline = class {
           if (store.isSeen(filePath, key)) continue;
           store.markSeen(filePath, key);
         }
+        if (def.validate) {
+          const result = await def.validate(entry);
+          const threshold = def.rejectBelow ?? 0;
+          if (!result.pass || result.score < threshold) {
+            def.onReject?.(entry, result);
+            continue;
+          }
+        }
         await def.persist(entry, filePath);
         processed++;
       }
@@ -1666,6 +1764,73 @@ function resolveEncryptedColumns(encrypted, allColumns) {
   return new Set(allColumns.filter((c) => !SKIP_COLUMNS.has(c)));
 }
 
+// src/search/embeddings.ts
+var EMBEDDINGS_TABLE = "_lattice_embeddings";
+function ensureEmbeddingsTable(adapter) {
+  adapter.run(`CREATE TABLE IF NOT EXISTS "${EMBEDDINGS_TABLE}" (
+    "table_name" TEXT NOT NULL,
+    "row_pk"     TEXT NOT NULL,
+    "embedding"  TEXT NOT NULL,
+    PRIMARY KEY ("table_name", "row_pk")
+  )`);
+}
+async function storeEmbedding(adapter, table, pk, row, config) {
+  const text = config.fields.map((f) => {
+    const v = row[f];
+    return v == null ? "" : String(v);
+  }).filter((s) => s.length > 0).join(" ");
+  if (text.length === 0) return;
+  const vector = await config.embed(text);
+  adapter.run(
+    `INSERT OR REPLACE INTO "${EMBEDDINGS_TABLE}" ("table_name", "row_pk", "embedding") VALUES (?, ?, ?)`,
+    [table, pk, JSON.stringify(vector)]
+  );
+}
+function removeEmbedding(adapter, table, pk) {
+  adapter.run(
+    `DELETE FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ? AND "row_pk" = ?`,
+    [table, pk]
+  );
+}
+function cosineSimilarity(a, b) {
+  const len = Math.min(a.length, b.length);
+  let dot = 0;
+  let magA = 0;
+  let magB = 0;
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  return denom === 0 ? 0 : dot / denom;
+}
+async function searchByEmbedding(adapter, table, queryText, config, topK, minScore, pkColumn = "id") {
+  const queryVector = await config.embed(queryText);
+  const stored = adapter.all(
+    `SELECT "row_pk", "embedding" FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
+    [table]
+  );
+  const scored = [];
+  for (const entry of stored) {
+    const vec = JSON.parse(entry.embedding);
+    const score = cosineSimilarity(queryVector, vec);
+    if (score >= minScore) {
+      scored.push({ pk: entry.row_pk, score });
+    }
+  }
+  scored.sort((a, b) => b.score - a.score);
+  const topResults = scored.slice(0, topK);
+  const results = [];
+  for (const { pk, score } of topResults) {
+    const row = adapter.get(`SELECT * FROM "${table}" WHERE "${pkColumn}" = ?`, [pk]);
+    if (row) {
+      results.push({ row, score });
+    }
+  }
+  return results;
+}
+
 // src/lattice.ts
 var Lattice = class {
   _adapter;
@@ -1684,6 +1849,8 @@ var Lattice = class {
   _encryptedTableColumns = /* @__PURE__ */ new Map();
   /** Raw encryption key passphrase from constructor options. */
   _encryptionKeyRaw;
+  /** Current task context string for relevance filtering. */
+  _taskContext = "";
   _auditHandlers = [];
   _renderHandlers = [];
   _writebackHandlers = [];
@@ -1710,7 +1877,7 @@ var Lattice = class {
     this._adapter = new SQLiteAdapter(dbPath, adapterOpts);
     this._schema = new SchemaManager();
     this._sanitizer = new Sanitizer(options.security);
-    this._render = new RenderEngine(this._schema, this._adapter);
+    this._render = new RenderEngine(this._schema, this._adapter, () => this._taskContext);
     this._reverseSync = new ReverseSyncEngine(this._schema, this._adapter);
     this._loop = new SyncLoop(this._render);
     this._writeback = new WritebackPipeline();
@@ -1734,8 +1901,10 @@ var Lattice = class {
   // -------------------------------------------------------------------------
   define(table, def) {
     this._assertNotInit("define");
+    const columns = def.rewardTracking ? { ...def.columns, _reward_total: "REAL DEFAULT 0", _reward_count: "INTEGER DEFAULT 0" } : def.columns;
     const compiledDef = {
       ...def,
+      columns,
       render: def.render ? compileRender(
         def,
         table,
@@ -1781,6 +1950,10 @@ var Lattice = class {
       const rows = this._adapter.all(`PRAGMA table_info("${tableName}")`);
       this._columnCache.set(tableName, new Set(rows.map((r) => r.name)));
     }
+    const hasEmbeddings = [...this._schema.getTables().values()].some((d) => d.embeddings);
+    if (hasEmbeddings) {
+      ensureEmbeddingsTable(this._adapter);
+    }
     this._setupEncryption();
     this._initialized = true;
     return Promise.resolve();
@@ -1810,6 +1983,21 @@ var Lattice = class {
     this._initialized = false;
   }
   // -------------------------------------------------------------------------
+  // Task context (for relevance filtering)
+  // -------------------------------------------------------------------------
+  /**
+   * Set the current task context string. Tables with a `relevanceFilter`
+   * will use this value to filter rows before rendering.
+   */
+  setTaskContext(context) {
+    this._taskContext = context;
+    return this;
+  }
+  /** Return the current task context string. */
+  getTaskContext() {
+    return this._taskContext;
+  }
+  // -------------------------------------------------------------------------
   // Encryption helpers
   // -------------------------------------------------------------------------
   _setupEncryption() {
@@ -1884,6 +2072,7 @@ var Lattice = class {
     const pkValue = rawPk != null ? String(rawPk) : "";
     this._sanitizer.emitAudit(table, "insert", pkValue);
     this._fireWriteHooks(table, "insert", rowWithPk, pkValue);
+    this._syncEmbedding(table, "insert", rowWithPk, pkValue);
     return Promise.resolve(pkValue);
   }
   /**
@@ -1951,6 +2140,11 @@ var Lattice = class {
     const auditId = typeof id === "string" ? id : JSON.stringify(id);
     this._sanitizer.emitAudit(table, "update", auditId);
     this._fireWriteHooks(table, "update", sanitized, auditId, Object.keys(sanitized));
+    const def = this._schema.getTables().get(table);
+    if (def?.embeddings) {
+      const fullRow = this._adapter.get(`SELECT * FROM "${table}" WHERE ${clause}`, pkParams);
+      if (fullRow) this._syncEmbedding(table, "update", fullRow, auditId);
+    }
     return Promise.resolve();
   }
   /**
@@ -1972,6 +2166,7 @@ var Lattice = class {
     const auditId = typeof id === "string" ? id : JSON.stringify(id);
     this._sanitizer.emitAudit(table, "delete", auditId);
     this._fireWriteHooks(table, "delete", { id: auditId }, auditId);
+    this._syncEmbedding(table, "delete", {}, auditId);
     return Promise.resolve();
   }
   get(table, id) {
@@ -2355,6 +2550,65 @@ var Lattice = class {
     const ms = unit === "h" ? num * 36e5 : unit === "d" ? num * 864e5 : num * 6e4;
     return new Date(Date.now() - ms).toISOString();
   }
+  // -------------------------------------------------------------------------
+  // Reward tracking
+  // -------------------------------------------------------------------------
+  /**
+   * Update reward scores for a row. The total reward is recalculated as
+   * the running average across all reward calls. Requires `rewardTracking`
+   * on the table definition.
+   */
+  reward(table, id, scores) {
+    const notInit = this._notInitError();
+    if (notInit) return notInit;
+    const def = this._schema.getTables().get(table);
+    if (!def?.rewardTracking) {
+      return Promise.reject(
+        new Error(`Table "${table}" does not have rewardTracking enabled`)
+      );
+    }
+    const vals = Object.values(scores);
+    if (vals.length === 0) return Promise.resolve();
+    const avg = vals.reduce((a, b) => a + b, 0) / vals.length;
+    const { clause, params: pkParams } = this._pkWhere(table, id);
+    this._adapter.run(
+      `UPDATE "${table}" SET "_reward_total" = ("_reward_total" * "_reward_count" + ?) / ("_reward_count" + 1), "_reward_count" = "_reward_count" + 1 WHERE ${clause}`,
+      [avg, ...pkParams]
+    );
+    return Promise.resolve();
+  }
+  // -------------------------------------------------------------------------
+  // Semantic search
+  // -------------------------------------------------------------------------
+  /**
+   * Search for rows by semantic similarity. Requires `embeddings` config
+   * on the table definition.
+   *
+   * @param table  - Table to search
+   * @param query  - Natural-language query text
+   * @param opts   - Search options (topK, minScore)
+   * @returns Matching rows with similarity scores, sorted best-first.
+   */
+  async search(table, query, opts = {}) {
+    const notInit = this._notInitError();
+    if (notInit) return notInit;
+    const def = this._schema.getTables().get(table);
+    if (!def?.embeddings) {
+      return Promise.reject(
+        new Error(`Table "${table}" does not have embeddings configured`)
+      );
+    }
+    const pkCol = this._schema.getPrimaryKey(table)[0] ?? "id";
+    return searchByEmbedding(
+      this._adapter,
+      table,
+      query,
+      def.embeddings,
+      opts.topK ?? 10,
+      opts.minScore ?? 0,
+      pkCol
+    );
+  }
   query(table, opts = {}) {
     const notInit = this._notInitError();
     if (notInit) return notInit;
@@ -2613,6 +2867,23 @@ var Lattice = class {
       }
     }
   }
+  /**
+   * Update or remove the embedding for a row.
+   * No-op if the table doesn't have `embeddings` configured.
+   */
+  _syncEmbedding(table, op, row, pk) {
+    const def = this._schema.getTables().get(table);
+    if (!def?.embeddings) return;
+    if (op === "delete") {
+      removeEmbedding(this._adapter, table, pk);
+      return;
+    }
+    storeEmbedding(this._adapter, table, pk, row, def.embeddings).catch((err) => {
+      for (const h of this._errorHandlers) {
+        h(err instanceof Error ? err : new Error(String(err)));
+      }
+    });
+  }
   _notInitError() {
     if (!this._initialized) {
       return Promise.reject(