latticesql 1.2.6 → 1.3.1

package/dist/index.cjs

@@ -35,6 +35,7 @@ __export(index_exports, {
   InMemoryStateStore: () => InMemoryStateStore,
   Lattice: () => Lattice,
   READ_ONLY_HEADER: () => READ_ONLY_HEADER,
+  applyTokenBudget: () => applyTokenBudget,
   applyWriteEntry: () => applyWriteEntry,
   autoUpdate: () => autoUpdate,
   contentHash: () => contentHash,
@@ -44,6 +45,7 @@ __export(index_exports, {
   deriveKey: () => deriveKey,
   encrypt: () => encrypt,
   entityFileNames: () => entityFileNames,
+  estimateTokens: () => estimateTokens,
   fixSchemaConflicts: () => fixSchemaConflicts,
   frontmatter: () => frontmatter,
   generateEntryId: () => generateEntryId,
@@ -87,7 +89,16 @@ function atomicWrite(filePath, content) {
   if (currentHash === newHash) return false;
   const tmp = (0, import_node_path.join)((0, import_node_os.tmpdir)(), `lattice-${(0, import_node_crypto2.randomBytes)(8).toString("hex")}.tmp`);
   (0, import_node_fs.writeFileSync)(tmp, content, "utf8");
-  (0, import_node_fs.renameSync)(tmp, filePath);
+  try {
+    (0, import_node_fs.renameSync)(tmp, filePath);
+  } catch (err) {
+    if (err.code === "EXDEV") {
+      (0, import_node_fs.copyFileSync)(tmp, filePath);
+      (0, import_node_fs.unlinkSync)(tmp);
+    } else {
+      throw err;
+    }
+  }
   return true;
 }
 function existingHash(filePath) {
@@ -399,6 +410,55 @@ var Sanitizer = class {
 var import_node_path4 = require("path");
 var import_node_fs4 = require("fs");
 
+// src/render/token-budget.ts
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function applyTokenBudget(rows, renderFn, budget, prioritizeBy) {
+  const fullContent = renderFn(rows);
+  if (estimateTokens(fullContent) <= budget) return fullContent;
+  if (rows.length === 0) return fullContent;
+  const prioritized = [...rows];
+  if (typeof prioritizeBy === "function") {
+    prioritized.sort(prioritizeBy);
+  } else if (typeof prioritizeBy === "string") {
+    const col = prioritizeBy;
+    prioritized.sort((a, b) => {
+      const va = a[col];
+      const vb = b[col];
+      if (va == null && vb == null) return 0;
+      if (va == null) return 1;
+      if (vb == null) return -1;
+      if (va < vb) return 1;
+      if (va > vb) return -1;
+      return 0;
+    });
+  }
+  let lo = 0;
+  let hi = prioritized.length;
+  let bestContent = "";
+  let bestCount = 0;
+  while (lo < hi) {
+    const mid = Math.ceil((lo + hi) / 2);
+    const content = renderFn(prioritized.slice(0, mid));
+    if (estimateTokens(content) <= budget) {
+      bestContent = content;
+      bestCount = mid;
+      lo = mid;
+      if (lo === hi) break;
+    } else {
+      hi = mid - 1;
+    }
+  }
+  if (bestCount === 0) {
+    bestContent = renderFn([]);
+  }
+  const tokens = estimateTokens(bestContent);
+  return bestContent + `
+
+[truncated: ${bestCount} of ${rows.length} rows rendered, ~${tokens} tokens]`;
+}
+
 // src/render/entity-query.ts
 var SAFE_COL_RE = /^[a-zA-Z0-9_]+$/;
 function effectiveFilters(opts) {
@@ -833,9 +893,11 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
 var RenderEngine = class {
   _schema;
   _adapter;
-  constructor(schema, adapter) {
+  _getTaskContext;
+  constructor(schema, adapter, getTaskContext) {
     this._schema = schema;
     this._adapter = adapter;
+    this._getTaskContext = getTaskContext ?? (() => "");
   }
   async render(outputDir) {
     const start = Date.now();
@@ -843,8 +905,38 @@ var RenderEngine = class {
     const counters = { skipped: 0 };
     for (const [name, def] of this._schema.getTables()) {
       let rows = this._schema.queryTable(this._adapter, name);
+      if (def.relevanceFilter) {
+        const ctx = this._getTaskContext();
+        rows = rows.filter((row) => def.relevanceFilter(row, ctx));
+      }
       if (def.filter) rows = def.filter(rows);
-      const content = def.render(rows);
+      if (def.rewardTracking) {
+        if (def.pruneBelow !== void 0) {
+          const threshold = def.pruneBelow;
+          const toPrune = rows.filter(
+            (r) => r._reward_count > 0 && r._reward_total < threshold
+          );
+          if (toPrune.length > 0) {
+            for (const r of toPrune) {
+              const pkCol = this._schema.getPrimaryKey(name)[0] ?? "id";
+              this._adapter.run(
+                `UPDATE "${name}" SET deleted_at = datetime('now') WHERE "${pkCol}" = ?`,
+                [r[pkCol]]
+              );
+            }
+            rows = rows.filter(
+              (r) => r._reward_count === 0 || r._reward_total >= threshold
+            );
+          }
+        }
+        if (!def.prioritizeBy) {
+          rows.sort((a, b) => (b._reward_total ?? 0) - (a._reward_total ?? 0));
+        }
+      }
+      if (def.enrich) {
+        for (const fn of def.enrich) rows = fn(rows);
+      }
+      const content = def.tokenBudget ? applyTokenBudget(rows, def.render, def.tokenBudget, def.prioritizeBy) : def.render(rows);
       const filePath = (0, import_node_path4.join)(outputDir, def.outputFile);
       if (atomicWrite(filePath, content)) {
         filesWritten.push(filePath);
@@ -1304,6 +1396,14 @@ var WritebackPipeline = class {
           if (store.isSeen(filePath, key)) continue;
           store.markSeen(filePath, key);
         }
+        if (def.validate) {
+          const result = await def.validate(entry);
+          const threshold = def.rejectBelow ?? 0;
+          if (!result.pass || result.score < threshold) {
+            def.onReject?.(entry, result);
+            continue;
+          }
+        }
         await def.persist(entry, filePath);
         processed++;
       }
@@ -1710,6 +1810,73 @@ function resolveEncryptedColumns(encrypted, allColumns) {
   return new Set(allColumns.filter((c) => !SKIP_COLUMNS.has(c)));
 }
 
+// src/search/embeddings.ts
+var EMBEDDINGS_TABLE = "_lattice_embeddings";
+function ensureEmbeddingsTable(adapter) {
+  adapter.run(`CREATE TABLE IF NOT EXISTS "${EMBEDDINGS_TABLE}" (
+    "table_name" TEXT NOT NULL,
+    "row_pk"     TEXT NOT NULL,
+    "embedding"  TEXT NOT NULL,
+    PRIMARY KEY ("table_name", "row_pk")
+  )`);
+}
+async function storeEmbedding(adapter, table, pk, row, config) {
+  const text = config.fields.map((f) => {
+    const v = row[f];
+    return v == null ? "" : String(v);
+  }).filter((s) => s.length > 0).join(" ");
+  if (text.length === 0) return;
+  const vector = await config.embed(text);
+  adapter.run(
+    `INSERT OR REPLACE INTO "${EMBEDDINGS_TABLE}" ("table_name", "row_pk", "embedding") VALUES (?, ?, ?)`,
+    [table, pk, JSON.stringify(vector)]
+  );
+}
+function removeEmbedding(adapter, table, pk) {
+  adapter.run(
+    `DELETE FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ? AND "row_pk" = ?`,
+    [table, pk]
+  );
+}
+function cosineSimilarity(a, b) {
+  const len = Math.min(a.length, b.length);
+  let dot = 0;
+  let magA = 0;
+  let magB = 0;
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(magA) * Math.sqrt(magB);
+  return denom === 0 ? 0 : dot / denom;
+}
+async function searchByEmbedding(adapter, table, queryText, config, topK, minScore, pkColumn = "id") {
+  const queryVector = await config.embed(queryText);
+  const stored = adapter.all(
+    `SELECT "row_pk", "embedding" FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ?`,
+    [table]
+  );
+  const scored = [];
+  for (const entry of stored) {
+    const vec = JSON.parse(entry.embedding);
+    const score = cosineSimilarity(queryVector, vec);
+    if (score >= minScore) {
+      scored.push({ pk: entry.row_pk, score });
+    }
+  }
+  scored.sort((a, b) => b.score - a.score);
+  const topResults = scored.slice(0, topK);
+  const results = [];
+  for (const { pk, score } of topResults) {
+    const row = adapter.get(`SELECT * FROM "${table}" WHERE "${pkColumn}" = ?`, [pk]);
+    if (row) {
+      results.push({ row, score });
+    }
+  }
+  return results;
+}
+
 // src/lattice.ts
 var Lattice = class {
   _adapter;
@@ -1728,6 +1895,8 @@ var Lattice = class {
   _encryptedTableColumns = /* @__PURE__ */ new Map();
   /** Raw encryption key passphrase from constructor options. */
   _encryptionKeyRaw;
+  /** Current task context string for relevance filtering. */
+  _taskContext = "";
   _auditHandlers = [];
   _renderHandlers = [];
   _writebackHandlers = [];
@@ -1754,7 +1923,7 @@ var Lattice = class {
     this._adapter = new SQLiteAdapter(dbPath, adapterOpts);
     this._schema = new SchemaManager();
     this._sanitizer = new Sanitizer(options.security);
-    this._render = new RenderEngine(this._schema, this._adapter);
+    this._render = new RenderEngine(this._schema, this._adapter, () => this._taskContext);
     this._reverseSync = new ReverseSyncEngine(this._schema, this._adapter);
     this._loop = new SyncLoop(this._render);
     this._writeback = new WritebackPipeline();
@@ -1778,8 +1947,10 @@ var Lattice = class {
   // -------------------------------------------------------------------------
   define(table, def) {
     this._assertNotInit("define");
+    const columns = def.rewardTracking ? { ...def.columns, _reward_total: "REAL DEFAULT 0", _reward_count: "INTEGER DEFAULT 0" } : def.columns;
     const compiledDef = {
       ...def,
+      columns,
       render: def.render ? compileRender(
         def,
         table,
@@ -1825,6 +1996,10 @@ var Lattice = class {
       const rows = this._adapter.all(`PRAGMA table_info("${tableName}")`);
       this._columnCache.set(tableName, new Set(rows.map((r) => r.name)));
     }
+    const hasEmbeddings = [...this._schema.getTables().values()].some((d) => d.embeddings);
+    if (hasEmbeddings) {
+      ensureEmbeddingsTable(this._adapter);
+    }
     this._setupEncryption();
     this._initialized = true;
     return Promise.resolve();
@@ -1854,6 +2029,21 @@ var Lattice = class {
     this._initialized = false;
   }
   // -------------------------------------------------------------------------
+  // Task context (for relevance filtering)
+  // -------------------------------------------------------------------------
+  /**
+   * Set the current task context string. Tables with a `relevanceFilter`
+   * will use this value to filter rows before rendering.
+   */
+  setTaskContext(context) {
+    this._taskContext = context;
+    return this;
+  }
+  /** Return the current task context string. */
+  getTaskContext() {
+    return this._taskContext;
+  }
+  // -------------------------------------------------------------------------
   // Encryption helpers
   // -------------------------------------------------------------------------
   _setupEncryption() {
@@ -1928,6 +2118,7 @@ var Lattice = class {
     const pkValue = rawPk != null ? String(rawPk) : "";
     this._sanitizer.emitAudit(table, "insert", pkValue);
     this._fireWriteHooks(table, "insert", rowWithPk, pkValue);
+    this._syncEmbedding(table, "insert", rowWithPk, pkValue);
     return Promise.resolve(pkValue);
   }
   /**
@@ -1995,6 +2186,11 @@ var Lattice = class {
     const auditId = typeof id === "string" ? id : JSON.stringify(id);
     this._sanitizer.emitAudit(table, "update", auditId);
     this._fireWriteHooks(table, "update", sanitized, auditId, Object.keys(sanitized));
+    const def = this._schema.getTables().get(table);
+    if (def?.embeddings) {
+      const fullRow = this._adapter.get(`SELECT * FROM "${table}" WHERE ${clause}`, pkParams);
+      if (fullRow) this._syncEmbedding(table, "update", fullRow, auditId);
+    }
     return Promise.resolve();
   }
   /**
@@ -2016,6 +2212,7 @@ var Lattice = class {
     const auditId = typeof id === "string" ? id : JSON.stringify(id);
     this._sanitizer.emitAudit(table, "delete", auditId);
     this._fireWriteHooks(table, "delete", { id: auditId }, auditId);
+    this._syncEmbedding(table, "delete", {}, auditId);
     return Promise.resolve();
   }
   get(table, id) {
@@ -2399,6 +2596,65 @@ var Lattice = class {
     const ms = unit === "h" ? num * 36e5 : unit === "d" ? num * 864e5 : num * 6e4;
     return new Date(Date.now() - ms).toISOString();
   }
+  // -------------------------------------------------------------------------
+  // Reward tracking
+  // -------------------------------------------------------------------------
+  /**
+   * Update reward scores for a row. The total reward is recalculated as
+   * the running average across all reward calls. Requires `rewardTracking`
+   * on the table definition.
+   */
+  reward(table, id, scores) {
+    const notInit = this._notInitError();
+    if (notInit) return notInit;
+    const def = this._schema.getTables().get(table);
+    if (!def?.rewardTracking) {
+      return Promise.reject(
+        new Error(`Table "${table}" does not have rewardTracking enabled`)
+      );
+    }
+    const vals = Object.values(scores);
+    if (vals.length === 0) return Promise.resolve();
+    const avg = vals.reduce((a, b) => a + b, 0) / vals.length;
+    const { clause, params: pkParams } = this._pkWhere(table, id);
+    this._adapter.run(
+      `UPDATE "${table}" SET "_reward_total" = ("_reward_total" * "_reward_count" + ?) / ("_reward_count" + 1), "_reward_count" = "_reward_count" + 1 WHERE ${clause}`,
+      [avg, ...pkParams]
+    );
+    return Promise.resolve();
+  }
+  // -------------------------------------------------------------------------
+  // Semantic search
+  // -------------------------------------------------------------------------
+  /**
+   * Search for rows by semantic similarity. Requires `embeddings` config
+   * on the table definition.
+   *
+   * @param table  - Table to search
+   * @param query  - Natural-language query text
+   * @param opts   - Search options (topK, minScore)
+   * @returns Matching rows with similarity scores, sorted best-first.
+   */
+  async search(table, query, opts = {}) {
+    const notInit = this._notInitError();
+    if (notInit) return notInit;
+    const def = this._schema.getTables().get(table);
+    if (!def?.embeddings) {
+      return Promise.reject(
+        new Error(`Table "${table}" does not have embeddings configured`)
+      );
+    }
+    const pkCol = this._schema.getPrimaryKey(table)[0] ?? "id";
+    return searchByEmbedding(
+      this._adapter,
+      table,
+      query,
+      def.embeddings,
+      opts.topK ?? 10,
+      opts.minScore ?? 0,
+      pkCol
+    );
+  }
   query(table, opts = {}) {
     const notInit = this._notInitError();
     if (notInit) return notInit;
@@ -2657,6 +2913,23 @@ var Lattice = class {
       }
     }
   }
+  /**
+   * Update or remove the embedding for a row.
+   * No-op if the table doesn't have `embeddings` configured.
+   */
+  _syncEmbedding(table, op, row, pk) {
+    const def = this._schema.getTables().get(table);
+    if (!def?.embeddings) return;
+    if (op === "delete") {
+      removeEmbedding(this._adapter, table, pk);
+      return;
+    }
+    storeEmbedding(this._adapter, table, pk, row, def.embeddings).catch((err) => {
+      for (const h of this._errorHandlers) {
+        h(err instanceof Error ? err : new Error(String(err)));
+      }
+    });
+  }
   _notInitError() {
     if (!this._initialized) {
       return Promise.reject(
@@ -3198,6 +3471,7 @@ async function autoUpdate(opts) {
   InMemoryStateStore,
   Lattice,
   READ_ONLY_HEADER,
+  applyTokenBudget,
   applyWriteEntry,
   autoUpdate,
   contentHash,
@@ -3207,6 +3481,7 @@ async function autoUpdate(opts) {
   deriveKey,
   encrypt,
   entityFileNames,
+  estimateTokens,
   fixSchemaConflicts,
   frontmatter,
   generateEntryId,

package/dist/index.d.cts

@@ -812,6 +812,90 @@ interface TableDefinition {
      * ```
      */
     relations?: Record<string, Relation>;
+    /**
+     * Enable semantic search for this table via embeddings.
+     *
+     * When configured, Lattice computes and stores vector embeddings for
+     * the specified text fields. Use `lattice.search(table, query, opts)`
+     * to retrieve rows by semantic similarity.
+     *
+     * The `embed` function is called to generate vectors — bring your own
+     * embedding model (OpenAI, local model, etc.).
+     *
+     * @example
+     * ```ts
+     * embeddings: {
+     *   fields: ['title', 'body'],
+     *   embed: async (text) => openai.embeddings.create({ input: text, model: 'text-embedding-3-small' }).then(r => r.data[0].embedding),
+     * }
+     * ```
+     */
+    embeddings?: EmbeddingsConfig;
+    /**
+     * Enable reward tracking for this table. When `true`, Lattice
+     * auto-adds `_reward_total REAL DEFAULT 0` and `_reward_count INTEGER
+     * DEFAULT 0` columns. Rows are sorted by `_reward_total DESC` before
+     * rendering (unless overridden by `prioritizeBy`).
+     *
+     * Use `lattice.reward(table, id, scores)` to update reward values.
+     */
+    rewardTracking?: boolean;
+    /**
+     * When `rewardTracking` is enabled, automatically soft-delete rows
+     * whose `_reward_total` falls below this threshold during rendering.
+     * Requires a `deleted_at` column on the table. Default: no pruning.
+     */
+    pruneBelow?: number;
+    /**
+     * Pipeline of enrichment functions applied to rows after query and
+     * filtering but before rendering. Each function receives the row
+     * array and returns a (possibly transformed) row array.
+     *
+     * Use enrichment hooks to cluster, annotate, summarize, or
+     * cross-reference rows without modifying the underlying data.
+     *
+     * @example
+     * ```ts
+     * enrich: [
+     *   (rows) => rows.map(r => ({ ...r, _age: Date.now() - new Date(r.created_at as string).getTime() })),
+     *   (rows) => rows.length > 50 ? [{ summary: `${rows.length} items` }] : rows,
+     * ]
+     * ```
+     */
+    enrich?: ((rows: Row[]) => Row[])[];
+    /**
+     * Dynamic filter that scores rows against the current task context
+     * (set via `lattice.setTaskContext()`). Called before `filter` and
+     * before rendering. Only rows for which the function returns `true`
+     * are included.
+     *
+     * The second argument is the current task-context string (empty string
+     * when none has been set).
+     *
+     * @example
+     * ```ts
+     * relevanceFilter: (row, ctx) =>
+     *   ctx ? String(row.body).toLowerCase().includes(ctx.toLowerCase()) : true
+     * ```
+     */
+    relevanceFilter?: (row: Row, taskContext: string) => boolean;
+    /**
+     * Maximum estimated token count for the rendered output.
+     * When the rendered content exceeds this budget, rows are pruned
+     * (lowest-priority first) and re-rendered with a truncation notice.
+     *
+     * Token count is estimated at ~4 characters per token.
+     */
+    tokenBudget?: number;
+    /**
+     * Controls row priority when `tokenBudget` forces pruning.
+     *
+     * - `string` — column name to sort by descending (highest value = highest priority).
+     * - `(a, b) => number` — custom comparator (positive = a has higher priority).
+     *
+     * When omitted, rows at the end of the query result are pruned first.
+     */
+    prioritizeBy?: string | ((a: Row, b: Row) => number);
 }
 interface MultiTableDefinition {
     /** Returns the "anchor" entities — one output file is produced per anchor */
@@ -823,6 +907,58 @@ interface MultiTableDefinition {
     /** Additional table names to query and pass into render */
     tables?: string[];
 }
+/**
+ * Configuration for embedding-based semantic search on a table.
+ */
+interface EmbeddingsConfig {
+    /** Column names whose values are concatenated and embedded. */
+    fields: string[];
+    /**
+     * Function that converts text into a numeric vector.
+     * Bring your own model — Lattice does not bundle an embedding provider.
+     */
+    embed: (text: string) => Promise<number[]>;
+}
+/**
+ * Options for `Lattice.search()`.
+ */
+interface SearchOptions {
+    /** Maximum number of results to return. Default: 10. */
+    topK?: number;
+    /**
+     * Minimum cosine similarity threshold (0–1). Results below this
+     * score are excluded. Default: 0.
+     */
+    minScore?: number;
+}
+/**
+ * A single search result returned by `Lattice.search()`.
+ */
+interface SearchResult {
+    /** The matched row from the source table. */
+    row: Row;
+    /** Cosine similarity score (0–1). */
+    score: number;
+}
+/**
+ * Dimension scores passed to `Lattice.reward()`.
+ * Values should be between 0 and 1. The total reward is the average
+ * of all provided dimension scores, accumulated over multiple calls.
+ */
+interface RewardScores {
+    [dimension: string]: number;
+}
+/**
+ * Result of a writeback validation check.
+ */
+interface WritebackValidationResult {
+    /** Whether the entry passed validation. */
+    pass: boolean;
+    /** Overall quality score (0–1). */
+    score: number;
+    /** Human-readable reason when validation fails. */
+    reason?: string;
+}
 interface WritebackDefinition {
     /** Path or glob to agent-written files */
     file: string;
@@ -843,6 +979,25 @@ interface WritebackDefinition {
     stateStore?: WritebackStateStore;
     /** Called after entries are processed for a file. Useful for archival. */
     onArchive?: (filePath: string) => void;
+    /**
+     * Optional validation hook. Called before `persist` for each entry.
+     * Return `{ pass: true, score }` to allow the write, or
+     * `{ pass: false, score, reason }` to reject it.
+     *
+     * When omitted, all parsed entries are persisted without validation.
+     */
+    validate?: (entry: unknown) => WritebackValidationResult | Promise<WritebackValidationResult>;
+    /**
+     * Minimum score threshold. Entries with `score < rejectBelow` are
+     * automatically rejected even if `validate` returns `pass: true`.
+     * Only meaningful when `validate` is set. Default: 0 (no threshold).
+     */
+    rejectBelow?: number;
+    /**
+     * Called for each entry that fails validation.
+     * Useful for logging or auditing rejected writes.
+     */
+    onReject?: (entry: unknown, result: WritebackValidationResult) => void;
 }
 interface QueryOptions {
     /**
@@ -1154,6 +1309,8 @@ declare class Lattice {
     private readonly _encryptedTableColumns;
     /** Raw encryption key passphrase from constructor options. */
     private readonly _encryptionKeyRaw?;
+    /** Current task context string for relevance filtering. */
+    private _taskContext;
     private readonly _auditHandlers;
     private readonly _renderHandlers;
     private readonly _writebackHandlers;
@@ -1178,6 +1335,13 @@ declare class Lattice {
      */
     migrate(migrations: Migration[]): Promise<void>;
     close(): void;
+    /**
+     * Set the current task context string. Tables with a `relevanceFilter`
+     * will use this value to filter rows before rendering.
+     */
+    setTaskContext(context: string): this;
+    /** Return the current task context string. */
+    getTaskContext(): string;
     private _setupEncryption;
     /** Encrypt applicable columns in a row before writing. Returns a new row. */
     private _encryptRow;
@@ -1258,6 +1422,22 @@ declare class Lattice {
     buildReport(config: ReportConfig): Promise<ReportResult>;
     /** Parse duration shorthand ('8h', '24h', '7d') into ISO timestamp. */
     private _resolveSince;
+    /**
+     * Update reward scores for a row. The total reward is recalculated as
+     * the running average across all reward calls. Requires `rewardTracking`
+     * on the table definition.
+     */
+    reward(table: string, id: PkLookup, scores: RewardScores): Promise<void>;
+    /**
+     * Search for rows by semantic similarity. Requires `embeddings` config
+     * on the table definition.
+     *
+     * @param table  - Table to search
+     * @param query  - Natural-language query text
+     * @param opts   - Search options (topK, minScore)
+     * @returns Matching rows with similarity scores, sorted best-first.
+     */
+    search(table: string, query: string, opts?: SearchOptions): Promise<SearchResult[]>;
     query(table: string, opts?: QueryOptions): Promise<Row[]>;
     count(table: string, opts?: CountOptions): Promise<number>;
     render(outputDir: string): Promise<RenderResult>;
@@ -1297,6 +1477,11 @@ declare class Lattice {
     private _buildFilters;
     /** Returns a rejected Promise if not initialized; null if ready. */
     private _fireWriteHooks;
+    /**
+     * Update or remove the embedding for a row.
+     * No-op if the table doesn't have `embeddings` configured.
+     */
+    private _syncEmbedding;
     private _notInitError;
     /**
      * Returns a rejected Promise if any of the given column names are not present
@@ -1512,6 +1697,25 @@ declare function parseConfigString(yamlContent: string, configDir: string): Pars
 
 declare function contentHash(content: string): string;
 
+/**
+ * Estimate the number of tokens in a string using a fast heuristic.
+ * Uses ~4 characters per token, a reasonable approximation for English
+ * text with typical LLM tokenizers (BPE/ByteBPE).
+ *
+ * Users who need exact counts should pre-process with their own tokenizer
+ * and use the `filter` hook to manually limit rows.
+ */
+declare function estimateTokens(text: string): number;
+/**
+ * Apply a token budget to rendered output.
+ * If the full render exceeds the budget, rows are sorted by priority,
+ * pruned via binary search, and re-rendered. A truncation footer is
+ * appended so the consuming agent knows context was limited.
+ *
+ * @returns Final content string, possibly with truncation footer.
+ */
+declare function applyTokenBudget(rows: Row[], renderFn: (rows: Row[]) => string, budget: number, prioritizeBy?: string | ((a: Row, b: Row) => number)): string;
+
 /**
  * Fix legacy schema conflicts before Lattice init().
  *
@@ -1827,4 +2031,4 @@ declare function autoUpdate(opts?: {
     quiet?: boolean;
 }): Promise<AutoUpdateResult>;
 
-export { type ApplyWriteResult, type AuditEvent, type AutoUpdateResult, type BelongsToRelation, type BelongsToSource, type BuiltinTemplateName, type CleanupOptions, type CleanupResult, type CountOptions, type CustomSource, DEFAULT_ENTRY_TYPES, DEFAULT_TYPE_ALIASES, type EnrichedSource, type EnrichmentLookup, type EntityContextDefinition, type EntityContextManifestEntry, type EntityFileManifestInfo, type EntityFileSource, type EntityFileSpec, type EntityProfileField, type EntityProfileSection, type EntityProfileTemplate, type EntityRenderSpec, type EntityRenderTemplate, type EntitySectionPerRow, type EntitySectionsTemplate, type EntityTableColumn, type EntityTableTemplate, type Filter, type FilterOp, type HasManyRelation, type HasManySource, InMemoryStateStore, type InitOptions, Lattice, type LatticeConfig, type LatticeConfigInput, type LatticeEntityDef, type LatticeEntityRenderSpec, type LatticeFieldDef, type LatticeFieldType, type LatticeManifest, type LatticeOptions, type LinkOptions, type ManyToManySource, type MarkdownTableColumn, type Migration, type MultiTableDefinition, type OrderBySpec, type ParseError, type ParseResult, type ParsedConfig, type PkLookup, type PrimaryKey, type QueryOptions, READ_ONLY_HEADER, type ReadOnlyHeaderOptions, type ReconcileOptions, type ReconcileResult, type Relation, type RenderHooks, type RenderResult, type RenderSpec, type ReportConfig, type ReportResult, type ReportSection, type ReportSectionResult, type ReverseSyncError, type ReverseSyncResult, type ReverseSyncUpdate, type Row, type SecurityOptions, type SeedConfig, type SeedLinkSpec, type SeedResult, type SelfSource, type SessionEntry, type SessionParseOptions, type SessionWriteEntry, type SessionWriteOp, type SessionWriteParseResult, type SourceQueryOptions, type StopFn, type SyncResult, type TableDefinition, type TemplateRenderSpec, type UpsertByNaturalKeyOptions, type WatchOptions, type WriteHook, type WriteHookContext, type WritebackDefinition, type WritebackStateStore, applyWriteEntry, autoUpdate, contentHash, createReadOnlyHeader, createSQLiteStateStore, decrypt, deriveKey, encrypt, entityFileNames, fixSchemaConflicts, frontmatter, generateEntryId, generateWriteEntryId, isEncrypted, isV1EntityFiles, manifestPath, markdownTable, normalizeEntityFiles, parseConfigFile, parseConfigString, parseMarkdownEntries, parseSessionMD, parseSessionWrites, readManifest, slugify, truncate, validateEntryId, writeManifest };
+export { type ApplyWriteResult, type AuditEvent, type AutoUpdateResult, type BelongsToRelation, type BelongsToSource, type BuiltinTemplateName, type CleanupOptions, type CleanupResult, type CountOptions, type CustomSource, DEFAULT_ENTRY_TYPES, DEFAULT_TYPE_ALIASES, type EmbeddingsConfig, type EnrichedSource, type EnrichmentLookup, type EntityContextDefinition, type EntityContextManifestEntry, type EntityFileManifestInfo, type EntityFileSource, type EntityFileSpec, type EntityProfileField, type EntityProfileSection, type EntityProfileTemplate, type EntityRenderSpec, type EntityRenderTemplate, type EntitySectionPerRow, type EntitySectionsTemplate, type EntityTableColumn, type EntityTableTemplate, type Filter, type FilterOp, type HasManyRelation, type HasManySource, InMemoryStateStore, type InitOptions, Lattice, type LatticeConfig, type LatticeConfigInput, type LatticeEntityDef, type LatticeEntityRenderSpec, type LatticeFieldDef, type LatticeFieldType, type LatticeManifest, type LatticeOptions, type LinkOptions, type ManyToManySource, type MarkdownTableColumn, type Migration, type MultiTableDefinition, type OrderBySpec, type ParseError, type ParseResult, type ParsedConfig, type PkLookup, type PrimaryKey, type QueryOptions, READ_ONLY_HEADER, type ReadOnlyHeaderOptions, type ReconcileOptions, type ReconcileResult, type Relation, type RenderHooks, type RenderResult, type RenderSpec, type ReportConfig, type ReportResult, type ReportSection, type ReportSectionResult, type ReverseSyncError, type ReverseSyncResult, type ReverseSyncUpdate, type RewardScores, type Row, type SearchOptions, type SearchResult, type SecurityOptions, type SeedConfig, type SeedLinkSpec, type SeedResult, type SelfSource, type SessionEntry, type SessionParseOptions, type SessionWriteEntry, type SessionWriteOp, type SessionWriteParseResult, type SourceQueryOptions, type StopFn, type SyncResult, type TableDefinition, type TemplateRenderSpec, type UpsertByNaturalKeyOptions, type WatchOptions, type WriteHook, type WriteHookContext, type WritebackDefinition, type WritebackStateStore, type WritebackValidationResult, applyTokenBudget, applyWriteEntry, autoUpdate, contentHash, createReadOnlyHeader, createSQLiteStateStore, decrypt, deriveKey, encrypt, entityFileNames, estimateTokens, fixSchemaConflicts, frontmatter, generateEntryId, generateWriteEntryId, isEncrypted, isV1EntityFiles, manifestPath, markdownTable, normalizeEntityFiles, parseConfigFile, parseConfigString, parseMarkdownEntries, parseSessionMD, parseSessionWrites, readManifest, slugify, truncate, validateEntryId, writeManifest };