latticesql 1.16.4 → 2.0.0

package/dist/index.cjs

@@ -58,8 +58,10 @@ __export(index_exports, {
   attachBlob: () => attachBlob,
   autoFtsColumns: () => autoFtsColumns,
   autoUpdate: () => autoUpdate,
+  classifyLinks: () => classifyLinks,
   configDir: () => configDir,
   contentHash: () => contentHash,
+  crawlUrl: () => crawlUrl,
   createReadOnlyHeader: () => createReadOnlyHeader,
   createSQLiteStateStore: () => createSQLiteStateStore,
   decrypt: () => decrypt,
@@ -68,11 +70,15 @@ __export(index_exports, {
   deleteToken: () => deleteToken,
   deriveCanonicalContexts: () => deriveCanonicalContexts,
   deriveKey: () => deriveKey,
+  describeImage: () => describeImage,
+  describePdf: () => describePdf,
   encrypt: () => encrypt,
+  enrichKnowledge: () => enrichKnowledge,
   ensureFtsIndex: () => ensureFtsIndex,
   ensureLatticeRoot: () => ensureLatticeRoot,
   entityFileNames: () => entityFileNames,
   estimateTokens: () => estimateTokens,
+  extractObjects: () => extractObjects,
   findLatticeRoot: () => findLatticeRoot,
   fixSchemaConflicts: () => fixSchemaConflicts,
   frontmatter: () => frontmatter,
@@ -101,9 +107,12 @@ __export(index_exports, {
   migrateLatticeData: () => migrateLatticeData,
   normalizeEntityFiles: () => normalizeEntityFiles,
   openTargetLatticeForMigration: () => openTargetLatticeForMigration,
+  organizeSource: () => organizeSource,
   parseConfigFile: () => parseConfigFile,
   parseConfigString: () => parseConfigString,
   parseMarkdownEntries: () => parseMarkdownEntries,
+  parseMatches: () => parseMatches,
+  parseObjects: () => parseObjects,
   parseSessionMD: () => parseSessionMD,
   parseSessionWrites: () => parseSessionWrites,
   probeCloud: () => probeCloud,
@@ -126,6 +135,7 @@ __export(index_exports, {
   saveDbCredentialForTeam: () => saveDbCredentialForTeam,
   setActiveWorkspace: () => setActiveWorkspace,
   slugify: () => slugify,
+  summarizeText: () => summarizeText,
   toSafeDirName: () => toSafeDirName,
   truncate: () => truncate,
   validateEntryId: () => validateEntryId,
@@ -430,6 +440,9 @@ var PostgresAdapter = class {
       connectionString: this._connectionString,
       max: this._poolSize
     });
+    this._pool.on("error", (err) => {
+      console.error("[latticesql] recovered from idle Postgres client error:", err.message);
+    });
     this._polyfillsReady = this._registerPolyfills();
     this._opened = true;
   }
@@ -964,6 +977,15 @@ var SchemaManager = class {
     }
     this._entityContexts.set(table, def);
   }
+  /**
+   * Register OR replace an entity context — unlike {@link defineEntityContext}
+   * (which throws on a redefine), this overwrites. Used to re-derive a
+   * runtime-created table's canonical context after a schema change (e.g. a new
+   * junction adds a hasMany rollup to the tables it links) without a reopen.
+   */
+  redefineEntityContext(table, def) {
+    this._entityContexts.set(table, def);
+  }
   getTables() {
     return this._tables;
   }
@@ -2039,20 +2061,25 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
 }
 
 // src/render/engine.ts
+var NOOP_RENDER = () => "";
 var RenderEngine = class {
   _schema;
   _adapter;
   _getTaskContext;
-  constructor(schema, adapter, getTaskContext) {
+  /** When true, skip the read + write for spec-less (no-op render) tables. */
+  _skipEmpty;
+  constructor(schema, adapter, getTaskContext, options) {
     this._schema = schema;
     this._adapter = adapter;
     this._getTaskContext = getTaskContext ?? (() => "");
+    this._skipEmpty = options?.skipEmpty ?? false;
   }
   async render(outputDir) {
     const start = Date.now();
     const filesWritten = [];
     const counters = { skipped: 0 };
     for (const [name, def] of this._schema.getTables()) {
+      if (this._skipEmpty && def.render === NOOP_RENDER) continue;
       let rows = await this._schema.queryTable(this._adapter, name);
       if (def.relevanceFilter) {
         const ctx = this._getTaskContext();
@@ -4386,7 +4413,9 @@ var Lattice = class _Lattice {
     this._adapter = options.adapter ?? buildAdapter(dbPath, options);
     this._schema = new SchemaManager();
     this._sanitizer = new Sanitizer(options.security);
-    this._render = new RenderEngine(this._schema, this._adapter, () => this._taskContext);
+    this._render = new RenderEngine(this._schema, this._adapter, () => this._taskContext, {
+      skipEmpty: options.renderSkipsEmpty ?? false
+    });
     this._reverseSync = new ReverseSyncEngine(this._schema, this._adapter);
     this._reverseSeedEngine = new ReverseSeedEngine(this._schema, this._adapter);
     this._loop = new SyncLoop(this._render);
@@ -4507,7 +4536,7 @@ var Lattice = class _Lattice {
         table,
         this._schema,
         this._adapter
-      ) : () => "",
+      ) : NOOP_RENDER,
       outputFile: def.outputFile ?? `.schema-only/${table}.md`,
       ...renderTemplateName ? { _renderTemplateName: renderTemplateName } : {}
     };
@@ -4523,6 +4552,15 @@ var Lattice = class _Lattice {
     this._schema.defineEntityContext(table, def);
     return this;
   }
+  /**
+   * Register or REPLACE an entity context (overwrites instead of throwing on a
+   * redefine — see {@link SchemaManager.redefineEntityContext}). Used to refresh
+   * a canonical context at runtime after a related schema change.
+   */
+  redefineEntityContext(table, def) {
+    this._schema.redefineEntityContext(table, def);
+    return this;
+  }
   /**
    * All entity contexts currently registered on this Lattice — both those
    * declared in `lattice.config.yml` and those added programmatically via
@@ -6548,6 +6586,39 @@ var NATIVE_ENTITY_DEFS = {
     },
     render: () => "",
     outputFile: ".lattice-native/notes.md"
+  },
+  chat_threads: {
+    // An assistant conversation. Native so chat history survives across
+    // sessions and is queryable/renderable like any other Lattice entity.
+    columns: {
+      id: "TEXT PRIMARY KEY",
+      title: "TEXT",
+      created_at: "TEXT NOT NULL DEFAULT (datetime('now'))",
+      updated_at: "TEXT NOT NULL DEFAULT (datetime('now'))",
+      deleted_at: "TEXT"
+    },
+    render: () => "",
+    outputFile: ".lattice-native/chat-threads.md"
+  },
+  chat_messages: {
+    // One turn (or feed entry) within a chat_thread.
+    columns: {
+      id: "TEXT PRIMARY KEY",
+      // Soft reference to chat_threads.id. Kept as a plain column (no FK)
+      // to match the generic, dialect-agnostic native-entity style.
+      thread_id: "TEXT",
+      // user | assistant | tool | feed | system
+      role: "TEXT NOT NULL DEFAULT 'user'",
+      // JSON payload: text, tool_use / tool_result blocks, attachments, or
+      // (for role='feed') the feed-event details.
+      content_json: "TEXT",
+      // ai | gui | cli | ingest — meaningful for role='feed'.
+      source: "TEXT",
+      created_at: "TEXT NOT NULL DEFAULT (datetime('now'))",
+      deleted_at: "TEXT"
+    },
+    render: () => "",
+    outputFile: ".lattice-native/chat-messages.md"
   }
 };
 var NATIVE_ENTITY_NAMES = new Set(Object.keys(NATIVE_ENTITY_DEFS));
@@ -8920,6 +8991,568 @@ function archiveLocalSqlite(dbPath) {
   }
   return backupPath;
 }
+
+// src/ai/llm-client.ts
+var import_node_module2 = require("module");
+var DEFAULT_MODEL = "claude-haiku-4-5";
+
+// src/ai/summarize.ts
+var SUMMARY_SYSTEM = 'You write a one or two sentence factual description of a document for a knowledge base, focused on what it is and what it contains. No preamble, no "This document". Plain text only.';
+async function summarizeText(client, text, name) {
+  const turn = await client.runTurn({
+    model: DEFAULT_MODEL,
+    system: SUMMARY_SYSTEM,
+    messages: [
+      {
+        role: "user",
+        content: `File name: ${name}
+
+Content:
+${text.slice(0, 12e3)}
+
+Describe it in 1-2 sentences.`
+      }
+    ],
+    tools: [],
+    onText: () => void 0
+  });
+  return turn.text.trim();
+}
+var CLASSIFY_SYSTEM = 'You decide which existing records a newly added document relates to. You are given a catalog of record types (with descriptions) and their records. Return ONLY a JSON array of {"table","id"} objects for records the document clearly relates to \u2014 an empty array if none. Output the JSON in a ```json fenced block and nothing else.';
+function buildCatalogBlock(catalog) {
+  return catalog.map((e) => {
+    const head = `## ${e.table}${e.description ? ` \u2014 ${e.description}` : ""}`;
+    const rows = e.records.map((r) => `- id=${r.id} | ${r.label}`).join("\n");
+    return `${head}
+${rows || "- (no records)"}`;
+  }).join("\n\n");
+}
+function parseMatches(raw, catalog) {
+  const fence = /```json\s*([\s\S]*?)```/i.exec(raw);
+  const body = fence ? fence[1] : raw;
+  let parsed;
+  try {
+    parsed = JSON.parse((body ?? "").trim());
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(parsed)) return [];
+  const valid = new Map(catalog.map((e) => [e.table, new Set(e.records.map((r) => r.id))]));
+  const out = [];
+  for (const item of parsed) {
+    if (!item || typeof item !== "object") continue;
+    const table = item.table;
+    const id = item.id;
+    if (typeof table === "string" && typeof id === "string" && valid.get(table)?.has(id)) {
+      out.push({ table, id });
+    }
+  }
+  return out;
+}
+var ID_RE = /^[a-z][a-z0-9_]*$/;
+var RESERVED_COLS = /* @__PURE__ */ new Set(["id", "deleted_at", "created_at", "updated_at"]);
+var EXTRACT_SYSTEM = 'You build a knowledge base by extracting the key structured objects a document is ABOUT \u2014 e.g. an invoice, a person, a project, a contract, a meeting. You are given the existing entity types (tables) and their columns. For each salient object: reuse an existing entity when one clearly fits; otherwise propose a NEW entity with a short snake_case PLURAL name and 2-6 simple snake_case columns. Extract only objects the document is genuinely about \u2014 prefer 1-3, never more than 3, and never invent data not in the document. Return ONLY a JSON array of objects {"entity","isNew","columns","values","label"}, where "values" is an OBJECT mapping each column name to its value \u2014 e.g. {"invoice_number":"INV-114","total":"6400"} \u2014 in a ```json fenced block.';
+function buildSchemaBlock(existing) {
+  if (existing.length === 0) return "(no entities yet \u2014 propose new ones)";
+  return existing.map((e) => `## ${e.table}
+columns: ${e.columns.join(", ")}`).join("\n\n");
+}
+function parseObjects(raw) {
+  const fence = /```json\s*([\s\S]*?)```/i.exec(raw);
+  let parsed;
+  try {
+    parsed = JSON.parse((fence ? fence[1] : raw)?.trim() ?? "");
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(parsed)) return [];
+  const out = [];
+  for (const item of parsed) {
+    if (!item || typeof item !== "object") continue;
+    const o = item;
+    const entity = typeof o.entity === "string" ? o.entity.trim().toLowerCase() : "";
+    const label = typeof o.label === "string" ? o.label.trim() : "";
+    if (!ID_RE.test(entity) || !label) continue;
+    let valuesRaw = {};
+    if (Array.isArray(o.values) && Array.isArray(o.columns)) {
+      o.columns.forEach((c, i) => {
+        valuesRaw[String(c)] = o.values[i];
+      });
+    } else if (o.values && typeof o.values === "object") {
+      valuesRaw = o.values;
+    }
+    const values = {};
+    for (const [k, v] of Object.entries(valuesRaw)) {
+      const col = k.trim().toLowerCase();
+      if (ID_RE.test(col) && !RESERVED_COLS.has(col) && (typeof v === "string" || typeof v === "number")) {
+        values[col] = String(v).slice(0, 2e3);
+      }
+    }
+    if (Object.keys(values).length === 0) continue;
+    const cols = Array.isArray(o.columns) ? o.columns.map((c) => String(c).trim().toLowerCase()).filter((c) => ID_RE.test(c) && !RESERVED_COLS.has(c)) : [];
+    const columns = Array.from(/* @__PURE__ */ new Set([...cols, ...Object.keys(values)])).slice(0, 8);
+    out.push({ entity, isNew: o.isNew === true, columns, values, label });
+    if (out.length >= 3) break;
+  }
+  return out;
+}
+async function extractObjects(client, text, name, existing, temperature) {
+  if (text.trim().length === 0) return [];
+  const turn = await client.runTurn({
+    model: DEFAULT_MODEL,
+    system: EXTRACT_SYSTEM,
+    messages: [
+      {
+        role: "user",
+        content: `# Existing entities
+${buildSchemaBlock(existing)}
+
+# Document: ${name}
+
+${text.slice(0, 12e3)}
+
+# Task
+Return the JSON array of objects to create.`
+      }
+    ],
+    tools: [],
+    ...temperature !== void 0 ? { temperature } : {},
+    onText: () => void 0
+  });
+  return parseObjects(turn.text);
+}
+async function classifyLinks(client, text, name, catalog) {
+  if (catalog.length === 0 || text.trim().length === 0) return [];
+  let captured = "";
+  const turn = await client.runTurn({
+    model: DEFAULT_MODEL,
+    system: CLASSIFY_SYSTEM,
+    messages: [
+      {
+        role: "user",
+        content: `# Catalog
+${buildCatalogBlock(catalog)}
+
+# Document: ${name}
+
+${text.slice(0, 12e3)}
+
+# Task
+Return the JSON array of matching {table,id}.`
+      }
+    ],
+    tools: [],
+    onText: (d) => {
+      captured += d;
+    }
+  });
+  return parseMatches(turn.text || captured, catalog);
+}
+
+// src/ai/organize.ts
+async function organizeSource(db, opts) {
+  const { fileId, text, name, catalog, client } = opts;
+  const linkTable = opts.linkTable ?? "file_links";
+  const fallbackTable = opts.fallbackTable ?? "notes";
+  const createIfNecessary = opts.createIfNecessary ?? true;
+  if (!client) {
+    return { skipped: true, description: "", linked: [], created: [], message: "" };
+  }
+  const linkExisting = opts.linkExisting ?? (async (m) => {
+    await db.insert(linkTable, {
+      file_id: fileId,
+      table_name: m.table,
+      row_id: m.id,
+      relevance: "related"
+    });
+    return true;
+  });
+  const createFallback = opts.createFallback ?? (async (title, body) => {
+    const id = await db.insert(fallbackTable, { title, body });
+    await db.insert(linkTable, {
+      file_id: fileId,
+      table_name: fallbackTable,
+      row_id: id,
+      relevance: "primary"
+    });
+    return { table: fallbackTable, id };
+  });
+  const description = (await summarizeText(client, text, name)).trim();
+  const matches = await classifyLinks(client, text, name, catalog);
+  const linked = [];
+  for (const m of matches) {
+    if (await linkExisting(m)) linked.push({ table: m.table, id: m.id });
+  }
+  const created = [];
+  if (linked.length === 0 && createIfNecessary && text.trim().length > 0) {
+    const title = name.replace(/\.[^./\\]+$/, "").trim() || "Note";
+    const body = description.length > 0 ? description : text.slice(0, 2e3);
+    const result = await createFallback(title, body);
+    if (result) created.push({ table: result.table, id: result.id, title });
+  }
+  return {
+    skipped: false,
+    description,
+    linked,
+    created,
+    message: buildMessage(linked, created)
+  };
+}
+function buildMessage(linked, created) {
+  const parts = [];
+  if (linked.length > 0) {
+    const byTable = /* @__PURE__ */ new Map();
+    for (const l of linked) byTable.set(l.table, (byTable.get(l.table) ?? 0) + 1);
+    const where = [...byTable.entries()].map(([t, n]) => `${String(n)} in ${t}`).join(", ");
+    parts.push(
+      `Linked it to ${String(linked.length)} existing record${linked.length === 1 ? "" : "s"} (${where}).`
+    );
+  }
+  for (const c of created) {
+    parts.push(
+      `Created a new ${singular(c.table)} "${c.title}" because it didn't fit any existing record.`
+    );
+  }
+  if (parts.length === 0) parts.push("Saved it; nothing else needed organizing.");
+  parts.push("You can change any of this anytime.");
+  return parts.join(" ");
+}
+function singular(table) {
+  if (/ies$/i.test(table)) return table.replace(/ies$/i, "y");
+  if (/s$/i.test(table) && !/ss$/i.test(table)) return table.replace(/s$/i, "");
+  return table;
+}
+
+// src/ai/crawl.ts
+var import_jsdom = require("jsdom");
+var import_readability = require("@mozilla/readability");
+var import_node_path18 = require("path");
+var import_node_module3 = require("module");
+var import_meta2 = {};
+var DEFAULT_MAX_BYTES2 = 25 * 1024 * 1024;
+var DEFAULT_TIMEOUT_MS = 3e4;
+var DEFAULT_UA = "LatticeSQL/2.0 (+https://latticesql.com)";
+async function crawlUrl(rawUrl, opts = {}) {
+  const u = await assertSafeUrl(rawUrl, opts.allowPrivate ?? false);
+  const fetchImpl = opts.fetcher ?? fetch;
+  const controller = new AbortController();
+  const timer = setTimeout(() => {
+    controller.abort();
+  }, opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+  let res;
+  try {
+    res = await safeFetch(u.toString(), fetchImpl, {
+      allowPrivate: opts.allowPrivate ?? false,
+      init: {
+        signal: controller.signal,
+        headers: {
+          "user-agent": opts.userAgent ?? DEFAULT_UA,
+          accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        }
+      }
+    });
+  } finally {
+    clearTimeout(timer);
+  }
+  if (!res.ok) {
+    throw new Error(`Lattice: crawl failed for ${rawUrl}: HTTP ${String(res.status)}`);
+  }
+  let mime = (res.headers.get("content-type") ?? "").split(";")[0]?.trim().toLowerCase() ?? "";
+  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES2;
+  const raw = Buffer.from(await res.arrayBuffer());
+  const body = raw.length > maxBytes ? raw.subarray(0, maxBytes) : raw;
+  const finalUrl = res.url || u.toString();
+  if (mime === "" || mime === "application/octet-stream") {
+    mime = await sniffMime(body) || mime;
+  }
+  const isHtml = mime.includes("html") || mime.includes("xml");
+  if (mime && !isHtml && !mime.startsWith("text/")) {
+    return {
+      url: finalUrl,
+      title: titleFromUrl(finalUrl),
+      text: body.toString("utf-8"),
+      excerpt: "",
+      mime,
+      byteLength: raw.length
+    };
+  }
+  const dom = new import_jsdom.JSDOM(body.toString("utf-8"), { url: finalUrl });
+  const doc = dom.window.document;
+  let title = (doc.title || "").trim();
+  let text = "";
+  let excerpt = "";
+  try {
+    const article = new import_readability.Readability(doc).parse();
+    if (article) {
+      text = article.textContent.trim();
+      const articleTitle = article.title.trim();
+      if (articleTitle.length > 0) title = articleTitle;
+      excerpt = article.excerpt.trim();
+    }
+  } catch {
+  }
+  if (text.length === 0) text = strippedBodyText(dom);
+  if (!opts.noJs && text.length < 200) {
+    const rendered = await renderViaPlaywright(finalUrl, opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+    if (rendered) {
+      const rdom = new import_jsdom.JSDOM(rendered, { url: finalUrl });
+      const rdoc = rdom.window.document;
+      try {
+        const a = new import_readability.Readability(rdoc).parse();
+        if (a && a.textContent.trim().length > text.length) {
+          text = a.textContent.trim();
+          if (a.title.trim().length > 0) title = a.title.trim();
+          if (a.excerpt.trim().length > 0) excerpt = a.excerpt.trim();
+        }
+      } catch {
+      }
+      if (text.length === 0) text = strippedBodyText(rdom);
+    }
+  }
+  if (title.length === 0) title = titleFromUrl(finalUrl);
+  return { url: finalUrl, title, text, excerpt, mime: mime || "text/html", byteLength: raw.length };
+}
+function strippedBodyText(dom) {
+  const doc = dom.window.document;
+  for (const el of Array.from(doc.querySelectorAll("script, style, noscript, template"))) {
+    el.remove();
+  }
+  return (doc.body.textContent ?? "").replace(/\s+\n/g, "\n").replace(/[ \t]{2,}/g, " ").trim();
+}
+function titleFromUrl(rawUrl) {
+  try {
+    const u = new URL(rawUrl);
+    const last = (0, import_node_path18.basename)(u.pathname);
+    return last && last !== "/" ? last : u.hostname;
+  } catch {
+    return rawUrl;
+  }
+}
+async function sniffMime(body) {
+  try {
+    const ft = await import("file-type");
+    const result = await ft.fileTypeFromBuffer(body);
+    return result?.mime ?? "";
+  } catch {
+    return "";
+  }
+}
+async function renderViaPlaywright(url, timeoutMs) {
+  let chromium;
+  try {
+    const importMetaUrl = import_meta2.url;
+    const req = importMetaUrl ? (0, import_node_module3.createRequire)(importMetaUrl) : require;
+    const pw = req("playwright");
+    chromium = pw.chromium;
+  } catch {
+    return null;
+  }
+  let browser = null;
+  try {
+    browser = await chromium.launch({ headless: true });
+    const page = await browser.newPage();
+    await page.goto(url, { waitUntil: "networkidle", timeout: timeoutMs });
+    return await page.content();
+  } catch {
+    return null;
+  } finally {
+    if (browser) await browser.close().catch(() => void 0);
+  }
+}
+
+// src/ai/enrich.ts
+var ENRICH_SYSTEM = 'You are writing the body of a knowledge-base entry by synthesizing several source documents into one coherent, factual summary. Integrate concrete facts (dates, names, amounts) and note relationships across sources. Do not invent anything; if the sources are thin, keep it short. Output ONLY the body markdown \u2014 no title, no headings like "Sources", no preamble, no fences.';
+async function enrichKnowledge(db, opts) {
+  const { client } = opts;
+  if (!client) return { skipped: true, enriched: [], examined: 0 };
+  const knowledgeTable = opts.knowledgeTable ?? "notes";
+  const bodyColumn = opts.bodyColumn ?? "body";
+  const linkTable = opts.linkTable ?? "file_links";
+  const sourceTable = opts.sourceTable ?? "files";
+  const sourceTextColumn = opts.sourceTextColumn ?? "extracted_text";
+  const minSources = opts.minSources ?? 2;
+  const thinBodyChars = opts.thinBodyChars ?? 500;
+  const maxObjects = opts.maxObjects ?? 40;
+  const links = await db.query(linkTable);
+  const objects = await db.query(knowledgeTable);
+  const sourceIdsByObject = /* @__PURE__ */ new Map();
+  for (const l of links) {
+    if (String(l.table_name) !== knowledgeTable) continue;
+    const rowId = String(l.row_id);
+    const arr = sourceIdsByObject.get(rowId);
+    if (arr) arr.push(String(l.file_id));
+    else sourceIdsByObject.set(rowId, [String(l.file_id)]);
+  }
+  const enriched = [];
+  let examined = 0;
+  for (const obj of objects) {
+    if (enriched.length >= maxObjects) break;
+    const idVal = obj.id;
+    const id = typeof idVal === "string" ? idVal : "";
+    if (id.length === 0) continue;
+    const sourceIds = sourceIdsByObject.get(id) ?? [];
+    if (sourceIds.length < minSources) continue;
+    const rawBody = obj[bodyColumn];
+    const currentBody = typeof rawBody === "string" ? rawBody : "";
+    if (currentBody.length >= thinBodyChars) continue;
+    examined++;
+    const snippets = [];
+    for (const sid of sourceIds) {
+      const src = await db.get(sourceTable, sid);
+      const rawText = src ? src[sourceTextColumn] : "";
+      if (typeof rawText === "string" && rawText.trim().length > 0) {
+        snippets.push(rawText.slice(0, 4e3));
+      }
+    }
+    if (snippets.length < minSources) continue;
+    const titleVal = obj.title ?? obj.name;
+    const title = typeof titleVal === "string" && titleVal.length > 0 ? titleVal : id;
+    const userBlock = `# Entry: ${title}
+
+Current body:
+${currentBody || "(empty)"}
+
+` + snippets.map((s, i) => `## Source ${String(i + 1)}
+${s}`).join("\n\n") + `
+
+# Task
+Write the improved body.`;
+    let newBody = "";
+    try {
+      const turn = await client.runTurn({
+        model: DEFAULT_MODEL,
+        system: ENRICH_SYSTEM,
+        messages: [{ role: "user", content: userBlock }],
+        tools: [],
+        onText: () => void 0
+      });
+      newBody = turn.text.trim();
+    } catch {
+      continue;
+    }
+    if (isBetter(newBody, currentBody)) {
+      await db.update(knowledgeTable, id, { [bodyColumn]: newBody });
+      enriched.push(id);
+    }
+  }
+  return { skipped: false, enriched, examined };
+}
+function isBetter(next, prev) {
+  if (next.length === 0) return false;
+  if (prev.trim().length < 40 && next.length > 120) return true;
+  return next.length > prev.length + 80;
+}
+
+// src/ai/vision.ts
+var import_node_module4 = require("module");
+var import_promises3 = require("fs/promises");
+var import_meta3 = {};
+var DEFAULT_PROMPT = "Describe this image for a knowledge base in 2-4 factual sentences: what it shows, any visible text, and notable details. No preamble.";
+var MAX_DIM = 1568;
+async function describeImage(auth, path2, opts = {}) {
+  const data = (await normalizeImage(path2, opts.maxBytes ?? 14e5)).toString("base64");
+  const sender = opts.sender ?? defaultSender(auth);
+  const text = await sender({
+    media_type: "image/jpeg",
+    data,
+    prompt: opts.prompt ?? DEFAULT_PROMPT,
+    model: opts.model ?? DEFAULT_MODEL
+  });
+  return text.trim();
+}
+var DEFAULT_PDF_PROMPT = "Read this document for a knowledge base. First transcribe its readable text, then add a 2-4 sentence factual summary of what it is and its key details. It may be a scanned/image-only PDF \u2014 read the text from the page images. No preamble.";
+async function describePdf(auth, path2, opts = {}) {
+  const buf = await (0, import_promises3.readFile)(path2);
+  const maxBytes = opts.maxBytes ?? 3e7;
+  if (buf.length > maxBytes) {
+    throw new Error(
+      `PDF too large for a direct model read (${String(buf.length)} > ${String(maxBytes)} bytes)`
+    );
+  }
+  const sender = opts.sender ?? defaultPdfSender(auth);
+  const text = await sender({
+    data: buf.toString("base64"),
+    prompt: opts.prompt ?? DEFAULT_PDF_PROMPT,
+    model: opts.model ?? DEFAULT_MODEL
+  });
+  return text.trim();
+}
+async function normalizeImage(path2, maxBytes) {
+  const sharpMod = await import("sharp");
+  const sharp = sharpMod.default;
+  let quality = 80;
+  let buf = await renderJpeg(sharp, path2, quality);
+  while (buf.length > maxBytes && quality > 35) {
+    quality -= 15;
+    buf = await renderJpeg(sharp, path2, quality);
+  }
+  return buf;
+}
+function renderJpeg(sharp, path2, quality) {
+  return sharp(path2).rotate().resize({ width: MAX_DIM, height: MAX_DIM, fit: "inside", withoutEnlargement: true }).jpeg({ quality }).toBuffer();
+}
+function defaultSender(auth) {
+  return async (input) => {
+    const importMetaUrl = import_meta3.url;
+    const req = importMetaUrl ? (0, import_node_module4.createRequire)(importMetaUrl) : require;
+    const sdk = req("@anthropic-ai/sdk");
+    const Anthropic = sdk.Anthropic ?? sdk.default;
+    if (!Anthropic) throw new Error("Could not resolve Anthropic from '@anthropic-ai/sdk'");
+    const config = {};
+    if (auth.authToken) config.authToken = auth.authToken;
+    else if (auth.apiKey) config.apiKey = auth.apiKey;
+    if (auth.betaHeader) config.defaultHeaders = { "anthropic-beta": auth.betaHeader };
+    const client = new Anthropic(config);
+    const res = await client.messages.create({
+      model: input.model,
+      max_tokens: 1024,
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "image",
+              source: { type: "base64", media_type: input.media_type, data: input.data }
+            },
+            { type: "text", text: input.prompt }
+          ]
+        }
+      ]
+    });
+    return res.content.filter((b) => b.type === "text").map((b) => b.text ?? "").join("");
+  };
+}
+function defaultPdfSender(auth) {
+  return async (input) => {
+    const importMetaUrl = import_meta3.url;
+    const req = importMetaUrl ? (0, import_node_module4.createRequire)(importMetaUrl) : require;
+    const sdk = req("@anthropic-ai/sdk");
+    const Anthropic = sdk.Anthropic ?? sdk.default;
+    if (!Anthropic) throw new Error("Could not resolve Anthropic from '@anthropic-ai/sdk'");
+    const config = {};
+    if (auth.authToken) config.authToken = auth.authToken;
+    else if (auth.apiKey) config.apiKey = auth.apiKey;
+    if (auth.betaHeader) config.defaultHeaders = { "anthropic-beta": auth.betaHeader };
+    const client = new Anthropic(config);
+    const res = await client.messages.create({
+      model: input.model,
+      max_tokens: 4096,
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "document",
+              source: { type: "base64", media_type: "application/pdf", data: input.data }
+            },
+            { type: "text", text: input.prompt }
+          ]
+        }
+      ]
+    });
+    return res.content.filter((b) => b.type === "text").map((b) => b.text ?? "").join("");
+  };
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   CONFIG_SUBDIR,
@@ -8950,8 +9583,10 @@ function archiveLocalSqlite(dbPath) {
   attachBlob,
   autoFtsColumns,
   autoUpdate,
+  classifyLinks,
   configDir,
   contentHash,
+  crawlUrl,
   createReadOnlyHeader,
   createSQLiteStateStore,
   decrypt,
@@ -8960,11 +9595,15 @@ function archiveLocalSqlite(dbPath) {
   deleteToken,
   deriveCanonicalContexts,
   deriveKey,
+  describeImage,
+  describePdf,
   encrypt,
+  enrichKnowledge,
   ensureFtsIndex,
   ensureLatticeRoot,
   entityFileNames,
   estimateTokens,
+  extractObjects,
   findLatticeRoot,
   fixSchemaConflicts,
   frontmatter,
@@ -8993,9 +9632,12 @@ function archiveLocalSqlite(dbPath) {
   migrateLatticeData,
   normalizeEntityFiles,
   openTargetLatticeForMigration,
+  organizeSource,
   parseConfigFile,
   parseConfigString,
   parseMarkdownEntries,
+  parseMatches,
+  parseObjects,
   parseSessionMD,
   parseSessionWrites,
   probeCloud,
@@ -9018,6 +9660,7 @@ function archiveLocalSqlite(dbPath) {
   saveDbCredentialForTeam,
   setActiveWorkspace,
   slugify,
+  summarizeText,
   toSafeDirName,
   truncate,
   validateEntryId,