latticesql 2.2.2 → 2.2.4

package/README.md

@@ -2137,8 +2137,9 @@ Optional extras, each enabled by its own key/binary:
 - **Voice** — set an OpenAI (Whisper) or ElevenLabs key to dictate into the composer.
 - **File ingest** — reference a local file or paste text; it becomes a row in the
   native `files` entity with extracted text + (with a Claude key) an
-  LLM-written description and links to related records. PDFs/office docs use the
-  optional [`markitdown`](https://github.com/microsoft/markitdown) CLI when installed.
+  LLM-written description and links to related records. Documents (PDF, Word,
+  PowerPoint, Excel, OpenDocument, EPUB, RTF) are parsed natively in-process —
+  no external CLI.
 
 Chat threads, files, and secrets are all stored as native Lattice entities.
 
@@ -2295,8 +2296,9 @@ the library API is unchanged and fully backwards-compatible.
   entity. A subscription **Connect** link (PKCE) appears when the `ANTHROPIC_OAUTH_*`
   values are set (see [`.env.example`](.env.example)).
 - **Drop files / paste text / images / URLs.** Sources become native `files` rows
-  (referenced, not copied) and are extracted — text via the optional `markitdown`
-  CLI, **images via Claude vision**, a pasted **URL crawled** for readable text —
+  (referenced, not copied) and are extracted — documents (PDF / Office /
+  OpenDocument / EPUB / RTF) parsed **natively in-process**, **images via Claude
+  vision**, a pasted **URL crawled** for readable text —
   then summarized with **Claude Haiku** and classified against your records, and
   **added, enriched, and linked** automatically, **auto-creating the junction table
   when none exists** (and a new object when a source fits nothing). All audited and

package/dist/cli.js

@@ -6279,7 +6279,7 @@ async function checkForUpdate(pkgName, currentVersion) {
 
 // src/gui/server.ts
 import { createServer } from "http";
-import { spawn as spawn3 } from "child_process";
+import { spawn as spawn2 } from "child_process";
 import {
   existsSync as existsSync21,
   mkdirSync as mkdirSync10,
@@ -7584,6 +7584,13 @@ var css = `
     .grants-panel .grants-title { font-weight: 600; margin-bottom: 6px; }
     .grants-panel .grants-row { display: flex; align-items: center; gap: 8px; padding: 3px 0; cursor: pointer; }
     .grants-panel .grants-row input { accent-color: var(--accent); }
+    /* Reconnect-required notice: a cloud opened via an unsupported direct
+       database connection serves no data until reconnected through a server. */
+    .cloud-reconnect {
+      padding: 10px 16px; font-size: 13px; line-height: 1.4;
+      background: rgba(239, 68, 68, 0.12); color: var(--text);
+      border-bottom: 1px solid rgba(239, 68, 68, 0.5);
+    }
 
     /* Inline create-row at the bottom of every table */
     tr.create-row td { background: var(--surface-2); }
@@ -9074,6 +9081,20 @@ var appJs = `
 
     window.addEventListener('hashchange', renderRoute);
 
+    // 2.2.3: a cloud reached via a raw postgres:// connection is refused (it
+    // can't enforce per-user access). The server serves no cloud data; tell the
+    // operator to reconnect through a user-authenticated server.
+    function initCloudReconnectNotice() {
+      fetchJson('/api/dbconfig').then(function (d) {
+        if (!d || !d.cloudReconnectRequired) return;
+        var bar = document.getElementById('cloud-reconnect');
+        if (!bar) return;
+        bar.textContent = 'This cloud is connected with a direct database connection, which is no longer supported. Reconnect through a server (sign in as a user) to access it securely.';
+        bar.hidden = false;
+      }).catch(function () { /* dbconfig unavailable (server mode) \u2014 nothing to show */ });
+    }
+    initCloudReconnectNotice();
+
     // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
     // Sidebar
     // \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
@@ -14198,6 +14219,7 @@ var guiAppHtml = `<!doctype html>
       </svg>
     </button>
   </header>
+  <div class="cloud-reconnect" id="cloud-reconnect" hidden></div>
   <div class="layout">
     <nav class="sidebar">
       <label class="sidebar-advanced toggle" title="Advanced mode \u2014 row/table editor instead of the file workspace">
@@ -19359,7 +19381,10 @@ async function dispatchDbConfigRoute(req, res, ctx) {
         // without a local `__lattice_team_connections` row (which doesn't
         // exist when the team cloud itself is the active database).
         teamId: ctx.teamMembership?.teamId ?? null,
-        myUserId: ctx.teamMembership?.myUserId ?? null
+        myUserId: ctx.teamMembership?.myUserId ?? null,
+        // 2.2.3: a direct postgres:// cloud connection is refused — the SPA
+        // shows a "reconnect through a server" prompt instead of cloud data.
+        cloudReconnectRequired: ctx.cloudReconnectRequired
       });
     });
     return true;
@@ -21533,10 +21558,538 @@ import { tmpdir as tmpdir2 } from "os";
 import { basename as basename10, extname as extname2, resolve as resolve8, join as join20 } from "path";
 
 // src/gui/ai/extract.ts
-import { readFile } from "fs/promises";
+import { readFile as readFile2 } from "fs/promises";
 import { extname, basename as basename7 } from "path";
-import { spawn as spawn2 } from "child_process";
+
+// src/gui/ai/doc-extractors.ts
+import { readFile } from "fs/promises";
 var MAX_TEXT = 2e5;
+var MAX_ENTRY_BYTES = 64 * 1024 * 1024;
+var MAX_TOTAL_BYTES = 256 * 1024 * 1024;
+var PDF_TIMEOUT_MS = 3e4;
+var textDecoder = new TextDecoder("utf-8");
+function decodeUtf8(bytes) {
+  return textDecoder.decode(bytes);
+}
+async function loadOptional(specifier) {
+  try {
+    return await import(specifier);
+  } catch {
+    return null;
+  }
+}
+function nullIfEmpty(s) {
+  const t = s.trim();
+  return t ? t : null;
+}
+function withTimeout(p, ms, label) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      reject(new Error(label));
+    }, ms);
+    timer.unref?.();
+  });
+  return Promise.race([
+    p.finally(() => {
+      clearTimeout(timer);
+    }),
+    timeout
+  ]);
+}
+function decodeXmlEntities(s) {
+  return s.replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&#x([0-9a-fA-F]+);/g, (_, h) => safeCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => safeCodePoint(parseInt(d, 10))).replace(/&amp;/g, "&");
+}
+function safeCodePoint(n) {
+  if (!Number.isFinite(n) || n < 0 || n > 1114111) return "";
+  try {
+    return String.fromCodePoint(n);
+  } catch {
+    return "";
+  }
+}
+function stripTags(s) {
+  let out = "";
+  let i = 0;
+  while (i < s.length) {
+    const lt = s.indexOf("<", i);
+    if (lt < 0) {
+      out += s.slice(i);
+      break;
+    }
+    out += s.slice(i, lt);
+    const gt = s.indexOf(">", lt + 1);
+    if (gt < 0) break;
+    i = gt + 1;
+  }
+  return out;
+}
+function isNameBoundary(code) {
+  return code === 32 || // space
+  code === 9 || // tab
+  code === 10 || // \n
+  code === 13 || // \r
+  code === 62 || // >
+  code === 47 || // /
+  Number.isNaN(code);
+}
+function eachElement(xml, tag, cb) {
+  const open = "<" + tag;
+  const close = "</" + tag + ">";
+  let i = 0;
+  while (i < xml.length) {
+    const s = xml.indexOf(open, i);
+    if (s < 0) break;
+    const ne = s + open.length;
+    if (!isNameBoundary(xml.charCodeAt(ne))) {
+      i = ne;
+      continue;
+    }
+    const gt = xml.indexOf(">", ne);
+    if (gt < 0) break;
+    const selfClose = xml.charCodeAt(gt - 1) === 47;
+    const attrs = xml.slice(ne, selfClose ? gt - 1 : gt);
+    if (selfClose) {
+      cb(attrs, "", s);
+      i = gt + 1;
+      continue;
+    }
+    const e = xml.indexOf(close, gt + 1);
+    if (e < 0) break;
+    cb(attrs, xml.slice(gt + 1, e), s);
+    i = e + close.length;
+  }
+}
+function stripElement(xml, tag) {
+  const open = "<" + tag;
+  const close = "</" + tag + ">";
+  let out = "";
+  let i = 0;
+  while (i < xml.length) {
+    const s = xml.indexOf(open, i);
+    if (s < 0) {
+      out += xml.slice(i);
+      break;
+    }
+    const ne = s + open.length;
+    if (!isNameBoundary(xml.charCodeAt(ne))) {
+      out += xml.slice(i, ne);
+      i = ne;
+      continue;
+    }
+    const gt = xml.indexOf(">", ne);
+    if (gt < 0) {
+      out += xml.slice(i);
+      break;
+    }
+    out += xml.slice(i, s);
+    if (xml.charCodeAt(gt - 1) === 47) {
+      i = gt + 1;
+      continue;
+    }
+    const e = xml.indexOf(close, gt + 1);
+    if (e < 0) break;
+    i = e + close.length;
+  }
+  return out;
+}
+function concatTagText(xml, tag) {
+  let out = "";
+  eachElement(xml, tag, (_, inner) => {
+    out += decodeXmlEntities(stripTags(inner));
+  });
+  return out;
+}
+function firstTagText(xml, tag) {
+  let found = "";
+  let done = false;
+  eachElement(xml, tag, (_, inner) => {
+    if (done) return;
+    found = inner;
+    done = true;
+  });
+  return found;
+}
+function stripHtml(html) {
+  const noScript = stripElement(stripElement(html, "script"), "style");
+  const text = decodeXmlEntities(stripTags(noScript));
+  return text.replace(/[ \t\f\r]+/g, " ").replace(/ *\n */g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+async function unzip(path2) {
+  const fflate = await loadOptional("fflate");
+  if (!fflate || typeof fflate.unzipSync !== "function") return null;
+  try {
+    const buf = await readFile(path2);
+    let total = 0;
+    return fflate.unzipSync(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), {
+      filter: (file) => {
+        const size = file.originalSize || 0;
+        if (size > MAX_ENTRY_BYTES) throw new Error("zip entry exceeds size cap");
+        total += size;
+        if (total > MAX_TOTAL_BYTES) throw new Error("zip total exceeds size cap");
+        return true;
+      }
+    });
+  } catch {
+    return null;
+  }
+}
+async function extractDocx(path2) {
+  const mod = await loadOptional("mammoth");
+  const lib = mod?.default ?? mod;
+  if (!lib || typeof lib.extractRawText !== "function") return null;
+  try {
+    const { value } = await lib.extractRawText({ path: path2 });
+    return nullIfEmpty(value);
+  } catch {
+    return null;
+  }
+}
+async function extractDoc(path2) {
+  const mod = await loadOptional(
+    "word-extractor"
+  );
+  const Ctor = mod && "default" in mod ? mod.default : mod;
+  if (typeof Ctor !== "function") return null;
+  try {
+    const doc = await new Ctor().extract(path2);
+    return nullIfEmpty(doc.getBody());
+  } catch {
+    return null;
+  }
+}
+async function extractPdf(path2) {
+  const unpdf = await loadOptional("unpdf");
+  if (!unpdf || typeof unpdf.getDocumentProxy !== "function") return null;
+  try {
+    const buf = await readFile(path2);
+    const data = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
+    const text = await withTimeout(
+      (async () => {
+        const pdf = await unpdf.getDocumentProxy(data);
+        const out = await unpdf.extractText(pdf, { mergePages: true });
+        return out.text;
+      })(),
+      PDF_TIMEOUT_MS,
+      "pdf extract timeout"
+    );
+    return nullIfEmpty(text);
+  } catch {
+    return null;
+  }
+}
+function partNumber(name) {
+  const m = /(\d+)\.xml$/.exec(name);
+  return m?.[1] ? parseInt(m[1], 10) : 0;
+}
+function slideText(xml) {
+  const paras = [];
+  eachElement(xml, "a:p", (_, inner) => {
+    const runs = concatTagText(inner, "a:t");
+    if (runs.trim()) paras.push(runs);
+  });
+  if (paras.length === 0) {
+    const runs = concatTagText(xml, "a:t");
+    if (runs.trim()) paras.push(runs);
+  }
+  return paras.join("\n");
+}
+async function extractPptx(path2) {
+  const entries = await unzip(path2);
+  if (!entries) return null;
+  const slides = Object.keys(entries).filter((n) => /^ppt\/slides\/slide\d+\.xml$/.test(n)).sort((a, b) => partNumber(a) - partNumber(b));
+  if (slides.length === 0) return null;
+  const parts = [];
+  let total = 0;
+  for (const n of slides) {
+    if (total >= MAX_TEXT) break;
+    const bytes = entries[n];
+    if (!bytes) continue;
+    const text = slideText(decodeUtf8(bytes)).replace(/[ \t]+/g, " ").trim();
+    if (text) {
+      parts.push(text);
+      total += text.length + 2;
+    }
+  }
+  return nullIfEmpty(parts.join("\n\n"));
+}
+async function extractXlsx(path2) {
+  const entries = await unzip(path2);
+  if (!entries) return null;
+  const shared = [];
+  const ssBytes = entries["xl/sharedStrings.xml"];
+  if (ssBytes) {
+    eachElement(decodeUtf8(ssBytes), "si", (_, inner) => {
+      shared.push(concatTagText(stripElement(inner, "rPh"), "t"));
+    });
+  }
+  const sheetNames = Object.keys(entries).filter((n) => /^xl\/worksheets\/sheet\d+\.xml$/.test(n)).sort((a, b) => partNumber(a) - partNumber(b));
+  const rowsOut = [];
+  let total = 0;
+  for (const n of sheetNames) {
+    if (total >= MAX_TEXT) break;
+    const bytes = entries[n];
+    if (!bytes) continue;
+    eachElement(decodeUtf8(bytes), "row", (_, rowInner) => {
+      if (total >= MAX_TEXT) return;
+      const cells = [];
+      eachElement(rowInner, "c", (attrs, body) => {
+        const type = /\bt="([^"]+)"/.exec(attrs)?.[1];
+        let val = "";
+        if (type === "s") {
+          const idx = parseInt(firstTagText(body, "v"), 10);
+          val = Number.isInteger(idx) ? shared[idx] ?? "" : "";
+        } else if (type === "inlineStr") {
+          val = concatTagText(body, "t");
+        } else {
+          val = decodeXmlEntities(firstTagText(body, "v"));
+        }
+        if (val) cells.push(val);
+      });
+      if (cells.length) {
+        const line = cells.join("	");
+        rowsOut.push(line);
+        total += line.length + 1;
+      }
+    });
+  }
+  return nullIfEmpty(rowsOut.join("\n"));
+}
+function odfWhitespace(s) {
+  return s.replace(/<text:tab\b[^>]{0,400}\/?>/g, "	").replace(/<text:line-break\b[^>]{0,400}\/?>/g, "\n").replace(
+    /<text:s\b[^>]{0,400}\btext:c="(\d+)"[^>]{0,400}\/?>/g,
+    (_, c) => " ".repeat(Math.min(parseInt(c, 10) || 1, 100))
+  ).replace(/<text:s\b[^>]{0,400}\/?>/g, " ");
+}
+function odfParagraph(inner) {
+  return decodeXmlEntities(stripTags(odfWhitespace(inner))).trim();
+}
+async function extractOdfText(path2) {
+  const entries = await unzip(path2);
+  if (!entries) return null;
+  const contentBytes = entries["content.xml"];
+  if (!contentBytes) return null;
+  const xml = decodeUtf8(contentBytes);
+  const items = [];
+  const collect = (_, inner, start) => {
+    const line = odfParagraph(inner);
+    if (line) items.push([start, line]);
+  };
+  eachElement(xml, "text:p", collect);
+  eachElement(xml, "text:h", collect);
+  items.sort((a, b) => a[0] - b[0]);
+  const lines = [];
+  let total = 0;
+  for (const [, line] of items) {
+    if (total >= MAX_TEXT) break;
+    lines.push(line);
+    total += line.length + 1;
+  }
+  return nullIfEmpty(lines.join("\n"));
+}
+async function extractOds(path2) {
+  const entries = await unzip(path2);
+  if (!entries) return null;
+  const contentBytes = entries["content.xml"];
+  if (!contentBytes) return null;
+  const xml = decodeUtf8(contentBytes);
+  const rows = [];
+  let total = 0;
+  eachElement(xml, "table:table-row", (_, rowInner) => {
+    if (total >= MAX_TEXT) return;
+    const cells = [];
+    eachElement(rowInner, "table:table-cell", (attrs, body) => {
+      const parts = [];
+      eachElement(body, "text:p", (__, p) => {
+        const t = odfParagraph(p);
+        if (t) parts.push(t);
+      });
+      let val = parts.join(" ").trim();
+      if (!val) {
+        const ov = /\boffice:(?:value|date-value|time-value|string-value|boolean-value)="([^"]*)"/.exec(
+          attrs
+        )?.[1];
+        if (ov) val = decodeXmlEntities(ov);
+      }
+      if (val) cells.push(val);
+    });
+    if (cells.length) {
+      const line = cells.join("	");
+      rows.push(line);
+      total += line.length + 1;
+    }
+  });
+  return nullIfEmpty(rows.join("\n"));
+}
+function normalizeZipPath(p) {
+  const parts = [];
+  for (const seg of p.split("/")) {
+    if (seg === "" || seg === ".") continue;
+    if (seg === "..") parts.pop();
+    else parts.push(seg);
+  }
+  return parts.join("/");
+}
+function resolveHref(baseDir, href) {
+  let h = href.split("#")[0]?.split("?")[0] ?? "";
+  try {
+    h = decodeURIComponent(h);
+  } catch {
+  }
+  return normalizeZipPath(baseDir + h);
+}
+async function extractEpub(path2) {
+  const entries = await unzip(path2);
+  if (!entries) return null;
+  let order = [];
+  const container = entries["META-INF/container.xml"];
+  const opfPath = container ? /full-path="([^"]+)"/.exec(decodeUtf8(container))?.[1] : void 0;
+  if (opfPath && entries[opfPath]) {
+    const opf = decodeUtf8(entries[opfPath]);
+    const manifest = {};
+    eachElement(opf, "item", (attrs) => {
+      const id = /\bid="([^"]+)"/.exec(attrs)?.[1];
+      const href = /\bhref="([^"]+)"/.exec(attrs)?.[1];
+      if (id && href) manifest[id] = href;
+    });
+    const baseDir = opfPath.includes("/") ? opfPath.slice(0, opfPath.lastIndexOf("/") + 1) : "";
+    eachElement(opf, "itemref", (attrs) => {
+      const idref = /\bidref="([^"]+)"/.exec(attrs)?.[1];
+      const href = idref ? manifest[idref] : void 0;
+      if (href) order.push(resolveHref(baseDir, href));
+    });
+  }
+  if (order.length === 0) {
+    order = Object.keys(entries).filter((n) => /\.x?html?$/i.test(n)).sort((a, b) => a.localeCompare(b, void 0, { numeric: true }));
+  }
+  const parts = [];
+  let total = 0;
+  for (const n of order) {
+    if (total >= MAX_TEXT) break;
+    const bytes = entries[n];
+    if (!bytes) continue;
+    const body = stripHtml(decodeUtf8(bytes));
+    if (body) {
+      parts.push(body);
+      total += body.length + 2;
+    }
+  }
+  return nullIfEmpty(parts.join("\n\n"));
+}
+var RTF_IGNORED_DESTINATIONS = /^\\(?:\*|(?:fonttbl|colortbl|stylesheet|info|pict|themedata|colorschememapping|latentstyles|datastore|listtable|listoverridetable|rsidtbl|generator|operator|xmlnstbl|wgrffmtfilter|mmathPr)(?![a-zA-Z]))/;
+function stripRtfDestinations(s) {
+  const kept = [];
+  let keepFrom = 0;
+  let i = 0;
+  while (i < s.length) {
+    if (s[i] === "{" && RTF_IGNORED_DESTINATIONS.test(s.slice(i + 1, i + 40))) {
+      const pre = s.slice(keepFrom, i);
+      kept.push(pre);
+      if (/\\[a-zA-Z]+$/.test(pre.slice(-40))) kept.push(" ");
+      let depth = 1;
+      let j = i + 1;
+      for (; j < s.length && depth > 0; j++) {
+        const ch = s[j];
+        if (ch === "\\")
+          j++;
+        else if (ch === "{") depth++;
+        else if (ch === "}") depth--;
+      }
+      i = j;
+      keepFrom = i;
+    } else {
+      i++;
+    }
+  }
+  kept.push(s.slice(keepFrom));
+  return kept.join("");
+}
+var CP1252_HIGH = {
+  128: 8364,
+  130: 8218,
+  131: 402,
+  132: 8222,
+  133: 8230,
+  134: 8224,
+  135: 8225,
+  136: 710,
+  137: 8240,
+  138: 352,
+  139: 8249,
+  140: 338,
+  142: 381,
+  145: 8216,
+  146: 8217,
+  147: 8220,
+  148: 8221,
+  149: 8226,
+  150: 8211,
+  151: 8212,
+  152: 732,
+  153: 8482,
+  154: 353,
+  155: 8250,
+  156: 339,
+  158: 382,
+  159: 376
+};
+function cp1252Char(byte) {
+  if (byte >= 128 && byte <= 159) {
+    const cp = CP1252_HIGH[byte];
+    return cp ? safeCodePoint(cp) : "";
+  }
+  return safeCodePoint(byte);
+}
+function rtfToText(rtf) {
+  let s = stripRtfDestinations(rtf);
+  s = s.replace(/\\'([0-9a-fA-F]{2})/g, (_, h) => cp1252Char(parseInt(h, 16)));
+  s = s.replace(/\\u(-?\d+)\s?\??/g, (_, d) => {
+    let n = parseInt(d, 10);
+    if (n < 0) n += 65536;
+    return safeCodePoint(n);
+  });
+  s = s.replace(/\\par[d]?\b/g, "\n").replace(/\\line\b/g, "\n").replace(/\\sect\b/g, "\n").replace(/\\page\b/g, "\n").replace(/\\tab\b/g, "	");
+  s = s.replace(/\\[a-zA-Z]+-?\d*\s?/g, "").replace(/\\[^a-zA-Z]/g, "");
+  s = s.replace(/[{}]/g, "");
+  return s.replace(/[ \t]+/g, (m) => m.includes("	") ? "	" : " ").replace(/[ \t]\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+async function extractRtf(path2) {
+  try {
+    const raw = await readFile(path2, "latin1");
+    if (!raw.startsWith("{\\rtf")) return null;
+    return nullIfEmpty(rtfToText(raw));
+  } catch {
+    return null;
+  }
+}
+async function extractDocument(path2, ext) {
+  switch (ext) {
+    case ".docx":
+      return extractDocx(path2);
+    case ".doc":
+      return extractDoc(path2);
+    case ".pdf":
+      return extractPdf(path2);
+    case ".pptx":
+      return extractPptx(path2);
+    case ".xlsx":
+      return extractXlsx(path2);
+    case ".odt":
+    case ".odp":
+      return extractOdfText(path2);
+    case ".ods":
+      return extractOds(path2);
+    case ".epub":
+      return extractEpub(path2);
+    case ".rtf":
+      return extractRtf(path2);
+    default:
+      return null;
+  }
+}
+
+// src/gui/ai/extract.ts
+var MAX_TEXT2 = 2e5;
 var CODE_LANGS = {
   ".ts": "typescript",
   ".tsx": "typescript",
@@ -21584,81 +22137,25 @@ var TEXT_EXT = /* @__PURE__ */ new Set([
   ".htm"
 ]);
 var TEXT_MIME = /^(text\/|application\/(json|xml|xhtml\+xml|x-yaml|yaml|toml))/;
-var MARKITDOWN_EXT = /* @__PURE__ */ new Set([
-  ".pdf",
-  ".docx",
-  ".doc",
-  ".pptx",
-  ".ppt",
-  ".xlsx",
-  ".xls",
-  ".epub",
-  ".rtf",
-  ".odt",
-  ".ods",
-  ".odp"
-]);
-var MARKITDOWN_TIMEOUT_MS = 12e4;
-var MARKITDOWN_MAX_BYTES = 5e7;
-function runMarkitdown(path2) {
-  return new Promise((resolve12) => {
-    const bin = process.env.MARKITDOWN_BIN ?? "markitdown";
-    let child;
-    try {
-      child = spawn2(bin, [path2], { stdio: ["ignore", "pipe", "ignore"] });
-    } catch {
-      resolve12(null);
-      return;
-    }
-    let out = "";
-    let bytes = 0;
-    let settled = false;
-    const finish = (v) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(timer);
-      resolve12(v);
-    };
-    const timer = setTimeout(() => {
-      child.kill();
-      finish(null);
-    }, MARKITDOWN_TIMEOUT_MS);
-    child.stdout.on("data", (c) => {
-      bytes += c.length;
-      if (bytes > MARKITDOWN_MAX_BYTES) {
-        child.kill();
-        finish(null);
-      } else {
-        out += c.toString("utf8");
-      }
-    });
-    child.on("error", () => {
-      finish(null);
-    });
-    child.on("close", (code) => {
-      finish(code === 0 && out.trim() ? out.trim() : null);
-    });
-  });
-}
 function languageOf(name) {
   return CODE_LANGS[extname(name).toLowerCase()] ?? null;
 }
 function truncate(s) {
-  return s.length > MAX_TEXT ? s.slice(0, MAX_TEXT) : s;
+  return s.length > MAX_TEXT2 ? s.slice(0, MAX_TEXT2) : s;
 }
 async function parseFile(path2, mimeHint, originalName) {
   const name = originalName ?? basename7(path2);
   const ext = extname(name).toLowerCase();
   const lang = languageOf(name);
   if (lang) {
-    return { text: truncate(await readFile(path2, "utf8")), language: lang };
+    return { text: truncate(await readFile2(path2, "utf8")), language: lang };
   }
   if (mimeHint && TEXT_MIME.test(mimeHint) || TEXT_EXT.has(ext)) {
-    return { text: truncate(await readFile(path2, "utf8")) };
+    return { text: truncate(await readFile2(path2, "utf8")) };
   }
-  if (MARKITDOWN_EXT.has(ext)) {
-    const md = await runMarkitdown(path2);
-    if (md) return { text: truncate(md) };
+  const doc = await extractDocument(path2, ext);
+  if (doc != null) {
+    return { text: truncate(doc) };
   }
   return { text: "", skip: true };
 }
@@ -21672,7 +22169,7 @@ function describe(text, mime, name) {
 
 // src/ai/vision.ts
 import { createRequire as createRequire5 } from "module";
-import { readFile as readFile2 } from "fs/promises";
+import { readFile as readFile3 } from "fs/promises";
 var DEFAULT_PROMPT = "Describe this image for a knowledge base in 2-4 factual sentences: what it shows, any visible text, and notable details. No preamble.";
 var MAX_DIM = 1568;
 async function describeImage(auth, path2, opts = {}) {
@@ -21688,7 +22185,7 @@ async function describeImage(auth, path2, opts = {}) {
 }
 var DEFAULT_PDF_PROMPT = "Read this document for a knowledge base. First transcribe its readable text, then add a 2-4 sentence factual summary of what it is and its key details. It may be a scanned/image-only PDF \u2014 read the text from the page images. No preamble.";
 async function describePdf(auth, path2, opts = {}) {
-  const buf = await readFile2(path2);
+  const buf = await readFile3(path2);
   const maxBytes = opts.maxBytes ?? 3e7;
   if (buf.length > maxBytes) {
     throw new Error(
@@ -22037,6 +22534,10 @@ function fileSlug(name, id) {
   const base = slugify(name.replace(/\.[^./\\]+$/, "")) || "file";
   return `${base}-${id.slice(0, 8)}`;
 }
+function fileIdentity(displayName, id) {
+  const label = displayName.trim() || "file";
+  return { slug: fileSlug(displayName, id), name: label, title: label };
+}
 var MIME_BY_EXT = {
   ".pdf": "application/pdf",
   ".png": "image/png",
@@ -22105,6 +22606,45 @@ function labelColumn(cols) {
   const text = Object.keys(cols).find((c) => !STRUCTURAL.has(c) && !c.endsWith("_id"));
   return text ?? null;
 }
+var TEXT_COL_RE = /\b(TEXT|VARCHAR|CHAR|CLOB|CHARACTER|STRING|NAME|CITEXT)\b/i;
+async function requiredTextFileColumns(db) {
+  const out = /* @__PURE__ */ new Set();
+  try {
+    if (db.getDialect() === "postgres") {
+      const rows = await allAsyncOrSync(
+        db.adapter,
+        `SELECT column_name AS name, data_type AS type, is_nullable, column_default AS dflt
+           FROM information_schema.columns
+          WHERE table_name = 'files' AND table_schema = current_schema()`
+      );
+      for (const r of rows) {
+        if (String(r.is_nullable).toUpperCase() === "NO" && r.dflt == null && TEXT_COL_RE.test(String(r.type))) {
+          out.add(String(r.name));
+        }
+      }
+    } else {
+      const rows = await allAsyncOrSync(db.adapter, `PRAGMA table_info("files")`);
+      for (const r of rows) {
+        if (Number(r.notnull) === 1 && r.dflt_value == null && Number(r.pk) === 0 && TEXT_COL_RE.test(String(r.type))) {
+          out.add(String(r.name));
+        }
+      }
+    }
+  } catch {
+  }
+  return out;
+}
+async function requiredFileDefaults(db, displayName, id, provided) {
+  const required = await requiredTextFileColumns(db);
+  const label = displayName.trim() || "file";
+  const out = {};
+  for (const col of required) {
+    if (STRUCTURAL.has(col)) continue;
+    if (provided[col] != null) continue;
+    out[col] = /slug/i.test(col) ? fileSlug(displayName, id) : label;
+  }
+  return out;
+}
 async function buildCatalog(db, descriptions) {
   const out = [];
   for (const name of db.getRegisteredTableNames()) {
@@ -22341,7 +22881,8 @@ function looksLikeUrl(s) {
   const t = s.trim();
   return /^https?:\/\/\S+$/i.test(t) && !/\s/.test(t);
 }
-function readBuffer2(req, maxBytes = 5e7) {
+var MAX_INGEST_BYTES = 5e7;
+function readBuffer2(req, maxBytes = MAX_INGEST_BYTES) {
   return new Promise((resolve_, reject) => {
     const chunks = [];
     let size = 0;
@@ -22405,9 +22946,19 @@ async function dispatchIngestRoute(req, res, ctx) {
       await rm(tmp, { force: true }).catch(() => void 0);
     }
     const fileId = crypto.randomUUID();
-    const { id: id2 } = await createRow(mctx, "files", {
+    const rawFilePath = typeof req.headers["x-filepath"] === "string" && req.headers["x-filepath"] || "";
+    let realPath = "";
+    if (rawFilePath) {
+      try {
+        realPath = decodeURIComponent(rawFilePath);
+      } catch {
+        realPath = rawFilePath;
+      }
+    }
+    const uploadRow = {
       id: fileId,
-      slug: fileSlug(name2, fileId),
+      ...fileIdentity(name2, fileId),
+      ...realPath ? { path: realPath } : {},
       original_name: name2,
       mime: mime2,
       size_bytes: buf.length,
@@ -22415,6 +22966,10 @@ async function dispatchIngestRoute(req, res, ctx) {
       description: describe(result.text, mime2, name2),
       extraction_status: result.skip ? "skipped" : "extracted",
       ...blob ? { ref_kind: "blob", blob_path: blob.blob_path, sha256: blob.sha256 } : {}
+    };
+    const { id: id2 } = await createRow(mctx, "files", {
+      ...await requiredFileDefaults(ctx.db, name2, fileId, uploadRow),
+      ...uploadRow
     });
     let suggestedLinks = [];
     if (!result.skip) {
@@ -22459,9 +23014,9 @@ async function dispatchIngestRoute(req, res, ctx) {
       }
     }
     const textFileId = crypto.randomUUID();
-    const { id: id2 } = await createRow(mctx, "files", {
+    const textRow = {
       id: textFileId,
-      slug: fileSlug(title, textFileId),
+      ...fileIdentity(title, textFileId),
       original_name: title,
       mime: mime2,
       size_bytes: Buffer.byteLength(content, "utf8"),
@@ -22469,6 +23024,10 @@ async function dispatchIngestRoute(req, res, ctx) {
       description: describe(content, mime2, title),
       extraction_status: "extracted",
       ...sourceUrl ? { ref_kind: "cloud_ref", ref_uri: sourceUrl, ref_provider: "web" } : {}
+    };
+    const { id: id2 } = await createRow(mctx, "files", {
+      ...await requiredFileDefaults(ctx.db, title, textFileId, textRow),
+      ...textRow
     });
     const suggestedLinks = await enrichOrFail(mctx, ctx.db, id2, content, title, ctx, res);
     if (suggestedLinks === null) return true;
@@ -22493,17 +23052,25 @@ async function dispatchIngestRoute(req, res, ctx) {
     sendJson5(res, { error: `file not found: ${abs}` }, 400);
     return true;
   }
+  if (size > MAX_INGEST_BYTES) {
+    sendJson5(res, { error: "file too large" }, 413);
+    return true;
+  }
   const name = basename10(abs);
   const mime = mimeFor(name);
   const localFileId = crypto.randomUUID();
-  const { id } = await createRow(mctx, "files", {
+  const localRow = {
     id: localFileId,
-    slug: fileSlug(name, localFileId),
+    ...fileIdentity(name, localFileId),
     path: abs,
     original_name: name,
     mime,
     size_bytes: size,
     extraction_status: "pending"
+  };
+  const { id } = await createRow(mctx, "files", {
+    ...await requiredFileDefaults(ctx.db, name, localFileId, localRow),
+    ...localRow
   });
   try {
     const result = await extractSource(ctx.db, abs, mime, name);
@@ -22608,7 +23175,7 @@ function sendText(res, body, status = 200, contentType = "text/plain; charset=ut
 function openUrl(url) {
   const command = process.platform === "darwin" ? "open" : process.platform === "win32" ? "cmd" : "xdg-open";
   const args = process.platform === "win32" ? ["/c", "start", "", url] : [url];
-  const child = spawn3(command, args, { stdio: "ignore", detached: true });
+  const child = spawn2(command, args, { stdio: "ignore", detached: true });
   child.unref();
 }
 function listen(server, port, host) {
@@ -22858,7 +23425,7 @@ function resolveOutputDirForConfig(configPath) {
   }
   return resolve9(base, "context");
 }
-async function openConfig(configPath, outputDir, autoRender = false) {
+async function openConfig(configPath, outputDir, autoRender = false, teamCloud = false) {
   const parsed = parseConfigFile(configPath);
   if (!/^postgres(ql)?:\/\//i.test(parsed.dbPath) && !parsed.dbPath.startsWith("file:") && parsed.dbPath !== ":memory:") {
     mkdirSync10(dirname11(parsed.dbPath), { recursive: true });
@@ -22987,6 +23554,7 @@ async function openConfig(configPath, outputDir, autoRender = false) {
     }
   }
   let teamContext = null;
+  let cloudReconnectRequired = false;
   if (db.getDialect() === "postgres") {
     let teamEnabled = false;
     try {
@@ -22994,7 +23562,14 @@ async function openConfig(configPath, outputDir, autoRender = false) {
     } catch {
       teamEnabled = false;
     }
-    if (!teamEnabled) {
+    const directGuiPostgres = !teamCloud && isPostgresUrl(parsed.dbPath);
+    if (directGuiPostgres) {
+      if (teamEnabled) {
+        cloudReconnectRequired = true;
+        teamEnabled = false;
+        validTables.clear();
+      }
+    } else if (!teamEnabled) {
       try {
         const rawDb = parseDocument3(readFileSync15(configPath, "utf8")).get("db");
         const dbLine = typeof rawDb === "string" ? rawDb.trim() : "";
@@ -23036,7 +23611,7 @@ async function openConfig(configPath, outputDir, autoRender = false) {
     }
   }
   let realtime = null;
-  if (db.getDialect() === "postgres") {
+  if (db.getDialect() === "postgres" && !cloudReconnectRequired) {
     try {
       realtime = new RealtimeBroker(parsed.dbPath);
       await realtime.start();
@@ -23075,6 +23650,8 @@ async function openConfig(configPath, outputDir, autoRender = false) {
     teamsClient,
     validTables,
     teamContext,
+    teamCloud,
+    cloudReconnectRequired,
     junctionTables,
     entityContextByTable,
     manifest,
@@ -23169,7 +23746,7 @@ async function disposeActive(active) {
 async function reopenSameConfig(active, autoRender) {
   const feed = active.feed;
   await disposeActive(active);
-  const next = await openConfig(active.configPath, active.outputDir, autoRender);
+  const next = await openConfig(active.configPath, active.outputDir, autoRender, active.teamCloud);
   next.feed = feed;
   return next;
 }
@@ -23304,7 +23881,7 @@ async function applySchemaConfig(active, entry, direction, autoRender) {
   for (const sql of ddl) await execSql(active.db, sql);
   saveConfigDoc(active.configPath, doc);
   await disposeActive(active);
-  return openConfig(active.configPath, active.outputDir, autoRender);
+  return openConfig(active.configPath, active.outputDir, autoRender, active.teamCloud);
 }
 function schemaReverseSummary(verb, entry) {
   const what = entry.operation.replace("schema.", "").replace(/_/g, " ");
@@ -23318,7 +23895,7 @@ async function startGuiServer(options) {
   const teamCloud = options.teamCloud ?? false;
   const autoRender = options.autoRender ?? false;
   const sessionId = crypto.randomUUID();
-  let active = await openConfig(configPath, outputDir, autoRender);
+  let active = await openConfig(configPath, outputDir, autoRender, teamCloud);
   const latticeRoot = findLatticeRoot(dirname11(configPath));
   let currentWorkspaceId = null;
   if (latticeRoot) {
@@ -24508,7 +25085,7 @@ data: ${JSON.stringify(data)}
           const paths = resolveWorkspacePaths(latticeRoot, ws);
           let next;
           try {
-            next = await openConfig(paths.configPath, paths.contextDir, autoRender);
+            next = await openConfig(paths.configPath, paths.contextDir, autoRender, teamCloud);
           } catch (e) {
             const err = e;
             sendJson(
@@ -24550,7 +25127,12 @@ data: ${JSON.stringify(data)}
           const newPaths = resolveWorkspacePaths(latticeRoot, created);
           let newActive;
           try {
-            newActive = await openConfig(newPaths.configPath, newPaths.contextDir, autoRender);
+            newActive = await openConfig(
+              newPaths.configPath,
+              newPaths.contextDir,
+              autoRender,
+              teamCloud
+            );
           } catch (e) {
             sendJson(
               res,
@@ -24609,7 +25191,12 @@ data: ${JSON.stringify(data)}
             const fbPaths = resolveWorkspacePaths(latticeRoot, fallback);
             let next;
             try {
-              next = await openConfig(fbPaths.configPath, fbPaths.contextDir, autoRender);
+              next = await openConfig(
+                fbPaths.configPath,
+                fbPaths.contextDir,
+                autoRender,
+                teamCloud
+              );
             } catch (e) {
               const err = e;
               const codePrefix = err.code ? `[${err.code}] ` : "";
@@ -24701,7 +25288,12 @@ data: ${JSON.stringify(data)}
           }
           let next;
           try {
-            next = await openConfig(newPath, resolveOutputDirForConfig(newPath), autoRender);
+            next = await openConfig(
+              newPath,
+              resolveOutputDirForConfig(newPath),
+              autoRender,
+              teamCloud
+            );
           } catch (e) {
             const err = e;
             console.error(`[dbconfig.switch] openConfig(${newPath}) failed:`, err);
@@ -24728,7 +25320,8 @@ data: ${JSON.stringify(data)}
           const next = await openConfig(
             newConfigPath,
             resolveOutputDirForConfig(newConfigPath),
-            autoRender
+            autoRender,
+            teamCloud
           );
           await disposeActive(active);
           active = next;
@@ -24770,7 +25363,8 @@ data: ${JSON.stringify(data)}
               next = await openConfig(
                 fallback.path,
                 resolveOutputDirForConfig(fallback.path),
-                autoRender
+                autoRender,
+                teamCloud
               );
             } catch (e) {
               const err = e;
@@ -25170,8 +25764,14 @@ data: ${JSON.stringify(data)}
               teamId: active.teamContext.teamId,
               myUserId: active.teamContext.myUserId
             } : null,
+            cloudReconnectRequired: active.cloudReconnectRequired,
             swap: async () => {
-              const next = await openConfig(active.configPath, active.outputDir, autoRender);
+              const next = await openConfig(
+                active.configPath,
+                active.outputDir,
+                autoRender,
+                active.teamCloud
+              );
               await disposeActive(active);
               active = next;
             }

package/dist/index.d.cts

@@ -4509,8 +4509,8 @@ interface PdfOptions {
 }
 /**
  * Read a PDF with Claude's native document support — works on text PDFs AND
- * scanned/image-only PDFs (no text layer), which `markitdown` cannot extract.
- * AI-gated; the model call is injectable for tests.
+ * scanned/image-only PDFs (no text layer), where in-process text extraction
+ * finds nothing. AI-gated; the model call is injectable for tests.
  */
 declare function describePdf(auth: ClaudeAuth, path: string, opts?: PdfOptions): Promise<string>;
 

package/dist/index.d.ts

@@ -4509,8 +4509,8 @@ interface PdfOptions {
 }
 /**
  * Read a PDF with Claude's native document support — works on text PDFs AND
- * scanned/image-only PDFs (no text layer), which `markitdown` cannot extract.
- * AI-gated; the model call is injectable for tests.
+ * scanned/image-only PDFs (no text layer), where in-process text extraction
+ * finds nothing. AI-gated; the model call is injectable for tests.
  */
 declare function describePdf(auth: ClaudeAuth, path: string, opts?: PdfOptions): Promise<string>;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latticesql",
-  "version": "2.2.2",
+  "version": "2.2.4",
   "description": "Persistent structured memory for AI agent systems — pluggable SQLite or Postgres backend, LLM context bridge",
   "type": "module",
   "main": "./dist/index.js",
@@ -57,10 +57,14 @@
   },
   "optionalDependencies": {
     "@anthropic-ai/sdk": "^0.71.2",
+    "fflate": "^0.8.3",
     "file-type": "^19.6.0",
+    "mammoth": "^1.12.0",
     "pg": "^8.11.0",
     "playwright": "^1.48.0",
-    "sharp": "^0.33.5"
+    "sharp": "^0.33.5",
+    "unpdf": "^1.6.2",
+    "word-extractor": "^1.0.4"
   },
   "devDependencies": {
     "@anthropic-ai/sdk": "^0.71.0",