latticesql 2.2.3 → 2.3.0

package/dist/cli.js

@@ -21571,10 +21571,14 @@ var textDecoder = new TextDecoder("utf-8");
 function decodeUtf8(bytes) {
   return textDecoder.decode(bytes);
 }
-async function loadOptional(specifier) {
+async function loadParser(specifier) {
   try {
     return await import(specifier);
-  } catch {
+  } catch (err) {
+    console.error(
+      `[latticesql] document parser "${specifier}" failed to load \u2014 document extraction is degraded (likely a broken/partial install or incompatible build). Reinstall dependencies (\`npm install\`). Cause:`,
+      err
+    );
     return null;
   }
 }
@@ -21716,7 +21720,7 @@ function stripHtml(html) {
   return text.replace(/[ \t\f\r]+/g, " ").replace(/ *\n */g, "\n").replace(/\n{3,}/g, "\n\n").trim();
 }
 async function unzip(path2) {
-  const fflate = await loadOptional("fflate");
+  const fflate = await loadParser("fflate");
   if (!fflate || typeof fflate.unzipSync !== "function") return null;
   try {
     const buf = await readFile(path2);
@@ -21735,7 +21739,7 @@ async function unzip(path2) {
   }
 }
 async function extractDocx(path2) {
-  const mod = await loadOptional("mammoth");
+  const mod = await loadParser("mammoth");
   const lib = mod?.default ?? mod;
   if (!lib || typeof lib.extractRawText !== "function") return null;
   try {
@@ -21746,7 +21750,7 @@ async function extractDocx(path2) {
   }
 }
 async function extractDoc(path2) {
-  const mod = await loadOptional(
+  const mod = await loadParser(
     "word-extractor"
   );
   const Ctor = mod && "default" in mod ? mod.default : mod;
@@ -21759,7 +21763,7 @@ async function extractDoc(path2) {
   }
 }
 async function extractPdf(path2) {
-  const unpdf = await loadOptional("unpdf");
+  const unpdf = await loadParser("unpdf");
   if (!unpdf || typeof unpdf.getDocumentProxy !== "function") return null;
   try {
     const buf = await readFile(path2);
@@ -22461,8 +22465,17 @@ function titleFromUrl(rawUrl) {
   }
 }
 async function sniffMime(body) {
+  let ft;
+  try {
+    ft = await import("file-type");
+  } catch (err) {
+    console.error(
+      `[latticesql] mime sniffer "file-type" failed to load \u2014 crawl mime detection is degraded (likely a broken/partial install). Reinstall dependencies (\`npm install\`). Cause:`,
+      err
+    );
+    return "";
+  }
   try {
-    const ft = await import("file-type");
     const result = await ft.fileTypeFromBuffer(body);
     return result?.mime ?? "";
   } catch {
@@ -22606,6 +22619,45 @@ function labelColumn(cols) {
   const text = Object.keys(cols).find((c) => !STRUCTURAL.has(c) && !c.endsWith("_id"));
   return text ?? null;
 }
+var TEXT_COL_RE = /\b(TEXT|VARCHAR|CHAR|CLOB|CHARACTER|STRING|NAME|CITEXT)\b/i;
+async function requiredTextFileColumns(db) {
+  const out = /* @__PURE__ */ new Set();
+  try {
+    if (db.getDialect() === "postgres") {
+      const rows = await allAsyncOrSync(
+        db.adapter,
+        `SELECT column_name AS name, data_type AS type, is_nullable, column_default AS dflt
+           FROM information_schema.columns
+          WHERE table_name = 'files' AND table_schema = current_schema()`
+      );
+      for (const r of rows) {
+        if (String(r.is_nullable).toUpperCase() === "NO" && r.dflt == null && TEXT_COL_RE.test(String(r.type))) {
+          out.add(String(r.name));
+        }
+      }
+    } else {
+      const rows = await allAsyncOrSync(db.adapter, `PRAGMA table_info("files")`);
+      for (const r of rows) {
+        if (Number(r.notnull) === 1 && r.dflt_value == null && Number(r.pk) === 0 && TEXT_COL_RE.test(String(r.type))) {
+          out.add(String(r.name));
+        }
+      }
+    }
+  } catch {
+  }
+  return out;
+}
+async function requiredFileDefaults(db, displayName, id, provided) {
+  const required = await requiredTextFileColumns(db);
+  const label = displayName.trim() || "file";
+  const out = {};
+  for (const col of required) {
+    if (STRUCTURAL.has(col)) continue;
+    if (provided[col] != null) continue;
+    out[col] = /slug/i.test(col) ? fileSlug(displayName, id) : label;
+  }
+  return out;
+}
 async function buildCatalog(db, descriptions) {
   const out = [];
   for (const name of db.getRegisteredTableNames()) {
@@ -22907,9 +22959,19 @@ async function dispatchIngestRoute(req, res, ctx) {
       await rm(tmp, { force: true }).catch(() => void 0);
     }
     const fileId = crypto.randomUUID();
-    const { id: id2 } = await createRow(mctx, "files", {
+    const rawFilePath = typeof req.headers["x-filepath"] === "string" && req.headers["x-filepath"] || "";
+    let realPath = "";
+    if (rawFilePath) {
+      try {
+        realPath = decodeURIComponent(rawFilePath);
+      } catch {
+        realPath = rawFilePath;
+      }
+    }
+    const uploadRow = {
       id: fileId,
       ...fileIdentity(name2, fileId),
+      ...realPath ? { path: realPath } : {},
       original_name: name2,
       mime: mime2,
       size_bytes: buf.length,
@@ -22917,6 +22979,10 @@ async function dispatchIngestRoute(req, res, ctx) {
       description: describe(result.text, mime2, name2),
       extraction_status: result.skip ? "skipped" : "extracted",
       ...blob ? { ref_kind: "blob", blob_path: blob.blob_path, sha256: blob.sha256 } : {}
+    };
+    const { id: id2 } = await createRow(mctx, "files", {
+      ...await requiredFileDefaults(ctx.db, name2, fileId, uploadRow),
+      ...uploadRow
     });
     let suggestedLinks = [];
     if (!result.skip) {
@@ -22961,7 +23027,7 @@ async function dispatchIngestRoute(req, res, ctx) {
       }
     }
     const textFileId = crypto.randomUUID();
-    const { id: id2 } = await createRow(mctx, "files", {
+    const textRow = {
       id: textFileId,
       ...fileIdentity(title, textFileId),
       original_name: title,
@@ -22971,6 +23037,10 @@ async function dispatchIngestRoute(req, res, ctx) {
       description: describe(content, mime2, title),
       extraction_status: "extracted",
       ...sourceUrl ? { ref_kind: "cloud_ref", ref_uri: sourceUrl, ref_provider: "web" } : {}
+    };
+    const { id: id2 } = await createRow(mctx, "files", {
+      ...await requiredFileDefaults(ctx.db, title, textFileId, textRow),
+      ...textRow
     });
     const suggestedLinks = await enrichOrFail(mctx, ctx.db, id2, content, title, ctx, res);
     if (suggestedLinks === null) return true;
@@ -23002,7 +23072,7 @@ async function dispatchIngestRoute(req, res, ctx) {
   const name = basename10(abs);
   const mime = mimeFor(name);
   const localFileId = crypto.randomUUID();
-  const { id } = await createRow(mctx, "files", {
+  const localRow = {
     id: localFileId,
     ...fileIdentity(name, localFileId),
     path: abs,
@@ -23010,6 +23080,10 @@ async function dispatchIngestRoute(req, res, ctx) {
     mime,
     size_bytes: size,
     extraction_status: "pending"
+  };
+  const { id } = await createRow(mctx, "files", {
+    ...await requiredFileDefaults(ctx.db, name, localFileId, localRow),
+    ...localRow
   });
   try {
     const result = await extractSource(ctx.db, abs, mime, name);

package/dist/index.cjs

@@ -9607,8 +9607,17 @@ function titleFromUrl(rawUrl) {
   }
 }
 async function sniffMime(body) {
+  let ft;
+  try {
+    ft = await import("file-type");
+  } catch (err) {
+    console.error(
+      `[latticesql] mime sniffer "file-type" failed to load \u2014 crawl mime detection is degraded (likely a broken/partial install). Reinstall dependencies (\`npm install\`). Cause:`,
+      err
+    );
+    return "";
+  }
   try {
-    const ft = await import("file-type");
     const result = await ft.fileTypeFromBuffer(body);
     return result?.mime ?? "";
   } catch {

package/dist/index.js

@@ -9472,8 +9472,17 @@ function titleFromUrl(rawUrl) {
   }
 }
 async function sniffMime(body) {
+  let ft;
+  try {
+    ft = await import("file-type");
+  } catch (err) {
+    console.error(
+      `[latticesql] mime sniffer "file-type" failed to load \u2014 crawl mime detection is degraded (likely a broken/partial install). Reinstall dependencies (\`npm install\`). Cause:`,
+      err
+    );
+    return "";
+  }
   try {
-    const ft = await import("file-type");
     const result = await ft.fileTypeFromBuffer(body);
     return result?.mime ?? "";
   } catch {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latticesql",
-  "version": "2.2.3",
+  "version": "2.3.0",
   "description": "Persistent structured memory for AI agent systems — pluggable SQLite or Postgres backend, LLM context bridge",
   "type": "module",
   "main": "./dist/index.js",
@@ -43,10 +43,16 @@
     "prepublishOnly": "npm run check:generic && npm run build && npm run typecheck && npm test"
   },
   "dependencies": {
+    "@anthropic-ai/sdk": "^0.71.2",
     "@mozilla/readability": "^0.5.0",
     "@scarf/scarf": "^1.4.0",
+    "fflate": "^0.8.3",
+    "file-type": "^19.6.0",
     "jsdom": "^25.0.1",
+    "mammoth": "^1.12.0",
+    "unpdf": "^1.6.2",
     "uuid": "^13.0.0",
+    "word-extractor": "^1.0.4",
     "yaml": "^2.8.3"
   },
   "scarfSettings": {
@@ -56,18 +62,11 @@
     "better-sqlite3": ">=11 <13"
   },
   "optionalDependencies": {
-    "@anthropic-ai/sdk": "^0.71.2",
-    "fflate": "^0.8.3",
-    "file-type": "^19.6.0",
-    "mammoth": "^1.12.0",
     "pg": "^8.11.0",
     "playwright": "^1.48.0",
-    "sharp": "^0.33.5",
-    "unpdf": "^1.6.2",
-    "word-extractor": "^1.0.4"
+    "sharp": "^0.33.5"
   },
   "devDependencies": {
-    "@anthropic-ai/sdk": "^0.71.0",
     "@eslint/js": "^9.0.0",
     "@playwright/test": "^1.48.0",
     "@types/better-sqlite3": "^7.6.12",