@jcode.labs/mimir 0.2.1 → 0.4.0

package/dist/ingest.js

@@ -1,20 +1,26 @@
+import { recordAccess } from "./access-log.js";
 import { chunkDocument } from "./chunking.js";
 import { loadConfig } from "./config.js";
 import { embedTexts } from "./embeddings.js";
 import { listSourceFiles } from "./files.js";
 import { parseFile } from "./parsing.js";
+import { redactText, totalRedactions } from "./redaction.js";
 import { openRowsTable, writeRows } from "./store.js";
 const EMBED_BATCH_SIZE = 32;
+const MAX_AUDIT_ROWS = 100_000;
 export async function ingest(options = {}) {
     const config = await loadConfig(String(options.cwd ?? process.cwd()));
     const files = await listSourceFiles(config);
     const allChunks = [];
     const errors = [];
+    const redactionCounts = [];
     let skippedFiles = 0;
     for (const file of files) {
         try {
             const parsed = await parseFile(file);
-            const chunks = chunkDocument(parsed, config.chunkSize, config.chunkOverlap);
+            const redacted = redactText(parsed.text, config);
+            redactionCounts.push(...redacted.counts);
+            const chunks = chunkDocument({ ...parsed, text: redacted.text }, config.chunkSize, config.chunkOverlap);
             if (chunks.length === 0) {
                 skippedFiles += 1;
             }
@@ -40,10 +46,16 @@ export async function ingest(options = {}) {
         }
     }
     await writeRows(rows, config);
+    await recordAccess(config, {
+        action: "ingest",
+        resultCount: rows.length,
+        redactions: totalRedactions(redactionCounts),
+    });
     return {
         indexedFiles: new Set(rows.map((row) => row.relativePath)).size,
         chunks: rows.length,
         skippedFiles,
+        redactions: totalRedactions(redactionCounts),
         errors,
     };
 }
@@ -61,7 +73,7 @@ export async function audit(cwd = process.cwd()) {
             totalChunks: 0,
         };
     }
-    const rows = (await table.query().limit(100000).toArray());
+    const rows = (await table.query().limit(MAX_AUDIT_ROWS).toArray());
     const counts = new Map();
     for (const row of rows) {
         counts.set(row.relativePath, (counts.get(row.relativePath) ?? 0) + 1);

package/dist/ingest.js.map

@@ -1 +1 @@
-{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAGrD,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAE3B,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,UAAyB,EAAE;IACtD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAgB,EAAE,CAAA;IACjC,MAAM,MAAM,GAA2B,EAAE,CAAA;IACzC,IAAI,YAAY,GAAG,CAAC,CAAA;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAA;YACpC,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YAC3E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,YAAY,IAAI,CAAC,CAAA;YACnB,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAI,CAAC,YAAY;gBACvB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAChE,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAgB,EAAE,CAAA;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAA;QACtD,MAAM,UAAU,GAAG,MAAM,UAAU,CACjC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAChC,MAAM,CACP,CAAA;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,CAAA;YAC3F,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;QACjC,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE7B,OAAO;QACL,YAAY,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;QAC/D,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,YAAY;QACZ,MAAM;KACP,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,YAAY,EAAE,EAAE;YAChB,cAAc;YACd,gBAAgB,EAAE,cAAc;YAChC,YAAY,EAAE,EAAE;YAChB,WAAW,EAAE,CAAC;SACf,CAAA;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAoC,CAAA;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAA;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IAEzC,OAAO;QACL,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aAChC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;aACtC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,cAAc;QACd,gBAAgB,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxE,YAAY,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9E,WAAW,EAAE,IAAI,CAAC,MAAM;KACzB,CAAA;AACH,CAAC"}
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAC5D,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAUrD,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,cAAc,GAAG,OAAO,CAAA;AAE9B,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,UAAyB,EAAE;IACtD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAgB,EAAE,CAAA;IACjC,MAAM,MAAM,GAA2B,EAAE,CAAA;IACzC,MAAM,eAAe,GAAqB,EAAE,CAAA;IAC5C,IAAI,YAAY,GAAG,CAAC,CAAA;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAA;YACpC,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAChD,eAAe,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAA;YACxC,MAAM,MAAM,GAAG,aAAa,CAC1B,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAClC,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,YAAY,CACpB,CAAA;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,YAAY,IAAI,CAAC,CAAA;YACnB,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAI,CAAC,YAAY;gBACvB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAChE,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAgB,EAAE,CAAA;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAA;QACtD,MAAM,UAAU,GAAG,MAAM,UAAU,CACjC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAChC,MAAM,CACP,CAAA;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,CAAA;YAC3F,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;QACjC,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7B,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,QAAQ;QAChB,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,UAAU,EAAE,eAAe,CAAC,eAAe,CAAC;KAC7C,CAAC,CAAA;IAEF,OAAO;QACL,YAAY,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;QAC/D,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,YAAY;QACZ,UAAU,EAAE,eAAe,CAAC,eAAe,CAAC;QAC5C,MAAM;KACP,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAC7C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAC3C,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,YAAY,EAAE,EAAE;YAChB,cAAc;YACd,gBAAgB,EAAE,cAAc;YAChC,YAAY,EAAE,EAAE;YAChB,WAAW,EAAE,CAAC;SACf,CAAA;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAE/D,CAAA;IACF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACvE,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAA;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IAEzC,OAAO;QACL,YAAY,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aAChC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;aACtC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,cAAc;QACd,gBAAgB,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxE,YAAY,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;QAC9E,WAAW,EAAE,IAAI,CAAC,MAAM;KACzB,CAAA;AACH,CAAC"}

package/dist/init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAkBA,wBAAsB,WAAW,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAwCxE"}
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAMA,wBAAsB,WAAW,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAwCxE"}

package/dist/init.js

@@ -1,27 +1,16 @@
 import { existsSync } from "node:fs";
 import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
+import { CONFIG_PATH, DEFAULT_CONFIG, KB_DIR, PRIVATE_DIR } from "./defaults.js";
 import { ensureMimirGitignore } from "./gitignore.js";
-const DEFAULT_CONFIG = {
-    rawDir: "private",
-    storageDir: ".kb/storage",
-    sourcesFile: ".kb/sources.txt",
-    tableName: "chunks",
-    ollamaHost: "http://localhost:11434",
-    embedModel: "nomic-embed-text",
-    llmModel: "gemma4:latest",
-    topK: 5,
-    chunkSize: 1200,
-    chunkOverlap: 150,
-};
 export async function initProject(cwd = process.cwd()) {
     const root = path.resolve(cwd);
-    const kbDir = path.join(root, ".kb");
-    const privateDir = path.join(root, "private");
+    const kbDir = path.join(root, KB_DIR);
+    const privateDir = path.join(root, PRIVATE_DIR);
     const created = [];
     await mkdir(kbDir, { recursive: true });
     await mkdir(privateDir, { recursive: true });
-    const configPath = path.join(kbDir, "config.json");
+    const configPath = path.join(root, CONFIG_PATH);
     if (!existsSync(configPath)) {
         await writeFile(configPath, `${JSON.stringify(DEFAULT_CONFIG, null, 2)}\n`, "utf8");
         created.push(path.relative(root, configPath));

package/dist/init.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAErD,MAAM,cAAc,GAAG;IACrB,MAAM,EAAE,SAAS;IACjB,UAAU,EAAE,aAAa;IACzB,WAAW,EAAE,iBAAiB;IAC9B,SAAS,EAAE,QAAQ;IACnB,UAAU,EAAE,wBAAwB;IACpC,UAAU,EAAE,kBAAkB;IAC9B,QAAQ,EAAE,eAAe;IACzB,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,IAAI;IACf,YAAY,EAAE,GAAG;CAClB,CAAA;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;IAC7C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;IAClD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QACnF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;IACnD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,SAAS,CACb,WAAW,EACX,8FAA8F,EAC9F,MAAM,CACP,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;IACrD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CACb,UAAU,EACV,6FAA6F,EAC7F,MAAM,CACP,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,IAAI,MAAM,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC"}
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAChF,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AAErD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;IAC/C,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QACnF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC,CAAA;IACnD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,SAAS,CACb,WAAW,EACX,8FAA8F,EAC9F,MAAM,CACP,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;IACrD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,MAAM,SAAS,CACb,UAAU,EACV,6FAA6F,EAC7F,MAAM,CACP,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;IAC/C,CAAC;IAED,IAAI,MAAM,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,OAAO,CAAA;AAChB,CAAC"}

package/dist/mcp.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AASA,wBAAsB,QAAQ,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAoEjE"}
+{"version":3,"file":"mcp.d.ts","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAeA,wBAAsB,QAAQ,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CA6EjE"}

package/dist/mcp.js

@@ -4,8 +4,13 @@ import { z } from "zod";
 import { loadConfig } from "./config.js";
 import { audit } from "./ingest.js";
 import { ask, search } from "./query.js";
+import { securityAudit } from "./security.js";
 import { countRows } from "./store.js";
 import { VERSION } from "./version.js";
+const queryToolInputSchema = z.object({
+    query: z.string().min(1),
+    topK: z.number().int().positive().optional(),
+});
 export async function serveMcp(cwd = process.cwd()) {
     const server = new McpServer({
         name: "mimir",
@@ -23,8 +28,14 @@ export async function serveMcp(cwd = process.cwd()) {
             rawDir: config.rawDir,
             storageDir: config.storageDir,
             sourcesFile: config.sourcesFile,
-            embedModel: config.embedModel,
-            llmModel: config.llmModel,
+            embeddingProvider: config.embeddingProvider,
+            embeddingModel: config.embeddingModel,
+            embeddingModelPath: config.embeddingModelPath,
+            transformersAllowRemoteModels: config.transformersAllowRemoteModels,
+            llmGeneration: false,
+            redactionEnabled: config.redaction.enabled,
+            mcpMaxTopK: config.mcpMaxTopK,
+            includeExtensions: config.includeExtensions,
             chunksIndexed,
         };
         return textResult(output);
@@ -32,24 +43,23 @@ export async function serveMcp(cwd = process.cwd()) {
     server.registerTool("mimir_search", {
         title: "Mimir Search",
         description: "Retrieve relevant passages from the local Mimir knowledge base.",
-        inputSchema: z.object({
-            query: z.string().min(1),
-            topK: z.number().int().positive().optional(),
-        }),
-    }, async ({ query, topK }) => textResult(await search(query, searchOptions(cwd, topK))));
+        inputSchema: queryToolInputSchema,
+    }, async ({ query, topK }) => textResult(await search(query, await searchOptions(cwd, topK))));
     server.registerTool("mimir_ask", {
         title: "Mimir Ask",
-        description: "Answer a question using local retrieved passages and the configured Ollama model.",
-        inputSchema: z.object({
-            query: z.string().min(1),
-            topK: z.number().int().positive().optional(),
-        }),
-    }, async ({ query, topK }) => textResult(await ask(query, searchOptions(cwd, topK))));
+        description: "Return cited retrieval context for a question without calling an LLM.",
+        inputSchema: queryToolInputSchema,
+    }, async ({ query, topK }) => textResult(await ask(query, await searchOptions(cwd, topK))));
     server.registerTool("mimir_audit", {
         title: "Mimir Audit",
         description: "Compare supported source files on disk with the current vector index.",
         inputSchema: z.object({}),
     }, async () => textResult(await audit(cwd)));
+    server.registerTool("mimir_security_audit", {
+        title: "Mimir Security Audit",
+        description: "Show local privacy, provider, redaction, MCP, and gitignore posture.",
+        inputSchema: z.object({}),
+    }, async () => textResult(await securityAudit(cwd)));
     await server.connect(new StdioServerTransport());
 }
 function textResult(value) {
@@ -62,7 +72,9 @@ function textResult(value) {
         ],
     };
 }
-function searchOptions(cwd, topK) {
-    return topK === undefined ? { cwd } : { cwd, topK };
+async function searchOptions(cwd, topK) {
+    const config = await loadConfig(cwd);
+    const boundedTopK = Math.min(topK ?? config.topK, config.mcpMaxTopK);
+    return { cwd, topK: boundedTopK };
 }
 //# sourceMappingURL=mcp.js.map

package/dist/mcp.js.map

@@ -1 +1 @@
-{"version":3,"file":"mcp.js","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAChF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAChD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO;KACjB,CAAC,CAAA;IAEF,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,0DAA0D;QACvE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE;QACT,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG;YACb,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,aAAa;SACd,CAAA;QAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC,CACF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,iEAAiE;QAC9E,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC7C,CAAC;KACH,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CACrF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,WAAW,EACX;QACE,KAAK,EAAE,WAAW;QAClB,WAAW,EACT,mFAAmF;QACrF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC7C,CAAC;KACH,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAClF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,aAAa,EACb;QACE,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,uEAAuE;QACpF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CACzC,CAAA;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAA;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;aACrC;SACF;KACF,CAAA;AACH,CAAC;AAED,SAAS,aAAa,CAAC,GAAW,EAAE,IAAwB;IAC1D,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAA;AACrD,CAAC"}
+{"version":3,"file":"mcp.js","sourceRoot":"","sources":["../src/mcp.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAA;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAA;AAChF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AACnC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAA;AAEF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IAChD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,OAAO;KACjB,CAAC,CAAA;IAEF,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,0DAA0D;QACvE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE;QACT,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG;YACb,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;YAC7C,6BAA6B,EAAE,MAAM,CAAC,6BAA6B;YACnE,aAAa,EAAE,KAAK;YACpB,gBAAgB,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YAC1C,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,aAAa;SACd,CAAA;QAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC,CACF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,iEAAiE;QAC9E,WAAW,EAAE,oBAAoB;KAClC,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,MAAM,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAC3F,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,WAAW,EACX;QACE,KAAK,EAAE,WAAW;QAClB,WAAW,EAAE,uEAAuE;QACpF,WAAW,EAAE,oBAAoB;KAClC,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CACxF,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,aAAa,EACb;QACE,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,uEAAuE;QACpF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,CACzC,CAAA;IAED,MAAM,CAAC,YAAY,CACjB,sBAAsB,EACtB;QACE,KAAK,EAAE,sBAAsB;QAC7B,WAAW,EAAE,sEAAsE;QACnF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KAC1B,EACD,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAA;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;aACrC;SACF;KACF,CAAA;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAW,EACX,IAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;IACpE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;AACnC,CAAC"}

package/dist/parsing.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parsing.d.ts","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE5D,wBAAsB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC,CA6BzE"}
+{"version":3,"file":"parsing.d.ts","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAI5D,wBAAsB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,cAAc,CAAC,CA8CzE"}

package/dist/parsing.js

@@ -1,13 +1,29 @@
 import { readFile } from "node:fs/promises";
+import { strFromU8, unzipSync } from "fflate";
 import { htmlToText } from "html-to-text";
 import { extractText, getDocumentProxy } from "unpdf";
 import YAML from "yaml";
+const MAX_OFFICE_XML_ENTRY_BYTES = 25_000_000;
 export async function parseFile(file) {
     let text;
     switch (file.extension) {
         case ".pdf":
             text = await parsePdf(file.absolutePath);
             break;
+        case ".docx":
+            text = await parseDocx(file.absolutePath);
+            break;
+        case ".pptx":
+            text = await parsePptx(file.absolutePath);
+            break;
+        case ".xlsx":
+            text = await parseXlsx(file.absolutePath);
+            break;
+        case ".odt":
+        case ".ods":
+        case ".odp":
+            text = await parseOpenDocument(file.absolutePath);
+            break;
         case ".html":
         case ".htm":
             text = htmlToText(await readFile(file.absolutePath, "utf8"), {
@@ -25,11 +41,133 @@ export async function parseFile(file) {
         case ".yml":
             text = YAML.stringify(YAML.parse(await readFile(file.absolutePath, "utf8")));
             break;
+        case ".rtf":
+            text = stripRtf(await readFile(file.absolutePath, "utf8"));
+            break;
         default:
             text = await readFile(file.absolutePath, "utf8");
     }
     return { file, text: normalizeText(text) };
 }
+async function parseDocx(filePath) {
+    const entries = unzipOfficeFile(await readFile(filePath));
+    return xmlEntriesToText(entries, [
+        /^word\/document\.xml$/u,
+        /^word\/header\d*\.xml$/u,
+        /^word\/footer\d*\.xml$/u,
+        /^word\/footnotes\.xml$/u,
+        /^word\/endnotes\.xml$/u,
+        /^word\/comments\.xml$/u,
+    ]);
+}
+async function parsePptx(filePath) {
+    const entries = unzipOfficeFile(await readFile(filePath));
+    return xmlEntriesToText(entries, [
+        /^ppt\/slides\/slide\d+\.xml$/u,
+        /^ppt\/notesSlides\/notesSlide\d+\.xml$/u,
+    ]);
+}
+async function parseXlsx(filePath) {
+    const entries = unzipOfficeFile(await readFile(filePath));
+    const sharedStrings = parseSharedStrings(entries.get("xl/sharedStrings.xml") ?? "");
+    const sheets = [...entries.entries()]
+        .filter(([name]) => /^xl\/worksheets\/sheet\d+\.xml$/u.test(name))
+        .sort(([a], [b]) => a.localeCompare(b));
+    const rows = [];
+    for (const [name, xml] of sheets) {
+        const values = parseSheetValues(xml, sharedStrings);
+        if (values.length > 0) {
+            rows.push(`# ${name}`, values.join("\n"));
+        }
+    }
+    return rows.join("\n\n");
+}
+async function parseOpenDocument(filePath) {
+    const entries = unzipOfficeFile(await readFile(filePath));
+    return xmlEntriesToText(entries, [/^content\.xml$/u, /^meta\.xml$/u]);
+}
+function unzipOfficeFile(buffer) {
+    const unzipped = unzipSync(new Uint8Array(buffer), {
+        filter: (file) => file.originalSize <= MAX_OFFICE_XML_ENTRY_BYTES,
+    });
+    const entries = new Map();
+    for (const [name, content] of Object.entries(unzipped)) {
+        if (name.endsWith(".xml")) {
+            entries.set(name, strFromU8(content));
+        }
+    }
+    return entries;
+}
+function xmlEntriesToText(entries, patterns) {
+    const parts = [];
+    for (const [name, xml] of [...entries.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        if (patterns.some((pattern) => pattern.test(name))) {
+            const text = xmlToText(xml);
+            if (text) {
+                parts.push(text);
+            }
+        }
+    }
+    return parts.join("\n\n");
+}
+function parseSharedStrings(xml) {
+    return [...xml.matchAll(/<si\b[\s\S]*?<\/si>/gu)].map(([item]) => xmlToText(item));
+}
+function parseSheetValues(xml, sharedStrings) {
+    const rows = [];
+    for (const rowMatch of xml.matchAll(/<row\b[\s\S]*?<\/row>/gu)) {
+        const rowXml = rowMatch[0];
+        const values = [...rowXml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/gu)]
+            .map((cellMatch) => {
+            const attrs = cellMatch[1] ?? "";
+            const cellXml = cellMatch[2] ?? "";
+            const inline = firstMatch(cellXml, /<is\b[\s\S]*?<\/is>/u);
+            if (inline) {
+                return xmlToText(inline);
+            }
+            const rawValue = firstMatch(cellXml, /<v>([\s\S]*?)<\/v>/u);
+            if (!rawValue) {
+                return "";
+            }
+            if (/\bt="s"/u.test(attrs)) {
+                return sharedStrings[Number.parseInt(rawValue, 10)] ?? "";
+            }
+            return decodeXmlEntities(rawValue);
+        })
+            .filter(Boolean);
+        if (values.length > 0) {
+            rows.push(values.join("\t"));
+        }
+    }
+    return rows;
+}
+function firstMatch(input, pattern) {
+    const match = input.match(pattern);
+    return match?.[1] ?? match?.[0] ?? "";
+}
+function xmlToText(xml) {
+    return normalizeText(decodeXmlEntities(xml
+        .replace(/<w:tab\/>/gu, " ")
+        .replace(/<w:br\/>/gu, "\n")
+        .replace(/<\/(?:w:p|a:p|text:p|text:h|table:table-row)>/gu, "\n")
+        .replace(/<[^>]+>/gu, " ")
+        .replace(/[ \t]{2,}/gu, " ")));
+}
+function stripRtf(input) {
+    return input
+        .replace(/\\par[d]?/gu, "\n")
+        .replace(/\\'[0-9a-fA-F]{2}/gu, " ")
+        .replace(/\\[a-zA-Z]+-?\d* ?/gu, " ")
+        .replace(/[{}]/gu, " ");
+}
+function decodeXmlEntities(input) {
+    return input
+        .replace(/&lt;/gu, "<")
+        .replace(/&gt;/gu, ">")
+        .replace(/&quot;/gu, '"')
+        .replace(/&apos;/gu, "'")
+        .replace(/&amp;/gu, "&");
+}
 async function parsePdf(filePath) {
     const buffer = await readFile(filePath);
     const pdf = await getDocumentProxy(new Uint8Array(buffer));

package/dist/parsing.js.map

@@ -1 +1 @@
-{"version":3,"file":"parsing.js","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACzC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAA;AACrD,OAAO,IAAI,MAAM,MAAM,CAAA;AAGvB,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,IAAI,IAAY,CAAA;IAEhB,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,KAAK,MAAM;YACT,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACxC,MAAK;QACP,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,UAAU,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE;gBAC3D,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE;oBACT,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;oBAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;iBACpC;aACF,CAAC,CAAA;YACF,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;YACrF,MAAK;QACP,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;YAC5E,MAAK;QACP;YACE,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAA;AAC5C,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACvC,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAA;IAC1D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3D,OAAO,MAAM,CAAC,IAAI,CAAA;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;SAC5B,IAAI,EAAE,CAAA;AACX,CAAC"}
+{"version":3,"file":"parsing.js","sourceRoot":"","sources":["../src/parsing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACzC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAA;AACrD,OAAO,IAAI,MAAM,MAAM,CAAA;AAGvB,MAAM,0BAA0B,GAAG,UAAU,CAAA;AAE7C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAgB;IAC9C,IAAI,IAAY,CAAA;IAEhB,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,KAAK,MAAM;YACT,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACxC,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACzC,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACzC,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACzC,MAAK;QACP,KAAK,MAAM,CAAC;QACZ,KAAK,MAAM,CAAC;QACZ,KAAK,MAAM;YACT,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YACjD,MAAK;QACP,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,UAAU,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE;gBAC3D,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE;oBACT,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE;oBAChD,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;iBACpC;aACF,CAAC,CAAA;YACF,MAAK;QACP,KAAK,OAAO;YACV,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;YACrF,MAAK;QACP,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;YAC5E,MAAK;QACP,KAAK,MAAM;YACT,IAAI,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;YAC1D,MAAK;QACP;YACE,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAA;AAC5C,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzD,OAAO,gBAAgB,CAAC,OAAO,EAAE;QAC/B,wBAAwB;QACxB,yBAAyB;QACzB,yBAAyB;QACzB,yBAAyB;QACzB,wBAAwB;QACxB,wBAAwB;KACzB,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzD,OAAO,gBAAgB,CAAC,OAAO,EAAE;QAC/B,+BAA+B;QAC/B,yCAAyC;KAC1C,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzD,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAA;IACnF,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAClC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACjE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzC,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QACnD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAC1B,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzD,OAAO,gBAAgB,CAAC,OAAO,EAAE,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAA;AACvE,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE;QACjD,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,IAAI,0BAA0B;KAClE,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;IACzC,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvD,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAA;QACvC,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,gBAAgB,CAAC,OAA4B,EAAE,QAAkB;IACxE,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAA;YAC3B,IAAI,IAAI,EAAE,CAAC;gBACT,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAC3B,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,OAAO,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;AACpF,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,aAAuB;IAC5D,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;QAC/D,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;QAC1B,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC;aACjE,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;YACjB,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAChC,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAClC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAA;YAC1D,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,SAAS,CAAC,MAAM,CAAC,CAAA;YAC1B,CAAC;YAED,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAA;YAC3D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,EAAE,CAAA;YACX,CAAC;YAED,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC3D,CAAC;YACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAA;QACpC,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC,CAAA;QAElB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,OAAe;IAChD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAClC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AACvC,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,aAAa,CAClB,iBAAiB,CACf,GAAG;SACA,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;SAC3B,OAAO,CAAC,iDAAiD,EAAE,IAAI,CAAC;SAChE,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAC/B,CACF,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,OAAO,KAAK;SACT,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC;SAC5B,OAAO,CAAC,qBAAqB,EAAE,GAAG,CAAC;SACnC,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC;SACpC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;AAC3B,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;AAC5B,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACvC,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAA;IAC1D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3D,OAAO,MAAM,CAAC,IAAI,CAAA;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK;SACT,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC;SAC5B,IAAI,EAAE,CAAA;AACX,CAAC"}

package/dist/query.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAUxE,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAoBhG;AAED,wBAAsB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,SAAS,CAAC,CAuCxF"}
+{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAUxE,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CA2BhG;AAED,wBAAsB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,SAAS,CAAC,CAsBxF"}

package/dist/query.js

@@ -1,4 +1,4 @@
-import { Ollama } from "ollama";
+import { recordAccess } from "./access-log.js";
 import { loadConfig } from "./config.js";
 import { embedText } from "./embeddings.js";
 import { openRowsTable } from "./store.js";
@@ -13,13 +13,20 @@ export async function search(query, options = {}) {
         .vectorSearch(vector)
         .limit(options.topK ?? config.topK)
         .toArray());
-    return rows.map((row) => ({
+    const results = rows.map((row) => ({
         source: row.source,
         relativePath: row.relativePath,
         chunkIndex: row.chunkIndex,
         text: row.text,
         distance: typeof row._distance === "number" ? row._distance : null,
     }));
+    await recordAccess(config, {
+        action: "search",
+        query,
+        topK: options.topK ?? config.topK,
+        resultCount: results.length,
+    });
+    return results;
 }
 export async function ask(query, options = {}) {
     const config = await loadConfig(String(options.cwd ?? process.cwd()));
@@ -30,27 +37,28 @@ export async function ask(query, options = {}) {
             sources,
         };
     }
-    const context = sources
-        .map((source, index) => `[${index + 1}] ${source.relativePath}#${source.chunkIndex}\n${source.text}`)
-        .join("\n\n---\n\n");
-    const client = new Ollama({ host: config.ollamaHost });
-    const response = await client.chat({
-        model: config.llmModel,
-        messages: [
-            {
-                role: "system",
-                content: "Answer only from the provided context. If the context is insufficient, say what is missing. Cite sources with [1], [2], etc.",
-            },
-            {
-                role: "user",
-                content: `Question:\n${query}\n\nContext:\n${context}`,
-            },
-        ],
-        stream: false,
+    await recordAccess(config, {
+        action: "ask",
+        query,
+        topK: options.topK ?? config.topK,
+        resultCount: sources.length,
     });
     return {
-        answer: response.message.content,
+        answer: retrievalOnlyAnswer(sources),
         sources,
     };
 }
+function retrievalOnlyAnswer(sources) {
+    const snippets = sources
+        .map((source, index) => {
+        const text = source.text.replace(/\s+/gu, " ").trim();
+        return `[${index + 1}] ${source.relativePath}#${source.chunkIndex}: ${text}`;
+    })
+        .join("\n\n");
+    return [
+        "Mimir returns retrieval context only. Use these passages as grounded context for your agent or LLM:",
+        "",
+        snippets,
+    ].join("\n");
+}
 //# sourceMappingURL=query.js.map

package/dist/query.js.map

@@ -1 +1 @@
-{"version":3,"file":"query.js","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAW1C,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;IACrE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAC7C,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK;SACtB,YAAY,CAAC,MAAM,CAAC;SACpB,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;SAClC,OAAO,EAAE,CAAgB,CAAA;IAE5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,QAAQ,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,CAAC,CAAA;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE;IAClE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAE5C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,2EAA2E;YACnF,OAAO;SACR,CAAA;IACH,CAAC;IAED,MAAM,OAAO,GAAG,OAAO;SACpB,GAAG,CACF,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAChB,IAAI,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC,IAAI,EAAE,CAC/E;SACA,IAAI,CAAC,aAAa,CAAC,CAAA;IAEtB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;IACtD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC,QAAQ;QACtB,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,QAAQ;gBACd,OAAO,EACL,8HAA8H;aACjI;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,cAAc,KAAK,iBAAiB,OAAO,EAAE;aACvD;SACF;QACD,MAAM,EAAE,KAAK;KACd,CAAC,CAAA;IAEF,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO;QAChC,OAAO;KACR,CAAA;AACH,CAAC"}
+{"version":3,"file":"query.js","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAW1C,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;IACrE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAC7C,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK;SACtB,YAAY,CAAC,MAAM,CAAC;SACpB,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;SAClC,OAAO,EAAE,CAAgB,CAAA;IAE5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,QAAQ,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,CAAC,CAAA;IACH,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,QAAQ;QAChB,KAAK;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI;QACjC,WAAW,EAAE,OAAO,CAAC,MAAM;KAC5B,CAAC,CAAA;IACF,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,KAAa,EAAE,UAAyB,EAAE;IAClE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAE5C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,2EAA2E;YACnF,OAAO;SACR,CAAA;IACH,CAAC;IAED,MAAM,YAAY,CAAC,MAAM,EAAE;QACzB,MAAM,EAAE,KAAK;QACb,KAAK;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI;QACjC,WAAW,EAAE,OAAO,CAAC,MAAM;KAC5B,CAAC,CAAA;IAEF,OAAO;QACL,MAAM,EAAE,mBAAmB,CAAC,OAAO,CAAC;QACpC,OAAO;KACR,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAuB;IAClD,MAAM,QAAQ,GAAG,OAAO;SACrB,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QACrD,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,UAAU,KAAK,IAAI,EAAE,CAAA;IAC9E,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAA;IAEf,OAAO;QACL,qGAAqG;QACrG,EAAE;QACF,QAAQ;KACT,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACd,CAAC"}

package/dist/redaction.d.ts

@@ -0,0 +1,7 @@
+import type { Config, RedactionCount } from "./types.js";
+export declare function redactText(input: string, config: Config): {
+    text: string;
+    counts: RedactionCount[];
+};
+export declare function totalRedactions(counts: RedactionCount[]): number;
+//# sourceMappingURL=redaction.d.ts.map

package/dist/redaction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redaction.d.ts","sourceRoot":"","sources":["../src/redaction.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAoB,MAAM,YAAY,CAAA;AAoC1E,wBAAgB,UAAU,CACxB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,CAyB5C;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAEhE"}

package/dist/redaction.js

@@ -0,0 +1,63 @@
+const BUILT_IN_PATTERNS = [
+    {
+        name: "private_key",
+        pattern: "-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----",
+        flags: "g",
+    },
+    {
+        name: "jwt",
+        pattern: "\\beyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\b",
+        flags: "g",
+    },
+    {
+        name: "api_token",
+        pattern: "\\b(?:sk|pk|ghp|gho|github_pat|npm)_[A-Za-z0-9_=-]{20,}\\b|\\b[A-Za-z0-9_-]{32,}\\.[A-Za-z0-9_-]{16,}\\.[A-Za-z0-9_-]{16,}\\b",
+        flags: "g",
+    },
+    {
+        name: "email",
+        pattern: "\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}\\b",
+        flags: "gi",
+    },
+    {
+        name: "iban",
+        pattern: "\\b[A-Z]{2}\\d{2}[A-Z0-9]{11,30}\\b",
+        flags: "g",
+    },
+    {
+        name: "credit_card",
+        pattern: "\\b(?:\\d[ -]*?){13,19}\\b",
+        flags: "g",
+    },
+];
+export function redactText(input, config) {
+    if (!config.redaction.enabled) {
+        return { text: input, counts: [] };
+    }
+    let text = input;
+    const counts = [];
+    const patterns = [
+        ...(config.redaction.builtIn ? BUILT_IN_PATTERNS : []),
+        ...config.redaction.patterns,
+    ];
+    for (const pattern of patterns) {
+        const regexp = compilePattern(pattern);
+        let count = 0;
+        text = text.replace(regexp, () => {
+            count += 1;
+            return pattern.replacement ?? `[REDACTED_${pattern.name.toUpperCase()}]`;
+        });
+        if (count > 0) {
+            counts.push({ name: pattern.name, count });
+        }
+    }
+    return { text, counts };
+}
+export function totalRedactions(counts) {
+    return counts.reduce((total, entry) => total + entry.count, 0);
+}
+function compilePattern(pattern) {
+    const flags = pattern.flags?.includes("g") ? pattern.flags : `${pattern.flags ?? ""}g`;
+    return new RegExp(pattern.pattern, flags);
+}
+//# sourceMappingURL=redaction.js.map

package/dist/redaction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redaction.js","sourceRoot":"","sources":["../src/redaction.ts"],"names":[],"mappings":"AAEA,MAAM,iBAAiB,GAAuB;IAC5C;QACE,IAAI,EAAE,aAAa;QACnB,OAAO,EAAE,8EAA8E;QACvF,KAAK,EAAE,GAAG;KACX;IACD;QACE,IAAI,EAAE,KAAK;QACX,OAAO,EAAE,2DAA2D;QACpE,KAAK,EAAE,GAAG;KACX;IACD;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EACL,+HAA+H;QACjI,KAAK,EAAE,GAAG;KACX;IACD;QACE,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,8CAA8C;QACvD,KAAK,EAAE,IAAI;KACZ;IACD;QACE,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qCAAqC;QAC9C,KAAK,EAAE,GAAG;KACX;IACD;QACE,IAAI,EAAE,aAAa;QACnB,OAAO,EAAE,4BAA4B;QACrC,KAAK,EAAE,GAAG;KACX;CACF,CAAA;AAED,MAAM,UAAU,UAAU,CACxB,KAAa,EACb,MAAc;IAEd,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAA;IACpC,CAAC;IAED,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,MAAM,MAAM,GAAqB,EAAE,CAAA;IACnC,MAAM,QAAQ,GAAG;QACf,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,GAAG,MAAM,CAAC,SAAS,CAAC,QAAQ;KAC7B,CAAA;IAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;QACtC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE;YAC/B,KAAK,IAAI,CAAC,CAAA;YACV,OAAO,OAAO,CAAC,WAAW,IAAI,aAAa,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAA;QAC1E,CAAC,CAAC,CAAA;QACF,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QAC5C,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;AACzB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAAwB;IACtD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,cAAc,CAAC,OAAyB;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,GAAG,CAAA;IACtF,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;AAC3C,CAAC"}

package/dist/security.d.ts

@@ -0,0 +1,3 @@
+import type { SecurityAuditReport } from "./types.js";
+export declare function securityAudit(cwd?: string): Promise<SecurityAuditReport>;
+//# sourceMappingURL=security.d.ts.map

package/dist/security.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"security.d.ts","sourceRoot":"","sources":["../src/security.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAErD,wBAAsB,aAAa,CAAC,GAAG,SAAgB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAqErF"}