aidimag 1.0.0

package/dist/knowledge/ingest.js

@@ -0,0 +1,305 @@
+/**
+ * Knowledge ingestion pipeline (KNOWLEDGEBASE_DESIGN.md).
+ *
+ * Drop docs into the `knowledge/` inbox → they're classified, (chunked and)
+ * summarized into typed claims → queued as proposals (source `knowledge:<doc>`,
+ * pin-on-approve) → a plain-text summary is written and the original is backed up
+ * to .aidimag/knowledge/processed/ before the inbox copy is removed.
+ * PDF and DOCX files have their text extracted (pdf-parse / mammoth) first.
+ *
+ * Trust gate: claims become PINNED memories only after `dim review` (unless the
+ * repo opts out with knowledge.requireReview = false). Nothing is ever deleted —
+ * unsupported files move to .aidimag/knowledge/skipped/ with a reason.
+ */
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync, statSync, renameSync, copyFileSync, } from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { KNOWLEDGE_EXTRACT_INSTRUCTIONS, buildExtractionUser, parseClaims, dedupeClaims } from "./extract.js";
+import { chunkText } from "./chunk.js";
+import { getTextProvider } from "./llm.js";
+const KB_DIR = path.join(".aidimag", "knowledge");
+// ── paths ───────────────────────────────────────────────────────────────────
+function inboxDir(root, cfg) {
+    return path.join(root, cfg.folder);
+}
+function kbDir(root) { return path.join(root, KB_DIR); }
+function processedDir(root) { return path.join(kbDir(root), "processed"); }
+function skippedDir(root) { return path.join(kbDir(root), "skipped"); }
+function manifestPath(root) { return path.join(kbDir(root), "manifest.json"); }
+function summaryPath(root, file) {
+    return path.join(kbDir(root), `${file}.summary.md`);
+}
+export function readManifest(root) {
+    try {
+        return JSON.parse(readFileSync(manifestPath(root), "utf8"));
+    }
+    catch {
+        return { version: 1, docs: [] };
+    }
+}
+function appendManifest(root, entry) {
+    const m = readManifest(root);
+    m.docs.push(entry);
+    mkdirSync(kbDir(root), { recursive: true });
+    writeFileSync(manifestPath(root), JSON.stringify(m, null, 2) + "\n");
+}
+// ── classification (read-only; no side effects) ───────────────────────────────
+function isBinary(buf) {
+    const n = Math.min(buf.length, 8192);
+    for (let i = 0; i < n; i++)
+        if (buf[i] === 0)
+            return true; // NUL byte → binary
+    try {
+        new TextDecoder("utf-8", { fatal: true }).decode(buf.subarray(0, n));
+        return false;
+    }
+    catch {
+        return true; // not valid UTF-8
+    }
+}
+// ── binary document extraction (.pdf / .docx → plain text) ────────────────────
+const BINARY_DOC_EXTS = new Set([".pdf", ".docx"]);
+/**
+ * Extract plain text from a binary document. Parsers are imported lazily so
+ * repos that never ingest PDFs/DOCX pay no startup cost.
+ */
+async function extractBinaryDocText(buf, ext) {
+    if (ext === ".pdf") {
+        // NOTE: import the lib entry directly — pdf-parse's index.js runs debug
+        // code (reads a test fixture) when loaded outside a CJS parent module.
+        const mod = await import("pdf-parse/lib/pdf-parse.js");
+        const pdfParse = mod.default;
+        const parsed = await pdfParse(buf);
+        return String(parsed.text ?? "");
+    }
+    if (ext === ".docx") {
+        const mammoth = await import("mammoth");
+        const res = await mammoth.extractRawText({ buffer: buf });
+        return res.value ?? "";
+    }
+    throw new Error(`no text extractor for ${ext}`);
+}
+/** Collapse extractor artifacts: >2 blank lines, trailing spaces. */
+function tidyExtractedText(text) {
+    return text.replace(/\r\n/g, "\n").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+export async function classifyInbox(root, cfg) {
+    const dir = inboxDir(root, cfg);
+    const pending = [];
+    const toSkip = [];
+    if (!existsSync(dir))
+        return { pending, toSkip };
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        if (!entry.isFile())
+            continue;
+        if (entry.name.startsWith("."))
+            continue; // .gitkeep and other dotfiles
+        const abs = path.join(dir, entry.name);
+        const ext = path.extname(entry.name).toLowerCase();
+        if (!cfg.extensions.includes(ext)) {
+            toSkip.push({ file: entry.name, reason: `unsupported type (${ext || "no extension"})` });
+            continue;
+        }
+        const size = statSync(abs).size;
+        if (size > cfg.maxBytes) {
+            toSkip.push({ file: entry.name, reason: `exceeds maxBytes (${size} > ${cfg.maxBytes})` });
+            continue;
+        }
+        const buf = readFileSync(abs);
+        let content;
+        if (BINARY_DOC_EXTS.has(ext)) {
+            // PDF/DOCX: extract text before summarization; extraction failures are
+            // skipped (never deleted) with the parser error as the reason.
+            try {
+                content = tidyExtractedText(await extractBinaryDocText(buf, ext));
+            }
+            catch (e) {
+                toSkip.push({ file: entry.name, reason: `text extraction failed (${e instanceof Error ? e.message : String(e)})` });
+                continue;
+            }
+            if (!content) {
+                toSkip.push({ file: entry.name, reason: "no extractable text (scanned/image-only document?)" });
+                continue;
+            }
+        }
+        else {
+            if (isBinary(buf)) {
+                toSkip.push({ file: entry.name, reason: "not text-decodable (looks binary)" });
+                continue;
+            }
+            content = buf.toString("utf8");
+            if (!content.trim()) {
+                toSkip.push({ file: entry.name, reason: "no content" });
+                continue;
+            }
+        }
+        pending.push({
+            file: entry.name,
+            abs,
+            content,
+            bytes: size,
+            hash: createHash("sha256").update(content).digest("hex"),
+        });
+    }
+    return { pending, toSkip };
+}
+function moveToSkipped(root, cfg, file, reason) {
+    const dest = skippedDir(root);
+    mkdirSync(dest, { recursive: true });
+    const src = path.join(inboxDir(root, cfg), file);
+    try {
+        renameSync(src, path.join(dest, file));
+    }
+    catch {
+        // cross-device or already gone — best-effort copy
+        try {
+            copyFileSync(src, path.join(dest, file));
+        }
+        catch { /* ignore */ }
+    }
+    writeFileSync(path.join(dest, `${file}.reason.txt`), `${reason}\nskipped: ${new Date().toISOString()}\n`);
+}
+// ── summarization (LLM path) ──────────────────────────────────────────────────
+async function summarizeWithLlm(provider, doc, cfg) {
+    const chunks = chunkText(doc.content, cfg.chunkBytes);
+    const all = [];
+    for (const chunk of chunks) {
+        const raw = await provider.generate(KNOWLEDGE_EXTRACT_INSTRUCTIONS, buildExtractionUser(doc.file, chunk));
+        all.push(...parseClaims(raw));
+    }
+    return dedupeClaims(all);
+}
+// ── finalize one doc: proposals → summary → backup → remove inbox copy ─────────
+/**
+ * Turn extracted claims into proposals and retire the inbox file safely.
+ * Used by both the LLM path and the MCP agent path (which supplies its own claims).
+ */
+export function finalizeDoc(store, root, cfg, doc, claims, via) {
+    const source = `knowledge:${doc.file}`;
+    const sourceRef = doc.hash.slice(0, 12);
+    const proposalIds = [];
+    for (const c of claims) {
+        const p = store.propose({
+            kind: c.kind,
+            claim: c.claim,
+            paths: c.paths,
+            symbols: c.symbols,
+            guardrailLevel: c.guardrailLevel,
+            source,
+            sourceRef,
+            rationale: c.rationale ?? `Extracted from ${doc.file}`,
+            evidence: [
+                { type: "HUMAN_ATTESTED", payload: `Extracted from knowledge doc ${doc.file} (${via})` },
+            ],
+        });
+        if (p)
+            proposalIds.push(p.id);
+    }
+    // Optional auto-approve (opt-out of the review gate). SAFETY: unlike
+    // human-reviewed approvals, auto-approved claims are NOT pinned — a machine
+    // wrote them and no human looked, so they must stay subject to decay and
+    // evidence checks rather than leading the generated context forever.
+    const memoryIds = [];
+    const pinned = cfg.requireReview === false;
+    if (pinned) {
+        for (const id of proposalIds) {
+            try {
+                memoryIds.push(store.approveProposal(id, { pinned: false }).id);
+            }
+            catch { /* ignore */ }
+        }
+    }
+    // Durable record + backup BEFORE removing the inbox copy (no data loss).
+    mkdirSync(kbDir(root), { recursive: true });
+    writeFileSync(summaryPath(root, doc.file), renderSummary(doc.file, doc.hash, via, claims, pinned));
+    if (cfg.backup) {
+        mkdirSync(processedDir(root), { recursive: true });
+        const src = doc.abs ?? path.join(inboxDir(root, cfg), doc.file);
+        if (existsSync(src))
+            copyFileSync(src, path.join(processedDir(root), doc.file));
+    }
+    // retire the inbox original
+    const inboxFile = doc.abs ?? path.join(inboxDir(root, cfg), doc.file);
+    try {
+        if (existsSync(inboxFile))
+            renameSync(inboxFile, path.join(processedDir(root), doc.file));
+    }
+    catch { /* already backed up */ }
+    appendManifest(root, {
+        file: doc.file, hash: doc.hash, date: new Date().toISOString(),
+        via, claimCount: claims.length, proposalIds, memoryIds,
+    });
+    return { file: doc.file, hash: doc.hash, claimCount: claims.length, proposalIds, memoryIds, pinned };
+}
+function renderSummary(file, hash, via, claims, pinned) {
+    const lines = [];
+    lines.push(`# Knowledge summary: ${file}`);
+    lines.push(`> source: ${file} · sha256: ${hash} · summarized by: ${via} · ${new Date().toISOString()}`);
+    lines.push(pinned
+        ? `> ⚠️ ${claims.length} claim(s) auto-approved (knowledge.requireReview:false) as ACTIVE but UNPINNED memories — no human reviewed them, so they stay subject to decay. Pin the keepers with \`dim pin\`.`
+        : `> ${claims.length} claim(s) queued as proposals — review with \`dim review\`.`);
+    lines.push("");
+    if (!claims.length) {
+        lines.push("_No durable claims were extracted from this document._");
+    }
+    for (const c of claims) {
+        const lvl = c.kind === "GUARDRAIL" && c.guardrailLevel ? ` (${c.guardrailLevel})` : "";
+        lines.push(`- **[${c.kind}${lvl}]** ${c.claim}`);
+        const scope = [...(c.paths ?? []), ...(c.symbols ?? [])];
+        if (scope.length)
+            lines.push(`  - scope: ${scope.join(", ")}`);
+        if (c.rationale)
+            lines.push(`  - why: ${c.rationale}`);
+    }
+    return lines.join("\n") + "\n";
+}
+// ── top-level: process the whole inbox (CLI / hook / watcher) ──────────────────
+export async function ingestAll(store, root, cfg) {
+    const { pending, toSkip } = await classifyInbox(root, cfg);
+    // move unsupported files aside (never deleted)
+    for (const s of toSkip)
+        moveToSkipped(root, cfg, s.file, s.reason);
+    const report = {
+        processed: [], pendingNoSummarizer: [], duplicates: [], skipped: toSkip, summarizer: null,
+    };
+    // resolve summarizer (LLM path); "agent"/"off" never summarize from the CLI
+    let provider = null;
+    if (cfg.summarizer === "auto" || cfg.summarizer === "llm") {
+        provider = await getTextProvider().catch(() => null);
+    }
+    report.summarizer = provider?.name ?? null;
+    const known = new Set(readManifest(root).docs.map((d) => d.hash));
+    for (const doc of pending) {
+        if (known.has(doc.hash)) {
+            // identical content already ingested — just retire the inbox copy
+            mkdirSync(processedDir(root), { recursive: true });
+            try {
+                renameSync(doc.abs, path.join(processedDir(root), doc.file));
+            }
+            catch { /* ignore */ }
+            report.duplicates.push(doc.file);
+            continue;
+        }
+        if (!provider) {
+            report.pendingNoSummarizer.push(doc.file); // leave in inbox for next time
+            continue;
+        }
+        const claims = await summarizeWithLlm(provider, doc, cfg);
+        report.processed.push(finalizeDoc(store, root, cfg, doc, claims, `llm:${provider.name}`));
+    }
+    return report;
+}
+export async function knowledgeStatus(root, cfg) {
+    const { pending, toSkip } = await classifyInbox(root, cfg);
+    const skippedOnDisk = existsSync(skippedDir(root))
+        ? readdirSync(skippedDir(root)).filter((f) => !f.endsWith(".reason.txt"))
+        : [];
+    return {
+        folder: cfg.folder,
+        pending,
+        unsupported: toSkip,
+        skippedOnDisk,
+        processed: readManifest(root).docs,
+    };
+}
+//# sourceMappingURL=ingest.js.map

package/dist/knowledge/llm.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Text-generation provider for knowledge summarization — the CLI/`dim knowledge sync`
+ * "LLM fallback" when no MCP agent is in the loop. Auto-detects the same way the
+ * embedding provider does: OpenAI if OPENAI_API_KEY is set, else a local Ollama, else
+ * none (in which case docs simply wait in the inbox until a provider/agent appears).
+ *
+ *   AIDIMAG_LLM = auto (default) | openai | ollama | off
+ *   AIDIMAG_OPENAI_CHAT_MODEL  (default gpt-4o-mini)
+ *   AIDIMAG_OLLAMA_CHAT_MODEL  (default llama3.1)
+ *   AIDIMAG_OLLAMA_URL         (default http://localhost:11434)
+ */
+export interface TextProvider {
+    readonly name: string;
+    readonly model: string;
+    /** Returns the model's raw text response (expected to be JSON for our prompts). */
+    generate(system: string, user: string): Promise<string>;
+}
+/** Check if running inside an MCP client (e.g., Cursor AI with MCP enabled). */
+export declare function isMcpAvailable(): boolean;
+/** Resolve the text/LLM provider (cached per process). null = none available. */
+export declare function getTextProvider(): Promise<TextProvider | null>;

package/dist/knowledge/llm.js

@@ -0,0 +1,110 @@
+/**
+ * Text-generation provider for knowledge summarization — the CLI/`dim knowledge sync`
+ * "LLM fallback" when no MCP agent is in the loop. Auto-detects the same way the
+ * embedding provider does: OpenAI if OPENAI_API_KEY is set, else a local Ollama, else
+ * none (in which case docs simply wait in the inbox until a provider/agent appears).
+ *
+ *   AIDIMAG_LLM = auto (default) | openai | ollama | off
+ *   AIDIMAG_OPENAI_CHAT_MODEL  (default gpt-4o-mini)
+ *   AIDIMAG_OLLAMA_CHAT_MODEL  (default llama3.1)
+ *   AIDIMAG_OLLAMA_URL         (default http://localhost:11434)
+ */
+const OLLAMA_URL = process.env.AIDIMAG_OLLAMA_URL ?? "http://localhost:11434";
+const OLLAMA_CHAT_MODEL = process.env.AIDIMAG_OLLAMA_CHAT_MODEL ?? "llama3.1";
+const OPENAI_CHAT_MODEL = process.env.AIDIMAG_OPENAI_CHAT_MODEL ?? "gpt-4o-mini";
+class OpenAiTextProvider {
+    name = "openai";
+    model = OPENAI_CHAT_MODEL;
+    async generate(system, user) {
+        const res = await fetch("https://api.openai.com/v1/chat/completions", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+            },
+            body: JSON.stringify({
+                model: this.model,
+                temperature: 0.1,
+                response_format: { type: "json_object" },
+                messages: [
+                    { role: "system", content: system },
+                    { role: "user", content: user },
+                ],
+            }),
+        });
+        if (!res.ok)
+            throw new Error(`OpenAI chat: HTTP ${res.status} ${await res.text()}`);
+        const body = (await res.json());
+        return body.choices[0]?.message?.content ?? "";
+    }
+}
+class OllamaTextProvider {
+    name = "ollama";
+    model = OLLAMA_CHAT_MODEL;
+    async generate(system, user) {
+        const res = await fetch(`${OLLAMA_URL}/api/generate`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                model: this.model,
+                system,
+                prompt: user,
+                format: "json",
+                stream: false,
+                options: { temperature: 0.1 },
+            }),
+        });
+        if (!res.ok)
+            throw new Error(`Ollama generate: HTTP ${res.status}`);
+        const body = (await res.json());
+        return body.response ?? "";
+    }
+    static async detect() {
+        try {
+            const ctl = new AbortController();
+            const t = setTimeout(() => ctl.abort(), 1500);
+            const res = await fetch(`${OLLAMA_URL}/api/tags`, { signal: ctl.signal });
+            clearTimeout(t);
+            if (!res.ok)
+                return null;
+            return new OllamaTextProvider();
+        }
+        catch {
+            return null;
+        }
+    }
+}
+let cached;
+/** Check if running inside an MCP client (e.g., Cursor AI with MCP enabled). */
+export function isMcpAvailable() {
+    // MCP clients communicate via stdio with the MCP server process
+    // When dim is invoked via MCP, stdin/stdout are connected to the client
+    return (typeof process.send === "function" || // IPC channel (some MCP clients)
+        !process.stdin.isTTY || // stdin is piped (MCP stdio transport)
+        process.env.MCP_CLIENT === "true" // explicit env var
+    );
+}
+/** Resolve the text/LLM provider (cached per process). null = none available. */
+export async function getTextProvider() {
+    if (cached !== undefined)
+        return cached;
+    const mode = (process.env.AIDIMAG_LLM ?? "auto").toLowerCase();
+    if (mode === "off")
+        return (cached = null);
+    if (mode === "openai") {
+        if (!process.env.OPENAI_API_KEY)
+            throw new Error("AIDIMAG_LLM=openai but OPENAI_API_KEY is not set");
+        return (cached = new OpenAiTextProvider());
+    }
+    if (mode === "ollama") {
+        const p = await OllamaTextProvider.detect();
+        if (!p)
+            throw new Error(`AIDIMAG_LLM=ollama but Ollama is not reachable at ${OLLAMA_URL}`);
+        return (cached = p);
+    }
+    // auto: try OpenAI first, then Ollama
+    if (process.env.OPENAI_API_KEY)
+        return (cached = new OpenAiTextProvider());
+    return (cached = await OllamaTextProvider.detect());
+}
+//# sourceMappingURL=llm.js.map

package/dist/mcp/server.d.ts

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+/**
+ * aidimag MCP server — exposes repo memory to any MCP-compatible agent
+ * (Claude Code, Cursor, Copilot, ...) over stdio.
+ *
+ * Tools: memory_search, memory_get_for_files, memory_write, memory_refute, memory_status,
+ *        commits_mine, context_note (passive in-chat fact capture), … — searches are logged so zero-hit
+ *        queries surface as coverage gaps (`dim gaps`).
+ * Resource: aidimag://digest — repo memory digest for session bootstrapping.
+ */
+export {};