gnosys 4.0.0

package/dist/lib/import.js

@@ -0,0 +1,365 @@
+/**
+ * Gnosys Bulk Import — Structured data → atomic memories.
+ * Supports CSV, JSON, JSONL formats with LLM or structured ingestion modes.
+ * Shared core used by both the MCP tool and CLI command.
+ */
+import { parse as csvParse } from "csv-parse/sync";
+import fs from "fs/promises";
+// ─── URL Safety ──────────────────────────────────────────────────────────
+function isSafeImportUrl(urlStr) {
+    try {
+        const url = new URL(urlStr);
+        if (url.protocol !== "http:" && url.protocol !== "https:")
+            return false;
+        const hostname = url.hostname;
+        if (hostname === "169.254.169.254" || hostname === "metadata.google.internal")
+            return false;
+        const ipv4Match = hostname.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/);
+        if (ipv4Match) {
+            const [, a, b] = ipv4Match.map(Number);
+            if (a === 10)
+                return false;
+            if (a === 172 && b >= 16 && b <= 31)
+                return false;
+            if (a === 192 && b === 168)
+                return false;
+            if (a === 169 && b === 254)
+                return false;
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── Parsing ─────────────────────────────────────────────────────────────
+async function loadData(data, format) {
+    let raw;
+    // Determine if data is a file path, URL, or inline
+    if (data.startsWith("http://") || data.startsWith("https://")) {
+        if (!isSafeImportUrl(data)) {
+            throw new Error(`Refusing to fetch unsafe URL: ${data}`);
+        }
+        const response = await fetch(data);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`);
+        }
+        raw = await response.text();
+    }
+    else if (data.trim().startsWith("[") ||
+        data.trim().startsWith("{") ||
+        data.includes(",") && data.includes("\n") && format === "csv") {
+        // Inline data — use as-is
+        raw = data;
+    }
+    else {
+        // File path
+        raw = await fs.readFile(data, "utf-8");
+    }
+    switch (format) {
+        case "csv":
+            return csvParse(raw, {
+                columns: true,
+                skip_empty_lines: true,
+                trim: true,
+            });
+        case "json": {
+            const parsed = JSON.parse(raw);
+            // Handle both arrays and objects with a data/foods/results array
+            if (Array.isArray(parsed))
+                return parsed;
+            // Common API response patterns
+            for (const key of ["data", "foods", "results", "items", "records", "vulnerabilities"]) {
+                if (Array.isArray(parsed[key]))
+                    return parsed[key];
+            }
+            throw new Error("JSON must be an array or an object with a recognizable array field (data, foods, results, items, records, vulnerabilities)");
+        }
+        case "jsonl":
+            return raw
+                .split("\n")
+                .filter((line) => line.trim())
+                .map((line) => JSON.parse(line));
+        default:
+            throw new Error(`Unsupported format: ${format}`);
+    }
+}
+function validateMapping(mapping) {
+    const errors = [];
+    const validTargets = ["title", "category", "content", "tags", "relevance"];
+    const hasTitle = Object.values(mapping).includes("title");
+    if (!hasTitle) {
+        errors.push("Mapping must include a field mapped to 'title'");
+    }
+    for (const [src, dst] of Object.entries(mapping)) {
+        if (!validTargets.includes(dst)) {
+            errors.push(`Invalid mapping target "${dst}" for field "${src}". Valid: ${validTargets.join(", ")}`);
+        }
+    }
+    return errors;
+}
+function applyMapping(record, mapping) {
+    const mapped = {};
+    const mappedSourceFields = new Set(Object.keys(mapping));
+    const extraParts = [];
+    for (const [src, dst] of Object.entries(mapping)) {
+        const value = record[src];
+        if (value === undefined || value === null)
+            continue;
+        switch (dst) {
+            case "title":
+                mapped.title = String(value);
+                break;
+            case "category":
+                mapped.category = String(value)
+                    .toLowerCase()
+                    .replace(/[^a-z0-9]+/g, "-")
+                    .replace(/^-|-$/g, "");
+                break;
+            case "content":
+                mapped.content = String(value);
+                break;
+            case "relevance":
+                mapped.relevance = String(value);
+                break;
+            case "tags":
+                // Accept comma-separated string or array
+                if (Array.isArray(value)) {
+                    mapped.tags = { domain: value.map(String) };
+                }
+                else {
+                    mapped.tags = {
+                        domain: String(value)
+                            .split(",")
+                            .map((t) => t.trim())
+                            .filter(Boolean),
+                    };
+                }
+                break;
+        }
+    }
+    // Collect unmapped fields as extra context for LLM
+    for (const [key, value] of Object.entries(record)) {
+        if (!mappedSourceFields.has(key) && value !== undefined && value !== null) {
+            const strVal = typeof value === "object" ? JSON.stringify(value) : String(value);
+            if (strVal.length > 0 && strVal !== "null") {
+                extraParts.push(`${key}: ${strVal}`);
+            }
+        }
+    }
+    return {
+        title: mapped.title || "Untitled",
+        category: mapped.category || "imported",
+        content: mapped.content || "",
+        tags: mapped.tags,
+        relevance: mapped.relevance,
+        extraContext: extraParts.join("\n"),
+    };
+}
+// ─── Core Import ─────────────────────────────────────────────────────────
+export async function performImport(store, ingestion, options) {
+    const startTime = Date.now();
+    const results = {
+        imported: [],
+        skipped: [],
+        failed: [],
+        totalProcessed: 0,
+        duration: 0,
+    };
+    const batchCommit = options.batchCommit !== false; // Default true
+    const concurrency = options.concurrency || 5;
+    const author = options.author || "ai";
+    const authority = options.authority || "imported";
+    // Phase 1: Parse data
+    options.onProgress?.({
+        processed: 0,
+        total: 0,
+        current: "Loading data...",
+        stage: "parsing",
+    });
+    let records = await loadData(options.data, options.format);
+    // Apply offset and limit
+    if (options.offset && options.offset > 0) {
+        records = records.slice(options.offset);
+    }
+    if (options.limit && options.limit > 0) {
+        records = records.slice(0, options.limit);
+    }
+    const total = records.length;
+    // Validate mapping
+    const mappingErrors = validateMapping(options.mapping);
+    if (mappingErrors.length > 0) {
+        throw new Error(`Invalid field mapping:\n${mappingErrors.join("\n")}`);
+    }
+    // Phase 2: Dedup
+    let existingTitles = new Set();
+    if (options.skipExisting) {
+        options.onProgress?.({
+            processed: 0,
+            total,
+            current: "Checking existing memories...",
+            stage: "deduplicating",
+        });
+        const existing = await store.getAllMemories();
+        existingTitles = new Set(existing.map((m) => m.frontmatter.title.toLowerCase()));
+    }
+    // Phase 3 & 4: Ingest + Write
+    if (options.mode === "llm" && !ingestion.isLLMAvailable) {
+        throw new Error("LLM mode requires a configured LLM provider. Set ANTHROPIC_API_KEY, configure Ollama, or use --mode structured.");
+    }
+    // Process in batches for LLM concurrency
+    for (let i = 0; i < records.length; i += concurrency) {
+        const batch = records.slice(i, Math.min(i + concurrency, records.length));
+        const batchPromises = batch.map(async (record, batchIdx) => {
+            const globalIdx = i + batchIdx;
+            const mapped = applyMapping(record, options.mapping);
+            // Dedup check
+            if (options.skipExisting &&
+                existingTitles.has(mapped.title.toLowerCase())) {
+                results.skipped.push(mapped.title);
+                return;
+            }
+            options.onProgress?.({
+                processed: globalIdx,
+                total,
+                current: mapped.title,
+                stage: "ingesting",
+            });
+            try {
+                let title;
+                let category;
+                let tags;
+                let relevance;
+                let content;
+                let confidence;
+                let filename;
+                if (options.mode === "llm") {
+                    // Build rich text input for LLM
+                    let rawInput = `Title: ${mapped.title}\nCategory: ${mapped.category}`;
+                    if (mapped.content)
+                        rawInput += `\n\n${mapped.content}`;
+                    if (mapped.extraContext)
+                        rawInput += `\n\nAdditional data:\n${mapped.extraContext}`;
+                    const result = await ingestion.ingest(rawInput);
+                    title = result.title;
+                    category = result.category;
+                    tags = result.tags;
+                    relevance = result.relevance;
+                    content = result.content;
+                    confidence = result.confidence;
+                    filename = result.filename;
+                }
+                else {
+                    // Structured mode — direct mapping
+                    title = mapped.title;
+                    category = mapped.category;
+                    tags = mapped.tags || {};
+                    relevance = mapped.relevance || "";
+                    content = mapped.content || mapped.extraContext || title;
+                    confidence = 0.8;
+                    filename = title
+                        .toLowerCase()
+                        .replace(/[^a-z0-9]+/g, "-")
+                        .replace(/^-|-$/g, "")
+                        .substring(0, 60);
+                }
+                if (options.dryRun) {
+                    results.imported.push({
+                        title,
+                        category,
+                        path: `${category}/${filename}.md`,
+                    });
+                    return;
+                }
+                // Write memory
+                const id = await store.generateId(category);
+                const today = new Date().toISOString().split("T")[0];
+                const frontmatter = {
+                    id,
+                    title,
+                    category,
+                    tags,
+                    relevance,
+                    author,
+                    authority,
+                    confidence,
+                    created: today,
+                    modified: today,
+                    status: "active",
+                    supersedes: null,
+                };
+                const fullContent = `# ${title}\n\n${content}`;
+                const relativePath = await store.writeMemory(category, `${filename}.md`, frontmatter, fullContent, { autoCommit: !batchCommit });
+                results.imported.push({ title, category, path: relativePath });
+            }
+            catch (err) {
+                results.failed.push({
+                    record: mapped.title || `Record #${globalIdx}`,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        });
+        // Wait for batch to complete
+        await Promise.all(batchPromises);
+        // Small delay between LLM batches to avoid rate limits
+        if (options.mode === "llm" &&
+            i + concurrency < records.length) {
+            await new Promise((r) => setTimeout(r, 200));
+        }
+    }
+    results.totalProcessed = total;
+    // Phase 5: Batch commit
+    if (!options.dryRun && batchCommit && results.imported.length > 0) {
+        options.onProgress?.({
+            processed: total,
+            total,
+            current: "Committing...",
+            stage: "committing",
+        });
+        await store.batchCommit(`Bulk import: ${results.imported.length} memories from ${options.format} (${options.mode} mode)`);
+    }
+    results.duration = Date.now() - startTime;
+    return results;
+}
+// ─── Helpers ─────────────────────────────────────────────────────────────
+/**
+ * Format a human-readable summary of import results.
+ */
+export function formatImportSummary(result) {
+    const lines = [];
+    const durationSec = (result.duration / 1000).toFixed(1);
+    lines.push(`Import complete in ${durationSec}s`);
+    lines.push(`  Imported: ${result.imported.length}`);
+    lines.push(`  Skipped:  ${result.skipped.length}`);
+    lines.push(`  Failed:   ${result.failed.length}`);
+    lines.push(`  Total:    ${result.totalProcessed}`);
+    if (result.failed.length > 0) {
+        lines.push("");
+        lines.push("Failures:");
+        for (const f of result.failed.slice(0, 10)) {
+            lines.push(`  - ${f.record}: ${f.error}`);
+        }
+        if (result.failed.length > 10) {
+            lines.push(`  ... and ${result.failed.length - 10} more`);
+        }
+    }
+    return lines.join("\n");
+}
+/**
+ * Estimate import duration for smart threshold guidance.
+ */
+export function estimateDuration(recordCount, mode, concurrency = 5) {
+    if (mode === "structured") {
+        // ~50ms per record for structured
+        const seconds = Math.ceil((recordCount * 50) / 1000);
+        return seconds < 60 ? `~${seconds}s` : `~${Math.ceil(seconds / 60)}m`;
+    }
+    // LLM mode: ~1.5s per record with concurrency batching
+    const seconds = Math.ceil((recordCount * 1500) / concurrency / 1000);
+    if (seconds < 60)
+        return `~${seconds}s`;
+    if (seconds < 3600)
+        return `~${Math.ceil(seconds / 60)}m`;
+    return `~${(seconds / 3600).toFixed(1)}h`;
+}
+//# sourceMappingURL=import.js.map

package/dist/lib/import.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"import.js","sourceRoot":"","sources":["../../src/lib/import.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,MAAM,aAAa,CAAC;AAqC7B,4EAA4E;AAE5E,SAAS,eAAe,CAAC,MAAc;IACrC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACxE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC9B,IAAI,QAAQ,KAAK,iBAAiB,IAAI,QAAQ,KAAK,0BAA0B;YAAE,OAAO,KAAK,CAAC;QAC5F,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACjE,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,IAAI,CAAC,KAAK,EAAE;gBAAE,OAAO,KAAK,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;gBAAE,OAAO,KAAK,CAAC;YAClD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;gBAAE,OAAO,KAAK,CAAC;YACzC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;gBAAE,OAAO,KAAK,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,4EAA4E;AAE5E,KAAK,UAAU,QAAQ,CACrB,IAAY,EACZ,MAAgC;IAEhC,IAAI,GAAW,CAAC;IAEhB,mDAAmD;IACnD,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9D,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpF,CAAC;QACD,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;SAAM,IACL,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAC3B,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAC3B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,KAAK,KAAK,EAC7D,CAAC;QACD,0BAA0B;QAC1B,GAAG,GAAG,IAAI,CAAC;IACb,CAAC;SAAM,CAAC;QACN,YAAY;QACZ,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,KAAK;YACR,OAAO,QAAQ,CAAC,GAAG,EAAE;gBACnB,OAAO,EAAE,IAAI;gBACb,gBAAgB,EAAE,IAAI;gBACtB,IAAI,EAAE,IAAI;aACX,CAAC,CAAC;QAEL,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC/B,iEAAiE;YACjE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,OAAO,MAAM,CAAC;YACzC,+BAA+B;YAC/B,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,CAAC,EAAE,CAAC;gBACtF,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;YACrD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,4HAA4H,CAC7H,CAAC;QACJ,CAAC;QAED,KAAK,OAAO;YACV,OAAO,GAAG;iBACP,KAAK,CAAC,IAAI,CAAC;iBACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iBAC7B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAErC;YACE,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAaD,SAAS,eAAe,CAAC,OAA+B;IACtD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,gBAAgB,GAAG,aAAa,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvG,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CACnB,MAA+B,EAC/B,OAA+B;IAE/B,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACzD,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;YAAE,SAAS;QAEpD,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,OAAO;gBACV,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC7B,MAAM;YACR,KAAK,UAAU;gBACb,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;qBAC5B,WAAW,EAAE;qBACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;qBAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBACzB,MAAM;YACR,KAAK,SAAS;gBACZ,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC/B,MAAM;YACR,KAAK,WAAW;gBACd,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBACjC,MAAM;YACR,KAAK,MAAM;gBACT,yCAAyC;gBACzC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzB,MAAM,CAAC,IAAI,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,GAAG;wBACZ,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;6BAClB,KAAK,CAAC,GAAG,CAAC;6BACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;6BACpB,MAAM,CAAC,OAAO,CAAC;qBACnB,CAAC;gBACJ,CAAC;gBACD,MAAM;QACV,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1E,MAAM,MAAM,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACjF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC3C,UAAU,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,MAAM,EAAE,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,UAAU;QACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU;QACvC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;QAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,YAAY,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;KACpC,CAAC;AACJ,CAAC;AAED,4EAA4E;AAE5E,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAkB,EAClB,SAA0B,EAC1B,OAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAiB;QAC5B,QAAQ,EAAE,EAAE;QACZ,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,EAAE;QACV,cAAc,EAAE,CAAC;QACjB,QAAQ,EAAE,CAAC;KACZ,CAAC;IAEF,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,eAAe;IAClE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC;IACtC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,UAAU,CAAC;IAElD,sBAAsB;IACtB,OAAO,CAAC,UAAU,EAAE,CAAC;QACnB,SAAS,EAAE,CAAC;QACZ,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,iBAAiB;QAC1B,KAAK,EAAE,SAAS;KACjB,CAAC,CAAC;IAEH,IAAI,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAE3D,yBAAyB;IACzB,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACvC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,mBAAmB;IACnB,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,2BAA2B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,iBAAiB;IACjB,IAAI,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,OAAO,CAAC,UAAU,EAAE,CAAC;YACnB,SAAS,EAAE,CAAC;YACZ,KAAK;YACL,OAAO,EAAE,+BAA+B;YACxC,KAAK,EAAE,eAAe;SACvB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;QAC9C,cAAc,GAAG,IAAI,GAAG,CACtB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CACvD,CAAC;IACJ,CAAC;IAED,8BAA8B;IAC9B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CACb,iHAAiH,CAClH,CAAC;IACJ,CAAC;IAED,yCAAyC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CACzB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAC1C,CAAC;QAEF,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE;YACzD,MAAM,SAAS,GAAG,CAAC,GAAG,QAAQ,CAAC;YAC/B,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;YAErD,cAAc;YACd,IACE,OAAO,CAAC,YAAY;gBACpB,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAC9C,CAAC;gBACD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnC,OAAO;YACT,CAAC;YAED,OAAO,CAAC,UAAU,EAAE,CAAC;gBACnB,SAAS,EAAE,SAAS;gBACpB,KAAK;gBACL,OAAO,EAAE,MAAM,CAAC,KAAK;gBACrB,KAAK,EAAE,WAAW;aACnB,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,IAAI,KAAa,CAAC;gBAClB,IAAI,QAAgB,CAAC;gBACrB,IAAI,IAA8B,CAAC;gBACnC,IAAI,SAAiB,CAAC;gBACtB,IAAI,OAAe,CAAC;gBACpB,IAAI,UAAkB,CAAC;gBACvB,IAAI,QAAgB,CAAC;gBAErB,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBAC3B,gCAAgC;oBAChC,IAAI,QAAQ,GAAG,UAAU,MAAM,CAAC,KAAK,eAAe,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACtE,IAAI,MAAM,CAAC,OAAO;wBAAE,QAAQ,IAAI,OAAO,MAAM,CAAC,OAAO,EAAE,CAAC;oBACxD,IAAI,MAAM,CAAC,YAAY;wBAAE,QAAQ,IAAI,yBAAyB,MAAM,CAAC,YAAY,EAAE,CAAC;oBAEpF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAChD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;oBACrB,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;oBAC3B,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;oBACnB,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;oBAC7B,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;oBACzB,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;oBAC/B,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;gBAC7B,CAAC;qBAAM,CAAC;oBACN,mCAAmC;oBACnC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;oBACrB,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;oBAC3B,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;oBACzB,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;oBACnC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,YAAY,IAAI,KAAK,CAAC;oBACzD,UAAU,GAAG,GAAG,CAAC;oBACjB,QAAQ,GAAG,KAAK;yBACb,WAAW,EAAE;yBACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;yBAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;yBACrB,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtB,CAAC;gBAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;wBACpB,KAAK;wBACL,QAAQ;wBACR,IAAI,EAAE,GAAG,QAAQ,IAAI,QAAQ,KAAK;qBACnC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,eAAe;gBACf,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAC5C,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAErD,MAAM,WAAW,GAAsB;oBACrC,EAAE;oBACF,KAAK;oBACL,QAAQ;oBACR,IAAI;oBACJ,SAAS;oBACT,MAAM;oBACN,SAAS;oBACT,UAAU;oBACV,OAAO,EAAE,KAAK;oBACd,QAAQ,EAAE,KAAK;oBACf,MAAM,EAAE,QAAiB;oBACzB,UAAU,EAAE,IAAI;iBACjB,CAAC;gBAEF,MAAM,WAAW,GAAG,KAAK,KAAK,OAAO,OAAO,EAAE,CAAC;gBAC/C,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,WAAW,CAC1C,QAAQ,EACR,GAAG,QAAQ,KAAK,EAChB,WAAW,EACX,WAAW,EACX,EAAE,UAAU,EAAE,CAAC,WAAW,EAAE,CAC7B,CAAC;gBAEF,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;YACjE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;oBAClB,MAAM,EAAE,MAAM,CAAC,KAAK,IAAI,WAAW,SAAS,EAAE;oBAC9C,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,6BAA6B;QAC7B,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAEjC,uDAAuD;QACvD,IACE,OAAO,CAAC,IAAI,KAAK,KAAK;YACtB,CAAC,GAAG,WAAW,GAAG,OAAO,CAAC,MAAM,EAChC,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO,CAAC,cAAc,GAAG,KAAK,CAAC;IAE/B,wBAAwB;IACxB,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,OAAO,CAAC,UAAU,EAAE,CAAC;YACnB,SAAS,EAAE,KAAK;YAChB,KAAK;YACL,OAAO,EAAE,eAAe;YACxB,KAAK,EAAE,YAAY;SACpB,CAAC,CAAC;QAEH,MAAM,KAAK,CAAC,WAAW,CACrB,gBAAgB,OAAO,CAAC,QAAQ,CAAC,MAAM,kBAAkB,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,IAAI,QAAQ,CACjG,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAC1C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,4EAA4E;AAE5E;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAoB;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAExD,KAAK,CAAC,IAAI,CAAC,sBAAsB,WAAW,GAAG,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACpD,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACnD,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IAEnD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,WAAmB,EACnB,IAA0B,EAC1B,cAAsB,CAAC;IAEvB,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QAC1B,kCAAkC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACrD,OAAO,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,CAAC;IACxE,CAAC;IAED,uDAAuD;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,WAAW,GAAG,IAAI,CAAC,CAAC;IACrE,IAAI,OAAO,GAAG,EAAE;QAAE,OAAO,IAAI,OAAO,GAAG,CAAC;IACxC,IAAI,OAAO,GAAG,IAAI;QAAE,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,CAAC;IAC1D,OAAO,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5C,CAAC"}

package/dist/lib/ingest.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Gnosys Smart Ingestion — Uses LLM to structure raw input into atomic memories.
+ * Accepts messy human input, produces clean markdown files with YAML frontmatter.
+ * Uses the LLM abstraction layer — works with Anthropic, Ollama, or any future provider.
+ */
+import { GnosysTagRegistry } from "./tags.js";
+import { GnosysStore } from "./store.js";
+import { GnosysConfig } from "./config.js";
+interface IngestResult {
+    title: string;
+    category: string;
+    tags: Record<string, string[]>;
+    relevance: string;
+    content: string;
+    confidence: number;
+    filename: string;
+    proposedNewTags?: {
+        category: string;
+        tag: string;
+    }[];
+}
+export declare class GnosysIngestion {
+    private provider;
+    private tagRegistry;
+    private store;
+    private config;
+    constructor(store: GnosysStore, tagRegistry: GnosysTagRegistry, config?: GnosysConfig);
+    get isLLMAvailable(): boolean;
+    get providerName(): string;
+    /**
+     * Ingest raw text and structure it into an atomic memory.
+     * Uses LLM if available, otherwise requires structured input.
+     */
+    ingest(rawInput: string): Promise<IngestResult>;
+    /**
+     * Create a memory from already-structured input (no LLM needed).
+     */
+    createStructured(input: {
+        title: string;
+        category: string;
+        tags: Record<string, string[]>;
+        relevance?: string;
+        content: string;
+        author?: "human" | "ai" | "human+ai";
+        authority?: "declared" | "observed" | "imported" | "inferred";
+        confidence?: number;
+    }): IngestResult;
+    private slugify;
+}
+export {};
+//# sourceMappingURL=ingest.d.ts.map

package/dist/lib/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/lib/ingest.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAkB,MAAM,aAAa,CAAC;AAG3D,UAAU,YAAY;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACvD;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAA4B;IAC5C,OAAO,CAAC,WAAW,CAAoB;IACvC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,MAAM,CAAe;gBAEjB,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,YAAY;IAcrF,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;;OAGG;IACG,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAmGrD;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE;QACtB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,UAAU,CAAC;QACrC,SAAS,CAAC,EAAE,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;QAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,YAAY;IAYhB,OAAO,CAAC,OAAO;CAOhB"}

package/dist/lib/ingest.js

@@ -0,0 +1,144 @@
+/**
+ * Gnosys Smart Ingestion — Uses LLM to structure raw input into atomic memories.
+ * Accepts messy human input, produces clean markdown files with YAML frontmatter.
+ * Uses the LLM abstraction layer — works with Anthropic, Ollama, or any future provider.
+ */
+import { DEFAULT_CONFIG } from "./config.js";
+import { getLLMProvider } from "./llm.js";
+export class GnosysIngestion {
+    provider = null;
+    tagRegistry;
+    store;
+    config;
+    constructor(store, tagRegistry, config) {
+        this.store = store;
+        this.tagRegistry = tagRegistry;
+        this.config = config || DEFAULT_CONFIG;
+        // Initialize LLM provider via abstraction layer
+        try {
+            this.provider = getLLMProvider(this.config, "structuring");
+        }
+        catch {
+            // Provider not available (e.g., no API key for Anthropic)
+            this.provider = null;
+        }
+    }
+    get isLLMAvailable() {
+        return this.provider !== null;
+    }
+    get providerName() {
+        return this.provider?.name || "none";
+    }
+    /**
+     * Ingest raw text and structure it into an atomic memory.
+     * Uses LLM if available, otherwise requires structured input.
+     */
+    async ingest(rawInput) {
+        if (!this.provider) {
+            const providerName = this.config.llm.defaultProvider;
+            throw new Error(providerName === "anthropic"
+                ? "No ANTHROPIC_API_KEY set. Smart ingestion requires an LLM. " +
+                    "Set the ANTHROPIC_API_KEY environment variable, switch to Ollama (gnosys config set provider ollama), or use gnosys_add_structured."
+                : `LLM provider "${providerName}" is not available. Check your configuration or use gnosys_add_structured.`);
+        }
+        const registry = this.tagRegistry.getRegistry();
+        const categories = await this.store.getCategories();
+        const systemPrompt = `You are Gnosys, a knowledge management system. Your job is to take raw, unstructured input and produce a structured atomic memory.
+
+An atomic memory is ONE concept, decision, or piece of knowledge per file. If the input contains multiple concepts, focus on the primary one.
+
+You must output valid JSON with these fields:
+- title: A clear, concise title for this memory
+- category: One of the existing categories or suggest a new one. Existing: ${categories.join(", ")}
+- tags: An object with tag categories as keys and arrays of tags as values. Use ONLY tags from the registry below. If no existing tag fits, include a "proposed_new_tags" array.
+- relevance: A keyword cloud (space-separated words, NOT sentences) describing every context where this memory would be useful. Include synonyms, related terms, abbreviations, and domain variations. Example: "auth authentication OAuth JWT login session tokens SSO identity credentials access-control permissions RBAC". Be generous — 15-30 keywords. This field powers discovery search.
+- content: The memory content as clean markdown prose (NOT including the title as an H1 — that's added automatically)
+- confidence: 0.0 to 1.0 — how confident you are this is a well-formed atomic memory
+- filename: A kebab-case filename (without .md extension)
+
+TAG REGISTRY:
+${Object.entries(registry)
+            .map(([cat, tags]) => `${cat}: ${tags.join(", ")}`)
+            .join("\n")}
+
+Rules:
+1. One concept per memory. If the input is about multiple things, pick the most important one.
+2. Use existing tags from the registry. Only propose new tags if nothing fits.
+3. The content should be self-contained — someone reading just this file should understand the concept.
+4. Be concise. Memories are meant to be quickly scanned, not deeply read.
+5. Write in third person or neutral voice, not first person.
+6. The relevance field is critical for discovery. Include all terms an agent might use to find this memory — think about what someone working on a related task would search for.`;
+        const text = await this.provider.generate(`Structure this into an atomic memory:\n\n${rawInput}`, { system: systemPrompt, maxTokens: 2000 });
+        // Parse JSON from response (handle markdown code blocks)
+        const jsonMatch = text.match(/```json\s*([\s\S]*?)```/) ||
+            text.match(/```\s*([\s\S]*?)```/) || [null, text];
+        const raw = JSON.parse(jsonMatch[1] || text);
+        // Sanitize parsed object — strip prototype-polluting keys
+        const parsed = {};
+        if (raw && typeof raw === "object") {
+            for (const [key, value] of Object.entries(raw)) {
+                if (key === "__proto__" || key === "constructor" || key === "prototype")
+                    continue;
+                parsed[key] = value;
+            }
+        }
+        // Validate and clean up tags
+        const cleanTags = {};
+        const proposedNewTags = [];
+        if (parsed.tags && typeof parsed.tags === "object") {
+            for (const [cat, tags] of Object.entries(parsed.tags)) {
+                if (Array.isArray(tags)) {
+                    const validTags = [];
+                    for (const tag of tags) {
+                        if (this.tagRegistry.hasTag(tag)) {
+                            validTags.push(tag);
+                        }
+                        else {
+                            proposedNewTags.push({ category: cat, tag });
+                        }
+                    }
+                    if (validTags.length > 0)
+                        cleanTags[cat] = validTags;
+                }
+            }
+        }
+        // Add any explicitly proposed new tags
+        if (parsed.proposed_new_tags) {
+            for (const item of parsed.proposed_new_tags) {
+                proposedNewTags.push(item);
+            }
+        }
+        return {
+            title: parsed.title || "Untitled Memory",
+            category: parsed.category || "uncategorized",
+            tags: cleanTags,
+            relevance: parsed.relevance || "",
+            content: parsed.content || rawInput,
+            confidence: parsed.confidence || 0.7,
+            filename: parsed.filename || this.slugify(parsed.title || "memory"),
+            proposedNewTags: proposedNewTags.length > 0 ? proposedNewTags : undefined,
+        };
+    }
+    /**
+     * Create a memory from already-structured input (no LLM needed).
+     */
+    createStructured(input) {
+        return {
+            title: input.title,
+            category: input.category,
+            tags: input.tags,
+            relevance: input.relevance || "",
+            content: input.content,
+            confidence: input.confidence || 0.8,
+            filename: this.slugify(input.title),
+        };
+    }
+    slugify(text) {
+        return text
+            .toLowerCase()
+            .replace(/[^a-z0-9]+/g, "-")
+            .replace(/^-|-$/g, "")
+            .substring(0, 60);
+    }
+}
+//# sourceMappingURL=ingest.js.map

package/dist/lib/ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/lib/ingest.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAgB,cAAc,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAe,cAAc,EAAE,MAAM,UAAU,CAAC;AAavD,MAAM,OAAO,eAAe;IAClB,QAAQ,GAAuB,IAAI,CAAC;IACpC,WAAW,CAAoB;IAC/B,KAAK,CAAc;IACnB,MAAM,CAAe;IAE7B,YAAY,KAAkB,EAAE,WAA8B,EAAE,MAAqB;QACnF,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,cAAc,CAAC;QAEvC,gDAAgD;QAChD,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC;YACP,0DAA0D;YAC1D,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC;IAChC,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,QAAQ,EAAE,IAAI,IAAI,MAAM,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,QAAgB;QAC3B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC;YACrD,MAAM,IAAI,KAAK,CACb,YAAY,KAAK,WAAW;gBAC1B,CAAC,CAAC,6DAA6D;oBAC7D,qIAAqI;gBACvI,CAAC,CAAC,iBAAiB,YAAY,4EAA4E,CAC9G,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QAChD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QAEpD,MAAM,YAAY,GAAG;;;;;;6EAMoD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;EAQhG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;aACvB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aAClD,IAAI,CAAC,IAAI,CAAC;;;;;;;;kLAQqK,CAAC;QAE/K,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CACvC,4CAA4C,QAAQ,EAAE,EACtD,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,EAAE,CAC1C,CAAC;QAEF,yDAAyD;QACzD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;QAE7C,0DAA0D;QAC1D,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YACnC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,EAAE,CAAC;gBAC1E,IAAI,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,aAAa,IAAI,GAAG,KAAK,WAAW;oBAAE,SAAS;gBAClF,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACtB,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,MAAM,SAAS,GAA6B,EAAE,CAAC;QAC/C,MAAM,eAAe,GAAwC,EAAE,CAAC;QAEhE,IAAI,MAAM,CAAC,IAAI,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;oBACxB,MAAM,SAAS,GAAa,EAAE,CAAC;oBAC/B,KAAK,MAAM,GAAG,IAAI,IAAgB,EAAE,CAAC;wBACnC,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;4BACjC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACtB,CAAC;6BAAM,CAAC;4BACN,eAAe,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;wBAC/C,CAAC;oBACH,CAAC;oBACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;wBAAE,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;QAED,uCAAuC;QACvC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC7B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC5C,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,iBAAiB;YACxC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,eAAe;YAC5C,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,QAAQ;YACnC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,GAAG;YACpC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC;YACnE,eAAe,EACb,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;SAC3D,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAShB;QACC,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;YAChC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,GAAG;YACnC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;SACpC,CAAC;IACJ,CAAC;IAEO,OAAO,CAAC,IAAY;QAC1B,OAAO,IAAI;aACR,WAAW,EAAE;aACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;aAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;aACrB,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtB,CAAC;CACF"}

package/dist/lib/lensing.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Gnosys Lensing — Filtered views of the memory store.
+ *
+ * A lens is a set of filter criteria applied to memories.
+ * Compound lenses combine multiple filters with AND/OR logic.
+ */
+import { Memory } from "./store.js";
+export interface LensFilter {
+    category?: string;
+    tags?: string[];
+    tagMatchMode?: "any" | "all";
+    status?: ("active" | "archived" | "superseded")[];
+    author?: ("human" | "ai" | "human+ai")[];
+    authority?: ("declared" | "observed" | "imported" | "inferred")[];
+    minConfidence?: number;
+    maxConfidence?: number;
+    createdAfter?: string;
+    createdBefore?: string;
+    modifiedAfter?: string;
+    modifiedBefore?: string;
+}
+export type CompoundOperator = "AND" | "OR";
+export interface CompoundLens {
+    filters: LensFilter[];
+    operator: CompoundOperator;
+}
+/**
+ * Apply a single lens filter to an array of memories.
+ */
+export declare function applyLens(memories: Memory[], lens: LensFilter): Memory[];
+/**
+ * Apply a compound lens (multiple filters combined with AND/OR).
+ */
+export declare function applyCompoundLens(memories: Memory[], compound: CompoundLens): Memory[];
+//# sourceMappingURL=lensing.d.ts.map

package/dist/lib/lensing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lensing.d.ts","sourceRoot":"","sources":["../../src/lib/lensing.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,YAAY,CAAC;AAEvD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,YAAY,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IAC7B,MAAM,CAAC,EAAE,CAAC,QAAQ,GAAG,UAAU,GAAG,YAAY,CAAC,EAAE,CAAC;IAClD,MAAM,CAAC,EAAE,CAAC,OAAO,GAAG,IAAI,GAAG,UAAU,CAAC,EAAE,CAAC;IACzC,SAAS,CAAC,EAAE,CAAC,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;IAClE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,gBAAgB,GAAG,KAAK,GAAG,IAAI,CAAC;AAE5C,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,QAAQ,EAAE,gBAAgB,CAAC;CAC5B;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,UAAU,GAAG,MAAM,EAAE,CAExE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,EAAE,EAClB,QAAQ,EAAE,YAAY,GACrB,MAAM,EAAE,CAcV"}