@soleri/core 2.4.0 → 2.5.0

package/dist/intake/content-classifier.js

@@ -0,0 +1,125 @@
+// ─── Content Classifier — LLM-based knowledge extraction ────────────────────
+//
+// Takes a text chunk and uses an LLM to classify it into structured
+// knowledge items. Graceful degradation: returns [] on any error.
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+export const VALID_TYPES = [
+    'pattern',
+    'anti-pattern',
+    'principle',
+    'concept',
+    'reference',
+    'workflow',
+    'idea',
+    'roadmap',
+];
+const VALID_SEVERITIES = ['critical', 'warning', 'suggestion'];
+export const CLASSIFICATION_PROMPT = `You are a knowledge extraction engine. Your job is to analyze a text chunk and extract structured knowledge items from it.
+
+For each distinct piece of knowledge you identify, produce an object with these fields:
+- type: one of ${JSON.stringify(VALID_TYPES)}
+- title: concise title, max 80 characters
+- description: 2-3 sentence summary of the knowledge
+- tags: 3-5 lowercase single-word or hyphenated tags
+- severity: one of "critical", "warning", "suggestion"
+
+Rules:
+- Extract ALL meaningful knowledge items from the text.
+- Each item must be self-contained and independently useful.
+- Use "critical" for must-know items, "warning" for important gotchas, "suggestion" for nice-to-know.
+- Tags should be specific and useful for search.
+- Respond with a pure JSON array of objects. No markdown fences, no explanation, no wrapping.
+- If the text contains no extractable knowledge, respond with an empty array: []`;
+// =============================================================================
+// CLASSIFIER
+// =============================================================================
+/**
+ * Classify a text chunk into structured knowledge items using an LLM.
+ *
+ * @param llm      - LLMClient instance
+ * @param chunkText - The text to classify
+ * @param citation  - Source citation (e.g. "book.pdf, pages 12-15")
+ * @returns Classified items, or [] on any error
+ */
+export async function classifyChunk(llm, chunkText, citation) {
+    try {
+        const result = await llm.complete({
+            provider: 'openai',
+            model: 'gpt-4o-mini',
+            systemPrompt: CLASSIFICATION_PROMPT,
+            userPrompt: chunkText,
+            maxTokens: 4096,
+            temperature: 0.3,
+            caller: 'intake',
+            task: 'classify',
+        });
+        const raw = parseJsonResponse(result.text);
+        if (!Array.isArray(raw))
+            return [];
+        return raw
+            .map((item) => sanitizeItem(item, citation))
+            .filter((item) => item !== null);
+    }
+    catch {
+        // Graceful degradation — never throw
+        return [];
+    }
+}
+// =============================================================================
+// HELPERS
+// =============================================================================
+/**
+ * Parse a JSON response, handling potential markdown fences the LLM
+ * might include despite instructions.
+ */
+function parseJsonResponse(text) {
+    const trimmed = text.trim();
+    // Strip markdown fences if present (defensive)
+    const fenceMatch = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?\s*```$/);
+    const jsonStr = fenceMatch ? fenceMatch[1] : trimmed;
+    return JSON.parse(jsonStr);
+}
+/**
+ * Validate and sanitize a single classified item.
+ * Returns null if the item is not salvageable.
+ */
+function sanitizeItem(raw, citation) {
+    if (!raw || typeof raw !== 'object')
+        return null;
+    const obj = raw;
+    // Type — must be a valid KnowledgeType
+    const type = typeof obj.type === 'string' ? obj.type : '';
+    if (!VALID_TYPES.includes(type))
+        return null;
+    // Title — required, truncate to 80 chars
+    const title = typeof obj.title === 'string' ? obj.title.slice(0, 80).trim() : '';
+    if (!title)
+        return null;
+    // Description — required
+    const description = typeof obj.description === 'string' ? obj.description.trim() : '';
+    if (!description)
+        return null;
+    // Tags — must be array of strings, cap at 5
+    const tags = Array.isArray(obj.tags)
+        ? obj.tags
+            .filter((t) => typeof t === 'string')
+            .map((t) => t.toLowerCase().trim())
+            .filter((t) => t.length > 0)
+            .slice(0, 5)
+        : [];
+    // Severity — default to 'suggestion' if invalid
+    const severity = VALID_SEVERITIES.includes(obj.severity)
+        ? obj.severity
+        : 'suggestion';
+    return {
+        type: type,
+        title,
+        description,
+        tags,
+        severity,
+        citation,
+    };
+}
+//# sourceMappingURL=content-classifier.js.map

package/dist/intake/content-classifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-classifier.js","sourceRoot":"","sources":["../../src/intake/content-classifier.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,EAAE;AACF,oEAAoE;AACpE,kEAAkE;AAKlE,gFAAgF;AAChF,YAAY;AACZ,gFAAgF;AAEhF,MAAM,CAAC,MAAM,WAAW,GAAoB;IAC1C,SAAS;IACT,cAAc;IACd,WAAW;IACX,SAAS;IACT,WAAW;IACX,UAAU;IACV,MAAM;IACN,SAAS;CACV,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,YAAY,CAAU,CAAC;AAGxE,MAAM,CAAC,MAAM,qBAAqB,GAAG;;;iBAGpB,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;;;;;;;;;;;;iFAYqC,CAAC;AAElF,gFAAgF;AAChF,aAAa;AACb,gFAAgF;AAEhF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAc,EACd,SAAiB,EACjB,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC;YAChC,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,aAAa;YACpB,YAAY,EAAE,qBAAqB;YACnC,UAAU,EAAE,SAAS;YACrB,SAAS,EAAE,IAAI;YACf,WAAW,EAAE,GAAG;YAChB,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,UAAU;SACjB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QAEnC,OAAO,GAAG;aACP,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;aACpD,MAAM,CAAC,CAAC,IAAI,EAA0B,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IAC7D,CAAC;IAAC,MAAM,CAAC;QACP,qCAAqC;QACrC,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,UAAU;AACV,gFAAgF;AAEhF;;;GAGG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,+CAA+C;IAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAErD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,GAAY,EAAE,QAAgB;IAClD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEjD,MAAM,GAAG,GAAG,GAA8B,CAAC;IAE3C,uCAAuC;IACvC,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAqB,CAAC;QAAE,OAAO,IAAI,CAAC;IAE9D,yCAAyC;IACzC,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,yBAAyB;IACzB,MAAM,WAAW,GAAG,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,IAAI,CAAC,WAAW;QAAE,OAAO,IAAI,CAAC;IAE9B,4CAA4C;IAC5C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,CAAC,CAAC,GAAG,CAAC,IAAI;aACL,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;aACjD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC;IAEP,gDAAgD;IAChD,MAAM,QAAQ,GAAa,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAoB,CAAC;QAC5E,CAAC,CAAE,GAAG,CAAC,QAAqB;QAC5B,CAAC,CAAC,YAAY,CAAC;IAEjB,OAAO;QACL,IAAI,EAAE,IAAqB;QAC3B,KAAK;QACL,WAAW;QACX,IAAI;QACJ,QAAQ;QACR,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/intake/dedup-gate.d.ts

@@ -0,0 +1,17 @@
+import type { Vault } from '../vault/vault.js';
+import type { ClassifiedItem } from './types.js';
+export declare const DEDUP_THRESHOLD = 0.85;
+export interface DedupResult {
+    item: ClassifiedItem;
+    isDuplicate: boolean;
+    bestMatchId?: string;
+    similarity: number;
+}
+/**
+ * Check new items against existing vault entries for duplicates using TF-IDF cosine similarity.
+ *
+ * Builds a shared IDF vocabulary from all texts (existing + new), computes TF-IDF vectors,
+ * and marks items as duplicates when cosine similarity >= DEDUP_THRESHOLD.
+ */
+export declare function dedupItems(items: ClassifiedItem[], vault: Vault): DedupResult[];
+//# sourceMappingURL=dedup-gate.d.ts.map

package/dist/intake/dedup-gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup-gate.d.ts","sourceRoot":"","sources":["../../src/intake/dedup-gate.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,cAAc,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,KAAK,EAAE,KAAK,GAAG,WAAW,EAAE,CA+D/E"}

package/dist/intake/dedup-gate.js

@@ -0,0 +1,66 @@
+// ─── Dedup Gate ───────────────────────────────────────────────────
+// TF-IDF cosine similarity check against existing vault entries.
+// Pure function: no side effects, no I/O beyond reading vault.
+import { tokenize, calculateTfIdf, cosineSimilarity, } from '../text/similarity.js';
+export const DEDUP_THRESHOLD = 0.85;
+/**
+ * Check new items against existing vault entries for duplicates using TF-IDF cosine similarity.
+ *
+ * Builds a shared IDF vocabulary from all texts (existing + new), computes TF-IDF vectors,
+ * and marks items as duplicates when cosine similarity >= DEDUP_THRESHOLD.
+ */
+export function dedupItems(items, vault) {
+    const existing = vault.exportAll().entries;
+    // Fast path: nothing in vault — everything is new
+    if (existing.length === 0) {
+        return items.map((item) => ({
+            item,
+            isDuplicate: false,
+            similarity: 0,
+        }));
+    }
+    // ── Build texts for vocabulary ──────────────────────────────────
+    const existingTexts = existing.map((e) => `${e.title} ${e.description}`);
+    const newTexts = items.map((i) => `${i.title} ${i.description}`);
+    const allTexts = [...existingTexts, ...newTexts];
+    const totalDocs = allTexts.length;
+    // ── Count document frequency per term ───────────────────────────
+    const docFreq = new Map();
+    for (const text of allTexts) {
+        const terms = new Set(tokenize(text));
+        for (const term of terms) {
+            docFreq.set(term, (docFreq.get(term) ?? 0) + 1);
+        }
+    }
+    // ── Build IDF vocabulary ────────────────────────────────────────
+    const vocabulary = new Map();
+    for (const [term, df] of docFreq) {
+        vocabulary.set(term, Math.log((totalDocs + 1) / (df + 1)) + 1);
+    }
+    // ── Compute TF-IDF vectors for existing entries ─────────────────
+    const existingVectors = existing.map((entry, idx) => ({
+        id: entry.id,
+        vec: calculateTfIdf(tokenize(existingTexts[idx]), vocabulary),
+    }));
+    // ── Score each new item against all existing entries ────────────
+    return items.map((item, idx) => {
+        const itemVec = calculateTfIdf(tokenize(newTexts[idx]), vocabulary);
+        let bestSimilarity = 0;
+        let bestMatchId;
+        for (const { id, vec } of existingVectors) {
+            const sim = cosineSimilarity(itemVec, vec);
+            if (sim > bestSimilarity) {
+                bestSimilarity = sim;
+                bestMatchId = id;
+            }
+        }
+        const isDuplicate = bestSimilarity >= DEDUP_THRESHOLD;
+        return {
+            item,
+            isDuplicate,
+            bestMatchId: isDuplicate ? bestMatchId : undefined,
+            similarity: bestSimilarity,
+        };
+    });
+}
+//# sourceMappingURL=dedup-gate.js.map

package/dist/intake/dedup-gate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup-gate.js","sourceRoot":"","sources":["../../src/intake/dedup-gate.ts"],"names":[],"mappings":"AAAA,qEAAqE;AACrE,iEAAiE;AACjE,+DAA+D;AAE/D,OAAO,EACL,QAAQ,EACR,cAAc,EACd,gBAAgB,GAEjB,MAAM,uBAAuB,CAAC;AAI/B,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AASpC;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,KAAuB,EAAE,KAAY;IAC9D,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC;IAE3C,kDAAkD;IAClD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1B,IAAI;YACJ,WAAW,EAAE,KAAK;YAClB,UAAU,EAAE,CAAC;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAED,mEAAmE;IACnE,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAG,CAAC,GAAG,aAAa,EAAE,GAAG,QAAQ,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC;IAElC,mEAAmE;IACnE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC;QACjC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,mEAAmE;IACnE,MAAM,eAAe,GAA6C,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC9F,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,GAAG,EAAE,cAAc,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC;KAC9D,CAAC,CAAC,CAAC;IAEJ,mEAAmE;IACnE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAEpE,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,WAA+B,CAAC;QAEpC,KAAK,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,eAAe,EAAE,CAAC;YAC1C,MAAM,GAAG,GAAG,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC3C,IAAI,GAAG,GAAG,cAAc,EAAE,CAAC;gBACzB,cAAc,GAAG,GAAG,CAAC;gBACrB,WAAW,GAAG,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,cAAc,IAAI,eAAe,CAAC;QAEtD,OAAO;YACL,IAAI;YACJ,WAAW;YACX,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YAClD,UAAU,EAAE,cAAc;SAC3B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/intake/intake-pipeline.d.ts

@@ -0,0 +1,63 @@
+import type { PersistenceProvider } from '../persistence/types.js';
+import type { Vault } from '../vault/vault.js';
+import type { LLMClient } from '../llm/llm-client.js';
+import type { IntakeConfig, IntakeChunk, IntakeJobRecord, IntakePreviewResult } from './types.js';
+/**
+ * Split concatenated PDF text into per-page segments.
+ *
+ * Strategy: split on form-feed characters first (common in pdf-parse output).
+ * If that yields fewer segments than expected, fall back to equal-length splits.
+ */
+export declare function splitIntoPages(text: string, numPages: number): string[];
+export declare class IntakePipeline {
+    private provider;
+    private vault;
+    private llm;
+    constructor(provider: PersistenceProvider, vault: Vault, llm: LLMClient);
+    private initSchema;
+    /**
+     * Parse a PDF, compute its file hash, create fixed-size page chunks,
+     * and persist the job + chunk records to the database.
+     */
+    ingestBook(config: IntakeConfig): Promise<IntakeJobRecord>;
+    /**
+     * Process up to `count` pending chunks for a job.
+     *
+     * For each chunk:
+     *   2. Extract page text from PDF
+     *   3. Classify via LLM
+     *   4. Dedup against vault
+     *   5. Store unique items
+     *
+     * When all chunks are done, finalizes the job (stage 6).
+     */
+    processChunks(jobId: string, count?: number): Promise<{
+        processed: number;
+        itemsStored: number;
+        itemsDeduped: number;
+        remaining: number;
+    }>;
+    /**
+     * Parse a page range from a PDF and classify it without storing.
+     * Useful for previewing what the pipeline would extract.
+     */
+    preview(config: IntakeConfig, pageStart: number, pageEnd: number): Promise<IntakePreviewResult>;
+    /**
+     * Get a job record by ID.
+     */
+    getJob(jobId: string): IntakeJobRecord | null;
+    /**
+     * List all intake jobs.
+     */
+    listJobs(): IntakeJobRecord[];
+    /**
+     * Get all chunks for a job.
+     */
+    getChunks(jobId: string): IntakeChunk[];
+    /**
+     * Sum stats from all chunks and mark the job as completed.
+     */
+    private finalizeJob;
+    private countPendingChunks;
+}
+//# sourceMappingURL=intake-pipeline.d.ts.map

package/dist/intake/intake-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"intake-pipeline.d.ts","sourceRoot":"","sources":["../../src/intake/intake-pipeline.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,eAAe,EACf,mBAAmB,EAGpB,MAAM,YAAY,CAAC;AAwBpB;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAoBvE;AAMD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,GAAG,CAAY;gBAEX,QAAQ,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS;IASvE,OAAO,CAAC,UAAU;IAiClB;;;OAGG;IACG,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;IAgEhE;;;;;;;;;;OAUG;IACG,aAAa,CACjB,KAAK,EAAE,MAAM,EACb,KAAK,GAAE,MAAU,GAChB,OAAO,CAAC;QACT,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IAwHF;;;OAGG;IACG,OAAO,CACX,MAAM,EAAE,YAAY,EACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,mBAAmB,CAAC;IAqB/B;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI;IAQ7C;;OAEG;IACH,QAAQ,IAAI,eAAe,EAAE;IAO7B;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,EAAE;IAUvC;;OAEG;IACH,OAAO,CAAC,WAAW;IAiCnB,OAAO,CAAC,kBAAkB;CAO3B"}

package/dist/intake/intake-pipeline.js

@@ -0,0 +1,373 @@
+// ─── Intake Pipeline ──────────────────────────────────────────────
+//
+// 6-stage pipeline for ingesting PDF books into the vault:
+//   1. Parse PDF + compute hash + create chunks → job record
+//   2. Extract page text for each chunk
+//   3. Classify chunk text via LLM
+//   4. Dedup classified items against vault
+//   5. Store unique items in vault
+//   6. Finalize job with aggregate stats
+//
+// SQLite-backed job tracking for resumable processing.
+import { createHash, randomUUID } from 'node:crypto';
+import { readFileSync, statSync } from 'node:fs';
+import { classifyChunk } from './content-classifier.js';
+import { dedupItems } from './dedup-gate.js';
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+const DEFAULT_CHUNK_SIZE = 10;
+/**
+ * Map KnowledgeType → IntelligenceEntry.type.
+ * Only 'pattern' and 'anti-pattern' map directly; everything else becomes 'rule'.
+ */
+function mapKnowledgeType(kt) {
+    if (kt === 'pattern')
+        return 'pattern';
+    if (kt === 'anti-pattern')
+        return 'anti-pattern';
+    return 'rule';
+}
+// =============================================================================
+// HELPERS
+// =============================================================================
+/**
+ * Split concatenated PDF text into per-page segments.
+ *
+ * Strategy: split on form-feed characters first (common in pdf-parse output).
+ * If that yields fewer segments than expected, fall back to equal-length splits.
+ */
+export function splitIntoPages(text, numPages) {
+    if (numPages <= 0)
+        return [text];
+    // Try form-feed split first
+    const ffPages = text.split('\f');
+    if (ffPages.length >= numPages) {
+        return ffPages.slice(0, numPages);
+    }
+    // Fallback: equal-length chunks
+    const chunkSize = Math.ceil(text.length / numPages);
+    const pages = [];
+    for (let i = 0; i < text.length; i += chunkSize) {
+        pages.push(text.slice(i, i + chunkSize));
+    }
+    // Pad with empty strings if we somehow got fewer
+    while (pages.length < numPages) {
+        pages.push('');
+    }
+    return pages;
+}
+// =============================================================================
+// PIPELINE
+// =============================================================================
+export class IntakePipeline {
+    provider;
+    vault;
+    llm;
+    constructor(provider, vault, llm) {
+        this.provider = provider;
+        this.vault = vault;
+        this.llm = llm;
+        this.initSchema();
+    }
+    // ─── Schema ──────────────────────────────────────────────────────
+    initSchema() {
+        this.provider.execSql(`
+      CREATE TABLE IF NOT EXISTS intake_jobs (
+        id TEXT PRIMARY KEY,
+        status TEXT NOT NULL,
+        config TEXT NOT NULL,
+        pdf_meta TEXT,
+        toc TEXT,
+        stats TEXT,
+        created_at INTEGER,
+        updated_at INTEGER,
+        completed_at INTEGER
+      );
+
+      CREATE TABLE IF NOT EXISTS intake_chunks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        job_id TEXT NOT NULL REFERENCES intake_jobs(id),
+        chunk_index INTEGER,
+        title TEXT,
+        page_start INTEGER,
+        page_end INTEGER,
+        status TEXT DEFAULT 'pending',
+        items_extracted INTEGER DEFAULT 0,
+        items_stored INTEGER DEFAULT 0,
+        items_deduped INTEGER DEFAULT 0,
+        error TEXT,
+        processed_at INTEGER
+      );
+    `);
+    }
+    // ─── Stage 1: Ingest Book ────────────────────────────────────────
+    /**
+     * Parse a PDF, compute its file hash, create fixed-size page chunks,
+     * and persist the job + chunk records to the database.
+     */
+    async ingestBook(config) {
+        const jobId = randomUUID();
+        const now = Math.floor(Date.now() / 1000);
+        const chunkPageSize = config.chunkPageSize ?? DEFAULT_CHUNK_SIZE;
+        // Read file
+        const fileBuffer = readFileSync(config.pdfPath);
+        const fileSize = statSync(config.pdfPath).size;
+        const fileHash = createHash('sha256').update(fileBuffer).digest('hex');
+        // Dynamic import of pdf-parse
+        const pdfParse = (await import('pdf-parse')).default;
+        const pdfData = await pdfParse(fileBuffer);
+        const totalPages = pdfData.numpages;
+        const pdfMeta = { totalPages, fileHash, fileSize };
+        // Create chunk definitions (fixed N-page windows)
+        const numChunks = Math.ceil(totalPages / chunkPageSize);
+        this.provider.transaction(() => {
+            // Insert job
+            this.provider.run(`INSERT INTO intake_jobs (id, status, config, pdf_meta, toc, stats, created_at, updated_at, completed_at)
+         VALUES (@id, @status, @config, @pdfMeta, @toc, @stats, @createdAt, @updatedAt, @completedAt)`, {
+                id: jobId,
+                status: 'initialized',
+                config: JSON.stringify(config),
+                pdfMeta: JSON.stringify(pdfMeta),
+                toc: null,
+                stats: null,
+                createdAt: now,
+                updatedAt: now,
+                completedAt: null,
+            });
+            // Insert chunk records
+            for (let i = 0; i < numChunks; i++) {
+                const pageStart = i * chunkPageSize + 1;
+                const pageEnd = Math.min((i + 1) * chunkPageSize, totalPages);
+                const chunkTitle = `${config.title} — pages ${pageStart}-${pageEnd}`;
+                this.provider.run(`INSERT INTO intake_chunks (job_id, chunk_index, title, page_start, page_end, status)
+           VALUES (@jobId, @chunkIndex, @title, @pageStart, @pageEnd, @status)`, {
+                    jobId,
+                    chunkIndex: i,
+                    title: chunkTitle,
+                    pageStart,
+                    pageEnd,
+                    status: 'pending',
+                });
+            }
+        });
+        return this.getJob(jobId);
+    }
+    // ─── Stages 2-5: Process Chunks ──────────────────────────────────
+    /**
+     * Process up to `count` pending chunks for a job.
+     *
+     * For each chunk:
+     *   2. Extract page text from PDF
+     *   3. Classify via LLM
+     *   4. Dedup against vault
+     *   5. Store unique items
+     *
+     * When all chunks are done, finalizes the job (stage 6).
+     */
+    async processChunks(jobId, count = 5) {
+        // Get pending chunks
+        const pendingChunks = this.provider.all(`SELECT * FROM intake_chunks WHERE job_id = @jobId AND status = 'pending' ORDER BY chunk_index ASC LIMIT @limit`, { jobId, limit: count });
+        if (pendingChunks.length === 0) {
+            const remaining = this.countPendingChunks(jobId);
+            return { processed: 0, itemsStored: 0, itemsDeduped: 0, remaining };
+        }
+        // Mark job as processing
+        this.provider.run(`UPDATE intake_jobs SET status = 'processing', updated_at = @now WHERE id = @id`, { id: jobId, now: Math.floor(Date.now() / 1000) });
+        // Re-read config and parse PDF
+        const job = this.getJob(jobId);
+        if (!job) {
+            return { processed: 0, itemsStored: 0, itemsDeduped: 0, remaining: 0 };
+        }
+        const fileBuffer = readFileSync(job.config.pdfPath);
+        const pdfParse = (await import('pdf-parse')).default;
+        const pdfData = await pdfParse(fileBuffer);
+        const totalPages = job.pdfMeta?.totalPages ?? pdfData.numpages;
+        const pages = splitIntoPages(pdfData.text, totalPages);
+        let totalStored = 0;
+        let totalDeduped = 0;
+        let processed = 0;
+        for (const chunkRow of pendingChunks) {
+            const chunkId = chunkRow.id;
+            const chunkIndex = chunkRow.chunk_index;
+            const pageStart = chunkRow.page_start;
+            const pageEnd = chunkRow.page_end;
+            try {
+                // Mark chunk processing
+                this.provider.run(`UPDATE intake_chunks SET status = 'processing' WHERE id = @id`, {
+                    id: chunkId,
+                });
+                // Stage 2: Extract page text (1-indexed → 0-indexed)
+                const chunkText = pages.slice(pageStart - 1, pageEnd).join('\n\n');
+                const citation = `${job.config.title}, pages ${pageStart}-${pageEnd}`;
+                // Stage 3: Classify
+                const classifiedItems = await classifyChunk(this.llm, chunkText, citation);
+                // Stage 4: Dedup
+                const dedupResults = dedupItems(classifiedItems, this.vault);
+                const uniqueItems = dedupResults.filter((r) => !r.isDuplicate);
+                const dupCount = dedupResults.filter((r) => r.isDuplicate).length;
+                // Stage 5: Store unique items in vault
+                let storedCount = 0;
+                for (let itemIdx = 0; itemIdx < uniqueItems.length; itemIdx++) {
+                    const result = uniqueItems[itemIdx];
+                    const entry = classifiedItemToEntry(result.item, job.config.domain, jobId, chunkIndex, itemIdx, job.config.tags);
+                    this.vault.add(entry);
+                    storedCount++;
+                }
+                // Update chunk record
+                const now = Math.floor(Date.now() / 1000);
+                this.provider.run(`UPDATE intake_chunks
+           SET status = 'completed', items_extracted = @extracted, items_stored = @stored, items_deduped = @deduped, processed_at = @now
+           WHERE id = @id`, {
+                    id: chunkId,
+                    extracted: classifiedItems.length,
+                    stored: storedCount,
+                    deduped: dupCount,
+                    now,
+                });
+                totalStored += storedCount;
+                totalDeduped += dupCount;
+                processed++;
+            }
+            catch (err) {
+                // Graceful degradation: mark chunk as failed, continue with others
+                const errorMsg = err instanceof Error ? err.message : String(err);
+                this.provider.run(`UPDATE intake_chunks SET status = 'failed', error = @error, processed_at = @now WHERE id = @id`, { id: chunkId, error: errorMsg, now: Math.floor(Date.now() / 1000) });
+                processed++;
+            }
+        }
+        // Update job timestamp
+        this.provider.run(`UPDATE intake_jobs SET updated_at = @now WHERE id = @id`, {
+            id: jobId,
+            now: Math.floor(Date.now() / 1000),
+        });
+        // Check remaining
+        const remaining = this.countPendingChunks(jobId);
+        if (remaining === 0) {
+            this.finalizeJob(jobId);
+        }
+        return { processed, itemsStored: totalStored, itemsDeduped: totalDeduped, remaining };
+    }
+    // ─── Preview ─────────────────────────────────────────────────────
+    /**
+     * Parse a page range from a PDF and classify it without storing.
+     * Useful for previewing what the pipeline would extract.
+     */
+    async preview(config, pageStart, pageEnd) {
+        const fileBuffer = readFileSync(config.pdfPath);
+        const pdfParse = (await import('pdf-parse')).default;
+        const pdfData = await pdfParse(fileBuffer);
+        const totalPages = pdfData.numpages;
+        const pages = splitIntoPages(pdfData.text, totalPages);
+        const chunkText = pages.slice(pageStart - 1, pageEnd).join('\n\n');
+        const citation = `${config.title}, pages ${pageStart}-${pageEnd}`;
+        const items = await classifyChunk(this.llm, chunkText, citation);
+        return {
+            items,
+            chunkText,
+            pageRange: { start: pageStart, end: pageEnd },
+        };
+    }
+    // ─── Queries ─────────────────────────────────────────────────────
+    /**
+     * Get a job record by ID.
+     */
+    getJob(jobId) {
+        const row = this.provider.get('SELECT * FROM intake_jobs WHERE id = @id', { id: jobId });
+        return row ? rowToJobRecord(row) : null;
+    }
+    /**
+     * List all intake jobs.
+     */
+    listJobs() {
+        const rows = this.provider.all('SELECT * FROM intake_jobs ORDER BY created_at DESC');
+        return rows.map(rowToJobRecord);
+    }
+    /**
+     * Get all chunks for a job.
+     */
+    getChunks(jobId) {
+        const rows = this.provider.all('SELECT * FROM intake_chunks WHERE job_id = @jobId ORDER BY chunk_index ASC', { jobId });
+        return rows.map(rowToChunk);
+    }
+    // ─── Stage 6: Finalize ──────────────────────────────────────────
+    /**
+     * Sum stats from all chunks and mark the job as completed.
+     */
+    finalizeJob(jobId) {
+        const chunks = this.provider.all('SELECT * FROM intake_chunks WHERE job_id = @jobId', { jobId });
+        let itemsExtracted = 0;
+        let itemsStored = 0;
+        let itemsDeduped = 0;
+        let itemsFailed = 0;
+        for (const chunk of chunks) {
+            const status = chunk.status;
+            if (status === 'completed') {
+                itemsExtracted += chunk.items_extracted ?? 0;
+                itemsStored += chunk.items_stored ?? 0;
+                itemsDeduped += chunk.items_deduped ?? 0;
+            }
+            else if (status === 'failed') {
+                itemsFailed++;
+            }
+        }
+        const stats = { itemsExtracted, itemsStored, itemsDeduped, itemsFailed };
+        const now = Math.floor(Date.now() / 1000);
+        this.provider.run(`UPDATE intake_jobs SET status = 'completed', stats = @stats, updated_at = @now, completed_at = @now WHERE id = @id`, { id: jobId, stats: JSON.stringify(stats), now });
+    }
+    // ─── Private helpers ─────────────────────────────────────────────
+    countPendingChunks(jobId) {
+        const result = this.provider.get(`SELECT COUNT(*) as count FROM intake_chunks WHERE job_id = @jobId AND status = 'pending'`, { jobId });
+        return result?.count ?? 0;
+    }
+}
+// =============================================================================
+// ROW MAPPERS
+// =============================================================================
+function rowToJobRecord(row) {
+    return {
+        id: row.id,
+        status: row.status,
+        config: JSON.parse(row.config),
+        pdfMeta: row.pdf_meta ? JSON.parse(row.pdf_meta) : null,
+        toc: row.toc ? JSON.parse(row.toc) : null,
+        stats: row.stats ? JSON.parse(row.stats) : null,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+        completedAt: row.completed_at ?? null,
+    };
+}
+function rowToChunk(row) {
+    return {
+        id: row.id,
+        jobId: row.job_id,
+        chunkIndex: row.chunk_index,
+        title: row.title ?? null,
+        pageStart: row.page_start,
+        pageEnd: row.page_end,
+        status: row.status,
+        itemsExtracted: row.items_extracted ?? 0,
+        itemsStored: row.items_stored ?? 0,
+        itemsDeduped: row.items_deduped ?? 0,
+        error: row.error ?? null,
+        processedAt: row.processed_at ?? null,
+    };
+}
+/**
+ * Convert a ClassifiedItem to an IntelligenceEntry for vault storage.
+ */
+function classifiedItemToEntry(item, domain, jobId, chunkIndex, itemIndex, extraTags) {
+    const entryType = mapKnowledgeType(item.type);
+    const tags = [...item.tags, ...(extraTags ?? [])];
+    return {
+        id: `intake-${jobId}-${chunkIndex}-${itemIndex}`,
+        type: entryType,
+        domain,
+        title: item.title,
+        severity: item.severity,
+        description: item.description,
+        context: item.citation,
+        tags,
+    };
+}
+//# sourceMappingURL=intake-pipeline.js.map