@steno-ai/engine 0.1.0

package/dist/extraction/llm-extractor.js

@@ -0,0 +1,238 @@
+import { buildFactExtractionPrompt, buildGraphExtractionPrompt } from './prompts.js';
+import { createEnrichedSegments } from './sliding-window.js';
+/**
+ * Two-pass extraction like Mem0:
+ * Pass 1: Extract facts as simple strings (focused, high quality)
+ * Pass 2: Extract entities + edges from the facts (separate concern)
+ */
+export async function extractWithLLM(config, input, existingFacts) {
+    let totalTokensIn = 0;
+    let totalTokensOut = 0;
+    // ── PASS 1: Fact extraction with Sliding Window Inference ──
+    // For long inputs, split into overlapping segments so the LLM can resolve
+    // pronouns and references using surrounding context (like Hydra DB).
+    const segments = createEnrichedSegments(input);
+    let factStrings = [];
+    let factEntries = [];
+    // Process segments (in parallel for speed, up to 4 at a time)
+    const segmentBatches = [];
+    for (let i = 0; i < segments.length; i += 4) {
+        segmentBatches.push(segments.slice(i, i + 4));
+    }
+    for (const batch of segmentBatches) {
+        const batchPromises = batch.map(async (seg) => {
+            const factMessages = buildFactExtractionPrompt(seg.contextWindow);
+            // Append existing facts for dedup context
+            if (existingFacts && existingFacts.length > 0) {
+                const factsBlock = existingFacts
+                    .map(f => `- [lineage: ${f.lineageId}] ${f.content}`)
+                    .join('\n');
+                factMessages[1].content += `\n\n--- EXISTING FACTS (skip duplicates, mark updates) ---\n${factsBlock}`;
+            }
+            // Also append already-extracted facts from previous segments to avoid duplicates
+            if (factEntries.length > 0) {
+                const alreadyExtracted = factEntries.map(f => `- ${f.text}`).join('\n');
+                factMessages[1].content += `\n\n--- ALREADY EXTRACTED (skip these) ---\n${alreadyExtracted}`;
+            }
+            try {
+                const factResponse = await config.llm.complete(factMessages, { temperature: 0, responseFormat: 'json' });
+                totalTokensIn += factResponse.tokensInput;
+                totalTokensOut += factResponse.tokensOutput;
+                const parsed = JSON.parse(factResponse.content);
+                const rawFacts = Array.isArray(parsed.facts) ? parsed.facts : [];
+                const entries = [];
+                for (const f of rawFacts) {
+                    if (typeof f === 'string') {
+                        const trimmed = f.trim();
+                        if (trimmed.length > 0)
+                            entries.push({ text: trimmed, importance: 0.5, sourceChunk: seg.contextWindow });
+                    }
+                    else if (f && typeof f === 'object') {
+                        const obj = f;
+                        const text = (typeof obj.t === 'string' ? obj.t : typeof obj.text === 'string' ? obj.text : '').trim();
+                        const importance = typeof obj.i === 'number' ? obj.i : typeof obj.importance === 'number' ? obj.importance : 0.5;
+                        const eventDate = obj.ed ? new Date(obj.ed) : undefined;
+                        const documentDate = obj.dd ? new Date(obj.dd) : undefined;
+                        if (text.length > 0)
+                            entries.push({
+                                text,
+                                importance: Math.max(0, Math.min(1, importance)),
+                                sourceChunk: seg.contextWindow,
+                                eventDate: eventDate && !isNaN(eventDate.getTime()) ? eventDate : undefined,
+                                documentDate: documentDate && !isNaN(documentDate.getTime()) ? documentDate : undefined,
+                            });
+                    }
+                }
+                return entries;
+            }
+            catch {
+                return [];
+            }
+        });
+        const batchResults = await Promise.all(batchPromises);
+        for (const entries of batchResults) {
+            factEntries.push(...entries);
+        }
+    }
+    // Deduplicate facts by content similarity (simple string match)
+    const seenContent = new Set();
+    factEntries = factEntries.filter(e => {
+        const key = e.text.toLowerCase().trim();
+        if (seenContent.has(key))
+            return false;
+        seenContent.add(key);
+        return true;
+    });
+    factStrings = factEntries.map(e => e.text);
+    if (factEntries.length === 0) {
+        return emptyResult(config.tier, config.llm.model);
+    }
+    if (factStrings.length === 0) {
+        return emptyResult(config.tier, config.llm.model);
+    }
+    // Build ExtractedFact objects from parsed entries with LLM-scored importance
+    const facts = factEntries.map(({ text, importance, sourceChunk, eventDate, documentDate }) => ({
+        content: text,
+        importance,
+        confidence: 0.8,
+        sourceType: 'conversation',
+        modality: 'text',
+        tags: [],
+        originalContent: input,
+        entityCanonicalNames: [],
+        sourceChunk,
+        eventDate,
+        documentDate,
+    }));
+    // ── Contextual memory wrappers ──
+    // Prepend source context so the embedding captures the full meaning.
+    // E.g. "User went to the gym" becomes "Context: <segment>... | Fact: User went to the gym"
+    // This is a transient field — only used at embedding time, never stored.
+    for (const fact of facts) {
+        const src = fact.sourceChunk ?? input;
+        const contextPrefix = src.length > 100
+            ? `Context: ${src.slice(0, 200).trim()}... | Fact: `
+            : `Context: ${src.trim()} | Fact: `;
+        fact.contextualContent = contextPrefix + fact.content;
+    }
+    // ── PASS 2: Graph extraction (entities + edges) from the facts ──
+    let entities = [];
+    let edges = [];
+    try {
+        const graphMessages = buildGraphExtractionPrompt(factStrings, config.entityTypes, config.domainEntityTypes);
+        const graphResponse = await config.llm.complete(graphMessages, { temperature: 0, responseFormat: 'json' });
+        totalTokensIn += graphResponse.tokensInput;
+        totalTokensOut += graphResponse.tokensOutput;
+        const graphParsed = JSON.parse(graphResponse.content);
+        // Parse entities
+        const seenEntities = new Set();
+        if (Array.isArray(graphParsed.entities)) {
+            for (const e of graphParsed.entities) {
+                if (!e || typeof e.name !== 'string')
+                    continue;
+                const entity = e;
+                const canonical = normalizeEntityName(entity.name);
+                if (canonical.length === 0 || seenEntities.has(canonical))
+                    continue;
+                seenEntities.add(canonical);
+                // Capture properties from domain entity types (e.g., {"company_size": "enterprise"})
+                const rawProps = entity.properties;
+                const properties = (rawProps && typeof rawProps === 'object' && !Array.isArray(rawProps))
+                    ? rawProps
+                    : {};
+                entities.push({
+                    name: canonical.split(' ').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' '),
+                    entityType: String(entity.entity_type ?? entity.type ?? 'concept'),
+                    canonicalName: canonical,
+                    properties,
+                });
+            }
+        }
+        // Parse edges
+        if (Array.isArray(graphParsed.edges)) {
+            for (const r of graphParsed.edges) {
+                if (!r)
+                    continue;
+                const rel = r;
+                const rawSource = typeof rel.source === 'string' ? rel.source :
+                    typeof rel.source_name === 'string' ? rel.source_name : null;
+                const rawTarget = typeof rel.target === 'string' ? rel.target :
+                    typeof rel.target_name === 'string' ? rel.target_name : null;
+                if (!rawSource || !rawTarget)
+                    continue;
+                const source = normalizeEntityName(rawSource);
+                const target = normalizeEntityName(rawTarget);
+                if (!source || !target)
+                    continue;
+                edges.push({
+                    sourceName: source,
+                    targetName: target,
+                    relation: String(rel.relation ?? 'related_to'),
+                    edgeType: isValidEdgeType(rel.edge_type) ? rel.edge_type : 'associative',
+                    confidence: 0.8,
+                });
+            }
+        }
+        // Link entities to facts by text match
+        for (const fact of facts) {
+            const contentLower = fact.content.toLowerCase();
+            for (const entity of entities) {
+                if (entity.canonicalName === 'user') {
+                    if (contentLower.startsWith('user ') || contentLower.includes(' user ')) {
+                        fact.entityCanonicalNames.push(entity.canonicalName);
+                    }
+                }
+                else if (entity.canonicalName.length >= 3 && contentLower.includes(entity.canonicalName)) {
+                    fact.entityCanonicalNames.push(entity.canonicalName);
+                }
+            }
+        }
+    }
+    catch {
+        // Graph pass failed — we still have facts, just no graph. That's OK.
+    }
+    return {
+        facts,
+        entities,
+        edges,
+        tier: config.tier,
+        confidence: 0.8,
+        tokensInput: totalTokensIn,
+        tokensOutput: totalTokensOut,
+        model: config.llm.model,
+    };
+}
+function emptyResult(tier, model) {
+    return {
+        facts: [],
+        entities: [],
+        edges: [],
+        tier,
+        confidence: 0,
+        tokensInput: 0,
+        tokensOutput: 0,
+        model,
+    };
+}
+function isValidEdgeType(t) {
+    return (typeof t === 'string' &&
+        ['associative', 'causal', 'temporal', 'contradictory', 'hierarchical'].includes(t));
+}
+/**
+ * Normalize an entity name to a clean canonical form.
+ */
+export function normalizeEntityName(raw) {
+    let name = raw.trim();
+    name = name.replace(/^[-–—*•#>]+\s*/g, '');
+    name = name.replace(/'s$/i, '');
+    name = name.replace(/\u2019s$/i, '');
+    name = name.replace(/^[^a-zA-Z0-9]+/, '');
+    name = name.replace(/[^a-zA-Z0-9]+$/, '');
+    const leadingNoise = /^(the|a|an|when|where|how|what|why|who|is|are|was|were|has|have|had|my|our|their|his|her|its|this|that|these|those)\s+/i;
+    name = name.replace(leadingNoise, '');
+    name = name.replace(leadingNoise, '');
+    name = name.replace(/\s+/g, ' ').trim();
+    name = name.toLowerCase();
+    return name;
+}
+//# sourceMappingURL=llm-extractor.js.map

package/dist/extraction/llm-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-extractor.js","sourceRoot":"","sources":["../../src/extraction/llm-extractor.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAyB,MAAM,cAAc,CAAC;AAC5G,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAS7D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAA0B,EAC1B,KAAa,EACb,aAA6D;IAE7D,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,8DAA8D;IAC9D,0EAA0E;IAC1E,qEAAqE;IACrE,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAE/C,IAAI,WAAW,GAAa,EAAE,CAAC;IAC/B,IAAI,WAAW,GAA4G,EAAE,CAAC;IAE9H,8DAA8D;IAC9D,MAAM,cAAc,GAAsB,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAC5C,MAAM,YAAY,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAElE,0CAA0C;YAC1C,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9C,MAAM,UAAU,GAAG,aAAa;qBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;qBACpD,IAAI,CAAC,IAAI,CAAC,CAAC;gBACd,YAAY,CAAC,CAAC,CAAE,CAAC,OAAO,IAAI,+DAA+D,UAAU,EAAE,CAAC;YAC1G,CAAC;YAED,iFAAiF;YACjF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxE,YAAY,CAAC,CAAC,CAAE,CAAC,OAAO,IAAI,+CAA+C,gBAAgB,EAAE,CAAC;YAChG,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzG,aAAa,IAAI,YAAY,CAAC,WAAW,CAAC;gBAC1C,cAAc,IAAI,YAAY,CAAC,YAAY,CAAC;gBAE5C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAA4B,CAAC;gBAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,MAAM,OAAO,GAA4G,EAAE,CAAC;gBAC5H,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACzB,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;wBACzB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;4BAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC;oBAC3G,CAAC;yBAAM,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;wBACtC,MAAM,GAAG,GAAG,CAA4B,CAAC;wBACzC,MAAM,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;wBACvG,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;wBACjH,MAAM,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,EAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;wBAClE,MAAM,YAAY,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,EAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;wBACrE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;4BAAE,OAAO,CAAC,IAAI,CAAC;gCAChC,IAAI;gCACJ,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gCAChD,WAAW,EAAE,GAAG,CAAC,aAAa;gCAC9B,SAAS,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;gCAC3E,YAAY,EAAE,YAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;6BACxF,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBACD,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACtD,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACnC,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACvC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE3C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,6EAA6E;IAC7E,MAAM,KAAK,GAAoB,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9G,OAAO,EAAE,IAAI;QACb,UAAU;QACV,UAAU,EAAE,GAAG;QACf,UAAU,EAAE,cAAuB;QACnC,QAAQ,EAAE,MAAe;QACzB,IAAI,EAAE,EAAE;QACR,eAAe,EAAE,KAAK;QACtB,oBAAoB,EAAE,EAAE;QACxB,WAAW;QACX,SAAS;QACT,YAAY;KACb,CAAC,CAAC,CAAC;IAEJ,mCAAmC;IACnC,qEAAqE;IACrE,2FAA2F;IAC3F,yEAAyE;IACzE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC;QACtC,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG;YACpC,CAAC,CAAC,YAAY,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,cAAc;YACpD,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC;QACtC,IAAI,CAAC,iBAAiB,GAAG,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC;IACxD,CAAC;IAED,mEAAmE;IACnE,IAAI,QAAQ,GAAsB,EAAE,CAAC;IACrC,IAAI,KAAK,GAAoB,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,0BAA0B,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC5G,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3G,aAAa,IAAI,aAAa,CAAC,WAAW,CAAC;QAC3C,cAAc,IAAI,aAAa,CAAC,YAAY,CAAC;QAE7C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAA4B,CAAC;QAEjF,iBAAiB;QACjB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,QAAqB,EAAE,CAAC;gBAClD,IAAI,CAAC,CAAC,IAAI,OAAQ,CAA6B,CAAC,IAAI,KAAK,QAAQ;oBAAE,SAAS;gBAC5E,MAAM,MAAM,GAAG,CAA4B,CAAC;gBAC5C,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,IAAc,CAAC,CAAC;gBAC7D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC;oBAAE,SAAS;gBACpE,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC5B,qFAAqF;gBACrF,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC;gBACnC,MAAM,UAAU,GAA4B,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAChH,CAAC,CAAC,QAAmC;oBACrC,CAAC,CAAC,EAAE,CAAC;gBAEP,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;oBACrF,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC;oBAClE,aAAa,EAAE,SAAS;oBACxB,UAAU;iBACX,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,cAAc;QACd,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,KAAkB,EAAE,CAAC;gBAC/C,IAAI,CAAC,CAAC;oBAAE,SAAS;gBACjB,MAAM,GAAG,GAAG,CAA4B,CAAC;gBACzC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9C,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC9E,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9C,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC9E,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS;oBAAE,SAAS;gBACvC,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;gBAC9C,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM;oBAAE,SAAS;gBACjC,KAAK,CAAC,IAAI,CAAC;oBACT,UAAU,EAAE,MAAM;oBAClB,UAAU,EAAE,MAAM;oBAClB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,YAAY,CAAC;oBAC9C,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;oBACxE,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,uCAAuC;QACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAChD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC9B,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;oBACpC,IAAI,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACxE,IAAI,CAAC,oBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACxD,CAAC;gBACH,CAAC;qBAAM,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;oBAC3F,IAAI,CAAC,oBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qEAAqE;IACvE,CAAC;IAED,OAAO;QACL,KAAK;QACL,QAAQ;QACR,KAAK;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,aAAa;QAC1B,YAAY,EAAE,cAAc;QAC5B,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK;KACxB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAoB,EAAE,KAAa;IACtD,OAAO;QACL,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,IAAI;QACJ,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,CAAU;IACjC,OAAO,CACL,OAAO,CAAC,KAAK,QAAQ;QACrB,CAAC,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,eAAe,EAAE,cAAc,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CACnF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IAC3C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACrC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,yHAAyH,CAAC;IAC/I,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC1B,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/extraction/pipeline.d.ts

@@ -0,0 +1,30 @@
+import type { StorageAdapter } from '../adapters/storage.js';
+import type { EmbeddingAdapter } from '../adapters/embedding.js';
+import type { LLMAdapter } from '../adapters/llm.js';
+import type { DomainEntityType } from '../config.js';
+import type { ExtractionInput, PipelineResult, ExtractedFact, ExtractedEntity } from './types.js';
+export interface PipelineConfig {
+    storage: StorageAdapter;
+    embedding: EmbeddingAdapter;
+    cheapLLM: LLMAdapter;
+    smartLLM?: LLMAdapter;
+    extractionTier?: 'heuristic_only' | 'cheap_only' | 'auto' | 'smart_only';
+    embeddingModel: string;
+    embeddingDim: number;
+    decayHalfLifeDays?: number;
+    decayNormalizationK?: number;
+    entityTypes?: string[];
+    domainEntityTypes?: DomainEntityType[];
+}
+export declare function inputToText(input: ExtractionInput): string;
+export declare function mergeFacts(heuristic: ExtractedFact[], llm: ExtractedFact[]): ExtractedFact[];
+export declare function mergeEntities(heuristic: ExtractedEntity[], llm: ExtractedEntity[]): ExtractedEntity[];
+export declare function runExtractionPipeline(config: PipelineConfig, input: ExtractionInput): Promise<PipelineResult>;
+/**
+ * Run extraction for a pre-created extraction record (from queue).
+ * Unlike runExtractionPipeline, this does NOT create the extraction record
+ * or perform hash-based dedup — both were already handled by the API route.
+ * It updates the existing record through the pipeline lifecycle.
+ */
+export declare function runExtractionFromQueue(config: PipelineConfig, extractionId: string, input: ExtractionInput): Promise<PipelineResult>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/extraction/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/extraction/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAc,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EAEhB,MAAM,YAAY,CAAC;AASpB,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,cAAc,CAAC;IACxB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,QAAQ,EAAE,UAAU,CAAC;IACrB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,cAAc,CAAC,EAAE,gBAAgB,GAAG,YAAY,GAAG,MAAM,GAAG,YAAY,CAAC;IACzE,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACxC;AAMD,wBAAgB,WAAW,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAiB1D;AAED,wBAAgB,UAAU,CACxB,SAAS,EAAE,aAAa,EAAE,EAC1B,GAAG,EAAE,aAAa,EAAE,GACnB,aAAa,EAAE,CAIjB;AAED,wBAAgB,aAAa,CAC3B,SAAS,EAAE,eAAe,EAAE,EAC5B,GAAG,EAAE,eAAe,EAAE,GACrB,eAAe,EAAE,CAQnB;AAwUD,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,cAAc,CAAC,CA0EzB;AAMD;;;;;GAKG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,cAAc,EACtB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,eAAe,GACrB,OAAO,CAAC,cAAc,CAAC,CAiBzB"}