@steno-ai/engine 0.1.0

package/src/extraction/heuristic.ts

@@ -0,0 +1,341 @@
+import nlp from 'compromise';
+import type { ExtractionResult, ExtractedFact, ExtractedEntity } from './types.js';
+
+// ---------------------------------------------------------------------------
+// Regex patterns
+// ---------------------------------------------------------------------------
+
+const REGEX = {
+  email: /\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b/g,
+  phone: /(?:\+?1[\s\-.]?)?\(?\d{3}\)?[\s\-.]?\d{3}[\s\-.]?\d{4}\b/g,
+  url: /https?:\/\/[^\s/$.?#].[^\s]*/g,
+  date: /\b(?:\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4}|\d{4}[\/\-]\d{2}[\/\-]\d{2}|(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}|\d{1,2}(?:st|nd|rd|th)?\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{4})\b/gi,
+  money: /\$\s?\d+(?:[.,]\d+)*(?:\.\d{2})?|\b\d+(?:[.,]\d+)*(?:\.\d{2})?\s?(?:dollars?|USD|EUR|GBP|euros?|pounds?)\b/gi,
+};
+
+// ---------------------------------------------------------------------------
+// Pattern rules for personal facts
+// ---------------------------------------------------------------------------
+
+interface PatternRule {
+  pattern: RegExp;
+  template: (match: RegExpMatchArray) => string;
+  importance: number;
+  tags: string[];
+}
+
+const PATTERN_RULES: PatternRule[] = [
+  // Health / allergy — importance 0.95
+  {
+    pattern: /\bi(?:'m| am)\s+allergic\s+to\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User is allergic to ${m[1]!.trim()}`,
+    importance: 0.95,
+    tags: ['health', 'allergy'],
+  },
+  {
+    pattern: /\bi\s+have\s+(?:a\s+)?(?:allergy|allergies)\s+to\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User is allergic to ${m[1]!.trim()}`,
+    importance: 0.95,
+    tags: ['health', 'allergy'],
+  },
+  {
+    pattern: /\bi\s+(?:have|suffer from|was diagnosed with)\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User has ${m[1]!.trim()}`,
+    importance: 0.95,
+    tags: ['health'],
+  },
+
+  // Name — importance 0.9
+  {
+    pattern: /\bmy\s+name\s+is\s+([A-Za-z][a-zA-Z\s\-']{1,40}?)(?:[.!?,]|$)/i,
+    template: (m) => `User's name is ${m[1]!.trim()}`,
+    importance: 0.9,
+    tags: ['identity', 'name'],
+  },
+  {
+    pattern: /\bthey\s+call\s+me\s+([A-Za-z][a-zA-Z\s\-']{1,40}?)(?:[.!?,]|$)/i,
+    template: (m) => `User's name is ${m[1]!.trim()}`,
+    importance: 0.9,
+    tags: ['identity', 'name'],
+  },
+
+  // Identity — importance 0.85
+  {
+    pattern: /\bi(?:'m| am)\s+a(?:n)?\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User is a ${m[1]!.trim()}`,
+    importance: 0.85,
+    tags: ['identity'],
+  },
+  {
+    pattern: /\bi(?:'m| am)\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)(?:[.!?,]|$)/,
+    template: (m) => `User is ${m[1]!.trim()}`,
+    importance: 0.85,
+    tags: ['identity'],
+  },
+
+  // Work / company — importance 0.8
+  {
+    pattern: /\bi\s+work\s+(?:at|for)\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User works at ${m[1]!.trim()}`,
+    importance: 0.8,
+    tags: ['work', 'company'],
+  },
+  {
+    pattern: /\bmy\s+(?:job|career|profession|occupation)\s+is\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User's job is ${m[1]!.trim()}`,
+    importance: 0.8,
+    tags: ['work'],
+  },
+  {
+    pattern: /\bi\s+(?:work|am employed)\s+as\s+(?:a(?:n)?\s+)?(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User works as ${m[1]!.trim()}`,
+    importance: 0.8,
+    tags: ['work'],
+  },
+
+  // Location — importance 0.7
+  {
+    pattern: /\bi\s+live\s+in\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User lives in ${m[1]!.trim()}`,
+    importance: 0.7,
+    tags: ['location'],
+  },
+  {
+    pattern: /\bi(?:'m| am)\s+from\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User is from ${m[1]!.trim()}`,
+    importance: 0.7,
+    tags: ['location'],
+  },
+  {
+    pattern: /\bi\s+(?:moved|relocated)\s+to\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User moved to ${m[1]!.trim()}`,
+    importance: 0.7,
+    tags: ['location'],
+  },
+
+  // Preferences (like/love/enjoy) — importance 0.6
+  {
+    pattern: /\bi\s+(?:really\s+)?(?:like|love|enjoy|adore)\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User likes ${m[1]!.trim()}`,
+    importance: 0.6,
+    tags: ['preference', 'like'],
+  },
+  {
+    pattern: /\bmy\s+favorite\s+(?:\w+\s+)?is\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User's favorite is ${m[1]!.trim()}`,
+    importance: 0.6,
+    tags: ['preference', 'like'],
+  },
+
+  // Dislikes — importance 0.6
+  {
+    pattern: /\bi\s+(?:really\s+)?(?:hate|dislike|can'?t stand|despise|detest)\s+(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User dislikes ${m[1]!.trim()}`,
+    importance: 0.6,
+    tags: ['preference', 'dislike'],
+  },
+
+  // Trivia / other — importance 0.3
+  {
+    pattern: /\bi\s+(?:think|believe|feel|guess|suppose)\s+(?:that\s+)?(.+?)(?:[.!?]|$)/i,
+    template: (m) => `User thinks ${m[1]!.trim()}`,
+    importance: 0.3,
+    tags: ['opinion'],
+  },
+];
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Strip a "role: " prefix like "user: " or "assistant: " from a line. */
+function stripRolePrefix(line: string): string {
+  return line.replace(/^[a-zA-Z_\-]+:\s*/, '');
+}
+
+function makeFact(
+  content: string,
+  originalContent: string,
+  importance: number,
+  confidence: number,
+  tags: string[],
+  entityCanonicalNames: string[] = [],
+): ExtractedFact {
+  return {
+    content,
+    importance,
+    confidence,
+    sourceType: 'conversation',
+    modality: 'text',
+    tags,
+    originalContent,
+    operation: 'add',
+    entityCanonicalNames,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Regex extraction (runs on the full text)
+// ---------------------------------------------------------------------------
+
+function extractRegex(text: string): ExtractedFact[] {
+  const facts: ExtractedFact[] = [];
+  const seenContents = new Set<string>();
+
+  function addFact(content: string, original: string, tags: string[]): void {
+    if (!seenContents.has(content)) {
+      seenContents.add(content);
+      facts.push(makeFact(content, original, 0.8, 0.9, tags));
+    }
+  }
+
+  // Emails
+  for (const match of text.matchAll(REGEX.email)) {
+    const email = match[0]!;
+    addFact(`User's email is ${email}`, email, ['contact', 'email']);
+  }
+
+  // Phone numbers
+  for (const match of text.matchAll(REGEX.phone)) {
+    const phone = match[0]!;
+    addFact(`User's phone number is ${phone}`, phone, ['contact', 'phone']);
+  }
+
+  // URLs
+  for (const match of text.matchAll(REGEX.url)) {
+    const url = match[0]!;
+    addFact(`User mentioned URL: ${url}`, url, ['url']);
+  }
+
+  // Dates
+  for (const match of text.matchAll(REGEX.date)) {
+    const date = match[0]!;
+    addFact(`Mentioned date: ${date}`, date, ['date']);
+  }
+
+  // Money
+  for (const match of text.matchAll(REGEX.money)) {
+    const amount = match[0]!;
+    addFact(`Mentioned monetary amount: ${amount}`, amount, ['money']);
+  }
+
+  return facts;
+}
+
+// ---------------------------------------------------------------------------
+// Pattern extraction (runs line by line, role prefix stripped)
+// ---------------------------------------------------------------------------
+
+function extractPatterns(lines: string[]): ExtractedFact[] {
+  const facts: ExtractedFact[] = [];
+  const seenContents = new Set<string>();
+
+  for (const rawLine of lines) {
+    const line = stripRolePrefix(rawLine).trim();
+    if (!line) continue;
+
+    for (const rule of PATTERN_RULES) {
+      // Reset lastIndex for global regexes (these aren't global, but be safe)
+      const match = line.match(rule.pattern);
+      if (match) {
+        const content = rule.template(match);
+        if (!seenContents.has(content)) {
+          seenContents.add(content);
+          facts.push(makeFact(content, line, rule.importance, 0.7, rule.tags));
+        }
+        // Only use the first matching rule per line to avoid overlapping facts
+        break;
+      }
+    }
+  }
+
+  return facts;
+}
+
+// ---------------------------------------------------------------------------
+// NER via compromise
+// ---------------------------------------------------------------------------
+
+function extractEntities(text: string): ExtractedEntity[] {
+  const doc = nlp(text);
+  const entities: ExtractedEntity[] = [];
+  const seenCanonical = new Set<string>();
+
+  function addEntity(name: string, entityType: string): void {
+    const trimmed = name.trim();
+    if (!trimmed) return;
+    const canonical = trimmed.toLowerCase();
+    if (!seenCanonical.has(`${entityType}:${canonical}`)) {
+      seenCanonical.add(`${entityType}:${canonical}`);
+      entities.push({
+        name: trimmed,
+        entityType,
+        canonicalName: canonical,
+        properties: {},
+      });
+    }
+  }
+
+  // People
+  const people = doc.people().out('array') as string[];
+  for (const name of people) {
+    addEntity(name, 'person');
+  }
+
+  // Organizations
+  const orgs = doc.organizations().out('array') as string[];
+  for (const org of orgs) {
+    addEntity(org, 'organization');
+  }
+
+  // Places
+  const places = doc.places().out('array') as string[];
+  for (const place of places) {
+    addEntity(place, 'location');
+  }
+
+  return entities;
+}
+
+// ---------------------------------------------------------------------------
+// Main export
+// ---------------------------------------------------------------------------
+
+export function extractHeuristic(text: string): ExtractionResult {
+  const lines = text.split('\n');
+
+  // 1. Regex extraction — run on full text
+  const regexFacts = extractRegex(text);
+  const regexContents = new Set(regexFacts.map((f) => f.content));
+
+  // 2. Pattern extraction — line by line, strip role prefix
+  const patternFacts = extractPatterns(lines);
+
+  // 3. De-duplicate: remove pattern facts whose content overlaps with regex facts
+  //    (e.g., if regex grabbed the email and a pattern also mentions it)
+  const filteredPatternFacts = patternFacts.filter((f) => !regexContents.has(f.content));
+
+  // 4. NER entities via compromise
+  const entities = extractEntities(text);
+
+  const facts = [...regexFacts, ...filteredPatternFacts];
+
+  // Overall confidence: weighted average or fixed 0.9 for regex, 0.7 for pattern
+  // Use the maximum confidence across extracted facts, or 0.9 if only regex results exist
+  const confidence = facts.length > 0
+    ? Math.max(...facts.map((f) => f.confidence))
+    : entities.length > 0
+      ? 0.6
+      : 0.5;
+
+  return {
+    facts,
+    entities,
+    edges: [],
+    tier: 'heuristic',
+    confidence,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}

package/src/extraction/index.ts

@@ -0,0 +1,9 @@
+export * from './types.js';
+export * from './hasher.js';
+export * from './prompts.js';
+export * from './heuristic.js';
+export * from './contradiction.js';
+export * from './entity-extractor.js';
+export * from './llm-extractor.js';
+export * from './dedup.js';
+export * from './pipeline.js';

package/src/extraction/llm-extractor.d.ts

@@ -0,0 +1,21 @@
+import type { LLMAdapter } from '../adapters/llm.js';
+import type { ExtractionResult } from './types.js';
+import type { ExtractionTier } from '../config.js';
+export interface LLMExtractorConfig {
+    llm: LLMAdapter;
+    tier: ExtractionTier;
+}
+/**
+ * Two-pass extraction like Mem0:
+ * Pass 1: Extract facts as simple strings (focused, high quality)
+ * Pass 2: Extract entities + edges from the facts (separate concern)
+ */
+export declare function extractWithLLM(config: LLMExtractorConfig, input: string, existingFacts?: Array<{
+    lineageId: string;
+    content: string;
+}>): Promise<ExtractionResult>;
+/**
+ * Normalize an entity name to a clean canonical form.
+ */
+export declare function normalizeEntityName(raw: string): string;
+//# sourceMappingURL=llm-extractor.d.ts.map

package/src/extraction/llm-extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-extractor.d.ts","sourceRoot":"","sources":["llm-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAE,gBAAgB,EAAiD,MAAM,YAAY,CAAC;AAClG,OAAO,KAAK,EAAE,cAAc,EAAY,MAAM,cAAc,CAAC;AAI7D,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,UAAU,CAAC;IAChB,IAAI,EAAE,cAAc,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,aAAa,CAAC,EAAE,KAAK,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAC5D,OAAO,CAAC,gBAAgB,CAAC,CAiL3B;AAsBD;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAavD"}

package/src/extraction/llm-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-extractor.js","sourceRoot":"","sources":["llm-extractor.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAyB,MAAM,cAAc,CAAC;AAC5G,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAO7D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAA0B,EAC1B,KAAa,EACb,aAA6D;IAE7D,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,8DAA8D;IAC9D,0EAA0E;IAC1E,qEAAqE;IACrE,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAE/C,IAAI,WAAW,GAAa,EAAE,CAAC;IAC/B,IAAI,WAAW,GAAgD,EAAE,CAAC;IAElE,8DAA8D;IAC9D,MAAM,cAAc,GAAsB,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAC5C,MAAM,YAAY,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAElE,0CAA0C;YAC1C,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9C,MAAM,UAAU,GAAG,aAAa;qBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;qBACpD,IAAI,CAAC,IAAI,CAAC,CAAC;gBACd,YAAY,CAAC,CAAC,CAAE,CAAC,OAAO,IAAI,+DAA+D,UAAU,EAAE,CAAC;YAC1G,CAAC;YAED,iFAAiF;YACjF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxE,YAAY,CAAC,CAAC,CAAE,CAAC,OAAO,IAAI,+CAA+C,gBAAgB,EAAE,CAAC;YAChG,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzG,aAAa,IAAI,YAAY,CAAC,WAAW,CAAC;gBAC1C,cAAc,IAAI,YAAY,CAAC,YAAY,CAAC;gBAE5C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAA4B,CAAC;gBAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,MAAM,OAAO,GAAgD,EAAE,CAAC;gBAChE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACzB,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;wBAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;wBACzB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;4BAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC3E,CAAC;yBAAM,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;wBACtC,MAAM,GAAG,GAAG,CAA4B,CAAC;wBACzC,MAAM,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;wBACvG,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;wBACjH,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;4BAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;oBAChG,CAAC;gBACH,CAAC;gBACD,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACtD,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACnC,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACvC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAE3C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,6EAA6E;IAC7E,MAAM,KAAK,GAAoB,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;QACxE,OAAO,EAAE,IAAI;QACb,UAAU;QACV,UAAU,EAAE,GAAG;QACf,UAAU,EAAE,cAAuB;QACnC,QAAQ,EAAE,MAAe;QACzB,IAAI,EAAE,EAAE;QACR,eAAe,EAAE,KAAK;QACtB,oBAAoB,EAAE,EAAE;KACzB,CAAC,CAAC,CAAC;IAEJ,mEAAmE;IACnE,IAAI,QAAQ,GAAsB,EAAE,CAAC;IACrC,IAAI,KAAK,GAAoB,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,0BAA0B,CAAC,WAAW,CAAC,CAAC;QAC9D,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3G,aAAa,IAAI,aAAa,CAAC,WAAW,CAAC;QAC3C,cAAc,IAAI,aAAa,CAAC,YAAY,CAAC;QAE7C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAA4B,CAAC;QAEjF,iBAAiB;QACjB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxC,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,QAAqB,EAAE,CAAC;gBAClD,IAAI,CAAC,CAAC,IAAI,OAAQ,CAA6B,CAAC,IAAI,KAAK,QAAQ;oBAAE,SAAS;gBAC5E,MAAM,MAAM,GAAG,CAA4B,CAAC;gBAC5C,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,IAAc,CAAC,CAAC;gBAC7D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC;oBAAE,SAAS;gBACpE,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC5B,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;oBACrF,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC;oBAClE,aAAa,EAAE,SAAS;oBACxB,UAAU,EAAE,EAAE;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,cAAc;QACd,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,KAAkB,EAAE,CAAC;gBAC/C,IAAI,CAAC,CAAC;oBAAE,SAAS;gBACjB,MAAM,GAAG,GAAG,CAA4B,CAAC;gBACzC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9C,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC9E,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBAC9C,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC9E,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS;oBAAE,SAAS;gBACvC,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;gBAC9C,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM;oBAAE,SAAS;gBACjC,KAAK,CAAC,IAAI,CAAC;oBACT,UAAU,EAAE,MAAM;oBAClB,UAAU,EAAE,MAAM;oBAClB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,YAAY,CAAC;oBAC9C,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;oBACxE,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,uCAAuC;QACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAChD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC9B,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;oBACpC,IAAI,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACxE,IAAI,CAAC,oBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACxD,CAAC;gBACH,CAAC;qBAAM,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;oBAC3F,IAAI,CAAC,oBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qEAAqE;IACvE,CAAC;IAED,OAAO;QACL,KAAK;QACL,QAAQ;QACR,KAAK;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,aAAa;QAC1B,YAAY,EAAE,cAAc;QAC5B,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK;KACxB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAoB,EAAE,KAAa;IACtD,OAAO;QACL,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,IAAI;QACJ,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,CAAU;IACjC,OAAO,CACL,OAAO,CAAC,KAAK,QAAQ;QACrB,CAAC,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,eAAe,EAAE,cAAc,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CACnF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IAC3C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACrC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,yHAAyH,CAAC;IAC/I,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC1B,OAAO,IAAI,CAAC;AACd,CAAC"}

package/src/extraction/llm-extractor.ts

@@ -0,0 +1,267 @@
+import type { LLMAdapter } from '../adapters/llm.js';
+import type { ExtractionResult, ExtractedFact, ExtractedEntity, ExtractedEdge } from './types.js';
+import type { ExtractionTier, EdgeType, DomainEntityType } from '../config.js';
+import { buildFactExtractionPrompt, buildGraphExtractionPrompt, buildExtractionPrompt } from './prompts.js';
+import { createEnrichedSegments } from './sliding-window.js';
+
+export interface LLMExtractorConfig {
+  llm: LLMAdapter;
+  tier: ExtractionTier;
+  entityTypes?: string[];
+  domainEntityTypes?: DomainEntityType[];
+}
+
+/**
+ * Two-pass extraction like Mem0:
+ * Pass 1: Extract facts as simple strings (focused, high quality)
+ * Pass 2: Extract entities + edges from the facts (separate concern)
+ */
+export async function extractWithLLM(
+  config: LLMExtractorConfig,
+  input: string,
+  existingFacts?: Array<{ lineageId: string; content: string }>,
+): Promise<ExtractionResult> {
+  let totalTokensIn = 0;
+  let totalTokensOut = 0;
+
+  // ── PASS 1: Fact extraction with Sliding Window Inference ──
+  // For long inputs, split into overlapping segments so the LLM can resolve
+  // pronouns and references using surrounding context (like Hydra DB).
+  const segments = createEnrichedSegments(input);
+
+  let factStrings: string[] = [];
+  let factEntries: Array<{ text: string; importance: number; sourceChunk: string; eventDate?: Date; documentDate?: Date }> = [];
+
+  // Process segments (in parallel for speed, up to 4 at a time)
+  const segmentBatches: typeof segments[] = [];
+  for (let i = 0; i < segments.length; i += 4) {
+    segmentBatches.push(segments.slice(i, i + 4));
+  }
+
+  for (const batch of segmentBatches) {
+    const batchPromises = batch.map(async (seg) => {
+      const factMessages = buildFactExtractionPrompt(seg.contextWindow);
+
+      // Append existing facts for dedup context
+      if (existingFacts && existingFacts.length > 0) {
+        const factsBlock = existingFacts
+          .map(f => `- [lineage: ${f.lineageId}] ${f.content}`)
+          .join('\n');
+        factMessages[1]!.content += `\n\n--- EXISTING FACTS (skip duplicates, mark updates) ---\n${factsBlock}`;
+      }
+
+      // Also append already-extracted facts from previous segments to avoid duplicates
+      if (factEntries.length > 0) {
+        const alreadyExtracted = factEntries.map(f => `- ${f.text}`).join('\n');
+        factMessages[1]!.content += `\n\n--- ALREADY EXTRACTED (skip these) ---\n${alreadyExtracted}`;
+      }
+
+      try {
+        const factResponse = await config.llm.complete(factMessages, { temperature: 0, responseFormat: 'json' });
+        totalTokensIn += factResponse.tokensInput;
+        totalTokensOut += factResponse.tokensOutput;
+
+        const parsed = JSON.parse(factResponse.content) as Record<string, unknown>;
+        const rawFacts = Array.isArray(parsed.facts) ? parsed.facts : [];
+        const entries: Array<{ text: string; importance: number; sourceChunk: string; eventDate?: Date; documentDate?: Date }> = [];
+        for (const f of rawFacts) {
+          if (typeof f === 'string') {
+            const trimmed = f.trim();
+            if (trimmed.length > 0) entries.push({ text: trimmed, importance: 0.5, sourceChunk: seg.contextWindow });
+          } else if (f && typeof f === 'object') {
+            const obj = f as Record<string, unknown>;
+            const text = (typeof obj.t === 'string' ? obj.t : typeof obj.text === 'string' ? obj.text : '').trim();
+            const importance = typeof obj.i === 'number' ? obj.i : typeof obj.importance === 'number' ? obj.importance : 0.5;
+            const eventDate = obj.ed ? new Date(obj.ed as string) : undefined;
+            const documentDate = obj.dd ? new Date(obj.dd as string) : undefined;
+            if (text.length > 0) entries.push({
+              text,
+              importance: Math.max(0, Math.min(1, importance)),
+              sourceChunk: seg.contextWindow,
+              eventDate: eventDate && !isNaN(eventDate.getTime()) ? eventDate : undefined,
+              documentDate: documentDate && !isNaN(documentDate.getTime()) ? documentDate : undefined,
+            });
+          }
+        }
+        return entries;
+      } catch {
+        return [];
+      }
+    });
+
+    const batchResults = await Promise.all(batchPromises);
+    for (const entries of batchResults) {
+      factEntries.push(...entries);
+    }
+  }
+
+  // Deduplicate facts by content similarity (simple string match)
+  const seenContent = new Set<string>();
+  factEntries = factEntries.filter(e => {
+    const key = e.text.toLowerCase().trim();
+    if (seenContent.has(key)) return false;
+    seenContent.add(key);
+    return true;
+  });
+  factStrings = factEntries.map(e => e.text);
+
+  if (factEntries.length === 0) {
+    return emptyResult(config.tier, config.llm.model);
+  }
+
+  if (factStrings.length === 0) {
+    return emptyResult(config.tier, config.llm.model);
+  }
+
+  // Build ExtractedFact objects from parsed entries with LLM-scored importance
+  const facts: ExtractedFact[] = factEntries.map(({ text, importance, sourceChunk, eventDate, documentDate }) => ({
+    content: text,
+    importance,
+    confidence: 0.8,
+    sourceType: 'conversation' as const,
+    modality: 'text' as const,
+    tags: [],
+    originalContent: input,
+    entityCanonicalNames: [],
+    sourceChunk,
+    eventDate,
+    documentDate,
+  }));
+
+  // ── Contextual memory wrappers ──
+  // Prepend source context so the embedding captures the full meaning.
+  // E.g. "User went to the gym" becomes "Context: <segment>... | Fact: User went to the gym"
+  // This is a transient field — only used at embedding time, never stored.
+  for (const fact of facts) {
+    const src = fact.sourceChunk ?? input;
+    const contextPrefix = src.length > 100
+      ? `Context: ${src.slice(0, 200).trim()}... | Fact: `
+      : `Context: ${src.trim()} | Fact: `;
+    fact.contextualContent = contextPrefix + fact.content;
+  }
+
+  // ── PASS 2: Graph extraction (entities + edges) from the facts ──
+  let entities: ExtractedEntity[] = [];
+  let edges: ExtractedEdge[] = [];
+
+  try {
+    const graphMessages = buildGraphExtractionPrompt(factStrings, config.entityTypes, config.domainEntityTypes);
+    const graphResponse = await config.llm.complete(graphMessages, { temperature: 0, responseFormat: 'json' });
+    totalTokensIn += graphResponse.tokensInput;
+    totalTokensOut += graphResponse.tokensOutput;
+
+    const graphParsed = JSON.parse(graphResponse.content) as Record<string, unknown>;
+
+    // Parse entities
+    const seenEntities = new Set<string>();
+    if (Array.isArray(graphParsed.entities)) {
+      for (const e of graphParsed.entities as unknown[]) {
+        if (!e || typeof (e as Record<string, unknown>).name !== 'string') continue;
+        const entity = e as Record<string, unknown>;
+        const canonical = normalizeEntityName(entity.name as string);
+        if (canonical.length === 0 || seenEntities.has(canonical)) continue;
+        seenEntities.add(canonical);
+        // Capture properties from domain entity types (e.g., {"company_size": "enterprise"})
+        const rawProps = entity.properties;
+        const properties: Record<string, unknown> = (rawProps && typeof rawProps === 'object' && !Array.isArray(rawProps))
+          ? rawProps as Record<string, unknown>
+          : {};
+
+        entities.push({
+          name: canonical.split(' ').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' '),
+          entityType: String(entity.entity_type ?? entity.type ?? 'concept'),
+          canonicalName: canonical,
+          properties,
+        });
+      }
+    }
+
+    // Parse edges
+    if (Array.isArray(graphParsed.edges)) {
+      for (const r of graphParsed.edges as unknown[]) {
+        if (!r) continue;
+        const rel = r as Record<string, unknown>;
+        const rawSource = typeof rel.source === 'string' ? rel.source :
+                         typeof rel.source_name === 'string' ? rel.source_name : null;
+        const rawTarget = typeof rel.target === 'string' ? rel.target :
+                         typeof rel.target_name === 'string' ? rel.target_name : null;
+        if (!rawSource || !rawTarget) continue;
+        const source = normalizeEntityName(rawSource);
+        const target = normalizeEntityName(rawTarget);
+        if (!source || !target) continue;
+        edges.push({
+          sourceName: source,
+          targetName: target,
+          relation: String(rel.relation ?? 'related_to'),
+          edgeType: isValidEdgeType(rel.edge_type) ? rel.edge_type : 'associative',
+          confidence: 0.8,
+        });
+      }
+    }
+
+    // Link entities to facts by text match
+    for (const fact of facts) {
+      const contentLower = fact.content.toLowerCase();
+      for (const entity of entities) {
+        if (entity.canonicalName === 'user') {
+          if (contentLower.startsWith('user ') || contentLower.includes(' user ')) {
+            fact.entityCanonicalNames!.push(entity.canonicalName);
+          }
+        } else if (entity.canonicalName.length >= 3 && contentLower.includes(entity.canonicalName)) {
+          fact.entityCanonicalNames!.push(entity.canonicalName);
+        }
+      }
+    }
+  } catch {
+    // Graph pass failed — we still have facts, just no graph. That's OK.
+  }
+
+  return {
+    facts,
+    entities,
+    edges,
+    tier: config.tier,
+    confidence: 0.8,
+    tokensInput: totalTokensIn,
+    tokensOutput: totalTokensOut,
+    model: config.llm.model,
+  };
+}
+
+function emptyResult(tier: ExtractionTier, model: string): ExtractionResult {
+  return {
+    facts: [],
+    entities: [],
+    edges: [],
+    tier,
+    confidence: 0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model,
+  };
+}
+
+function isValidEdgeType(t: unknown): t is EdgeType {
+  return (
+    typeof t === 'string' &&
+    ['associative', 'causal', 'temporal', 'contradictory', 'hierarchical'].includes(t)
+  );
+}
+
+/**
+ * Normalize an entity name to a clean canonical form.
+ */
+export function normalizeEntityName(raw: string): string {
+  let name = raw.trim();
+  name = name.replace(/^[-–—*•#>]+\s*/g, '');
+  name = name.replace(/'s$/i, '');
+  name = name.replace(/\u2019s$/i, '');
+  name = name.replace(/^[^a-zA-Z0-9]+/, '');
+  name = name.replace(/[^a-zA-Z0-9]+$/, '');
+  const leadingNoise = /^(the|a|an|when|where|how|what|why|who|is|are|was|were|has|have|had|my|our|their|his|her|its|this|that|these|those)\s+/i;
+  name = name.replace(leadingNoise, '');
+  name = name.replace(leadingNoise, '');
+  name = name.replace(/\s+/g, ' ').trim();
+  name = name.toLowerCase();
+  return name;
+}

package/src/extraction/pipeline.d.ts

@@ -0,0 +1,27 @@
+import type { StorageAdapter } from '../adapters/storage.js';
+import type { EmbeddingAdapter } from '../adapters/embedding.js';
+import type { LLMAdapter } from '../adapters/llm.js';
+import type { ExtractionInput, PipelineResult, ExtractedFact, ExtractedEntity } from './types.js';
+export interface PipelineConfig {
+    storage: StorageAdapter;
+    embedding: EmbeddingAdapter;
+    cheapLLM: LLMAdapter;
+    smartLLM?: LLMAdapter;
+    extractionTier?: 'heuristic_only' | 'cheap_only' | 'auto' | 'smart_only';
+    embeddingModel: string;
+    embeddingDim: number;
+    decayHalfLifeDays?: number;
+    decayNormalizationK?: number;
+}
+export declare function inputToText(input: ExtractionInput): string;
+export declare function mergeFacts(heuristic: ExtractedFact[], llm: ExtractedFact[]): ExtractedFact[];
+export declare function mergeEntities(heuristic: ExtractedEntity[], llm: ExtractedEntity[]): ExtractedEntity[];
+export declare function runExtractionPipeline(config: PipelineConfig, input: ExtractionInput): Promise<PipelineResult>;
+/**
+ * Run extraction for a pre-created extraction record (from queue).
+ * Unlike runExtractionPipeline, this does NOT create the extraction record
+ * or perform hash-based dedup — both were already handled by the API route.
+ * It updates the existing record through the pipeline lifecycle.
+ */
+export declare function runExtractionFromQueue(config: PipelineConfig, extractionId: string, input: ExtractionInput): Promise<PipelineResult>;
+//# sourceMappingURL=pipeline.d.ts.map