@vellumai/assistant 0.5.6 → 0.5.8

package/src/memory/items-extractor.ts

@@ -1,21 +1,24 @@
-import { and, eq, like, sql } from "drizzle-orm";
+import { and, desc, eq, like, sql } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
 import { getConfig } from "../config/loader.js";
 import type { MemoryExtractionConfig } from "../config/types.js";
+import { getAssistantName } from "../daemon/identity-helpers.js";
+import { resolveGuardianPersona } from "../prompts/persona-resolver.js";
 import { buildCoreIdentityContext } from "../prompts/system-prompt.js";
 import {
-  createTimeout,
   extractToolUse,
   getConfiguredProvider,
   userMessage,
 } from "../providers/provider-send-message.js";
+import { BackendUnavailableError } from "../util/errors.js";
 import { getLogger } from "../util/logger.js";
 import { truncate } from "../util/truncate.js";
 import { maybeEnqueueConversationStartersJob } from "./conversation-starters-cadence.js";
 import { getDb } from "./db.js";
 import { computeMemoryFingerprint } from "./fingerprint.js";
 import { enqueueMemoryJob } from "./jobs-store.js";
+import { upsertJournalMemoriesFromDisk } from "./journal-memory.js";
 import { extractTextFromStoredMessageContent } from "./message-content.js";
 import { withQdrantBreaker } from "./qdrant-circuit-breaker.js";
 import { getQdrantClient } from "./qdrant-client.js";
@@ -31,7 +34,8 @@ export type MemoryItemKind =
   | "project"
   | "decision"
   | "constraint"
-  | "event";
+  | "event"
+  | "journal";
 
 export type OverrideConfidence = "explicit" | "tentative" | "inferred";
 
@@ -55,8 +59,16 @@ const VALID_KINDS = new Set<string>([
   "decision",
   "constraint",
   "event",
+  "journal",
 ]);
 
+/**
+ * Kinds the LLM is allowed to produce during extraction. Excludes "journal"
+ * because journal memories are created directly from disk files — any
+ * LLM-produced journal items would be silently dropped, wasting tokens.
+ */
+const EXTRACTION_KINDS = [...VALID_KINDS].filter((k) => k !== "journal");
+
 /** Maps old kind names to their new equivalents for graceful migration. */
 const KIND_MIGRATION_MAP: Record<string, MemoryItemKind> = {
   profile: "identity",
@@ -150,6 +162,7 @@ function buildExtractionSystemPrompt(
     statement: string;
   }>,
   messageRole: string,
+  userPersona?: string | null,
 ): string {
   // Build the fixed instruction body first so we can measure it and allocate
   // the remaining budget to identity context.
@@ -182,17 +195,57 @@ For each item, provide:
 Rules:
 - Only extract genuinely memorable information. Skip pleasantries, filler, and transient discussion.
 - Do NOT extract information about what tools the assistant used or what files it read — only extract substantive facts about the user, their projects, and their preferences.
-- Do NOT extract claims about actions the assistant performed, outcomes it achieved, or progress it reported (e.g., "I booked an appointment", "I sent the email"). Only extract facts stated by the user or from external sources — the assistant's self-reports are not reliable memory material.
 - Do NOT extract raw code snippets, JSON fragments, YAML, configuration values, log output, or data structures. Only extract the human-readable meaning or intent behind such content, not the literal syntax.
 - Prefer fewer high-quality items over many low-quality ones.
-- If the message contains no memorable information, return an empty array.`;
+- If the message contains no memorable information, return an empty array.
+- The preceding conversation context (if provided) is for disambiguation only. Extract items ONLY from the final message after the --- separator, not from the context messages.`;
+
+  // Try to extract user name from persona text
+  let userName = "the user";
+  if (userPersona) {
+    const nameMatch = userPersona.match(/\*\*Name:\*\*\s*(.+)/);
+    if (nameMatch) {
+      userName = nameMatch[1].trim();
+    }
+  }
 
   if (messageRole === "assistant") {
     instructions += `
 
-IMPORTANT: The message below is from the ASSISTANT, not the user. Do NOT attribute the assistant's own statements, feelings, self-descriptions, or introspection to the user. Only extract facts about the user, the world, or the project that the assistant is referencing or relaying — NOT the assistant's own identity, uncertainty, or behavior. If the assistant is simply talking about itself (e.g., introducing itself, expressing uncertainty about its own purpose), extract nothing.`;
+IMPORTANT: The message below is from the ASSISTANT. You may extract facts about actions taken, decisions made, and outcomes achieved. However, do NOT attribute the assistant's own identity, personality, or self-descriptions to the user. If the assistant is just introducing itself or expressing uncertainty about its own nature, extract nothing.`;
   }
 
+  instructions += `
+
+## Examples
+
+Good extractions from user messages:
+- "I'm a backend engineer at Acme Corp, mostly working with Go and PostgreSQL"
+  → kind: identity, subject: "Role at Acme Corp", statement: "${userName} is a backend engineer at Acme Corp, works primarily with Go and PostgreSQL"
+
+- "Always use semantic commits in this repo. I hate squash merges."
+  → kind: constraint, subject: "Git conventions", statement: "${userName} requires semantic commit messages. Strongly dislikes squash merges."
+
+- "We decided to go with Redis for the cache layer because DynamoDB was too expensive at our read volume"
+  → kind: decision, subject: "Cache layer choice", statement: "${userName} chose Redis over DynamoDB for caching due to cost at high read volumes"
+
+Good extractions from assistant messages:
+- "Based on your earlier mention, I see you're using Next.js 14 with the app router for the dashboard project."
+  → kind: project, subject: "Dashboard tech stack", statement: "${userName}'s dashboard project uses Next.js 14 with the app router"
+
+- "Since you mentioned your team follows trunk-based development, I'll keep the changes in a single commit."
+  → kind: constraint, subject: "Team branching strategy", statement: "${userName}'s team follows trunk-based development"
+
+- "I've refactored the auth middleware to use JWT validation and added rate limiting to the login endpoint."
+  → kind: project, subject: "Auth middleware changes", statement: "Auth middleware was refactored to use JWT validation with rate limiting on the login endpoint"
+
+Do NOT extract:
+- "I'll check that file for you" → assistant operational statement with no lasting information
+- "I think the best approach would be to refactor this" → speculative, no action taken yet
+- "The tests passed" → transient status
+- "Sure, sounds good" → filler
+- "\`\`\`json {"key": "val"} \`\`\`" → raw code/data, extract meaning not syntax`;
+
   if (existingItems.length > 0) {
     instructions += `\n\nExisting memory items (use these to identify supersession targets — set \`supersedes\` to the item ID if the new information replaces one of these):\n`;
     for (const item of existingItems) {
@@ -204,9 +257,10 @@ IMPORTANT: The message below is from the ASSISTANT, not the user. Do NOT attribu
   // generic "User ..." labels. Budget is dynamically computed: whatever
   // remains after the fixed instructions fits within the system prompt
   // ceiling, preventing oversized prompts from exceeding the provider input
-  // window (which would cause sendMessage to error and fall back to
-  // lower-quality pattern-based extraction).
-  const rawIdentityContext = buildCoreIdentityContext();
+  // window (which would cause sendMessage to error).
+  const rawIdentityContext = buildCoreIdentityContext(
+    userPersona ? { userPersona } : undefined,
+  );
 
   let prompt = "";
   if (rawIdentityContext) {
@@ -316,186 +370,200 @@ async function extractItemsWithLLM(
   extractionConfig: MemoryExtractionConfig,
   scopeId: string,
   messageRole: string,
+  precedingMessages: Array<{ role: string; content: string }>,
+  userPersona?: string | null,
 ): Promise<ExtractedItem[]> {
   const provider = await getConfiguredProvider();
   if (!provider) {
-    log.debug(
-      "Configured provider unavailable for LLM extraction, falling back to pattern-based",
+    throw new BackendUnavailableError(
+      "Provider unavailable for memory extraction",
     );
-    return extractItemsPatternBased(text, scopeId);
   }
 
-  try {
-    const { signal, cleanup } = createTimeout(15000);
-
-    try {
-      // Query existing items to give the LLM supersession context
-      const existingItems = queryExistingItemsForContext(scopeId, text);
-      const systemPrompt = buildExtractionSystemPrompt(
-        existingItems,
-        messageRole,
-      );
+  // Query existing items to give the LLM supersession context
+  const existingItems = queryExistingItemsForContext(scopeId, text);
+  const systemPrompt = buildExtractionSystemPrompt(
+    existingItems,
+    messageRole,
+    userPersona,
+  );
+
+  const assistantName = getAssistantName() ?? "the assistant";
+  const messagePrefix =
+    messageRole === "assistant"
+      ? `[This message is from ${assistantName}]\n\n`
+      : `[This message is from the user]\n\n`;
+
+  // Build user content with optional preceding conversation context
+  const contextParts: string[] = [];
+  for (const msg of precedingMessages) {
+    const msgText = extractTextFromStoredMessageContent(msg.content);
+    if (msgText.length === 0) continue;
+    const roleLabel =
+      msg.role === "assistant"
+        ? (getAssistantName() ?? "assistant")
+        : "user";
+    contextParts.push(`[${roleLabel}]: ${msgText}`);
+  }
+  let userContent = `${messagePrefix}${text}`;
+  if (contextParts.length > 0) {
+    userContent = `Preceding conversation context:\n${contextParts.join("\n\n")}\n\n---\n\nMessage to extract from:\n${messagePrefix}${text}`;
+  }
 
-      const messagePrefix =
-        messageRole === "assistant"
-          ? "[This message is from the assistant]\n\n"
-          : "";
-      const response = await provider.sendMessage(
-        [userMessage(`${messagePrefix}${text}`)],
-        [
-          {
-            name: "store_memory_items",
-            description: "Store extracted memory items from the message",
-            input_schema: {
-              type: "object" as const,
-              properties: {
-                items: {
-                  type: "array",
-                  items: {
-                    type: "object",
-                    properties: {
-                      kind: {
-                        type: "string",
-                        enum: [...VALID_KINDS],
-                        description: "Category of memory item",
-                      },
-                      subject: {
-                        type: "string",
-                        description:
-                          "Short label (2-8 words) for what this is about",
-                      },
-                      statement: {
-                        type: "string",
-                        description:
-                          "Relationship-rich factual statement to remember (1-2 sentences). Include relational context.",
-                      },
-                      confidence: {
-                        type: "number",
-                        description:
-                          "Confidence that this is accurate (0.0-1.0)",
-                      },
-                      importance: {
-                        type: "number",
-                        description:
-                          "How valuable this is to remember (0.0-1.0)",
-                      },
-                      supersedes: {
-                        type: ["string", "null"],
-                        description:
-                          "ID of the existing memory item this replaces, or null if not replacing anything",
-                      },
-                      overrideConfidence: {
-                        type: "string",
-                        enum: ["explicit", "tentative", "inferred"],
-                        description:
-                          "How confident you are that this overrides an existing item: explicit (clear override), tentative (ambiguous), inferred (weak signal)",
-                      },
-                    },
-                    required: [
-                      "kind",
-                      "subject",
-                      "statement",
-                      "confidence",
-                      "importance",
-                      "supersedes",
-                      "overrideConfidence",
-                    ],
+  const response = await provider.sendMessage(
+    [userMessage(userContent)],
+    [
+      {
+        name: "store_memory_items",
+        description: "Store extracted memory items from the message",
+        input_schema: {
+          type: "object" as const,
+          properties: {
+            items: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  kind: {
+                    type: "string",
+                    enum: EXTRACTION_KINDS,
+                    description: "Category of memory item",
+                  },
+                  subject: {
+                    type: "string",
+                    description:
+                      "Short label (2-8 words) for what this is about",
+                  },
+                  statement: {
+                    type: "string",
+                    description:
+                      "Relationship-rich factual statement to remember (1-2 sentences). Include relational context.",
+                  },
+                  confidence: {
+                    type: "number",
+                    description: "Confidence that this is accurate (0.0-1.0)",
+                  },
+                  importance: {
+                    type: "number",
+                    description: "How valuable this is to remember (0.0-1.0)",
+                  },
+                  supersedes: {
+                    type: ["string", "null"],
+                    description:
+                      "ID of the existing memory item this replaces, or null if not replacing anything",
+                  },
+                  overrideConfidence: {
+                    type: "string",
+                    enum: ["explicit", "tentative", "inferred"],
+                    description:
+                      "How confident you are that this overrides an existing item: explicit (clear override), tentative (ambiguous), inferred (weak signal)",
                   },
                 },
+                required: [
+                  "kind",
+                  "subject",
+                  "statement",
+                  "confidence",
+                  "importance",
+                  "supersedes",
+                  "overrideConfidence",
+                ],
               },
-              required: ["items"],
             },
           },
-        ],
-        systemPrompt,
-        {
-          config: {
-            modelIntent: extractionConfig.modelIntent,
-            max_tokens: 1024,
-            tool_choice: { type: "tool" as const, name: "store_memory_items" },
-          },
-          signal,
+          required: ["items"],
         },
-      );
-      cleanup();
+      },
+    ],
+    systemPrompt,
+    {
+      config: {
+        modelIntent: extractionConfig.modelIntent,
+        tool_choice: { type: "tool" as const, name: "store_memory_items" },
+      },
+    },
+  );
 
-      const toolBlock = extractToolUse(response);
-      if (!toolBlock) {
-        log.warn(
-          "No tool_use block in LLM extraction response, falling back to pattern-based",
-        );
-        return extractItemsPatternBased(text, scopeId);
-      }
+  const toolBlock = extractToolUse(response);
+  if (!toolBlock) {
+    throw new Error("No tool_use block in LLM extraction response");
+  }
 
-      const input = toolBlock.input as { items?: LLMExtractedItem[] };
-      if (!Array.isArray(input.items)) {
-        log.warn(
-          "Invalid items in LLM extraction response, falling back to pattern-based",
-        );
-        return extractItemsPatternBased(text, scopeId);
-      }
+  const input = toolBlock.input as { items?: LLMExtractedItem[] };
+  if (!Array.isArray(input.items)) {
+    throw new Error("Invalid items structure in LLM extraction response");
+  }
 
-      // Build set of known existing item IDs for supersession validation
-      const existingItemIds = new Set(existingItems.map((e) => e.id));
-
-      const items: ExtractedItem[] = [];
-      for (const raw of input.items) {
-        // Apply kind migration map for old kind names, then validate
-        const resolvedKind = KIND_MIGRATION_MAP[raw.kind] ?? raw.kind;
-        if (!VALID_KINDS.has(resolvedKind)) continue;
-        if (!raw.subject || !raw.statement) continue;
-        const subject = truncate(String(raw.subject), 80, "");
-        const statement = truncate(String(raw.statement), 500, "");
-        const confidence = clampUnitInterval(parseScore(raw.confidence, 0.5));
-        const importance = clampUnitInterval(parseScore(raw.importance, 0.5));
-        const fingerprint = computeMemoryFingerprint(
-          scopeId,
-          resolvedKind,
-          subject,
-          statement,
-        );
+  // Build set of known existing item IDs for supersession validation
+  const existingItemIds = new Set(existingItems.map((e) => e.id));
 
-        // Validate supersedes: must reference a known existing item ID.
-        // Reject hallucinated IDs that don't match any item we showed the LLM.
-        const rawSupersedes =
-          typeof raw.supersedes === "string" && raw.supersedes.length > 0
-            ? raw.supersedes
-            : null;
-        const supersedes =
-          rawSupersedes && existingItemIds.has(rawSupersedes)
-            ? rawSupersedes
-            : null;
-        const supersedesRejected = !!rawSupersedes && !supersedes;
-        const overrideConfidence = VALID_OVERRIDE_CONFIDENCES.has(
-          raw.overrideConfidence,
-        )
-          ? (raw.overrideConfidence as OverrideConfidence)
-          : "inferred";
-
-        items.push({
-          kind: resolvedKind as MemoryItemKind,
-          subject,
-          statement,
-          confidence,
-          importance,
-          fingerprint,
-          supersedes,
-          overrideConfidence,
-          supersedesRejected,
-        });
-      }
+  const items: ExtractedItem[] = [];
+  for (const raw of input.items) {
+    // Apply kind migration map for old kind names, then validate
+    const resolvedKind = KIND_MIGRATION_MAP[raw.kind] ?? raw.kind;
+    if (resolvedKind === "journal") continue; // journal memories created directly from disk
+    if (!VALID_KINDS.has(resolvedKind)) continue;
+    if (!raw.subject || !raw.statement) continue;
+    const subject = String(raw.subject).trim();
+    const statement = String(raw.statement).trim();
+    const confidence = clampUnitInterval(parseScore(raw.confidence, 0.5));
+    const importance = clampUnitInterval(parseScore(raw.importance, 0.5));
+    const fingerprint = computeMemoryFingerprint(
+      scopeId,
+      resolvedKind,
+      subject,
+      statement,
+    );
 
-      return deduplicateItems(items);
-    } finally {
-      cleanup();
-    }
+    // Validate supersedes: must reference a known existing item ID.
+    // Reject hallucinated IDs that don't match any item we showed the LLM.
+    const rawSupersedes =
+      typeof raw.supersedes === "string" && raw.supersedes.length > 0
+        ? raw.supersedes
+        : null;
+    const supersedes =
+      rawSupersedes && existingItemIds.has(rawSupersedes)
+        ? rawSupersedes
+        : null;
+    const supersedesRejected = !!rawSupersedes && !supersedes;
+    const overrideConfidence = VALID_OVERRIDE_CONFIDENCES.has(
+      raw.overrideConfidence,
+    )
+      ? (raw.overrideConfidence as OverrideConfidence)
+      : "inferred";
+
+    items.push({
+      kind: resolvedKind as MemoryItemKind,
+      subject,
+      statement,
+      confidence,
+      importance,
+      fingerprint,
+      supersedes,
+      overrideConfidence,
+      supersedesRejected,
+    });
+  }
+
+  return deduplicateItems(items);
+}
+
+/**
+ * Fire conversation starters generation when journal memories were created.
+ * Wrapped in try/catch so failures never propagate to the caller.
+ */
+function triggerConversationStartersIfNeeded(
+  count: number,
+  scopeId: string,
+): void {
+  if (count <= 0) return;
+  try {
+    maybeEnqueueConversationStartersJob(scopeId);
   } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
     log.warn(
-      { err: message },
-      "LLM extraction failed, falling back to pattern-based",
+      { err: err instanceof Error ? err.message : String(err) },
+      "Failed to check conversation starters cadence",
     );
-    return extractItemsPatternBased(text, scopeId);
   }
 }
 
@@ -513,6 +581,7 @@ export async function extractAndUpsertMemoryItemsForMessage(
       role: messages.role,
       content: messages.content,
       createdAt: messages.createdAt,
+      conversationId: messages.conversationId,
     })
     .from(messages)
     .where(eq(messages.id, messageId))
@@ -520,28 +589,77 @@ export async function extractAndUpsertMemoryItemsForMessage(
 
   if (!message) return 0;
 
+  // Fetch up to 6 preceding messages from the same conversation for
+  // disambiguation context (e.g. resolving "that framework" or "yes, do it").
+  const effectiveConversationId = conversationId ?? message.conversationId;
+  const precedingMessages = effectiveConversationId
+    ? db
+        .select({ role: messages.role, content: messages.content })
+        .from(messages)
+        .where(
+          and(
+            eq(messages.conversationId, effectiveConversationId),
+            sql`${messages.createdAt} < ${message.createdAt}`,
+          ),
+        )
+        .orderBy(desc(messages.createdAt))
+        .limit(6)
+        .all()
+        .reverse()
+    : [];
+
+  const effectiveScopeId = scopeId ?? "default";
+
+  // Directly create journal memories from any journal files written during
+  // this message, bypassing LLM extraction (which would summarize/rewrite them).
+  // This must run before the extraction guards (semantic density, useLLM, etc.)
+  // because journal disk scanning is independent of LLM extraction.
+  let journalUpserted = 0;
+  if (message.role === "assistant") {
+    journalUpserted = upsertJournalMemoriesFromDisk(
+      message.createdAt,
+      effectiveScopeId,
+      messageId,
+    );
+  }
+
   const text = extractTextFromStoredMessageContent(message.content);
   if (!hasSemanticDensity(text)) {
     log.debug(
       { messageId },
       "Skipping extraction — message lacks semantic density",
     );
-    return 0;
+    triggerConversationStartersIfNeeded(journalUpserted, effectiveScopeId);
+    return journalUpserted;
   }
 
   const config = getConfig();
   const extractionConfig = config.memory.extraction;
-  const effectiveScopeId = scopeId ?? "default";
-  const extracted = extractionConfig.useLLM
-    ? await extractItemsWithLLM(
-        text,
-        extractionConfig,
-        effectiveScopeId,
-        message.role,
-      )
-    : extractItemsPatternBased(text, effectiveScopeId);
 
-  if (extracted.length === 0) return 0;
+  // Resolve the guardian's persona to provide personality-aware extraction
+  // context. Currently uses the guardian persona for all conversations —
+  // non-guardian conversations are rare and the guardian's persona provides
+  // better extraction context than none.
+  const userPersona = resolveGuardianPersona();
+
+  if (!extractionConfig.useLLM) {
+    triggerConversationStartersIfNeeded(journalUpserted, effectiveScopeId);
+    return journalUpserted;
+  }
+
+  const extracted = await extractItemsWithLLM(
+    text,
+    extractionConfig,
+    effectiveScopeId,
+    message.role,
+    precedingMessages,
+    userPersona,
+  );
+
+  if (extracted.length === 0) {
+    triggerConversationStartersIfNeeded(journalUpserted, effectiveScopeId);
+    return journalUpserted;
+  }
 
   // Guard: re-check after the async LLM call. The event loop yields during
   // extractItemsWithLLM, so another task could have marked the conversation
@@ -551,13 +669,10 @@ export async function extractAndUpsertMemoryItemsForMessage(
       { messageId, conversationId },
       "Skipping upsert — conversation marked failed during extraction",
     );
-    return 0;
+    triggerConversationStartersIfNeeded(journalUpserted, effectiveScopeId);
+    return journalUpserted;
   }
 
-  // Determine verification state from message role
-  const verificationState =
-    message.role === "user" ? "user_reported" : "assistant_inferred";
-
   let upserted = 0;
   for (const item of extracted) {
     const now = Date.now();
@@ -577,13 +692,21 @@ export async function extractAndUpsertMemoryItemsForMessage(
     let effectiveStatus: string = "active";
     if (existing) {
       memoryItemId = existing.id;
-      // Promote verification state if re-seen from a more trusted source
-      const promotedState =
-        existing.verificationState === "assistant_inferred" &&
-        verificationState === "user_reported"
-          ? "user_reported"
-          : existing.verificationState;
       effectiveStatus = "active";
+      // Preserve sourceType for tool-sourced items — extraction should not
+      // demote items the user explicitly saved.
+      const effectiveSourceType =
+        existing.sourceType === "tool" ? "tool" : "extraction";
+
+      // Dual-write verificationState alongside sourceType for client compat.
+      // Promote from assistant_inferred → user_reported when re-seen from user.
+      const effectiveVerificationState =
+        message.role === "user" || existing.verificationState === "user_reported"
+          ? "user_reported"
+          : existing.verificationState === "user_confirmed"
+            ? "user_confirmed"
+            : "assistant_inferred";
+
       db.update(memoryItems)
         .set({
           status: effectiveStatus,
@@ -594,7 +717,9 @@ export async function extractAndUpsertMemoryItemsForMessage(
             Math.max(existing.importance ?? 0, item.importance),
           ),
           lastSeenAt: Math.max(existing.lastSeenAt, seenAt),
-          verificationState: promotedState,
+          sourceType: effectiveSourceType,
+          sourceMessageRole: message.role,
+          verificationState: effectiveVerificationState,
         })
         .where(eq(memoryItems.id, existing.id))
         .run();
@@ -610,7 +735,11 @@ export async function extractAndUpsertMemoryItemsForMessage(
           confidence: item.confidence,
           importance: item.importance,
           fingerprint: item.fingerprint,
-          verificationState,
+          sourceType: "extraction",
+          sourceMessageRole: message.role,
+          // Dual-write verificationState for client compat
+          verificationState:
+            message.role === "user" ? "user_reported" : "assistant_inferred",
           scopeId: effectiveScopeId,
           firstSeenAt: message.createdAt,
           lastSeenAt: seenAt,
@@ -701,11 +830,9 @@ export async function extractAndUpsertMemoryItemsForMessage(
     }
 
     // Fallback subject-match supersession: only when the LLM did not
-    // explicitly handle supersession for this item. This preserves the
-    // original behavior for pattern-based extraction and items without
-    // LLM-directed supersession. Skip items whose supersedes ID was
-    // rejected (hallucinated) — they should coexist, not trigger
-    // subject-based replacement.
+    // explicitly handle supersession for this item. Skip items whose
+    // supersedes ID was rejected (hallucinated) — they should coexist,
+    // not trigger subject-based replacement.
     if (
       !item.supersedes &&
       !item.supersedesRejected &&
@@ -730,7 +857,7 @@ export async function extractAndUpsertMemoryItemsForMessage(
       .values({
         memoryItemId,
         messageId,
-        evidence: truncate(item.statement, 500, ""),
+        evidence: item.statement,
         createdAt: now,
       })
       .onConflictDoNothing()
@@ -739,139 +866,19 @@ export async function extractAndUpsertMemoryItemsForMessage(
     enqueueMemoryJob("embed_item", { itemId: memoryItemId });
   }
 
+  upserted += journalUpserted;
+
   log.debug(
     { messageId, extracted: extracted.length, upserted },
     "Extracted memory items from message",
   );
 
   // Trigger conversation starters generation when new items are upserted
-  if (upserted > 0) {
-    try {
-      maybeEnqueueConversationStartersJob(effectiveScopeId);
-    } catch (err) {
-      log.warn(
-        { err: err instanceof Error ? err.message : String(err) },
-        "Failed to check conversation starters cadence",
-      );
-    }
-  }
+  triggerConversationStartersIfNeeded(upserted, effectiveScopeId);
 
   return upserted;
 }
 
-// ── Pattern-based extraction (fallback) ────────────────────────────────
-
-function extractItemsPatternBased(
-  text: string,
-  scopeId: string = "default",
-): ExtractedItem[] {
-  const sentences = text
-    .split(/[\n\r]+|(?<=[.!?])\s+/)
-    .map((s) => s.trim())
-    .filter((s) => s.length >= 20 && s.length <= 500);
-
-  const items: ExtractedItem[] = [];
-  for (const sentence of sentences) {
-    const lower = sentence.toLowerCase();
-    const classification = classifySentence(lower);
-    if (!classification) continue;
-    const subject = inferSubject(sentence, classification.kind);
-    const statement = sentence.replace(/\s+/g, " ").trim();
-    const fingerprint = computeMemoryFingerprint(
-      scopeId,
-      classification.kind,
-      subject,
-      statement,
-    );
-    items.push({
-      kind: classification.kind,
-      subject,
-      statement,
-      confidence: classification.confidence,
-      importance: classification.importance,
-      fingerprint,
-      supersedes: null,
-      overrideConfidence: "inferred" as OverrideConfidence,
-    });
-  }
-
-  return deduplicateItems(items);
-}
-
-function classifySentence(
-  lower: string,
-): { kind: MemoryItemKind; confidence: number; importance: number } | null {
-  if (
-    includesAny(lower, [
-      "i prefer",
-      "prefer to",
-      "favorite",
-      "i like",
-      "i dislike",
-    ])
-  ) {
-    return { kind: "preference", confidence: 0.78, importance: 0.7 };
-  }
-  if (
-    includesAny(lower, [
-      "my name is",
-      "i am ",
-      "i work as",
-      "i live in",
-      "timezone",
-    ])
-  ) {
-    return { kind: "identity", confidence: 0.72, importance: 0.8 };
-  }
-  if (includesAny(lower, ["project", "repository", "repo", "codebase"])) {
-    return { kind: "project", confidence: 0.68, importance: 0.6 };
-  }
-  if (
-    includesAny(lower, ["we decided", "decision", "chosen approach", "we will"])
-  ) {
-    return { kind: "decision", confidence: 0.75, importance: 0.7 };
-  }
-  if (
-    includesAny(lower, ["todo", "to do", "next step", "follow up", "need to"])
-  ) {
-    return { kind: "project", confidence: 0.74, importance: 0.6 };
-  }
-  if (
-    includesAny(lower, [
-      "must",
-      "cannot",
-      "should not",
-      "constraint",
-      "requirement",
-    ])
-  ) {
-    return { kind: "constraint", confidence: 0.7, importance: 0.7 };
-  }
-  if (includesAny(lower, ["remember", "important", "fact", "noted"])) {
-    return { kind: "identity", confidence: 0.62, importance: 0.5 };
-  }
-  return null;
-}
-
-function inferSubject(sentence: string, kind: MemoryItemKind): string {
-  const trimmed = sentence.trim();
-  if (kind === "project") {
-    const match = trimmed.match(
-      /(?:project|repo(?:sitory)?)\s+([A-Za-z0-9._/-]{2,80})/i,
-    );
-    if (match) return match[1];
-  }
-  const words = trimmed.split(/\s+/).slice(0, 6).join(" ");
-  return truncate(words, 80, "");
-}
-
-function includesAny(text: string, needles: string[]): boolean {
-  for (const needle of needles) {
-    if (text.includes(needle)) return true;
-  }
-  return false;
-}
-
 // ── Helpers ────────────────────────────────────────────────────────────
 
 function deduplicateItems(items: ExtractedItem[]): ExtractedItem[] {