@remnic/core 9.49.0 → 9.50.0

package/src/extraction-prompt.ts

@@ -18,6 +18,7 @@ export const EXTRACTION_RESPONSE_SHAPE = `{
     "confidence": 0.0,
     "tags": ["<tag>"],
     "entityRef": "<optional normalized-name>",
+    "cueAnchors": [{"type": "<anchor-type>", "value": "<entity-or-topic plus key aspect>"}],
     "promptedByQuestion": "<optional source-grounded question>",
     "quote": "<optional exact contiguous source span>",
     "scope": "<optional project-or-global>",
@@ -40,12 +41,18 @@ export const EXTRACTION_RESPONSE_SHAPE = `{
   "profileUpdates": ["<source-grounded profile update>"],
   "questions": [{"question": "<source-grounded unresolved question>", "context": "<source-grounded context>", "priority": 0.0}],
   "identityReflection": "<conversation-grounded agent reflection>",
+  "episodeTitle": "<six-to-eight word conversation segment title>",
   "relationships": [{"source": "<normalized-name>", "target": "<normalized-name>", "label": "<source-grounded relationship>"}]
 }`;
 export const EXTRACTION_RESPONSE_PLACEHOLDERS: Record<string, true> = {};
 for (const placeholder of EXTRACTION_RESPONSE_SHAPE.match(/<[^<>\r\n]+>/g) ?? []) {
   EXTRACTION_RESPONSE_PLACEHOLDERS[placeholder] = true;
 }
+export const CUE_ANCHOR_PROMPT_INSTRUCTION = `- For each fact, emit at most 3 "cueAnchors".
+- Each cue anchor value must be a source-grounded search hook of at most 120 characters, in the form "<main entity or topic> <key aspect>".
+- Set each cue anchor type to entity, file, tool, outcome, constraint, or date.
+- Omit cue anchors when the conversation does not support a useful search hook.
+- Example: "Alice booked Bistro Max for Mike's surprise party" can use {"type":"entity","value":"Mike surprise party"} and {"type":"entity","value":"Alice restaurant booking"}.`;
 
 /**
  * Bi-temporal event-time extraction instruction (#1578 PR2). Emitted on
@@ -67,7 +74,11 @@ When a fact states when it became or stopped being true, copy that explicit temp
  * `ambientCapture` marks input from an always-on recorder, which may carry
  * speech the user never authored (issue #2294).
  */
-export function buildExtractionInstructions(config: PluginConfig, existingEntities?: string[], ambientCapture = false): string {
+export function buildExtractionInstructions(
+  config: PluginConfig,
+  existingEntities?: string[],
+  ambientCapture = false
+): string {
   const lifecycleCaps = resolveMemoryLifecycleCapabilities(config);
   return `You are a memory extraction system. Analyze the following conversation and extract durable, reusable memories.
 
@@ -81,8 +92,12 @@ Memory categories:
 - principle: Durable rules, values, or operating beliefs (e.g., "never use Chat Completions API")
 - commitment: Promises, obligations, or deadlines (e.g., "deploy by Friday", "call accountant Monday")
 - moment: Emotionally significant events or milestones (e.g., "first successful deployment of engram")
-- skill: Capabilities the user or agent has demonstrated (e.g., "user is proficient with Kubernetes")${resolveRecallAuxiliaryCapabilities(config).causalRuleExtraction ? `
-- rule: Causal rules discovered through experience (format: "IF <condition> THEN <action/outcome>", e.g., "IF Shopify API returns 401 THEN the admin token is missing read_products scope")` : ""}
+- skill: Capabilities the user or agent has demonstrated (e.g., "user is proficient with Kubernetes")${
+    resolveRecallAuxiliaryCapabilities(config).causalRuleExtraction
+      ? `
+- rule: Causal rules discovered through experience (format: "IF <condition> THEN <action/outcome>", e.g., "IF Shopify API returns 401 THEN the admin token is missing read_products scope")`
+      : ""
+  }
 - procedure: A reusable workflow the user wants remembered the same way across sessions. Set category to "procedure". Use "content" for a short title that includes explicit trigger phrasing (e.g. "When you deploy to production…", "Whenever you ship a release…"). Add "procedureSteps": an array of at least two objects {"order": number, "intent": "concrete step description"} in execution order. Optional per-step "toolCall": {"kind": "…", "signature": "…"}, "expectedOutcome", "optional": true.
 - reasoning_trace: A stored solution chain / chain-of-thought the user walked through to solve a problem (e.g. "Here's how I debugged the latency spike: first I checked…, then I…, finally I…"). Set category to "reasoning_trace". Use "content" for a short title summarising the problem (e.g. "How I debugged the staging latency spike"). Add "reasoningTrace": {"steps": [{"order": number, "description": "what happened at this step"}, …], "finalAnswer": "the conclusion or answer", "observedOutcome": "optional confirmation of how it played out"}. Require at least two ordered steps AND a finalAnswer. Use this category only when the user explicitly narrates their reasoning — not for ordinary decisions (use "decision") or reusable workflows (use "procedure").
 
@@ -98,6 +113,7 @@ Rules:
 - Entity references should use normalized names (lowercase, hyphenated: "jane-doe", "acme-corp")
 - CRITICAL: Entity names must be CANONICAL. Always use the hyphenated multi-word form: "acme-corp" NOT "acmecorp" or "acme". "jane-doe" NOT "janedoe" or "jane". If unsure, prefer the most specific full name.
 - Avoid creating entities typed as "other" when a more specific type fits (company, project, tool, person, place)
+${CUE_ANCHOR_PROMPT_INSTRUCTION}
 - When entity facts clearly belong under a durable named heading, add them to entity.structuredSections as {key, title, facts}. Example person headings: "Beliefs", "Communication Style", "Building / Working On". Leave structuredSections empty when no stable heading fits.
 - Tags should be concise and reusable (e.g., "coding-style", "personal", "tools")
 - When a fact contains measurable, categorical, or precisely valued data, include a "structuredAttributes" field with key-value string pairs (e.g., {"price": "29.99", "brand": "Sony"}, {"date": "2024-03-15", "location": "SF"}, {"chosen": "PostgreSQL", "rejected": "MongoDB"}). Only for concrete values, not narrative content.
@@ -105,17 +121,29 @@ Rules:
   * Explicit (0.95-1.0): Direct user statements — "I prefer X", "my name is Y"
   * Implied (0.70-0.94): Strong contextual inference — user consistently does X, clear from conversation flow
   * Inferred (0.40-0.69): Pattern recognition — reasonable guess from limited evidence
-  * Speculative (0.00-0.39): Tentative hypothesis — weak signal, needs future confirmation. Speculative memories auto-expire after 30 days if not confirmed.${ambientCapture ? AMBIENT_SPECULATIVE_TIER_CLAUSE : ""}${config.provenance?.enabled ? `
-- Source quotes: For each fact, include a "quote" field containing the EXACT verbatim words from the conversation that support the fact. Copy a contiguous span from a single speaker turn (not a paraphrase, not a summary). Cap at ~300 characters. This grounds every memory in the literal utterance that created it.` : ""}
-- For commitments: include any deadline or timeframe mentioned${lifecycleCaps.extractionScopeClassification ? `
+  * Speculative (0.00-0.39): Tentative hypothesis — weak signal, needs future confirmation. Speculative memories auto-expire after 30 days if not confirmed.${ambientCapture ? AMBIENT_SPECULATIVE_TIER_CLAUSE : ""}${
+    config.provenance?.enabled
+      ? `
+- Source quotes: For each fact, include a "quote" field containing the EXACT verbatim words from the conversation that support the fact. Copy a contiguous span from a single speaker turn (not a paraphrase, not a summary). Cap at ~300 characters. This grounds every memory in the literal utterance that created it.`
+      : ""
+  }
+- For commitments: include any deadline or timeframe mentioned${
+    lifecycleCaps.extractionScopeClassification
+      ? `
 
 Scope classification:
 For each fact, set "scope" to one of:
 - "global" — knowledge that applies across projects: core framework/library bugs, API behavior patterns, user preferences (editor, language, style), general coding patterns, infrastructure knowledge, technology facts not tied to one codebase
 - "project" — knowledge specific to one codebase: file paths, environment configs, deployment details, project-specific workarounds, team/stakeholder info tied to one project, repo-specific conventions. Tool, command, or CLI-flag instructions TIED TO ONE AGENT are also "project", because the same tool name means different things in different agent integrations (a "search" tool may search repository code in one agent and the web in another); when keeping such an agent-tied instruction, the fact text MUST name the originating agent as a leading "In <agent>," clause (e.g. "In Pi, use the search tool with a repository path."). A tool/command fact that holds in every agent and every repo is NOT agent-tied — leave it "global" and do not add a qualifier (e.g. "\`git status --short\` emits compact output"). Examples: "In Pi, the search tool takes a repository path" -> "project"; "\`git status --short\` emits compact output" -> "global".
-When in doubt, prefer "project" — it is safer to keep knowledge scoped narrowly.` : ""}
-${ambientCapture ? `${AMBIENT_CAPTURE_PROMPT_SECTION}
-` : ""}
+When in doubt, prefer "project" — it is safer to keep knowledge scoped narrowly.`
+      : ""
+  }
+${
+  ambientCapture
+    ? `${AMBIENT_CAPTURE_PROMPT_SECTION}
+`
+    : ""
+}
 Entity creation rules (STRICT):
 - Only create entities for DURABLE things: real people, companies, products, tools, ongoing projects
 - NEVER create entities for transient items: individual PRs, branches, Jira tickets, meetings, agent task IDs, log files, database tables, cron job runs, sessions
@@ -123,12 +151,16 @@ Entity creation rules (STRICT):
 - Prefer attaching facts to broad parent entities rather than creating sub-entities. E.g., "acme-store uses Algolia for search" is a fact on entity "acme-store", NOT a new entity "acme-store-algolia-connector"
 - The entity list should be SHORT — think "things that would have their own Wikipedia page" not "things mentioned in passing"
 
-${existingEntities && existingEntities.length > 0 ? `
+${
+  existingEntities && existingEntities.length > 0
+    ? `
 KNOWN ENTITIES (use these exact names when referencing existing things):
 ${existingEntities.join(", ")}
 
 When you see something that matches a known entity, use THAT name exactly. Only create a NEW entity if nothing in this list represents it.
-` : ""}
+`
+    : ""
+}
 ${eventTimePromptInstruction(config)}
 Also extract relationships between entities mentioned in the conversation.
 - Format: {source: "entity-name", target: "entity-name", label: "relationship description"}
@@ -136,6 +168,8 @@ Also extract relationships between entities mentioned in the conversation.
 - Only include clear, durable relationships (e.g., "works at", "created", "manages", "uses")
 - Use normalized entity names (e.g., "person-jane-doe", "company-acme-corp")
 
+
+Also emit "episodeTitle" as a six-to-eight word title for this conversation segment.
 Questions are optional. Include only source-grounded unresolved questions that would be useful in future sessions; otherwise return an empty array.
 
 Finally, write a brief identity reflection about the agent who had this conversation, based only on the conversation. Do not write about the extraction process.`;

package/src/extraction.ts

@@ -4,6 +4,7 @@ import { delinearize } from "./delinearize.js";
 import { LocalLlmClient } from "./local-llm.js";
 import { shouldRunProactivePass } from "./proactive-contention.js";
 import { FallbackLlmClient, fallbackLlmRuntimeContextFromConfig, gatewayTaskChainOptions } from "./fallback-llm.js";
+import { revalidateDependentsViaLlm } from "./orchestration/dependency-revalidation.js";
 import {
   ExtractionResultSchema,
   ConsolidationResultSchema,
@@ -47,7 +48,14 @@ import { applyGroundingWithConnector, headerConnector, renderExtractionConversat
 import { isMemoryCategory } from "./write-envelope.js";
 import { classifyExtractionThrownError, classifyFallbackParseFailure } from "./extraction-error-classification.js";
 import { AMBIENT_CAPTURE_PROMPT_SECTION_COMPACT, clampAmbientCaptureConfidence } from "./ambient-provenance.js";
-import { EXTRACTION_RESPONSE_PLACEHOLDERS, EXTRACTION_RESPONSE_SHAPE, buildExtractionInstructions, eventTimePromptInstruction } from "./extraction-prompt.js";
+import { CUE_ANCHOR_PROMPT_INSTRUCTION, EXTRACTION_RESPONSE_SHAPE, buildExtractionInstructions, eventTimePromptInstruction } from "./extraction-prompt.js";
+import {
+  containsExtractionPlaceholder,
+  extractionAttributes,
+  extractionCueAnchors,
+  extractionText,
+  isPlainRecord,
+} from "./extraction-normalization.js";
 export { classifyExtractionThrownError, classifyFallbackParseFailure } from "./extraction-error-classification.js";
 import { resolveLocalLlmCapabilities, resolveMemoryLifecycleCapabilities, resolvePipelineProcessingCapabilities, resolveRecallAuxiliaryCapabilities } from "./capabilities.js";
 
@@ -71,34 +79,6 @@ const CONSOLIDATION_RESPONSE_SCHEMA = `{
   "entityUpdates": [{"name": "person-jane-doe", "type": "person", "facts": ["Now leads the backend team", "Recently migrated the user service to TypeScript"]}]
 }`;
 
-function isPlainRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-function containsExtractionPlaceholder(value: unknown): boolean {
-  if (typeof value === "string") return EXTRACTION_RESPONSE_PLACEHOLDERS[value.trim()] === true;
-  if (Array.isArray(value)) return value.some(containsExtractionPlaceholder);
-  return isPlainRecord(value) && Object.values(value).some(containsExtractionPlaceholder);
-}
-
-function extractionText(value: unknown): string | undefined {
-  if (typeof value !== "string") return undefined;
-  const text = value.trim();
-  return text.length > 0 && !containsExtractionPlaceholder(text) ? text : undefined;
-}
-
-function extractionAttributes(value: unknown): Record<string, string> | undefined {
-  if (!isPlainRecord(value)) return undefined;
-  const attributes: Record<string, string> = {};
-  for (const [key, candidate] of Object.entries(value)) {
-    const normalizedKey = extractionText(key);
-    const normalizedValue = extractionText(candidate);
-    if (normalizedKey !== undefined && normalizedValue !== undefined) {
-      attributes[normalizedKey] = normalizedValue;
-    }
-  }
-  return Object.keys(attributes).length > 0 ? attributes : undefined;
-}
 
 function extractionEntityType(value: unknown): ExtractedEntityResult["type"] | undefined {
   const type = extractionText(value);
@@ -268,9 +248,13 @@ export class ExtractionEngine {
         if (resolvePipelineProcessingCapabilities(this.config).delinearize) {
           content = delinearize(content, result.entities, ts);
         }
-        return { ...fact, content };
+        return {
+          ...fact,
+          content,
+          cueAnchors: extractionCueAnchors(fact.cueAnchors),
+        };
       });
-    return { ...result, facts };
+    return { ...result, facts, episodeTitle: extractionText(result.episodeTitle) };
   }
 
   private hasExtractionOutputs(result: ExtractionResult): boolean {
@@ -338,6 +322,7 @@ export class ExtractionEngine {
                   })
                 : [],
               entityRef: extractionText(f.entityRef),
+              cueAnchors: extractionCueAnchors(f.cueAnchors),
               promptedByQuestion: extractionText(f.promptedByQuestion),
               scope:
                 f.scope === "global" || f.scope === "project" ? f.scope : undefined,
@@ -398,6 +383,7 @@ export class ExtractionEngine {
       profileUpdates,
       questions,
       identityReflection: extractionText(parsed?.identityReflection),
+      episodeTitle: extractionText(parsed?.episodeTitle),
       relationships,
     };
   }
@@ -1448,7 +1434,7 @@ Rules:
 - Use normalized, hyphenated entity names and keep the entity list short.
 - Keep facts standalone. Skip transient task state and operational noise such as routine scheduler, monitoring, or automation status.
 - Add structuredAttributes only for concrete values.
-- Include at most five durable relationships.${this.config.provenance?.enabled ? `
+${CUE_ANCHOR_PROMPT_INSTRUCTION}\n- Include at most five durable relationships.${this.config.provenance?.enabled ? `
 - Each fact must include a quote copied verbatim from one contiguous conversation span.` : ""}${lifecycleCaps.extractionScopeClassification ? `
 - Set each fact scope to "global" for cross-project knowledge or "project" for codebase-specific knowledge. Tool, command, or CLI-flag instructions tied to one agent are "project" (the same tool name can mean different things across agents); when keeping one, begin the fact with a leading "In <agent>," clause naming that agent.` : ""}${ambientCapture ? AMBIENT_CAPTURE_PROMPT_SECTION_COMPACT : ""}
 ${eventTimePromptInstruction(this.config)}
@@ -2455,6 +2441,36 @@ Respond with valid JSON matching this schema:
       return null;
     }
   }
+  async revalidateDependents(
+    superseded: { id: string; content: string },
+    replacement: { id: string; content: string } | null,
+    dependents: Array<{ id: string; category: string; content: string }>,
+    signal?: AbortSignal,
+  ): Promise<{
+    verdicts: Array<{
+      memoryId: string;
+      verdict: "still_valid" | "invalidated" | "uncertain";
+      reason?: string;
+    }>;
+  }> {
+    return revalidateDependentsViaLlm(
+      {
+        shouldUseLocalLlm: this.shouldUseLocalLlm,
+        shouldUseDirectClient: this.shouldUseDirectClient,
+        localLlm: this.localLlm,
+        fallbackLlm: this.fallbackLlm,
+        client: this.client,
+        config: this.config,
+        withGatewayAgent: this.withGatewayAgent.bind(this),
+        parseJsonObject: this.parseJsonObject,
+      },
+      superseded,
+      replacement,
+      dependents,
+      signal,
+    );
+  }
+
 
   /**
    * Suggest links between a new memory and existing memories (Phase 3A).

package/src/faithfulness-graph-guard.test.ts

@@ -23,6 +23,7 @@ import test from "node:test";
 import { parseConfig } from "./config.js";
 import { Orchestrator } from "./orchestrator.js";
 import { readEdges } from "./graph.js";
+import { searchHarmonicRetrieval } from "./harmonic-retrieval.js";
 import type { ExtractionResult, PluginConfig } from "./types.js";
 import type { LocalLlmClient } from "./local-llm.js";
 import type { BufferTurn, ConversationThread } from "./types.js";
@@ -96,6 +97,8 @@ async function makeHarness(): Promise<OrchHarness> {
     extractionMinChars: 0,
     extractionMinImportanceLevel: "trivial",
     inlineSourceAttributionEnabled: false,
+    harmonicRetrievalEnabled: true,
+    abstractionAnchorsEnabled: true,
   });
   const orchestrator = new Orchestrator(config);
   return { orchestrator, memoryDir };
@@ -190,6 +193,20 @@ test("pending_review fact gets no graph edge; an active fact with the same entit
       !edgesAfter1.some((e) => e.from.includes(id1!) || e.from.includes(id0!)),
       "neither fact0 nor fact1 originates an entity edge",
     );
+    const harmonicAfter1 = await searchHarmonicRetrieval({
+      memoryDir,
+      query: "widget factory",
+      maxResults: 10,
+      anchorsEnabled: true,
+    });
+    assert.ok(
+      harmonicAfter1.some((result) => result.node.sourceMemoryIds?.includes(id0!)),
+      "active fact must appear in harmonic node sources",
+    );
+    assert.ok(
+      !harmonicAfter1.some((result) => result.node.sourceMemoryIds?.includes(id1!)),
+      "pending_review fact must not appear in harmonic node sources",
+    );
 
     // Control: an ACTIVE fact with the same entityRef DOES create entity edges
     // (proving the graph is wired and only pending_review is excluded).
@@ -219,6 +236,16 @@ test("pending_review fact gets no graph edge; an active fact with the same entit
       !edgesAfter2.some((e) => e.from.includes(id1!)),
       "pending_review fact1 must never originate an entity edge even after a later active fact",
     );
+    const harmonicAfter2 = await searchHarmonicRetrieval({
+      memoryDir,
+      query: "widget factory",
+      maxResults: 10,
+      anchorsEnabled: true,
+    });
+    assert.ok(
+      harmonicAfter2.some((result) => result.node.sourceMemoryIds?.includes(id2!)),
+      "active fact must appear in harmonic node sources",
+    );
   } finally {
     await rm(memoryDir, { recursive: true, force: true });
   }