@remnic/core 9.3.700 → 9.3.702

package/src/event-order-recall.test.ts

@@ -216,6 +216,59 @@ test("event order recall keeps summary within the section budget", async () => {
   assert.match(recalled, /^## Chronological event evidence/);
 });
 
+test("event order recall budgets the outline BEFORE evidence so the total never exceeds maxChars (#1539 fix)", async () => {
+  // Issue #1539 step 4 (the ONE declared behavior change): pre-fix, event-order
+  // built the evidence pack at the FULL budget, then appended the chronology
+  // outline + summary, then clipped — which could silently exceed maxChars
+  // (the outline consumed space the evidence pack had already filled). Post-fix,
+  // the summary (incl. outline) is computed FIRST and its length subtracted from
+  // the evidence budget, so evidence + summary = budget exactly.
+  //
+  // This test uses a fixture rich enough to produce a non-empty chronology
+  // outline, then asserts the total fits the budget with the outline present.
+  const sessionId = "event-order-outline-budget-fix";
+  const engine = new FakeEventOrderEngine(sessionId, [
+    {
+      turn_index: 10,
+      role: "user",
+      content:
+        "I first brought up passive voice reduction and checklist in my writing process.",
+    },
+    {
+      turn_index: 30,
+      role: "user",
+      content:
+        "Later I discussed webinar planning and promotion for the workshop series.",
+    },
+    {
+      turn_index: 50,
+      role: "user",
+      content:
+        "After that I focused on engagement and incentives discussion with the team.",
+    },
+  ]);
+
+  const maxChars = 600;
+  const recalled = await buildEventOrderRecallSection({
+    engine,
+    sessionId,
+    query:
+      "Can you walk me through the order in which I brought up different aspects of my personal and professional progress throughout our conversations, in order? Mention ONLY and ONLY three items.",
+    maxChars,
+    maxScanWindowTurns: 4,
+  });
+
+  assert.ok(recalled.length > 0);
+  // The fix guarantees the total (evidence + summary + outline) fits the budget.
+  assert.ok(
+    recalled.length <= maxChars,
+    `expected recalled section length ${recalled.length} to fit ${maxChars}`,
+  );
+  // The outline must be present — the fix guarantees it is budgeted for, not
+  // clipped away by evidence that consumed the full budget.
+  assert.match(recalled, /Chronology outline:/);
+});
+
 test("event order recall returns relevant user turns in chronological order", async () => {
   const sessionId = "event-order-core";
   const engine = new FakeEventOrderEngine(sessionId, [

package/src/event-order-recall.ts

@@ -1,6 +1,11 @@
 import { buildEvidencePack, type EvidencePackItem } from "./evidence-pack.js";
 import type { ExplicitCueRecallEngine } from "./explicit-cue-recall.js";
 
+import {
+  unifiedDedupeAndRank,
+  type RankedEvidenceItem,
+} from "./recall-pipeline-stages.js";
+
 export interface EventOrderRecallOptions {
   engine: ExplicitCueRecallEngine | null | undefined;
   // event-order reads a SINGLE LCM session key. Unlike the relevance-ranked
@@ -18,10 +23,6 @@ export interface EventOrderRecallOptions {
   title?: string;
 }
 
-interface RankedEventItem extends EvidencePackItem {
-  rank: number;
-}
-
 const DEFAULT_SCAN_WINDOW_TURNS = 12;
 const DEFAULT_SCAN_WINDOW_TOKENS = 24_000;
 const DEFAULT_MAX_ITEMS = 24;
@@ -67,15 +68,14 @@ export async function buildEventOrderRecallSection(
   }
 
   const title = options.title ?? "Chronological event evidence";
-  const evidence = buildEvidencePack(ranked, {
-    title,
-    maxChars: budget,
-    maxItemChars: options.maxItemChars,
-  });
-  if (!evidence) {
-    return "";
-  }
 
+  // Issue #1539 step 4 (the ONE declared behavior change): budget the outline
+  // BEFORE building evidence, matching guidance/targeted-fact. Previously
+  // event-order built the evidence pack at the FULL budget, then appended the
+  // chronology outline, then clipped the total — which could silently exceed
+  // maxChars (the outline consumed space the evidence pack had already filled).
+  // Now the summary (incl. outline) is computed first and its length subtracted
+  // from the evidence budget, so evidence + summary = budget exactly.
   const requested = parseRequestedItemCount(options.query);
   const outlineSource = requested
     ? rankEventOrderItemsForOutline(items, options.query).slice(0, maxItems)
@@ -87,8 +87,20 @@ export async function buildEventOrderRecallSection(
     outline,
     "Use these turns to preserve the order in which the user raised the topics.",
   ].filter(Boolean).join(" ");
+  const summaryInsert = `\n\n${summary}`;
+  const evidenceBudget = Math.max(0, budget - summaryInsert.length);
+  const evidence = buildEvidencePack(ranked, {
+    title,
+    maxChars: evidenceBudget,
+    maxItemChars: options.maxItemChars,
+  });
+  if (!evidence) {
+    // If no evidence fit the residual budget, still emit the summary clipped to
+    // budget (matches targeted-fact's empty-evidence-with-summary fallback).
+    return clipSectionToBudget(`## ${title}${summaryInsert}`, budget);
+  }
   return clipSectionToBudget(
-    evidence.replace(`## ${title}`, `## ${title}\n\n${summary}`),
+    evidence.replace(`## ${title}`, `## ${title}${summaryInsert}`),
     budget,
   );
 }
@@ -148,23 +160,28 @@ function rankAndSelectEventOrderItems(
   items: EvidencePackItem[],
   options: EventOrderRecallOptions,
   maxItems: number,
-): RankedEventItem[] {
+): RankedEvidenceItem[] {
   const requested = parseRequestedItemCount(options.query);
-  const rankedByScore = items
-    .map((item) => ({
-      ...item,
-      rank: scoreEventOrderItem(item, options.query),
-    }))
-    .filter((item) => item.rank >= 6)
-    .sort((left, right) => {
-      if (right.rank !== left.rank) return right.rank - left.rank;
-      const leftTurn = typeof left.turnIndex === "number" ? left.turnIndex : Number.MAX_SAFE_INTEGER;
-      const rightTurn = typeof right.turnIndex === "number" ? right.turnIndex : Number.MAX_SAFE_INTEGER;
-      if (leftTurn !== rightTurn) return leftTurn - rightTurn;
-      return left.content.localeCompare(right.content);
-    });
+  // Issue #1539 PR6: rank + threshold-filter + sort now route through the
+  // unified spine. Event-order's three declared divergences are named config
+  // fields: rankThreshold:6 (was an undocumented inline .filter), ASC sort
+  // direction (was a byte-identical comparator copy with flipped direction),
+  // and dedupByContent:false (event-order keeps distinct turns — turn_index is
+  // local to each session_id, so chronology must not collapse same-body turns).
+  // The spine's id-dedup is a no-op here: event-order reads a single session
+  // with unique per-turn ids (no cross-key merge, per the #1505 opt-out).
+  // Content is already cue-appended during collection; the spine scores on the
+  // (already-transformed) item.content, matching the prior inline behavior.
+  const rankedByScore = unifiedDedupeAndRank(items, {
+    query: options.query,
+    intents: [],
+    scoreEvidence: (item, q) => scoreEventOrderItem(item, q),
+    rankThreshold: 6,
+    dedupByContent: false,
+    turnIndexSortDirection: "asc",
+  });
 
-  const selectedById = new Map<string, RankedEventItem>();
+  const selectedById = new Map<string, RankedEvidenceItem>();
   if (requested) {
     const queryTerms = extractEventOrderTerms(options.query);
     const tailReserve = Math.min(
@@ -254,13 +271,20 @@ function rankAndSelectEventOrderItems(
 function rankEventOrderItemsForOutline(
   items: readonly EvidencePackItem[],
   query: string,
-): RankedEventItem[] {
-  return items
-    .map((item) => ({
-      ...item,
-      rank: scoreEventOrderItem(item, query),
-    }))
-    .filter((item) => item.rank >= 6);
+): RankedEvidenceItem[] {
+  // Issue #1539 PR6: the outline source uses the same declared config as the
+  // main rank pass (rankThreshold:6, dedupByContent:false) but preserves
+  // collection order via preserveInsertionOrder — the outline renders turns in
+  // the order they were collected (chronological by session scan), not in
+  // rank-sorted order.
+  return unifiedDedupeAndRank(items, {
+    query,
+    intents: [],
+    scoreEvidence: (item, q) => scoreEventOrderItem(item, q),
+    rankThreshold: 6,
+    dedupByContent: false,
+    preserveInsertionOrder: true,
+  });
 }
 
 function eventItemSelectionKey(item: EvidencePackItem): string {
@@ -527,7 +551,7 @@ function appendChronologicalCues(content: string, query: string): string {
 }
 
 function buildChronologyOutline(
-  items: readonly RankedEventItem[],
+  items: readonly RankedEvidenceItem[],
   query: string,
   requested?: number,
 ): string {
@@ -584,7 +608,7 @@ function buildChronologyOutline(
 }
 
 function collectChronologyOutlineCandidates(
-  items: readonly RankedEventItem[],
+  items: readonly RankedEvidenceItem[],
   query: string,
 ): Array<{ turn: number; label: string; priority: number }> {
   const terms = extractEventOrderTerms(query);

package/src/explicit-cue-recall.ts

@@ -3,6 +3,7 @@ import {
   gatherAcrossReadSessions,
   resolveLcmReadSessionIds,
 } from "./lcm-fallback-read.js";
+import { unifiedDedupeAndRank } from "./recall-pipeline-stages.js";
 
 export interface ExplicitCueRecallEngine {
   expandContext(
@@ -410,7 +411,26 @@ export async function buildExplicitCueRecallSection(
     includeContentLexicalCues: options.includeContentLexicalCues,
     includeStructuredPlanCues: options.includeStructuredPlanCues,
   });
-  return buildEvidencePack(evidenceItems, {
+  // Issue #1539 PR5: route through the unified spine to declare explicit-cue's
+  // policy explicitly. The call is behavior-idempotent — items are already
+  // deduped by `seenTurns` during collection (so the spine's id-dedup matches
+  // exactly), the constant scorer assigns rank=0 which `buildEvidencePack`
+  // ignores, and `preserveInsertionOrder` skips the sort so the deliberate
+  // value ordering (turn references -> content cues -> lexical cues, by evidence
+  // type then read-key priority) is preserved byte-for-byte.
+  //
+  // The value is the DECLARED config: a fifth recall pipeline would be a config
+  // file, and this config documents explicit-cue's opt-outs (no scoring, id-only
+  // dedup, insertion-order preservation) as named fields instead of embedded
+  // code. See recall-pipeline-stages.ts for the spine contract.
+  const rankedEvidenceItems = unifiedDedupeAndRank(evidenceItems, {
+    query: "",
+    intents: [],
+    scoreEvidence: () => 0,
+    dedupByContent: false,
+    preserveInsertionOrder: true,
+  });
+  return buildEvidencePack(rankedEvidenceItems, {
     title: "Explicit Cue Evidence",
     query: evidenceFocusQuery,
     maxChars,

package/src/extraction.ts

@@ -37,6 +37,7 @@ import { ProfilingCollector } from "./profiling.js";
 import { normalizeProcedureSteps } from "./procedural/procedure-types.js";
 import { normalizeReasoningTrace } from "./reasoning-trace-types.js";
 import { looksLikeMechanicalTelemetryTranscript } from "./telemetry-transcript.js";
+import { buildFactProvenance, type ProvenanceTurnInput } from "./provenance.js";
 
 type ExtractionQuestion = ExtractionResult["questions"][number];
 type ExtractedFactResult = ExtractionResult["facts"][number];
@@ -269,6 +270,15 @@ export class ExtractionEngine {
                     : null;
               return candidate ? normalizeReasoningTrace(candidate) ?? undefined : undefined;
             })(),
+            // Issue #1575 PR 2: the LLM provides a verbatim supporting quote
+            // per fact. Optional/nullable per repo gotcha 6 (OpenAI Responses
+            // API emits null for absent optional fields). The post-parse
+            // validator (buildFactProvenance) locates this quote in the
+            // buffered turns and builds verified ProvenanceSource[] entries.
+            quote:
+              typeof f?.quote === "string" && f.quote.trim().length > 0
+                ? f.quote
+                : undefined,
           }))
           .filter((f: any) => f.content.length > 0)
       : [];
@@ -736,7 +746,10 @@ export class ExtractionEngine {
       `Return at most ${maxAdditional} additional high-confidence memory candidates that were omitted from the base extraction.`,
       "Only include information directly supported by the conversation. Do not speculate. Do not repeat the base extraction.",
       "Return only valid JSON with this shape:",
-      '{"facts":[{"category":"fact","content":"...","confidence":0.0,"tags":["..."],"entityRef":"optional","promptedByQuestion":"optional"}],"profileUpdates":["..."],"entities":[{"name":"...","type":"person","facts":["..."],"structuredSections":[{"key":"beliefs","title":"Beliefs","facts":["..."]}],"promptedByQuestion":"optional"}],"relationships":[{"source":"...","target":"...","label":"...","promptedByQuestion":"optional"}]}',
+      '{"facts":[{"category":"fact","content":"...","confidence":0.0,"tags":["..."],"entityRef":"optional","promptedByQuestion":"optional","quote":"optional verbatim span from a single turn"}],"profileUpdates":["..."],"entities":[{"name":"...","type":"person","facts":["..."],"structuredSections":[{"key":"beliefs","title":"Beliefs","facts":["..."]}],"promptedByQuestion":"optional"}],"relationships":[{"source":"...","target":"...","label":"...","promptedByQuestion":"optional"}]}',
+      this.config.provenance?.enabled
+        ? '- Source quotes: For each fact, include a "quote" field with the EXACT verbatim words from the conversation that support the fact (a contiguous span from a single turn, not a paraphrase). Cap at ~300 chars.'
+        : "",
       "",
       "Base extracted facts (do not repeat):",
       factsPreview || "(none)",
@@ -963,6 +976,73 @@ export class ExtractionEngine {
     return null;
   }
 
+  /**
+   * Attach claim-level provenance spans to each fact in the extraction result
+   * (issue #1575 PR 2). Runs once at write time, after sanitize + proactive
+   * pass so ALL facts (base + proactive additions) get verified spans before
+   * the result is returned for persistence.
+   *
+   * The validator locates each fact's LLM-provided `quote` in the buffered
+   * turns and builds a `ProvenanceSource[]` with verified offsets. Never
+   * throws, never drops a fact — an unverifiable span is a tagged state, not
+   * a silent failure (rule 34 spirit). When `provenance.enabled` is false,
+   * this is a no-op (byte-identical to pre-feature behavior, rule 39).
+   */
+  private attachProvenanceToResult(
+    result: ExtractionResult,
+    turns: ReadonlyArray<{
+      content: string;
+      sessionKey?: string;
+      logicalSessionKey?: string;
+      timestamp: string;
+      turnFingerprint?: string;
+    }>,
+  ): ExtractionResult {
+    // Even when provenance is disabled, strip the transient LLM-provided
+    // `quote` field so it does not leak through the persist pipeline (the
+    // enabled path strips it after validation; the disabled path must match).
+    // quote is never persisted to frontmatter, but carrying it risks it
+    // surfacing in content-hash dedup or downstream in-memory consumers
+    // (cursor thread dHiY).
+    if (!this.config.provenance?.enabled) {
+      if (result.facts.length === 0) return result;
+      return {
+        ...result,
+        facts: result.facts.map((fact) => {
+          if (fact.quote === undefined) return fact;
+          const { quote: _stripped, ...rest } = fact;
+          return rest;
+        }),
+      };
+    }
+    if (result.facts.length === 0) return result;
+    const provenanceTurns: ProvenanceTurnInput[] = turns.map((t) => ({
+      content: t.content,
+      sessionKey: t.sessionKey,
+      logicalSessionKey: t.logicalSessionKey,
+      timestamp: t.timestamp,
+      turnId: t.turnFingerprint,
+    }));
+    const facts = result.facts.map((fact) => {
+      const built = buildFactProvenance(
+        fact.quote,
+        provenanceTurns,
+        this.config.provenance,
+      );
+      // Strip the transient `quote` field — it has served its purpose (the
+      // validator consumed it) and must NOT leak into the persisted ExtractedFact
+      // shape or the content-hash dedup key (rule 23 / checklist §13).
+      const { quote: _stripped, ...factWithoutQuote } = fact;
+      return {
+        ...factWithoutQuote,
+        ...(built.sources && built.sources.length > 0 ? { sources: built.sources } : {}),
+        ...(built.provenance !== "none" ? { provenance: built.provenance } : {}),
+        ...(built.requireSpansPending ? { requireSpansPending: true } : {}),
+      };
+    });
+    return { ...result, facts };
+  }
+
   async extract(turns: BufferTurn[], existingEntities?: string[]): Promise<ExtractionResult> {
 
     // Guard: skip if buffer is empty or all turns are whitespace-only
@@ -1037,7 +1117,8 @@ export class ExtractionEngine {
           this.emit({ kind: "llm_end", traceId, model: this.config.localLlmModel, operation: "extraction", durationMs });
           log.debug(`extraction: used local LLM — ${localResult.facts.length} facts, ${localResult.entities.length} entities`);
           const sanitized = this.sanitizeExtractionResult(localResult, messageTimestamp);
-          return await this.applyProactiveQuestionPass(conversation, sanitized);
+          const finalResult = await this.applyProactiveQuestionPass(conversation, sanitized);
+          return this.attachProvenanceToResult(finalResult, boundedTurns);
         }
         // Local failed, fall back if allowed
         if (!this.config.localLlmFallback) {
@@ -1080,7 +1161,8 @@ export class ExtractionEngine {
           this.emit({ kind: "llm_end", traceId, model: this.config.model, operation: "extraction", durationMs });
           log.debug(`extraction: used direct client (${this.config.model}) — ${directResult.facts.length} facts, ${directResult.entities.length} entities`);
           const sanitized = this.sanitizeExtractionResult(directResult, messageTimestamp);
-          return await this.applyProactiveQuestionPass(conversation, sanitized);
+          const finalResult = await this.applyProactiveQuestionPass(conversation, sanitized);
+          return this.attachProvenanceToResult(finalResult, boundedTurns);
         }
         // Emit error event so Opik sees the direct client failure before fallback.
         // Wrapped in try/catch so a subscriber error doesn't break the fallback path.
@@ -1180,7 +1262,8 @@ export class ExtractionEngine {
           questions: result.questions ?? [],
           identityReflection: result.identityReflection ?? undefined,
         } as ExtractionResult, messageTimestamp);
-        return await this.applyProactiveQuestionPass(conversation, sanitized);
+        const finalResult = await this.applyProactiveQuestionPass(conversation, sanitized);
+        return this.attachProvenanceToResult(finalResult, boundedTurns);
       }
 
       this.emit({
@@ -1291,7 +1374,8 @@ These are durable insights - capture them:
 === Rules ===
 - Extract only NEW information worth remembering across sessions
 - Skip transient details (file paths, current errors, temporary states, agent actions)
-- Confidence: Explicit (0.95-1.0), Implied (0.70-0.94), Inferred (0.40-0.69), Speculative (0.00-0.39)
+- Confidence: Explicit (0.95-1.0), Implied (0.70-0.94), Inferred (0.40-0.69), Speculative (0.00-0.39)${this.config.provenance?.enabled ? `
+- Source quotes: For each fact, include a "quote" field with the EXACT verbatim words from the conversation that support the fact (copy a contiguous span from a single turn, not a paraphrase). Cap at ~300 chars.` : ""}
 - Corrections get highest confidence (0.95+)
 - Each fact should be standalone and self-contained
 - Lines labelled [context user] or [context assistant] are reference context only. Use them to resolve pronouns and adjacent question/answer pairs, but do not extract a memory stated only in context lines unless a normal [user] or [assistant] line confirms or completes it.
@@ -1549,7 +1633,8 @@ Rules:
   * Explicit (0.95-1.0): Direct user statements — "I prefer X", "my name is Y"
   * Implied (0.70-0.94): Strong contextual inference — user consistently does X, clear from conversation flow
   * Inferred (0.40-0.69): Pattern recognition — reasonable guess from limited evidence
-  * Speculative (0.00-0.39): Tentative hypothesis — weak signal, needs future confirmation. Speculative memories auto-expire after 30 days if not confirmed.
+  * Speculative (0.00-0.39): Tentative hypothesis — weak signal, needs future confirmation. Speculative memories auto-expire after 30 days if not confirmed.${this.config.provenance?.enabled ? `
+- Source quotes: For each fact, include a "quote" field containing the EXACT verbatim words from the conversation that support the fact. Copy a contiguous span from a single speaker turn (not a paraphrase, not a summary). Cap at ~300 characters. This grounds every memory in the literal utterance that created it.` : ""}
 - For commitments: include any deadline or timeframe mentioned${this.config.extractionScopeClassificationEnabled ? `
 
 Scope classification: