@remnic/core 9.3.742 → 9.3.743

package/src/orchestration/contradiction-linking-coordinator.ts

@@ -0,0 +1,336 @@
+/**
+ * Contradiction-linking coordinator — extracted from the orchestrator (issue #1526).
+ *
+ * Owns the write-path semantic-analysis subsystem: contradiction detection
+ * (QMD similarity search + LLM verification), deferred contradiction
+ * auto-resolve (#1645 — retires superseded memories only after the new write's
+ * tombstone status is known), and memory link suggestion. Behavior-preserving
+ * move from orchestrator.ts — no logic changes; the orchestrator constructs one
+ * instance and keeps thin delegating methods so existing call sites (writeMemory
+ * fact-write branches) continue to work.
+ *
+ * Config, storage, search, namespace resolution, and the extraction engine are
+ * accessed through getter callbacks (not captured at construction) so that
+ * post-construction reassignment of the orchestrator's live fields is honored.
+ * This mirrors the RecallRerankCoordinator / ConversationIndexCoordinator
+ * accessor pattern.
+ */
+
+import type {
+  MemoryLink,
+  PluginConfig,
+  QmdSearchResult,
+} from "../types.js";
+// StorageManager type comes from the package barrel (type-only) so this
+// module does not add a direct storage.ts import (ratchet #1533).
+import type { StorageManager } from "../index.js";
+import type { ExtractionEngine } from "../extraction.js";
+import { log } from "../logger.js";
+import { resolveIndexingCapabilities } from "../capabilities.js";
+import { deindexMemory } from "../temporal-index.js";
+
+/** Result type of {@link ContradictionLinkingCoordinator.checkForContradiction}. */
+export interface ContradictionResult {
+  supersededId: string;
+  confidence: number;
+  reason: string;
+  supersededPath: string;
+  supersededCreated: string;
+  supersededTags: string[];
+}
+
+/**
+ * Subset of {@link ContradictionResult} used by
+ * {@link ContradictionLinkingCoordinator.applyDeferredContradictionResolve}.
+ * The deferred resolve runs AFTER writeMemory and only needs the supersede
+ * target fields (not confidence, which the caller already consumed).
+ */
+export interface ContradictionResolveTarget {
+  supersededId: string;
+  reason: string;
+  supersededPath: string;
+  supersededCreated: string;
+  supersededTags: string[];
+}
+
+/**
+ * Coordinator for the contradiction-detection + memory-linking subsystem.
+ *
+ * The three methods mirror the orchestrator's former private methods
+ * `checkForContradiction`, `applyDeferredContradictionResolve`, and
+ * `suggestLinksForMemory` byte-for-byte in logic.
+ */
+export class ContradictionLinkingCoordinator {
+  private readonly getConfig: () => PluginConfig;
+  private readonly isSearchAvailable: () => boolean;
+  private readonly searchAcrossNamespaces: (options: {
+    query: string;
+    namespaces?: string[];
+    maxResults?: number;
+    mode?: "search" | "hybrid" | "bm25" | "vector";
+  }) => Promise<QmdSearchResult[]>;
+  private readonly extractMemoryIdsFromResults: (
+    results: QmdSearchResult[],
+  ) => string[];
+  private readonly namespaceFromPath: (p: string) => string;
+  private readonly storageForNamespace: (
+    namespace: string,
+  ) => Promise<StorageManager>;
+  private readonly getExtraction: () => ExtractionEngine;
+
+  constructor(options: {
+    getConfig: () => PluginConfig;
+    isSearchAvailable: () => boolean;
+    searchAcrossNamespaces: (options: {
+      query: string;
+      namespaces?: string[];
+      maxResults?: number;
+      mode?: "search" | "hybrid" | "bm25" | "vector";
+    }) => Promise<QmdSearchResult[]>;
+    extractMemoryIdsFromResults: (results: QmdSearchResult[]) => string[];
+    namespaceFromPath: (p: string) => string;
+    storageForNamespace: (namespace: string) => Promise<StorageManager>;
+    getExtraction: () => ExtractionEngine;
+  }) {
+    this.getConfig = options.getConfig;
+    this.isSearchAvailable = options.isSearchAvailable;
+    this.searchAcrossNamespaces = options.searchAcrossNamespaces;
+    this.extractMemoryIdsFromResults = options.extractMemoryIdsFromResults;
+    this.namespaceFromPath = options.namespaceFromPath;
+    this.storageForNamespace = options.storageForNamespace;
+    this.getExtraction = options.getExtraction;
+  }
+
+  /**
+   * Check if a new memory contradicts an existing one.
+   * Uses QMD to find similar memories, then LLM to verify contradiction.
+   */
+  async checkForContradiction(
+    content: string,
+    category: string,
+    namespaceScope: string,
+  ): Promise<ContradictionResult | null> {
+    if (!this.isSearchAvailable()) return null;
+
+    const config = this.getConfig();
+
+    // Search for similar memories
+    const results = await this.searchAcrossNamespaces({
+      query: content,
+      namespaces: [namespaceScope],
+      maxResults: 5,
+      mode: "search",
+    });
+
+    for (const result of results) {
+      // Check similarity threshold
+      if (result.score < config.contradictionSimilarityThreshold) {
+        continue;
+      }
+
+      // Get the existing memory
+      const memoryId = this.extractMemoryIdsFromResults([result])[0];
+      if (!memoryId) continue;
+
+      const resultNamespace = this.namespaceFromPath(result.path);
+      if (resultNamespace !== namespaceScope) continue;
+      const resultStorage = await this.storageForNamespace(resultNamespace);
+      const existingMemory = await resultStorage.getMemoryById(memoryId);
+      if (!existingMemory) continue;
+
+      // Skip memories already resolved or explicitly forgotten. Other
+      // non-active statuses remain valid contradiction candidates.
+      if (
+        existingMemory.frontmatter.status === "superseded" ||
+        existingMemory.frontmatter.status === "forgotten"
+      ) {
+        continue;
+      }
+
+      // Verify contradiction with LLM
+      const extraction = this.getExtraction();
+      const verification = await extraction.verifyContradiction(
+        { content, category },
+        {
+          id: existingMemory.frontmatter.id,
+          content: existingMemory.content,
+          category: existingMemory.frontmatter.category,
+          created: existingMemory.frontmatter.created,
+        },
+      );
+
+      if (!verification) continue;
+
+      // Check if it's a real contradiction with high confidence
+      if (
+        verification.isContradiction &&
+        verification.confidence >= config.contradictionMinConfidence
+      ) {
+        // When the LLM says the existing memory is newer (whichIsNewer ===
+        // "first") the incoming fact is the stale one in both resolve modes —
+        // log and continue so the caller never marks contradictionDetected and
+        // the semantic-skip gate can discard the outdated write normally.
+        if (verification.whichIsNewer === "first") {
+          log.info(
+            `detected contradiction (confidence: ${verification.confidence}): ${existingMemory.frontmatter.id} vs new memory — existing is newer, incoming fact is stale`,
+          );
+          continue;
+        }
+
+        // The new fact is newer than the existing one. When auto-resolve is
+        // enabled, the caller retires the old memory AFTER the new write lands
+        // and its tombstone status is known (#1645: defer auto-resolve until
+        // tombstone status is known — a tombstone-blocked new write must not
+        // retire the only active copy, leaving neither memory active). When
+        // disabled, the old memory stays active for manual review. The
+        // contradiction info is returned in both cases so the caller can apply
+        // the deferred retire post-write.
+
+        // Return the contradiction info regardless of auto-resolve setting.
+        // The caller uses this to set `contradictionDetected=true` which
+        // prevents the semantic-skip guard from silently dropping a
+        // legitimately contradictory update (the regression this fixes).
+        log.info(
+          `detected contradiction (confidence: ${verification.confidence}): ${existingMemory.frontmatter.id} vs new memory${config.contradictionAutoResolve ? " (auto-resolved)" : " (queued for manual review)"}`,
+        );
+        return {
+          supersededId: existingMemory.frontmatter.id,
+          confidence: verification.confidence,
+          reason: verification.reasoning,
+          supersededPath: existingMemory.path,
+          supersededCreated: existingMemory.frontmatter.created,
+          supersededTags: existingMemory.frontmatter.tags ?? [],
+        };
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * #1645: Complete the deferred contradiction auto-resolve after writeMemory
+   * returns and the new write's tombstone status is known. Retires the old
+   * memory + deindexes it ONLY when the new write is genuinely active (not
+   * tombstone-blocked / pending_review). A blocked write must not retire the
+   * only active copy — deferring here closes the "contradictionAutoResolve
+   * supersedes before tombstone status is known" defect class.
+   */
+  async applyDeferredContradictionResolve(
+    contradiction: ContradictionResolveTarget | null | undefined,
+    storage: StorageManager,
+    newMemoryId: string,
+    postWriteGuard: boolean,
+  ): Promise<void> {
+    const config = this.getConfig();
+    // #1645 yG2: clear the pre-write `supersedes` link from a blocked row so
+    // it doesn't claim to supersede a still-active memory (best-effort).
+    if (postWriteGuard && contradiction && config.contradictionAutoResolve) {
+      try {
+        const blockedRow = await storage.getMemoryById(newMemoryId);
+        if (blockedRow?.frontmatter?.supersedes) {
+          await storage.writeMemoryFrontmatter(blockedRow, { supersedes: undefined });
+        }
+      } catch (err) {
+        log.warn(
+          `contradiction auto-resolve supersedes clear failed for blocked ${newMemoryId}: ${err}`,
+        );
+      }
+    }
+    if (
+      !contradiction ||
+      !config.contradictionAutoResolve ||
+      postWriteGuard
+    ) {
+      return;
+    }
+    try {
+      await storage.supersedeMemory(
+        contradiction.supersededId,
+        newMemoryId,
+        contradiction.reason,
+      );
+      if (
+        resolveIndexingCapabilities(config).queryAwareIndexing &&
+        contradiction.supersededPath
+      ) {
+        deindexMemory(
+          config.memoryDir,
+          contradiction.supersededPath,
+          contradiction.supersededCreated,
+          contradiction.supersededTags,
+        );
+      }
+    } catch (err) {
+      log.warn(
+        `contradiction auto-resolve supersede failed for ${contradiction.supersededId}: ${err}`,
+      );
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+  // Memory Linking (Phase 3A)
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Suggest links for a new memory based on similar existing memories.
+   */
+  async suggestLinksForMemory(
+    content: string,
+    category: string,
+    namespaceScope: string,
+  ): Promise<MemoryLink[]> {
+    if (!this.isSearchAvailable()) return [];
+
+    // Search for related memories
+    const results = await this.searchAcrossNamespaces({
+      query: content,
+      namespaces: [namespaceScope],
+      maxResults: 5,
+      mode: "search",
+    });
+    if (results.length === 0) return [];
+
+    // Get full memory details for candidates
+    const candidates: Array<{ id: string; content: string; category: string }> =
+      [];
+    for (const result of results) {
+      const memoryId = this.extractMemoryIdsFromResults([result])[0];
+      if (!memoryId) continue;
+
+      const resultNamespace = this.namespaceFromPath(result.path);
+      if (resultNamespace !== namespaceScope) continue;
+      const resultStorage = await this.storageForNamespace(resultNamespace);
+      const memory = await resultStorage.getMemoryById(memoryId);
+      if (
+        memory &&
+        memory.frontmatter.status !== "superseded" &&
+        memory.frontmatter.status !== "forgotten"
+      ) {
+        candidates.push({
+          id: memory.frontmatter.id,
+          content: memory.content,
+          category: memory.frontmatter.category,
+        });
+      }
+    }
+
+    if (candidates.length === 0) return [];
+
+    // Ask LLM for link suggestions
+    const extraction = this.getExtraction();
+    const suggestions = await extraction.suggestLinks(
+      { content, category },
+      candidates,
+    );
+
+    if (!suggestions || suggestions.links.length === 0) return [];
+
+    // Convert to MemoryLink format
+    return suggestions.links.map((link) => ({
+      targetId: link.targetId,
+      linkType: link.linkType,
+      strength: link.strength,
+      reason: link.reason || undefined,
+    }));
+  }
+}

package/src/orchestration/recall-section-coordinator.ts

@@ -0,0 +1,349 @@
+/**
+ * Recall section coordinator — extracted from the orchestrator (issue #1526).
+ *
+ * Owns the recall-output section-budgeting subsystem: section enablement
+ * lookups, per-section maxChars/number resolution, section bucket appends,
+ * budget-aware section truncation, and final ordered assembly of recall
+ * sections within the configured character budget. Also builds the
+ * LastRecallBudgetSummary and collects the sources used for X-ray tracing.
+ *
+ * Behavior-preserving move from orchestrator.ts — no logic changes; the
+ * orchestrator constructs one instance and keeps thin delegating methods so
+ * existing call sites (recallInternal, consolidation) continue to work.
+ *
+ * Config is accessed through a getter callback (not captured at construction)
+ * so that post-construction reassignment of the orchestrator's live
+ * `config` field is honored. This mirrors the ConversationIndexCoordinator /
+ * TierMigrationCoordinator accessor pattern.
+ */
+
+import type { PluginConfig, RecallSectionConfig } from "../types.js";
+import type { LastRecallBudgetSummary } from "../recall-state.js";
+
+/**
+ * Coordinator for the recall section budgeting + assembly subsystem.
+ *
+ * All methods are pure with respect to the coordinator's own state — they
+ * read config via the getter and operate on the caller-provided section
+ * buckets. No caches are held because the config-derived values are cheap
+ * to recompute and the buckets are transient (per-recall-call).
+ */
+export class RecallSectionCoordinator {
+  private readonly getConfig: () => PluginConfig;
+  private readonly resolveSectionEnabled: (
+    sectionId: string,
+    defaultEnabled: boolean,
+  ) => boolean;
+
+  constructor(options: {
+    getConfig: () => PluginConfig;
+    resolveSectionEnabled?: (
+      sectionId: string,
+      defaultEnabled: boolean,
+    ) => boolean;
+  }) {
+    this.getConfig = options.getConfig;
+    // Default to the coordinator's own config-based check so standalone usage
+    // (e.g. config-recall-pipeline tests) keeps self-gating. The orchestrator
+    // injects its own resolver so appendRecallSection flows through the same
+    // overridable isRecallSectionEnabled that the recallInternal computation
+    // gates use — keeping compute-side and append-side enablement coherent
+    // (seam 13). This preserves the recall-hardening test seam where tests
+    // override orchestrator.isRecallSectionEnabled to isolate sections.
+    this.resolveSectionEnabled =
+      options.resolveSectionEnabled ??
+      ((id, def) => this.isRecallSectionEnabled(id, def));
+  }
+
+  getRecallSectionEntry(
+    sectionId: string,
+  ): RecallSectionConfig | undefined {
+    const pipeline = Array.isArray(this.getConfig().recallPipeline)
+      ? this.getConfig().recallPipeline
+      : [];
+    return pipeline.find((entry) => entry.id === sectionId);
+  }
+
+  isRecallSectionEnabled(
+    sectionId: string,
+    defaultEnabled: boolean = true,
+  ): boolean {
+    const entry = this.getRecallSectionEntry(sectionId);
+    if (!entry) return defaultEnabled;
+    return entry.enabled !== false;
+  }
+
+  isSpecializedRecallSectionEnabled(
+    sectionId: string,
+    topLevelEnabled: boolean,
+  ): boolean {
+    const entry = this.getRecallSectionEntry(sectionId);
+    if (!entry) return topLevelEnabled;
+    return entry.enabled === true || (topLevelEnabled && entry.enabled !== false);
+  }
+
+  getRecallSectionMaxChars(
+    sectionId: string,
+  ): number | null | undefined {
+    const entry = this.getRecallSectionEntry(sectionId);
+    if (!entry) return undefined;
+    if (entry.maxChars === null) return null;
+    if (typeof entry.maxChars !== "number") return undefined;
+    return Math.max(0, Math.floor(entry.maxChars));
+  }
+
+  getRecallSectionNumber(
+    sectionId: string,
+    key: keyof RecallSectionConfig,
+  ): number | undefined {
+    const entry = this.getRecallSectionEntry(sectionId);
+    if (!entry) return undefined;
+    const value = entry[key];
+    if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
+    return Math.max(0, Math.floor(value));
+  }
+
+  appendRecallSection(
+    sectionBuckets: Map<string, string[]>,
+    sectionId: string,
+    content: string,
+  ): boolean {
+    // Returns true when the section was actually appended to sectionBuckets,
+    // false when it was dropped (disabled, empty, or maxChars===0). Callers
+    // that need to know whether injection occurred (e.g. xray annotation for
+    // peer-profile) must gate on this return value rather than on whether the
+    // section text was computed (Codex P2 finding, PR #764).
+    if (!this.resolveSectionEnabled(sectionId, true)) return false;
+    const trimmed = content.trim();
+    if (trimmed.length === 0) return false;
+
+    const maxChars = this.getRecallSectionMaxChars(sectionId);
+    let finalContent = trimmed;
+    if (maxChars === 0) return false;
+    if (typeof maxChars === "number" && finalContent.length > maxChars) {
+      finalContent = `${finalContent.slice(0, maxChars)}\n\n...(trimmed)\n`;
+    }
+
+    const existing = sectionBuckets.get(sectionId) ?? [];
+    existing.push(finalContent);
+    sectionBuckets.set(sectionId, existing);
+    return true;
+  }
+
+  truncateRecallSectionToBudget(
+    content: string,
+    maxChars: number,
+  ): string {
+    if (maxChars <= 0) return "";
+    if (content.length <= maxChars) return content;
+    const suffix = "\n\n...(memory context trimmed)";
+    if (maxChars <= suffix.length) {
+      return content.slice(0, maxChars);
+    }
+    return `${content.slice(0, maxChars - suffix.length)}${suffix}`;
+  }
+
+  protectedRecallSectionIds(
+    sectionBuckets: Map<string, string[]>,
+  ): Set<string> {
+    const protectedIds = new Set<string>();
+    if ((sectionBuckets.get("memories")?.length ?? 0) > 0) {
+      protectedIds.add("memories");
+    }
+    return protectedIds;
+  }
+
+  protectedRecallReservationChars(content: string): number {
+    const headingBoundary = content.indexOf("\n\n");
+    const headingChars =
+      headingBoundary >= 0 ? headingBoundary + 2 : Math.min(content.length, 24);
+    return Math.min(content.length, Math.max(headingChars, 24));
+  }
+
+  estimateReservedRecallBudget(
+    entries: Array<{ id: string; content: string }>,
+    startIndex: number,
+    protectedIds: Set<string>,
+    alreadyIncludedCount: number,
+  ): number {
+    const separatorLength = "\n\n---\n\n".length;
+    let reserved = 0;
+    let simulatedIncluded = alreadyIncludedCount;
+    for (let i = startIndex; i < entries.length; i += 1) {
+      const entry = entries[i];
+      if (!entry || !protectedIds.has(entry.id)) continue;
+      if (simulatedIncluded > 0) {
+        reserved += separatorLength;
+      }
+      reserved += this.protectedRecallReservationChars(entry.content);
+      simulatedIncluded += 1;
+    }
+    return reserved;
+  }
+
+  getRecallBudgetChars(override?: number): number {
+    if (
+      typeof override === "number" &&
+      Number.isFinite(override) &&
+      override >= 0
+    ) {
+      return Math.floor(override);
+    }
+    const configuredBudget = this.getConfig().recallBudgetChars;
+    if (
+      typeof configuredBudget === "number" &&
+      Number.isFinite(configuredBudget) &&
+      configuredBudget >= 0
+    ) {
+      return Math.floor(configuredBudget);
+    }
+    const tokenBudget = this.getConfig().maxMemoryTokens;
+    if (
+      typeof tokenBudget === "number" &&
+      Number.isFinite(tokenBudget) &&
+      tokenBudget >= 0
+    ) {
+      return Math.floor(tokenBudget * 4);
+    }
+    return 0;
+  }
+
+  assembleRecallSections(
+    sectionBuckets: Map<string, string[]>,
+    budgetOverride?: number,
+  ): {
+    sections: string[];
+    includedIds: string[];
+    omittedIds: string[];
+    truncated: boolean;
+    finalChars: number;
+  } {
+    const orderedEntries: Array<{ id: string; content: string }> = [];
+    const pipeline = Array.isArray(this.getConfig().recallPipeline)
+      ? this.getConfig().recallPipeline
+      : [];
+    const orderedIds = pipeline
+      .filter((entry) => entry.enabled !== false)
+      .map((entry) => entry.id);
+    const seen = new Set<string>();
+
+    for (const id of orderedIds) {
+      const chunks = sectionBuckets.get(id);
+      if (!chunks || chunks.length === 0) continue;
+      orderedEntries.push({ id, content: chunks.join("\n\n") });
+      seen.add(id);
+    }
+
+    for (const [id, chunks] of sectionBuckets.entries()) {
+      if (seen.has(id)) continue;
+      if (chunks.length === 0) continue;
+      orderedEntries.push({ id, content: chunks.join("\n\n") });
+    }
+
+    const budget = this.getRecallBudgetChars(budgetOverride);
+    if (budget === 0) {
+      return {
+        sections: [],
+        includedIds: [],
+        omittedIds: orderedEntries.map((entry) => entry.id),
+        truncated: orderedEntries.length > 0,
+        finalChars: 0,
+      };
+    }
+
+    const separator = "\n\n---\n\n";
+    const protectedIds = this.protectedRecallSectionIds(sectionBuckets);
+    const sections: string[] = [];
+    const includedIds: string[] = [];
+    const omittedIds: string[] = [];
+    let usedChars = 0;
+    let truncated = false;
+
+    for (let index = 0; index < orderedEntries.length; index += 1) {
+      const entry = orderedEntries[index]!;
+      const separatorChars = sections.length > 0 ? separator.length : 0;
+      const reserve = protectedIds.has(entry.id)
+        ? 0
+        : this.estimateReservedRecallBudget(
+            orderedEntries,
+            index + 1,
+            protectedIds,
+            sections.length + 1,
+          );
+      const availableForEntry = budget - usedChars - separatorChars - reserve;
+      if (availableForEntry <= 0) {
+        omittedIds.push(entry.id);
+        truncated = true;
+        continue;
+      }
+      const finalContent = this.truncateRecallSectionToBudget(
+        entry.content,
+        availableForEntry,
+      );
+      if (!finalContent) {
+        omittedIds.push(entry.id);
+        truncated = true;
+        continue;
+      }
+      if (finalContent.length < entry.content.length) {
+        truncated = true;
+      }
+      sections.push(finalContent);
+      includedIds.push(entry.id);
+      usedChars += separatorChars + finalContent.length;
+    }
+
+    return {
+      sections,
+      includedIds,
+      omittedIds,
+      truncated,
+      finalChars: usedChars,
+    };
+  }
+
+  buildLastRecallBudgetSummary(options: {
+    requestedTopK?: number;
+    recallResultLimit: number;
+    qmdFetchLimit: number;
+    qmdHybridFetchLimit: number;
+    finalContextChars?: number;
+    truncated?: boolean;
+    includedSections?: string[];
+    omittedSections?: string[];
+  }): LastRecallBudgetSummary {
+    return {
+      requestedTopK: options.requestedTopK,
+      appliedTopK: options.recallResultLimit,
+      recallBudgetChars: this.getRecallBudgetChars(),
+      maxMemoryTokens: this.getConfig().maxMemoryTokens,
+      qmdFetchLimit: options.qmdFetchLimit,
+      qmdHybridFetchLimit: options.qmdHybridFetchLimit,
+      finalContextChars: options.finalContextChars,
+      truncated: options.truncated,
+      includedSections: [...(options.includedSections ?? [])],
+      omittedSections: [...(options.omittedSections ?? [])],
+    };
+  }
+
+  collectLastRecallSources(
+    sectionBuckets: Map<string, string[]>,
+    recallSource:
+      | "none"
+      | "hot_qmd"
+      | "hot_embedding"
+      | "cold_fallback"
+      | "recent_scan",
+  ): string[] {
+    const used = new Set<string>();
+    if (recallSource !== "none") {
+      used.add(recallSource);
+    }
+    for (const [sectionId, chunks] of sectionBuckets.entries()) {
+      if (chunks.length > 0) {
+        used.add(sectionId);
+      }
+    }
+    return [...used];
+  }
+}