@remnic/core 9.3.713 → 9.3.715

package/src/orchestrator.ts

@@ -126,6 +126,7 @@ import { RelevanceStore } from "./relevance.js";
 import { NegativeExampleStore } from "./negative.js";
 import {
   LastRecallStore,
+  RecallHandleHistoryStore,
   type LastRecallBudgetSummary,
   TierMigrationStatusStore,
   clampGraphRecallExpandedEntries,
@@ -134,6 +135,14 @@ import {
   type TierMigrationCycleSummary,
   type TierMigrationStatusSnapshot,
 } from "./recall-state.js";
+import {
+  buildHandleIndexForResults,
+  MEMORY_ID_PATTERN,
+  parseIdOrHandle,
+  resolveHandle,
+  stripHandles,
+  type RecallSnapshotIds,
+} from "./recall-handles.js";
 import {
   buildXraySnapshot,
   type RecallFilterTrace,
@@ -1722,8 +1731,16 @@ export function resolveRecentThreadMemoryPaths(options: {
     buildMemoryPathById(options.allMemsForGraph, options.storageDir);
   if (pathById.size === 0) return [];
 
+  // #1635 (defensive): skip pending_review ids from legacy episode sets.
+  const pendingReviewIds = new Set<string>(
+    (options.allMemsForGraph ?? [])
+      .filter((m) => m.frontmatter.status === "pending_review" && m.frontmatter.id)
+      .map((m) => m.frontmatter.id as string),
+  );
+
   return options.threadEpisodeIds
     .filter((id) => id !== options.currentMemoryId)
+    .filter((id) => !pendingReviewIds.has(id))
     .slice(-maxRecent)
     .map((id) => pathById.get(id))
     .filter((p): p is string => typeof p === "string" && p.length > 0);
@@ -1849,6 +1866,11 @@ export class Orchestrator {
    * callers already destructure `persistedIds` by position.
    */
   private lastPersistExtractionDeferredCount: number = 0;
+  /**
+   * Side-channel (#1635): pending_review persisted ids from the last
+   * persistExtraction; runExtraction excludes them from the thread episode set.
+   */
+  private lastPersistExtractionPendingReviewIds: string[] = [];
   private readonly _fastGatewayLlm: FallbackLlmClient | null;
 
   get fastGatewayLlm(): FallbackLlmClient | null {
@@ -1872,6 +1894,7 @@ export class Orchestrator {
   readonly relevance: RelevanceStore;
   readonly negatives: NegativeExampleStore;
   readonly lastRecall: LastRecallStore;
+  readonly handleHistory: RecallHandleHistoryStore;
   readonly tierMigrationStatus: TierMigrationStatusStore;
   /**
    * In-memory X-ray snapshot from the most recent `recall()` call that
@@ -2797,6 +2820,9 @@ export class Orchestrator {
     this.relevance = new RelevanceStore(config.memoryDir);
     this.negatives = new NegativeExampleStore(config.memoryDir);
     this.lastRecall = new LastRecallStore(config.memoryDir);
+    this.handleHistory = new RecallHandleHistoryStore(config.memoryDir, {
+      maxDepth: config.recallHandleSnapshotDepth,
+    });
     this.tierMigrationStatus = new TierMigrationStatusStore(config.memoryDir);
     this.sessionObserver = new SessionObserverState({
       memoryDir: config.memoryDir,
@@ -3241,6 +3267,7 @@ export class Orchestrator {
       await this.relevance.load();
       await this.negatives.load();
       await this.lastRecall.load();
+      await this.handleHistory.load();
       await this.tierMigrationStatus.load();
       await this.sessionObserver.load();
       this.runtimePolicyValues = await this.policyRuntime.loadRuntimeValues();
@@ -11299,7 +11326,7 @@ export class Orchestrator {
         this.appendRecallSection(
           sectionBuckets,
           "workspace-context",
-          this.formatQmdResults("Workspace Context", globalResults),
+          this.formatQmdResults("Workspace Context", globalResults, sessionKey),
         );
       }
 
@@ -12073,6 +12100,22 @@ export class Orchestrator {
             backendDegradations.length > 0 ? backendDegradations : undefined,
         })
         .catch((err) => log.debug(`last recall record failed: ${err}`));
+      // Issue #1582 — record the admitted memory-id set for handle resolution.
+      // Only when handles are enabled: if injection is off, no handle is ever
+      // rendered, so there is nothing to resolve and we skip the write.
+      if (this.config.recallMemoryHandles && recalledMemoryIds.length > 0) {
+        // Only handle-eligible ids aid resolution; filter out non-memory .md
+        // basenames (entity reconstructions) so the resolver never records a
+        // citation target that cannot be loaded.
+        const handleEligibleIds = recalledMemoryIds.filter((id) =>
+          MEMORY_ID_PATTERN.test(id),
+        );
+        if (handleEligibleIds.length > 0) {
+          this.handleHistory
+            .record(sessionKey, handleEligibleIds)
+            .catch((err) => log.debug(`handle history record failed: ${err}`));
+        }
+      }
     }
     if (sessionKey) {
       this.queueEvalShadowRecall({
@@ -12165,9 +12208,17 @@ export class Orchestrator {
           ? sessionKey
           : "default";
     const captureTimestamp = new Date().toISOString();
+    // Issue #1582 hygiene §2 — strip any echoed `[m:xxxx]` handle before the
+    // turn enters the extraction buffer so handles never become memory content
+    // or get QMD-indexed (rule 23). Gated on the feature flag: when handles are
+    // off none are ever injected, so there is nothing to strip and the buffer
+    // stays byte-identical to the pre-#1582 path.
+    const bufferedContent = this.config.recallMemoryHandles
+      ? stripHandles(content)
+      : content;
     const turn: BufferTurn = {
       role,
-      content,
+      content: bufferedContent,
       timestamp: captureTimestamp,
       // #1578: anchor live-capture turns to wall-clock when bi-temporal is on;
       // replay/import turns carry sourceValidAt explicitly (codex P1).
@@ -13423,23 +13474,16 @@ export class Orchestrator {
     }
 
     // Batch-append persisted IDs so non-fact memories (entities/questions) are
-    // always attached to the thread.
+    // always attached to the thread. The helper excludes pending_review ids (#1635).
     if (
       this.config.threadingEnabled &&
       threadIdForExtraction &&
       persistedIds.length > 0
     ) {
-      try {
-        await this.threading.appendEpisodeIds(
-          threadIdForExtraction,
-          persistedIds,
-        );
-      } catch (err) {
-        log.warn(
-          "[threading] appendEpisodeIds failed after persistence (non-fatal)",
-          err,
-        );
-      }
+      await this.appendPersistedThreadEpisodes(
+        threadIdForExtraction,
+        persistedIds,
+      );
     }
 
     // Thread title update for the already-established thread context.
@@ -13793,9 +13837,11 @@ export class Orchestrator {
       // to avoid a maintenance hazard where the two guard paths could diverge.
       return attachCitation(content, citationContext, citationTemplate);
     };
+    const persistedIds: string[] = [];
     const supersessionOrderingAt = (validAt?: string): string =>
       validAt && validAt.length > 0 ? validAt : new Date().toISOString();
-    const persistedIds: string[] = [];
+    // #1635: pending_review persisted ids, excluded from the thread episode set below.
+    const pendingReviewPersistedIds: string[] = [];
     const persistedIdsByStorage = new Map<
       string,
       { storage: StorageManager; ids: string[] }
@@ -13803,11 +13849,18 @@ export class Orchestrator {
     const trackPersistedId = (
       targetStorage: StorageManager,
       id: string,
-      options: { includeReturnedIds?: boolean } = {},
+      options: {
+        includeReturnedIds?: boolean;
+        /** #1635: keep this id out of the persisted thread episode set. */
+        pendingReview?: boolean;
+      } = {},
     ): void => {
       if (options.includeReturnedIds !== false) {
         persistedIds.push(id);
       }
+      if (options.pendingReview) {
+        pendingReviewPersistedIds.push(id);
+      }
       const key = targetStorage.dir;
       const existing = persistedIdsByStorage.get(key);
       if (existing) {
@@ -15309,7 +15362,9 @@ export class Orchestrator {
           log.debug(
             `chunked memory ${parentId} into ${chunkResult.chunks.length} chunks`,
           );
-          trackPersistedId(targetStorage, parentId);
+          trackPersistedId(targetStorage, parentId, {
+            pendingReview: faithfulnessEnforceStatus === "pending_review",
+          });
           // #1576 (cursor Medium): keep pending_review ids out of threadEpisodeIdsForGraph — else later active facts build thread-predecessor edges to an unfaithful memory.
           if (
             faithfulnessEnforceStatus !== "pending_review" &&
@@ -15595,8 +15650,9 @@ export class Orchestrator {
             source: "extraction",
           }),
         );
-        trackPersistedId(targetStorage, memoryId);
-        // #1576 (cursor Medium): same thread-episode guard on the non-chunked path.
+        trackPersistedId(targetStorage, memoryId, {
+          pendingReview: faithfulnessEnforceStatus === "pending_review",
+        });
         if (
           faithfulnessEnforceStatus !== "pending_review" &&
           threadEpisodeIdsForGraph &&
@@ -15906,9 +15962,35 @@ export class Orchestrator {
       log.debug(`temporal-index update error (non-fatal): ${err}`),
     );
 
+    // #1635: surface pending_review ids so the thread episode set excludes them.
+    this.lastPersistExtractionPendingReviewIds = pendingReviewPersistedIds;
     // Return the persisted fact IDs for threading
     return persistedIds;
   }
+  /**
+   * Append persisted ids to the thread episode set, excluding pending_review
+   * fact ids (#1635) so they don't re-seed predecessor edges. Fail-open like
+   * the raw appendEpisodeIds call it replaces.
+   */
+  private async appendPersistedThreadEpisodes(
+    threadId: string,
+    persistedIds: string[],
+  ): Promise<void> {
+    const pendingReviewIds = this.lastPersistExtractionPendingReviewIds ?? [];
+    const episodeIds =
+      pendingReviewIds.length > 0
+        ? persistedIds.filter((id) => !pendingReviewIds.includes(id))
+        : persistedIds;
+    if (episodeIds.length === 0) return;
+    try {
+      await this.threading.appendEpisodeIds(threadId, episodeIds);
+    } catch (err) {
+      log.warn(
+        "[threading] appendEpisodeIds failed after persistence (non-fatal)",
+        err,
+      );
+    }
+  }
 
   private async indexPersistedMemory(
     storage: StorageManager,
@@ -17455,13 +17537,28 @@ export class Orchestrator {
     }
   }
 
-  private formatQmdResults(title: string, results: QmdSearchResult[]): string {
+  private formatQmdResults(
+    title: string,
+    results: QmdSearchResult[],
+    sessionKey?: string,
+  ): string {
+    // Issue #1582 — handles are only rendered when a session key is available:
+    // resolution requires the handle history to have been recorded for this
+    // session, which only happens when sessionKey is present. Rendering handles
+    // without recording would show tokens a user can never resolve (cursor
+    // review). The rendering logic itself lives in the pure handles module.
+    const handleByIndex = buildHandleIndexForResults(
+      results,
+      this.config.recallMemoryHandles === true && sessionKey != null,
+    );
     const lines = results.map((r, i) => {
       const snippet = r.snippet
         ? r.snippet.slice(0, 500).replace(/\n/g, " ")
         : "(no preview)";
       const source = typeof r.line === "number" ? `${r.path}:${r.line}` : r.path;
-      return `[${i + 1}] ${source} (score: ${r.score.toFixed(3)})\n${snippet}`;
+      const head = `[${i + 1}] ${source} (score: ${r.score.toFixed(3)})\n${snippet}`;
+      const handle = handleByIndex.get(i);
+      return handle ? `${head.replace(/\s+$/, "")} ${handle}` : head;
     });
     return `## ${title}\n\n${lines.join("\n\n")}`;
   }
@@ -17826,7 +17923,7 @@ export class Orchestrator {
     this.appendRecallSection(
       options.sectionBuckets,
       sectionId,
-      this.formatQmdResults(options.title, options.results),
+      this.formatQmdResults(options.title, options.results, options.sessionKey),
     );
   }
 
@@ -19661,6 +19758,54 @@ export class Orchestrator {
     return this.lastRecall.get(sessionKey);
   }
 
+  /**
+   * Issue #1582 — resolve a memory handle (`[m:4f2a]` / bare hex) to its memory
+   * id against THIS session's recent recall history. Returns the id on a unique
+   * hit, `null` on a miss, and throws on ambiguity (callers must disambiguate —
+   * rule 34/51). Resolution is per-session and snapshot-scoped; there is no
+   * global handle→id map (rule 42).
+   */
+  resolveMemoryHandle(handle: string, sessionKey: string): string | null {
+    if (!sessionKey) return null;
+    const depth = this.config.recallHandleSnapshotDepth;
+    const snapshots: RecallSnapshotIds[] = this.handleHistory
+      .recent(sessionKey, depth)
+      .map((ids) => ({ memoryIds: ids }));
+    const result = resolveHandle(handle, snapshots, depth);
+    if (result.ok) return result.memoryId;
+    if (result.reason === "ambiguous") {
+      throw new Error(
+        `Memory handle ${handle} is ambiguous in session ${sessionKey}: ` +
+          `${result.candidates.join(", ")}. Cite a unique memory id instead.`,
+      );
+    }
+    return null;
+  }
+
+  /**
+   * Issue #1582 — resolve a caller reference that may be EITHER a memory id or
+   * a handle to a concrete memory id. Handles resolve against the session's
+   * recall history; raw ids pass through unchanged. The single shared helper
+   * every surface (memory_get / feedback / outcome / correction targetIds)
+   * calls so handle resolution has one path (rule 22).
+   */
+  resolveMemoryIdOrHandle(ref: string, sessionKey?: string): string {
+    const parsed = parseIdOrHandle(ref);
+    if (!parsed.isHandle) return parsed.value;
+    if (!sessionKey) {
+      throw new Error(
+        `Memory handle ${ref} cannot be resolved without a session key.`,
+      );
+    }
+    const resolved = this.resolveMemoryHandle(parsed.value, sessionKey);
+    if (resolved === null) {
+      throw new Error(
+        `Memory handle ${ref} was not found in the recent recall history for session ${sessionKey}.`,
+      );
+    }
+    return resolved;
+  }
+
   /**
    * Check if a new memory contradicts an existing one.
    * Uses QMD to find similar memories, then LLM to verify contradiction.