@remnic/core 10.0.39 → 10.0.40

package/src/extraction.ts

@@ -38,7 +38,7 @@ import {
 import { applyGroundingWithConnector, headerConnector, renderExtractionConversation, resolveSourceConnector, type ExtractionGroundingContext } from "./source-agent-qualifier.js";
 import { isMemoryCategory } from "./write-envelope.js";
 export { classifyExtractionThrownError, classifyFallbackParseFailure, formatExtractionParseFailureLog } from "./extraction-error-classification.js";
-import { classifyExtractionThrownError, classifyFallbackParseFailure, formatExtractionParseFailureLog } from "./extraction-error-classification.js";
+import { LocalLlmRejectedError, classifyExtractionThrownError, classifyFallbackParseFailure, formatExtractionParseFailureLog, localRejectionStatus, type LocalFailureDiag } from "./extraction-error-classification.js";
 import { AMBIENT_CAPTURE_PROMPT_SECTION_COMPACT, clampAmbientCaptureConfidence } from "./ambient-provenance.js";
 import {
   CONSOLIDATION_RESPONSE_SCHEMA,
@@ -1244,7 +1244,7 @@ export class ExtractionEngine {
             entities: [],
             questions: [],
             extractionFailure: "local_llm_error",
-            extractionFailureClass: "provider_retryable",
+            extractionFailureClass: err instanceof LocalLlmRejectedError ? "parse_empty" : classifyExtractionThrownError(err),
           };
         }
         log.info("extraction: local LLM error, falling back to gateway default AI:", err);
@@ -1494,9 +1494,8 @@ ${extractionResponseShape(this.config.extraction?.spanMode ?? "off")}
 Conversation:
 ${truncatedConversation}`;
 
-    log.debug(
-      `extractWithLocalLlm: calling localLlm.chatCompletion with prompt length ${localPrompt.length}...`,
-    );
+    log.debug(`extractWithLocalLlm: calling localLlm.chatCompletion with prompt length ${localPrompt.length}...`);
+    const failureDiag: LocalFailureDiag = {};
     const response = await this.localLlm.chatCompletion(
       [
         { role: "system", content: "You are a memory extraction system. Output valid JSON only." },
@@ -1512,9 +1511,12 @@ ${truncatedConversation}`;
           : undefined,
         priority: "background",
         signal,
+        failureDiag,
       },
     );
     if (!response?.content) {
+      const rejected = localRejectionStatus(failureDiag);
+      if (rejected !== undefined) throw new LocalLlmRejectedError(rejected); // the endpoint is up and refuses this input
       log.debug("extractWithLocalLlm: chatCompletion returned null or empty content");
       return null;
     }
@@ -1547,12 +1549,10 @@ ${truncatedConversation}`;
     log.debug("extractWithLocalLlm: JSON parse failed, attempting partial extraction...");
     const partial = this.extractPartialFacts(content);
     if (partial.facts.length > 0 || partial.entities.length > 0) {
-      log.debug(
-        `extractWithLocalLlm: extracted ${partial.facts.length} partial facts from truncated JSON`,
-      );
+      log.debug(`extractWithLocalLlm: extracted ${partial.facts.length} partial facts from truncated JSON`);
       return partial;
     }
-    return null;
+    throw new LocalLlmRejectedError(); // the endpoint answered and nothing in the answer is usable
   }
 
   /**

package/src/namespaces/search.ts

@@ -700,7 +700,7 @@ export function normalizeQmdResultPath(resultPath: string, collection: string):
   }
   return value;
 }
-function resolveNamespaceResultPath(
+export function resolveNamespaceResultPath(
   memoryDir: string,
   collection: string,
   resultPath: string,

package/src/orchestration/extraction-run.test.ts

@@ -88,6 +88,7 @@ const migrationSummary: TierMigrationCycleSummary = {
 
 interface Harness {
   config: PluginConfig;
+  buffer: SmartBuffer;
   storageForNs: (ns: string) => Promise<StorageManager>;
   setStorageForDelay: (delayMs: number) => void;
   newCoordinator: () => ExtractionRunCoordinator;
@@ -105,7 +106,7 @@ interface Harness {
     coord: ExtractionRunCoordinator,
     content: string,
     opts?: { force?: boolean; failClosed?: boolean; writeNamespaceOverride?: string },
-  ) => Promise<{ status: string; reason?: string }>;
+  ) => Promise<{ status: string; reason?: string; failureClass?: string; retainedBuffer?: boolean }>;
   cleanup: () => Promise<void>;
   passiveCapture: () => { principal?: string; namespace?: string } | null;
   setThreadingBlock: (blocked: boolean) => void;
@@ -201,6 +202,7 @@ async function makeHarness(overrides: Record<string, unknown> = {}): Promise<Har
 
   return {
     config,
+    buffer,
     storageForNs,
     setStorageForDelay: (delayMs) => {
       storageForDelayMs = delayMs;
@@ -243,7 +245,7 @@ async function makeHarness(overrides: Record<string, unknown> = {}): Promise<Har
         forceExtractionAttempt: opts.force === true,
         writeNamespaceOverride: opts.writeNamespaceOverride,
       });
-      return { status: result.status, reason: result.reason };
+      return { status: result.status, reason: result.reason, failureClass: result.failureClass, retainedBuffer: result.retainedBuffer };
     },
     cleanup: async () => {
       StorageManager.clearAllStaticCaches();
@@ -1336,3 +1338,82 @@ test("#3140: a successful flush heals the recorded LLM health", async () => {
     await h.cleanup();
   }
 });
+
+// ---------------------------------------------------------------------------
+// Wiring the trigger drain depends on (#3266): what a real run hands back, and what it asks of the buffer
+// ---------------------------------------------------------------------------
+
+test("a retained failure carries the extraction's failure class, so the drain can tell an outage from a bad chunk", async () => {
+  for (const cls of ["provider_retryable", "auth_config", "parse_empty"] as const) {
+    const h = await makeHarness({ extractionRetryScheduleMs: [3_600_000], extractionBreakerFailureThreshold: 100 });
+    try {
+      h.setRespond(() => failureResult(cls));
+      const result = await h.run(h.newCoordinator(), `class-${cls}`);
+      assert.equal(result.retainedBuffer, true, `${cls}: the window is kept for a retry`);
+      assert.equal(result.failureClass, cls, `${cls}: the class reaches the run result`);
+    } finally {
+      await h.cleanup();
+    }
+  }
+});
+
+test("a failure with no class defaults to provider_retryable on the run result, like the retry bookkeeping", async () => {
+  const h = await makeHarness({ extractionRetryScheduleMs: [3_600_000], extractionBreakerFailureThreshold: 100 });
+  try {
+    h.setRespond(() => ({ ...failureResult("parse_empty"), extractionFailureClass: undefined }));
+    const result = await h.run(h.newCoordinator(), "class-default");
+    assert.equal(result.failureClass, "provider_retryable");
+  } finally {
+    await h.cleanup();
+  }
+});
+
+test("a run without a failure reports no failure class", async () => {
+  const h = await makeHarness();
+  try {
+    h.setRespond(() => ({ ...successResult(), facts: [] }));
+    const result = await h.run(h.newCoordinator(), "class-none");
+    assert.equal(result.failureClass, undefined);
+  } finally {
+    await h.cleanup();
+  }
+});
+
+test("the retention window a chunk run is given reaches the buffer as the processed snapshot (#3266)", async () => {
+  const h = await makeHarness();
+  try {
+    const seen: Array<readonly BufferTurn[] | undefined> = [];
+    const original = h.buffer.retainDeferredTurns.bind(h.buffer);
+    h.buffer.retainDeferredTurns = async (key, turns, max, processed) => {
+      seen.push(processed);
+      await original(key, turns, max, processed);
+    };
+    const chunk = makeTurns("chunk-turns");
+    const window = [...makeTurns("earlier-turns"), ...chunk];
+    h.setRespond(() => successResult());
+    await h.newCoordinator().runExtraction(chunk, {
+      skipCharThreshold: true,
+      skipUserTurnThreshold: true,
+      clearBufferAfterExtraction: true,
+      bufferKey: "window",
+      failOnExtractionFailure: false,
+      forceExtractionAttempt: false,
+      retentionWindow: window,
+    });
+    assert.equal(seen.length, 1, "the run asked the buffer to update its retention slot once");
+    assert.deepEqual(seen[0], window, "with the window it was given, not just its own chunk");
+
+    seen.length = 0;
+    await h.newCoordinator().runExtraction(chunk, {
+      skipCharThreshold: true,
+      skipUserTurnThreshold: true,
+      clearBufferAfterExtraction: true,
+      bufferKey: "window",
+      failOnExtractionFailure: false,
+      forceExtractionAttempt: false,
+    });
+    assert.deepEqual(seen[0], chunk, "without a window the run keeps naming its own turns");
+  } finally {
+    await h.cleanup();
+  }
+});

package/src/orchestration/qmd-result-resolver.ts

@@ -26,12 +26,13 @@
  * RecallRerankCoordinator / ConversationIndexCoordinator accessor pattern.
  */
 
+import { existsSync } from "node:fs";
 import path from "node:path";
 import { resolveNamespaceCapabilities } from "../capabilities.js";
 import type { StorageManager } from "../index.js";
 import { log } from "../logger.js";
 import { namespaceIdentityFromToken } from "../namespaces/identity.js";
-import { normalizeQmdResultPath } from "../namespaces/search.js";
+import { normalizeQmdResultPath, resolveNamespaceResultPath } from "../namespaces/search.js";
 import { SecureStoreLockedError } from "../secure-store/index.js";
 import { isPathInsideStorageRoot } from "../storage-paths.js";
 import { isSupportPassportPrivateMemory } from "../support-passport/card-projection.js";
@@ -136,6 +137,30 @@ export function qmdResultPathCandidates(storageDir: string, resultPath: string):
   return [...candidates];
 }
 
+/** Canonicalize flat hot-memory identities before merging with native hits (#3194).
+ * This is a path-only map: no candidates, collections or namespace scopes are removed.
+ * Cold/explicit foreign collections keep their existing owner-resolution path.
+ */
+export function normalizeFlatQmdResults(
+  results: QmdSearchResult[],
+  config: PluginConfig,
+  collection = config.qmdCollection,
+): QmdSearchResult[] {
+  if (!collection || resolveNamespaceCapabilities(config).namespaces || collection !== config.qmdCollection) return results;
+  return results.map((result) => ({
+    ...result,
+    path: withHotFactsPath(config.memoryDir, resolveNamespaceResultPath(config.memoryDir, collection, result.path)),
+  }));
+}
+
+/** A hot-facts collection registered at the facts/ subtree returns date-relative paths (#2111); meet the loaded memory's path. */
+function withHotFactsPath(memoryDir: string, resolved: string): string {
+  const relative = path.relative(path.resolve(memoryDir), resolved).split(path.sep).join("/");
+  if (!/^\d{4}-\d{2}-\d{2}\//.test(relative) || existsSync(resolved)) return resolved;
+  const hot = path.join(path.resolve(memoryDir), "facts", relative);
+  return existsSync(hot) ? hot : resolved;
+}
+
 /**
  * Extract a non-empty `dir` from a storage, returning `null` when the
  * property is absent or not a non-empty string at runtime. Proxy-backed

package/src/orchestration/recall-search-pipeline.ts

@@ -27,7 +27,7 @@ import { inferIntentFromText, intentCompatibilityScore } from "../intent.js";
 import { log } from "../logger.js";
 import { NamespaceStorageRouter } from "../namespaces/storage.js";
 import { NegativeExampleStore } from "../negative.js";
-import { qmdCollectionPathParts } from "./qmd-result-resolver.js";
+import { normalizeFlatQmdResults, qmdCollectionPathParts } from "./qmd-result-resolver.js";
 import type { RecallRerankCoordinator, RecallResultPartitionSink } from "./recall-rerank-coordinator.js";
 import type { GraphRecallExpansionOptions, GraphRecallExpansionResult } from "./graph-recall-coordinator.js";
 import { RelevanceStore } from "../relevance.js";
@@ -394,7 +394,7 @@ export class RecallSearchPipelineCoordinator {
         break;
       }
 
-      const primaryResults = options.collection
+      let primaryResults = options.collection
         ? options.abortSignal
           ? await this.deps.qmd.search(
               prompt,
@@ -426,6 +426,7 @@ export class RecallSearchPipelineCoordinator {
               onDegradation: options.onDegradation,
             },
           });
+      primaryResults = normalizeFlatQmdResults(primaryResults, this.deps.config, options.collection);
       lastPrimaryResultCount = primaryResults.length;
       lastHybridResultCount = 0;
       lastHybridTopUpUsed = false;
@@ -452,7 +453,7 @@ export class RecallSearchPipelineCoordinator {
           // uniform across primary + top-up (gotcha #39). Issue #1335 (codex review #1422).
           lastHybridTopUpSkippedReason = "lex_strategy";
         } else {
-          const hybridResults = options.collection
+          let hybridResults = options.collection
             ? await this.deps.qmd.hybridSearch(
                 prompt,
                 options.collection,
@@ -474,6 +475,7 @@ export class RecallSearchPipelineCoordinator {
                   onDegradation: options.onDegradation,
                 },
               });
+          hybridResults = normalizeFlatQmdResults(hybridResults, this.deps.config, options.collection);
           lastHybridResultCount = hybridResults.length;
           lastHybridTopUpUsed = hybridResults.length > 0;
           if (hybridResults.length > 0) {
@@ -740,9 +742,7 @@ export class RecallSearchPipelineCoordinator {
         // limps home later cannot mutate a recorded recall's collector.
         coldQmdObserverActive = false;
         if (longTerm.length > 0) {
-          log.debug(
-            `cold-tier recall source=cold-qmd collection=${coldCollection} hits=${longTerm.length}`,
-          );
+          log.debug(`cold-tier recall source=cold-qmd collection=${coldCollection} hits=${longTerm.length}`);
         }
       }
     }
@@ -940,9 +940,7 @@ export class RecallSearchPipelineCoordinator {
       }
     }
     if (caps.rerank && this.deps.config.rerankProvider === "cloud") {
-      log.debug(
-        "rerankProvider=cloud is reserved/experimental in v2.2.0; skipping rerank",
-      );
+      log.debug("rerankProvider=cloud is reserved/experimental in v2.2.0; skipping rerank");
     }
 
     if (caps.recallTrustScore && results.length > 0) {
@@ -1149,6 +1147,9 @@ export class RecallSearchPipelineCoordinator {
       : null;
     const buildRecallRow = (r: QmdSearchResult, memory: MemoryFile | undefined): QmdSearchResult => ({
       ...r,
+      // Render loaded flat memories, not QMD's front-matter/chunk preview (#3194).
+      snippet: memory && !resolveNamespaceCapabilities(this.deps.config).namespaces ? memory.content : r.snippet,
+      line: memory && !resolveNamespaceCapabilities(this.deps.config).namespaces ? undefined : r.line,
       sourceConnector: memory ? memory.frontmatter.sourceConnector : undefined,
       origin: memory ? memory.frontmatter.origin : undefined,
       // #1952: an active state view carries the chain fields the inject
@@ -1358,9 +1359,7 @@ export class RecallSearchPipelineCoordinator {
       }
     }
     if (connectorPartitionFilteredCount > 0) {
-      log.debug(
-        `connector partition filter removed ${connectorPartitionFilteredCount} tool-scoped candidates from recall`,
-      );
+      log.debug(`connector partition filter removed ${connectorPartitionFilteredCount} tool-scoped candidates from recall`);
     }
     if (supportPassportFilteredCount > 0) {
       log.debug(