@remnic/core 9.45.3 → 9.45.4

package/src/orchestration/external-wiki-recall-guard.test.ts

@@ -0,0 +1,94 @@
+import assert from "node:assert/strict";
+import { mkdir, symlink } from "node:fs/promises";
+import path from "node:path";
+import test from "node:test";
+
+import type { QmdSearchResult } from "../types.js";
+import * as wikiGuard from "../external-wiki-guard.js";
+import { withTempDir } from "../testing/tmp-dir.js";
+import {
+  filterRecallCandidates,
+  isExternalWikiCollectionName,
+  isGenericRecallExcludedPath,
+} from "./generic-recall-paths.js";
+
+const { assertExternalWikiRootOutsideMemoryDir } = wikiGuard;
+const assertCanonicalIsolation = (
+  wikiGuard as typeof wikiGuard & {
+    assertExternalWikiRootOutsideMemoryDirCanonical(memoryDir: string, rootDir: string): Promise<void>;
+  }
+).assertExternalWikiRootOutsideMemoryDirCanonical;
+
+function hit(path: string, snippet: string): QmdSearchResult {
+  return { docid: path, path, score: 1, snippet };
+}
+
+const OVERLAPPING_QUERY = "bounded context budgets";
+const MEMORY_HIT = hit("openclaw-engram/facts/context-budget.md", `The operator chose ${OVERLAPPING_QUERY}.`);
+const WIKI_HIT = hit(
+  "qmd://external-wiki-reading/wiki/context-budgets.md",
+  `A compiled guide to ${OVERLAPPING_QUERY}.`
+);
+
+const MEMORY_POLICY = {
+  qmdCollection: "openclaw-engram",
+  qmdColdCollection: "openclaw-engram-cold",
+};
+
+test("default recall excludes overlapping external wiki hits before applying its limit", () => {
+  const recalled = filterRecallCandidates([WIKI_HIT, MEMORY_HIT], {
+    namespacesEnabled: false,
+    recallNamespaces: [],
+    resolveNamespace: () => "default",
+    limit: 1,
+    pathPolicy: MEMORY_POLICY,
+  });
+
+  assert.deepEqual(recalled, [MEMORY_HIT]);
+  assert.equal(isGenericRecallExcludedPath(WIKI_HIT.path, MEMORY_POLICY, "qmd"), true);
+});
+
+test("the recall-only guard leaves on-demand external wiki search results available", () => {
+  const onDemandResults = [WIKI_HIT];
+
+  assert.equal(onDemandResults[0]?.path, WIKI_HIT.path);
+  assert.match(onDemandResults[0]?.snippet ?? "", new RegExp(OVERLAPPING_QUERY));
+});
+
+test("external wiki collection ids cannot masquerade as memory collections", () => {
+  assert.equal(isExternalWikiCollectionName("external-wiki-reading"), true);
+  assert.equal(isExternalWikiCollectionName("external-wiki-reading--default"), true);
+  assert.equal(isExternalWikiCollectionName("openclaw-engram"), false);
+  assert.equal(isExternalWikiCollectionName("my-external-wiki-reading"), false);
+});
+
+test("external wiki roots cannot enter the primary memory collection walk", () => {
+  assert.throws(
+    () => assertExternalWikiRootOutsideMemoryDir("/data/memory", "/data/memory"),
+    /must be outside memoryDir/
+  );
+  assert.throws(
+    () => assertExternalWikiRootOutsideMemoryDir("/data/memory", "/data/memory/compiled-wiki"),
+    /must be outside memoryDir/
+  );
+  assert.doesNotThrow(() => assertExternalWikiRootOutsideMemoryDir("/data/memory", "/data/compiled-wiki"));
+});
+
+test("external wiki root isolation resolves filesystem aliases", async () => {
+  assert.equal(typeof assertCanonicalIsolation, "function");
+  await withTempDir(async (rootDir) => {
+    const memoryDir = path.join(rootDir, "memory");
+    const nestedWiki = path.join(memoryDir, "compiled-wiki");
+    const wikiAlias = path.join(rootDir, "wiki-alias");
+    await mkdir(nestedWiki, { recursive: true });
+    await symlink(nestedWiki, wikiAlias);
+    const memoryAlias = path.join(rootDir, "memory-alias");
+    await symlink(memoryDir, memoryAlias);
+
+    await assert.rejects(() => assertCanonicalIsolation(memoryDir, wikiAlias), /must be outside memoryDir/);
+    await assert.rejects(
+      () => assertCanonicalIsolation(memoryDir, path.join(memoryAlias, "future-wiki")),
+      /must be outside memoryDir/
+    );
+  }, "remnic-external-wiki-guard-");
+});

package/src/orchestration/extraction-persist.ts

@@ -419,6 +419,7 @@ export class ExtractionPersistCoordinator {
       source: string;
       sources?: ProvenanceSource[];
       provenance?: "verified" | "unverified" | "none";
+      toolScoped?: true;
     }): Promise<void> => {
       if (
         !scopeProfileWritePlan ||
@@ -536,6 +537,7 @@ export class ExtractionPersistCoordinator {
             contentHashSource: options.category === "fact" ? dedupContent : rawContent,
             ...(options.sources && options.sources.length > 0 ? { sources: options.sources } : {}),
             ...(options.provenance ? { provenance: options.provenance } : {}),
+            ...(options.toolScoped ? { toolScoped: true as const } : {}),
           });
           const promotedId = targetPromotion.id;
           // #1645: if the TARGET namespace's own tombstone blocked this promotion,
@@ -615,8 +617,12 @@ export class ExtractionPersistCoordinator {
       sources?: ProvenanceSource[];
       provenance?: "verified" | "unverified" | "none";
     }): Promise<void> => {
-      await promoteMemoryToProfileTargets(options);
-      if (lifecycleCaps.extractionScopeClassification && withholdToolScopedFromSharedNamespace(options)) return;
+      const toolScoped = withholdToolScopedFromSharedNamespace(options);
+      await promoteMemoryToProfileTargets({
+        ...options,
+        ...(toolScoped ? { toolScoped: true as const } : {}),
+      });
+      if (lifecycleCaps.extractionScopeClassification && toolScoped) return;
       if (
         !shouldPromoteToShared(
           this.deps.config,
@@ -939,6 +945,7 @@ export class ExtractionPersistCoordinator {
           // Claim-level provenance spans (issue #1575 PR 2).
           ...(options.sources && options.sources.length > 0 ? { sources: options.sources } : {}),
           ...(options.provenance ? { provenance: options.provenance } : {}),
+          ...(toolScoped ? { toolScoped: true as const } : {}),
         });
         const promotedId = sharedPromotion.id;
         // #1645: if the shared namespace's own tombstone blocked this promotion,
@@ -2019,6 +2026,11 @@ export class ExtractionPersistCoordinator {
           ? "extraction-proactive"
           : "extraction";
       const extractionSourceConnector = sourceContext?.sourceConnector;
+      const factToolScoped = withholdToolScopedFromSharedNamespace({
+        content: fact.content,
+        sourceConnector: extractionSourceConnector,
+        procedureSteps: fact.procedureSteps,
+      });
 
       // Check for contradictions before writing (Phase 2B).
       // NOTE: This block was moved above the chunking branch so that the
@@ -2227,6 +2239,7 @@ export class ExtractionPersistCoordinator {
             // PR #2016: never defer here — fallible chunk/artifact writes follow this
             // durable parent .md; flush the hash now so a throw can't strand it (dup).
             deferHashIndexSave: false,
+            ...(factToolScoped ? { toolScoped: true as const } : {}),
           });
           const parentId = parentWrite.id;
           // #1645: surface the tombstone block and gate active post-write paths
@@ -2310,6 +2323,7 @@ export class ExtractionPersistCoordinator {
                   ...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
                   ...(fact.provenance ? { provenance: fact.provenance } : {}),
                   ...(extractionSourceConnector ? { sourceConnector: extractionSourceConnector } : {}),
+                  ...(factToolScoped ? { toolScoped: true as const } : {}),
                 },
               );
             }
@@ -2457,6 +2471,7 @@ export class ExtractionPersistCoordinator {
                 intentActionType: inferredIntent?.actionType,
                 intentEntityTypes: inferredIntent?.entityTypes,
                 ...(extractionSourceConnector ? { sourceConnector: extractionSourceConnector } : {}),
+                ...(factToolScoped ? { toolScoped: true as const } : {}),
               });
             }
             // v8.2: graph edge building for chunked memories. #1576: skip pending_review.
@@ -2598,6 +2613,7 @@ export class ExtractionPersistCoordinator {
         // #1909: defer the per-fact index flush to the batch save only when
         // fact dedup is on (the batch saver is a no-op otherwise).
         deferHashIndexSave: factDedupEnabled,
+        ...(factToolScoped ? { toolScoped: true as const } : {}),
       });
       const memoryId = factWrite.id;
       // #1645: surface the tombstone block; gate active post-write paths like #1576
@@ -2762,6 +2778,7 @@ export class ExtractionPersistCoordinator {
             intentActionType: inferredIntent?.actionType,
             intentEntityTypes: inferredIntent?.entityTypes,
             ...(extractionSourceConnector ? { sourceConnector: extractionSourceConnector } : {}),
+            ...(factToolScoped ? { toolScoped: true as const } : {}),
           });
         }
         // Register in the target storage content-hash index after successful

package/src/orchestration/extraction-run-helpers.ts

@@ -309,11 +309,11 @@ export async function runExtractionPostPersist(
 }
 
 /**
- * Advance the extraction liveness watermark for a successfully parsed empty
- * extraction (#2223) and record its processed fingerprint when applicable.
- * Fingerprinted/import runs treat the metadata save as load-bearing (dedupe
- * state) and propagate failures; normal live runs treat the stamp as
- * diagnostic liveness and fail open. Deadline errors always propagate.
+ * Record a processed fingerprint when applicable and advance the extraction
+ * liveness watermark only for a provider-backed empty success. Fingerprinted
+ * imports treat the metadata save as load-bearing dedupe state; normal live
+ * runs treat the stamp as diagnostic and fail open. Deadline errors always
+ * propagate.
  */
 export async function commitEmptySuccessExtraction(args: {
   storage: StorageManager;
@@ -321,6 +321,7 @@ export async function commitEmptySuccessExtraction(args: {
   extractionFingerprint: string | null;
   shouldPersistProcessedFingerprint: boolean;
   hasExtractionFailure: boolean;
+  extractionSkippedReason: ExtractionResult["extractionSkippedReason"];
   recordProcessedExtractionFingerprint: (storage: StorageManager, fingerprint: string, meta: MetaState) => Promise<unknown>;
   runDeadlineAware: <T>(operation: () => Promise<T>, phase: string, clearTimerOnError?: boolean) => Promise<T>;
   isDeadlineError: (error: unknown) => boolean;
@@ -331,26 +332,43 @@ export async function commitEmptySuccessExtraction(args: {
     extractionFingerprint,
     shouldPersistProcessedFingerprint,
     hasExtractionFailure,
+    extractionSkippedReason,
     recordProcessedExtractionFingerprint,
     runDeadlineAware,
     isDeadlineError,
   } = args;
-  let meta = existingMeta;
-  if (extractionFingerprint && shouldPersistProcessedFingerprint && !hasExtractionFailure) {
-    meta ??= await storage.loadMeta();
-    await recordProcessedExtractionFingerprint(storage, extractionFingerprint, meta);
-  }
-  if (!hasExtractionFailure) {
-    const livenessStampFatal = shouldPersistProcessedFingerprint;
-    try {
-      meta ??= await runDeadlineAware(() => storage.loadMeta(), "during_empty_success_liveness_load", false);
+  const shouldRecordFingerprint =
+    extractionFingerprint !== null &&
+    shouldPersistProcessedFingerprint &&
+    !hasExtractionFailure;
+  const shouldStampLiveness =
+    !hasExtractionFailure && extractionSkippedReason === undefined;
+  if (!shouldRecordFingerprint && !shouldStampLiveness) return;
+
+  const metadataFailureIsFatal = shouldRecordFingerprint;
+  try {
+    const meta =
+      existingMeta ??
+      await runDeadlineAware(
+        () => storage.loadMeta(),
+        "during_empty_success_liveness_load",
+        false,
+      );
+    if (shouldRecordFingerprint) {
+      await recordProcessedExtractionFingerprint(storage, extractionFingerprint, meta);
+    }
+    if (shouldStampLiveness) {
       meta.extractionCount += 1;
       meta.lastExtractionAt = new Date().toISOString();
-      await runDeadlineAware(() => storage.saveMeta(meta!), "during_empty_success_liveness_save", false);
-    } catch (err) {
-      if (isDeadlineError(err)) throw err;
-      if (livenessStampFatal) throw err;
-      log.warn("runExtraction: failed to stamp empty-success liveness watermark (non-fatal)", err);
     }
+    await runDeadlineAware(
+      () => storage.saveMeta(meta),
+      "during_empty_success_liveness_save",
+      false,
+    );
+  } catch (err) {
+    if (isDeadlineError(err)) throw err;
+    if (metadataFailureIsFatal) throw err;
+    log.warn("runExtraction: failed to stamp empty-success liveness watermark (non-fatal)", err);
   }
 }

package/src/orchestration/extraction-run.test.ts

@@ -93,7 +93,11 @@ interface Harness {
   setRespond: (fn: (turns: BufferTurn[]) => ExtractionResult | Promise<ExtractionResult>) => void;
   setPassiveCapture: (fn: () => void | Promise<void>) => void;
   setMemoryBox: (fn: () => void | Promise<void>) => void;
+  setBufferClear: (fn: () => void | Promise<void>) => void;
   setPersist: (fn: () => void | Promise<void>) => void;
+  setFingerprintRecorder: (
+    fn: ExtractionRunCoordinatorDeps["recordProcessedExtractionFingerprint"],
+  ) => void;
   recordedProcessedCount: () => number;
   run: (
     coord: ExtractionRunCoordinator,
@@ -140,6 +144,9 @@ async function makeHarness(overrides: Record<string, unknown> = {}): Promise<Har
   let persistHandler: () => void | Promise<void> = async () => {};
   let memoryBoxHandler: () => void | Promise<void> = async () => {};
   let recordedProcessedCount = 0;
+  let fingerprintRecorder:
+    ExtractionRunCoordinatorDeps["recordProcessedExtractionFingerprint"] =
+      async () => {};
 
   let threadingBlocked = false;
   let persistCalls = 0;
@@ -184,8 +191,9 @@ async function makeHarness(overrides: Record<string, unknown> = {}): Promise<Har
     requestQmdMaintenance: () => {},
     runTierMigrationCycle: async () => migrationSummary,
     getLastPersistExtractionDeferredCount: () => 0,
-    recordProcessedExtractionFingerprint: async () => {
+    recordProcessedExtractionFingerprint: async (storage, fingerprint, meta) => {
       recordedProcessedCount += 1;
+      await fingerprintRecorder(storage, fingerprint, meta);
     },
   });
 
@@ -210,6 +218,14 @@ async function makeHarness(overrides: Record<string, unknown> = {}): Promise<Har
     setMemoryBox: (fn) => {
       memoryBoxHandler = fn;
     },
+    setBufferClear: (fn) => {
+      buffer.clearAfterExtraction = async () => {
+        await fn();
+      };
+    },
+    setFingerprintRecorder: (fn) => {
+      fingerprintRecorder = fn;
+    },
     setPersist: (fn) => {
       persistHandler = fn;
     },
@@ -685,6 +701,32 @@ test("circuit breaker: a forced flush still calls extract() while the breaker is
   }
 });
 
+test("circuit breaker: a forced local prefilter skip does not heal an open breaker", async () => {
+  const h = await makeHarness({
+    extractionRetryScheduleMs: [3_600_000],
+    extractionBreakerFailureThreshold: 1,
+    extractionBreakerCooldownMs: 3_600_000,
+  });
+  try {
+    const coord = h.newCoordinator();
+    h.setRespond(() => failureResult("provider_retryable"));
+    await h.run(coord, "fp-1");
+    assert.equal(coord.getExtractionResilienceStatus().breaker.state, "open");
+
+    h.setRespond(() => ({
+      ...emptySuccessResult(),
+      extractionSkippedReason: "mechanical_telemetry",
+    }));
+    const result = await h.run(coord, "fp-2", { force: true });
+
+    assert.equal(result.reason, "empty_extraction_result");
+    assert.equal(coord.getExtractionResilienceStatus().breaker.state, "open");
+    assert.equal(coord.getExtractionResilienceStatus().breaker.consecutiveFailures, 1);
+  } finally {
+    await h.cleanup();
+  }
+});
+
 test("circuit breaker: auth_config failure opens the breaker immediately (no hot loop)", async () => {
   const h = await makeHarness({
     extractionRetryScheduleMs: [3_600_000],
@@ -1013,7 +1055,7 @@ test("extraction liveness (#2223): a normal live empty success advances lastExtr
     const result = await coord.runExtraction(liveTurns("nothing-durable-here"), {
       skipCharThreshold: true,
       skipUserTurnThreshold: true,
-      clearBufferAfterExtraction: false,
+      clearBufferAfterExtraction: true,
       bufferKey: "nothing-durable-here",
     });
 
@@ -1030,6 +1072,133 @@ test("extraction liveness (#2223): a normal live empty success advances lastExtr
   }
 });
 
+test("extraction liveness (#2227): prefilter skips do not advance the watermark", async () => {
+  for (const extractionSkippedReason of [
+    "conversation_only_non_memory",
+    "mechanical_telemetry",
+  ] as const) {
+    const h = await makeHarness();
+    try {
+      h.setRespond(() => ({ ...emptySuccessResult(), extractionSkippedReason }));
+      const storage = await h.storageForNs("default");
+      const before = await storage.loadMeta();
+
+      const result = await h.newCoordinator().runExtraction(liveTurns(extractionSkippedReason), {
+        skipCharThreshold: true,
+        skipUserTurnThreshold: true,
+        clearBufferAfterExtraction: true,
+        bufferKey: extractionSkippedReason,
+      });
+
+      assert.equal(result.reason, "empty_extraction_result");
+      const after = await storage.loadMeta();
+      assert.equal(after.extractionCount, before.extractionCount);
+      assert.equal(after.lastExtractionAt, before.lastExtractionAt);
+    } finally {
+      await h.cleanup();
+    }
+  }
+});
+
+test("extraction liveness (#2227): passive-correction failure prevents a live empty-success stamp", async () => {
+  const h = await makeHarness();
+  try {
+    h.setRespond(() => emptySuccessResult());
+    h.setPassiveCapture(() => {
+      throw new Error("passive capture failed");
+    });
+    const storage = await h.storageForNs("default");
+    const before = await storage.loadMeta();
+
+    await assert.rejects(
+      h.newCoordinator().runExtraction(liveTurns("passive-capture-failure"), {
+        skipCharThreshold: true,
+        skipUserTurnThreshold: true,
+        clearBufferAfterExtraction: true,
+        bufferKey: "passive-capture-failure",
+      }),
+      /passive capture failed/,
+    );
+
+    const after = await storage.loadMeta();
+    assert.equal(after.extractionCount, before.extractionCount);
+    assert.equal(after.lastExtractionAt, before.lastExtractionAt);
+  } finally {
+    await h.cleanup();
+  }
+});
+
+test("extraction liveness (#2227): buffer-clear failure prevents a live empty-success stamp", async () => {
+  const h = await makeHarness();
+  try {
+    h.setRespond(() => emptySuccessResult());
+    h.setBufferClear(() => {
+      throw new Error("buffer clear failed");
+    });
+    const storage = await h.storageForNs("default");
+    const before = await storage.loadMeta();
+
+    await assert.rejects(
+      h.newCoordinator().runExtraction(liveTurns("buffer-clear-failure"), {
+        skipCharThreshold: true,
+        skipUserTurnThreshold: true,
+        clearBufferAfterExtraction: true,
+        bufferKey: "buffer-clear-failure",
+      }),
+      /buffer clear failed/,
+    );
+
+    const after = await storage.loadMeta();
+    assert.equal(after.extractionCount, before.extractionCount);
+    assert.equal(after.lastExtractionAt, before.lastExtractionAt);
+  } finally {
+    await h.cleanup();
+  }
+});
+
+test("extraction liveness (#2227): a fingerprinted prefilter skip persists dedupe state without stamping liveness", async () => {
+  const h = await makeHarness();
+  try {
+    h.setRespond(() => ({
+      ...emptySuccessResult(),
+      extractionSkippedReason: "mechanical_telemetry",
+    }));
+    h.setFingerprintRecorder(async (_storage, fingerprint, meta) => {
+      assert.ok(meta, "empty-success commit supplies preloaded metadata");
+      meta.processedExtractionFingerprints = [
+        ...(meta.processedExtractionFingerprints ?? []),
+        { fingerprint, observedAt: new Date().toISOString() },
+      ];
+    });
+    const storage = await h.storageForNs("default");
+    const before = await storage.loadMeta();
+    const baselineFingerprintCount =
+      before.processedExtractionFingerprints?.length ?? 0;
+
+    const result = await h.newCoordinator().runExtraction(
+      makeTurns("fingerprinted-prefilter"),
+      {
+        skipCharThreshold: true,
+        skipUserTurnThreshold: true,
+        clearBufferAfterExtraction: true,
+        bufferKey: "fingerprinted-prefilter",
+      },
+    );
+
+    assert.equal(result.reason, "empty_extraction_result");
+    const after = await storage.loadMeta();
+    assert.equal(after.extractionCount, before.extractionCount);
+    assert.equal(after.lastExtractionAt, before.lastExtractionAt);
+    assert.equal(
+      after.processedExtractionFingerprints?.length,
+      baselineFingerprintCount + 1,
+    );
+    assert.equal(h.recordedProcessedCount(), 1);
+  } finally {
+    await h.cleanup();
+  }
+});
+
 test("extraction liveness (#2223): a fingerprinted empty success advances the watermark once and records the fingerprint once", async () => {
   const h = await makeHarness();
   try {

package/src/orchestration/extraction-run.ts

@@ -979,7 +979,7 @@ export class ExtractionRunCoordinator {
           // pre-#1908 clear-buffer behavior applies below.
           log.warn("runExtraction: failed to load meta for retry bookkeeping (non-fatal)", err);
         }
-      } else {
+      } else if (result.extractionSkippedReason === undefined) {
         // Provider responded without failure → breaker heals; clear any
         // parked backoff for this fingerprint so it proceeds normally.
         this.resetProviderBreakerOnSuccess();
@@ -1016,19 +1016,20 @@ export class ExtractionRunCoordinator {
           { extractionFailure }
         );
       }
-      await commitEmptySuccessExtraction({
+      const emptySuccessCommit = {
         storage,
         meta,
         extractionFingerprint,
         shouldPersistProcessedFingerprint,
         hasExtractionFailure: Boolean(extractionFailure),
+        extractionSkippedReason: result.extractionSkippedReason,
         recordProcessedExtractionFingerprint: this.deps.recordProcessedExtractionFingerprint,
         runDeadlineAware,
-        isDeadlineError: (error) => error instanceof ExtractionDeadlineError,
-      });
-      // Correction-only turns that meet char/user-turn thresholds but yield
-      // zero facts still need passive capture (review: "empty extraction skips
-      // capture"). selfNamespace/principal already resolved above.
+        isDeadlineError: (error: unknown) => error instanceof ExtractionDeadlineError,
+      };
+      if (shouldPersistProcessedFingerprint) {
+        await commitEmptySuccessExtraction(emptySuccessCommit);
+      }
       await runPassiveCapture(normalizedTurns as BufferTurn[], {
         sessionKey,
         principal,
@@ -1037,16 +1038,13 @@ export class ExtractionRunCoordinator {
         isLiveSession: clearBufferAfterExtraction,
       });
       if (recordedRetryFailure) {
-        // Retain the failed turns so the backoff gate can re-attempt them after
-        // nextEligibleAt. Clearing here would lose the data the retry state
-        // points at (cursor high + codex P1). Trigger/forced paths already
-        // retain via clearBufferAfterExtraction=false; this covers the normal
-        // live-session path. Overflow is bounded by MAX_BUFFER_ENTRY_COUNT
-        // with a loud eviction log.
         log.debug("runExtraction: retaining buffer for backoff retry after failure");
       } else {
         await clearBuffer();
       }
+      if (!shouldPersistProcessedFingerprint) {
+        await commitEmptySuccessExtraction(emptySuccessCommit);
+      }
       return { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
     }