@remnic/core 9.66.0 → 9.66.2

package/src/orchestration/extraction-persist.ts

@@ -83,6 +83,7 @@ import {
 } from "../fallback-llm.js";
 import { EmbeddingFallback } from "../embedding-fallback.js";
 import { decideSemanticDedup, type SemanticDedupDecision, type SemanticDedupHit } from "../dedup/semantic.js";
+import { applyNoveltyGate, embeddingsFromCosineHits } from "../dedup/novelty-gate.js";
 import { selectRouteRule, type RouteRule, type RoutingEngineOptions } from "../routing/engine.js";
 import { ThreadingManager } from "../threading.js";
 import { NamespaceStorageRouter } from "../namespaces/storage.js";
@@ -1919,26 +1920,24 @@ export class ExtractionPersistCoordinator {
           : undefined;
       const faithfulnessEnforceStatus = faithfulnessGateStatus ?? requireSpansPendingStatus ?? injectionScreenStatus;
 
-      // Issue #373 — write-time semantic similarity guard. Hook runs after
-      // the exact content-hash miss and the importance gate so that:
-      //   (a) paraphrased near-duplicates never reach writeMemory(), and
-      //   (b) low-importance facts that will be dropped never trigger an
-      //       embedding lookup (avoids unnecessary API latency/cost).
-      // Fails open when the embedding backend is unavailable.
-      //
-      // Defense in depth (PR #399 review): decideSemanticDedup already
-      // catches lookup errors internally, and the embedding fetch is
-      // bounded by a timeout in embedding-fallback.ts. We still wrap the
-      // whole call in its own try/catch here so that any unexpected
-      // rejection (future refactors, misbehaving custom backends, etc.)
-      // can never block the persist loop — a failure in the dedup path
-      // must always default to "not a duplicate".
-      // Track a pending semantic-skip decision (populated inside the block
-      // below). The actual drop happens AFTER contradiction detection so that
-      // a high-similarity update/correction is linked as a superseding
-      // contradiction rather than silently dropped.
+      const novelty = await applyNoveltyGate({
+        enabled: this.deps.config["noveltyGateEnabled"] === true,
+        addThreshold: this.deps.config.noveltyAddThreshold,
+        noopThreshold: this.deps.config.noveltyNoopThreshold,
+        lookup: async () =>
+          embeddingsFromCosineHits(
+            await this.deps.semanticDedupLookup(
+              fact.content,
+              this.deps.config.semanticDedupCandidates,
+              targetStorage,
+            ),
+            // Same connector scoping as decideSemanticDedup below: a
+            // cross-connector neighbor must not noop this candidate.
+            { candidateConnector: sourceContext?.sourceConnector },
+          ),
+      });
       let pendingSemanticSkip: (SemanticDedupDecision & { action: "skip" }) | null = null;
-      if (resolvePipelineProcessingCapabilities(this.deps.config).semanticDedup && injectionScreenStatus !== "pending_review") {
+      if (novelty.decision !== "add" && resolvePipelineProcessingCapabilities(this.deps.config).semanticDedup && injectionScreenStatus !== "pending_review") {
         let semanticDecision: SemanticDedupDecision;
         // UUI2: skip embedding lookup for the rest of this batch once we know
         // the backend is unavailable. The flag is reset per-batch (set to false
@@ -2100,17 +2099,17 @@ export class ExtractionPersistCoordinator {
       // gate's contract is "persists with status: pending_review, never
       // silently dropped" (issue #1576).
       if (
-        pendingSemanticSkip &&
+        (pendingSemanticSkip || novelty.decision === "noop") &&
         !contradictionDetected &&
         !isCorrection &&
         faithfulnessEnforceStatus !== "pending_review"
       ) {
         log.debug(
-          `dedup: skipping semantic near-duplicate fact "${fact.content
+          `dedup: skipping ${pendingSemanticSkip ? "semantic near-duplicate" : "novelty-noop"} fact "${fact.content
             .slice(0, 60)
-            .replace(/\s+/g, " ")}…" score=${pendingSemanticSkip.topScore.toFixed(
+            .replace(/\s+/g, " ")}…" score=${(pendingSemanticSkip?.topScore ?? novelty.score).toFixed(
             3,
-          )} neighbor=${pendingSemanticSkip.topId}`,
+          )} neighbor=${pendingSemanticSkip?.topId ?? novelty.neighborId}`,
         );
         dedupedCount++;
         // Do NOT add fact.content to contentHashIndex here. No memory was

package/src/orchestration/generic-recall-paths.ts

@@ -9,6 +9,7 @@ import {
   isLocationDayPath,
   isMeetingRecordPath,
 } from "./orchestrator-helpers.js";
+import { isJournalDayPath } from "../activity/journal.js";
 import { OKF_RESERVED_BASENAMES } from "../okf/type-mapping.js";
 
 
@@ -137,6 +138,7 @@ export function isSearchExcludedPath(
       isArtifactMemoryPath(candidate) ||
       isNamespacedActivityDigestPath(candidate, policy.memoryDir) ||
       isNamespacedLocationDayPath(candidate, policy.memoryDir) ||
+      isNamespacedJournalDayPath(candidate, policy.memoryDir) ||
       isMeetingRecordPath(candidate)
     ) {
       return true;
@@ -179,6 +181,15 @@ function isNamespacedLocationDayPath(candidate: string, memoryDir?: string): boo
   return isLocationDayPath(namespaced[2] ?? "", memoryDir);
 }
 
+function isNamespacedJournalDayPath(candidate: string, memoryDir?: string): boolean {
+  if (isJournalDayPath(candidate, memoryDir)) return true;
+  if (memoryDir === undefined || memoryDir.length === 0) return false;
+  const relative = path.relative(memoryDir, path.resolve(memoryDir, candidate));
+  const namespaced = /^namespaces[\\/]([^\\/]+)[\\/](.*)$/.exec(relative);
+  if (namespaced === null) return false;
+  return isJournalDayPath(namespaced[2] ?? "", memoryDir);
+}
+
 /** A path as given, plus its form with a leading COLLECTION segment removed. */
 function collectionSpellings(
   filePath: string,

package/src/orchestration/recall-entry.ts

@@ -20,6 +20,7 @@ import { ProfilingCollector } from "../profiling.js";
 import { trustResultFor, type TrustStageResultItem } from "../trust-score-stage.js";
 import type { RecallResultFormatter } from "./recall-result-formatter.js";
 import type { IdentityInjectionMode, PluginConfig, QmdSearchResult } from "../types.js";
+import { applyRecallStateViews } from "../recall-state-view-wire.js";
 
 export interface RecallEntryDeps {
   appendRecallSection(
@@ -274,9 +275,13 @@ export class RecallEntryCoordinator {
   }): void {
     const sectionId = "memories";
     const trustByPath = options.trustByPath ?? null;
-    const injectable = trustByPath
-      ? options.results.filter((r) => !trustResultFor(trustByPath, r)?.quarantined)
-      : options.results;
+    const injectable = applyRecallStateViews(
+      trustByPath
+        ? options.results.filter((r) => !trustResultFor(trustByPath, r)?.quarantined)
+        : options.results,
+      options.retrievalQuery,
+      this.deps.config,
+    );
     if (injectable.length === 0) return;
 
     const formatted = this.deps.recallResultFormatter.formatQmdResultEntries(

package/src/recall-deep-access.test.ts

@@ -0,0 +1,58 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { runDeepRecallAccess } from "./recall-deep-access.js";
+
+const QUERY = "who met at the cafe";
+
+test("budget 0 stops immediately with a tagged refusal", () => {
+  const result = runDeepRecallAccess({ query: QUERY, budget: 0, policyName: "expand-once" });
+  assert.equal(result.ok, false);
+  assert.deepEqual(result.refusal, { tag: "budget_exhausted" });
+  assert.equal(result.trace.length, 1);
+  assert.equal(result.trace[0]?.action, "stop");
+  assert.equal(result.state.budget, 0);
+  assert.equal(result.state.query, QUERY);
+  assert.deepEqual(result.state.workingSet, []);
+  assert.deepEqual(result.state.frontier, []);
+  assert.equal(result.traceJson, JSON.stringify(result.trace));
+});
+
+test("unknown policyName is rejected", () => {
+  assert.throws(
+    () => runDeepRecallAccess({ query: QUERY, budget: 2, policyName: "refine-twice" }),
+    /unknown deep recall access policy/,
+  );
+  assert.throws(
+    () => runDeepRecallAccess({ query: QUERY, budget: 2, policyName: "refine-twice" }),
+    /stop, expand-once/,
+  );
+});
+
+test("stop policy ends without expanding", () => {
+  const result = runDeepRecallAccess({ query: QUERY, budget: 3, policyName: "stop" });
+  assert.equal(result.ok, true);
+  assert.equal(result.refusal, undefined);
+  assert.equal(result.trace.length, 1);
+  assert.equal(result.trace[0]?.action, "stop");
+  assert.equal(result.state.budget, 3);
+  assert.deepEqual(result.state.workingSet, []);
+});
+
+test("expand-once expands once then stops", () => {
+  const result = runDeepRecallAccess({ query: QUERY, budget: 3, policyName: "expand-once" });
+  assert.equal(result.ok, true);
+  assert.deepEqual(
+    result.trace.map((step) => step.action),
+    ["expand", "stop"],
+  );
+  assert.equal(result.state.budget, 2);
+});
+
+test("same input produces the same trace JSON", () => {
+  const input = { query: QUERY, budget: 4, policyName: "expand-once" as const };
+  const first = runDeepRecallAccess(input);
+  const second = runDeepRecallAccess(input);
+  assert.equal(first.traceJson, second.traceJson);
+  assert.equal(first.traceJson, JSON.stringify(first.trace));
+});

package/src/recall-deep-access.ts

@@ -0,0 +1,71 @@
+/**
+ * Named-policy access facade for the deep-recall stepper (issue #2332 leftover).
+ *
+ * Not the hot path. Surfaces and parseConfig wait. Policy names are
+ * allow-listed. Budget 0 returns a tagged refusal without a policy call.
+ */
+
+import { runDeepRecall, type DeepRecallResult, type DeepRecallState } from "./recall-deep.js";
+
+export const DEEP_RECALL_ACCESS_POLICIES = ["stop", "expand-once"] as const;
+
+export type DeepRecallAccessPolicyName = (typeof DEEP_RECALL_ACCESS_POLICIES)[number];
+
+export interface DeepRecallAccessInput {
+  query: string;
+  budget: number;
+  policyName: string;
+}
+
+export interface DeepRecallAccessRefusal {
+  tag: "budget_exhausted";
+}
+
+export interface DeepRecallAccessResult {
+  ok: boolean;
+  refusal?: DeepRecallAccessRefusal;
+  state: DeepRecallState;
+  trace: DeepRecallResult["trace"];
+  traceJson: string;
+}
+
+export function isDeepRecallAccessPolicyName(value: unknown): value is DeepRecallAccessPolicyName {
+  return typeof value === "string" && (DEEP_RECALL_ACCESS_POLICIES as readonly string[]).includes(value);
+}
+
+export function runDeepRecallAccess(input: DeepRecallAccessInput): DeepRecallAccessResult {
+  if (!isDeepRecallAccessPolicyName(input.policyName)) {
+    throw new Error(
+      `unknown deep recall access policy: ${JSON.stringify(input.policyName)}. Valid: ${DEEP_RECALL_ACCESS_POLICIES.join(", ")}`,
+    );
+  }
+
+  const start: DeepRecallState = { query: input.query, workingSet: [], frontier: [], budget: input.budget };
+  if (!Number.isFinite(input.budget) || input.budget <= 0) {
+    const result = runDeepRecall(start, () => {
+      throw new Error("policy must not run when budget is 0");
+    });
+    return {
+      ok: false,
+      refusal: { tag: "budget_exhausted" },
+      state: result.state,
+      trace: result.trace,
+      traceJson: JSON.stringify(result.trace),
+    };
+  }
+
+  let expanded = false;
+  const result = runDeepRecall(start, () => {
+    if (input.policyName === "expand-once" && !expanded) {
+      expanded = true;
+      return "expand";
+    }
+    return "stop";
+  });
+  return {
+    ok: true,
+    state: result.state,
+    trace: result.trace,
+    traceJson: JSON.stringify(result.trace),
+  };
+}

package/src/recall-deep-render.test.ts

@@ -0,0 +1,68 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  runDeepRecall,
+  type DeepRecallState,
+} from "./recall-deep.js";
+import { renderDeepRecallTrace } from "./recall-deep-render.js";
+
+function seed(extra: Partial<DeepRecallState> = {}): DeepRecallState {
+  return {
+    query: "who met at the cafe",
+    workingSet: [],
+    frontier: ["m-b", "m-c"],
+    budget: 2,
+    ...extra,
+  };
+}
+
+test("empty working set prints (empty)", () => {
+  const result = runDeepRecall(seed({ budget: 0 }), () => {
+    throw new Error("policy must not run when budget is 0");
+  });
+  assert.equal(
+    renderDeepRecallTrace(result),
+    [
+      "# Deep recall",
+      "",
+      "- query: who met at the cafe",
+      "- budget: 0",
+      "- stop: budget exhausted",
+      "",
+      "## Steps",
+      "",
+      "1. stop budget=0 workingSet=(empty) frontier=m-b, m-c",
+      "",
+    ].join("\n"),
+  );
+});
+
+test("expand then stop snapshot", () => {
+  const result = runDeepRecall(seed({ workingSet: ["m-seed"], budget: 2 }), (state) =>
+    state.workingSet.includes("m-b") ? "stop" : "expand",
+  );
+  assert.equal(
+    renderDeepRecallTrace(result),
+    [
+      "# Deep recall",
+      "",
+      "- query: who met at the cafe",
+      "- budget: 1",
+      "- stop: stop",
+      "",
+      "## Steps",
+      "",
+      "1. expand budget=1 workingSet=m-seed, m-b frontier=m-c",
+      "2. stop budget=1 workingSet=m-seed, m-b frontier=m-c",
+      "",
+    ].join("\n"),
+  );
+});
+
+test("same result renders the same bytes", () => {
+  const policy = () => "stop";
+  const first = renderDeepRecallTrace(runDeepRecall(seed({ budget: 3 }), policy));
+  const second = renderDeepRecallTrace(runDeepRecall(seed({ budget: 3 }), policy));
+  assert.equal(first, second);
+});

package/src/recall-deep-render.ts

@@ -0,0 +1,32 @@
+/**
+ * Markdown renderer for a deep-recall trace (issue #2332 leftover).
+ *
+ * Deterministic. Surfaces wait. Empty working set prints (empty).
+ */
+
+import type { DeepRecallResult } from "./recall-deep.js";
+
+export function renderDeepRecallTrace(result: DeepRecallResult): string {
+  const last = result.trace[result.trace.length - 1];
+  const stop = !last ? "none" : last.budget <= 0 ? "budget exhausted" : last.action;
+  const steps =
+    result.trace.length === 0
+      ? ["(empty)"]
+      : result.trace.map((step, index) => {
+          const workingSet = step.workingSet.length === 0 ? "(empty)" : step.workingSet.join(", ");
+          const frontier = step.frontier.length === 0 ? "(empty)" : step.frontier.join(", ");
+          return `${index + 1}. ${step.action} budget=${step.budget} workingSet=${workingSet} frontier=${frontier}`;
+        });
+  return [
+    "# Deep recall",
+    "",
+    `- query: ${result.state.query}`,
+    `- budget: ${result.state.budget}`,
+    `- stop: ${stop}`,
+    "",
+    "## Steps",
+    "",
+    ...steps,
+    "",
+  ].join("\n");
+}

package/src/recall-deep.test.ts

@@ -0,0 +1,72 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  runDeepRecall,
+  runDeepRecallStep,
+  type DeepRecallState,
+} from "./recall-deep.js";
+
+function seed(extra: Partial<DeepRecallState> = {}): DeepRecallState {
+  return {
+    query: "who met at the cafe",
+    workingSet: ["m-seed"],
+    frontier: ["m-b", "m-c"],
+    budget: 2,
+    ...extra,
+  };
+}
+
+test("budget 0 stops without calling the policy", () => {
+  const result = runDeepRecall(seed({ budget: 0 }), () => {
+    throw new Error("policy must not run when budget is 0");
+  });
+  assert.equal(result.trace.length, 1);
+  assert.equal(result.trace[0]?.action, "stop");
+  assert.equal(result.state.budget, 0);
+  assert.deepEqual(result.state.workingSet, ["m-seed"]);
+  assert.deepEqual(result.state.frontier, ["m-b", "m-c"]);
+});
+
+test("expand consumes budget and pulls the next frontier item", () => {
+  const result = runDeepRecallStep(seed({ budget: 2 }), "expand");
+  assert.equal(result.step.action, "expand");
+  assert.equal(result.state.budget, 1);
+  assert.deepEqual(result.state.workingSet, ["m-seed", "m-b"]);
+  assert.deepEqual(result.state.frontier, ["m-c"]);
+});
+
+test("stop ends while budget remains", () => {
+  let calls = 0;
+  const result = runDeepRecall(seed({ budget: 4 }), () => {
+    calls += 1;
+    return "stop";
+  });
+  assert.equal(calls, 1);
+  assert.equal(result.trace.length, 1);
+  assert.equal(result.trace[0]?.action, "stop");
+  assert.equal(result.state.budget, 4);
+  assert.deepEqual(result.state.workingSet, ["m-seed"]);
+});
+
+test("unknown action throws", () => {
+  assert.throws(() => runDeepRecallStep(seed(), "rewrite"), /unknown deep recall action/);
+  assert.throws(
+    () => runDeepRecall(seed(), () => "rewrite"),
+    /unknown deep recall action/,
+  );
+});
+
+test("same state and policy produce the same trace", () => {
+  const policy = (state: DeepRecallState) => (state.budget > 1 ? "expand" : "refine");
+  const start = seed({ budget: 3, frontier: ["m-z", "m-a"] });
+  const first = runDeepRecall(start, policy);
+  const second = runDeepRecall(start, policy);
+  assert.deepEqual(first, second);
+  assert.deepEqual(
+    first.trace.map((step) => step.action),
+    ["expand", "expand", "refine", "stop"],
+  );
+  assert.equal(start.budget, 3);
+  assert.deepEqual(start.frontier, ["m-z", "m-a"]);
+});

package/src/recall-deep.ts

@@ -0,0 +1,113 @@
+/**
+ * Budgeted REFINE/EXPAND/STOP loop (issue #2332 first slice).
+ *
+ * Pure state stepper. Policy is injected. No LLM, no orchestrator recall,
+ * no IO. Surfaces and parseConfig wait for a later PR — config.ts is at
+ * its fileSizeGrandfather ceiling.
+ *
+ * Budget 0 stops without calling the policy. Unknown actions throw.
+ */
+export const DEEP_RECALL_ACTIONS = ["refine", "expand", "stop"] as const;
+
+export type DeepRecallAction = (typeof DEEP_RECALL_ACTIONS)[number];
+
+export interface DeepRecallState {
+  query: string;
+  workingSet: readonly string[];
+  frontier: readonly string[];
+  budget: number;
+}
+
+export interface DeepRecallTraceStep {
+  action: DeepRecallAction;
+  budget: number;
+  workingSet: readonly string[];
+  frontier: readonly string[];
+}
+
+export interface DeepRecallStepResult {
+  state: DeepRecallState;
+  step: DeepRecallTraceStep;
+}
+
+export interface DeepRecallResult {
+  state: DeepRecallState;
+  trace: readonly DeepRecallTraceStep[];
+}
+
+export type DeepRecallPolicy = (state: DeepRecallState) => string;
+
+export function isDeepRecallAction(value: unknown): value is DeepRecallAction {
+  return typeof value === "string" && (DEEP_RECALL_ACTIONS as readonly string[]).includes(value);
+}
+
+function snapshot(state: DeepRecallState): DeepRecallState {
+  return {
+    query: state.query,
+    workingSet: [...state.workingSet],
+    frontier: [...state.frontier],
+    budget: state.budget,
+  };
+}
+
+export function runDeepRecallStep(state: DeepRecallState, action: string): DeepRecallStepResult {
+  if (!isDeepRecallAction(action)) {
+    throw new Error(
+      `unknown deep recall action: ${JSON.stringify(action)}. Valid: ${DEEP_RECALL_ACTIONS.join(", ")}`,
+    );
+  }
+  if (!Number.isFinite(state.budget) || state.budget <= 0 || action === "stop") {
+    const stopped = snapshot(state);
+    return {
+      state: stopped,
+      step: {
+        action: "stop",
+        budget: stopped.budget,
+        workingSet: stopped.workingSet,
+        frontier: stopped.frontier,
+      },
+    };
+  }
+
+  const next = snapshot(state);
+  next.budget = state.budget - 1;
+  if (action === "expand") {
+    const [head, ...rest] = next.frontier;
+    if (head !== undefined && !next.workingSet.includes(head)) {
+      next.workingSet = [...next.workingSet, head];
+    }
+    next.frontier = rest;
+  }
+  return {
+    state: next,
+    step: {
+      action,
+      budget: next.budget,
+      workingSet: next.workingSet,
+      frontier: next.frontier,
+    },
+  };
+}
+
+export function runDeepRecall(state: DeepRecallState, policy: DeepRecallPolicy): DeepRecallResult {
+  const trace: DeepRecallTraceStep[] = [];
+  let current = snapshot(state);
+  while (true) {
+    if (!Number.isFinite(current.budget) || current.budget <= 0) {
+      const stopped = snapshot(current);
+      trace.push({
+        action: "stop",
+        budget: stopped.budget,
+        workingSet: stopped.workingSet,
+        frontier: stopped.frontier,
+      });
+      return { state: stopped, trace };
+    }
+    const stepped = runDeepRecallStep(current, policy(current));
+    trace.push(stepped.step);
+    current = stepped.state;
+    if (stepped.step.action === "stop") {
+      return { state: current, trace };
+    }
+  }
+}

package/src/recall-navigate.test.ts

@@ -0,0 +1,84 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  expandRecallNode,
+  traverseRecallLink,
+  type RecallNavNode,
+} from "./recall-navigate.js";
+
+function node(extra: Partial<RecallNavNode> = {}): RecallNavNode {
+  return {
+    id: "m1",
+    disclosure: "chunk",
+    ...extra,
+  };
+}
+
+test("expandRecallNode budget 0 is a no-op unavailable", () => {
+  const source = node({
+    payloads: { section: "full section body" },
+  });
+  const result = expandRecallNode(source, { budget: 0 });
+  assert.equal(result.status, "unavailable");
+  if (result.status === "unavailable") {
+    assert.equal(result.reason, "budget_off");
+  }
+  assert.equal("node" in result, false);
+  assert.equal(source.disclosure, "chunk");
+});
+
+test("traverseRecallLink budget 0 is a no-op unavailable", () => {
+  const result = traverseRecallLink(
+    node({
+      links: [{ targetId: "m2", linkType: "supports" }],
+    }),
+    "supports",
+    { budget: 0 },
+  );
+  assert.equal(result.status, "unavailable");
+  if (result.status === "unavailable") {
+    assert.equal(result.reason, "budget_off");
+  }
+  assert.equal("neighbors" in result, false);
+});
+
+test("unknown linkType throws", () => {
+  assert.throws(
+    () => traverseRecallLink(node({ links: [] }), "related", { budget: 10 }),
+    /unknown recall nav linkType/,
+  );
+});
+
+test("traverseRecallLink returns neighbors in deterministic targetId order", () => {
+  const result = traverseRecallLink(
+    node({
+      links: [
+        { targetId: "z-mem", linkType: "contradicts" },
+        { targetId: "a-mem", linkType: "contradicts" },
+        { targetId: "m-mem", linkType: "contradicts" },
+        { targetId: "skip-me", linkType: "supports" },
+      ],
+    }),
+    "contradicts",
+    { budget: 100 },
+  );
+  assert.equal(result.status, "ok");
+  if (result.status !== "ok") return;
+  assert.deepEqual(
+    result.neighbors.map((row) => row.id),
+    ["a-mem", "m-mem", "z-mem"],
+  );
+});
+
+test("no matching links is empty, not unavailable", () => {
+  const result = traverseRecallLink(
+    node({
+      links: [{ targetId: "x", linkType: "supports" }],
+    }),
+    "contradicts",
+    { budget: 10 },
+  );
+  assert.equal(result.status, "empty");
+  assert.equal("neighbors" in result, false);
+});