@remnic/core 9.3.613 → 9.3.615

package/src/operator-toolkit.ts

@@ -89,6 +89,7 @@ interface QmdRuntimeLike {
   isAvailable(): boolean;
   ensureCollection(
     memoryDir: string,
+    collectionOrExecution?: string | { signal?: AbortSignal },
     execution?: { signal?: AbortSignal },
   ): Promise<"present" | "missing" | "unknown" | "skipped">;
   debugStatus(): string;
@@ -730,7 +731,10 @@ export async function runOperatorSetup(options: OperatorSetupOptions): Promise<O
 
   const qmdAvailable = await options.orchestrator.qmd.probe();
   const collectionState = options.orchestrator.config.qmdEnabled
-    ? await options.orchestrator.qmd.ensureCollection(options.orchestrator.config.memoryDir)
+    ? await options.orchestrator.qmd.ensureCollection(
+        options.orchestrator.config.memoryDir,
+        options.orchestrator.config.qmdCollection,
+      )
     : "skipped";
   const nativeKnowledgeStatus = await summarizeNativeKnowledgeStatus(options.orchestrator.config);
 
@@ -1108,7 +1112,7 @@ export async function runOperatorDoctor(options: OperatorDoctorOptions): Promise
 
   const qmdAvailable = await options.orchestrator.qmd.probe();
   const collectionState = config.qmdEnabled
-    ? await options.orchestrator.qmd.ensureCollection(config.memoryDir)
+    ? await options.orchestrator.qmd.ensureCollection(config.memoryDir, config.qmdCollection)
     : "skipped";
   checks.push({
     key: "qmd",

package/src/orchestrator.ts

@@ -263,6 +263,7 @@ import {
 import { normalizeReplaySessionKey, type ReplayTurn } from "./replay/types.js";
 import type { ImportTurn } from "./bulk-import/types.js";
 import {
+  type AgentPersonaModelConfig,
   confidenceTier,
   type MemoryIntent,
   type MemorySummary,
@@ -529,6 +530,32 @@ export interface RecallModeDecision {
   effectiveMode: RecallPlanMode;
   graphExpandedIntentDetected: boolean;
   graphReason?: string;
+  /**
+   * Where `plannedMode` came from (issue #1367 / Option C). `"heuristic"` for
+   * the regex planner; `"llm"` when the LLM planner classified it; and
+   * `"heuristic-fallback"` when the LLM was enabled but errored/timed out and we
+   * fell back. Absent on the synchronous heuristic-only path.
+   */
+  plannerSource?: "heuristic" | "llm" | "heuristic-fallback";
+  /** Short rationale from the planner (for telemetry / x-ray). */
+  plannerReason?: string;
+  /** Wall-clock spent in the LLM planner call, when one was made. */
+  plannerLatencyMs?: number;
+  /** True when the LLM planner was enabled but fell back to the heuristic. */
+  plannerFallbackUsed?: boolean;
+  /** Model that served the LLM classification, when one was used. */
+  plannerModelUsed?: string;
+  /**
+   * The regex-heuristic baseline mode, captured whenever the LLM planner ran
+   * (any source). Lets operators compare planned-vs-heuristic during rollout —
+   * distinct from `plannedMode`, which on the LLM path is the LLM's choice.
+   */
+  plannerHeuristicMode?: RecallPlanMode;
+  /**
+   * In shadow mode, the mode the LLM *would* have chosen (recorded for
+   * comparison) while `effectiveMode` stays on the heuristic decision.
+   */
+  shadowLlmMode?: RecallPlanMode;
 }
 
 /**
@@ -1052,16 +1079,27 @@ export function resolveEffectiveRecallMode(options: {
   return resolveRecallModeDecision(options).effectiveMode;
 }
 
-export function resolveRecallModeDecision(options: {
+interface RecallModeGraphOptions {
   plannerEnabled: boolean;
   graphRecallEnabled: boolean;
   multiGraphMemoryEnabled: boolean;
   graphExpandedIntentEnabled?: boolean;
   prompt: string;
-}): RecallModeDecision {
-  let plannedMode: RecallPlanMode = options.plannerEnabled
-    ? planRecallMode(options.prompt)
-    : "full";
+}
+
+/**
+ * Apply the graph-mode overlay + gating to a planner-produced mode.
+ *
+ * Shared by the heuristic ({@link resolveRecallModeDecision}) and LLM
+ * ({@link resolveRecallModeDecisionAsync}) paths so graph promotion and the
+ * "graph disabled → fall back to full" gating behave identically regardless of
+ * which planner produced `plannedModeRaw` (gotcha #39).
+ */
+function finalizeRecallModeDecision(
+  plannedModeRaw: RecallPlanMode,
+  options: RecallModeGraphOptions,
+): RecallModeDecision {
+  let plannedMode: RecallPlanMode = plannedModeRaw;
   const graphExpandedIntentDetected =
     options.plannerEnabled &&
     options.graphExpandedIntentEnabled === true &&
@@ -1089,6 +1127,74 @@ export function resolveRecallModeDecision(options: {
   };
 }
 
+export function resolveRecallModeDecision(options: RecallModeGraphOptions): RecallModeDecision {
+  const plannedMode: RecallPlanMode = options.plannerEnabled
+    ? planRecallMode(options.prompt)
+    : "full";
+  return finalizeRecallModeDecision(plannedMode, options);
+}
+
+/**
+ * Async recall-mode decision with optional LLM-based planning (issue #1367 /
+ * Option C). Falls back to the heuristic decision when the LLM planner is
+ * disabled, in shadow mode, or unavailable/failed — so this is always safe to
+ * await on the recall hot path. Provider-agnostic: the LLM call routes through
+ * the gateway/fallback chain.
+ *
+ * `recallPlannerEnabled === false` keeps the legacy "always full" behavior and
+ * skips the LLM entirely (the planner as a whole is off).
+ */
+export async function resolveRecallModeDecisionAsync(
+  options: RecallModeGraphOptions & {
+    config: PluginConfig;
+    hints?: string[];
+    llm?: FallbackLlmClient;
+    signal?: AbortSignal;
+  },
+): Promise<RecallModeDecision> {
+  const heuristicDecision = resolveRecallModeDecision(options);
+
+  // Planner globally off, or LLM planning not opted into → heuristic only.
+  if (!options.plannerEnabled || !options.config.recallPlannerLlmEnabled) {
+    return heuristicDecision;
+  }
+
+  const { planRecallModeLLM } = await import("./recall-planner-llm.js");
+  const planned = await planRecallModeLLM(
+    options.prompt,
+    options.hints,
+    options.config,
+    options.llm,
+    options.signal,
+  );
+
+  // Shadow mode: record what the LLM would have chosen but keep the heuristic
+  // effective decision (safe rollout / comparison — gotcha #30).
+  if (options.config.recallPlannerShadowMode) {
+    return {
+      ...heuristicDecision,
+      plannerSource: planned.source,
+      plannerReason: `shadow:${planned.reason}`,
+      plannerLatencyMs: planned.latencyMs,
+      plannerFallbackUsed: planned.fallbackUsed,
+      plannerModelUsed: planned.modelUsed,
+      plannerHeuristicMode: planned.heuristicMode,
+      shadowLlmMode: planned.mode,
+    };
+  }
+
+  const llmDecision = finalizeRecallModeDecision(planned.mode, options);
+  return {
+    ...llmDecision,
+    plannerSource: planned.source,
+    plannerReason: planned.reason,
+    plannerLatencyMs: planned.latencyMs,
+    plannerFallbackUsed: planned.fallbackUsed,
+    plannerModelUsed: planned.modelUsed,
+    plannerHeuristicMode: planned.heuristicMode,
+  };
+}
+
 export function hasIdentityRecoveryIntent(prompt: string): boolean {
   const text = typeof prompt === "string" ? prompt.toLowerCase() : "";
   if (!text) return false;
@@ -2512,9 +2618,11 @@ export class Orchestrator {
                       namespace,
                       { signal: collectionCheckAbort.signal },
                     )
-                  : this.qmd.ensureCollection(this.config.memoryDir, {
-                      signal: collectionCheckAbort.signal,
-                    }),
+                  : this.qmd.ensureCollection(
+                      this.config.memoryDir,
+                      this.config.qmdCollection,
+                      { signal: collectionCheckAbort.signal },
+                    ),
                 collectionCheckAbort,
                 namespace,
               );
@@ -2860,7 +2968,7 @@ export class Orchestrator {
         namespace,
         state: this.config.namespacesEnabled
           ? await this.namespaceSearchRouter.ensureNamespaceCollection(namespace, { signal })
-          : await this.qmd.ensureCollection(this.config.memoryDir, { signal }),
+          : await this.qmd.ensureCollection(this.config.memoryDir, this.config.qmdCollection, { signal }),
       })),
     );
 
@@ -3390,22 +3498,31 @@ export class Orchestrator {
 
     // Use FallbackLlmClient for LLM calls (same pattern as causal-consolidation.ts)
     // Honor semanticConsolidationModel: "auto" = primary, "fast" = local fast, or specific model
-    const { FallbackLlmClient } = await import("./fallback-llm.js");
+    const { FallbackLlmClient, gatewayTaskChainOptions } = await import("./fallback-llm.js");
     const useGateway = this.config.modelSource === "gateway";
     const modelSetting = this.config.semanticConsolidationModel;
     if (modelSetting === "fast" && this.fastLlm && !useGateway) {
       log.info("[semantic-consolidation] using fast local LLM for synthesis");
     }
-    const gatewayAgentId = useGateway
-      ? (modelSetting === "fast" && this.config.fastGatewayAgentId
-          ? this.config.fastGatewayAgentId
-          : this.config.gatewayAgentId || undefined)
-      : undefined;
+    // Gateway routing: an explicit "fast" setting keeps the fast persona chain
+    // (the operator's deliberate fast-tier choice). Otherwise route through the
+    // shared task-chain resolution so taskModelChain applies to semantic
+    // consolidation like every other background task (gotcha #22). Issue #1365.
+    const gatewayChainOptions: { modelChain?: AgentPersonaModelConfig; agentId?: string } =
+      !useGateway
+        ? {}
+        : modelSetting === "fast"
+          ? (this.config.fastGatewayAgentId
+              ? { agentId: this.config.fastGatewayAgentId }
+              : this.config.gatewayAgentId
+                ? { agentId: this.config.gatewayAgentId }
+                : {})
+          : gatewayTaskChainOptions(this.config);
     const llm = new FallbackLlmClient(
       this.config.gatewayConfig,
       fallbackLlmRuntimeContextFromConfig(this.config),
     );
-    if (!llm.isAvailable(gatewayAgentId) && !(modelSetting === "fast" && this.fastLlm && !useGateway)) {
+    if (!llm.isAvailable(gatewayChainOptions) && !(modelSetting === "fast" && this.fastLlm && !useGateway)) {
       log.warn(
         "[semantic-consolidation] no LLM available — skipping synthesis",
       );
@@ -3454,7 +3571,7 @@ export class Orchestrator {
         let response: { content: string } | null = null;
         if (useGateway) {
           // Gateway model source — use the appropriate agent chain
-          response = await llm.chatCompletion(messages, { ...llmOpts, agentId: gatewayAgentId });
+          response = await llm.chatCompletion(messages, { ...llmOpts, ...gatewayChainOptions });
         } else if (modelSetting === "fast" && this.fastLlm) {
           const fastResult = await this.fastLlm.chatCompletion(messages, {
             operation: "semantic-consolidation",
@@ -5628,6 +5745,12 @@ export class Orchestrator {
       ) {
         if (debugSearchOptions?.intent) {
           lastHybridTopUpSkippedReason = "intent_hint_active";
+        } else if (this.config.qmdSearchStrategy === "lex") {
+          // BM25-only strategy: a hybrid top-up runs vectorSearch (see
+          // QmdClient.hybridSearch), which would reintroduce the vector path the
+          // operator opted out of. Keep "lex" BM25-only end-to-end so the gate is
+          // uniform across primary + top-up (gotcha #39). Issue #1335 (codex review #1422).
+          lastHybridTopUpSkippedReason = "lex_strategy";
         } else {
           const hybridResults = options.collection
             ? await this.qmd.hybridSearch(
@@ -6382,16 +6505,44 @@ export class Orchestrator {
     let identityInjectedChars = 0;
     let identityInjectionTruncated = false;
     timings.queryPolicy = `${queryPolicy.promptShape}/${queryPolicy.retrievalBudgetMode}${queryPolicy.skipConversationRecall ? "/skip-conv" : ""}`;
-    const recallDecision = resolveRecallModeDecision({
+    const recallModeDecisionOptions = {
       plannerEnabled: this.config.recallPlannerEnabled,
       graphRecallEnabled: this.config.graphRecallEnabled,
       multiGraphMemoryEnabled: this.config.multiGraphMemoryEnabled,
       graphExpandedIntentEnabled:
         this.config.graphExpandedIntentEnabled === true,
       prompt,
-    });
-    this.profiler.endSpan("planning", profileTraceId);
+    };
     const requestedMode = options.mode;
+    // When the caller forces a mode, skip the (async, possibly LLM-backed)
+    // planner entirely — the decision is overridden anyway. Otherwise consult
+    // the LLM planner when opted in (issue #1367 / Option C); it falls back to
+    // the heuristic on disable / shadow / timeout / error.
+    const recallDecision =
+      requestedMode !== undefined
+        ? resolveRecallModeDecision(recallModeDecisionOptions)
+        : await resolveRecallModeDecisionAsync({
+            ...recallModeDecisionOptions,
+            config: this.config,
+            signal: options.abortSignal,
+          });
+    if (
+      this.config.recallPlannerTelemetryEnabled &&
+      recallDecision.plannerSource &&
+      recallDecision.plannerSource !== "heuristic"
+    ) {
+      log.debug(
+        `[recall-planner] mode=${recallDecision.shadowLlmMode ?? recallDecision.effectiveMode} ` +
+          `source=${recallDecision.plannerSource} ` +
+          `planned=${recallDecision.plannedMode} ` +
+          `heuristic=${recallDecision.plannerHeuristicMode ?? recallDecision.plannedMode} ` +
+          `model=${recallDecision.plannerModelUsed ?? "n/a"} ` +
+          `latencyMs=${recallDecision.plannerLatencyMs ?? 0} ` +
+          `fallback=${recallDecision.plannerFallbackUsed ?? false}` +
+          (recallDecision.shadowLlmMode ? " (shadow)" : ""),
+      );
+    }
+    this.profiler.endSpan("planning", profileTraceId);
     const recallMode: RecallPlanMode =
       requestedMode ?? recallDecision.effectiveMode;
     const queryIntent = inferIntentFromText(retrievalQuery);
@@ -7780,6 +7931,8 @@ export class Orchestrator {
           maxResults: qmdFetchLimit,
           memoryDir: this.config.memoryDir,
           searchOptions: qmdSearchOptions,
+          searchStrategy: this.config.qmdSearchStrategy,
+          subprocessStrategy: this.config.qmdSubprocessStrategy,
         });
         const cachedQmd = getCachedQmdRecall<Exclude<QmdPhaseResult, null>>(
           qmdCacheKey,

package/src/qmd-client.test.ts

@@ -43,3 +43,88 @@ test("QmdClient rechecks daemon availability before returning unavailable", asyn
   assert.equal(results.length, 1);
   assert.equal(results[0]?.transport, "daemon");
 });
+
+type SubprocessInternals = {
+  available: boolean;
+  runQmdCommand: (args: string[]) => Promise<{ stdout: string; stderr: string }>;
+  searchViaSubprocess: (
+    query: string,
+    collection: string,
+    maxResults: number,
+  ) => Promise<QmdSearchResult[]>;
+  searchGlobalViaSubprocess: (query: string, maxResults: number) => Promise<QmdSearchResult[]>;
+};
+
+function captureSubprocessArgs(client: QmdClient): string[][] {
+  const calls: string[][] = [];
+  const internals = client as unknown as SubprocessInternals;
+  internals.available = true;
+  internals.runQmdCommand = async (args: string[]) => {
+    calls.push(args);
+    return { stdout: "[]", stderr: "" };
+  };
+  return calls;
+}
+
+test("subprocess fallback defaults to `qmd query` for scoped and global recall", async () => {
+  const client = new QmdClient("memories", 3, {});
+  const calls = captureSubprocessArgs(client);
+  const internals = client as unknown as SubprocessInternals;
+
+  await internals.searchViaSubprocess("hermes deployment", "memories", 3);
+  await internals.searchGlobalViaSubprocess("hermes deployment", 3);
+
+  assert.equal(calls[0]?.[0], "query", "scoped fallback must default to `qmd query`");
+  assert.equal(calls[1]?.[0], "query", "global fallback must default to `qmd query`");
+});
+
+test("qmdSubprocessStrategy 'search' applies BM25 to scoped AND global recall (gotcha #39)", async () => {
+  // Cursor #1422 review: the gate must be uniform across every subprocess path,
+  // not just the scoped one.
+  const client = new QmdClient("memories", 3, { qmdSubprocessStrategy: "search" });
+  const calls = captureSubprocessArgs(client);
+  const internals = client as unknown as SubprocessInternals;
+
+  await internals.searchViaSubprocess("hermes deployment", "memories", 3);
+  await internals.searchGlobalViaSubprocess("hermes deployment", 3);
+
+  assert.equal(calls[0]?.[0], "search", "scoped fallback must honor BM25 opt-in");
+  assert.equal(calls[1]?.[0], "search", "global fallback must honor BM25 opt-in");
+  // Global BM25 must NOT pass a collection flag.
+  assert.ok(!calls[1]?.includes("-c"), "global BM25 search must not include -c");
+});
+
+test("QMD search cache key isolates results by strategy (codex review on #1422)", async () => {
+  // Two clients with different strategies must not serve each other's cached
+  // results for the same query/collection within the global cache TTL.
+  function makeClient(opts: Record<string, unknown>): {
+    client: QmdClient;
+    calls: string[][];
+  } {
+    const client = new QmdClient("memories", 3, opts);
+    const internals = client as unknown as SubprocessInternals & {
+      daemonAvailable: boolean;
+    };
+    internals.available = true;
+    internals.daemonAvailable = false;
+    const calls: string[][] = [];
+    internals.runQmdCommand = async (args: string[]) => {
+      calls.push(args);
+      return { stdout: "[]", stderr: "" };
+    };
+    return { client, calls };
+  }
+
+  // Unique query avoids colliding with cache entries from other tests.
+  const query = "strategy-cache-isolation-probe-xyz";
+  const a = makeClient({ qmdSearchStrategy: "hybrid" });
+  const b = makeClient({ qmdSearchStrategy: "lex" });
+
+  await a.client.search(query, "memories", 3);
+  await b.client.search(query, "memories", 3);
+
+  // If the cache key ignored strategy, b would hit a's cached entry and never
+  // invoke the subprocess. Both must register their own subprocess call.
+  assert.equal(a.calls.length, 1, "first strategy populates its own cache entry");
+  assert.equal(b.calls.length, 1, "second strategy must NOT reuse the first's cached result");
+});

package/src/qmd-recall-cache.test.ts

@@ -97,6 +97,22 @@ test("qmd recall cache key reflects all defined search options", () => {
   assert.notEqual(left, right);
 });
 
+test("qmd recall cache key reflects search and subprocess strategy (codex review #1422)", () => {
+  const base = {
+    query: "api rate limit",
+    namespaces: ["a", "b"],
+    recallMode: "minimal" as const,
+    maxResults: 4,
+    memoryDir: "/tmp/engram-a",
+  };
+  const hybrid = buildQmdRecallCacheKey({ ...base, searchStrategy: "hybrid", subprocessStrategy: "query" });
+  const lex = buildQmdRecallCacheKey({ ...base, searchStrategy: "lex", subprocessStrategy: "query" });
+  const bm25Fallback = buildQmdRecallCacheKey({ ...base, searchStrategy: "hybrid", subprocessStrategy: "search" });
+
+  assert.notEqual(hybrid, lex, "different search strategies must not share a recall cache entry");
+  assert.notEqual(hybrid, bm25Fallback, "different subprocess strategies must not share a recall cache entry");
+});
+
 test("qmd recall cache returns cloned values so callers cannot mutate cached entries", () => {
   clearQmdRecallCache();
   const key = buildQmdRecallCacheKey({

package/src/qmd-recall-cache.ts

@@ -22,6 +22,11 @@ export interface QmdRecallCacheKeyOptions {
   memoryDir?: string;
   collection?: string;
   searchOptions?: SearchQueryOptions;
+  // QMD search/subprocess strategies change the recalled results, so they must
+  // participate in the cache key — otherwise a different strategy's cached QMD
+  // phase is served within the TTL (gotcha #37). Issue #1335 (codex review #1422).
+  searchStrategy?: string;
+  subprocessStrategy?: string;
 }
 
 const qmdRecallCache = new Map<string, QmdRecallCacheEntry>();
@@ -61,6 +66,8 @@ export function buildQmdRecallCacheKey(
     memoryDir: normalizePathScope(options.memoryDir),
     collection: options.collection ?? "",
     searchOptions: normalizeSearchOptions(options.searchOptions),
+    searchStrategy: options.searchStrategy ?? "",
+    subprocessStrategy: options.subprocessStrategy ?? "",
   });
 }
 

package/src/qmd.test.ts

@@ -3,6 +3,7 @@ import test from "node:test";
 
 import { parseConfig } from "./config.js";
 import {
+  buildDefaultStructuredSearches,
   getQmdCommandName,
   getQmdPostInstallProbeTargets,
   parseQmdVersionOutput,
@@ -67,6 +68,59 @@ test("resolveQmdCapabilities gates qmd 2.5 features by installed version", () =>
   assert.equal(v201.legacySkillInstall, true);
 });
 
+test("buildDefaultStructuredSearches defaults to the full hybrid lex+vec+hyde plan", () => {
+  // Default (no strategy argument) must preserve historical behavior so an
+  // upgrade never silently drops vector/HyDE recall. Issue #1335.
+  const searches = buildDefaultStructuredSearches("hermes deployment");
+  assert.deepEqual(
+    searches.map((s) => s.type),
+    ["lex", "vec", "hyde"],
+  );
+});
+
+test("buildDefaultStructuredSearches honors the hybrid strategy explicitly", () => {
+  const searches = buildDefaultStructuredSearches("hermes deployment", undefined, "hybrid");
+  assert.deepEqual(
+    searches.map((s) => s.type),
+    ["lex", "vec", "hyde"],
+  );
+});
+
+test("buildDefaultStructuredSearches lex-vec strategy drops the expensive HyDE leg", () => {
+  const searches = buildDefaultStructuredSearches("hermes deployment", undefined, "lex-vec");
+  assert.deepEqual(
+    searches.map((s) => s.type),
+    ["lex", "vec"],
+  );
+});
+
+test("buildDefaultStructuredSearches lex strategy is BM25-only", () => {
+  const searches = buildDefaultStructuredSearches("hermes deployment", undefined, "lex");
+  assert.deepEqual(
+    searches.map((s) => s.type),
+    ["lex"],
+  );
+});
+
+test("buildDefaultStructuredSearches lets an explicit per-call override win over the strategy", () => {
+  // Per-call structuredSearches override must still take precedence regardless
+  // of the configured default strategy.
+  const searches = buildDefaultStructuredSearches(
+    "hermes deployment",
+    { structuredSearches: [{ type: "vec", query: "hermes deployment" }] },
+    "lex",
+  );
+  assert.deepEqual(
+    searches.map((s) => s.type),
+    ["vec"],
+  );
+});
+
+test("buildDefaultStructuredSearches returns empty for blank queries under any strategy", () => {
+  assert.deepEqual(buildDefaultStructuredSearches("   ", undefined, "hybrid"), []);
+  assert.deepEqual(buildDefaultStructuredSearches("   ", undefined, "lex"), []);
+});
+
 test("shouldAutoUpgradeQmd only upgrades below Remnic supported version", () => {
   assert.equal(shouldAutoUpgradeQmd("qmd 2.1.0", "2.5.3"), true);
   assert.equal(shouldAutoUpgradeQmd("qmd 2.5.3", "2.5.3"), false);