@remnic/core 9.7.15 → 9.8.0

package/src/prioritized-embed.ts

@@ -0,0 +1,123 @@
+/**
+ * Prioritized embedding: debounce and batch-trigger collection-level QMD embeds
+ * so fresh memory writes become searchable within minutes. The QMD CLI does not
+ * support per-file embed targeting; each trigger runs `qmd embed -c <collection>`
+ * which embeds all pending files. Batching avoids hammering the CLI on every write.
+ *
+ * Writes are keyed by namespace so a write persisted into a non-default namespace
+ * is embedded against that namespace's own QMD collection/backend, not the default
+ * collection (review thread: namespace-aware prioritized embedding).
+ *
+ * Extracted from orchestrator.ts to keep that file under the structural ratchet.
+ */
+
+const EMBED_FLUSH_MS = 30_000;
+const EMBED_BATCH_MAX = 50;
+
+export interface PrioritizedEmbeddingHandle {
+  enqueue: (filePath: string, namespace?: string) => void;
+  dispose: () => void;
+}
+
+interface EmbedBackend {
+  embedFiles?: (paths: string[]) => Promise<unknown>;
+}
+
+export function installPrioritizedEmbedding(
+  getBackendForNamespace: (namespace: string) => unknown | Promise<unknown>,
+  logDebug: (msg: string) => void,
+): PrioritizedEmbeddingHandle {
+  const pendingByNamespace = new Map<string, string[]>();
+  let embedFlushTimer: ReturnType<typeof setTimeout> | null = null;
+  let disposed = false;
+
+  const scheduleFlush = (): void => {
+    if (disposed || embedFlushTimer !== null) return;
+    embedFlushTimer = setTimeout(flushEmbedBatch, EMBED_FLUSH_MS);
+    // Do not keep the process alive for the flush delay — short-lived CLI
+    // and import commands must exit promptly (review thread: timer unref).
+    embedFlushTimer.unref?.();
+  };
+
+  const flushNamespace = async (namespace: string, paths: string[]): Promise<void> => {
+    let backend: EmbedBackend | null | undefined;
+    try {
+      backend = (await Promise.resolve(getBackendForNamespace(namespace))) as
+        | EmbedBackend
+        | null
+        | undefined;
+    } catch {
+      if (paths.length > 0) scheduleFlush();
+      return;
+    }
+    if (!backend || typeof backend.embedFiles !== "function") {
+      if (paths.length > 0) scheduleFlush();
+      return;
+    }
+    const batch = paths.splice(0, EMBED_BATCH_MAX);
+    if (paths.length === 0 && pendingByNamespace.get(namespace) === paths) {
+      pendingByNamespace.delete(namespace);
+    }
+    if (batch.length === 0) return;
+    backend.embedFiles(batch).then((ok) => {
+      if (ok === false) {
+        requeue(namespace, paths, batch);
+        return;
+      }
+      // Collection-level embed drains the entire backlog. If new writes
+      // were appended to the tail while the embed was in flight, reschedule
+      // so they get picked up; otherwise clear the queue.
+      if (pendingByNamespace.get(namespace) === paths) {
+        if (paths.length > 0) {
+          scheduleFlush();
+        } else {
+          pendingByNamespace.delete(namespace);
+        }
+      }
+    }).catch(() => {
+      requeue(namespace, paths, batch);
+    });
+  };
+
+  const requeue = (namespace: string, paths: string[], batch: string[]): void => {
+    const live = pendingByNamespace.get(namespace);
+    if (live && live !== paths) {
+      live.unshift(...batch);
+    } else {
+      paths.unshift(...batch);
+      if (!disposed) pendingByNamespace.set(namespace, paths);
+    }
+    scheduleFlush();
+  };
+
+  const flushEmbedBatch = (): void => {
+    embedFlushTimer = null;
+    for (const [namespace, paths] of [...pendingByNamespace]) {
+      void flushNamespace(namespace, paths);
+    }
+  };
+
+  const dispose = (): void => {
+    disposed = true;
+    if (embedFlushTimer) {
+      clearTimeout(embedFlushTimer);
+      embedFlushTimer = null;
+    }
+    pendingByNamespace.clear();
+  };
+
+  const enqueue = (filePath: string, namespace?: string): void => {
+    if (disposed) return;
+    const key = namespace ?? "default";
+    let paths = pendingByNamespace.get(key);
+    if (!paths) {
+      paths = [];
+      pendingByNamespace.set(key, paths);
+    }
+    paths.push(filePath);
+    scheduleFlush();
+    logDebug(`prioritized embed: queued ${filePath} in namespace ${key} (${paths.length} pending)`);
+  };
+
+  return { enqueue, dispose };
+}

package/src/proactive-contention.test.ts

@@ -0,0 +1,80 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { LocalLlmClient } from "./local-llm.js";
+import { shouldRunProactivePass } from "./proactive-contention.js";
+import type { PluginConfig } from "./types.js";
+
+function proactiveConfig(overrides: Partial<PluginConfig> = {}): PluginConfig {
+  return {
+    proactiveExtractionTimeoutMs: 2500,
+    proactiveExtractionMaxTokens: 900,
+    proactiveExtractionSkipWhenLocalLlmBusy: true,
+    ...overrides,
+  } as unknown as PluginConfig;
+}
+
+function fakeClient(contended: boolean): LocalLlmClient {
+  return { isBackgroundLaneContended: () => contended } as unknown as LocalLlmClient;
+}
+
+test("shouldRunProactivePass: budget disables skip the pass", () => {
+  const busy = fakeClient(true);
+  assert.equal(shouldRunProactivePass(proactiveConfig(), 0, true, fakeClient(false)), false);
+  assert.equal(
+    shouldRunProactivePass(proactiveConfig({ proactiveExtractionTimeoutMs: 0 }), 2, true, fakeClient(false)),
+    false,
+  );
+  assert.equal(
+    shouldRunProactivePass(proactiveConfig({ proactiveExtractionMaxTokens: 0 }), 2, true, fakeClient(false)),
+    false,
+  );
+  // A busy lane must not mask a budget disable either.
+  assert.equal(shouldRunProactivePass(proactiveConfig({ proactiveExtractionMaxTokens: 0 }), 2, true, busy), false);
+});
+
+test("shouldRunProactivePass: skips only a local extraction against a busy background lane (issue #2011)", () => {
+  // Local extractor + busy lane + gate on → skip.
+  assert.equal(shouldRunProactivePass(proactiveConfig(), 2, true, fakeClient(true)), false);
+  // Local extractor + idle lane → run.
+  assert.equal(shouldRunProactivePass(proactiveConfig(), 2, true, fakeClient(false)), true);
+  // Cloud extractor (no single-lane saturation) → run even if the local lane is busy.
+  assert.equal(shouldRunProactivePass(proactiveConfig(), 2, false, fakeClient(true)), true);
+  // Gate disabled → run even when the local lane is busy.
+  assert.equal(
+    shouldRunProactivePass(proactiveConfig({ proactiveExtractionSkipWhenLocalLlmBusy: false }), 2, true, fakeClient(true)),
+    true,
+  );
+});
+
+test("LocalLlmClient.isBackgroundLaneContended reflects background queue and in-flight state", () => {
+  const client = new LocalLlmClient({
+    localLlmEnabled: true,
+    localLlmModel: "test-local-model",
+    localLlmUrl: "http://127.0.0.1:1234",
+    localLlmTimeoutMs: 1_000,
+  } as unknown as PluginConfig);
+
+  // Test seam: reach the client's private queue state to simulate lane occupancy.
+  const internals = client as unknown as {
+    requestQueues: Record<string, unknown[]>;
+    queueProcessing: Set<string>;
+  };
+
+  assert.equal(client.isBackgroundLaneContended(), false);
+
+  // A queued background request contends the lane.
+  internals.requestQueues.background.push({});
+  assert.equal(client.isBackgroundLaneContended(), true);
+  internals.requestQueues.background.length = 0;
+  assert.equal(client.isBackgroundLaneContended(), false);
+
+  // An in-flight background request contends the lane too.
+  internals.queueProcessing.add("background");
+  assert.equal(client.isBackgroundLaneContended(), true);
+
+  // A recall-critical request on the separate priority slot does not.
+  internals.queueProcessing.clear();
+  internals.queueProcessing.add("recall-critical");
+  assert.equal(client.isBackgroundLaneContended(), false);
+});

package/src/proactive-contention.ts

@@ -0,0 +1,39 @@
+import type { LocalLlmClient } from "./local-llm.js";
+import type { PluginConfig } from "./types.js";
+
+/**
+ * Decide whether the optional proactive extraction second-pass should run.
+ *
+ * The proactive pass is fail-soft enrichment: it re-sends the full conversation
+ * to generate follow-up questions on top of the base extraction. On a saturated
+ * single-lane local LLM host (one Ollama runner), an optional pass queued behind
+ * an in-flight extraction loses its short client deadline while it waits, 500s,
+ * and wastes prefill GPU-seconds without ever producing output (issue #2011).
+ *
+ * Returns false (skip the pass, keep the base extraction unchanged) when:
+ * - the pass is disabled by budget config (`0` question / timeout / token caps),
+ * - or `proactiveExtractionSkipWhenLocalLlmBusy` is on, the extraction is running
+ *   against the local LLM, and that client's background lane is already busy —
+ *   so the optional pass would only queue behind concurrent extraction work.
+ *
+ * The cloud-fallback path has no single-lane saturation, so the busy-lane skip
+ * only applies when the local LLM is the active extractor.
+ */
+export function shouldRunProactivePass(
+  config: PluginConfig,
+  maxAdditional: number,
+  usingLocalLlm: boolean,
+  localLlm: LocalLlmClient,
+): boolean {
+  if (maxAdditional === 0) return false;
+  if (config.proactiveExtractionTimeoutMs === 0) return false;
+  if (config.proactiveExtractionMaxTokens === 0) return false;
+  if (
+    config.proactiveExtractionSkipWhenLocalLlmBusy &&
+    usingLocalLlm &&
+    localLlm.isBackgroundLaneContended()
+  ) {
+    return false;
+  }
+  return true;
+}

package/src/procedural/procedure-miner.test.ts

@@ -1,4 +1,5 @@
 import test from "node:test";
+import type { SealedMemoryEnvelope } from "../write-envelope.js";
 import assert from "node:assert/strict";
 import os from "node:os";
 import path from "node:path";
@@ -42,7 +43,7 @@ function makeTrajectory(
 
 test("procedure mining deduplicates by full cluster hash instead of truncated cluster text", async () => {
   const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-procedure-miner-"));
-  const written: Array<{ structuredAttributes: Record<string, string> }> = [];
+  const written: Array<{ structuredAttributes: Record<string, string>; status?: string }> = [];
   const storage = {
     async readAllMemories() {
       return written.map((entry, index) => ({
@@ -54,13 +55,12 @@ test("procedure mining deduplicates by full cluster hash instead of truncated cl
         },
       }));
     },
-    async writeMemory(
-      _category: string,
-      _body: string,
-      options: { structuredAttributes?: Record<string, string> },
-    ) {
+    async writeSealedMemory(envelope: SealedMemoryEnvelope, extras: { status?: string }) {
       written.push({
-        structuredAttributes: options.structuredAttributes ?? {},
+        structuredAttributes: envelope.rawStructuredAttributes
+          ? { ...envelope.rawStructuredAttributes }
+          : {},
+        status: extras.status,
       });
       return { id: `procedure-${written.length}`, tombstoneBlocked: false };
     },
@@ -95,6 +95,12 @@ test("procedure mining deduplicates by full cluster hash instead of truncated cl
     assert.equal(result.proceduresWritten, 2);
     assert.equal(new Set(written.map((entry) => entry.structuredAttributes.procedure_cluster)).size, 1);
     assert.equal(new Set(written.map((entry) => entry.structuredAttributes.procedure_cluster_hash)).size, 2);
+    // §21 review: the status extra is parameter-dependent behavior — these
+    // fixtures do not meet the auto-promote gate, so both land in review.
+    assert.deepEqual(
+      written.map((entry) => entry.status),
+      ["pending_review", "pending_review"],
+    );
   } finally {
     await rm(memoryDir, { recursive: true, force: true });
   }
@@ -102,7 +108,7 @@ test("procedure mining deduplicates by full cluster hash instead of truncated cl
 
 test("procedure mining serializes concurrent writes for the same cluster", async () => {
   const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-procedure-miner-concurrent-"));
-  const written: Array<{ structuredAttributes: Record<string, string> }> = [];
+  const written: Array<{ structuredAttributes: Record<string, string>; status?: string }> = [];
   const storage = {
     async readAllMemories() {
       return written.map((entry, index) => ({
@@ -114,14 +120,13 @@ test("procedure mining serializes concurrent writes for the same cluster", async
         },
       }));
     },
-    async writeMemory(
-      _category: string,
-      _body: string,
-      options: { structuredAttributes?: Record<string, string> },
-    ) {
+    async writeSealedMemory(envelope: SealedMemoryEnvelope, extras: { status?: string }) {
       await new Promise((resolve) => setTimeout(resolve, 20));
       written.push({
-        structuredAttributes: options.structuredAttributes ?? {},
+        structuredAttributes: envelope.rawStructuredAttributes
+          ? { ...envelope.rawStructuredAttributes }
+          : {},
+        status: extras.status,
       });
       return { id: `procedure-${written.length}`, tombstoneBlocked: false };
     },

package/src/procedural/procedure-miner.ts

@@ -15,6 +15,7 @@ import {
 import { buildProcedurePersistBody, normalizeProcedureSteps, type ProcedureStep } from "./procedure-types.js";
 import { clusterByKey } from "./reinforcement-core.js";
 import { log } from "../logger.js";
+import { composeMemoryEnvelope, STRUCTURED_ATTRIBUTE_LIMITS } from "../write-envelope.js";
 
 /** Must match truncation on `procedure_cluster` structured attribute (dedupe + storage). */
 const PROCEDURE_CLUSTER_ATTR_MAX = 500;
@@ -23,6 +24,19 @@ const PROCEDURE_CLUSTER_LOCK_TIMEOUT_MS = 30_000;
 
 const inProcessClusterWriteLocks = new Map<string, Promise<void>>();
 
+/** Join ids with commas, keeping only WHOLE ids within maxLength. */
+function capJoinedIds(ids: string[], maxLength: number): string {
+  const kept: string[] = [];
+  let length = 0;
+  for (const id of ids) {
+    const addition = kept.length === 0 ? id.length : id.length + 1;
+    if (length + addition > maxLength) break;
+    kept.push(id);
+    length += addition;
+  }
+  return kept.join(",");
+}
+
 export interface ProcedureMiningResult {
   clustersProcessed: number;
   proceduresWritten: number;
@@ -231,20 +245,37 @@ export async function runProcedureMining(options: {
         return false;
       }
 
-      await options.storage.writeMemory("procedure", body, {
-        source: "procedure-miner",
-        status: promote ? "active" : "pending_review",
-        tags: ["procedure-miner", "causal-trajectory"],
-        structuredAttributes: {
-          procedure_cluster: key.slice(0, PROCEDURE_CLUSTER_ATTR_MAX),
-          procedure_cluster_hash: clusterHash,
-          trajectory_ids: group
-            .map((g) => g.trajectoryId)
-            .join(",")
-            .slice(0, 1900),
-          trajectory_count: String(group.length),
-          success_rate: rate.toFixed(4),
+      // Sealed-envelope write (issue #1989 PR4): mined from causal
+      // trajectories (machine data) — salvage; drops are warn-logged.
+      const procedureEnvelope = composeMemoryEnvelope(
+        {
+          content: body,
+          category: "procedure",
+          tags: ["procedure-miner", "causal-trajectory"],
+          structuredAttributes: {
+            procedure_cluster: key.slice(0, PROCEDURE_CLUSTER_ATTR_MAX),
+            procedure_cluster_hash: clusterHash,
+            // Capped to the envelope attribute-value limit AT AN ID
+            // BOUNDARY (#2022 review): the legacy 1900-char slice now
+            // exceeds STRUCTURED_ATTRIBUTE_LIMITS.maxValueLength, and a
+            // salvage drop would lose ALL trajectory provenance instead
+            // of the tail.
+            trajectory_ids: capJoinedIds(
+              group.map((g) => g.trajectoryId),
+              STRUCTURED_ATTRIBUTE_LIMITS.maxValueLength,
+            ),
+            trajectory_count: String(group.length),
+            success_rate: rate.toFixed(4),
+          },
         },
+        { source: "procedure-miner" },
+        { salvage: true },
+      );
+      if (procedureEnvelope.salvageNotes.length > 0) {
+        log.warn(`procedure-miner write salvaged invalid fields: ${procedureEnvelope.salvageNotes.join("; ")}`);
+      }
+      await options.storage.writeSealedMemory(procedureEnvelope, {
+        status: promote ? "active" : "pending_review",
       });
       return true;
     });

package/src/qmd-client.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { QmdClient } from "./qmd.js";
+import { QmdClient, parseQmdStatusOutput } from "./qmd.js";
 import type { QmdSearchResult } from "./types.js";
 
 test("QmdClient rechecks daemon availability before returning unavailable", async () => {
@@ -260,3 +260,113 @@ test("QMD search cache key isolates results by strategy (codex review on #1422)"
   assert.equal(a.calls.length, 1, "first strategy populates its own cache entry");
   assert.equal(b.calls.length, 1, "second strategy must NOT reuse the first's cached result");
 });
+
+test("parseQmdStatusOutput parses full status output", () => {
+  const stdout = [
+    "Collection: remnic-memory",
+    "Total files: 1200",
+    "Embedded: 1150",
+    "Pending: 50",
+    "Oldest pending: 2h",
+  ].join("\n");
+  const report = parseQmdStatusOutput(stdout);
+  assert.equal(report.totalFiles, 1200);
+  assert.equal(report.embeddedFiles, 1150);
+  assert.equal(report.pendingEmbeddings, 50);
+  assert.equal(report.oldestPendingAgeMs, 2 * 60 * 60 * 1000);
+  assert.equal(report.raw, stdout);
+});
+
+test("parseQmdStatusOutput parses documented QMD status format", () => {
+  const stdout = [
+    "Collection: remnic-memory",
+    "Total: 1,200 files indexed",
+    "Vectors: 1,150 embedded",
+    "Pending: 50",
+    "Oldest pending: 2h",
+  ].join("\n");
+  const report = parseQmdStatusOutput(stdout);
+  assert.equal(report.totalFiles, 1200);
+  assert.equal(report.embeddedFiles, 1150);
+  assert.equal(report.pendingEmbeddings, 50);
+  assert.equal(report.oldestPendingAgeMs, 2 * 60 * 60 * 1000);
+});
+
+test("parseQmdStatusOutput parses time units correctly", () => {
+  assert.equal(parseQmdStatusOutput("Oldest pending: 45s").oldestPendingAgeMs, 45_000);
+  assert.equal(parseQmdStatusOutput("Oldest pending: 3m").oldestPendingAgeMs, 180_000);
+  assert.equal(parseQmdStatusOutput("Oldest pending: 500ms").oldestPendingAgeMs, 500);
+  assert.equal(parseQmdStatusOutput("no age here").oldestPendingAgeMs, null);
+});
+
+test("parseQmdStatusOutput returns nulls for unparseable output", () => {
+  const report = parseQmdStatusOutput("garbage output");
+  assert.equal(report.totalFiles, null);
+  assert.equal(report.embeddedFiles, null);
+  assert.equal(report.pendingEmbeddings, null);
+  assert.equal(report.oldestPendingAgeMs, null);
+});
+
+test("QmdClient.status() returns parsed report on success", async () => {
+  const client = new QmdClient("test-col", 3);
+  const internals = client as unknown as {
+    available: boolean;
+    qmdCapabilities: { safeStatusDeviceProbe: boolean };
+    runQmdCommand: (args: string[]) => Promise<{ stdout: string; code: number }>;
+  };
+  internals.available = true;
+  internals.qmdCapabilities = { safeStatusDeviceProbe: true };
+  let capturedArgs: string[] = [];
+  internals.runQmdCommand = async (args: string[]) => {
+    capturedArgs = args;
+    return { stdout: "Total files: 10\nEmbedded: 8\nPending: 2\nOldest pending: 5m", code: 0 };
+  };
+  const report = await client.status();
+  assert.deepEqual(capturedArgs, ["status", "-c", "test-col"]);
+  assert.equal(report.pendingEmbeddings, 2);
+  assert.equal(report.oldestPendingAgeMs, 300_000);
+});
+
+test("QmdClient.status() returns null fields when unavailable", async () => {
+  const client = new QmdClient("test-col", 3);
+  const internals = client as unknown as { available: boolean };
+  internals.available = false;
+  const report = await client.status();
+  assert.equal(report.pendingEmbeddings, null);
+  assert.equal(report.oldestPendingAgeMs, null);
+});
+
+test("QmdClient.embedFiles() runs update then embed", async () => {
+  const client = new QmdClient("test-col", 3);
+  const internals = client as unknown as {
+    available: boolean;
+    runQmdCommand: (args: string[]) => Promise<{ stdout: string; code: number }>;
+  };
+  internals.available = true;
+  const allCalls: string[][] = [];
+  internals.runQmdCommand = async (args: string[]) => {
+    allCalls.push(args);
+    return { stdout: "ok", code: 0 };
+  };
+  const result = await client.embedFiles(["/mem/a.md", "/mem/b.md"]);
+  assert.equal(result, true);
+  assert.equal(allCalls.length, 2);
+  assert.deepEqual(allCalls[0], ["update", "-c", "test-col"]);
+  assert.deepEqual(allCalls[1], ["embed", "-c", "test-col"]);
+});
+
+test("QmdClient.embedFiles() returns false for empty paths", async () => {
+  const client = new QmdClient("test-col", 3);
+  const internals = client as unknown as { available: boolean };
+  internals.available = true;
+  const result = await client.embedFiles([]);
+  assert.equal(result, false);
+});
+
+test("QmdClient.embedFiles() returns false when unavailable", async () => {
+  const client = new QmdClient("test-col", 3);
+  const internals = client as unknown as { available: boolean };
+  internals.available = false;
+  const result = await client.embedFiles(["/mem/a.md"]);
+  assert.equal(result, false);
+});