@remnic/core 10.0.6 → 10.0.8

package/src/recall-harmonic-qmd-ordering.test.ts

@@ -0,0 +1,304 @@
+// #3188: park only harmonic's source read and observe real QMD publication.
+
+import assert from "node:assert/strict";
+import path from "node:path";
+import type { TestContext } from "node:test";
+import test from "node:test";
+
+import { recordAbstractionNode } from "./abstraction-nodes.js";
+import { parseConfig } from "./config.js";
+import { rebuildMemoryProjection } from "./maintenance/rebuild-memory-projection.js";
+import { Orchestrator } from "./orchestrator.js";
+import {
+  RecallSectionCoordinator,
+  type RecallSectionAppendOptions,
+  type RecallSectionBuckets,
+} from "./orchestration/recall-section-coordinator.js";
+import { getRecallTimings } from "./recall-timings.js";
+import type { RecallInternalCoordinator } from "./orchestration/recall-internal.js";
+import { StorageManager } from "./storage.js";
+import type { SearchBackend } from "./search/port.js";
+import { cleanupDir, eventually, mkTempMemoryDir } from "./testing/orchestrator-lite.js";
+import type { MemoryFile, PluginConfig, QmdSearchResult } from "./types.js";
+
+const RECALL_PROMPT = "how does the atlas source lookup behave?";
+
+/**
+ * Complete typed SearchBackend stub (no casts): answers availability and
+ * returns one settled QMD candidate from `search`; every other read surface
+ * is empty. Mirrors the recall-hardening backend-stub shape.
+ */
+function settledQmdBackend(result: QmdSearchResult): SearchBackend {
+  return {
+    async probe() {
+      return true;
+    },
+    isAvailable() {
+      return true;
+    },
+    debugStatus() {
+      return "backend=settled-qmd-stub";
+    },
+    async search() {
+      return [result];
+    },
+    async searchGlobal() {
+      return [];
+    },
+    async bm25Search() {
+      return [];
+    },
+    async vectorSearch() {
+      return [];
+    },
+    async hybridSearch() {
+      return [];
+    },
+    async update() {},
+    async updateCollection() {},
+    async embed() {},
+    async embedCollection() {},
+    async ensureCollection() {
+      return "present";
+    },
+  };
+}
+
+interface OrderingHarness {
+  memoryDir: string;
+  orchestrator: Orchestrator;
+  config: PluginConfig;
+  qmdMemory: MemoryFile;
+  harmonicSource: MemoryFile;
+  /** Releases the parked harmonic source read (idempotent). */
+  releaseHarmonicRead: () => void;
+  /** Section ids in append order, via the section coordinator prototype. */
+  appendedSectionIds: string[];
+  /** How often the harmonic source read was asked for (and parked). */
+  harmonicReadCalls: { count: number };
+  recall: () => Promise<string>;
+}
+
+/**
+ * Seed two active memories — one returned as the settled QMD candidate, one
+ * projected into a real abstraction-node store row — then boot the real
+ * orchestrator over them. The projection rebuild keeps the harmonic read on
+ * the indexed exact-path branch, so intercepting that one
+ * `StorageManager.readMemoryByPath` call parks harmonic and nothing else.
+ */
+async function makeOrderingHarness(
+  t: TestContext,
+  mode: "park" | "fail",
+): Promise<OrderingHarness> {
+  const memoryDir = await mkTempMemoryDir("recall-harmonic-order-");
+  const storage = new StorageManager(memoryDir);
+  await storage.ensureDirectories();
+
+  const { memory: qmdMemory } = await storage.writeMemory(
+    "fact",
+    "the atlas qmd sentinel confirms deployment cadence",
+    { source: "test" },
+  );
+  const { memory: harmonicSource } = await storage.writeMemory(
+    "fact",
+    "atlas harmonic source lookup body",
+    { source: "test" },
+  );
+  await recordAbstractionNode({
+    memoryDir,
+    node: {
+      schemaVersion: 1,
+      nodeId: "ordering-under-qmd",
+      recordedAt: new Date().toISOString(),
+      sessionKey: "agent:test",
+      kind: "topic",
+      abstractionLevel: "meso",
+      title: "atlas source lookup",
+      summary: "atlas source lookup",
+      sourceMemoryIds: [harmonicSource.frontmatter.id],
+    },
+  });
+  await rebuildMemoryProjection({ memoryDir, storage, dryRun: false });
+
+  // Defaults only — the ordering fix must not move any budget, and the
+  // default candidate/namespace sets stay untouched.
+  const config = parseConfig({
+    openaiApiKey: "sk-test",
+    memoryDir,
+    workspaceDir: path.join(memoryDir, "workspace"),
+    qmdEnabled: true,
+    embeddingFallbackEnabled: false,
+    recallPlannerEnabled: false,
+    sharedContextEnabled: false,
+    harmonicRetrievalEnabled: true,
+    queryAwareIndexingEnabled: false,
+    parallelRetrievalEnabled: false,
+    initGateTimeoutMs: 1000,
+  });
+
+  const orchestrator = new Orchestrator(config);
+  orchestrator.qmd = settledQmdBackend({
+    docid: qmdMemory.frontmatter.id,
+    path: qmdMemory.path,
+    score: 0.9,
+    snippet: qmdMemory.content,
+  });
+
+  const appendedSectionIds: string[] = [];
+  const originalAppend = RecallSectionCoordinator.prototype.appendRecallSection;
+  t.mock.method(
+    RecallSectionCoordinator.prototype,
+    "appendRecallSection",
+    function recordAppendedSection(
+      this: RecallSectionCoordinator,
+      sectionBuckets: RecallSectionBuckets,
+      sectionId: string,
+      content: string,
+      options?: RecallSectionAppendOptions,
+    ): boolean {
+      appendedSectionIds.push(sectionId);
+      return originalAppend.call(this, sectionBuckets, sectionId, content, options);
+    },
+  );
+
+  const harmonicGate = Promise.withResolvers<void>();
+  const harmonicReadCalls = { count: 0 };
+  const originalRead = StorageManager.prototype.readMemoryByPath;
+  const harmonicSourcePath = path.resolve(harmonicSource.path);
+  t.mock.method(
+    StorageManager.prototype,
+    "readMemoryByPath",
+    function interceptHarmonicSourceRead(
+      this: StorageManager,
+      filePath: string,
+    ): Promise<MemoryFile | null> {
+      if (path.resolve(filePath) !== harmonicSourcePath) {
+        return originalRead.call(this, filePath);
+      }
+      if (mode === "fail") {
+        return Promise.reject(new Error("synthetic harmonic source read failure"));
+      }
+      harmonicReadCalls.count += 1;
+      return harmonicGate.promise.then(() => originalRead.call(this, filePath));
+    },
+  );
+
+  // Invoke the existing private delegate without booting background services.
+  const recallSeam = orchestrator as unknown as Pick<RecallInternalCoordinator, "recallInternal">
+    & Pick<RecallSectionCoordinator, "isRecallSectionEnabled">;
+  t.mock.method(recallSeam, "isRecallSectionEnabled", (id: string) =>
+    id === "memories" || id === "harmonic-retrieval");
+
+  return {
+    memoryDir,
+    orchestrator,
+    config,
+    qmdMemory,
+    harmonicSource,
+    releaseHarmonicRead: () => harmonicGate.resolve(),
+    appendedSectionIds,
+    harmonicReadCalls,
+    recall: () => recallSeam.recallInternal(RECALL_PROMPT, "agent:test", { mode: "full" }),
+  };
+}
+
+test("settled QMD publishes while the harmonic read is still parked; harmonic renders after release", async (t) => {
+  const harness = await makeOrderingHarness(t, "park");
+  const { memoryDir, orchestrator, config, appendedSectionIds, harmonicReadCalls } =
+    harness;
+  let recallPromise: Promise<string> | undefined;
+  try {
+    // The move must not touch any deadline: pin the configured defaults.
+    assert.equal(config.recallOuterTimeoutMs, 75_000);
+    assert.equal(config.recallCoreDeadlineMs, 75_000);
+    assert.equal(config.recallEnrichmentDeadlineMs, 60_000);
+
+    recallPromise = harness.recall();
+
+    // The harmonic read starts during phase 1 and is now parked.
+    await eventually(
+      async () => (harmonicReadCalls.count > 0 ? true : null),
+      "harmonic source read reached the parked seam",
+    );
+
+    // The fix: the settled QMD candidate is formatted and published while
+    // harmonic is still parked. Under the pre-fix order the assembly blocked
+    // on harmonic first, so this probe could never succeed.
+    await eventually(
+      async () => (appendedSectionIds.includes("memories") ? true : null),
+      "QMD memories section published while harmonic is parked",
+    );
+    assert.equal(
+      appendedSectionIds.includes("harmonic-retrieval"),
+      false,
+      "harmonic must not append before its read is released",
+    );
+
+    harness.releaseHarmonicRead();
+    const context = await recallPromise;
+
+    // Harmonic still publishes afterward, with its candidate content, and
+    // QMD publication strictly precedes the harmonic append.
+    assert.ok(
+      appendedSectionIds.includes("harmonic-retrieval"),
+      "harmonic section must append after the read is released",
+    );
+    assert.ok(
+      appendedSectionIds.lastIndexOf("memories") <
+        appendedSectionIds.indexOf("harmonic-retrieval"),
+      `QMD publication must precede the harmonic append, got: ${appendedSectionIds.join(",")}`,
+    );
+    assert.ok(context.includes(harness.qmdMemory.content), "QMD candidate must render");
+    assert.ok(
+      context.includes("## Harmonic Retrieval"),
+      `harmonic section must render, got: ${context.slice(0, 600)}`,
+    );
+    assert.ok(
+      context.includes(harness.harmonicSource.content),
+      "harmonic candidate content must render",
+    );
+
+    // Budget parity: harmonic completed inside the untouched shared
+    // enrichment budget. A cancelled/timed-out harmonic drops its timing
+    // entry (timeout strings do not sanitize to numbers), so a numeric
+    // harmonicRetrieval timing proves the move neither shortened nor spent
+    // the budget ahead of QMD.
+    const record = getRecallTimings(config).find(
+      (candidate) => candidate.timingsMs.harmonicRetrieval !== undefined,
+    );
+    assert.ok(record, "harmonic must complete within the shared enrichment budget");
+    assert.ok(record.timingsMs.qmd !== undefined, "QMD timing must be recorded");
+  } finally {
+    harness.releaseHarmonicRead();
+    await recallPromise?.catch(() => undefined);
+    await orchestrator.destroy();
+    await cleanupDir(memoryDir);
+  }
+});
+
+test("a failed harmonic source read degrades without taking QMD publication down", async (t) => {
+  const harness = await makeOrderingHarness(t, "fail");
+  const { memoryDir, orchestrator, qmdMemory, appendedSectionIds } = harness;
+  try {
+    const context = await harness.recall();
+
+    assert.ok(
+      appendedSectionIds.includes("memories"),
+      "QMD must still publish when the harmonic read fails",
+    );
+    assert.equal(
+      appendedSectionIds.includes("harmonic-retrieval"),
+      false,
+      "a failed harmonic read must append no harmonic section",
+    );
+    assert.ok(context.includes(qmdMemory.content), "QMD candidate must render");
+    assert.ok(
+      !context.includes("## Harmonic Retrieval"),
+      "no harmonic section may render after a failed read",
+    );
+  } finally {
+    harness.releaseHarmonicRead();
+    await orchestrator.destroy();
+    await cleanupDir(memoryDir);
+  }
+});

package/src/testing/orchestrator-lite.ts

@@ -18,6 +18,8 @@ import { getAbstractionNodeStoreStatus, type AbstractionNodeStoreStatus } from "
 import type { Orchestrator } from "../orchestrator.js";
 import type { ExtractionRunCoordinatorDeps } from "../orchestration/extraction-run.js";
 import type { BufferTurn, ExtractionResult, PluginConfig } from "../types.js";
+import { sleep } from "../utils/sleep.js";
+export { sleep } from "../utils/sleep.js";
 
 /**
  * Offline-safe config: QMD off, embedding fallback off, planner off (so the
@@ -76,13 +78,6 @@ export function pastIso(): string {
   return new Date(Date.now() - 3600_000).toISOString();
 }
 
-/** Linear async sleep (shared by the retrying cleanup and pollers). */
-export function sleep(ms: number): Promise<void> {
-  const { promise, resolve } = Promise.withResolvers<void>();
-  setTimeout(resolve, ms);
-  return promise;
-}
-
 /**
  * Remove a temp memory dir, retrying transient ENOTEMPTY/EBUSY races: the
  * orchestrator keeps best-effort background writers that can still be running

package/src/utils/serialize-mutations.ts

@@ -22,6 +22,7 @@
 import { randomUUID } from "node:crypto";
 import { link, mkdir, open, readFile, rename, stat, unlink, utimes } from "node:fs/promises";
 import path from "node:path";
+import { sleep } from "./sleep.js";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // 1. serializeMutations — keyed async serialization with rejection recovery
@@ -732,16 +733,3 @@ async function refreshHeldLock(
   }
   return lockHeldBySelf(held);
 }
-
-function sleep(ms: number): Promise<void> {
-  // Manual deferred instead of Promise.withResolvers (ES2024) — plugin-openclaw's
-  // standalone tsconfig targets ES2022 lib and this module is reachable from its
-  // type graph, so withResolvers would TS2550 there (same fix as
-  // extraction-faithfulness.ts:467).
-  return new Promise<void>((resolve) => {
-    // NOT unref'd: this polls inside an awaited acquire loop, so the caller's
-    // await chain keeps the loop alive; unref would let Node exit mid-poll when
-    // nothing else is pending (the heartbeat interval IS unref'd separately).
-    setTimeout(resolve, ms);
-  });
-}

package/src/utils/sleep.test.ts

@@ -0,0 +1,32 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { sleep } from "./sleep.js";
+
+test("sleep preserves delays, stays referenced, and settles only when its timer fires", async (t) => {
+  let fire: () => void = () => assert.fail("timer was not scheduled");
+  const delays: number[] = [];
+  const unref = t.mock.fn();
+  t.mock.method(globalThis, "setTimeout", (callback: () => void, ms: number) => {
+    delays.push(ms);
+    fire = callback;
+    return { unref };
+  });
+  const clear = t.mock.method(globalThis, "clearTimeout", () => {});
+
+  for (const ms of [0, 10, 25, 100, 250, 1_000]) {
+    let settled = false;
+    const waiting = sleep(ms).then((value) => {
+      settled = true;
+      assert.equal(value, undefined);
+    });
+    await Promise.resolve();
+    assert.equal(settled, false, "sleep must not resolve before its timer");
+    fire();
+    await waiting;
+    assert.equal(settled, true);
+  }
+
+  assert.deepEqual(delays, [0, 10, 25, 100, 250, 1_000]);
+  assert.equal(unref.mock.callCount(), 0, "awaited poll timers must keep the process alive");
+  assert.equal(clear.mock.callCount(), 0, "the non-cancellable helper must not cancel its timers");
+});

package/src/utils/sleep.ts

@@ -0,0 +1,5 @@
+/** Wait without cancellation; keep the timer referenced so awaited polling can finish. */
+export function sleep(ms: number): Promise<void> {
+  // Constructor form keeps this reachable utility compatible with ES2022 adapter type graphs.
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/chunk-7PZHH4KC.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/memory-projection-batch.ts"],"sourcesContent":["import type { MemoryProjectionCurrentState } from \"./types.js\";\nimport {\n  memoryCurrentSelectExpressions,\n  openProjectionReadonly,\n  parseCurrentRow,\n} from \"./memory-projection-store.js\";\n\nexport interface ProjectedMemoryStateBatch {\n  states: Map<string, MemoryProjectionCurrentState>;\n  /** Meta `rebuiltAt`; null when the projection never recorded one. */\n  rebuiltAt: string | null;\n  /**\n   * True only when the writer recorded that `memory_current` covers every active\n   * memory (an unscoped rebuild). Projections written before this flag existed,\n   * and scoped rebuilds, read as false.\n   */\n  complete: boolean;\n}\n\n/**\n * Read many current-state rows over ONE readonly connection, plus the meta the\n * recall path needs to decide whether the index can be trusted. Ids absent from\n * `memory_current` are simply missing from the map. Null means the projection\n * could not be opened or read.\n */\nexport function readProjectedMemoryStates(\n  memoryDir: string,\n  memoryIds: Iterable<string>,\n): ProjectedMemoryStateBatch | null {\n  const db = openProjectionReadonly(memoryDir);\n  if (!db) return null;\n  try {\n    const currentSelect = memoryCurrentSelectExpressions(db);\n    const select = db.prepare(\n      `\n        SELECT\n          memory_id,\n          category,\n          status,\n          lifecycle_state,\n          path_rel,\n          created_at,\n          updated_at,\n          archived_at,\n          superseded_at,\n          entity_ref,\n          source,\n          confidence,\n          confidence_tier,\n          memory_kind,\n          access_count,\n          last_accessed,\n          ${currentSelect.tagsJson},\n          ${currentSelect.previewText},\n          ${currentSelect.privateRecord}\n        FROM memory_current\n        WHERE memory_id = ?\n      `,\n    );\n    const states = new Map<string, MemoryProjectionCurrentState>();\n    for (const memoryId of memoryIds) {\n      const state = parseCurrentRow(memoryDir, select.get(memoryId) as Record<string, unknown> | undefined);\n      if (state) states.set(memoryId, state);\n    }\n    const meta = new Map<string, unknown>(\n      (db.prepare(\"SELECT key, value FROM meta WHERE key IN ('rebuiltAt', 'complete')\").all() as Array<{\n        key: string;\n        value: unknown;\n      }>).map((row) => [row.key, row.value]),\n    );\n    const rebuiltAt = meta.get(\"rebuiltAt\");\n    return {\n      states,\n      rebuiltAt: typeof rebuiltAt === \"string\" ? rebuiltAt : null,\n      complete: meta.get(\"complete\") === \"true\",\n    };\n  } catch {\n    return null;\n  } finally {\n    db.close();\n  }\n}\n"],"mappings":";;;;;;;AAyBO,SAAS,0BACd,WACA,WACkC;AAClC,QAAM,KAAK,uBAAuB,SAAS;AAC3C,MAAI,CAAC,GAAI,QAAO;AAChB,MAAI;AACF,UAAM,gBAAgB,+BAA+B,EAAE;AACvD,UAAM,SAAS,GAAG;AAAA,MAChB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,YAkBM,cAAc,QAAQ;AAAA,YACtB,cAAc,WAAW;AAAA,YACzB,cAAc,aAAa;AAAA;AAAA;AAAA;AAAA,IAInC;AACA,UAAM,SAAS,oBAAI,IAA0C;AAC7D,eAAW,YAAY,WAAW;AAChC,YAAM,QAAQ,gBAAgB,WAAW,OAAO,IAAI,QAAQ,CAAwC;AACpG,UAAI,MAAO,QAAO,IAAI,UAAU,KAAK;AAAA,IACvC;AACA,UAAM,OAAO,IAAI;AAAA,MACd,GAAG,QAAQ,oEAAoE,EAAE,IAAI,EAGlF,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,IAAI,KAAK,CAAC;AAAA,IACvC;AACA,UAAM,YAAY,KAAK,IAAI,WAAW;AACtC,WAAO;AAAA,MACL;AAAA,MACA,WAAW,OAAO,cAAc,WAAW,YAAY;AAAA,MACvD,UAAU,KAAK,IAAI,UAAU,MAAM;AAAA,IACrC;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT,UAAE;AACA,OAAG,MAAM;AAAA,EACX;AACF;","names":[]}