@remnic/core 10.0.97 → 10.0.99

package/src/orchestration/backlog-scheduler.ts

@@ -70,6 +70,8 @@ export interface BacklogSchedulerOptions {
   pollMs?: number;
   retryBaseMs?: number;
   retryMaxMs?: number;
+  /** Bound on `stop()`'s wait for aborted drains to settle. */
+  stopSettleTimeoutMs?: number;
   now?: () => number;
 }
 
@@ -114,6 +116,9 @@ interface KeyState {
 const DEFAULT_POLL_MS = 15_000;
 const DEFAULT_RETRY_BASE_MS = 30_000;
 const DEFAULT_RETRY_MAX_MS = 15 * 60_000;
+// Aborted drains normally settle within seconds (a chunk boundary); this only
+// bounds the pathological case, well inside the daemon's stop budget.
+const DEFAULT_STOP_SETTLE_TIMEOUT_MS = 30_000;
 
 export class BacklogScheduler {
   private readonly deps: BacklogSchedulerDeps;
@@ -141,6 +146,7 @@ export class BacklogScheduler {
       pollMs: options.pollMs ?? DEFAULT_POLL_MS,
       retryBaseMs: options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS,
       retryMaxMs: options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS,
+      stopSettleTimeoutMs: options.stopSettleTimeoutMs ?? DEFAULT_STOP_SETTLE_TIMEOUT_MS,
       now: options.now,
     };
   }
@@ -155,7 +161,10 @@ export class BacklogScheduler {
   }
 
   /** Idempotent. Stops the timer, aborts in-flight drains (their claims stay
-   *  retryable) and waits for them to settle. */
+   *  retryable) and waits up to `stopSettleTimeoutMs` for them to settle. A
+   *  drain that ignores abort is named in a warning and left behind: the
+   *  caller's later shutdown phases (orchestrator flush and dispose) must
+   *  still run before the service manager's kill. */
   async stop(): Promise<void> {
     if (this.stopped) return;
     this.stopped = true;
@@ -168,7 +177,19 @@ export class BacklogScheduler {
       entry.controller.abort(new Error("backlog scheduler stopping"));
       aborts.push(entry.settled);
     }
-    await Promise.allSettled(aborts);
+    const bound = Promise.withResolvers<"timeout">();
+    const boundTimer = setTimeout(() => bound.resolve("timeout"), this.options.stopSettleTimeoutMs);
+    try {
+      const result = await Promise.race([Promise.allSettled(aborts), bound.promise]);
+      if (result === "timeout") {
+        log.warn(
+          `backlog scheduler stop: ${this.inFlight.size} drain(s) ignored abort for ${this.options.stopSettleTimeoutMs}ms; ` +
+            `continuing shutdown (stuck keys: ${[...this.inFlight.keys()].join(", ")})`,
+        );
+      }
+    } finally {
+      clearTimeout(boundTimer);
+    }
   }
 
   /** Bounded snapshot for ops/watchers: counters + per-key states + live sets. */

package/src/orchestration/extraction-queue-bounded-retention.test.ts

@@ -299,6 +299,26 @@ test("abort while queued: settled task starts, skips extraction, queue drains",
   assert.equal(settled.length, 1, "queued task settles exactly once from the abort");
 });
 
+test("abort while queued: onTaskExit fires at once, not after the queue head finishes", async () => {
+  const h = makeHarness();
+  const controller = new AbortController();
+  // Occupies the single worker; the drain stays blocked until release().
+  await h.coordinator.processTurn("user", "running", "k");
+  let exited = 0;
+  await h.coordinator.queueBufferedExtraction([makeTurn("queued then aborted")], "trigger_mode", {
+    bufferKey: "k",
+    preemptable: true,
+    abortSignal: controller.signal,
+    onTaskExit: () => { exited += 1; },
+  });
+  controller.abort();
+  // A claim-lease owner waiting on this exit must not be held by an unrelated
+  // running extraction: nothing of this task ever ran.
+  assert.equal(exited, 1, "queued task that never started reports exit on abort");
+  await h.release();
+  assert.equal(exited, 1, "the later no-op closure run does not report a second exit");
+});
+
 test("sharing bound: distinct reference slots equal the union, not the snapshot sum", async () => {
   const h = makeHarness();
   const turns = 30;

package/src/orchestration/orchestrator-init.ts

@@ -115,6 +115,7 @@ export class OrchestratorInitCoordinator {
     | { epoch: number; controller: AbortController }
     | undefined;
   private readonly startupStorageWrites = new Set<Promise<void>>();
+  private readonly startupWarmups = new Set<Promise<void>>();
 
   constructor(
     private readonly deps: OrchestratorInitDeps,
@@ -138,6 +139,11 @@ export class OrchestratorInitCoordinator {
     }
   }
 
+  /** Wait for background startup warm-ups; destroy() only, never the deferredReady gate. */
+  async drainStartupWarmups(): Promise<void> {
+    await Promise.allSettled([...this.startupWarmups]);
+  }
+
   async drainEntityMigrations(): Promise<void> {
     await Promise.allSettled([...this.startupStorageWrites]);
     const drains = [this.deps.storage.drainEntityCanonicalIdMigration()];
@@ -657,6 +663,25 @@ export class OrchestratorInitCoordinator {
 
     if (signal.aborted || !this.isCurrentStartupEpoch(epoch)) return;
 
+    // Build the default namespace's authoritative fact-hash index now, in the
+    // background. The first explicit save otherwise pays the whole hot+cold
+    // corpus rebuild (measured 3.1 s CPU, 11 to 33 s wall on jarvis-ml). A save
+    // that arrives mid-build joins this same in-flight promise inside
+    // StorageManager, so the rebuild never runs twice. Not awaited: the
+    // deferredReady gate must not wait on it, and a failure only means the first
+    // save builds the index itself, as before.
+    const factHashWarm = (async () => {
+      const storage = await this.deps.storageRouter.storageFor(this.deps.config.defaultNamespace);
+      await storage.isFactContentHashAuthoritative();
+    })().catch((err) => {
+      log.debug(`fact-hash index startup warm-up failed (non-fatal): ${err}`);
+    });
+    // Tracked apart from startupStorageWrites: destroy() waits for it so the
+    // rebuild never touches state/ after teardown, but the deferredReady gate
+    // (which drains startupStorageWrites) must not.
+    this.startupWarmups.add(factHashWarm);
+    void factHashWarm.finally(() => this.startupWarmups.delete(factHashWarm));
+
     // Warmup: run cheap searches to pre-load QMD embedding models and the
     // embedding-fallback JSON index so the first real recall is fast.
     const warmupPromises: Promise<void>[] = [];

package/src/orchestration/task-exit.ts

@@ -0,0 +1,27 @@
+/**
+ * Exit-once bookkeeping for a queued extraction task (`onTaskExit`).
+ *
+ * A task settled (abort, queue deadline) before its closure ever ran has
+ * nothing to exit, so `settled()` reports exit at once; a claim-lease owner
+ * waiting on the exit is then not held behind an unrelated running extraction.
+ * A closure that did start reports exit when it finishes. Either way the
+ * callback fires exactly once.
+ */
+export function createTaskExit(onExit?: () => void) {
+  let started = false;
+  let reported = false;
+  const report = (): void => {
+    if (reported) return;
+    reported = true;
+    onExit?.();
+  };
+  return {
+    start: () => {
+      started = true;
+    },
+    settled: () => {
+      if (!started) report();
+    },
+    report,
+  };
+}

package/src/orchestration/turn-ingestion.ts

@@ -60,6 +60,7 @@ import {
 // The class lives in ../buffer-admission.js next to BufferAdmissionRefusedError;
 // re-exported here so existing importers keep their import path.
 import { ReplayAdmissionRefusedError, StorageTargetRequiredError } from "../buffer-admission.js";
+import { createTaskExit } from "./task-exit.js";
 export { ReplayAdmissionRefusedError, StorageTargetRequiredError };
 
 export interface TurnIngestionDeps {
@@ -798,13 +799,7 @@ export class TurnIngestionCoordinator {
       abortSignal?: AbortSignal;
       /** Automatic buffered work only. Replay and import must not set this. */
       preemptable?: boolean;
-      /**
-       * Invoked when the queued closure has FULLY exited (after settlement).
-       * Settlement can fire before the closure exits — e.g. an abort listener
-       * settles while persistence is still in flight — so callers that
-       * release resources on settlement (claim leases) can use this to wait
-       * for the actual task exit first.
-       */
+      /** Fires once when the task is done: closure exited, or settled before it started. */
       onTaskExit?: () => void;
       /** Fail-closed force-flush only. Not implied by failOnExtractionFailure. */
       preemptRunning?: boolean;
@@ -845,6 +840,7 @@ export class TurnIngestionCoordinator {
         : undefined;
     let timeout: ReturnType<typeof setTimeout> | undefined;
     let settled = false;
+    const exit = createTaskExit(options.onTaskExit);
     const clearQueueWaitTimer = (): void => {
       if (timeout) {
         clearTimeout(timeout);
@@ -867,6 +863,7 @@ export class TurnIngestionCoordinator {
       clearQueueWaitTimer();
       clearAbortListener();
       options.onTaskSettled?.(error, result);
+      exit.settled();
       return true;
     };
     if (options.abortSignal) {
@@ -910,6 +907,7 @@ export class TurnIngestionCoordinator {
     const queuedTurnsSource: { chain: SnapshotChain } | { direct: BufferTurn[] } =
       sharingNode ? { chain: sharingNode } : { direct: turnsToExtract };
     const accepted = this.deps.extractionQueueCoordinator.enqueue(async (preemptHandle) => {
+      exit.start();
       try {
         await this.deps.extractionQueueCoordinator.runPreemptable(preemptHandle, preemptable, options.abortSignal, async (abortSignal) => {
       if (settled) return;
@@ -1094,13 +1092,13 @@ export class TurnIngestionCoordinator {
       });
       } finally {
         if (sharingNode) this.snapshotSharing.release(sharingNode);
-        options.onTaskExit?.();
+        exit.report();
       }
     }, { front: options.preemptRunning === true, key: bufferKey });
     if (!accepted) {
       if (sharingNode) this.snapshotSharing.release(sharingNode);
       settleTask(new Error("extraction queue is stopped"));
-      options.onTaskExit?.();
+      exit.report();
       return;
     }
 

package/src/orchestrator-init.test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
 import { existsSync } from "node:fs";
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import test from "node:test";
@@ -1595,3 +1595,134 @@ for (const namespace of ["default", "shared"]) {
     }
   });
 }
+
+test("daemon start builds the fact-hash index once in the background and a save right after start joins it", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-warm-fact-hash-"));
+  const orchestrator = new Orchestrator(
+    parseConfig({
+      memoryDir,
+      workspaceDir: path.join(memoryDir, "workspace"),
+      qmdEnabled: false,
+      namespacesEnabled: true,
+      namespaceCatalogEnabled: false,
+      defaultNamespace: "default",
+      sharedNamespace: "shared",
+      namespacePolicies: [],
+      embeddingFallbackEnabled: false,
+      identityContinuityEnabled: false,
+      transcriptEnabled: false,
+      hourlySummariesEnabled: false,
+      compoundingEnabled: false,
+      knowledgeIndexEnabled: false,
+    }),
+  );
+  let releaseCorpus: (() => void) | undefined;
+  try {
+    const storage = await orchestrator.getStorage("default");
+    const realRead = storage.readAllMemories.bind(storage);
+    let corpusReads = 0;
+    let signalStarted!: () => void;
+    const rebuildStarted = new Promise<void>((resolve) => { signalStarted = resolve; });
+    const corpusHeld = new Promise<void>((resolve) => { releaseCorpus = resolve; });
+    storage.readAllMemories = async () => {
+      corpusReads += 1;
+      signalStarted();
+      await corpusHeld;
+      return realRead();
+    };
+
+    assert.notEqual(await withTestDeadline(orchestrator.initialize()), "deadline", "initialize must not wait on the held rebuild");
+    // SAFETY: test reads the private deferredReady gate, as the neighbouring tests do.
+    assert.notEqual(
+      await withTestDeadline((orchestrator as any).deferredReady),
+      "deadline",
+      "deferredReady must not wait on the held rebuild",
+    );
+    let giveUp: NodeJS.Timeout | undefined;
+    const outcome = await Promise.race([
+      rebuildStarted.then(() => "started" as const),
+      new Promise<"not-started">((resolve) => { giveUp = setTimeout(() => resolve("not-started"), 3_000); }),
+    ]);
+    clearTimeout(giveUp);
+    assert.equal(outcome, "started", "start must begin the fact-hash corpus rebuild before any save arrives");
+
+    // Two saves landing while the warm-up still runs: both wait on that one flight.
+    const saves = Promise.all([storage.lookupContentHash("first save"), storage.lookupContentHash("second save")]);
+    releaseCorpus?.();
+    const results = await withTestDeadline(saves);
+    if (results === "deadline") assert.fail("saves joining the warm-up never settled");
+    assert.equal(corpusReads, 1, "the saves must not start a second corpus rebuild");
+    assert.deepEqual(results.map((r) => r.authoritative), [true, true]);
+  } finally {
+    releaseCorpus?.();
+    await orchestrator.destroy().catch(() => undefined);
+    await rm(memoryDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 });
+  }
+});
+
+test("destroy persists a buffered turn before it waits on a running fact-hash warm-up", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-warm-destroy-flush-"));
+  const marker = "w7p-buffered-turn-marker";
+  const orchestrator = new Orchestrator(
+    parseConfig({
+      memoryDir,
+      workspaceDir: path.join(memoryDir, "workspace"),
+      qmdEnabled: false,
+      namespacesEnabled: true,
+      namespaceCatalogEnabled: false,
+      defaultNamespace: "default",
+      sharedNamespace: "shared",
+      namespacePolicies: [],
+      bufferSaveDebounceMs: 600_000,
+      embeddingFallbackEnabled: false,
+      identityContinuityEnabled: false,
+      transcriptEnabled: false,
+      hourlySummariesEnabled: false,
+      compoundingEnabled: false,
+      knowledgeIndexEnabled: false,
+    }),
+  );
+  let releaseCorpus: (() => void) | undefined;
+  try {
+    const storage = await orchestrator.getStorage("default");
+    const realRead = storage.readAllMemories.bind(storage);
+    let signalStarted!: () => void;
+    const rebuildStarted = new Promise<void>((resolve) => { signalStarted = resolve; });
+    const corpusHeld = new Promise<void>((resolve) => { releaseCorpus = resolve; });
+    storage.readAllMemories = async () => {
+      signalStarted();
+      await corpusHeld;
+      return realRead();
+    };
+    assert.notEqual(await withTestDeadline(orchestrator.initialize()), "deadline", "initialize must not wait on the held rebuild");
+    assert.notEqual(await withTestDeadline(rebuildStarted), "deadline", "the warm-up must start the rebuild");
+
+    await orchestrator.buffer.addTurn("warm-destroy-key", {
+      role: "user",
+      content: marker,
+      timestamp: new Date().toISOString(),
+    });
+    const bufferOnDisk = async (): Promise<boolean> => {
+      for (const name of await readdir(memoryDir, { recursive: true })) {
+        if ((await readFile(path.join(memoryDir, name), "utf8").catch(() => "")).includes(marker)) return true;
+      }
+      return false;
+    };
+    assert.equal(await bufferOnDisk(), false, "precondition: the debounced save has not run");
+
+    const destroying = orchestrator.destroy();
+    const startedAt = Date.now();
+    let persisted = false;
+    while (Date.now() - startedAt < 1_500 && !persisted) {
+      persisted = await bufferOnDisk();
+      if (!persisted) await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+    assert.equal(persisted, true, "the buffered turn must reach disk within 1.5 s of destroy() while the corpus read is held");
+    releaseCorpus?.();
+    assert.notEqual(await withTestDeadline(destroying), "deadline", "destroy must finish once the rebuild is released");
+  } finally {
+    releaseCorpus?.();
+    await orchestrator.destroy().catch(() => undefined);
+    await rm(memoryDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 });
+  }
+});

package/src/orchestrator.ts

@@ -1025,6 +1025,9 @@ export class Orchestrator {
     } catch (err) {
       bufferFlushError = err;
     }
+    // After the buffer flush: a long fact-hash warm-up must never delay
+    // persisting unsent turns (a kill during the wait would lose them).
+    await this._orchestratorInitCoordinator?.drainStartupWarmups();
     const extractionDrainError = await drainExtractionAndShutdownDependencyPropagation(
       this.extractionQueueCoordinator,
       this._dependencyPropagationDelivery,