@remnic/core 9.3.757 → 9.3.759

package/src/orchestration/recall-internal.ts

@@ -326,6 +326,7 @@ export interface RecallInternalDeps {
   ): number | undefined;
   getStorage(namespace?: string): Promise<StorageManager>;
   readonly handleHistory: RecallHandleHistoryStore;
+  trackRecallBackgroundWrite(promise: Promise<void>, label: string): void;
   isRecallSectionEnabled(
     sectionId: string,
     defaultEnabled?: boolean,
@@ -908,8 +909,8 @@ export class RecallInternalCoordinator {
         }
       }
       if (sessionKey) {
-        this.deps.lastRecall
-          .record({
+        this.deps.trackRecallBackgroundWrite(
+          this.deps.lastRecall.record({
             sessionKey,
             query: retrievalQuery,
             memoryIds: [],
@@ -935,8 +936,9 @@ export class RecallInternalCoordinator {
               injectedChars: identityInjectedChars,
               truncated: identityInjectionTruncated,
             },
-          })
-          .catch((err) => log.debug(`last recall record failed: ${err}`));
+          }),
+          "last recall record",
+        );
       }
       if (sessionKey) {
         this.deps.queueEvalShadowRecall({
@@ -5364,8 +5366,8 @@ export class RecallInternalCoordinator {
 
     if (sessionKey) {
       throwIfRecallAborted(options.abortSignal);
-      this.deps.lastRecall
-        .record({
+      this.deps.trackRecallBackgroundWrite(
+        this.deps.lastRecall.record({
           sessionKey,
           query: retrievalQuery,
           memoryIds: recalledMemoryIds,
@@ -5397,8 +5399,9 @@ export class RecallInternalCoordinator {
           // reviews on #1544).
           backendDegradations:
             backendDegradations.length > 0 ? backendDegradations : undefined,
-        })
-        .catch((err) => log.debug(`last recall record failed: ${err}`));
+        }),
+        "last recall record",
+      );
       // Issue #1582 — record the admitted memory-id set for handle resolution.
       // Only when handles are enabled: if injection is off, no handle is ever
       // rendered, so there is nothing to resolve and we skip the write.
@@ -5410,9 +5413,10 @@ export class RecallInternalCoordinator {
           MEMORY_ID_PATTERN.test(id),
         );
         if (handleEligibleIds.length > 0) {
-          this.deps.handleHistory
-            .record(sessionKey, handleEligibleIds)
-            .catch((err) => log.debug(`handle history record failed: ${err}`));
+          this.deps.trackRecallBackgroundWrite(
+            this.deps.handleHistory.record(sessionKey, handleEligibleIds),
+            "handle history record",
+          );
         }
       }
     }

package/src/orchestration/recall-introspection.ts

@@ -66,6 +66,7 @@ export interface RecallIntrospectionDeps {
   getStorage(namespace?: string): Promise<StorageManager>;
   readonly graphRecallCoordinator: GraphRecallCoordinator;
   readonly lastRecall: LastRecallStore;
+  trackRecallBackgroundWrite(promise: Promise<void>, label: string): void;
   resolveStateDirForNamespace(
     namespace: string,
   ): Promise<string>;
@@ -500,6 +501,10 @@ export class RecallIntrospectionCoordinator {
           log.debug(`direct-answer observation chain error: ${err}`);
         }
       });
+    this.deps.trackRecallBackgroundWrite(
+      this.deps.directAnswerObservationChain,
+      "direct-answer observation",
+    );
   }
 
   async annotateDirectAnswerTier(

package/src/orchestrator-background-writes.test.ts

@@ -0,0 +1,101 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { parseConfig } from "./config.js";
+import { Orchestrator } from "./orchestrator.js";
+
+test("destroy cancels maintenance and waits for tracked writes before disposing search backends", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-background-writes-"));
+  const orchestrator = new Orchestrator(
+    parseConfig({
+      openaiApiKey: "sk-test",
+      memoryDir,
+      workspaceDir: path.join(memoryDir, "workspace"),
+      qmdEnabled: false,
+      knowledgeIndexEnabled: false,
+      identityContinuityEnabled: false,
+      transcriptEnabled: false,
+      hourlySummariesEnabled: false,
+      compoundingEnabled: false,
+    })
+  );
+  const maintenanceScheduler = (orchestrator as unknown as { maintenanceScheduler: { dispose(): void } })
+    .maintenanceScheduler;
+  const originalMaintenanceDispose = maintenanceScheduler.dispose.bind(maintenanceScheduler);
+  let maintenanceDisposed = false;
+  maintenanceScheduler.dispose = () => {
+    maintenanceDisposed = true;
+    originalMaintenanceDispose();
+  };
+  const writeGate = Promise.withResolvers<void>();
+  let destroySettled = false;
+  let destroyPromise: Promise<void> | undefined;
+
+  try {
+    orchestrator.trackRecallBackgroundWrite(writeGate.promise, "test background write");
+    destroyPromise = orchestrator.destroy().then(() => {
+      destroySettled = true;
+    });
+
+    await new Promise<void>((resolve) => setImmediate(resolve));
+    assert.equal(destroySettled, false, "destroy must remain pending while a tracked write is pending");
+    assert.equal(maintenanceDisposed, true, "destroy must cancel maintenance before waiting for writes");
+
+    writeGate.resolve();
+    await destroyPromise;
+    assert.equal(destroySettled, true);
+  } finally {
+    writeGate.resolve();
+    await (destroyPromise ?? orchestrator.destroy());
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("destroy waits for a recall-initiated last-recall write", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-recall-background-write-"));
+  const orchestrator = new Orchestrator(
+    parseConfig({
+      openaiApiKey: "sk-test",
+      memoryDir,
+      workspaceDir: path.join(memoryDir, "workspace"),
+      qmdEnabled: false,
+      knowledgeIndexEnabled: false,
+      identityContinuityEnabled: false,
+      transcriptEnabled: false,
+      hourlySummariesEnabled: false,
+      compoundingEnabled: false,
+    })
+  );
+  const writeStarted = Promise.withResolvers<void>();
+  const writeGate = Promise.withResolvers<void>();
+  const originalRecord = orchestrator.lastRecall.record.bind(orchestrator.lastRecall);
+  orchestrator.lastRecall.record = async (...args: Parameters<typeof originalRecord>) => {
+    writeStarted.resolve();
+    await writeGate.promise;
+    await originalRecord(...args);
+  };
+  let destroySettled = false;
+  let destroyPromise: Promise<void> | undefined;
+
+  try {
+    await orchestrator.initialize();
+    await orchestrator.recall("recall state drain contract", "session-background-write");
+    await writeStarted.promise;
+    destroyPromise = orchestrator.destroy().then(() => {
+      destroySettled = true;
+    });
+
+    await new Promise<void>((resolve) => setImmediate(resolve));
+    assert.equal(destroySettled, false, "destroy must wait for the recall-initiated state write");
+
+    writeGate.resolve();
+    await destroyPromise;
+    assert.equal(destroySettled, true);
+  } finally {
+    writeGate.resolve();
+    await (destroyPromise ?? orchestrator.destroy());
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/orchestrator.ts

@@ -120,6 +120,7 @@ import { NamespaceReadFanoutCoordinator } from "./orchestration/namespace-read-f
 import { selfDeps } from "./orchestration/self-deps.js";
 import { RecallEntryCoordinator } from "./orchestration/recall-entry.js";
 import { SessionContextCoordinator } from "./orchestration/session-context.js";
+import { drainRecallWrites, trackRecallWrite } from "./orchestration/recall-background-writes.js";
 import {
   abortRecallError,
   buildCompressionGuidelinesMarkdown,
@@ -135,9 +136,8 @@ import {
   type QueryAwarePrefilter,
   type RecallInvocationOptions,
 } from "./orchestration/orchestrator-helpers.js";
-// Issue #1526 seam 25: module-level helpers moved to
-// orchestration/orchestrator-helpers.ts; re-exported here so existing
-// importers (coordinators, tests, root shims) keep working.
+// Issue #1526 seam 25: helpers moved to orchestration/orchestrator-helpers.ts and are
+// re-exported here so existing importers (coordinators, tests, root shims) keep working.
 export {
   BulkImportBatchPartialFailureError,
   COMPACTION_SIGNAL_MAX_AGE_MS,
@@ -701,8 +701,7 @@ export class Orchestrator {
   /** Read by NamespaceReadFanoutCoordinator (seam 26), hence not `private`. */
   static readonly ARTIFACT_STATUS_CACHE_TTL_MS = 60_000;
 
-  // Access tracking buffer (Phase 1A)
-  // Maps memoryId -> {count, lastAccessed} for batched updates
+  // Batched access-tracking counts and timestamps (Phase 1A).
   private accessTrackingBuffer: Map<
     string,
     { count: number; lastAccessed: string }
@@ -887,9 +886,7 @@ export class Orchestrator {
   private utilityRuntimeValues: UtilityRuntimeValues | null = null;
   private evalShadowWriteChain: Promise<void> = Promise.resolve();
 
-  // Pending background observation-mode direct-answer annotations (#518).
-  // Tracks fire-and-forget `annotateDirectAnswerTier` calls so callers (tests,
-  // waitForDirectAnswerObservationIdle) can await settlement.
+  // Pending fire-and-forget direct-answer tier annotations (#518).
   private directAnswerObservationChain: Promise<void> = Promise.resolve();
 
   // Initialization gate: recall() awaits this before proceeding
@@ -934,8 +931,9 @@ export class Orchestrator {
     }
   }
 
-  private async disposeSearchBackendIfNeeded(): Promise<void> {
-    await (this.qmd as { dispose?: () => void | Promise<void> }).dispose?.();
+  /** Track a recall-side write so destroy() can wait for it before teardown. @internal */
+  trackRecallBackgroundWrite(promise: Promise<void>, label: string): void {
+    trackRecallWrite(this, promise, label);
   }
 
   /**
@@ -952,8 +950,9 @@ export class Orchestrator {
       this.wearablesAutoSyncHandle = null;
     }
     this.maintenanceScheduler.dispose();
+    await drainRecallWrites(this);
     await this.namespaceSearchRouter.dispose();
-    await this.disposeSearchBackendIfNeeded();
+    await (this.qmd as { dispose?: () => void | Promise<void> }).dispose?.();
     if (this.conversationQmd && this.conversationQmd !== this.qmd) {
       await (this.conversationQmd as { dispose?: () => void | Promise<void> }).dispose?.();
     }
@@ -3713,8 +3712,9 @@ export class Orchestrator {
     if (
       this.accessTrackingBuffer.size >= this.config.accessTrackingBufferMaxSize
     ) {
-      this.flushAccessTracking().catch((err) =>
-        log.debug(`background access tracking flush failed: ${err}`),
+      this.trackRecallBackgroundWrite(
+        this.flushAccessTracking(),
+        "background access tracking flush",
       );
     }
   }

package/src/storage/identity-continuity-store.ts

@@ -0,0 +1,223 @@
+/**
+ * Identity-continuity persistence extracted from StorageManager.
+ *
+ * The host keeps the public API while this store owns anchors, incidents,
+ * audits, and improvement-loop files. Dependencies stay live through
+ * `selfDeps`, preserving secure-store and catalog-write behavior.
+ */
+
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import {
+  closeContinuityIncidentRecord,
+  createContinuityIncidentRecord,
+  parseContinuityImprovementLoops,
+  parseContinuityIncident,
+  reviewContinuityLoopInMarkdown,
+  serializeContinuityIncident,
+  upsertContinuityLoopInMarkdown,
+} from "../identity-continuity.js";
+import { SecureStoreLockedError } from "../secure-store/secure-fs.js";
+import type {
+  ContinuityImprovementLoop,
+  ContinuityIncidentCloseInput,
+  ContinuityIncidentOpenInput,
+  ContinuityIncidentRecord,
+  ContinuityLoopReviewInput,
+  ContinuityLoopUpsertInput,
+} from "../types.js";
+import { isErrnoCode } from "../utils/errno.js";
+
+export interface IdentityContinuityStoreDeps {
+  ensureDirectories(): Promise<void>;
+  generateId(prefix?: string): string;
+  readonly identityAnchorPath: string;
+  readonly identityIncidentsDir: string;
+  readonly identityAuditsWeeklyDir: string;
+  readonly identityAuditsMonthlyDir: string;
+  readonly identityImprovementLoopsPath: string;
+  readStorageSecureFile(filePath: string): Promise<string>;
+  writeStorageSecureFile(filePath: string, content: string | Buffer): Promise<void>;
+}
+
+export class IdentityContinuityStore {
+  constructor(private readonly deps: IdentityContinuityStoreDeps) {}
+
+  async writeIdentityAnchor(content: string): Promise<void> {
+    await this.deps.ensureDirectories();
+    await this.deps.writeStorageSecureFile(this.deps.identityAnchorPath, content);
+  }
+
+  async readIdentityAnchor(): Promise<string | null> {
+    try {
+      return await this.deps.readStorageSecureFile(this.deps.identityAnchorPath);
+    } catch (err) {
+      if (!isErrnoCode(err, "ENOENT")) throw err;
+      return null;
+    }
+  }
+
+  async appendContinuityIncident(input: ContinuityIncidentOpenInput): Promise<ContinuityIncidentRecord> {
+    await this.deps.ensureDirectories();
+    const now = new Date();
+    const nowIso = now.toISOString();
+    const date = nowIso.slice(0, 10);
+    const id = this.deps.generateId("incident");
+    const incident = createContinuityIncidentRecord(id, input, nowIso);
+    const filePath = path.join(this.deps.identityIncidentsDir, `${date}-${id}.md`);
+    await this.deps.writeStorageSecureFile(filePath, serializeContinuityIncident(incident));
+    return { ...incident, filePath };
+  }
+
+  async readContinuityIncidents(
+    limit = 200,
+    state: "open" | "closed" | "all" = "all"
+  ): Promise<ContinuityIncidentRecord[]> {
+    const normalizedLimit = Number.isFinite(limit) ? Math.floor(limit) : 0;
+    const cappedLimit = Math.max(0, normalizedLimit);
+    if (cappedLimit === 0) return [];
+
+    try {
+      const candidates = await this.readContinuityIncidentFileNames();
+      const incidents: ContinuityIncidentRecord[] = [];
+
+      for (const file of candidates) {
+        if (incidents.length >= cappedLimit) break;
+        const filePath = path.join(this.deps.identityIncidentsDir, file);
+        try {
+          const raw = await this.deps.readStorageSecureFile(filePath);
+          const parsed = parseContinuityIncident(raw);
+          if (!parsed) continue;
+          if (state !== "all" && parsed.state !== state) continue;
+          incidents.push({ ...parsed, filePath });
+        } catch (err) {
+          if (err instanceof SecureStoreLockedError) throw err;
+          // Fail-open on malformed/missing files.
+        }
+      }
+      return incidents;
+    } catch (err) {
+      if (err instanceof SecureStoreLockedError) throw err;
+      if (!isErrnoCode(err, "ENOENT")) throw err;
+      return [];
+    }
+  }
+
+  async closeContinuityIncident(
+    id: string,
+    closure: ContinuityIncidentCloseInput
+  ): Promise<ContinuityIncidentRecord | null> {
+    const directFilePath = await this.findContinuityIncidentFilePathById(id);
+    const target = directFilePath ? await this.readContinuityIncidentFile(directFilePath) : null;
+    if (!target || !directFilePath) return null;
+    if (target.state === "closed") return target;
+
+    const closed = closeContinuityIncidentRecord(target, closure, new Date().toISOString());
+    await this.deps.writeStorageSecureFile(directFilePath, serializeContinuityIncident(closed));
+    return { ...closed, filePath: directFilePath };
+  }
+
+  async writeIdentityAudit(period: "weekly" | "monthly", key: string, content: string): Promise<string> {
+    await this.deps.ensureDirectories();
+    const safeKey = this.sanitizeIdentityAuditKey(key);
+    const dir = period === "weekly" ? this.deps.identityAuditsWeeklyDir : this.deps.identityAuditsMonthlyDir;
+    const filePath = path.join(dir, `${safeKey}.md`);
+    await this.deps.writeStorageSecureFile(filePath, content);
+    return filePath;
+  }
+
+  async readIdentityAudit(period: "weekly" | "monthly", key: string): Promise<string | null> {
+    try {
+      const safeKey = this.sanitizeIdentityAuditKey(key);
+      const dir = period === "weekly" ? this.deps.identityAuditsWeeklyDir : this.deps.identityAuditsMonthlyDir;
+      return await this.deps.readStorageSecureFile(path.join(dir, `${safeKey}.md`));
+    } catch (err) {
+      if (err instanceof Error && err.message === "Invalid identity audit key") return null;
+      if (!isErrnoCode(err, "ENOENT")) throw err;
+      return null;
+    }
+  }
+
+  async writeIdentityImprovementLoops(content: string): Promise<void> {
+    await this.deps.ensureDirectories();
+    await this.deps.writeStorageSecureFile(this.deps.identityImprovementLoopsPath, content);
+  }
+
+  async readIdentityImprovementLoops(): Promise<string | null> {
+    try {
+      return await this.deps.readStorageSecureFile(this.deps.identityImprovementLoopsPath);
+    } catch (err) {
+      if (!isErrnoCode(err, "ENOENT")) throw err;
+      return null;
+    }
+  }
+
+  async readIdentityImprovementLoopRegister(): Promise<ContinuityImprovementLoop[]> {
+    const raw = await this.readIdentityImprovementLoops();
+    if (!raw) return [];
+    return parseContinuityImprovementLoops(raw);
+  }
+
+  async upsertIdentityImprovementLoop(input: ContinuityLoopUpsertInput): Promise<ContinuityImprovementLoop> {
+    const nowIso = new Date().toISOString();
+    const raw = await this.readIdentityImprovementLoops();
+    const { markdown, loop } = upsertContinuityLoopInMarkdown(raw, input, nowIso);
+    await this.writeIdentityImprovementLoops(markdown);
+    return loop;
+  }
+
+  async reviewIdentityImprovementLoop(
+    id: string,
+    input: ContinuityLoopReviewInput
+  ): Promise<ContinuityImprovementLoop | null> {
+    const raw = await this.readIdentityImprovementLoops();
+    const { markdown, loop } = reviewContinuityLoopInMarkdown(raw, id, input, new Date().toISOString());
+    if (!loop) return null;
+    await this.writeIdentityImprovementLoops(markdown);
+    return loop;
+  }
+
+  private async readContinuityIncidentFileNames(): Promise<string[]> {
+    const files = await readdir(this.deps.identityIncidentsDir);
+    return files
+      .filter((file) => file.endsWith(".md"))
+      .sort()
+      .reverse();
+  }
+
+  private async readContinuityIncidentFile(filePath: string): Promise<ContinuityIncidentRecord | null> {
+    try {
+      const raw = await this.deps.readStorageSecureFile(filePath);
+      const parsed = parseContinuityIncident(raw);
+      return parsed ? { ...parsed, filePath } : null;
+    } catch (err) {
+      if (err instanceof SecureStoreLockedError) throw err;
+      return null;
+    }
+  }
+
+  private async findContinuityIncidentFilePathById(id: string): Promise<string | null> {
+    const fileNames = await this.readContinuityIncidentFileNames();
+    const directMatch = fileNames.find((name) => name.endsWith(`-${id}.md`));
+    if (directMatch) {
+      const directPath = path.join(this.deps.identityIncidentsDir, directMatch);
+      const parsed = await this.readContinuityIncidentFile(directPath);
+      if (parsed?.id === id) return directPath;
+    }
+
+    for (const fileName of fileNames) {
+      const filePath = path.join(this.deps.identityIncidentsDir, fileName);
+      const parsed = await this.readContinuityIncidentFile(filePath);
+      if (parsed?.id === id) return filePath;
+    }
+    return null;
+  }
+
+  private sanitizeIdentityAuditKey(key: string): string {
+    const trimmed = key.trim();
+    if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(trimmed) || trimmed.includes("..")) {
+      throw new Error("Invalid identity audit key");
+    }
+    return trimmed;
+  }
+}

package/src/storage-contract/round-trip.test.ts

@@ -12,6 +12,8 @@
  */
 
 import assert from "node:assert/strict";
+import { utimes } from "node:fs/promises";
+import path from "node:path";
 import type { MemoryCategory } from "../types.js";
 import test from "node:test";
 
@@ -212,6 +214,9 @@ test("question-queue: writeQuestion → readQuestions → resolveQuestion round-
     // resolve marks it resolved
     const resolved = await storage.resolveQuestion(id);
     assert.equal(resolved, true);
+    // Directory mtimes are only a cross-process hint and may be coarser than
+    // the cache timestamp. Same-process resolution must invalidate directly.
+    await utimes(path.join(storage.dir, "questions"), new Date(0), new Date(0));
 
     const afterResolve = await storage.readQuestions({ unresolvedOnly: true });
     assert.ok(!afterResolve.some((x) => x.id === id), "resolved question must not appear in unresolvedOnly");