@remnic/core 9.48.1 → 9.49.1

package/src/in-flight-reads.ts

@@ -14,8 +14,20 @@ import type { MemoryFile } from "./types.js";
  * an unlocked manager's decrypted scan must NOT be handed to a locked or
  * differently-keyed manager for the same dir — that would bypass the keyId
  * isolation getCachedMemories enforces. Locked/plaintext stores use keyId "".
+ *
+ * Since issue #2307 the registry also owns the scan's CANCELLATION, because
+ * whether a shared scan may be cancelled is purely a question of who is still
+ * attached to it — see `beginCoalescedScan`.
  */
-const inFlightReadsByKey = new Map<string, Promise<MemoryFile[]>>();
+type InFlightEntry = {
+  read: Promise<MemoryFile[]>;
+  /** Readers still awaiting this scan. At zero the scan has no reason to finish. */
+  waiters: number;
+  /** Withdraw the slot and abort the scan. Absent for an uncancellable scan. */
+  cancel?: () => void;
+};
+
+const inFlightReadsByKey = new Map<string, InFlightEntry>();
 
 const DIR_SEP = "\u0000";
 
@@ -23,12 +35,113 @@ function composeKey(baseDir: string, keyId: string): string {
   return `${baseDir}${DIR_SEP}${keyId}`;
 }
 
-export function getInFlightRead(baseDir: string, keyId = ""): Promise<MemoryFile[]> | undefined {
-  return inFlightReadsByKey.get(composeKey(baseDir, keyId));
+/**
+ * A reader's attachment to a shared scan. `leave()` is idempotent per reader and
+ * must be called when that reader stops waiting — because its own signal fired, or
+ * because the scan settled.
+ */
+export interface CoalescedScanWaiter {
+  leave(): void;
+}
+
+/**
+ * Cancellation lifecycle for a coalesced corpus scan (issue #2307).
+ *
+ * The rule is reference counting, not ownership: **the scan is cancelled when its
+ * LAST waiter leaves, and never before.** One reader can therefore not cancel work
+ * another still needs, and two readers that both give up do not leave a full disk
+ * scan running for nobody — which was the abandoned I/O this whole change exists to
+ * remove.
+ *
+ * On the last leave the slot is withdrawn from the registry BEFORE the controller
+ * fires, so a reader arriving later cannot attach to a promise already doomed to
+ * reject with an `AbortError` it never asked for; it starts a fresh scan instead.
+ */
+export interface CoalescedScan extends CoalescedScanWaiter {
+  /** The signal the shared scan itself observes. */
+  readonly scanSignal: AbortSignal;
+  /** Publish the scan to the registry and arm cancellation. */
+  arm(read: Promise<MemoryFile[]>): void;
+}
+
+/** Build the per-reader `leave` closure: decrement once, cancel at zero. */
+function trackWaiter(entry: InFlightEntry, callerSignal?: AbortSignal): CoalescedScanWaiter {
+  let left = false;
+  const leave = () => {
+    if (left) return;
+    left = true;
+    callerSignal?.removeEventListener("abort", onAbort);
+    entry.waiters -= 1;
+    if (entry.waiters <= 0) entry.cancel?.();
+  };
+  // Named so the listener can be removed again; `once` alone would leak the
+  // reference for a reader that leaves normally.
+  function onAbort() {
+    leave();
+  }
+  callerSignal?.addEventListener("abort", onAbort, { once: true });
+  return { leave };
 }
 
-export function setInFlightRead(baseDir: string, keyId: string, read: Promise<MemoryFile[]>): void {
-  inFlightReadsByKey.set(composeKey(baseDir, keyId), read);
+/**
+ * Attach to an existing scan for (baseDir, keyId), or `undefined` when none is in
+ * flight. The caller MUST `leave()` once it stops awaiting the returned promise.
+ */
+export function attachInFlightReader(
+  baseDir: string,
+  keyId = "",
+  callerSignal?: AbortSignal,
+): { read: Promise<MemoryFile[]>; waiter: CoalescedScanWaiter } | undefined {
+  const entry = inFlightReadsByKey.get(composeKey(baseDir, keyId));
+  if (entry === undefined) return undefined;
+  entry.waiters += 1;
+  return { read: entry.read, waiter: trackWaiter(entry, callerSignal) };
+}
+
+/** Start a cancellable coalesced scan as its first waiter. */
+export function beginCoalescedScan(
+  baseDir: string,
+  keyId: string,
+  callerSignal?: AbortSignal,
+): CoalescedScan {
+  const controller = new AbortController();
+  let waiter: CoalescedScanWaiter | undefined;
+  return {
+    scanSignal: controller.signal,
+    arm(read: Promise<MemoryFile[]>): void {
+      // `waiters: 1` IS the starter; trackWaiter only builds its leave closure
+      // (attaching readers do their own increment).
+      const entry: InFlightEntry = {
+        read,
+        waiters: 1,
+        cancel: () => {
+          deleteInFlightRead(baseDir, keyId, read);
+          controller.abort();
+        },
+      };
+      inFlightReadsByKey.set(composeKey(baseDir, keyId), entry);
+      waiter = trackWaiter(entry, callerSignal);
+    },
+    leave(): void {
+      waiter?.leave();
+    },
+  };
+}
+
+/**
+ * Register a scan promise directly, with no cancellation attached.
+ *
+ * For callers that own a promise rather than a scan they can abort — tests that
+ * seed a stale in-flight read, and any future producer outside `beginCoalescedScan`.
+ * The entry has no `cancel`, so waiters leaving is a no-op: an externally-owned
+ * promise is not this module's to abort.
+ */
+export function setInFlightRead(
+  baseDir: string,
+  keyId: string,
+  read: Promise<MemoryFile[]>,
+): void {
+  inFlightReadsByKey.set(composeKey(baseDir, keyId), { read, waiters: 1 });
 }
 
 /**
@@ -41,7 +154,7 @@ export function deleteInFlightRead(
   expected?: Promise<MemoryFile[]>,
 ): void {
   const key = composeKey(baseDir, keyId);
-  if (expected === undefined || inFlightReadsByKey.get(key) === expected) {
+  if (expected === undefined || inFlightReadsByKey.get(key)?.read === expected) {
     inFlightReadsByKey.delete(key);
   }
 }

package/src/index.ts

@@ -508,7 +508,8 @@ export {
   type RemnicChatGptMemoryCard,
   type RemnicChatGptMemoryInspectorResult,
 } from "./mcp-memory-inspector-app.js";
-
+export * from "./coding/pre-action-gate.js";
+export * from "./causal-trajectory.js";
 // Coding-agent subsystem (issue #569)
 export {
   resolveGitContext,
@@ -1442,10 +1443,7 @@ export {
   type ForkLineage,
 } from "./transfer/capsule-fork.js";
 
-// ---------------------------------------------------------------------------
-// Coding-graph engine contract (#1551 PR1) — types owned by core,
-// implemented by @remnic/coding-graph (à-la-carte optional).
-// ---------------------------------------------------------------------------
+// Coding-graph engine contract (#1551 PR1), implemented by optional @remnic/coding-graph.
 
 export {
   CODING_GRAPH_ENGINE_VERSION,
@@ -1463,3 +1461,4 @@ export type {
   SymbolIR,
 } from "./coding/coding-graph-types.js";
 export type { ResolvedScopeProfilePlan } from "./namespaces/scope-profiles.js";
+export * from "./maintenance/atomic-file.js";

package/src/json-store.ts

@@ -1,19 +1,53 @@
+import { randomUUID } from "node:crypto";
+import type { Dirent } from "node:fs";
+import {
+  lstat,
+  mkdir,
+  open,
+  readFile,
+  readdir,
+  rename,
+  unlink,
+  utimes,
+  writeFile,
+  type FileHandle,
+} from "node:fs/promises";
 import path from "node:path";
-import { readFile, readdir } from "node:fs/promises";
 
-export async function listJsonFiles(dir: string): Promise<string[]> {
+function hasErrorCode(error: unknown, code: string): boolean {
+  return typeof error === "object" && error !== null && "code" in error && (error as { code?: unknown }).code === code;
+}
+
+async function listJsonFilesStrictInner(dir: string, allowMissingDirectory: boolean): Promise<string[]> {
+  let entries: Dirent[];
   try {
-    const entries = await readdir(dir, { withFileTypes: true });
-    const out: string[] = [];
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
-      if (entry.isDirectory()) {
-        out.push(...(await listJsonFiles(fullPath)));
-      } else if (entry.isFile() && entry.name.endsWith(".json")) {
-        out.push(fullPath);
-      }
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch (error) {
+    if (allowMissingDirectory && hasErrorCode(error, "ENOENT")) return [];
+    throw error;
+  }
+  const out: string[] = [];
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      out.push(...(await listJsonFilesStrictInner(fullPath, false)));
+    } else if (entry.isFile() && entry.name.endsWith(".json")) {
+      out.push(fullPath);
     }
-    return out.sort();
+  }
+  return out.sort();
+}
+
+export async function listJsonFilesStrict(
+  dir: string,
+  options: { allowMissingDirectory?: boolean } = {}
+): Promise<string[]> {
+  return listJsonFilesStrictInner(dir, options.allowMissingDirectory === true);
+}
+
+export async function listJsonFiles(dir: string): Promise<string[]> {
+  try {
+    return await listJsonFilesStrict(dir);
   } catch {
     return [];
   }
@@ -37,6 +71,314 @@ export async function listNamedFiles(dir: string, fileName: string): Promise<str
   }
 }
 
+const jsonStoreMutationTails = new Map<string, Promise<void>>();
+
+const JSON_STORE_LOCK_RETRY_MS = 25;
+const JSON_STORE_LOCK_STALE_MS = 60_000;
+const JSON_STORE_LOCK_TIMEOUT_MS = 10_000;
+
+type JsonStoreMutationLockTestHooks = {
+  beforeLockOwnerWrite?: (lockPath: string, ownerToken: string) => Promise<void> | void;
+  beforeLockOwnershipConfirm?: (lockPath: string, ownerToken: string) => Promise<void> | void;
+  afterReclaimGuardWrite?: (reclaimPath: string, ownerToken: string) => Promise<void> | void;
+  retryMs?: number;
+  staleLockMs?: number;
+  timeoutMs?: number;
+};
+
+let jsonStoreMutationLockTestHooks: JsonStoreMutationLockTestHooks | null = null;
+
+export function setJsonStoreMutationLockTestHooks(hooks: JsonStoreMutationLockTestHooks | null): void {
+  jsonStoreMutationLockTestHooks = hooks;
+}
+
+function jsonStoreLockRetryMs(): number {
+  return jsonStoreMutationLockTestHooks?.retryMs ?? JSON_STORE_LOCK_RETRY_MS;
+}
+
+function jsonStoreLockStaleMs(): number {
+  return jsonStoreMutationLockTestHooks?.staleLockMs ?? JSON_STORE_LOCK_STALE_MS;
+}
+
+function jsonStoreLockTimeoutMs(): number {
+  return jsonStoreMutationLockTestHooks?.timeoutMs ?? JSON_STORE_LOCK_TIMEOUT_MS;
+}
+
+async function closeAndRemoveJsonStoreReclaimGuard(reclaimPath: string, reclaimHandle: FileHandle): Promise<void> {
+  await reclaimHandle.close().catch(() => undefined);
+  await unlink(reclaimPath).catch((error) => {
+    if (!hasErrorCode(error, "ENOENT")) throw error;
+  });
+}
+
+async function removeStaleJsonStoreReclaimGuard(reclaimPath: string): Promise<void> {
+  let initial: JsonStoreLockState;
+  try {
+    initial = await readJsonStoreLockState(reclaimPath);
+  } catch (error) {
+    if (hasErrorCode(error, "ENOENT")) return;
+    throw error;
+  }
+  if (Date.now() - initial.mtimeMs <= jsonStoreLockStaleMs()) return;
+
+  let ownerPid: number | undefined;
+  try {
+    const metadata = JSON.parse(initial.content) as { pid?: unknown };
+    ownerPid = typeof metadata.pid === "number" && Number.isInteger(metadata.pid) ? metadata.pid : undefined;
+  } catch {
+    ownerPid = undefined;
+  }
+  if (ownerPid !== undefined && processIsRunning(ownerPid)) return;
+
+  let confirmed: JsonStoreLockState;
+  try {
+    confirmed = await readJsonStoreLockState(reclaimPath);
+  } catch (error) {
+    if (hasErrorCode(error, "ENOENT")) return;
+    throw error;
+  }
+  if (!sameJsonStoreLockState(initial, confirmed)) return;
+  await unlink(reclaimPath).catch((error) => {
+    if (!hasErrorCode(error, "ENOENT")) throw error;
+  });
+}
+
+async function acquireJsonStoreReclaimGuard(reclaimPath: string): Promise<FileHandle | undefined> {
+  try {
+    return await open(reclaimPath, "wx", 0o600);
+  } catch (error) {
+    if (!hasErrorCode(error, "EEXIST")) throw error;
+    await removeStaleJsonStoreReclaimGuard(reclaimPath);
+    try {
+      return await open(reclaimPath, "wx", 0o600);
+    } catch (retryError) {
+      if (hasErrorCode(retryError, "EEXIST")) return undefined;
+      throw retryError;
+    }
+  }
+}
+
+function waitForJsonStoreLock(): Promise<void> {
+  const { promise, resolve } = Promise.withResolvers<void>();
+  setTimeout(resolve, jsonStoreLockRetryMs());
+  return promise;
+}
+
+function processIsRunning(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return !hasErrorCode(error, "ESRCH");
+  }
+}
+
+type JsonStoreLockState = {
+  mtimeMs: number;
+  size: number;
+  ino: number | undefined;
+  content: string;
+};
+
+async function readJsonStoreLockState(lockPath: string): Promise<JsonStoreLockState> {
+  const lockStat = await lstat(lockPath);
+  if (!lockStat.isFile()) {
+    throw new Error(`JSON store mutation lock is not a regular file: ${lockPath}`);
+  }
+  return {
+    mtimeMs: lockStat.mtimeMs,
+    size: lockStat.size,
+    ino: typeof lockStat.ino === "number" ? lockStat.ino : undefined,
+    content: await readFile(lockPath, "utf8"),
+  };
+}
+
+function sameJsonStoreLockState(left: JsonStoreLockState, right: JsonStoreLockState): boolean {
+  return (
+    left.mtimeMs === right.mtimeMs &&
+    left.size === right.size &&
+    left.ino === right.ino &&
+    left.content === right.content
+  );
+}
+
+async function removeJsonStoreMutationLockIfOwned(lockPath: string, token: string): Promise<void> {
+  try {
+    const state = await readJsonStoreLockState(lockPath);
+    const metadata = JSON.parse(state.content) as { token?: unknown };
+    if (metadata.token === token) await unlink(lockPath);
+  } catch (error) {
+    if (!hasErrorCode(error, "ENOENT") && !(error instanceof SyntaxError)) throw error;
+  }
+}
+
+async function removeStaleJsonStoreLock(lockPath: string): Promise<void> {
+  const reclaimPath = `${lockPath}.reclaim`;
+  const reclaimHandle = await acquireJsonStoreReclaimGuard(reclaimPath);
+  if (!reclaimHandle) return;
+  try {
+    await reclaimHandle.writeFile(JSON.stringify({ pid: process.pid }), "utf8");
+    await jsonStoreMutationLockTestHooks?.afterReclaimGuardWrite?.(reclaimPath, String(process.pid));
+
+    let initial: JsonStoreLockState;
+    try {
+      initial = await readJsonStoreLockState(lockPath);
+    } catch (error) {
+      if (hasErrorCode(error, "ENOENT")) return;
+      throw error;
+    }
+    if (Date.now() - initial.mtimeMs <= jsonStoreLockStaleMs()) return;
+    let ownerPid: number | undefined;
+    try {
+      const metadata = JSON.parse(initial.content) as { pid?: unknown };
+      ownerPid = typeof metadata.pid === "number" && Number.isInteger(metadata.pid) ? metadata.pid : undefined;
+    } catch {
+      ownerPid = undefined;
+    }
+    if (ownerPid !== undefined && processIsRunning(ownerPid)) return;
+
+    let confirmed: JsonStoreLockState;
+    try {
+      confirmed = await readJsonStoreLockState(lockPath);
+    } catch (error) {
+      if (hasErrorCode(error, "ENOENT")) return;
+      throw error;
+    }
+    if (!sameJsonStoreLockState(initial, confirmed)) return;
+    await unlink(lockPath).catch((error) => {
+      if (!hasErrorCode(error, "ENOENT")) throw error;
+    });
+  } finally {
+    await closeAndRemoveJsonStoreReclaimGuard(reclaimPath, reclaimHandle);
+  }
+}
+
+async function confirmJsonStoreMutationLockOwnership(
+  lockPath: string,
+  token: string,
+  deadline: number
+): Promise<boolean> {
+  const reclaimPath = `${lockPath}.reclaim`;
+  while (true) {
+    const reclaimHandle = await acquireJsonStoreReclaimGuard(reclaimPath);
+    if (!reclaimHandle) {
+      if (Date.now() >= deadline) return false;
+      await waitForJsonStoreLock();
+      continue;
+    }
+    try {
+      try {
+        const state = await readJsonStoreLockState(lockPath);
+        const metadata = JSON.parse(state.content) as { token?: unknown };
+        return metadata.token === token;
+      } catch (error) {
+        if (hasErrorCode(error, "ENOENT") || error instanceof SyntaxError) return false;
+        throw error;
+      }
+    } finally {
+      await closeAndRemoveJsonStoreReclaimGuard(reclaimPath, reclaimHandle);
+    }
+  }
+}
+
+async function acquireJsonStoreMutationFileLock(key: string): Promise<() => Promise<void>> {
+  const lockPath = `${path.resolve(key)}.mutation.lock`;
+  await mkdir(path.dirname(lockPath), { recursive: true });
+  const token = randomUUID();
+  const deadline = Date.now() + jsonStoreLockTimeoutMs();
+  while (true) {
+    let handle: FileHandle;
+    try {
+      handle = await open(lockPath, "wx", 0o600);
+    } catch (error) {
+      if (!hasErrorCode(error, "EEXIST")) throw error;
+      await removeStaleJsonStoreLock(lockPath).catch((staleError) => {
+        if (!hasErrorCode(staleError, "ENOENT")) throw staleError;
+      });
+      if (Date.now() >= deadline) {
+        throw new Error(`Timed out acquiring JSON store mutation lock: ${lockPath}`);
+      }
+      await waitForJsonStoreLock();
+      continue;
+    }
+    try {
+      await jsonStoreMutationLockTestHooks?.beforeLockOwnerWrite?.(lockPath, token);
+      await handle.writeFile(JSON.stringify({ pid: process.pid, token }), "utf8");
+    } finally {
+      await handle.close().catch(() => undefined);
+    }
+    await jsonStoreMutationLockTestHooks?.beforeLockOwnershipConfirm?.(lockPath, token);
+
+    if (!(await confirmJsonStoreMutationLockOwnership(lockPath, token, deadline))) {
+      await removeJsonStoreMutationLockIfOwned(lockPath, token);
+      if (Date.now() >= deadline) {
+        throw new Error(`Timed out acquiring JSON store mutation lock: ${lockPath}`);
+      }
+      await waitForJsonStoreLock();
+      continue;
+    }
+
+    const heartbeat = setInterval(
+      () => {
+        void readFile(lockPath, "utf8")
+          .then((content) => {
+            const metadata = JSON.parse(content) as { token?: unknown };
+            if (metadata.token !== token) return;
+            const now = new Date();
+            return utimes(lockPath, now, now);
+          })
+          .catch(() => undefined);
+      },
+      Math.max(1, Math.floor(jsonStoreLockStaleMs() / 3))
+    );
+    heartbeat.unref();
+    return async () => {
+      clearInterval(heartbeat);
+      try {
+        const metadata = JSON.parse(await readFile(lockPath, "utf8")) as { token?: unknown };
+        if (metadata.token === token) await unlink(lockPath);
+      } catch (error) {
+        if (!hasErrorCode(error, "ENOENT")) throw error;
+      }
+    };
+  }
+}
+
+export async function withJsonStoreMutationLock<T>(key: string, operation: () => Promise<T>): Promise<T> {
+  const normalizedKey = path.resolve(key);
+  const previous = jsonStoreMutationTails.get(normalizedKey) ?? Promise.resolve();
+  const { promise: current, resolve: release } = Promise.withResolvers<void>();
+  const tail = previous.catch(() => undefined).then(() => current);
+  jsonStoreMutationTails.set(normalizedKey, tail);
+  await previous.catch(() => undefined);
+  let releaseFileLock: (() => Promise<void>) | undefined;
+  try {
+    releaseFileLock = await acquireJsonStoreMutationFileLock(normalizedKey);
+    return await operation();
+  } finally {
+    try {
+      await releaseFileLock?.();
+    } finally {
+      release();
+      if (jsonStoreMutationTails.get(normalizedKey) === tail) {
+        jsonStoreMutationTails.delete(normalizedKey);
+      }
+    }
+  }
+}
+
 export async function readJsonFile(filePath: string): Promise<unknown> {
   return JSON.parse(await readFile(filePath, "utf8")) as unknown;
 }
+
+export async function writeJsonFileAtomic(filePath: string, value: unknown): Promise<void> {
+  await mkdir(path.dirname(filePath), { recursive: true });
+  const tempPath = `${filePath}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
+  try {
+    await writeFile(tempPath, JSON.stringify(value, null, 2), "utf8");
+    await rename(tempPath, filePath);
+  } catch (error) {
+    await unlink(tempPath).catch(() => undefined);
+    throw error;
+  }
+}