@remnic/core 9.48.1 → 9.49.1

package/src/storage.ts

@@ -16,6 +16,8 @@ import {
 import { appendFileSync, createReadStream, mkdirSync, readFileSync, statSync, type Dirent } from "node:fs";
 import { createHash, randomUUID } from "node:crypto";
 import { normalizeContent, computeContentHash } from "./content-hash.js";
+import { raceAbort } from "./abort-error.js";
+import { checkCorpusReadAbort, type CorpusReadOptions } from "./corpus-read-cancellation.js";
 import { selectArtifactMatches, type ArtifactSearchOptions } from "./artifact-search.js";
 import path from "node:path";
 import { log } from "./logger.js";
@@ -71,8 +73,8 @@ import {
   updateCacheOnWrite,
 } from "./memory-cache.js";
 import {
-  getInFlightRead,
-  setInFlightRead,
+  attachInFlightReader,
+  beginCoalescedScan,
   deleteInFlightRead,
   deleteInFlightReadsForDir,
   clearInFlightReads,
@@ -363,6 +365,15 @@ function serializeFrontmatter(fm: MemoryFrontmatter): string {
   if (fm.tombstoneBlockTier) lines.push(`tombstoneBlockTier: ${fm.tombstoneBlockTier}`);
   if (fm.supersededBy) lines.push(`supersededBy: ${fm.supersededBy}`);
   if (fm.supersededAt) lines.push(`supersededAt: ${fm.supersededAt}`);
+  if (fm.supersessionCause) {
+    if (fm.supersessionCause !== "direct" && fm.supersessionCause !== "dependency") {
+      throw new Error(
+        `serializeFrontmatter: invalid supersessionCause ${JSON.stringify(fm.supersessionCause)} — expected "direct" | "dependency"`,
+      );
+    }
+    lines.push(`supersessionCause: ${fm.supersessionCause}`);
+  }
+  if (fm.invalidatedBy) lines.push(`invalidatedBy: ${fm.invalidatedBy}`);
   if (fm.archivedAt) lines.push(`archivedAt: ${fm.archivedAt}`);
   // Issue #680 — explicit fact lifecycle.  Emit only when present so legacy
   // memories round-trip unchanged; readers default `valid_at` to `created`.
@@ -891,6 +902,11 @@ export function parseFrontmatter(raw: string): { frontmatter: MemoryFrontmatter;
       supersededBy: fm.supersededBy || undefined,
       supersededAt: fm.supersededAt || undefined,
       archivedAt: fm.archivedAt || undefined,
+      supersessionCause:
+        fm.supersessionCause === "direct" || fm.supersessionCause === "dependency"
+          ? fm.supersessionCause
+          : undefined,
+      invalidatedBy: fm.invalidatedBy || undefined,
       // Issue #680 — explicit fact lifecycle round-trip.
       valid_at: fm.validAt || undefined,
       invalid_at: fm.invalidAt || undefined,
@@ -4347,8 +4363,8 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return id;
   }
 
-  private async readAllArtifactsCached(): Promise<MemoryFile[]> {
-    return this.memoryReadStore.readAllArtifactsCached();
+  private async readAllArtifactsCached(options?: CorpusReadOptions): Promise<MemoryFile[]> {
+    return this.memoryReadStore.readAllArtifactsCached(options);
   }
 
   async searchArtifacts(
@@ -4357,7 +4373,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     options: ArtifactSearchOptions = {},
   ): Promise<MemoryFile[]> {
     return selectArtifactMatches(
-      () => this.readAllArtifactsCached(),
+      () => this.readAllArtifactsCached({ abortSignal: options.abortSignal }),
       query,
       maxResults,
       options,
@@ -4474,7 +4490,8 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return memories;
   }
 
-  async readAllMemories(): Promise<MemoryFile[]> {
+  async readAllMemories(options?: CorpusReadOptions): Promise<MemoryFile[]> {
+    checkCorpusReadAbort(options);
     if (this.hotMemoriesCacheEnabled) {
       const cached = getCachedMemories(
         this.baseDir,
@@ -4492,25 +4509,51 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     // The in-flight slot get/set/delete also key on this snapshot so a mid-scan
     // key change can't orphan the slot registered under the old identity.
     const keyId = this.hotCacheKeyId();
-    const inFlight = getInFlightRead(this.baseDir, keyId);
-    if (inFlight) return this.rememberMemorySnapshots(await inFlight);
+    const inFlight = attachInFlightReader(this.baseDir, keyId, options?.abortSignal);
+    if (inFlight) {
+      // A joiner never cancels the shared scan on its own, but it does stop waiting
+      // on it, and its departure counts: if it was the last waiter the scan is
+      // cancelled (issue #2307 review).
+      try {
+        return this.rememberMemorySnapshots(
+          await raceAbort(inFlight.read, options?.abortSignal, "corpus read aborted"),
+        );
+      } finally {
+        inFlight.waiter.leave();
+      }
+    }
 
+    // Sole-waiter cancellation lives with the registry that decides whether this
+    // scan is still cancellable at all — see `beginCoalescedScan` (issue #2307).
+    const scan = beginCoalescedScan(this.baseDir, keyId, options?.abortSignal);
     const readPromise = (async (): Promise<MemoryFile[]> => {
       const version = this.getMemoryCorpusVersion();
-      const memories = await this._readAllMemoriesFromDisk();
+      const memories = await this._readAllMemoriesFromDisk({ abortSignal: scan.scanSignal });
       // Publish only if neither the corpus version NOR the key identity changed
       // during the scan. The version guard prevents clobbering a newer patched +
       // re-keyed entry; the keyId guard prevents publishing this key's decrypted
-      // corpus under a since-changed identity (Codex Medium, #1902).
+      // corpus under a since-changed identity (Codex Medium, #1902). A cancelled
+      // scan never arrives here — the checkpoints throw — so a partial corpus is
+      // never published.
       if (this.hotMemoriesCacheEnabled && this.getMemoryCorpusVersion() === version && this.hotCacheKeyId() === keyId) {
         setCachedMemories(this.baseDir, memories, version, keyId, this.hotCacheTtlMs());
       }
       return memories;
     })();
-    setInFlightRead(this.baseDir, keyId, readPromise);
+    scan.arm(readPromise);
     try {
-      return this.rememberMemorySnapshots(await readPromise);
+      // The starter races its own signal too. Refcounting means a sole waiter's
+      // abort rejects the scan itself, but while another reader keeps the scan
+      // alive the starter would otherwise sit here and be handed the full corpus
+      // after cancelling — the joiner path's contract, applied symmetrically
+      // (issue #2307 review). The scan is NOT aborted here: it belongs to the
+      // readers still waiting, and a completed scan is still worth caching for
+      // them.
+      return this.rememberMemorySnapshots(
+        await raceAbort(readPromise, options?.abortSignal, "corpus read aborted"),
+      );
     } finally {
+      scan.leave();
       deleteInFlightRead(this.baseDir, keyId, readPromise);
     }
   }
@@ -4769,7 +4812,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
    * active memory file paths without parsing frontmatter — safe on a 100k+
    * corpus, unlike readAllMemories().
    */
-  async collectActiveMemoryPaths(options?: { propagateReadErrors?: boolean }): Promise<string[]> {
+  async collectActiveMemoryPaths(
+    options?: { propagateReadErrors?: boolean } & CorpusReadOptions,
+  ): Promise<string[]> {
     return this.memoryReadStore.collectActiveMemoryPaths(options);
   }
 
@@ -4793,12 +4838,19 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return `${this.getMemoryCorpusVersion()}:${this.readColdWriteVersion()}`;
   }
 
-  private async readParsedMemoriesFromPaths(filePaths: string[], batchSize?: number): Promise<MemoryFile[]> {
+  private async readParsedMemoriesFromPaths(
+    filePaths: string[],
+    batchSize?: number,
+    options?: CorpusReadOptions,
+  ): Promise<MemoryFile[]> {
     if (filePaths.length === 0) return [];
 
     const normalizedBatchSize = this.normalizeMemoryReadBatchSize(batchSize);
     const memories: MemoryFile[] = [];
     for (let i = 0; i < filePaths.length; i += normalizedBatchSize) {
+      // Per batch: the parse phase is as long as the walk on a large corpus, and
+      // an abandoned caller must not keep paying for it (issue #2307).
+      checkCorpusReadAbort(options);
       const batch = filePaths.slice(i, i + normalizedBatchSize);
       const results = await Promise.all(
         batch.map(async (fullPath) => {
@@ -4831,6 +4883,10 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
         if (memory !== null) memories.push(memory);
       }
     }
+    // Once more after the loop: a signal that fired during the LAST batch's await
+    // would otherwise let this return successfully, publishing to the cache and
+    // handing results to a caller that had already given up (issue #2307 review).
+    checkCorpusReadAbort(options);
     return memories;
   }
 
@@ -4939,9 +4995,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return this.memoryReadStore.readMemoriesWindow(options);
   }
 
-  private async _readAllMemoriesFromDisk(): Promise<MemoryFile[]> {
-    const filePaths = await this.collectActiveMemoryPaths();
-    return this.readParsedMemoriesFromPaths(filePaths, 50);
+  private async _readAllMemoriesFromDisk(options?: CorpusReadOptions): Promise<MemoryFile[]> {
+    const filePaths = await this.collectActiveMemoryPaths(options);
+    return this.readParsedMemoriesFromPaths(filePaths, 50, options);
   }
 
   async readAllColdMemories(): Promise<MemoryFile[]> {
@@ -6430,8 +6486,8 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   // Scoring + Knowledge Index (Knowledge Graph v7.0)
   // ---------------------------------------------------------------------------
 
-  async readAllEntityFiles(): Promise<EntityFile[]> {
-    return this.entityStore.readAllEntityFiles();
+  async readAllEntityFiles(options?: CorpusReadOptions): Promise<EntityFile[]> {
+    return this.entityStore.readAllEntityFiles(options);
   }
 
   /**

package/src/temporal-supersession.ts

@@ -12,10 +12,12 @@
  * cached `readAllMemories()` path so cost is amortized with the rest of the
  * write pipeline.
  */
-import type { MemoryFile, MemoryFrontmatter } from "./types.js";
+import type { MemoryFile, MemoryFrontmatter, PluginConfig } from "./types.js";
+import type { ExtractionEngine } from "./extraction.js";
 import type { StorageManager } from "./storage.js";
 import { log } from "./logger.js";
 import { effectiveValidAt } from "./temporal-validity.js";
+import { propagateInvalidation } from "./orchestration/dependency-propagation.js";
 
 /**
  * Shared normalization for supersession key components.
@@ -244,6 +246,10 @@ export async function applyTemporalSupersession(args: {
    * stale relative to the incoming promotion event (PR #402 Finding Uyui).
    */
   useCallerTimestamp?: boolean;
+  /** Optional orchestration dependencies for one-hop propagation. */
+  extraction?: ExtractionEngine;
+  config?: PluginConfig;
+  namespaceScope?: string;
 }): Promise<TemporalSupersessionResult> {
   const empty: TemporalSupersessionResult = { supersededIds: [], matchedKeys: [] };
   if (!args.enabled) return empty;
@@ -509,6 +515,28 @@ export async function applyTemporalSupersession(args: {
             );
           }
         }
+        if (args.extraction && args.config && args.namespaceScope) {
+          try {
+            await propagateInvalidation(
+              {
+                storage: args.storage,
+                extraction: args.extraction,
+                config: args.config,
+              },
+              {
+                oldMemory: fresh,
+                replacementId: args.newMemoryId,
+                replacementContent: newMemoryFile?.content ?? null,
+                cause: "temporal_supersession",
+                namespaceScope: args.namespaceScope,
+              },
+            );
+          } catch (propagationErr) {
+            log.warn(
+              `temporal-supersession: dependency propagation failed for ${fresh.frontmatter.id}: ${propagationErr}`,
+            );
+          }
+        }
       }
     } catch (err) {
       log.warn(

package/src/testing/orchestrator-lite.ts

@@ -14,6 +14,7 @@ import os from "node:os";
 import path from "node:path";
 
 import { parseConfig } from "../config.js";
+import { getAbstractionNodeStoreStatus, type AbstractionNodeStoreStatus } from "../abstraction-nodes.js";
 import type { Orchestrator } from "../orchestrator.js";
 import type { ExtractionRunCoordinatorDeps } from "../orchestration/extraction-run.js";
 import type { BufferTurn, ExtractionResult, PluginConfig } from "../types.js";
@@ -25,10 +26,7 @@ import type { BufferTurn, ExtractionResult, PluginConfig } from "../types.js";
  * init gate floored so a recall() that never ran initialize() does not block.
  * Mirrors tests/orchestrator-characterization.test.ts:makeConfig.
  */
-export function makeLifecycleConfig(
-  memoryDir: string,
-  overrides: Record<string, unknown> = {},
-): PluginConfig {
+export function makeLifecycleConfig(memoryDir: string, overrides: Record<string, unknown> = {}): PluginConfig {
   return parseConfig({
     openaiApiKey: "sk-test",
     memoryDir,
@@ -47,6 +45,27 @@ export function makeLifecycleConfig(
   });
 }
 
+/** Enable the production harmonic-construction persistence path for a lifecycle row. */
+export function makeHarmonicLifecycleConfig(memoryDir: string, overrides: Record<string, unknown> = {}): PluginConfig {
+  return makeLifecycleConfig(memoryDir, {
+    harmonicRetrievalEnabled: true,
+    ...overrides,
+  });
+}
+
+/** Read and validate abstraction nodes written by the production store path. */
+export function harmonicNodeStoreStatus(
+  memoryDir: string,
+  abstractionNodeStoreDir?: string
+): Promise<AbstractionNodeStoreStatus> {
+  return getAbstractionNodeStoreStatus({
+    memoryDir,
+    abstractionNodeStoreDir,
+    enabled: true,
+    anchorsEnabled: false,
+  });
+}
+
 /** Create a fresh temp memory dir with a stable `remnic-lifecycle-<label>-` prefix. */
 export function mkTempMemoryDir(label: string): Promise<string> {
   return mkdtemp(path.join(os.tmpdir(), `remnic-lifecycle-${label}-`));
@@ -95,7 +114,7 @@ interface ExtractionClientSeam {
  */
 export function stubExtraction(
   orchestrator: Orchestrator,
-  factory: (turns: BufferTurn[], call: number) => ExtractionResult | Promise<ExtractionResult>,
+  factory: (turns: BufferTurn[], call: number) => ExtractionResult | Promise<ExtractionResult>
 ): BufferTurn[][] {
   const calls: BufferTurn[][] = [];
   // The `extraction` field is private and structurally unexpressible from
@@ -143,7 +162,7 @@ interface PersistExtractionSeam {
  */
 export function stubPersistExtraction(
   orchestrator: Orchestrator,
-  factory?: (args: PersistExtractionArgs, call: number) => string[] | Promise<string[]>,
+  factory?: (args: PersistExtractionArgs, call: number) => string[] | Promise<string[]>
 ): PersistExtractionArgs[] {
   const calls: PersistExtractionArgs[] = [];
   // `persistExtraction` is replaced in place; the named cast keeps the
@@ -191,7 +210,7 @@ export async function seedFactFile(memoryDir: string, id: string, content: strin
       content,
       "",
     ].join("\n"),
-    "utf-8",
+    "utf-8"
   );
   return file;
 }
@@ -222,7 +241,7 @@ export async function memoryFilesContaining(root: string, needle: string): Promi
 export async function eventually<T>(
   probe: () => Promise<T | null | undefined | false>,
   label: string,
-  timeoutMs = 5000,
+  timeoutMs = 5000
 ): Promise<T> {
   const startedAt = Date.now();
   for (;;) {