@remnic/core 10.0.51 → 10.0.53

package/src/orchestration/extraction-persist.ts

@@ -68,10 +68,7 @@ import {
 } from "../source-attribution.js";
 import { classifyMemoryKind } from "../himem.js";
 import { shouldPromoteGlobalFactToShared, withholdToolScopedFromSharedNamespace } from "../tool-scoped-memory.js";
-import {
-  buildBehaviorSignalsForMemory,
-  dedupeBehaviorSignalsByMemoryAndHash,
-} from "../behavior-signals.js";
+import { buildBehaviorSignalsForMemory } from "../behavior-signals.js";
 import { buildProcedurePersistBody } from "../procedural/procedure-types.js";
 import { stripAttributesSuffix } from "../structured-attributes.js";
 import { resolveWriteSubject } from "../memory-subject.js";
@@ -103,7 +100,7 @@ import type {
 } from "../types.js";
 import {
   createBatchPromotedCopyProbe,
-  flushDeferredFactHashOnFailure,
+  finalizeExtractionCommittedWrites,
   makeSubjectGuardAllows,
   profileAutoPromotionAllows,
   readActiveMemoriesBothTiers,
@@ -120,7 +117,7 @@ import {
   resolvePersistedMemoryRelativePath,
 } from "../orchestrator.js";
 import type { HarmonicConstructionInput } from "../harmonic-construction.js";
-import { enqueueMergedTargetForHarmonicConstruction, persistConstructedHarmonicRecords } from "./harmonic-construction-persist.js";
+import { enqueueMergedTargetForHarmonicConstruction } from "./harmonic-construction-persist.js";
 import { applySemanticMergeAtPersist, runMergedTargetPostEffects, writeMergedVerbatimArtifact } from "./semantic-merge-persist.js";
 import { buildMergedTargetPromotionPayload } from "./semantic-merge-promotion-payload.js";
 import { persistMergedTargetThreadEpisode } from "./semantic-merge-commit-effects.js";
@@ -291,6 +288,7 @@ export class ExtractionPersistCoordinator {
         events: [...events],
       });
     };
+
     const sharedProfileLayer = scopeProfileWritePlan?.layers.find(
       (layer) =>
         layer.id === "serverShared" &&
@@ -1106,6 +1104,32 @@ export class ExtractionPersistCoordinator {
       0,
       this.deps.config.extractionMaxProfileUpdatesPerRun,
     );
+    const finalizeCommittedWrites = async (scheduleQmd: boolean): Promise<void> => {
+      await finalizeExtractionCommittedWrites({
+        saveContentHashIndexes: () => this.deps.saveContentHashIndexes(),
+        behaviorSignalsByStorage: behaviorSignalsByStorage.values(),
+        harmonic: {
+          enabled: harmonicConstructionEnabled,
+          entries: harmonicFactsByStorage.values(),
+          baseStorageDir: storage.dir,
+          abstractionNodeStoreDir:
+            !scopeProfileWritePlan && harmonicBaseNamespace === this.deps.config.defaultNamespace
+              ? this.deps.config.abstractionNodeStoreDir
+              : undefined,
+          sessionKey: sourceContext?.sessionKey,
+          validAt: sourceContext?.validAt,
+          episodeTitle: result.episodeTitle,
+          anchorsEnabled: harmonicAnchorsEnabled,
+          entityMentions: entities,
+        },
+        updateTemporalTagIndexes: (target, ids) =>
+          this.deps.updateTemporalTagIndexes(target, ids),
+        fallbackStorage: storage,
+        persistedIdsByStorage,
+        requestQmdMaintenance: () => this.deps.requestQmdMaintenance(),
+        scheduleQmd,
+      });
+    };
 
     if (
       facts.length < result.facts.length ||
@@ -2583,6 +2607,25 @@ export class ExtractionPersistCoordinator {
       // so a blocked fact creates no active shared copy / supersession / graph entry.
       const tombstoneBlocked = factWrite.tombstoneBlocked;
       const postWriteGuard = faithfulnessEnforceStatus === "pending_review" || tombstoneBlocked;
+      // #3231: the fact is durable now. Track it before any post-write step
+      // that can throw, so the batch-end finalization on a failure path still
+      // covers it (a retry exact-dedupes the fact and would not re-queue it).
+      trackBehaviorSignals(
+        targetStorage,
+        buildBehaviorSignalsForMemory({
+          memoryId,
+          category: writeCategory,
+          content: fact.content,
+          namespace: this.deps.storageDirNamespace(targetStorage.dir),
+          confidence: fact.confidence,
+          source: "extraction",
+        }),
+      );
+      trackPersistedId(targetStorage, memoryId, {
+        pendingReview: postWriteGuard,
+        category: writeCategory,
+        harmonicFact,
+      });
       // #1645: defer contradiction auto-resolve until tombstone status is
       // known (see applyDeferredContradictionResolve).
       try {
@@ -2596,7 +2639,9 @@ export class ExtractionPersistCoordinator {
           await anchorSnapshots.remove(targetStorage, contradiction!.supersededId);
         }
       } catch (err) {
-        await flushDeferredFactHashOnFailure(() => this.deps.saveContentHashIndexes(), factDedupEnabled);
+        await finalizeCommittedWrites(true).catch((finalizeErr) =>
+          log.warn(`committed-write finalization after post-write failure failed: ${finalizeErr}`),
+        );
         throw err;
       }
       if (routedRuleId) {
@@ -2632,22 +2677,6 @@ export class ExtractionPersistCoordinator {
         }
       }
       try {
-        trackBehaviorSignals(
-          targetStorage,
-          buildBehaviorSignalsForMemory({
-            memoryId,
-            category: writeCategory,
-            content: fact.content,
-            namespace: this.deps.storageDirNamespace(targetStorage.dir),
-            confidence: fact.confidence,
-            source: "extraction",
-          }),
-        );
-        trackPersistedId(targetStorage, memoryId, {
-          pendingReview: postWriteGuard,
-          category: writeCategory,
-          harmonicFact,
-        });
         await anchorSnapshots.replace(targetStorage, memoryId, writeCategory, memoryPathById);
         if (
           !postWriteGuard &&
@@ -2782,9 +2811,12 @@ export class ExtractionPersistCoordinator {
           );
         }
       } catch (err) {
-        // PR #2016: flush the deferred fact-hash before propagating so a durable
-        // .md never outlives a missing shared fact-hash index entry.
-        await flushDeferredFactHashOnFailure(() => this.deps.saveContentHashIndexes(), factDedupEnabled);
+        // PR #2016 / #3231: finalize deferred side effects for facts already
+        // durable before propagating so hash/behavior/harmonic/temporal/QMD
+        // still land on a partial batch.
+        await finalizeCommittedWrites(true).catch((finalizeErr) =>
+          log.warn(`committed-write finalization after post-write failure failed: ${finalizeErr}`),
+        );
         throw err;
       }
     }
@@ -2807,7 +2839,6 @@ export class ExtractionPersistCoordinator {
       durableNonFactTouchRecorded = true;
       touchBaseNonFactNamespace();
     };
-    // #1955 review: entity fields are screened before entering the index (screenEntityForIndex).
     const entityScreenOn = resolveSecurityCapabilities(this.deps.config).injectionScreen;
     const entityScreenProfile = resolveSecurityCapabilities(this.deps.config).injectionScreenProfile;
     for (const entity of entities) {
@@ -2829,7 +2860,10 @@ export class ExtractionPersistCoordinator {
           recordDurableNonFactWrite();
         }
       } catch (err) {
-        log.warn(`persistExtraction: entity write failed: ${err}`);
+        await finalizeCommittedWrites(true).catch((finalizeErr) =>
+          log.warn(`committed-write finalization after entity failure failed: ${finalizeErr}`),
+        );
+        throw err;
       }
     }
     // Persist entity relationships (v7.0). #1955 review: source/label/target are screened.
@@ -2929,28 +2963,9 @@ export class ExtractionPersistCoordinator {
       touchBaseNonFactNamespace();
     }
 
-    // Save any content-hash indexes touched during the batch via the removal-
-    // aware, lock-held reconciling save (issue #1909). Round 11: the fact-hash
-    // index is rebuilt from the corpus on every restart (no ready marker), so
-    // this on-disk write is the in-process persisted view — never a cross-restart
-    // trust anchor. A deferred write whose addContentHashDedup threw is still
-    // safe: the fact .md is durable and the next restart's rebuild includes it.
-    await this.deps.saveContentHashIndexes().catch((err) => {
-      log.warn(`content-hash index save failed: ${err}`);
-    });
-
-    for (const {
-      storage: targetStorage,
-      events,
-    } of behaviorSignalsByStorage.values()) {
-      const dedupedSignals = dedupeBehaviorSignalsByMemoryAndHash(events);
-      if (dedupedSignals.length === 0) continue;
-      await targetStorage
-        .appendBehaviorSignals(dedupedSignals)
-        .catch((err) =>
-          log.warn(`appendBehaviorSignals failed (non-fatal): ${err}`),
-        );
-    }
+    // Shared batch-end finalization (hash, behavior, harmonic, temporal).
+    // Failure paths call the same helper with scheduleQmd=true (#3231).
+    await finalizeCommittedWrites(false);
 
     const dedupSuffix = `${dedupedCount > 0 ? ` (${dedupedCount} deduped)` : ""}${semanticMergedCount > 0 ? ` (${semanticMergedCount} merged)` : ""}`;
     const gatedSuffix =
@@ -2962,33 +2977,7 @@ export class ExtractionPersistCoordinator {
     log.info(
       `persisted: ${facts.length - dedupedCount - importanceGatedCount - judgeGatedCount - redactionGatedCount - semanticMergedCount} facts${dedupSuffix}${gatedSuffix}${judgeSuffix}${redactionSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`,
     );
-    if (harmonicConstructionEnabled && harmonicFactsByStorage.size > 0) {
-      await persistConstructedHarmonicRecords({
-        entries: harmonicFactsByStorage.values(),
-        baseStorageDir: storage.dir,
-        abstractionNodeStoreDir:
-          !scopeProfileWritePlan && harmonicBaseNamespace === this.deps.config.defaultNamespace
-            ? this.deps.config.abstractionNodeStoreDir
-            : undefined,
-        sessionKey: sourceContext?.sessionKey,
-        validAt: sourceContext?.validAt,
-        episodeTitle: result.episodeTitle,
-        anchorsEnabled: harmonicAnchorsEnabled,
-        entityMentions: entities,
-      });
-    }
-    // Update temporal + tag indexes (v8.1) — fire-and-forget, fail-open
-    void (async () => {
-      if (persistedIdsByStorage.size === 0) {
-        await this.deps.updateTemporalTagIndexes(storage, []);
-        return;
-      }
-      for (const entry of persistedIdsByStorage.values()) {
-        await this.deps.updateTemporalTagIndexes(entry.storage, entry.ids);
-      }
-    })().catch((err) =>
-      log.debug(`temporal-index update error (non-fatal): ${err}`),
-    );
+
     // #1635: surface pending_review ids so the thread episode set excludes them.
     this.deps.setLastPersistExtractionPendingReviewIds(pendingReviewPersistedIds);
     // Return the persisted fact IDs for threading

package/src/orchestration/lifecycle-pending-fallback-drain.ts

@@ -0,0 +1,97 @@
+/**
+ * Fallback lifecycle pending-append drain for targets with no secure
+ * StorageManager. Moved verbatim out of maintenance.ts (issue #2033) so that
+ * file stays under the structural size ceiling; behavior is unchanged.
+ */
+
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+import { log } from "../logger.js";
+import {
+  drainPendingLifecycleLedgerIfAny,
+  pendingLifecycleLedgerDir,
+  type LifecyclePendingIo,
+} from "../storage/memory-lifecycle-ledger-access.js";
+import { probeEncryptedRegularFileHeader } from "../secure-store/secure-fs.js";
+import { listContainedSpillFiles } from "../utils/path-containment.js";
+
+/**
+ * Drain a filesystem-fallback target's pending lifecycle-append spill when no
+ * secure StorageManager is available (catalog-disabled namespace; issue
+ * #2033). Uses the safe path-scoped PLAINTEXT context this fallback already
+ * operates in: a keyless drain can only fold plaintext spills into a plaintext
+ * ledger, so it refuses when the ledger or ANY spill is encrypted at rest,
+ * leaving those pending for a keyed (catalog-path) drain — exactly mirroring
+ * the fallback compaction's encrypted-ledger deferral. Non-fatal; a fast
+ * no-op when nothing is pending. The shared spill lister enforces the symlink/
+ * containment guard on every spill file it reads or deletes.
+ */
+export async function drainPendingLifecycleAppendsForPath(memoryDir: string): Promise<void> {
+  const ledgerPath = path.join(memoryDir, "state", "memory-lifecycle-ledger.jsonl");
+  const spillDir = pendingLifecycleLedgerDir(ledgerPath);
+  // Enumerate spills through the SAME guarded lister the actual drain uses
+  // (listContainedSpillFiles): it rejects a symlinked/non-directory spill dir
+  // and skips any symlink/FIFO/device/escaping entry, returning only regular
+  // files contained in the dir. Never raw-readdir + open() an untrusted name:
+  // opening a FIFO would block until a writer appears and a symlink would
+  // redirect the probe outside the dir (#2033). Enumerate BOTH live `*.jsonl`
+  // spills and crash-orphaned `*.jsonl.claimed` files: a drain that died
+  // between claiming a spill (rename to `.claimed`) and committing it leaves
+  // the ONLY copy of those rows as a claimed orphan. Gating solely on live
+  // spills would early-return here whenever the pending dir holds nothing but
+  // orphans, stranding those rows forever; proceeding lets
+  // drainPendingLifecycleLedgerIfAny -> recoverOrphanedClaims restore and
+  // re-commit them under the ledger lock (#2033).
+  let spillFiles: string[];
+  let claimedOrphans: string[];
+  try {
+    spillFiles = await listContainedSpillFiles(spillDir);
+    claimedOrphans = await listContainedSpillFiles(spillDir, ".jsonl.claimed");
+  } catch (err) {
+    log.debug(`fallback lifecycle pending scan failed (non-fatal) for ${memoryDir}: ${err}`);
+    return;
+  }
+  if (spillFiles.length === 0 && claimedOrphans.length === 0) return;
+  // The pending encryption state is authoritative only UNDER the ledger lock:
+  // drainPendingLifecycleLedgerIfAny waits for the lock and re-enumerates the
+  // pending dir, so a secure-store writer that spills an encrypted file — or a
+  // keyed compaction that rewrites the ledger encrypted — while this fallback
+  // waits would defeat any pre-lock probe (#2033). Probe each file at the
+  // moment of use inside the held lock instead: readSecure refuses an
+  // encrypted spill (the fold reads each spill lazily and propagates the throw
+  // BEFORE claiming that spill, leaving it intact) and the append refuses a
+  // ledger that became encrypted (the fold rolls the claims back). Either way
+  // the plaintext IO
+  // never reads or writes ciphertext; the keyed catalog-path drain handles the
+  // deferred rows.
+  const io: LifecyclePendingIo = {
+    writeSecure: async (filePath, payload) => {
+      await mkdir(path.dirname(filePath), { recursive: true });
+      await writeFile(filePath, payload, "utf-8");
+    },
+    readSecure: async (filePath) => {
+      if (await probeEncryptedRegularFileHeader(filePath)) {
+        throw new Error(`fallback lifecycle drain deferred: encrypted spill at ${filePath}`);
+      }
+      return readFile(filePath, "utf-8");
+    },
+  };
+  try {
+    await drainPendingLifecycleLedgerIfAny(
+      ledgerPath,
+      io,
+      async (payload) => {
+        if (await probeEncryptedRegularFileHeader(ledgerPath)) {
+          throw new Error(`fallback lifecycle drain deferred: ledger became encrypted at ${ledgerPath}`);
+        }
+        await appendFile(ledgerPath, payload, "utf-8");
+      },
+      async () => {
+        await mkdir(path.dirname(ledgerPath), { recursive: true });
+      },
+    );
+  } catch (err) {
+    log.debug(`fallback lifecycle pending drain failed (non-fatal) for ${memoryDir}: ${err}`);
+  }
+}