@remnic/core 9.3.745 → 9.3.746

package/src/orchestrator.ts

@@ -107,6 +107,7 @@ import { RecallSectionCoordinator } from "./orchestration/recall-section-coordin
 import { QmdResultResolver, qmdCollectionPathParts, qmdResultPathCandidates } from "./orchestration/qmd-result-resolver.js";
 import { ContradictionLinkingCoordinator } from "./orchestration/contradiction-linking-coordinator.js";
 import { ExtractionRunCoordinator, type ExtractionRunResult } from "./orchestration/extraction-run.js";
+import { ConsolidationRunCoordinator } from "./orchestration/consolidation-run.js";
 import { ExtractionPersistCoordinator } from "./orchestration/extraction-persist.js";
 export { hasIdentityRecoveryIntent, resolveEffectiveIdentityInjectionMode } from "./orchestration/recall-result-formatter.js";
 import {
@@ -1773,6 +1774,43 @@ export class Orchestrator {
     }
     return this._extractionRunCoordinator;
   }
+  /**
+   * Consolidation-run coordinator (issue #1526 seam 17). Owns
+   * the full consolidation maintenance pass (LLM merge/invalidate/update,
+   * entity merge, commitment/TTL cleanup, lifecycle policy, compression
+   * guideline learning, tier migration, fact archival, semantic consolidation,
+   * identity consolidation, profile consolidation, summarization, topic
+   * extraction, TMT rebuild). The orchestrator delegates and injects
+   * its own methods + coordinators.
+   */
+  private _consolidationRunCoordinator: ConsolidationRunCoordinator | undefined;
+
+  private get consolidationRunCoordinator(): ConsolidationRunCoordinator {
+    if (!this._consolidationRunCoordinator) {
+      this._consolidationRunCoordinator = new ConsolidationRunCoordinator({
+        config: this.config,
+        getStorage: () => this.storage,
+        getStorageRouter: () => this.storageRouter,
+        getExtraction: () => this.extraction,
+        embeddingFallback: this.embeddingFallback,
+        tmtBuilder: this.tmtBuilder,
+        consolidationObservers: this.consolidationObservers,
+        getAccessTrackingBuffer: () => this.accessTrackingBuffer,
+        lifecyclePolicyCoordinator: this.lifecyclePolicyCoordinator,
+        compressionGuidelineCoordinator: this.compressionGuidelineCoordinator,
+        semanticConsolidationCoordinator: this.semanticConsolidationCoordinator,
+        entitySynthesisCoordinator: this.entitySynthesisCoordinator,
+        recallSectionCoordinator: this.recallSectionCoordinator,
+        tierMigrationCoordinator: this.tierMigrationCoordinator,
+        flushAccessTracking: () => this.flushAccessTracking(),
+        indexPersistedMemory: (storage, memoryId) => this.indexPersistedMemory(storage, memoryId),
+        autoConsolidateIdentity: () => this.autoConsolidateIdentity(),
+        fastChatCompletion: (messages, options) => this.fastChatCompletion(messages, options),
+      });
+    }
+    return this._consolidationRunCoordinator;
+  }
+
   /**
    * Extraction-persist coordinator (issue #1526 seam 16). Owns the
    * `persistExtraction` pipeline. Lazy: created on first access so
@@ -12489,503 +12527,7 @@ export class Orchestrator {
     merged: number;
     invalidated: number;
   }> {
-    const lifecycleCaps = resolveMemoryLifecycleCapabilities(this.config);
-    log.info("running consolidation pass");
-    let merged = 0;
-    let invalidated = 0;
-    // Tracks whether any consolidation memory-item action (UPDATE / MERGE /
-    // INVALIDATE) durably rewrote memory state. A consolidation pass that only
-    // mutates memory items (no profile/entity updates) still changes the default
-    // namespace's data, so its catalog `lastWriteAt` must refresh too (NIBOi).
-    let memoryItemMutated = false;
-
-    // Flush access tracking buffer first
-    if (this.accessTrackingBuffer.size > 0) {
-      await this.flushAccessTracking();
-    }
-
-    let allMemories = await this.storage.readAllMemories();
-    if (allMemories.length < 5) {
-      return { memoriesProcessed: allMemories.length, merged, invalidated };
-    }
-
-    const recent = allMemories
-      .sort(
-        (a, b) =>
-          new Date(b.frontmatter.created).getTime() -
-          new Date(a.frontmatter.created).getTime(),
-      )
-      .slice(0, 20);
-
-    const older = allMemories.sort(
-      (a, b) =>
-        new Date(a.frontmatter.created).getTime() -
-        new Date(b.frontmatter.created).getTime(),
-    );
-
-    const profile = await this.storage.readProfile();
-    const result = await this.extraction.consolidate(recent, older, profile);
-
-    // Build a lookup map from the already-loaded corpus to avoid repeated
-    // readAllMemories() scans inside getMemoryById for pre-action deindex reads.
-    const memoryLookup = resolveIndexingCapabilities(this.config).queryAwareIndexing
-      ? new Map(allMemories.map((m) => [m.frontmatter.id, m]))
-      : null;
-
-    for (const item of result.items) {
-      switch (item.action) {
-        case "INVALIDATE": {
-          // Capture path/frontmatter before invalidation for index cleanup
-          const toInvalidate = resolveIndexingCapabilities(this.config).queryAwareIndexing
-            ? (memoryLookup?.get(item.existingId) ?? null)
-            : null;
-          if (await this.storage.invalidateMemory(item.existingId)) {
-            invalidated += 1;
-            memoryItemMutated = true;
-            await this.embeddingFallback.removeFromIndex(item.existingId);
-            if (toInvalidate?.path && toInvalidate.frontmatter?.created) {
-              deindexMemory(
-                this.config.memoryDir,
-                toInvalidate.path,
-                toInvalidate.frontmatter.created,
-                toInvalidate.frontmatter.tags ?? [],
-              );
-            }
-          }
-          break;
-        }
-        case "UPDATE":
-          if (item.updatedContent) {
-            await this.storage.updateMemory(
-              item.existingId,
-              item.updatedContent,
-              {
-                lineage: [item.existingId],
-              },
-            );
-            memoryItemMutated = true;
-            await this.indexPersistedMemory(this.storage, item.existingId);
-            // updateMemory() only changes content/updated/lineage — path, created, and tags
-            // are preserved, so the temporal/tag index entry is already correct; no reindex needed.
-          }
-          break;
-        case "MERGE":
-          if (item.updatedContent && item.mergeWith) {
-            await this.storage.updateMemory(
-              item.existingId,
-              item.updatedContent,
-              {
-                supersedes: item.mergeWith,
-                lineage: [item.existingId, item.mergeWith],
-              },
-            );
-            memoryItemMutated = true;
-            await this.indexPersistedMemory(this.storage, item.existingId);
-            // updateMemory() only changes content/updated/supersedes/lineage — path, created, and tags
-            // are preserved, so the temporal/tag index entry for the survivor is already correct.
-            // Capture before invalidation for index cleanup
-            const toMergeInvalidate = resolveIndexingCapabilities(this.config).queryAwareIndexing
-              ? (memoryLookup?.get(item.mergeWith) ?? null)
-              : null;
-            if (await this.storage.invalidateMemory(item.mergeWith)) {
-              invalidated += 1;
-              merged += 1;
-              await this.embeddingFallback.removeFromIndex(item.mergeWith);
-              if (
-                toMergeInvalidate?.path &&
-                toMergeInvalidate.frontmatter?.created
-              ) {
-                deindexMemory(
-                  this.config.memoryDir,
-                  toMergeInvalidate.path,
-                  toMergeInvalidate.frontmatter.created,
-                  toMergeInvalidate.frontmatter.tags ?? [],
-                );
-              }
-            }
-          }
-          break;
-      }
-    }
-
-    if (result.profileUpdates.length > 0) {
-      await this.storage.appendToProfile(result.profileUpdates);
-    }
-
-    for (const entity of result.entityUpdates) {
-      const safeFacts = Array.isArray((entity as any)?.facts)
-        ? (entity as any).facts.filter((f: any) => typeof f === "string")
-        : [];
-      await this.storage.writeEntity(entity.name, entity.type, safeFacts, {
-        source: "consolidation",
-        structuredSections: Array.isArray((entity as any)?.structuredSections)
-          ? (entity as any).structuredSections
-          : undefined,
-      });
-    }
-
-    // Catalog write touch accounting (issue #1499 sweep): consolidation persists
-    // durable mutations directly to the default-namespace `this.storage`, bypassing
-    // the extraction write path. We do NOT touch here — later maintenance steps in
-    // this same function (entity-file merges, expired-commitment / TTL cleanup,
-    // fact archival) can ALSO mutate the namespace on a run with no LLM outputs
-    // (NIjwl). So we accumulate every durable mutation into `memoryItemMutated` and
-    // record ONE consolidated touch AFTER all mutation-producing steps complete,
-    // just before returning (rule #25: touch after the write commits). LLM
-    // profile/entity updates and memory-item actions (UPDATE / MERGE / INVALIDATE)
-    // count here (NIBOi).
-    if (result.profileUpdates.length > 0 || result.entityUpdates.length > 0) {
-      memoryItemMutated = true;
-    }
-
-    // Merge fragmented entity files
-    const entitiesMerged = await this.storage.mergeFragmentedEntities();
-    if (entitiesMerged > 0) {
-      memoryItemMutated = true;
-      log.info(`merged ${entitiesMerged} fragmented entity files`);
-    }
-
-    if (resolvePresentationCapabilities(this.config).entitySummary) {
-      try {
-        const synthesized = await this.processEntitySynthesisQueue(
-          this.config.defaultNamespace,
-          5,
-        );
-        if (synthesized > 0) {
-          // Entity synthesis rewrites entity files — a durable namespace mutation,
-          // so record it for the catalog touch even when it is the only change in
-          // the pass (codex). Otherwise lastWriteAt goes stale.
-          memoryItemMutated = true;
-          log.info(`refreshed ${synthesized} entity syntheses`);
-        }
-      } catch (err) {
-        log.debug(`entity synthesis pass failed: ${err}`);
-      }
-    }
-
-    // Clean expired commitments
-    const deletedCommitments = await this.storage.cleanExpiredCommitments(
-      this.config.commitmentDecayDays,
-    );
-    if (deletedCommitments.length > 0) {
-      memoryItemMutated = true;
-      log.info(`cleaned ${deletedCommitments.length} expired commitments`);
-      if (resolveIndexingCapabilities(this.config).queryAwareIndexing) {
-        for (const m of deletedCommitments) {
-          deindexMemory(
-            this.config.memoryDir,
-            m.path,
-            m.frontmatter.created,
-            m.frontmatter.tags ?? [],
-          );
-        }
-      }
-    }
-
-    if (
-      resolveCreationMemoryCapabilities(this.config).creationMemory &&
-      resolveCreationMemoryCapabilities(this.config).commitmentLedger &&
-      resolveCreationMemoryCapabilities(this.config).commitmentLifecycle
-    ) {
-      try {
-        const lifecycle = await applyCommitmentLedgerLifecycle({
-          memoryDir: this.config.memoryDir,
-          commitmentLedgerDir: this.config.commitmentLedgerDir,
-          enabled: true,
-          decayDays: this.config.commitmentDecayDays,
-        });
-        if (
-          lifecycle.transitionedToExpired.length > 0 ||
-          lifecycle.deletedResolved.length > 0
-        ) {
-          memoryItemMutated = true;
-          log.info(
-            `commitment ledger lifecycle: expired ${lifecycle.transitionedToExpired.length}, cleaned ${lifecycle.deletedResolved.length}`,
-          );
-        }
-      } catch (err) {
-        log.debug(`commitment ledger lifecycle pass failed: ${err}`);
-      }
-    }
-
-    // Clean memories past their TTL (speculative memories auto-expire)
-    const deletedTTL = await this.storage.cleanExpiredTTL();
-    if (deletedTTL.length > 0) {
-      memoryItemMutated = true;
-      log.info(`cleaned ${deletedTTL.length} TTL-expired memories`);
-      if (resolveIndexingCapabilities(this.config).queryAwareIndexing) {
-        for (const m of deletedTTL) {
-          deindexMemory(
-            this.config.memoryDir,
-            m.path,
-            m.frontmatter.created,
-            m.frontmatter.tags ?? [],
-          );
-        }
-      }
-    }
-
-    // v8.3 Lifecycle policy pass — deterministic promotion/decay metadata
-    if (lifecycleCaps.lifecyclePolicy) {
-      try {
-        const lightSleepStartedAt = new Date().toISOString();
-        const lifecycleCorpus = await this.storage.readAllMemories();
-        // Lifecycle frontmatter writes count as durable mutations for the catalog
-        // touch below (codex NR-tS), even when no other consolidation step set
-        // memoryItemMutated.
-        if ((await this.runLifecyclePolicyPass(lifecycleCorpus)) > 0) {
-          memoryItemMutated = true;
-        }
-        await this.recordScheduledDreamsPhaseRun(
-          "lightSleep",
-          lifecycleCorpus.length,
-          `scheduled lifecycle policy pass assessed ${lifecycleCorpus.length} memories`,
-          {
-            startedAt: lightSleepStartedAt,
-            completedAt: new Date().toISOString(),
-          },
-        );
-      } catch (err) {
-        log.warn(`lifecycle policy pass failed (ignored): ${err}`);
-      }
-    }
-
-    // v8.3 Compression guideline learning pass (default off, fail-open).
-    await this.runCompressionGuidelineLearningPass();
-
-    try {
-      const deepSleepStartedAt = new Date().toISOString();
-      // Tier migrations move/rewrite memory files; count them as durable
-      // mutations for the catalog touch below (codex NThSW).
-      const tierMigration = await this.runTierMigrationCycle(this.storage, "maintenance");
-      if (tierMigration.migrated > 0) memoryItemMutated = true;
-      allMemories = await this.storage.readAllMemories();
-
-      // Fact archival pass (v6.0) — move old, low-importance, rarely-accessed facts to archive/
-      if (resolveRecallEnhancementCapabilities(this.config).factArchival) {
-        const archived = await this.runFactArchival(allMemories);
-        if (archived > 0) {
-          memoryItemMutated = true;
-          log.info(`archived ${archived} old low-importance facts`);
-        }
-      }
-      await this.recordScheduledDreamsPhaseRun(
-        "deepSleep",
-        allMemories.length,
-        `scheduled deep-sleep maintenance assessed ${allMemories.length} memories`,
-        {
-          startedAt: deepSleepStartedAt,
-          completedAt: new Date().toISOString(),
-        },
-      );
-    } catch (err) {
-      log.warn(`deep-sleep maintenance pass failed (ignored): ${err}`);
-      try {
-        allMemories = await this.storage.readAllMemories();
-      } catch (readErr) {
-        log.warn(`deep-sleep maintenance recovery read failed: ${readErr}`);
-        throw err;
-      }
-    }
-
-    // Semantic consolidation pass — find similar memories, synthesize canonical versions
-    if (resolveConsolidationCapabilities(this.config).semanticConsolidation) {
-      try {
-        const stateFilePath = path.join(
-          this.config.memoryDir,
-          "state",
-          "semantic-consolidation-last-run.json",
-        );
-        let shouldRun = true;
-        try {
-          const stateRaw = await readFile(stateFilePath, "utf-8");
-          const stateData = JSON.parse(stateRaw) as { lastRunAt?: string };
-          if (stateData.lastRunAt) {
-            const lastRunMs = new Date(stateData.lastRunAt).getTime();
-            const intervalMs =
-              this.config.semanticConsolidationIntervalHours * 60 * 60 * 1000;
-            if (Date.now() - lastRunMs < intervalMs) {
-              shouldRun = false;
-              log.debug(
-                "[semantic-consolidation] skipping — not enough time since last run",
-              );
-            }
-          }
-        } catch {
-          // No state file yet — first run
-        }
-
-        if (shouldRun) {
-          const remStartedAt = new Date().toISOString();
-          const semResult = await this.runSemanticConsolidation();
-          let remItemsProcessed = allMemories.length;
-          try {
-            allMemories = await this.storage.readAllMemories();
-            remItemsProcessed = allMemories.length;
-          } catch (err) {
-            log.warn(
-              `[semantic-consolidation] post-run telemetry refresh failed (non-fatal): ${err}`,
-            );
-          }
-          await this.recordScheduledDreamsPhaseRun(
-            "rem",
-            remItemsProcessed,
-            `scheduled REM consolidation found ${semResult.clustersFound} clusters`,
-            {
-              startedAt: remStartedAt,
-              completedAt: new Date().toISOString(),
-            },
-          );
-          if (semResult.memoriesArchived > 0) {
-            log.info(
-              `[semantic-consolidation] archived ${semResult.memoriesArchived} memories during maintenance`,
-            );
-          }
-          // Only persist last-run timestamp if the run succeeded (had no errors or made progress)
-          if (semResult.errors === 0 || semResult.memoriesArchived > 0) {
-            const stateDir = path.join(this.config.memoryDir, "state");
-            await mkdir(stateDir, { recursive: true });
-            await writeFile(
-              stateFilePath,
-              JSON.stringify({ lastRunAt: new Date().toISOString() }),
-              "utf-8",
-            );
-          }
-        }
-      } catch (err) {
-        log.warn(
-          `[semantic-consolidation] maintenance pass failed (non-fatal): ${err}`,
-        );
-      }
-    }
-
-    // Auto-consolidate IDENTITY.md if it's getting large
-    if (resolveRecallEnhancementCapabilities(this.config).identity) {
-      await this.autoConsolidateIdentity();
-    }
-
-    // Auto-consolidate profile.md if it exceeds max lines
-    const profileSection = this.getRecallSectionEntry("profile");
-    const profileConsolidationTriggerLines =
-      typeof profileSection?.consolidateTriggerLines === "number"
-        ? Math.max(0, Math.floor(profileSection.consolidateTriggerLines))
-        : undefined;
-    const profileConsolidationTargetLines =
-      typeof profileSection?.consolidateTargetLines === "number"
-        ? Math.max(0, Math.floor(profileSection.consolidateTargetLines))
-        : 50;
-    if (
-      await this.storage.profileNeedsConsolidation(
-        profileConsolidationTriggerLines,
-      )
-    ) {
-      log.info("profile.md exceeds max lines — running smart consolidation");
-      const currentProfile = await this.storage.readProfile();
-      if (currentProfile) {
-        const profileResult = await this.extraction.consolidateProfile(
-          currentProfile,
-          profileConsolidationTargetLines,
-        );
-        if (profileResult) {
-          await this.storage.writeProfile(profileResult.consolidatedProfile);
-          // Profile consolidation rewrites profile.md — a durable namespace
-          // mutation; record it for the catalog touch even when it is the only
-          // change in the pass (codex). Otherwise lastWriteAt goes stale.
-          memoryItemMutated = true;
-          log.info(
-            `profile.md consolidated: removed ${profileResult.removedCount} items — ${profileResult.summary}`,
-          );
-        }
-      }
-    }
-
-    // Memory Summarization (Phase 4A)
-    if (resolvePipelineProcessingCapabilities(this.config).summarization) {
-      await this.runSummarization(allMemories);
-    }
-
-    // Topic Extraction (Phase 4B)
-    if (resolvePipelineProcessingCapabilities(this.config).topicExtraction) {
-      await this.runTopicExtraction(allMemories);
-    }
-
-    const meta = await this.storage.loadMeta();
-    meta.lastConsolidationAt = new Date().toISOString();
-    await this.storage.saveMeta(meta);
-
-    // Temporal Memory Tree (v8.2) — rebuild nodes from all memories, fail-open
-    if (lifecycleCaps.temporalMemoryTree) {
-      try {
-        const tmtEntries = allMemories
-          .filter(
-            (m) =>
-              m.frontmatter.status !== "superseded" && m.frontmatter.status !== "archived" &&
-              m.frontmatter.status !== "forgotten" && m.frontmatter.status !== "pending_review", // #1576: unfaithful queue items must not feed TMT clusters
-          )
-          .map((m) => ({
-            path: m.path,
-            id: m.frontmatter.id,
-            created: m.frontmatter.created,
-            content: m.content,
-          }));
-        await this.tmtBuilder.maybeRebuildNodes(
-          tmtEntries,
-          async (texts, level) => {
-            const prompt = `You are a memory archivist. Summarize the following ${level}-level memories into 3–5 sentences, preserving key facts, decisions, and preferences.\n\n${texts.map((t, i) => `[${i + 1}] ${t}`).join("\n\n")}`;
-            const response = await this.fastChatCompletion(
-              [
-                {
-                  role: "system",
-                  content:
-                    "Respond with a 3–5 sentence narrative summary. No JSON, just plain prose.",
-                },
-                { role: "user", content: prompt },
-              ],
-              {
-                temperature: 0.3,
-                maxTokens: this.config.tmtSummaryMaxTokens,
-                operation: "tmt_summary",
-                priority: "background",
-              },
-            );
-            return response?.content?.trim() || texts.slice(0, 3).join(" ");
-          },
-        );
-      } catch (err) {
-        log.warn(`tmt: consolidation hook failed (ignored): ${err}`);
-      }
-    }
-
-    if (this.consolidationObservers.size > 0) {
-      const observation: ConsolidationObservation = {
-        runAt: new Date().toISOString(),
-        recentMemories: recent,
-        existingMemories: older.slice(-50),
-        profile,
-        result,
-        merged,
-        invalidated,
-      };
-      for (const observer of this.consolidationObservers) {
-        try {
-          await observer(observation);
-        } catch (err) {
-          log.warn(`consolidation observer failed (ignored): ${err}`);
-        }
-      }
-    }
-
-    // Consolidated catalog write touch — belt-and-suspenders for cleanup-only
-    // passes that mutate the store via delete-only paths (entity merges, TTL
-    // cleanup) without triggering the storage chokepoint's post-write hook.
-    // Gated on memoryItemMutated (set by every durable mutation including cleanup-only passes)
-    // Best-effort and failure-tolerant.
-    if (memoryItemMutated) {
-      this.storageRouter.recordWrite(this.config.defaultNamespace, this.storage.dir);
-    }
-
-    log.info("consolidation complete");
-    return { memoriesProcessed: allMemories.length, merged, invalidated };
+    return this.consolidationRunCoordinator.run();
   }
 
   async optimizeCompressionGuidelines(options?: {
@@ -13004,36 +12546,10 @@ export class Orchestrator {
 
   // Issue #1526 (seam 4): compression-guideline learning moved to
   // CompressionGuidelineCoordinator. Thin delegation keeps the
-  // consolidation-loop hook + recall-section hook call sites stable.
-  private async runCompressionGuidelineLearningPass(): Promise<void> {
-    return this.compressionGuidelineCoordinator.runCompressionGuidelineLearningPass();
-  }
-
   private async buildCompressionGuidelineRecallSection(): Promise<string | null> {
     return this.compressionGuidelineCoordinator.buildCompressionGuidelineRecallSection();
   }
 
-  private async recordScheduledDreamsPhaseRun(
-    phase: "lightSleep" | "rem" | "deepSleep",
-    itemsProcessed: number,
-    notes: string,
-    timing: { startedAt?: string; completedAt?: string } = {},
-  ): Promise<void> {
-    try {
-      const { recordDreamsPhaseRun } = await import("./maintenance/dreams-ledger.js");
-      await recordDreamsPhaseRun({
-        memoryDir: this.storage.dir,
-        phase,
-        trigger: "scheduled",
-        itemsProcessed,
-        notes,
-        startedAt: timing.startedAt,
-        completedAt: timing.completedAt,
-      });
-    } catch (error) {
-      log.debug(`dreams ledger scheduled ${phase} write failed (non-fatal): ${error}`);
-    }
-  }
 
   async runLifecyclePolicyNow(storage: StorageManager = this.storage): Promise<{ memoriesAssessed: number }> {
     const lifecycleCorpus = await storage.readAllMemories();
@@ -13049,21 +12565,6 @@ export class Orchestrator {
   ): Promise<number> {
     return this.lifecyclePolicyCoordinator.runLifecyclePolicyPass(allMemories, storage);
   }
-  private async runFactArchival(
-    allMemories: import("./types.js").MemoryFile[],
-  ): Promise<number> {
-    return this.lifecyclePolicyCoordinator.runFactArchival(allMemories);
-  }
-  private async runSummarization(
-    allMemories: import("./types.js").MemoryFile[],
-  ): Promise<void> {
-    return this.lifecyclePolicyCoordinator.runSummarization(allMemories);
-  }
-  private async runTopicExtraction(
-    allMemories: import("./types.js").MemoryFile[],
-  ): Promise<void> {
-    return this.lifecyclePolicyCoordinator.runTopicExtraction(allMemories);
-  }
   /** Threshold (bytes) at which IDENTITY.md reflections get auto-consolidated */
   private static readonly IDENTITY_CONSOLIDATE_THRESHOLD = 8_000;