@remnic/core 9.7.15 → 9.8.0

package/src/orchestration/recall-section-coordinator.ts

@@ -20,6 +20,26 @@
 import type { PluginConfig, RecallSectionConfig } from "../types.js";
 import type { LastRecallBudgetSummary } from "../recall-state.js";
 
+export interface RecallSectionAppendOptions {
+  atomic?: boolean;
+  memoryId?: string;
+  memoryPath?: string;
+  memoryNamespace?: string;
+}
+
+export interface RecallSectionChunk {
+  content: string;
+  atomic?: boolean;
+  memoryId?: string;
+  memoryPath?: string;
+  memoryNamespace?: string;
+}
+
+export type RecallSectionBuckets = Map<
+  string,
+  Array<string | RecallSectionChunk>
+>;
+
 /**
  * Coordinator for the recall section budgeting + assembly subsystem.
  *
@@ -104,15 +124,11 @@ export class RecallSectionCoordinator {
   }
 
   appendRecallSection(
-    sectionBuckets: Map<string, string[]>,
+    sectionBuckets: RecallSectionBuckets,
     sectionId: string,
     content: string,
+    options: RecallSectionAppendOptions = {},
   ): boolean {
-    // Returns true when the section was actually appended to sectionBuckets,
-    // false when it was dropped (disabled, empty, or maxChars===0). Callers
-    // that need to know whether injection occurred (e.g. xray annotation for
-    // peer-profile) must gate on this return value rather than on whether the
-    // section text was computed (Codex P2 finding, PR #764).
     if (!this.resolveSectionEnabled(sectionId, true)) return false;
     const trimmed = content.trim();
     if (trimmed.length === 0) return false;
@@ -120,16 +136,50 @@ export class RecallSectionCoordinator {
     const maxChars = this.getRecallSectionMaxChars(sectionId);
     let finalContent = trimmed;
     if (maxChars === 0) return false;
-    if (typeof maxChars === "number" && finalContent.length > maxChars) {
-      finalContent = `${finalContent.slice(0, maxChars)}\n\n...(trimmed)\n`;
+    if (
+      !options.atomic &&
+      typeof maxChars === "number" &&
+      finalContent.length > maxChars
+    ) {
+      finalContent =
+        sectionId === "profile"
+          ? this.truncateProfileToBoundary(finalContent, maxChars)
+          : `${finalContent.slice(0, maxChars)}\n\n...(trimmed)\n`;
     }
-
+    if (finalContent.length === 0) return false;
     const existing = sectionBuckets.get(sectionId) ?? [];
-    existing.push(finalContent);
+
+    if (options.atomic) {
+      existing.push({
+        content: finalContent,
+        atomic: true,
+        ...(options.memoryId ? { memoryId: options.memoryId } : {}),
+        ...(options.memoryPath ? { memoryPath: options.memoryPath } : {}),
+        ...(options.memoryNamespace
+          ? { memoryNamespace: options.memoryNamespace }
+          : {}),
+      });
+    } else {
+      existing.push(finalContent);
+    }
     sectionBuckets.set(sectionId, existing);
     return true;
   }
 
+  truncateProfileToBoundary(content: string, maxChars: number): string {
+    if (maxChars <= 0) return "";
+    if (content.length <= maxChars) return content;
+    const suffix = "\n\n...(profile context trimmed)";
+    const contentLimit = maxChars - suffix.length;
+    if (contentLimit <= 0) {
+      const boundary = content.lastIndexOf("\n", maxChars);
+      return content.slice(0, boundary > 0 ? boundary : maxChars).trimEnd();
+    }
+    const boundary = content.lastIndexOf("\n", contentLimit);
+    const end = boundary > 0 ? boundary : contentLimit;
+    return `${content.slice(0, end).trimEnd()}${suffix}`;
+  }
+
   truncateRecallSectionToBudget(
     content: string,
     maxChars: number,
@@ -143,43 +193,6 @@ export class RecallSectionCoordinator {
     return `${content.slice(0, maxChars - suffix.length)}${suffix}`;
   }
 
-  protectedRecallSectionIds(
-    sectionBuckets: Map<string, string[]>,
-  ): Set<string> {
-    const protectedIds = new Set<string>();
-    if ((sectionBuckets.get("memories")?.length ?? 0) > 0) {
-      protectedIds.add("memories");
-    }
-    return protectedIds;
-  }
-
-  protectedRecallReservationChars(content: string): number {
-    const headingBoundary = content.indexOf("\n\n");
-    const headingChars =
-      headingBoundary >= 0 ? headingBoundary + 2 : Math.min(content.length, 24);
-    return Math.min(content.length, Math.max(headingChars, 24));
-  }
-
-  estimateReservedRecallBudget(
-    entries: Array<{ id: string; content: string }>,
-    startIndex: number,
-    protectedIds: Set<string>,
-    alreadyIncludedCount: number,
-  ): number {
-    const separatorLength = "\n\n---\n\n".length;
-    let reserved = 0;
-    let simulatedIncluded = alreadyIncludedCount;
-    for (let i = startIndex; i < entries.length; i += 1) {
-      const entry = entries[i];
-      if (!entry || !protectedIds.has(entry.id)) continue;
-      if (simulatedIncluded > 0) {
-        reserved += separatorLength;
-      }
-      reserved += this.protectedRecallReservationChars(entry.content);
-      simulatedIncluded += 1;
-    }
-    return reserved;
-  }
 
   getRecallBudgetChars(override?: number): number {
     if (
@@ -209,7 +222,7 @@ export class RecallSectionCoordinator {
   }
 
   assembleRecallSections(
-    sectionBuckets: Map<string, string[]>,
+    sectionBuckets: RecallSectionBuckets,
     budgetOverride?: number,
   ): {
     sections: string[];
@@ -217,88 +230,297 @@ export class RecallSectionCoordinator {
     omittedIds: string[];
     truncated: boolean;
     finalChars: number;
+    includedMemoryIds: string[];
+    includedMemoryPaths: string[];
+    includedMemoryNamespaces: Array<string | undefined>;
+    omittedMemoryIds: string[];
   } {
-    const orderedEntries: Array<{ id: string; content: string }> = [];
+    type OrderedSection = { id: string; chunks: RecallSectionChunk[] };
+    const normalizeChunk = (
+      chunk: string | RecallSectionChunk,
+    ): RecallSectionChunk =>
+      typeof chunk === "string" ? { content: chunk } : chunk;
+    const orderedSections: OrderedSection[] = [];
     const pipeline = Array.isArray(this.getConfig().recallPipeline)
       ? this.getConfig().recallPipeline
       : [];
     const orderedIds = pipeline
-      .filter((entry) => entry.enabled !== false)
+      .filter(
+        (entry) =>
+          entry.enabled !== false &&
+          this.resolveSectionEnabled(entry.id, true),
+      )
       .map((entry) => entry.id);
     const seen = new Set<string>();
 
     for (const id of orderedIds) {
+      if (seen.has(id)) continue;
       const chunks = sectionBuckets.get(id);
       if (!chunks || chunks.length === 0) continue;
-      orderedEntries.push({ id, content: chunks.join("\n\n") });
+      orderedSections.push({
+        id,
+        chunks: chunks.map(normalizeChunk),
+      });
       seen.add(id);
     }
 
     for (const [id, chunks] of sectionBuckets.entries()) {
-      if (seen.has(id)) continue;
-      if (chunks.length === 0) continue;
-      orderedEntries.push({ id, content: chunks.join("\n\n") });
+      if (
+        seen.has(id) ||
+        chunks.length === 0 ||
+        !this.resolveSectionEnabled(id, true)
+      ) {
+        continue;
+      }
+      orderedSections.push({
+        id,
+        chunks: chunks.map(normalizeChunk),
+      });
+      seen.add(id);
     }
 
     const budget = this.getRecallBudgetChars(budgetOverride);
+    const candidateMemoryChunks =
+      orderedSections
+        .find((section) => section.id === "memories")
+        ?.chunks.filter((chunk) => chunk.atomic && chunk.memoryId)
+        .map((chunk) => ({
+          id: chunk.memoryId!,
+          namespace: chunk.memoryNamespace,
+        })) ?? [];
+    const candidateMemoryIds = candidateMemoryChunks.map((chunk) => chunk.id);
     if (budget === 0) {
       return {
         sections: [],
         includedIds: [],
-        omittedIds: orderedEntries.map((entry) => entry.id),
-        truncated: orderedEntries.length > 0,
+        omittedIds: orderedSections.map((entry) => entry.id),
+        truncated: orderedSections.length > 0,
         finalChars: 0,
+        includedMemoryIds: [],
+        includedMemoryPaths: [],
+        includedMemoryNamespaces: [],
+        omittedMemoryIds: candidateMemoryIds,
       };
     }
 
     const separator = "\n\n---\n\n";
-    const protectedIds = this.protectedRecallSectionIds(sectionBuckets);
-    const sections: string[] = [];
-    const includedIds: string[] = [];
-    const omittedIds: string[] = [];
+    const sectionById = new Map(
+      orderedSections.map((section) => [section.id, section]),
+    );
+    const allocationOrder = orderedSections.map((section) => section.id);
+    const memorySection = sectionById.get("memories");
+    const memoryIndex = allocationOrder.indexOf("memories");
+    const memoryBudget = this.getRecallSectionMaxChars("memories") ?? budget;
+    const memoryAllocationBudget = Math.min(budget, memoryBudget);
+    let firstAtomicMemoryReserveChars = 0;
+    for (const chunk of memorySection?.chunks ?? []) {
+      if (chunk.atomic && chunk.content.length <= memoryAllocationBudget) {
+        firstAtomicMemoryReserveChars = chunk.content.length;
+        break;
+      }
+    }
+    const includedMemoryIds: string[] = [];
+    const includedMemoryPaths: string[] = [];
+    const includedMemoryNamespaces: Array<string | undefined> = [];
     let usedChars = 0;
     let truncated = false;
+    const selected = new Map<string, string>();
 
-    for (let index = 0; index < orderedEntries.length; index += 1) {
-      const entry = orderedEntries[index]!;
-      const separatorChars = sections.length > 0 ? separator.length : 0;
-      const reserve = protectedIds.has(entry.id)
-        ? 0
-        : this.estimateReservedRecallBudget(
-            orderedEntries,
-            index + 1,
-            protectedIds,
-            sections.length + 1,
-          );
-      const availableForEntry = budget - usedChars - separatorChars - reserve;
-      if (availableForEntry <= 0) {
-        omittedIds.push(entry.id);
+    for (const id of allocationOrder) {
+      const section = sectionById.get(id);
+      if (!section || selected.has(id)) continue;
+      const separatorChars = selected.size > 0 ? separator.length : 0;
+      const available = budget - usedChars - separatorChars;
+      const sectionMaxChars = this.getRecallSectionMaxChars(id);
+      const reservesFirstMemory =
+        memoryIndex > allocationOrder.indexOf(id) &&
+        firstAtomicMemoryReserveChars > 0;
+      const memoryReserve = reservesFirstMemory
+        ? firstAtomicMemoryReserveChars + separator.length
+        : 0;
+      const availableAfterMemoryReserve = available - memoryReserve;
+      const allocatedSectionAvailable =
+        typeof sectionMaxChars === "number"
+          ? Math.min(availableAfterMemoryReserve, sectionMaxChars)
+          : availableAfterMemoryReserve;
+      if (allocatedSectionAvailable <= 0) {
         truncated = true;
         continue;
       }
-      const finalContent = this.truncateRecallSectionToBudget(
-        entry.content,
-        availableForEntry,
-      );
+
+
+      const atomicChunks = section.chunks.filter((chunk) => chunk.atomic);
+      let finalContent: string;
+      if (id !== "memories" || atomicChunks.length === 0) {
+        const content = section.chunks.map((chunk) => chunk.content).join("\n\n");
+        const profileLimit =
+          id === "profile"
+            ? Math.min(
+                Math.floor(
+                  budget *
+                    (typeof this.getConfig().recallProfileMaxRatio === "number"
+                      ? this.getConfig().recallProfileMaxRatio
+                      : 0.3),
+                ),
+                this.getRecallSectionMaxChars("profile") ?? budget,
+              )
+            : allocatedSectionAvailable;
+        const boundedContent =
+          id === "profile"
+            ? this.truncateProfileToBoundary(
+                content,
+                Math.min(profileLimit, allocatedSectionAvailable),
+              )
+            : content;
+        finalContent =
+          id === "profile"
+            ? boundedContent
+            : this.truncateRecallSectionToBudget(
+                boundedContent,
+                allocatedSectionAvailable,
+              );
+        if (!finalContent) {
+          truncated = true;
+          continue;
+        }
+        if (finalContent.length < content.length) truncated = true;
+      } else {
+        const firstAtomicIndex = section.chunks.findIndex((chunk) => chunk.atomic);
+        let rendered = "";
+        let includedAtomicCount = 0;
+        let includedPostAtomicContent = false;
+        const postAtomicContents: string[] = [];
+        let includedLeadingContent = false;
+        for (const [index, chunk] of section.chunks.entries()) {
+          if (
+            !chunk.atomic &&
+            includedAtomicCount === 0 &&
+            index < firstAtomicIndex
+          ) {
+            const candidate = rendered
+              ? `${rendered}\n\n${chunk.content}`
+              : chunk.content;
+            if (candidate.length <= allocatedSectionAvailable) {
+              rendered = candidate;
+              includedLeadingContent = true;
+            } else {
+              truncated = true;
+            }
+            continue;
+          }
+
+          const candidate = rendered
+            ? `${rendered}\n\n${chunk.content}`
+            : chunk.content;
+          if (candidate.length > allocatedSectionAvailable) {
+            if (
+              chunk.atomic &&
+              includedAtomicCount === 0 &&
+              includedLeadingContent &&
+              chunk.content.length <= allocatedSectionAvailable
+            ) {
+              rendered = chunk.content;
+              includedLeadingContent = false;
+              includedAtomicCount = 1;
+              // Push id/path/namespace together so the parallel tracking arrays
+              // never misalign when a chunk carries only one of them (#2020).
+              if (chunk.memoryId && chunk.memoryPath) {
+                includedMemoryIds.push(chunk.memoryId);
+                includedMemoryPaths.push(chunk.memoryPath);
+                includedMemoryNamespaces.push(chunk.memoryNamespace);
+              }
+              continue;
+            }
+            if (
+              !chunk.atomic &&
+              includedAtomicCount === 0 &&
+              includedLeadingContent &&
+              chunk.content.length <= allocatedSectionAvailable
+            ) {
+              rendered = chunk.content;
+              includedLeadingContent = false;
+              includedPostAtomicContent = true;
+              postAtomicContents.push(chunk.content);
+              continue;
+            }
+            truncated = true;
+            continue;
+          }
+
+          rendered = candidate;
+          if (chunk.atomic) {
+            includedAtomicCount += 1;
+            if (chunk.memoryId && chunk.memoryPath) {
+              includedMemoryIds.push(chunk.memoryId);
+              includedMemoryPaths.push(chunk.memoryPath);
+              includedMemoryNamespaces.push(chunk.memoryNamespace);
+            }
+          } else {
+            includedPostAtomicContent = true;
+            postAtomicContents.push(chunk.content);
+          }
+        }
+        if (includedAtomicCount === 0) {
+          if (includedPostAtomicContent) {
+            rendered = postAtomicContents.join("\n\n");
+          } else {
+            rendered = "";
+            truncated = true;
+          }
+        } else if (includedAtomicCount < atomicChunks.length) {
+          truncated = true;
+        }
+        finalContent = rendered;
+      }
+
       if (!finalContent) {
-        omittedIds.push(entry.id);
         truncated = true;
         continue;
       }
-      if (finalContent.length < entry.content.length) {
-        truncated = true;
-      }
-      sections.push(finalContent);
-      includedIds.push(entry.id);
+      selected.set(id, finalContent);
       usedChars += separatorChars + finalContent.length;
     }
 
+    const sections: string[] = [];
+    const includedIds: string[] = [];
+    const omittedIds: string[] = [];
+    const emittedIds = new Set<string>();
+    for (const section of orderedSections) {
+      if (emittedIds.has(section.id)) continue;
+      emittedIds.add(section.id);
+      const content = selected.get(section.id);
+      if (content) {
+        sections.push(content);
+        includedIds.push(section.id);
+      } else {
+        omittedIds.push(section.id);
+      }
+    }
     return {
       sections,
       includedIds,
       omittedIds,
       truncated,
       finalChars: usedChars,
+      includedMemoryIds,
+      includedMemoryPaths,
+      includedMemoryNamespaces,
+      omittedMemoryIds: (() => {
+        // Key by (namespace, id): the same memory id can exist in more than one
+        // namespace, so an id-only subtraction would hide a budget-dropped copy
+        // when another namespace's copy was included (#2020).
+        const includedKeys = new Set(
+          includedMemoryIds.map(
+            (id, i) => `${includedMemoryNamespaces[i] ?? ""}\u0000${id}`,
+          ),
+        );
+        return candidateMemoryChunks
+          .filter(
+            (chunk) =>
+              !includedKeys.has(`${chunk.namespace ?? ""}\u0000${chunk.id}`),
+          )
+          .map((chunk) => chunk.id);
+      })(),
     };
   }
 
@@ -310,6 +532,10 @@ export class RecallSectionCoordinator {
     finalContextChars?: number;
     truncated?: boolean;
     includedSections?: string[];
+    includedMemoryIds?: string[];
+    includedMemoryPaths?: string[];
+    includedMemoryNamespaces?: Array<string | undefined>;
+    omittedMemoryIds?: string[];
     omittedSections?: string[];
   }): LastRecallBudgetSummary {
     return {
@@ -321,13 +547,17 @@ export class RecallSectionCoordinator {
       qmdHybridFetchLimit: options.qmdHybridFetchLimit,
       finalContextChars: options.finalContextChars,
       truncated: options.truncated,
+      includedMemoryIds: [...(options.includedMemoryIds ?? [])],
+      includedMemoryPaths: [...(options.includedMemoryPaths ?? [])],
+      includedMemoryNamespaces: [...(options.includedMemoryNamespaces ?? [])],
+      omittedMemoryIds: [...(options.omittedMemoryIds ?? [])],
       includedSections: [...(options.includedSections ?? [])],
       omittedSections: [...(options.omittedSections ?? [])],
     };
   }
 
   collectLastRecallSources(
-    sectionBuckets: Map<string, string[]>,
+    sectionBuckets: RecallSectionBuckets,
     recallSource:
       | "none"
       | "hot_qmd"

package/src/orchestration/semantic-consolidation-coordinator.ts

@@ -42,13 +42,14 @@ import {
   gatewayTaskChainOptions,
 } from "../fallback-llm.js";
 import { resolveIndexingCapabilities, resolveConsolidationCapabilities } from "../capabilities.js";
-import { deindexMemoryAsync } from "../temporal-index.js";
+import { deindexMemoriesBatchAsync } from "../temporal-index-batch.js";
 import { runPeerProfileReasoner } from "../peers/index.js";
 import type { StorageManager } from "../index.js";
 import type { LocalLlmClient } from "../local-llm.js";
 import type { EmbeddingFallback } from "../embedding-fallback.js";
 import type { PluginConfig, MemoryFile, AgentPersonaModelConfig } from "../types.js";
 import { log } from "../logger.js";
+import { composeMemoryEnvelope } from "../write-envelope.js";
 import { resolveConversationContextCapabilities } from "../capabilities.js";
 
 /** Dependencies injected by the orchestrator. All stable references or
@@ -289,18 +290,30 @@ export class SemanticConsolidationCoordinator {
         }
 
         // Write the canonical memory
-        const { id: canonicalId, tombstoneBlocked: canonicalBlocked } = await targetStorage.writeMemory(
-          newest.frontmatter.category,
-          canonicalContent,
+        // Sealed-envelope write (issue #1989 PR4): canonical content is
+        // LLM-merged and tags are unioned from stored rows — salvage so a
+        // limit-violating union cannot abort consolidation (warn-logged).
+        const canonicalEnvelope = composeMemoryEnvelope(
           {
-            actor: "semantic-consolidation",
+            content: canonicalContent,
+            category: newest.frontmatter.category,
             confidence: newest.frontmatter.confidence,
             tags: [
               ...new Set(
                 cluster.memories.flatMap((m) => m.frontmatter.tags ?? []),
               ),
             ],
-            source: "semantic-consolidation",
+          },
+          { source: "semantic-consolidation" },
+          { salvage: true },
+        );
+        if (canonicalEnvelope.salvageNotes.length > 0) {
+          log.warn(`semantic-consolidation write salvaged invalid fields: ${canonicalEnvelope.salvageNotes.join("; ")}`);
+        }
+        const { id: canonicalId, tombstoneBlocked: canonicalBlocked } = await targetStorage.writeSealedMemory(
+          canonicalEnvelope,
+          {
+            actor: "semantic-consolidation",
             lineage: lineageIds,
             derivedFrom: derivedFromEntries.length > 0 ? derivedFromEntries : undefined,
             derivedVia: operator,
@@ -317,44 +330,63 @@ export class SemanticConsolidationCoordinator {
         result.memoriesConsolidated++;
 
         // Archive originals
-        for (const m of cluster.memories) {
-          const archiveResult = await targetStorage.archiveMemory(m, {
-            actor: "semantic-consolidation",
-            reasonCode: "semantic-consolidation",
-            relatedMemoryIds: [canonicalId],
-          });
-          if (archiveResult) {
-            // Remove from the same storage-scoped content-hash index that
-            // originally deduped this memory.
-            await this.deps.removeContentHashForMemory(
-              targetStorage,
-              m,
-              "semantic-consolidation",
-            );
-            // Best-effort index cleanup: a failure here (e.g. on-disk index save
-            // under disk-full) must NOT abort the archival loop and thereby skip
-            // the catalog write touch below for an already-durable canonical write
-            // (kilo NV0mh).
-            try {
-              await this.deps.embeddingFallback.removeFromIndex(m.frontmatter.id);
-              if (
-                resolveIndexingCapabilities(this.config).queryAwareIndexing &&
-                m.path &&
-                m.frontmatter?.created
-              ) {
-                await deindexMemoryAsync(
-                  targetStorage.dir,
-                  m.path,
-                  m.frontmatter.created,
-                  m.frontmatter.tags ?? [],
+        // Collect deindex entries and remove them in one batch rather than a full
+        // index read-modify-write per archived source. Guard is preserved: entries
+        // are only collected when queryAwareIndexing is on.
+        const clusterDeindexBatch: Array<{ path: string; createdAt: string; tags: string[] }> = [];
+        try {
+          for (const m of cluster.memories) {
+            const archiveResult = await targetStorage.archiveMemory(m, {
+              actor: "semantic-consolidation",
+              reasonCode: "semantic-consolidation",
+              relatedMemoryIds: [canonicalId],
+            });
+            if (archiveResult) {
+              // Remove from the same storage-scoped content-hash index that
+              // originally deduped this memory.
+              await this.deps.removeContentHashForMemory(
+                targetStorage,
+                m,
+                "semantic-consolidation",
+              );
+              // Best-effort index cleanup: a failure here (e.g. on-disk index save
+              // under disk-full) must NOT abort the archival loop and thereby skip
+              // the catalog write touch below for an already-durable canonical write
+              // (kilo NV0mh).
+              try {
+                await this.deps.embeddingFallback.removeFromIndex(m.frontmatter.id);
+                if (
+                  resolveIndexingCapabilities(this.config).queryAwareIndexing &&
+                  m.path &&
+                  m.frontmatter?.created
+                ) {
+                  clusterDeindexBatch.push({
+                    path: m.path,
+                    createdAt: m.frontmatter.created,
+                    tags: m.frontmatter.tags ?? [],
+                  });
+                }
+              } catch (cleanupErr) {
+                log.warn(
+                  `[semantic-consolidation] index cleanup failed (non-fatal): ${cleanupErr}`,
                 );
               }
+              result.memoriesArchived++;
+            }
+          }
+        } finally {
+          // Best-effort batch index cleanup, flushed in `finally` so sources
+          // already archived on disk are de-indexed even if a later archive
+          // iteration throws. Never abort the cluster (and thereby skip the
+          // catalog touch in the outer `finally`) on an index write failure.
+          if (clusterDeindexBatch.length > 0) {
+            try {
+              await deindexMemoriesBatchAsync(targetStorage.dir, clusterDeindexBatch);
             } catch (cleanupErr) {
               log.warn(
                 `[semantic-consolidation] index cleanup failed (non-fatal): ${cleanupErr}`,
               );
             }
-            result.memoriesArchived++;
           }
         }