@remnic/core 9.7.14 → 9.8.0

package/src/orchestration/batch-deindex-flush.test.ts

@@ -0,0 +1,146 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+import {
+  LifecyclePolicyCoordinator,
+  type LifecyclePolicyCoordinatorDeps,
+} from "./lifecycle-policy-coordinator.js";
+import { indexMemoryAsync } from "../temporal-index.js";
+import type { StorageManager } from "../index.js";
+import type { EmbeddingFallback } from "../embedding-fallback.js";
+import type { ExtractionEngine } from "../extraction.js";
+import type { MemoryFile, PluginConfig } from "../types.js";
+
+// Regression for the #1911B Cursor finding: the maintenance de-index loops
+// collect removals for a single batch flush. If a LATER iteration throws before
+// the flush, memories already archived/invalidated on disk must still be
+// de-indexed — the flush now lives in a `finally`. runFactArchival is the
+// smallest self-contained loop with this shape; the same finally guard is
+// applied to the consolidation-run INVALIDATE/MERGE and semantic-consolidation
+// archive loops.
+
+function makeFact(id: string, path: string, created: string): MemoryFile {
+  const memory = {
+    path,
+    content: `fact ${id}`,
+    frontmatter: {
+      id,
+      category: "fact",
+      created,
+      tags: ["project/remnic", "shared", `flat-${id}`],
+    },
+  };
+  // Test fixture: MemoryFile carries many optional fields the archival loop
+  // never reads; a full literal would be noise. Narrow, deliberate cast.
+  return memory as unknown as MemoryFile;
+}
+
+async function temporalDates(memoryDir: string): Promise<Record<string, string[]>> {
+  const raw: unknown = JSON.parse(
+    await readFile(join(memoryDir, "state", "index_time.json"), "utf8"),
+  );
+  if (raw && typeof raw === "object" && "dates" in raw) {
+    return raw.dates as Record<string, string[]>;
+  }
+  throw new Error("temporal index missing dates map");
+}
+
+test("#1911B fact-archival flushes prior de-index entries when a later iteration throws", async () => {
+  const memoryDir = await mkdtemp(join(tmpdir(), "remnic-batch-deindex-flush-"));
+  try {
+    const memA = makeFact("mem-a", "/tmp/mem-a.md", "2020-01-01T09:00:00.000Z");
+    const memB = makeFact("mem-b", "/tmp/mem-b.md", "2020-02-02T09:00:00.000Z");
+
+    // Seed the on-disk temporal index so a successful de-index is observable as
+    // the removal of the memory's date bucket.
+    await indexMemoryAsync(memoryDir, memA.path, memA.frontmatter.created, memA.frontmatter.tags ?? []);
+    await indexMemoryAsync(memoryDir, memB.path, memB.frontmatter.created, memB.frontmatter.tags ?? []);
+
+    const beforeDates = await temporalDates(memoryDir);
+    assert.deepEqual(beforeDates["2020-01-01"], [memA.path]);
+    assert.deepEqual(beforeDates["2020-02-02"], [memB.path]);
+
+    const archived: string[] = [];
+    const fakeStorage = {
+      // Both memories archive successfully on disk.
+      async archiveMemory(memory: MemoryFile): Promise<string> {
+        archived.push(memory.frontmatter.id);
+        return memory.frontmatter.id;
+      },
+    };
+    // runFactArchival only calls archiveMemory on the storage; the rest of the
+    // StorageManager surface is unused here.
+    const storage = fakeStorage as unknown as StorageManager;
+
+    const embeddingFallback = {
+      async removeFromIndex(): Promise<void> {},
+    } as unknown as EmbeddingFallback;
+
+    // Minimal config: the archival loop reads only these fields plus the
+    // queryAwareIndexing capability flag. A full PluginConfig literal is noise.
+    const config = {
+      memoryDir,
+      queryAwareIndexingEnabled: true,
+      factArchivalAgeDays: 0,
+      factArchivalProtectedCategories: [],
+      factArchivalMaxImportance: 1,
+      factArchivalMaxAccessCount: 1000,
+    } as unknown as PluginConfig;
+
+    const boom = new Error("content-hash removal exploded on a later iteration");
+    const deps: LifecyclePolicyCoordinatorDeps = {
+      config,
+      getStorage: () => storage,
+      extraction: {} as unknown as ExtractionEngine,
+      embeddingFallback,
+      getEffectiveLifecycleThresholds: () => ({
+        promoteHeatThreshold: 0,
+        staleDecayThreshold: 0,
+        archiveDecayThreshold: 0,
+      }),
+      // Throws for mem-b — the later iteration — after mem-b is already archived
+      // AND queued for de-index, but before its content-hash cleanup completes.
+      // Asserts the production call passes the target storage instance and the
+      // fact-archival context.
+      async removeContentHashForMemory(targetStorage, memory, context): Promise<void> {
+        assert.equal(
+          targetStorage,
+          storage,
+          "removeContentHashForMemory must receive the target storage instance",
+        );
+        assert.equal(
+          context,
+          "fact-archival",
+          "removeContentHashForMemory must receive the fact-archival context",
+        );
+        if (memory.frontmatter.id === "mem-b") throw boom;
+      },
+      async saveContentHashIndexes(): Promise<void> {},
+    };
+
+    const coordinator = new LifecyclePolicyCoordinator(deps);
+
+    await assert.rejects(
+      () => coordinator.runFactArchival([memA, memB]),
+      /content-hash removal exploded/,
+    );
+
+    // Both were archived on disk before the failure.
+    assert.deepEqual(archived, ["mem-a", "mem-b"]);
+
+    // Both facts were archived on disk AND queued for de-index — the queue push
+    // now precedes the throwing content-hash cleanup — so the finally-path flush
+    // must remove BOTH date buckets even though a later iteration threw. This
+    // fails on either ordering regression: move the flush out of `finally` and
+    // nothing is de-indexed (mem-a fails); move the queue push back after the
+    // throwing cleanup and mem-b is never queued (mem-b fails).
+    const afterDates = await temporalDates(memoryDir);
+    assert.equal(afterDates["2020-01-01"], undefined, "mem-a must be de-indexed via the finally flush");
+    assert.equal(afterDates["2020-02-02"], undefined, "mem-b must be de-indexed even though later cleanup threw");
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/orchestration/consolidation-run.ts

@@ -30,7 +30,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { log } from "../logger.js";
 import { applyCommitmentLedgerLifecycle } from "../commitment-ledger.js";
 import { recordDreamsPhaseRun } from "../maintenance/dreams-ledger.js";
-import { deindexMemoryAsync } from "../temporal-index.js";
+import { deindexMemoriesBatchAsync } from "../temporal-index-batch.js";
 import { isActiveMemoryStatus } from "../memory-lifecycle-ledger-utils.js";
 import {
   resolveConsolidationCapabilities,
@@ -153,80 +153,91 @@ export class ConsolidationRunCoordinator {
       ? new Map(allMemories.map((m) => [m.frontmatter.id, m]))
       : null;
 
-    for (const item of result.items) {
-      switch (item.action) {
-        case "INVALIDATE": {
-          // Capture path/frontmatter before invalidation for index cleanup
-          const toInvalidate = resolveIndexingCapabilities(config).queryAwareIndexing
-            ? (memoryLookup?.get(item.existingId) ?? null)
-            : null;
-          if (await storage.invalidateMemory(item.existingId)) {
-            invalidated += 1;
-            memoryItemMutated = true;
-            await this.deps.embeddingFallback.removeFromIndex(item.existingId);
-            if (toInvalidate?.path && toInvalidate.frontmatter?.created) {
-              await deindexMemoryAsync(
-                config.memoryDir,
-                toInvalidate.path,
-                toInvalidate.frontmatter.created,
-                toInvalidate.frontmatter.tags ?? [],
-              );
-            }
-          }
-          break;
-        }
-        case "UPDATE":
-          if (item.updatedContent) {
-            await storage.updateMemory(
-              item.existingId,
-              item.updatedContent,
-              {
-                lineage: [item.existingId],
-              },
-            );
-            memoryItemMutated = true;
-            await this.deps.indexPersistedMemory(storage, item.existingId);
-            // updateMemory() only changes content/updated/lineage — path, created, and tags
-            // are preserved, so the temporal/tag index entry is already correct; no reindex needed.
-          }
-          break;
-        case "MERGE":
-          if (item.updatedContent && item.mergeWith) {
-            await storage.updateMemory(
-              item.existingId,
-              item.updatedContent,
-              {
-                supersedes: item.mergeWith,
-                lineage: [item.existingId, item.mergeWith],
-              },
-            );
-            memoryItemMutated = true;
-            await this.deps.indexPersistedMemory(storage, item.existingId);
-            // updateMemory() only changes content/updated/supersedes/lineage — path, created, and tags
-            // are preserved, so the temporal/tag index entry for the survivor is already correct.
-            // Capture before invalidation for index cleanup
-            const toMergeInvalidate = resolveIndexingCapabilities(config).queryAwareIndexing
-              ? (memoryLookup?.get(item.mergeWith) ?? null)
+    // Collect deindex entries from INVALIDATE/MERGE actions and de-index them in
+    // one batch, instead of a full index read-modify-write per memory. The
+    // queryAwareIndexing guard is preserved: memoryLookup is null when it is
+    // disabled, so toInvalidate/toMergeInvalidate stay null and nothing is
+    // collected. The flush runs in `finally` so memories already invalidated on
+    // disk are still de-indexed if a later iteration throws; it runs exactly once
+    // (normal completion or throw) and any loop error still propagates after it.
+    const itemsDeindexBatch: Array<{ path: string; createdAt: string; tags: string[] }> = [];
+
+    try {
+      for (const item of result.items) {
+        switch (item.action) {
+          case "INVALIDATE": {
+            // Capture path/frontmatter before invalidation for index cleanup
+            const toInvalidate = resolveIndexingCapabilities(config).queryAwareIndexing
+              ? (memoryLookup?.get(item.existingId) ?? null)
               : null;
-            if (await storage.invalidateMemory(item.mergeWith)) {
+            if (await storage.invalidateMemory(item.existingId)) {
               invalidated += 1;
-              merged += 1;
-              await this.deps.embeddingFallback.removeFromIndex(item.mergeWith);
-              if (
-                toMergeInvalidate?.path &&
-                toMergeInvalidate.frontmatter?.created
-              ) {
-                await deindexMemoryAsync(
-                  config.memoryDir,
-                  toMergeInvalidate.path,
-                  toMergeInvalidate.frontmatter.created,
-                  toMergeInvalidate.frontmatter.tags ?? [],
-                );
+              memoryItemMutated = true;
+              await this.deps.embeddingFallback.removeFromIndex(item.existingId);
+              if (toInvalidate?.path && toInvalidate.frontmatter?.created) {
+                itemsDeindexBatch.push({
+                  path: toInvalidate.path,
+                  createdAt: toInvalidate.frontmatter.created,
+                  tags: toInvalidate.frontmatter.tags ?? [],
+                });
               }
             }
+            break;
           }
-          break;
+          case "UPDATE":
+            if (item.updatedContent) {
+              await storage.updateMemory(
+                item.existingId,
+                item.updatedContent,
+                {
+                  lineage: [item.existingId],
+                },
+              );
+              memoryItemMutated = true;
+              await this.deps.indexPersistedMemory(storage, item.existingId);
+              // updateMemory() only changes content/updated/lineage — path, created, and tags
+              // are preserved, so the temporal/tag index entry is already correct; no reindex needed.
+            }
+            break;
+          case "MERGE":
+            if (item.updatedContent && item.mergeWith) {
+              await storage.updateMemory(
+                item.existingId,
+                item.updatedContent,
+                {
+                  supersedes: item.mergeWith,
+                  lineage: [item.existingId, item.mergeWith],
+                },
+              );
+              memoryItemMutated = true;
+              await this.deps.indexPersistedMemory(storage, item.existingId);
+              // updateMemory() only changes content/updated/supersedes/lineage — path, created, and tags
+              // are preserved, so the temporal/tag index entry for the survivor is already correct.
+              // Capture before invalidation for index cleanup
+              const toMergeInvalidate = resolveIndexingCapabilities(config).queryAwareIndexing
+                ? (memoryLookup?.get(item.mergeWith) ?? null)
+                : null;
+              if (await storage.invalidateMemory(item.mergeWith)) {
+                invalidated += 1;
+                merged += 1;
+                await this.deps.embeddingFallback.removeFromIndex(item.mergeWith);
+                if (
+                  toMergeInvalidate?.path &&
+                  toMergeInvalidate.frontmatter?.created
+                ) {
+                  itemsDeindexBatch.push({
+                    path: toMergeInvalidate.path,
+                    createdAt: toMergeInvalidate.frontmatter.created,
+                    tags: toMergeInvalidate.frontmatter.tags ?? [],
+                  });
+                }
+              }
+            }
+            break;
+        }
       }
+    } finally {
+      await deindexMemoriesBatchAsync(config.memoryDir, itemsDeindexBatch);
     }
 
     if (result.profileUpdates.length > 0) {
@@ -292,14 +303,14 @@ export class ConsolidationRunCoordinator {
       memoryItemMutated = true;
       log.info(`cleaned ${deletedCommitments.length} expired commitments`);
       if (resolveIndexingCapabilities(config).queryAwareIndexing) {
-        for (const m of deletedCommitments) {
-          await deindexMemoryAsync(
-            config.memoryDir,
-            m.path,
-            m.frontmatter.created,
-            m.frontmatter.tags ?? [],
-          );
-        }
+        await deindexMemoriesBatchAsync(
+          config.memoryDir,
+          deletedCommitments.map((m) => ({
+            path: m.path,
+            createdAt: m.frontmatter.created,
+            tags: m.frontmatter.tags ?? [],
+          })),
+        );
       }
     }
 
@@ -335,14 +346,14 @@ export class ConsolidationRunCoordinator {
       memoryItemMutated = true;
       log.info(`cleaned ${deletedTTL.length} TTL-expired memories`);
       if (resolveIndexingCapabilities(config).queryAwareIndexing) {
-        for (const m of deletedTTL) {
-          await deindexMemoryAsync(
-            config.memoryDir,
-            m.path,
-            m.frontmatter.created,
-            m.frontmatter.tags ?? [],
-          );
-        }
+        await deindexMemoriesBatchAsync(
+          config.memoryDir,
+          deletedTTL.map((m) => ({
+            path: m.path,
+            createdAt: m.frontmatter.created,
+            tags: m.frontmatter.tags ?? [],
+          })),
+        );
       }
     }
 

package/src/orchestration/extraction-envelope.ts

@@ -0,0 +1,94 @@
+import {
+  composeMemoryEnvelope,
+  TAG_LIMITS,
+  type MemoryWriteInput,
+  type SealedMemoryEnvelope,
+  type WriteContext,
+} from "../write-envelope.js";
+import { normalizeTags } from "../recall-tag-filter.js";
+import { log } from "../logger.js";
+
+/**
+ * Envelope helpers for the extraction persistence pipeline (issue #1989 PR2).
+ *
+ * Extraction input is MACHINE-GENERATED: one malformed LLM field must never
+ * abort a whole persistence batch that legacy `writeMemory` would have
+ * accepted. Every extraction-side compose therefore runs in salvage mode,
+ * and every drop is warn-logged (rule 34 — visible, never silent).
+ */
+
+/**
+ * Compose an extraction-side envelope in salvage mode and warn-log any
+ * salvage notes. The single compose wrapper for all extraction-persist
+ * write sites, so note logging cannot be forgotten at a new site.
+ */
+export function composeSalvagedExtractionEnvelope(
+  input: MemoryWriteInput,
+  ctx: WriteContext,
+): SealedMemoryEnvelope {
+  const envelope = composeMemoryEnvelope(input, ctx, { salvage: true });
+  if (envelope.salvageNotes.length > 0) {
+    log.warn(`extraction write salvaged invalid fields: ${envelope.salvageNotes.join("; ")}`);
+  }
+  return envelope;
+}
+
+/**
+ * System marker tags ("shared-promotion", "<target>-promotion", "chunked")
+ * must survive the tag cap — the CLI and stats identify chunked parents and
+ * promotions through them (#2014/#2017 review round). Normalize the source
+ * tags first (same trim/dedupe salvage would apply), reserve one slot for
+ * the marker, and warn when source tags are dropped to make room.
+ */
+export function withReservedMarkerTag(sourceTags: string[], marker: string): string[] {
+  // Drop overlong tags BEFORE budgeting (round 3): normalizeTags only
+  // trims/dedupes, so a >256-char tag would consume a budget slot here and
+  // then be salvage-dropped at compose time — squeezing out a valid later
+  // tag for nothing.
+  const usable = sourceTags.filter(
+    (tag) => typeof tag !== "string" || tag.trim().length <= TAG_LIMITS.maxTagLength,
+  );
+  const normalized = normalizeTags(usable) ?? [];
+  const budget = TAG_LIMITS.maxTags - 1;
+  if (normalized.length > budget) {
+    log.warn(
+      `extraction tags exceed ${budget}+marker cap; keeping the first ${budget} of ${normalized.length} and the ${JSON.stringify(marker)} marker`,
+    );
+  }
+  const capped = normalized.length > budget ? normalized.slice(0, budget) : normalized;
+  return [...capped, marker];
+}
+
+/**
+ * Pure salvage-compose probe: reproduce the SURVIVING entityRef and raw
+ * attribute pairs a salvage write of these fields would persist
+ * (deterministic, side-effect-free). Returns null when strict envelope
+ * preconditions reject (such input never reached a salvage write either) —
+ * callers fail open to their raw fields.
+ */
+export function probeSalvageSurvivingFields(input: {
+  content: string;
+  category: MemoryWriteInput["category"];
+  structuredAttributes?: Record<string, string>;
+  entityRef?: string;
+}): { entityRef?: string; structuredAttributes?: Record<string, string> } | null {
+  try {
+    const probe = composeSalvagedExtractionEnvelope(
+      {
+        content: input.content,
+        category: input.category,
+        structuredAttributes: input.structuredAttributes,
+        entityRef: input.entityRef,
+      },
+      { source: "bitemporal-backfill-probe" },
+    );
+    return {
+      entityRef: probe.entityRef,
+      structuredAttributes: probe.rawStructuredAttributes
+        ? { ...probe.rawStructuredAttributes }
+        : undefined,
+    };
+  } catch {
+    return null;
+  }
+}

package/src/orchestration/extraction-persist-promotion.ts

@@ -0,0 +1,134 @@
+/**
+ * Auto-promotion gating helpers for the extraction-persist pipeline
+ * (extracted from extraction-persist.ts to keep that file under the
+ * structural line-count ceiling; issue #1909 / seam 16).
+ *
+ * Pure decision helpers that answer "may this fact be auto-promoted?" for
+ * both scope-profile targets and the server-shared namespace. Behavior is a
+ * verbatim move from the closures that previously lived inside
+ * `persistExtraction`; `this.deps.*` state is threaded in as explicit
+ * parameters so the functions stay free of coordinator instance state.
+ */
+
+import { StorageManager } from "../index.js";
+import { log } from "../logger.js";
+import { confidenceTier } from "../types.js";
+import {
+  resolveNamespaceCapabilities,
+  resolveRecallEnhancementCapabilities,
+} from "../capabilities.js";
+import type { MemoryFile, PluginConfig } from "../types.js";
+import type { ResolvedScopeProfilePlan } from "../namespaces/scope-profiles.js";
+
+export const confidenceTierOrder = [
+  "explicit",
+  "implied",
+  "inferred",
+  "speculative",
+] as const;
+
+export function profileAutoPromotionAllows(
+  scopeProfileWritePlan: ResolvedScopeProfilePlan | null | undefined,
+  category: string,
+  confidence: number,
+): boolean {
+  if (!scopeProfileWritePlan) return false;
+  const actualTier = confidenceTier(confidence);
+  const actualRank = confidenceTierOrder.indexOf(actualTier);
+  if (actualRank === -1) return false;
+  const autoPromote = scopeProfileWritePlan.profile.autoPromote;
+  if (!autoPromote.enabled) return false;
+  if (!autoPromote.categories.includes(category as any)) return false;
+  const minimumRank = confidenceTierOrder.indexOf(autoPromote.minConfidenceTier);
+  return minimumRank !== -1 && actualRank <= minimumRank;
+}
+
+export function sharedAutoPromotionAllows(
+  config: PluginConfig,
+  scopeProfileWritePlan: ResolvedScopeProfilePlan | null | undefined,
+  category: string,
+  confidence: number,
+): boolean {
+  if (!scopeProfileWritePlan) {
+    const actualTier = confidenceTier(confidence);
+    const actualRank = confidenceTierOrder.indexOf(actualTier);
+    if (actualRank === -1) return false;
+    if (!resolveRecallEnhancementCapabilities(config).autoPromoteToShared) return false;
+    if (!config.autoPromoteToSharedCategories.includes(category as any))
+      return false;
+    const minimumRank = confidenceTierOrder.indexOf(
+      config.autoPromoteMinConfidenceTier,
+    );
+    return minimumRank !== -1 && actualRank <= minimumRank;
+  }
+  return (
+    scopeProfileWritePlan.profile.autoPromote.targets.includes("serverShared") &&
+    profileAutoPromotionAllows(scopeProfileWritePlan, category, confidence)
+  );
+}
+
+export function shouldPromoteToShared(
+  config: PluginConfig,
+  scopeProfileWritePlan: ResolvedScopeProfilePlan | null | undefined,
+  profileAllowsSharedWrites: boolean,
+  storageDirNamespace: (storageDir: string) => string,
+  targetStorage: StorageManager,
+  category: string,
+  confidence: number,
+): boolean {
+  if (
+    !resolveNamespaceCapabilities(config).namespaces ||
+    !profileAllowsSharedWrites ||
+    !sharedAutoPromotionAllows(config, scopeProfileWritePlan, category, confidence)
+  )
+    return false;
+  if (
+    storageDirNamespace(targetStorage.dir) ===
+    config.sharedNamespace
+  )
+    return false;
+  return true;
+}
+
+/**
+ * Read the active-copy corpus across BOTH storage tiers (hot + cold).
+ *
+ * #2016 cold-tier finding: the authoritative content-hash rebuild unions the
+ * hot and cold tiers, so `hasFactContentHash()` can report a hit for a
+ * fact/procedure whose only active copy was demoted to `cold/`. A promotion or
+ * dedup confirmation scan that reads `readAllMemories()` (hot) alone misses
+ * that copy, so it either writes a duplicate hot copy or skips a needed
+ * temporal backfill. Scanning both tiers keeps the confirmation coherent with
+ * the hash index. Cold reads are folded in only when the cold tier is
+ * non-empty so hot-only namespaces incur no extra allocation.
+ */
+export async function readActiveMemoriesBothTiers(
+  storage: StorageManager,
+): Promise<MemoryFile[]> {
+  const [hotMems, coldMems] = await Promise.all([
+    storage.readAllMemories(),
+    storage.readAllColdMemories(),
+  ]);
+  return coldMems.length === 0 ? hotMems : [...hotMems, ...coldMems];
+}
+
+/**
+ * PR #2016: after `writeSealedMemory` has made a fact `.md` durable with its
+ * fact-hash flush deferred (#1909 batching), a throw in the fallible post-write
+ * work (contradiction resolve, indexing, shared promotion, artifact write)
+ * would strand the deferred parent hash — the shared fact-hash index on disk
+ * would be missing the parent until the next corpus rebuild, letting a peer
+ * re-extract a duplicate. Flush the batch save before the error propagates so
+ * the durable-hash invariant holds on every path. No-op when dedup is off
+ * (`deferred=false` → the write already flushed and the batch saver is a no-op).
+ * Best-effort: a flush failure is logged and never masks the original error.
+ */
+export async function flushDeferredFactHashOnFailure(
+  saveContentHashIndexes: () => Promise<void>,
+  deferred: boolean,
+): Promise<void> {
+  if (!deferred) return;
+  await saveContentHashIndexes().catch((err) =>
+    log.warn(`content-hash flush after post-write failure failed: ${err}`),
+  );
+}