@remnic/core 9.3.699 → 9.3.700

package/src/maintenance/forget.ts

@@ -22,6 +22,8 @@
  */
 
 import type { StorageManager } from "../storage.js";
+import { buildRetiredFactTombstoneInputs } from "../lifecycle/tombstones.js";
+import { supersessionKeysForFact } from "../temporal-supersession.js";
 import type { MemoryFile } from "../types.js";
 
 export interface ForgetMemoryRequest {
@@ -100,6 +102,45 @@ export async function forgetMemory(
     actor: "remnic-forget",
     reasonCode: "operator_forget",
   });
+  // Issue #1579 thread OchiF: emit a LIVE tombstone at forget time so a
+  // forgotten FACT cannot resurrect via re-extraction/import before an
+  // operator runs `remnic doctor --rebuild-tombstones`. Without this, the
+  // non-resurrection chokepoint in writeMemory has nothing to match — the
+  // forgotten fact's content hash is gone from the active index but no
+  // tombstone exists until a manual rebuild derives one — so the same fact
+  // written again through writeMemory becomes active. Mirrors supersedeMemory
+  // (contradiction) and applyTemporalSupersession via the shared
+  // buildRetiredFactTombstoneInputs helper. Best-effort (rule 34): a tombstone
+  // append failure must not fail the forget, which already succeeded on disk.
+  if (memory.frontmatter.category === "fact") {
+    for (const input of buildRetiredFactTombstoneInputs(
+      {
+        id,
+        content: memory.content,
+        ...(memory.frontmatter.contentHash
+          ? { contentHash: memory.frontmatter.contentHash }
+          : {}),
+        ...(memory.frontmatter.entityRef
+          ? { entityRef: memory.frontmatter.entityRef }
+          : {}),
+        ...(memory.frontmatter.structuredAttributes
+          ? { structuredAttributes: memory.frontmatter.structuredAttributes }
+          : {}),
+      },
+      {
+        reason: "retraction",
+        createdBy: "user_correction",
+        createdAt: forgottenAt,
+        supersessionKeysForFact,
+      },
+    )) {
+      try {
+        await storage.appendTombstone(input);
+      } catch {
+        // Best-effort — the forget already succeeded on disk.
+      }
+    }
+  }
   return {
     id,
     path: memory.path,

package/src/maintenance/pattern-reinforcement.ts

@@ -33,6 +33,8 @@
  */
 
 import { clusterByKey } from "../procedural/reinforcement-core.js";
+import { buildRetiredFactTombstoneInputs, type TombstoneReason, type TombstoneCreatedBy } from "../lifecycle/tombstones.js";
+import { supersessionKeysForFact } from "../temporal-supersession.js";
 import type {
   MemoryFile,
   MemoryFrontmatter,
@@ -50,6 +52,21 @@ export interface PatternReinforcementStorage {
     memory: MemoryFile,
     patch: Partial<MemoryFrontmatter>,
   ): Promise<boolean>;
+  /**
+   * Best-effort tombstone append for retired duplicates (issue #1579 thread
+   * Oblq9). Optional so unit-test stubs need not implement it; the production
+   * StorageManager satisfies it via its appendTombstone chokepoint.
+   */
+  appendTombstone?(input: {
+    reason: TombstoneReason;
+    createdBy: TombstoneCreatedBy;
+    sourceMemoryId: string;
+    contentHash?: string;
+    rawContent: string;
+    entityRef?: string;
+    supersessionKey?: string;
+    createdAt: string;
+  }): Promise<string | null>;
 }
 
 export interface PatternReinforcementOptions {
@@ -345,6 +362,36 @@ export async function runPatternReinforcement(
       await storage.writeMemoryFrontmatter(dup, patch);
       supersededIds.push(dup.frontmatter.id);
       result.duplicatesSuperseded += 1;
+      // Issue #1579 thread Oblq9 / Ocn0C / OcoPs: emit a tombstone so the
+      // retired duplicate cannot resurrect through re-extraction / import.
+      // Pattern reinforcement retires via writeMemoryFrontmatter which
+      // bypasses the writeMemory chokepoint; without this emit, the same
+      // content re-extracted before a manual rebuild would be stored active.
+      // Threads Ocn0C/OcoPs: emit one tombstone PER derived supersession key
+      // via buildRetiredFactTombstoneInputs (same helper as supersedeMemory +
+      // forgetMemory), not a single entityRef-only record — a paraphrased
+      // re-extraction with the same entity/attribute can only be caught by
+      // the keyed tier. Best-effort (the interface method is optional; a
+      // failure must not fail the reinforcement job).
+      if (storage.appendTombstone) {
+        for (const input of buildRetiredFactTombstoneInputs(
+          {
+            id: dup.frontmatter.id,
+            content: dup.content,
+            contentHash: dup.frontmatter.contentHash,
+            entityRef: dup.frontmatter.entityRef,
+            structuredAttributes: dup.frontmatter.structuredAttributes,
+          },
+          { reason: "supersession", createdBy: "supersession", createdAt: nowIso, supersessionKeysForFact },
+        )) {
+          try {
+            await storage.appendTombstone(input);
+          } catch {
+            // Best-effort: a tombstone append failure must not fail the
+            // reinforcement job (gotcha #13 / rule 34 spirit).
+          }
+        }
+      }
     }
 
     result.clusters.push({

package/src/memory-provenance.ts

@@ -328,6 +328,18 @@ function inferSafetyReasons(input: {
   if (frontmatter.verificationState === "disputed") {
     reasons.push("verification=disputed");
   }
+  // Issue #1579 — x-ray visibility (rule 34: never a silent drop). When a
+  // memory carries `blockedBy`, it was tombstone-blocked at the storage
+  // chokepoint. Surface the tombstone id (and the matched tier when known)
+  // in the safety reasons so every recall x-ray consumer (CLI / HTTP / MCP)
+  // can attribute the pending_review state to a tombstone rather than a
+  // generic low-confidence review.
+  if (typeof frontmatter.blockedBy === "string" && frontmatter.blockedBy.length > 0) {
+    reasons.push(`tombstone-blocked=${frontmatter.blockedBy}`);
+    if (frontmatter.tombstoneBlockTier) {
+      reasons.push(`tombstone-tier=${frontmatter.tombstoneBlockTier}`);
+    }
+  }
   if (stale) reasons.push("stale=true");
 
   const boundaryScopes = userContextScopes.filter(isUserBoundaryScope);

package/src/namespaces/storage.ts

@@ -297,6 +297,9 @@ export class NamespaceStorageRouter {
     // #1522: install the post-write catalog touch at the chokepoint — every
     // successful write on this StorageManager records the namespace touch.
     this.bindCatalogWriteHook(sm, ns);
+    // #1579: apply the tombstone non-resurrection config so every namespace
+    // storage enforces the invariant at its own writeMemory chokepoint.
+    this.applyTombstonesConfig(sm, ns);
     this.cache.set(ns, sm);
     this.notifyResolved(ns, root);
     return sm;
@@ -387,6 +390,36 @@ export class NamespaceStorageRouter {
     sm.onCatalogWrite = () => this.touchCatalogWrite(namespace, sm.dir);
   }
 
+  /**
+   * Install the tombstone config on an externally-constructed StorageManager
+   * (issue #1579). Mirrors `bindCatalogWriteHook` — used for the legacy
+   * default-namespace storage that bypasses the router. Router-created
+   * storages are wired inline in `storageFor()` via `applyTombstonesConfig`.
+   */
+  bindTombstonesConfig(
+    sm: StorageManager,
+    namespace: string,
+    config: { enabled: boolean; semanticMatch: boolean; semanticThreshold: number },
+  ): void {
+    this.tombstonesGlobalConfig = { ...config };
+    sm.setTombstonesConfig({ ...config, namespace });
+  }
+
+  private tombstonesGlobalConfig: {
+    enabled: boolean;
+    semanticMatch: boolean;
+    semanticThreshold: number;
+  } = { enabled: false, semanticMatch: false, semanticThreshold: 0.9 };
+
+  /**
+   * Apply the tombstone config to a router-created StorageManager. Called
+   * inline in `storageFor()` so every namespace storage enforces the
+   * non-resurrection invariant, namespace-scoped (rule 42).
+   */
+  private applyTombstonesConfig(sm: StorageManager, namespace: string): void {
+    sm.setTombstonesConfig({ ...this.tombstonesGlobalConfig, namespace });
+  }
+
   /**
    * Post-write catalog touch (issue #1522 chokepoint). Called by every
    * StorageManager's post-write hook AFTER a successful write. Best-effort

package/src/operator-toolkit.ts

@@ -1326,6 +1326,12 @@ export async function runOperatorDoctor(options: OperatorDoctorOptions): Promise
     details: namespaceMaintenanceHealth,
   });
 
+  // Tombstone non-resurrection invariant (issue #1579). Reports active
+  // tombstone count, last append, corrupted-line count, and rebuild
+  // staleness so operators can verify the invariant is enforced and know
+  // when a rebuild is warranted. Informational: never errors on its own.
+  checks.push(await summarizeTombstoneStatus(options.orchestrator.storage));
+
   const summary = checks.reduce(
     (acc, check) => {
       acc[check.status] += 1;
@@ -1871,6 +1877,77 @@ export async function summarizeObservationThroughput(
   }
 }
 
+/**
+ * Tombstone non-resurrection invariant status for `remnic doctor`
+ * (issue #1579).
+ *
+ * Reports:
+ *   - active tombstone count (excluding revoked)
+ *   - revoked count (re-allowed via the review queue)
+ *   - last append timestamp
+ *   - corrupted-line count (skipped, not crashed — rule 34/18)
+ *   - rebuild staleness (whether the in-memory index loaded cleanly)
+ *
+ * Always informational: a disabled or empty tombstone store is the expected
+ * cold-install state and is never an error. A non-zero corrupted-line count
+ * surfaces as `warn` so operators know a rebuild (`remnic doctor
+ * --rebuild-tombstones`) will repair the log.
+ */
+export async function summarizeTombstoneStatus(
+  storage: StorageManager,
+): Promise<OperatorDoctorCheck> {
+  try {
+    const stats = await storage.getTombstoneStats();
+    if (stats === null) {
+      return {
+        key: "tombstones",
+        status: "ok",
+        summary: "Tombstone non-resurrection invariant is disabled (tombstonesEnabled=false).",
+        details: { enabled: false },
+      };
+    }
+    const corrupted = stats.corruptedLines > 0;
+    const parts = [
+      `${stats.count} active tombstone(s)`,
+      ...(stats.revoked > 0 ? [`${stats.revoked} revoked`] : []),
+    ];
+    if (stats.lastAppendAt) {
+      parts.push(`last append ${stats.lastAppendAt}`);
+    }
+    if (corrupted) {
+      parts.push(`${stats.corruptedLines} corrupted line(s) skipped`);
+    }
+    return {
+      key: "tombstones",
+      status: corrupted ? "warn" : "ok",
+      summary:
+        `Tombstone invariant enabled. ${parts.join(", ")}. ` +
+        (corrupted
+          ? "Run `remnic doctor --rebuild-tombstones` to repair the log."
+          : "Re-observation of retired facts is blocked at the storage chokepoint."),
+      remediation: corrupted
+        ? "Rebuild the tombstone log from retired memories on disk."
+        : undefined,
+      details: {
+        enabled: true,
+        count: stats.count,
+        revoked: stats.revoked,
+        lastAppendAt: stats.lastAppendAt,
+        corruptedLines: stats.corruptedLines,
+        loaded: stats.loaded,
+      },
+    };
+  } catch (err) {
+    return {
+      key: "tombstones",
+      status: "warn",
+      summary: "Could not read tombstone store stats.",
+      remediation: "Ensure the memory state directory is readable and rerun `remnic doctor`.",
+      details: { error: String(err) },
+    };
+  }
+}
+
 export async function summarizeConsolidationProvenance(
   storage: StorageManager,
   config: Pick<PluginConfig, "memoryDir"> & { versioningSidecarDir?: string },

package/src/orchestrator.ts

@@ -2728,6 +2728,22 @@ export class Orchestrator {
       maxVersionsPerPage: config.versioningMaxPerPage,
       sidecarDir: config.versioningSidecarDir,
     });
+    // Wire the tombstone non-resurrection invariant (issue #1579) on the
+    // primary default-namespace storage. Router-created namespace storages
+    // get the same config via NamespaceStorageRouter.bindTombstonesConfig
+    // (installed below) so every write path funnels through the chokepoint.
+    const { tombstonesEnabled, tombstonesSemanticMatch, tombstonesSemanticThreshold } = config;
+    this.storage.setTombstonesConfig({
+      enabled: tombstonesEnabled,
+      semanticMatch: tombstonesSemanticMatch,
+      semanticThreshold: tombstonesSemanticThreshold,
+      namespace: config.defaultNamespace,
+    });
+    this.storageRouter.bindTombstonesConfig(this.storage, config.defaultNamespace, {
+      enabled: tombstonesEnabled,
+      semanticMatch: tombstonesSemanticMatch,
+      semanticThreshold: tombstonesSemanticThreshold,
+    });
     // Wire at-rest encryption (issue #690 PR 3/4).
     // If secureStoreEnabled, check whether the keyring already holds a key
     // for this memory dir (e.g. operator unlocked before daemon restart).

package/src/review/index.ts

@@ -29,7 +29,7 @@ export interface ReviewItem {
   /** Created date */
   created: string;
   /** Reason it's in review */
-  reviewReason: "low_confidence" | "suggestion" | "contradiction" | "duplicate";
+  reviewReason: "low_confidence" | "suggestion" | "contradiction" | "duplicate" | "tombstone_blocked";
   /** Additional context */
   context?: string;
 }
@@ -45,6 +45,8 @@ export interface ReviewResult {
   updatedPath?: string;
   /** Status message */
   message: string;
+  /** Tombstone id cleared by an approve action (for the caller to revoke). */
+  clearedTombstoneId?: string;
 }
 
 export interface ReviewListResult {
@@ -70,6 +72,15 @@ export interface ReviewOptions {
 export interface ReviewActionOptions {
   /** Match the threshold used when listing review items (default: 0.7) */
   confidenceThreshold?: number;
+  /**
+   * Revocation hook (issue #1579). When approving a memory whose frontmatter
+   * carries `blockedBy: <tombstoneId>`, the hook fires so the caller (CLI /
+   * orchestrator) can append a `kind: "revocation"` tombstone entry —
+   * re-allowing the content. Fire-and-forget: a revocation failure MUST NOT
+   * fail the approval (gotcha #13). The hook receives the tombstone id and
+   * the memory id.
+   */
+  onApproveBlockedMemory?: (tombstoneId: string, memoryId: string) => void | Promise<void>;
 }
 
 interface ReviewFileMatch {
@@ -211,7 +222,7 @@ export function listReviewItems(options: ReviewOptions): ReviewListResult {
       if (!fm?.id) return false;
 
       const confidence = parseConfidence(fm.confidence, 1);
-      if (confidence >= confidenceThreshold) return false;
+      if (confidence >= confidenceThreshold && !(typeof fm.blockedBy === "string" && fm.blockedBy.length > 0)) return false;
       if (parseBoolean(fm.reviewDismissed)) return false;
 
       // Skip if already in items
@@ -226,7 +237,7 @@ export function listReviewItems(options: ReviewOptions): ReviewListResult {
         source: (fm.source as string) ?? "unknown",
         filePath,
         created: (fm.created as string) ?? new Date().toISOString(),
-        reviewReason: "low_confidence",
+        reviewReason: (typeof fm.blockedBy === "string" && fm.blockedBy.length > 0) ? "tombstone_blocked" : "low_confidence",
       });
       return isLimitReached();
     });
@@ -276,10 +287,32 @@ function approveItem(
   const fm = parseFrontmatter(content);
   if (!fm) return { itemId, action: "approve", message: "Could not parse frontmatter" };
 
+  // Issue #1579 — when approving a tombstone-blocked memory, fire the
+  // revocation hook so the content is re-allowed (append-only log: a
+  // `kind: "revocation"` entry supersedes the tombstone at lookup). Clear
+  // blockedBy/tombstoneBlockTier on the promoted memory so it is fully
+  // active. Fire-and-forget: a hook failure never fails the approval.
+  const blockedBy = typeof fm.blockedBy === "string" ? fm.blockedBy : null;
+  if (blockedBy && options.onApproveBlockedMemory) {
+    try {
+      const maybe = options.onApproveBlockedMemory(blockedBy, itemId);
+      if (maybe && typeof (maybe as Promise<void>).then === "function") {
+        void (maybe as Promise<void>).catch(() => undefined);
+      }
+    } catch {
+      /* fire-and-forget — never fail the approval (gotcha #13) */
+    }
+  }
+
   const updatedContent = updateFrontmatterFields(content, {
     confidence: "0.9",
     confidenceTier: "high",
     reviewDismissed: null,
+    // Clear the tombstone-block markers so the promoted memory is active.
+    // Clear the tombstone-block markers AND restore active status so the
+    // promoted memory is fully active in recall + provenance (issue #1579
+    // review: leaving status: pending_review kept it invisible to recall).
+    ...(blockedBy ? { blockedBy: null, tombstoneBlockTier: null, status: "active" } : {}),
   });
 
   if (found.location === "category") {
@@ -289,6 +322,7 @@ function approveItem(
       action: "approve",
       updatedPath: found.filePath,
       message: "Approved low-confidence memory in place with confidence 0.9",
+      ...(blockedBy ? { clearedTombstoneId: blockedBy } : {}),
     };
   }
 
@@ -310,6 +344,7 @@ function approveItem(
     action: "approve",
     updatedPath: promotedPath,
     message: `Promoted to ${category} with confidence 0.9`,
+    ...(blockedBy ? { clearedTombstoneId: blockedBy } : {}),
   };
 }
 
@@ -418,6 +453,16 @@ function isLowConfidenceReviewCandidate(
   options: ReviewActionOptions,
 ): boolean {
   const threshold = options.confidenceThreshold ?? DEFAULT_CONFIDENCE_THRESHOLD;
+  // Issue #1579: tombstone-blocked facts are ALWAYS review candidates,
+  // regardless of confidence. The block sets status: pending_review +
+  // blockedBy; these must be visible in the review queue so an operator
+  // can approve (revoke the tombstone) or dismiss them. Without this, a
+  // high-confidence blocked fact is visible on disk but absent from the
+  // review UI — a silent drop, which violates the non-resurrection design
+  // (rule 34: visible, not silent).
+  if (typeof fm.blockedBy === "string" && fm.blockedBy.length > 0) {
+    return !parseBoolean(fm.reviewDismissed);
+  }
   return (
     parseConfidence(fm.confidence, 1) < threshold &&
     !parseBoolean(fm.reviewDismissed)