@remnic/core 9.49.0 → 9.50.0

package/src/orchestration/dependency-propagation.ts

@@ -0,0 +1,263 @@
+/**
+ * Bounded, one-hop invalidation propagation for linked memories.
+ *
+ * Propagation hooks stay at orchestration call sites. They never live inside
+ * StorageManager, so propagation supersessions cannot recurse.
+ */
+import type { ExtractionEngine } from "../extraction.js";
+import { log } from "../logger.js";
+import type { StorageManager } from "../index.js";
+import type {
+  DependencyPropagationConfig,
+  MemoryFile,
+  MemoryFrontmatter,
+  MemoryLinkType,
+  PluginConfig,
+} from "../types.js";
+
+export interface PropagationEvent {
+  /** Snapshot captured before the primary supersession or deletion. */
+  oldMemory: { content: string; frontmatter: MemoryFrontmatter };
+  replacementId: string | null;
+  replacementContent: string | null;
+  cause:
+    | "contradiction"
+    | "temporal_supersession"
+    | "consolidation_invalidate"
+    | "consolidation_merge";
+  namespaceScope: string;
+}
+
+export interface PropagationResult {
+  dependentsFound: number;
+  invalidated: number;
+  stillValid: number;
+  uncertain: number;
+  skipped: "disabled" | "no_dependents" | "llm_error" | "timeout" | null;
+}
+
+function propagationConfig(config: PluginConfig): DependencyPropagationConfig {
+  return config.dependencyPropagation;
+}
+
+function isEnabled(value: unknown): boolean {
+  if (value === true) return true;
+  if (typeof value !== "string") return false;
+  return ["true", "1", "yes", "on"].includes(value.trim().toLowerCase());
+}
+
+const ACTIVE_STATUS = "active";
+
+/**
+ * Discover exactly one-hop dependents from a memory snapshot.
+ *
+ * A supports edge on the dying memory points forward to its dependent.
+ * follows and configured references edges point back to the dying memory.
+ */
+export function findDependents(
+  memories: MemoryFile[],
+  oldMemory: Pick<PropagationEvent["oldMemory"], "frontmatter">,
+  linkTypes: readonly MemoryLinkType[] = ["supports", "follows"],
+): MemoryFile[] {
+  const allowed = new Set(linkTypes);
+  const ids = new Set<string>();
+  const byId = new Map(memories.map((memory) => [memory.frontmatter.id, memory]));
+
+  // Forward: supports and references edges on the dying memory point to
+  // dependents that lose evidence or a referenced anchor.
+  for (const link of oldMemory.frontmatter.links ?? []) {
+    if (link.linkType !== "supports" && link.linkType !== "references") continue;
+    if (!allowed.has(link.linkType)) continue;
+    const dependent = byId.get(link.targetId);
+    if (dependent && (dependent.frontmatter.status ?? ACTIVE_STATUS) === ACTIVE_STATUS) {
+      ids.add(dependent.frontmatter.id);
+    }
+  }
+
+  const oldId = oldMemory.frontmatter.id;
+  for (const memory of memories) {
+    if ((memory.frontmatter.status ?? ACTIVE_STATUS) !== ACTIVE_STATUS) continue;
+    if (memory.frontmatter.id === oldId) continue;
+    for (const link of memory.frontmatter.links ?? []) {
+      if (link.targetId !== oldId) continue;
+      if (link.linkType !== "follows" && link.linkType !== "references") continue;
+      if (!allowed.has(link.linkType)) continue;
+      ids.add(memory.frontmatter.id);
+    }
+  }
+
+  return Array.from(ids)
+    .map((id) => byId.get(id))
+    .filter((memory): memory is MemoryFile => memory !== undefined)
+    .sort((a, b) => {
+      const left = a.frontmatter.id;
+      const right = b.frontmatter.id;
+      return left < right ? -1 : left > right ? 1 : 0;
+    });
+}
+
+function emptyResult(skipped: PropagationResult["skipped"]): PropagationResult {
+  return { dependentsFound: 0, invalidated: 0, stillValid: 0, uncertain: 0, skipped };
+}
+
+function logResult(event: PropagationEvent, result: PropagationResult): void {
+  const oldId = event.oldMemory.frontmatter.id;
+  log.info(
+    `dependency-propagation ${JSON.stringify({
+      cause: event.cause,
+      oldId,
+      dependentsFound: result.dependentsFound,
+      invalidated: result.invalidated,
+      stillValid: result.stillValid,
+      uncertain: result.uncertain,
+      skipped: result.skipped,
+    })}`,
+  );
+}
+
+class PropagationTimeoutError extends Error {
+  constructor() {
+    super("dependency propagation timed out");
+    this.name = "PropagationTimeoutError";
+  }
+}
+
+function isTimeoutError(error: unknown, signal: AbortSignal): boolean {
+  return signal.aborted || error instanceof PropagationTimeoutError ||
+    (error instanceof Error && (error.name === "AbortError" || /timeout|timed out/i.test(error.message)));
+}
+
+/**
+ * Revalidate one-hop dependents after a successful primary operation.
+ * The caller's storage is already namespace-scoped; this function never
+ * constructs another storage or scans a different namespace.
+ */
+export async function propagateInvalidation(
+  deps: { storage: StorageManager; extraction: ExtractionEngine; config: PluginConfig },
+  event: PropagationEvent,
+): Promise<PropagationResult> {
+  const config = propagationConfig(deps.config);
+  if (!isEnabled(config.enabled)) {
+    const result = emptyResult("disabled");
+    logResult(event, result);
+    return result;
+  }
+
+  const maxDependents = config.maxDependents;
+  if (maxDependents <= 0) {
+    const result = emptyResult("no_dependents");
+    logResult(event, result);
+    return result;
+  }
+
+  let memories: MemoryFile[];
+  try {
+    memories = await deps.storage.readAllMemories();
+  } catch (error) {
+    log.warn(`dependency propagation discovery failed for ${event.oldMemory.frontmatter.id}: ${error}`);
+    const result = emptyResult("llm_error");
+    logResult(event, result);
+    return result;
+  }
+  // Cold-tier active dependents are not scanned in v1: the supersede and
+  // frontmatter write paths operate on hot storage. Cold discovery without
+  // cold write support would surface dependents it cannot act on.
+
+  const discovered = findDependents(
+    memories,
+    event.oldMemory,
+    config.linkTypes,
+  ).filter((memory) => memory.frontmatter.id !== event.replacementId);
+  const dependents = discovered.slice(0, Math.max(0, Math.floor(maxDependents)));
+
+  if (dependents.length === 0) {
+    const result = emptyResult("no_dependents");
+    logResult(event, result);
+    return result;
+  }
+
+  const controller = new AbortController();
+  const timeoutMs = config.timeoutMs;
+  const timeout = Number.isFinite(timeoutMs) && timeoutMs > 0
+    ? setTimeout(() => controller.abort(new PropagationTimeoutError()), timeoutMs)
+    : undefined;
+
+  let verdicts: Array<{ memoryId: string; verdict: "still_valid" | "invalidated" | "uncertain"; reason?: string }>;
+  try {
+    const response = await deps.extraction.revalidateDependents(
+      { id: event.oldMemory.frontmatter.id, content: event.oldMemory.content },
+      event.replacementId && event.replacementContent !== null
+        ? { id: event.replacementId, content: event.replacementContent }
+        : null,
+      dependents.map((memory) => ({
+        id: memory.frontmatter.id,
+        category: memory.frontmatter.category,
+        content: memory.content,
+      })),
+      controller.signal,
+    );
+    verdicts = response.verdicts;
+    if (controller.signal.aborted) {
+      if (timeout !== undefined) clearTimeout(timeout);
+      const result = emptyResult("timeout");
+      logResult(event, result);
+      return result;
+    }
+  } catch (error) {
+    const result = emptyResult(isTimeoutError(error, controller.signal) ? "timeout" : "llm_error");
+    log.warn(`dependency propagation revalidation failed for ${event.oldMemory.frontmatter.id}: ${error}`);
+    logResult(event, result);
+    if (timeout !== undefined) clearTimeout(timeout);
+    return result;
+  }
+  if (timeout !== undefined) clearTimeout(timeout);
+
+  let invalidated = 0;
+  let stillValid = 0;
+  let uncertain = 0;
+  for (const dependent of dependents) {
+    const verdict = verdicts.find((candidate) => candidate.memoryId === dependent.frontmatter.id);
+    if (verdict?.verdict === "invalidated") {
+      if (config.dryRun) {
+        invalidated += 1;
+        continue;
+      }
+      try {
+        const replacementId = event.replacementId ?? event.oldMemory.frontmatter.id;
+        const superseded = await deps.storage.supersedeMemory(
+          dependent.frontmatter.id,
+          replacementId,
+          `dependency_propagation:${event.cause}`,
+        );
+        if (!superseded) continue;
+        const current = await deps.storage.getMemoryById(dependent.frontmatter.id);
+        if (current) {
+          await deps.storage.writeMemoryFrontmatter(current, {
+            supersessionCause: "dependency",
+            invalidatedBy: event.oldMemory.frontmatter.id,
+          });
+        }
+        invalidated += 1;
+      } catch (error) {
+        log.warn(`dependency propagation write failed for ${dependent.frontmatter.id}: ${error}`);
+      }
+      continue;
+    }
+    if (verdict?.verdict === "still_valid") {
+      stillValid += 1;
+    } else {
+      uncertain += 1;
+      log.info(`dependency propagation uncertain for ${dependent.frontmatter.id}`);
+    }
+  }
+
+  const result: PropagationResult = {
+    dependentsFound: dependents.length,
+    invalidated,
+    stillValid,
+    uncertain,
+    skipped: null,
+  };
+  logResult(event, result);
+  return result;
+}

package/src/orchestration/dependency-revalidation.ts

@@ -0,0 +1,173 @@
+/**
+ * LLM revalidation of dependent memories after a support link breaks.
+ *
+ * Extracted from ExtractionEngine to keep extraction.ts under its ratchet
+ * ceiling. The ExtractionEngine.revalidateDependents method delegates here.
+ */
+import type { LocalLlmClient } from "../local-llm.js";
+import type { FallbackLlmClient, FallbackLlmOptions } from "../fallback-llm.js";
+
+export interface RevalidationVerdict {
+  memoryId: string;
+  verdict: "still_valid" | "invalidated" | "uncertain";
+  reason?: string;
+}
+
+export interface RevalidationDeps {
+  shouldUseLocalLlm: boolean;
+  shouldUseDirectClient: boolean;
+  localLlm: LocalLlmClient;
+  fallbackLlm: FallbackLlmClient;
+  client: unknown;
+  config: { model: string; localLlmFallback?: boolean };
+  withGatewayAgent: (opts: {
+    temperature: number;
+    maxTokens: number;
+    signal?: AbortSignal;
+  }) => FallbackLlmOptions;
+  parseJsonObject: (raw: string | null) => unknown;
+}
+
+function isPlainRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function buildPrompt(
+  superseded: { id: string; content: string },
+  replacement: { id: string; content: string } | null,
+  dependents: Array<{ id: string; category: string; content: string }>,
+): { input: string; instructionText: string } {
+  const dependentPrompt = dependents
+    .map(
+      (memory, index) =>
+        `[${index + 1}] id: ${memory.id} | category: ${memory.category}\n${memory.content}`,
+    )
+    .join("\n\n");
+  const input = `SUPERSEDED MEMORY (id: ${superseded.id}):
+${superseded.content}
+
+REPLACEMENT (id: ${replacement?.id ?? "none"}):
+${replacement?.content ?? "none — the memory was removed without replacement"}
+
+DEPENDENTS TO REVALIDATE (return exactly one verdict per id):
+${dependentPrompt}`;
+  const instructionText = `You revalidate dependent memory claims after a supporting memory changes.
+For each requested memory, decide whether its claim still holds.
+Return valid JSON only:
+{"verdicts":[{"memoryId":"id","verdict":"still_valid|invalidated|uncertain","reason":"brief reason"}]}
+Never invent memory IDs. Use "uncertain" when the evidence is insufficient.`;
+  return { input, instructionText };
+}
+
+function normalizeVerdicts(
+  parsed: unknown,
+  dependents: Array<{ id: string }>,
+): RevalidationVerdict[] {
+  const raw: unknown[] = isPlainRecord(parsed) && Array.isArray(parsed.verdicts) ? parsed.verdicts : [];
+  const requested = new Set(dependents.map((memory) => memory.id));
+  const seen = new Set<string>();
+  const normalized: RevalidationVerdict[] = [];
+  for (const candidate of raw) {
+    if (!isPlainRecord(candidate)) continue;
+    const memoryId = typeof candidate.memoryId === "string" ? candidate.memoryId.trim() : "";
+    if (!requested.has(memoryId) || seen.has(memoryId)) continue;
+    seen.add(memoryId);
+    const verdict =
+      candidate.verdict === "still_valid" ||
+      candidate.verdict === "invalidated" ||
+      candidate.verdict === "uncertain"
+        ? candidate.verdict
+        : "uncertain";
+    normalized.push({
+      memoryId,
+      verdict,
+      ...(typeof candidate.reason === "string" ? { reason: candidate.reason } : {}),
+    });
+  }
+  return dependents.map((memory) => {
+    const verdict = normalized.find((candidate) => candidate.memoryId === memory.id);
+    return verdict ?? { memoryId: memory.id, verdict: "uncertain" as const };
+  });
+}
+
+export async function revalidateDependentsViaLlm(
+  deps: RevalidationDeps,
+  superseded: { id: string; content: string },
+  replacement: { id: string; content: string } | null,
+  dependents: Array<{ id: string; category: string; content: string }>,
+  signal?: AbortSignal,
+): Promise<{ verdicts: RevalidationVerdict[] }> {
+  const { input, instructionText } = buildPrompt(superseded, replacement, dependents);
+  const maxTokens = Math.max(1024, dependents.length * 256);
+
+  let rawContent: string | null = null;
+  if (deps.shouldUseLocalLlm) {
+    try {
+      const response = await deps.localLlm.chatCompletion(
+        [
+          { role: "system", content: instructionText },
+          { role: "user", content: input },
+        ],
+        {
+          temperature: 0.2,
+          maxTokens,
+          operation: "dependency_revalidation",
+          priority: "background",
+          signal,
+        },
+      );
+      rawContent = response?.content ?? null;
+      if (!rawContent && !deps.config.localLlmFallback) {
+        throw new Error("local LLM returned no dependency revalidation output");
+      }
+    } catch (error) {
+      if (!deps.config.localLlmFallback || signal?.aborted) throw error;
+    }
+  }
+
+  if (rawContent === null && !deps.shouldUseDirectClient) {
+    const response = await deps.fallbackLlm.chatCompletion(
+      [
+        { role: "system", content: instructionText },
+        { role: "user", content: input },
+      ],
+      deps.withGatewayAgent({ temperature: 0.2, maxTokens, signal }),
+    );
+    rawContent = response?.content ?? null;
+    if (!rawContent) throw new Error("fallback LLM returned no dependency revalidation output");
+  }
+
+  if (rawContent === null && deps.shouldUseDirectClient) {
+    const chatClient = deps.client as unknown as {
+      chat: {
+        completions: {
+          create(
+            request: {
+              model: string;
+              messages: Array<{ role: string; content: string }>;
+              max_tokens?: number;
+            },
+            options?: { signal?: AbortSignal },
+          ): Promise<{ choices?: Array<{ message?: { content?: string } }> }>;
+        };
+      };
+    };
+    const response = await chatClient.chat.completions.create(
+      {
+        model: deps.config.model,
+        messages: [
+          { role: "system", content: instructionText },
+          { role: "user", content: input },
+        ],
+        max_tokens: maxTokens,
+      },
+      { signal },
+    );
+    rawContent = response.choices?.[0]?.message?.content ?? null;
+    if (!rawContent?.trim()) {
+      throw new Error("direct client returned no dependency revalidation output");
+    }
+  }
+
+  return { verdicts: normalizeVerdicts(deps.parseJsonObject(rawContent), dependents) };
+}

package/src/orchestration/extraction-persist-deps.ts

@@ -0,0 +1,93 @@
+import type { GraphConstructionCapabilitySet } from "../capabilities.js";
+import type { SemanticDedupHit } from "../dedup/semantic.js";
+import type { EmbeddingFallback } from "../embedding-fallback.js";
+import type { FaithfulnessGateCounters } from "../extraction-faithfulness.js";
+import type { ExtractionEngine } from "../extraction.js";
+import type { JudgeVerdict } from "../extraction-judge.js";
+import type { StorageManager } from "../index.js";
+import type { LocalLlmClient } from "../local-llm.js";
+import type { NamespaceStorageRouter } from "../namespaces/storage.js";
+import type { RouteRule, RoutingEngineOptions } from "../routing/engine.js";
+import type { SearchBackend } from "../search/port.js";
+import type { ThreadingManager } from "../threading.js";
+import type { MemoryFile, MemoryLink, PluginConfig } from "../types.js";
+
+export interface ExtractionPersistDeps {
+  config: PluginConfig;
+  getStorageRouter: () => NamespaceStorageRouter;
+  getThreading: () => ThreadingManager;
+  getExtraction: () => ExtractionEngine;
+  getLocalLlm: () => LocalLlmClient;
+  getQmd: () => SearchBackend;
+  getJudgeVerdictCache: () => Map<string, JudgeVerdict>;
+  getJudgeDeferCounts: () => Map<string, number>;
+  getFaithfulnessCounters: () => FaithfulnessGateCounters;
+  getEmbeddingFallback: () => EmbeddingFallback;
+  setLastPersistExtractionDeferredCount: (value: number) => void;
+  setLastPersistExtractionPendingReviewIds: (ids: string[]) => void;
+  addContentHashDedup: (targetStorage: StorageManager, content: string) => Promise<void>;
+  hasContentHashDedup: (targetStorage: StorageManager, content: string) => Promise<boolean>;
+  backfillTemporalBoundsOnDedupHit: (
+    targetStorage: StorageManager,
+    dedupContent: string,
+    bounds: {
+      invalidAt?: string;
+      validFrom?: string;
+      observedAt?: string;
+      eventTimeSource?: "extracted" | "assumed";
+    },
+    entityRef?: string,
+    sourceConnector?: string
+  ) => Promise<void>;
+  saveContentHashIndexes: () => Promise<void>;
+  artifactTypeForCategory: (
+    category: string
+  ) => "decision" | "constraint" | "todo" | "definition" | "commitment" | "correction" | "fact";
+  loadRoutingRules: () => Promise<RouteRule[]>;
+  routeEngineOptions: () => RoutingEngineOptions;
+  semanticDedupLookup: (content: string, limit: number, targetStorage: StorageManager) => Promise<SemanticDedupHit[]>;
+  checkForContradiction: (
+    content: string,
+    category: string,
+    namespaceScope: string
+  ) => Promise<{
+    supersededId: string;
+    confidence: number;
+    reason: string;
+    supersededPath: string;
+    supersededCreated: string;
+    supersededTags: string[];
+  } | null>;
+  applyDeferredContradictionResolve: (
+    contradiction:
+      | {
+          supersededId: string;
+          reason: string;
+          supersededPath: string;
+          supersededCreated: string;
+          supersededTags: string[];
+        }
+      | null
+      | undefined,
+    storage: StorageManager,
+    newMemoryId: string,
+    postWriteGuard: boolean
+  ) => Promise<void>;
+  suggestLinksForMemory: (content: string, category: string, namespaceScope: string) => Promise<MemoryLink[]>;
+  storageDirNamespace: (storageDir: string) => string;
+  indexPersistedMemory: (storage: StorageManager, memoryId: string) => Promise<void>;
+  buildGraphEdge: (
+    storage: StorageManager,
+    memoryRelPath: string,
+    entityRef: string | undefined,
+    memoryId: string,
+    factContent: string,
+    allMemsForGraph: MemoryFile[] | null | undefined,
+    memoryPathById: Map<string, string>,
+    threadIdForEdge: string | undefined,
+    threadEpisodeIdsForGraph: string[] | undefined,
+    fallbackCausalPredecessor: string | undefined,
+    graphCaps?: GraphConstructionCapabilitySet
+  ) => Promise<void>;
+  updateTemporalTagIndexes: (storage: StorageManager, persistedIds: string[]) => Promise<void>;
+}