@remnic/core 9.27.0 → 9.29.0

package/src/extraction-source-grounding.ts

@@ -0,0 +1,50 @@
+import type { ExtractionResult } from "./types.js";
+import { delinearize } from "./delinearize.js";
+import {
+  applyExtractionSourceGrounding as applyGroundingRules,
+  filterExtractionResultBySource as filterGroundingRules,
+} from "./extraction-source-grounding-rules.js";
+import type {
+  ExtractionGroundingRoleSources,
+  ExtractionSourceGroundingOptions,
+} from "./extraction-source-grounding-rules.js";
+
+export type { ExtractionGroundingRoleSources, ExtractionSourceGroundingOptions } from "./extraction-source-grounding-rules.js";
+
+export function applyExtractionSourceGrounding(
+  result: ExtractionResult,
+  sourceText: string,
+  assertionSourceText: string = sourceText,
+  roleAssertionSources: ExtractionGroundingRoleSources | undefined,
+  messageTimestamp: Date | undefined,
+  options: ExtractionSourceGroundingOptions,
+): ExtractionResult {
+  const sourceForGrounding = (source: string | undefined): string | undefined =>
+    source === undefined || !options.anchorTemporalExpressions
+      ? source
+      : delinearize(source, result.entities, messageTimestamp ?? new Date());
+  const groundedRoleSources = roleAssertionSources === undefined
+    ? undefined
+    : {
+      profile: sourceForGrounding(roleAssertionSources.profile),
+      identity: sourceForGrounding(roleAssertionSources.identity),
+    };
+  return applyGroundingRules(
+    result,
+    sourceForGrounding(sourceText) ?? sourceText,
+    sourceForGrounding(assertionSourceText) ?? assertionSourceText,
+    groundedRoleSources,
+    messageTimestamp,
+    options,
+  );
+}
+
+export function filterExtractionResultBySource(
+  result: ExtractionResult,
+  source: string,
+  assertionSource?: string,
+  roleAssertionSources?: ExtractionGroundingRoleSources,
+  eventTimeNormalizer?: (eventTime: string) => string,
+): ExtractionResult {
+  return filterGroundingRules(result, source, assertionSource, roleAssertionSources, eventTimeNormalizer);
+}

package/src/extraction.ts

@@ -40,6 +40,7 @@ import { normalizeProcedureSteps } from "./procedural/procedure-types.js";
 import { normalizeReasoningTrace } from "./reasoning-trace-types.js";
 import { looksLikeMechanicalTelemetryTranscript } from "./telemetry-transcript.js";
 import { buildFactProvenance, type ProvenanceTurnInput } from "./provenance.js";
+import { applyExtractionSourceGrounding, type ExtractionGroundingRoleSources } from "./extraction-source-grounding.js";
 import { isMemoryCategory } from "./write-envelope.js";
 import { classifyExtractionThrownError, classifyFallbackParseFailure } from "./extraction-error-classification.js";
 export { classifyExtractionThrownError, classifyFallbackParseFailure } from "./extraction-error-classification.js";
@@ -656,6 +657,10 @@ export class ExtractionEngine {
   private async applyProactiveQuestionPass(
     conversation: string,
     base: ExtractionResult,
+    groundingSource: string,
+    assertionSource: string,
+    messageTimestamp?: Date,
+    roleAssertionSources?: ExtractionGroundingRoleSources,
   ): Promise<ExtractionResult> {
     if (!resolvePipelineProcessingCapabilities(this.config).proactiveExtraction) return base;
     const maxAdditional = Math.max(0, Math.floor(this.config.maxProactiveQuestionsPerExtraction));
@@ -670,8 +675,16 @@ export class ExtractionEngine {
         proactive,
         maxAdditional,
       );
-      if (!this.hasExtractionOutputs(proactiveAdditions)) return base;
-      return this.mergeProactiveExtractionPass(base, proactiveAdditions, maxAdditional);
+      const sanitizedAdditions = this.sanitizeExtractionResult(proactiveAdditions, messageTimestamp);
+      const groundedAdditions = this.applySourceGrounding(
+        sanitizedAdditions,
+        groundingSource,
+        assertionSource,
+        roleAssertionSources,
+        messageTimestamp,
+      );
+      if (!this.hasExtractionOutputs(groundedAdditions)) return base;
+      return this.mergeProactiveExtractionPass(base, groundedAdditions, maxAdditional);
     } catch (err) {
       log.debug(`proactive extraction question pass failed (ignored): ${err}`);
       return base;
@@ -875,7 +888,7 @@ export class ExtractionEngine {
         if (localResponse?.content) {
           const parsed = this.parseProactiveExtractionResultFromText(localResponse.content.trim());
           if (parsed) {
-            return this.sanitizeExtractionResult(parsed);
+            return parsed;
           }
         }
         if (!this.config.localLlmFallback) {
@@ -906,12 +919,10 @@ export class ExtractionEngine {
     if (!fallbackResult) {
       return { facts: [], profileUpdates: [], entities: [], questions: [] };
     }
-    return this.sanitizeExtractionResult(
-      this.normalizeExtractionResultPayload({
-        ...fallbackResult,
-        questions: [],
-      }),
-    );
+    return this.normalizeExtractionResultPayload({
+      ...fallbackResult,
+      questions: [],
+    });
   }
 
   private mergeProactiveExtractionPass(
@@ -1133,6 +1144,40 @@ export class ExtractionEngine {
     return { ...result, facts };
   }
 
+  private applySourceGrounding(
+    result: ExtractionResult,
+    sourceText: string,
+    assertionSourceText: string = sourceText,
+    roleAssertionSources?: ExtractionGroundingRoleSources,
+    messageTimestamp?: Date,
+  ): ExtractionResult {
+    const capabilities = resolvePipelineProcessingCapabilities(this.config);
+    return applyExtractionSourceGrounding(
+      result,
+      sourceText,
+      assertionSourceText,
+      roleAssertionSources,
+      messageTimestamp,
+      {
+        sourceGrounding: capabilities.sourceGrounding,
+        anchorTemporalExpressions: capabilities.delinearize,
+      },
+    );
+  }
+
+  private finalizeExtractionResult(
+    result: ExtractionResult,
+    turns: ReadonlyArray<{
+      content: string;
+      sessionKey?: string;
+      logicalSessionKey?: string;
+      timestamp: string;
+      turnFingerprint?: string;
+    }>,
+  ): ExtractionResult {
+    return this.attachProvenanceToResult(result, turns);
+  }
+
   async extract(turns: BufferTurn[], existingEntities?: string[]): Promise<ExtractionResult> {
     const lifecycleCaps = resolveMemoryLifecycleCapabilities(this.config);
 
@@ -1158,6 +1203,24 @@ export class ExtractionEngine {
         return `[${roleLabel}] ${t.content}`;
       })
       .join("\n\n");
+
+    const groundingSource = boundedTurns
+      .map((turn) => turn.content)
+      .join("\n\n");
+    const assertionSource = boundedTurns
+      .filter((turn) => turn.extractionContextOnly !== true)
+      .map((turn) => turn.content)
+      .join("\n\n");
+    const roleAssertionSources: ExtractionGroundingRoleSources = {
+      profile: boundedTurns
+        .filter((turn) => turn.role === "user" && turn.extractionContextOnly !== true)
+        .map((turn) => turn.content)
+        .join("\n\n"),
+      identity: boundedTurns
+        .filter((turn) => turn.role === "assistant" && turn.extractionContextOnly !== true)
+        .map((turn) => turn.content)
+        .join("\n\n"),
+    };
     if (conversation.trim().length === 0) {
       log.debug("extraction skipped — conversation only contained non-memory work-layer context");
       return { facts: [], profileUpdates: [], entities: [], questions: [] };
@@ -1213,8 +1276,22 @@ export class ExtractionEngine {
           this.emit({ kind: "llm_end", traceId, model: this.config.localLlmModel, operation: "extraction", durationMs });
           log.debug(`extraction: used local LLM — ${localResult.facts.length} facts, ${localResult.entities.length} entities`);
           const sanitized = this.sanitizeExtractionResult(localResult, messageTimestamp);
-          const finalResult = await this.applyProactiveQuestionPass(conversation, sanitized);
-          return this.attachProvenanceToResult(finalResult, boundedTurns);
+          const grounded = this.applySourceGrounding(
+            sanitized,
+            groundingSource,
+            assertionSource,
+            roleAssertionSources,
+            messageTimestamp,
+          );
+          const finalResult = await this.applyProactiveQuestionPass(
+            conversation,
+            grounded,
+            groundingSource,
+            assertionSource,
+            messageTimestamp,
+            roleAssertionSources,
+          );
+          return this.finalizeExtractionResult(finalResult, boundedTurns);
         }
         // Local failed, fall back if allowed
         if (!this.config.localLlmFallback) {
@@ -1260,8 +1337,22 @@ export class ExtractionEngine {
           this.emit({ kind: "llm_end", traceId, model: this.config.model, operation: "extraction", durationMs });
           log.debug(`extraction: used direct client (${this.config.model}) — ${directResult.facts.length} facts, ${directResult.entities.length} entities`);
           const sanitized = this.sanitizeExtractionResult(directResult, messageTimestamp);
-          const finalResult = await this.applyProactiveQuestionPass(conversation, sanitized);
-          return this.attachProvenanceToResult(finalResult, boundedTurns);
+          const grounded = this.applySourceGrounding(
+            sanitized,
+            groundingSource,
+            assertionSource,
+            roleAssertionSources,
+            messageTimestamp,
+          );
+          const finalResult = await this.applyProactiveQuestionPass(
+            conversation,
+            grounded,
+            groundingSource,
+            assertionSource,
+            messageTimestamp,
+            roleAssertionSources,
+          );
+          return this.finalizeExtractionResult(finalResult, boundedTurns);
         }
         // Emit error event so Opik sees the direct client failure before fallback.
         // Wrapped in try/catch so a subscriber error doesn't break the fallback path.
@@ -1345,8 +1436,22 @@ export class ExtractionEngine {
         );
         const normalized = this.normalizeExtractionResultPayload(result);
         const sanitized = this.sanitizeExtractionResult(normalized, messageTimestamp);
-        const finalResult = await this.applyProactiveQuestionPass(conversation, sanitized);
-        return this.attachProvenanceToResult(finalResult, boundedTurns);
+        const grounded = this.applySourceGrounding(
+          sanitized,
+          groundingSource,
+          assertionSource,
+          roleAssertionSources,
+          messageTimestamp,
+        );
+        const finalResult = await this.applyProactiveQuestionPass(
+          conversation,
+          grounded,
+          groundingSource,
+          assertionSource,
+          messageTimestamp,
+          roleAssertionSources,
+        );
+        return this.finalizeExtractionResult(finalResult, boundedTurns);
       }
 
       this.emit({

package/src/memory-projection-mutations.ts

@@ -0,0 +1,132 @@
+import { lstat, realpath } from "node:fs/promises";
+import { isErrnoCode } from "./utils/errno.js";
+import { assertPathInsideRoot } from "./utils/path-containment.js";
+import { getMemoryProjectionPath, initializeMemoryProjectionDb } from "./memory-projection-store.js";
+import { type BetterSqlite3Database, openBetterSqlite3 } from "./runtime/better-sqlite.js";
+
+const PROJECTION_WRITE_BUSY_TIMEOUT_MS = 100;
+const PROJECTION_WRITE_RETRY_COUNT = 4;
+const PROJECTION_WRITE_RETRY_DELAY_MS = 100;
+
+function isSqliteBusyError(error: unknown): boolean {
+  const messages: string[] = [];
+  const seen = new Set<unknown>();
+  let current: unknown = error;
+  while (current !== null && typeof current === "object" && !seen.has(current)) {
+    seen.add(current);
+    if (current instanceof Error) messages.push(current.message);
+    if (!("cause" in current)) break;
+    current = current.cause;
+  }
+  const lower = messages.join("\n").toLowerCase();
+  return lower.includes("database is locked") || lower.includes("sqlite_busy");
+}
+
+async function validateProjectionPath(
+  memoryDir: string,
+  projectionPath: string,
+): Promise<boolean> {
+  let projectionStat;
+  try {
+    projectionStat = await lstat(projectionPath);
+  } catch (error) {
+    if (isErrnoCode(error, "ENOENT")) return false;
+    throw error;
+  }
+  if (projectionStat.isSymbolicLink() || !projectionStat.isFile()) {
+    throw new Error(`Refusing memory projection rewrite through unsafe database path: ${projectionPath}.`);
+  }
+  const [memoryRoot, projectionReal] = await Promise.all([
+    realpath(memoryDir),
+    realpath(projectionPath),
+  ]);
+  assertPathInsideRoot(memoryRoot, projectionReal, projectionPath);
+  return true;
+}
+
+async function rewriteProjectedEntityReferenceRows(
+  memoryDir: string,
+  entries: ReadonlyArray<readonly [string, string]>,
+  memoryId?: string,
+): Promise<void> {
+  if (memoryId !== undefined && memoryId.length === 0) {
+    throw new Error("Projection memoryId must not be empty.");
+  }
+  const scopedMemory = memoryId !== undefined;
+  const validEntries = entries.filter(
+    ([previousEntityRef, nextEntityRef]) =>
+      previousEntityRef.length > 0 && nextEntityRef.length > 0 && previousEntityRef !== nextEntityRef,
+  );
+  if (validEntries.length === 0) return;
+
+  const projectionPath = getMemoryProjectionPath(memoryDir);
+  if (!(await validateProjectionPath(memoryDir, projectionPath))) return;
+
+  let lastError: unknown;
+  for (let attempt = 0; attempt < PROJECTION_WRITE_RETRY_COUNT; attempt += 1) {
+    if (!(await validateProjectionPath(memoryDir, projectionPath))) return;
+    let db: BetterSqlite3Database | null = null;
+    try {
+      db = openBetterSqlite3(projectionPath, { fileMustExist: true });
+      db.pragma(`busy_timeout = ${PROJECTION_WRITE_BUSY_TIMEOUT_MS}`);
+      initializeMemoryProjectionDb(db);
+      const updateCurrent = db.prepare(
+        scopedMemory
+          ? "UPDATE memory_current SET entity_ref = ? WHERE memory_id = ? AND entity_ref = ?"
+          : "UPDATE memory_current SET entity_ref = ? WHERE entity_ref = ?",
+      );
+      const updateMentions = db.prepare(
+        scopedMemory
+          ? `
+            UPDATE memory_entity_mentions
+            SET entity_ref = ?
+            WHERE memory_id = ? AND entity_ref = ? AND mention_source = 'frontmatter.entityRef'
+          `
+          : `
+            UPDATE memory_entity_mentions
+            SET entity_ref = ?
+            WHERE entity_ref = ? AND mention_source = 'frontmatter.entityRef'
+          `,
+      );
+      const rewrite = db.transaction(() => {
+        for (const [previousEntityRef, nextEntityRef] of validEntries) {
+          const params = scopedMemory
+            ? [nextEntityRef, memoryId, previousEntityRef]
+            : [nextEntityRef, previousEntityRef];
+          updateCurrent.run(...params);
+          updateMentions.run(...params);
+        }
+      });
+      rewrite();
+      return;
+    } catch (error) {
+      lastError = error;
+      if (!isSqliteBusyError(error) || attempt === PROJECTION_WRITE_RETRY_COUNT - 1) {
+        throw error;
+      }
+      const { promise, resolve } = Promise.withResolvers<void>();
+      setTimeout(resolve, PROJECTION_WRITE_RETRY_DELAY_MS);
+      await promise;
+    } finally {
+      db?.close();
+    }
+  }
+
+  throw lastError instanceof Error ? lastError : new Error("Projection rewrite failed.");
+}
+
+export async function rewriteProjectedMemoryEntityReference(
+  memoryDir: string,
+  memoryId: string,
+  previousEntityRef: string,
+  nextEntityRef: string,
+): Promise<void> {
+  await rewriteProjectedEntityReferenceRows(memoryDir, [[previousEntityRef, nextEntityRef]], memoryId);
+}
+
+export async function rewriteProjectedEntityReferences(
+  memoryDir: string,
+  replacements: Readonly<Record<string, string>>,
+): Promise<void> {
+  await rewriteProjectedEntityReferenceRows(memoryDir, Object.entries(replacements));
+}

package/src/namespaces/storage.ts

@@ -284,6 +284,7 @@ export class NamespaceStorageRouter {
       root = await this.defaultNamespaceRoot();
       const cached = this.cache.get(ns);
       if (cached && cached.dir === root) {
+        await this.applySecureStoreConfig(cached);
         this.notifyResolved(ns, root);
         return cached;
       }
@@ -291,6 +292,7 @@ export class NamespaceStorageRouter {
       const cached = this.cache.get(ns);
       root = await this.namespaceRoot(ns);
       if (cached && cached.dir === root) {
+        await this.applySecureStoreConfig(cached);
         this.notifyResolved(ns, root);
         return cached;
       }
@@ -335,7 +337,7 @@ export class NamespaceStorageRouter {
     // recovery paths (e.g. the CLI lifecycle-ledger rebuild) refuse an encrypted
     // ledger they could actually rewrite, and namespace writes could silently
     // fall back to plaintext. Mirror the orchestrator's setup exactly.
-    this.applySecureStoreConfig(sm);
+    await this.applySecureStoreConfig(sm);
     this.cache.set(ns, sm);
     // #2019: let the orchestrator wire write-triggered prioritized embedding on
     // every router-created namespace storage, not just the primary default store.
@@ -478,12 +480,12 @@ export class NamespaceStorageRouter {
    * root keyring id — exactly the id the orchestrator and `secure-store unlock`
    * register under. No-op when secure-store is disabled.
    */
-  private applySecureStoreConfig(sm: StorageManager): void {
+  private async applySecureStoreConfig(sm: StorageManager): Promise<void> {
     if (!resolveRecallAuxiliaryCapabilities(this.config).secureStore) return;
     sm.setSecureStoreRequired(true);
     const existingKey = keyring.getKey(secureStoreDir(this.config.memoryDir));
     if (existingKey) {
-      sm.setSecureStoreKey(existingKey, this.config.secureStoreEncryptOnWrite);
+      await sm.setSecureStoreKeyAndWait(existingKey, this.config.secureStoreEncryptOnWrite);
     }
   }
 

package/src/operator-doctor-corpus.ts

@@ -23,7 +23,9 @@ import {
   computeCorpusWatermarks,
   resolveCorpusNamespaceRoots,
 } from "./corpus-watermark.js";
+import { summarizeReplicaDivergence } from "./operator-doctor-replica.js";
 import type { OperatorDoctorCheck } from "./operator-toolkit.js";
+import type { ResolveSecretRefFn } from "./resolve-auth-token.js";
 import type { PluginConfig } from "./types.js";
 
 const DIGEST_PREFIX_LENGTH = 12;
@@ -93,3 +95,42 @@ export async function summarizeCorpusWatermark(
     details: { corpus: watermarks },
   };
 }
+
+/**
+ * Extract the watermark array this check stores under `details.corpus` (issue
+ * #2149). The replica-divergence doctor check reuses the local watermark set
+ * computed here rather than rescanning the corpus, and consumes it through this
+ * helper so the `details.corpus` shape stays owned by the module that produces it.
+ */
+export function corpusWatermarksFromCheck(check: OperatorDoctorCheck): CorpusWatermark[] {
+  const details = check.details;
+  if (!details || typeof details !== "object" || !("corpus" in details) || !Array.isArray(details.corpus)) {
+    return [];
+  }
+  // This check always stores a CorpusWatermark[] under `corpus` (guarded above).
+  return details.corpus as CorpusWatermark[];
+}
+
+/**
+ * Build the corpus-watermark and replica-divergence doctor checks together.
+ *
+ * They are paired deliberately: the replica comparison consumes the corpus
+ * check's watermarks (one scan, not two) AND its completeness — an incomplete
+ * local census must not let the replica check certify convergence. The pairing
+ * lives HERE, next to the corpus check + `corpusWatermarksFromCheck` it consumes
+ * and the `PluginConfig` it needs, so the sibling replica module stays a light
+ * tsup DTS entry free of the heavy `PluginConfig` type graph (round 6, coderabbit).
+ */
+export async function summarizeCorpusAndReplica(
+  config: PluginConfig,
+  storageFactory: (dir: string) => CorpusStorage,
+  resolveSecretRef?: ResolveSecretRefFn | null,
+): Promise<OperatorDoctorCheck[]> {
+  const watermarkCheck = await summarizeCorpusWatermark(config, storageFactory);
+  const replicaCheck = await summarizeReplicaDivergence(
+    config.replicaPeers,
+    corpusWatermarksFromCheck(watermarkCheck),
+    { resolveSecretRef, localCensusComplete: watermarkCheck.status === "ok" },
+  );
+  return [watermarkCheck, replicaCheck];
+}