@remnic/core 9.3.628 → 9.3.630

package/src/wearables/memory-gen.ts

@@ -28,7 +28,16 @@
  */
 
 import { scoreImportance } from "../importance.js";
+import type { JudgeBatchResult, JudgeCandidate } from "../extraction-judge.js";
+import { getVerdictKind } from "../extraction-judge.js";
 import { describeErrorForOperator } from "./errors.js";
+import {
+  computeTrustScore,
+  decideSmart,
+  findCorroboration,
+  type CorroborationContext,
+  type TrustEvidence,
+} from "./trust.js";
 import type {
   BufferTurn,
   ExtractedFact,
@@ -54,7 +63,7 @@ import type {
 export function memoryStatusForMode(
   mode: WearableMemoryMode,
 ): Extract<MemoryStatus, "active" | "pending_review"> {
-  return mode === "auto" ? "active" : "pending_review";
+  return mode === "auto" || mode === "smart" ? "active" : "pending_review";
 }
 
 /** Narrow writer interface satisfied by `StorageManager`. */
@@ -75,19 +84,70 @@ export interface WearableMemoryWriter {
     },
   ): Promise<string>;
   hasFactContentHash(content: string): Promise<boolean>;
+  /**
+   * Locate an earlier wearable write of the same content (any status).
+   * Optional: enables in-place promotion when re-scoring with stronger
+   * evidence.
+   */
+  findWearableMemoryByContent?(
+    content: string,
+  ): Promise<{ id: string; status: MemoryStatus | undefined } | null>;
+  /**
+   * Promote a pending_review wearable memory to active, merging trust
+   * evidence. Returns false when missing or no longer pending.
+   */
+  promoteWearableMemory?(
+    id: string,
+    attributeUpdates: Record<string, string>,
+    confidence?: number,
+  ): Promise<boolean>;
+  /**
+   * Demote a pending_review wearable memory to rejected on an explicit
+   * judge-reject re-verdict. Returns false when missing or no longer
+   * pending. Active rows are never auto-demoted.
+   */
+  demoteWearableMemory?(
+    id: string,
+    attributeUpdates: Record<string, string>,
+  ): Promise<boolean>;
 }
 
 export interface WearableMemoryGenDeps {
   extract(turns: BufferTurn[]): Promise<ExtractionResult>;
   writer: WearableMemoryWriter;
+  /**
+   * LLM-as-judge batch evaluation (the existing extraction judge).
+   * Absent when no judge is wired (degraded smart mode: trust scoring
+   * runs on confidence x sourceTrust + corroboration alone).
+   */
+  judgeFacts?(candidates: JudgeCandidate[]): Promise<JudgeBatchResult>;
+  /**
+   * Corroboration evidence for smart mode: other sources' same-day
+   * transcript tokens + existing active memories. Absent disables
+   * corroboration boosts.
+   */
+  corroboration?: CorroborationContext;
 }
 
 export interface WearableMemoryGenResult {
   created: number;
+  /** Earlier borderline writes promoted to active by new evidence. */
+  promoted: number;
+  /** Earlier pending writes retired by a fresh judge-reject verdict. */
+  demoted: number;
   skipped: number;
   skippedByReason: Record<string, number>;
   /** Non-fatal problems (e.g. the extraction engine erroring). */
   warnings: string[];
+  /**
+   * True when every conversation was extracted (the pass may still
+   * carry degraded-mode warnings, e.g. judge unavailable). False only
+   * when extraction itself aborted mid-day — the signal callers use to
+   * re-run the pass on the next sync. A degraded-but-complete pass
+   * must NOT re-run forever: its facts are already written and dedup
+   * would suppress improvements anyway.
+   */
+  completed: boolean;
 }
 
 export const WEARABLE_SOURCE_PREFIX = "wearable";
@@ -171,6 +231,70 @@ interface GatedCandidate {
   conversation: WearableConversation;
 }
 
+interface ScoredCandidate {
+  trust: number;
+  verdict?: "accept" | "reject" | "defer";
+  evidence: TrustEvidence;
+}
+
+/**
+ * Smart-mode scoring: one judge batch call for the whole day, then
+ * per-fact trust = confidence x sourceTrust + judge/corroboration
+ * boosts. A judge failure degrades gracefully (warned once; scoring
+ * continues on confidence + corroboration alone).
+ */
+async function scoreCandidates(
+  novel: GatedCandidate[],
+  settings: WearableSourceSettings,
+  deps: WearableMemoryGenDeps,
+  result: WearableMemoryGenResult,
+): Promise<Map<number, ScoredCandidate>> {
+  const scored = new Map<number, ScoredCandidate>();
+  if (novel.length === 0) return scored;
+
+  let verdicts: Map<number, "accept" | "reject" | "defer"> | undefined;
+  if (deps.judgeFacts) {
+    const judgeCandidates: JudgeCandidate[] = novel.map((candidate) => ({
+      text: candidate.fact.content,
+      category: candidate.fact.category,
+      confidence:
+        typeof candidate.fact.confidence === "number"
+          ? candidate.fact.confidence
+          : 0.7,
+      tags: candidate.fact.tags ?? [],
+      importanceLevel: candidate.importance.level,
+    }));
+    try {
+      const judgeResult = await deps.judgeFacts(judgeCandidates);
+      verdicts = new Map();
+      for (const [index, verdict] of judgeResult.verdicts) {
+        verdicts.set(index, getVerdictKind(verdict));
+      }
+    } catch (err) {
+      result.warnings.push(
+        `extraction judge unavailable for this pass: ${describeErrorForOperator(err)} — trust scoring continued without judge verdicts`,
+      );
+    }
+  }
+
+  const corroboration: CorroborationContext = deps.corroboration ?? {
+    otherSourceDayTokens: new Map(),
+    existingMemories: [],
+  };
+  novel.forEach((candidate, index) => {
+    const evidence = findCorroboration(candidate.fact.content, corroboration);
+    const verdict = verdicts?.get(index);
+    const trust = computeTrustScore({
+      extractionConfidence: candidate.fact.confidence,
+      sourceTrust: settings.sourceTrust,
+      judgeVerdict: verdict,
+      evidence,
+    });
+    scored.set(index, { trust, ...(verdict !== undefined ? { verdict } : {}), evidence });
+  });
+  return scored;
+}
+
 /**
  * Run extraction + gates over a day's conversations and persist the
  * survivors. Returns counts for the sync summary.
@@ -185,9 +309,12 @@ export async function generateWearableMemories(
 ): Promise<WearableMemoryGenResult> {
   const result: WearableMemoryGenResult = {
     created: 0,
+    promoted: 0,
+    demoted: 0,
     skipped: 0,
     skippedByReason: {},
     warnings: [],
+    completed: true,
   };
   if (settings.memoryMode === "off") return result;
 
@@ -215,6 +342,7 @@ export async function generateWearableMemories(
       result.warnings.push(
         `extraction failed for ${sourceId}/${date} (conversation ${conversation.id}): ${describeErrorForOperator(err)} — the memory pass for this day retries on the next sync`,
       );
+      result.completed = false;
       break;
     }
     for (const fact of extraction.facts) {
@@ -227,7 +355,11 @@ export async function generateWearableMemories(
         skip("unsupported-category");
         continue;
       }
+      // In smart mode the trust bands subsume the hard confidence
+      // floor — a borderline fact belongs in the review band, not on
+      // the floor. The pre-filter applies to review/auto modes only.
       if (
+        settings.memoryMode !== "smart" &&
         typeof fact.confidence === "number" &&
         fact.confidence < settings.minConfidence
       ) {
@@ -254,33 +386,177 @@ export async function generateWearableMemories(
 
   // Drop candidates that already exist in storage BEFORE applying the
   // day cap so duplicates never consume cap slots that novel,
-  // lower-scoring candidates should get (Codex P2 on PR #1458).
+  // lower-scoring candidates should get (Codex P2 on PR #1458). In
+  // smart mode a duplicate of a PENDING_REVIEW write is kept aside as
+  // a promotion candidate — corroboration that arrives after the
+  // original borderline write (another device syncing the same day)
+  // must be able to promote it in place (Cursor review on PR #1462).
   const novel: GatedCandidate[] = [];
+  const promotable: GatedCandidate[] = [];
   for (const candidate of candidates) {
     if (await deps.writer.hasFactContentHash(candidate.fact.content)) {
-      skip("duplicate-existing");
+      if (
+        settings.memoryMode === "smart" &&
+        deps.writer.findWearableMemoryByContent !== undefined &&
+        deps.writer.promoteWearableMemory !== undefined
+      ) {
+        promotable.push(candidate);
+      } else {
+        skip("duplicate-existing");
+      }
       continue;
     }
     novel.push(candidate);
   }
 
-  // Day cap: keep the most important candidates. Stable ordering —
-  // score desc, then content asc so equal scores compare 0-consistent.
-  novel.sort((a, b) => {
-    if (a.importance.score > b.importance.score) return -1;
-    if (a.importance.score < b.importance.score) return 1;
-    if (a.fact.content < b.fact.content) return -1;
-    if (a.fact.content > b.fact.content) return 1;
+  // Re-score promotion candidates with TODAY'S evidence; promote the
+  // ones that now clear the auto threshold. Never consumes day-cap
+  // slots (no new memory is written).
+  if (promotable.length > 0) {
+    const promoteScores = await scoreCandidates(promotable, settings, deps, result);
+    for (const [index, candidate] of promotable.entries()) {
+      const scored = promoteScores.get(index);
+      const decision = scored
+        ? decideSmart(scored.trust, scored.verdict, settings)
+        : undefined;
+      if (!scored || !decision) {
+        skip("duplicate-existing");
+        continue;
+      }
+      // A fresh judge-REJECT retires the stored row — but only a
+      // pending_review one. Active rows are never auto-demoted: an
+      // operator approval or accrued recall signals must not be
+      // overturned by one later LLM verdict; contradiction scans and
+      // supersession own active-row retirement (Cursor review on PR
+      // #1462, round 7).
+      if (scored.verdict === "reject") {
+        if (deps.writer.demoteWearableMemory !== undefined) {
+          const existingForDemote = await deps.writer.findWearableMemoryByContent!(
+            candidate.fact.content,
+          );
+          if (
+            existingForDemote &&
+            existingForDemote.status === "pending_review" &&
+            (await deps.writer.demoteWearableMemory(existingForDemote.id, {
+              trustScore: scored.trust.toFixed(3),
+              trustDecision: "demoted-by-rejection",
+              judgeVerdict: "reject",
+            }))
+          ) {
+            result.demoted += 1;
+            continue;
+          }
+        }
+        skip("duplicate-existing");
+        continue;
+      }
+      if (decision.outcome !== "active") {
+        skip("duplicate-existing");
+        continue;
+      }
+      const existing = await deps.writer.findWearableMemoryByContent!(
+        candidate.fact.content,
+      );
+      if (!existing || existing.status !== "pending_review") {
+        skip("duplicate-existing");
+        continue;
+      }
+      const promoted = await deps.writer.promoteWearableMemory!(
+        existing.id,
+        {
+          trustScore: scored.trust.toFixed(3),
+          trustDecision: "promoted-by-corroboration",
+          ...(scored.verdict !== undefined ? { judgeVerdict: scored.verdict } : {}),
+          ...(scored.evidence.corroboratedBySources.length > 0
+            ? { corroboratedBySources: scored.evidence.corroboratedBySources.join(",") }
+            : {}),
+          ...(scored.evidence.supportingMemoryId !== undefined
+            ? { supportingMemoryId: scored.evidence.supportingMemoryId }
+            : {}),
+        },
+        scored.trust,
+      );
+      if (promoted) {
+        result.promoted += 1;
+      } else {
+        skip("duplicate-existing");
+      }
+    }
+  }
+
+  // Smart mode: judge + trust scoring decide active/review/drop per
+  // fact. The judge runs ONE batch call for the whole day; trust
+  // combines extraction confidence x sourceTrust with corroboration
+  // boosts (cross-device agreement, existing-memory support).
+  let trustById = new Map<number, ScoredCandidate>();
+  if (settings.memoryMode === "smart") {
+    trustById = await scoreCandidates(novel, settings, deps, result);
+  }
+
+  // Smart decisions run BEFORE the day cap so dropped facts (judge
+  // rejections, below-trust) never consume cap slots that surviving
+  // candidates ranked past position N should get (Cursor review on PR
+  // #1462).
+  interface Writable {
+    candidate: GatedCandidate;
+    index: number;
+    status: MemoryStatus;
+    trustAttributes: Record<string, string>;
+  }
+  const modeStatus = memoryStatusForMode(settings.memoryMode);
+  const writable: Writable[] = [];
+  novel.forEach((candidate, index) => {
+    if (settings.memoryMode !== "smart") {
+      writable.push({ candidate, index, status: modeStatus, trustAttributes: {} });
+      return;
+    }
+    const scored = trustById.get(index);
+    if (!scored) return;
+    const decision = decideSmart(scored.trust, scored.verdict, settings);
+    if (decision.outcome === "drop") {
+      skip(decision.reason);
+      return;
+    }
+    writable.push({
+      candidate,
+      index,
+      status: decision.outcome === "active" ? "active" : "pending_review",
+      trustAttributes: {
+        trustScore: scored.trust.toFixed(3),
+        trustDecision: decision.reason,
+        ...(scored.verdict !== undefined ? { judgeVerdict: scored.verdict } : {}),
+        ...(scored.evidence.corroboratedBySources.length > 0
+          ? { corroboratedBySources: scored.evidence.corroboratedBySources.join(",") }
+          : {}),
+        ...(scored.evidence.supportingMemoryId !== undefined
+          ? { supportingMemoryId: scored.evidence.supportingMemoryId }
+          : {}),
+      },
+    });
+  });
+
+  // Day cap over the SURVIVORS: strongest by trust in smart mode, by
+  // importance otherwise. Stable ordering with a content tiebreak.
+  const strength = (entry: Writable): number =>
+    settings.memoryMode === "smart"
+      ? trustById.get(entry.index)?.trust ?? 0
+      : entry.candidate.importance.score;
+  writable.sort((a, b) => {
+    const sa = strength(a);
+    const sb = strength(b);
+    if (sa > sb) return -1;
+    if (sa < sb) return 1;
+    if (a.candidate.fact.content < b.candidate.fact.content) return -1;
+    if (a.candidate.fact.content > b.candidate.fact.content) return 1;
     return 0;
   });
   const cap = settings.maxMemoriesPerDay;
-  const kept = cap > 0 ? novel.slice(0, cap) : novel;
-  if (novel.length > kept.length) {
-    skip("over-day-cap", novel.length - kept.length);
+  const kept = cap > 0 ? writable.slice(0, cap) : writable;
+  if (writable.length > kept.length) {
+    skip("over-day-cap", writable.length - kept.length);
   }
 
-  const status = memoryStatusForMode(settings.memoryMode);
-  for (const candidate of kept) {
+  for (const { candidate, index, status, trustAttributes } of kept) {
     const tags = [
       ...new Set([
         ...(candidate.fact.tags ?? []),
@@ -290,7 +566,10 @@ export async function generateWearableMemories(
       ]),
     ];
     await deps.writer.writeMemory(candidate.fact.category, candidate.fact.content, {
-      confidence: candidate.fact.confidence,
+      confidence:
+        settings.memoryMode === "smart"
+          ? trustById.get(index)?.trust
+          : candidate.fact.confidence,
       tags,
       source: wearableSourceLabel(sourceId),
       importance: candidate.importance,
@@ -300,6 +579,7 @@ export async function generateWearableMemories(
         wearableSource: sourceId,
         wearableDate: date,
         wearableConversationId: candidate.conversation.id,
+        ...trustAttributes,
       },
       contentHashSource: candidate.fact.content,
       status,
@@ -365,15 +645,26 @@ export async function writeDailyDigestMemory(
  * review queue. Always `pending_review` regardless of memoryMode — the
  * provider's extraction quality is outside Remnic's control.
  */
+/** Native-source trust prior reduction (provider extraction quality). */
+const NATIVE_TRUST_FACTOR = 0.9;
+
 export async function importNativeMemories(
   sourceId: string,
   memories: WearableNativeMemory[],
   alreadyImportedIds: ReadonlySet<string>,
-  writer: WearableMemoryWriter,
-): Promise<{ imported: number; importedIds: string[] }> {
+  settings: WearableSourceSettings,
+  deps: WearableMemoryGenDeps,
+): Promise<{ imported: number; importedIds: string[]; warnings: string[] }> {
   let imported = 0;
   const importedIds: string[] = [];
+  const warnings: string[] = [];
   const seenContent = new Set<string>();
+  const smart = settings.importNativeMemories === "smart";
+
+  // Smart path: one judge batch over the novel items, like transcript
+  // facts, with a reduced source prior — provider extraction quality is
+  // outside Remnic's control.
+  const novel: WearableNativeMemory[] = [];
   for (const memory of memories) {
     const content = memory.content?.trim();
     if (!content) continue;
@@ -381,13 +672,79 @@ export async function importNativeMemories(
     // Intra-run + cross-run dedup: the storage hash index only learns a
     // fact after its write lands, so same-content items within one page
     // batch need the local set.
-    if (seenContent.has(content) || (await writer.hasFactContentHash(content))) {
+    if (seenContent.has(content) || (await deps.writer.hasFactContentHash(content))) {
       importedIds.push(memory.id);
       continue;
     }
     seenContent.add(content);
-    await writer.writeMemory("fact", content, {
-      confidence: 0.6,
+    novel.push({ ...memory, content });
+  }
+
+  let verdicts: Map<number, "accept" | "reject" | "defer"> | undefined;
+  if (smart && deps.judgeFacts && novel.length > 0) {
+    try {
+      const judgeResult = await deps.judgeFacts(
+        novel.map((memory) => ({
+          text: memory.content,
+          category: "fact",
+          confidence: 0.7,
+          tags: memory.tags ?? [],
+        })),
+      );
+      verdicts = new Map();
+      for (const [index, verdict] of judgeResult.verdicts) {
+        verdicts.set(index, getVerdictKind(verdict));
+      }
+    } catch (err) {
+      warnings.push(
+        `extraction judge unavailable for native import: ${describeErrorForOperator(err)} — trust scoring continued without judge verdicts`,
+      );
+    }
+  }
+  const corroboration: CorroborationContext = deps.corroboration ?? {
+    otherSourceDayTokens: new Map(),
+    existingMemories: [],
+  };
+
+  for (const [index, memory] of novel.entries()) {
+    const content = memory.content;
+    let status: MemoryStatus = "pending_review";
+    let trustAttributes: Record<string, string> = {};
+    let confidence = 0.6;
+    if (smart) {
+      const evidence = findCorroboration(content, corroboration);
+      const verdict = verdicts?.get(index);
+      const trust = computeTrustScore({
+        extractionConfidence: undefined,
+        sourceTrust: settings.sourceTrust * NATIVE_TRUST_FACTOR,
+        judgeVerdict: verdict,
+        evidence,
+      });
+      const decision = decideSmart(trust, verdict, settings);
+      if (decision.outcome === "drop") {
+        // Deliberately NOT recorded in importedIds: a dropped native
+        // fact re-fetches and re-scores on later syncs, so corpus or
+        // corroboration support that arrives later can still admit it.
+        // The judge verdict cache keeps repeated rejections cheap
+        // (Cursor review on PR #1462).
+        continue;
+      }
+      status = decision.outcome === "active" ? "active" : "pending_review";
+      confidence = trust;
+      trustAttributes = {
+        trustScore: trust.toFixed(3),
+        trustDecision: decision.reason,
+        ...(verdict !== undefined ? { judgeVerdict: verdict } : {}),
+        ...(evidence.corroboratedBySources.length > 0
+          ? { corroboratedBySources: evidence.corroboratedBySources.join(",") }
+          : {}),
+        ...(evidence.supportingMemoryId !== undefined
+          ? { supportingMemoryId: evidence.supportingMemoryId }
+          : {}),
+      };
+    }
+    await deps.writer.writeMemory("fact", content, {
+      confidence,
       tags: [
         ...new Set([
           ...(memory.tags ?? []),
@@ -402,12 +759,13 @@ export async function importNativeMemories(
       structuredAttributes: {
         wearableSource: sourceId,
         wearableNativeId: memory.id,
+        ...trustAttributes,
       },
       contentHashSource: content,
-      status: "pending_review",
+      status,
     });
     imported += 1;
     importedIds.push(memory.id);
   }
-  return { imported, importedIds };
+  return { imported, importedIds, warnings };
 }