@shadowforge0/aquifer-memory 1.5.9 → 1.6.0

package/core/finalization-review.js

@@ -0,0 +1,319 @@
+'use strict';
+
+const TYPE_LABELS = {
+  state: '狀態',
+  decision: '決策',
+  fact: '事實',
+  preference: '偏好',
+  constraint: '限制',
+  entity_note: '註記',
+  open_loop: '未完成',
+  conclusion: '判斷',
+};
+
+const SESSION_START_TYPE_PRIORITY = {
+  state: 0,
+  open_loop: 1,
+  constraint: 2,
+  preference: 3,
+  decision: 4,
+  fact: 5,
+  conclusion: 6,
+  entity_note: 7,
+};
+
+const AUTHORITY_PRIORITY = {
+  user_explicit: 0,
+  executable_evidence: 1,
+  manual: 2,
+  system: 3,
+  verified_summary: 4,
+  llm_inference: 5,
+  raw_transcript: 6,
+};
+
+const MEMORY_KEYS = [
+  'summary',
+  'title',
+  'decision',
+  'item',
+  'conclusion',
+  'statement',
+  'fact',
+  'preference',
+  'constraint',
+  'state',
+  'note',
+  'text',
+  'value',
+];
+
+const STRUCTURED_FIELDS = [
+  ['states', 'state'],
+  ['state', 'state'],
+  ['decisions', 'decision'],
+  ['important_facts', 'fact'],
+  ['facts', 'fact'],
+  ['preferences', 'preference'],
+  ['constraints', 'constraint'],
+  ['conclusions', 'conclusion'],
+  ['entity_notes', 'entity_note'],
+  ['open_loops', 'open_loop'],
+];
+
+const DEFAULT_OMIT = [
+  '整段逐字稿、工具輸出、debug 訊息',
+  'DB row id、hash、message count 這類 audit 欄位',
+  '已作廢、隔離、錯誤或 superseded 的記憶',
+];
+
+function normalizeText(value) {
+  return String(value || '').trim().replace(/\s+/g, ' ');
+}
+
+function sanitizeHumanText(value) {
+  return normalizeText(value)
+    .replace(/\bDB Write Plan\b/g, 'DB 寫入計畫')
+    .replace(/\bLegacy Continuity Text\b/g, '舊 handoff 包裝文字')
+    .replace(/\bStructured Summary\b/g, 'structured summary 原始欄位')
+    .replace(/\braw JSON\b/gi, '原始 JSON');
+}
+
+function stripTerminalPunctuation(value) {
+  return normalizeText(value).replace(/[。.!?！？]+$/g, '');
+}
+
+function comparable(value) {
+  return stripTerminalPunctuation(value).toLowerCase();
+}
+
+function firstText(value) {
+  if (typeof value === 'string') return normalizeText(value);
+  if (!value || typeof value !== 'object') return '';
+  for (const key of MEMORY_KEYS) {
+    const text = normalizeText(value[key]);
+    if (text) return text;
+  }
+  const payload = value.payload && typeof value.payload === 'object' ? value.payload : null;
+  if (payload) return firstText(payload);
+  return '';
+}
+
+function memoryTypeOf(value) {
+  if (!value || typeof value !== 'object') return 'memory';
+  return value.memoryType || value.memory_type || value.type || 'memory';
+}
+
+function labelFor(type) {
+  return TYPE_LABELS[type] || TYPE_LABELS[String(type || '').toLowerCase()] || '記憶';
+}
+
+function pushUnique(out, text) {
+  const normalized = sanitizeHumanText(text);
+  if (!normalized) return;
+  const key = comparable(normalized);
+  if (!key || out.some(item => comparable(item) === key)) return;
+  out.push(normalized);
+}
+
+function asLine(type, text, suffix = '') {
+  const body = normalizeText(text);
+  if (!body) return '';
+  return `${labelFor(type)}：${body}${suffix}`;
+}
+
+function truncate(text, max = 220) {
+  const normalized = sanitizeHumanText(text);
+  if (normalized.length <= max) return normalized;
+  return `${normalized.slice(0, max - 1)}...`;
+}
+
+function addStructuredItems(out, structuredSummary = {}, filter = null) {
+  for (const [field, type] of STRUCTURED_FIELDS) {
+    if (filter && !filter(type)) continue;
+    const items = Array.isArray(structuredSummary[field]) ? structuredSummary[field] : [];
+    for (const item of items) {
+      const text = firstText(item);
+      if (!text) continue;
+      const owner = type === 'open_loop' && item && typeof item === 'object' && normalizeText(item.owner)
+        ? `（owner: ${normalizeText(item.owner)}）`
+        : '';
+      pushUnique(out, asLine(type, text, owner));
+    }
+  }
+}
+
+function promotedMemoryLines(memoryResults = []) {
+  const lines = [];
+  for (const result of memoryResults || []) {
+    if (!result || result.action !== 'promote') continue;
+    const memory = result.memory || result.record || result.candidate || {};
+    const type = memoryTypeOf(memory);
+    if (type === 'open_loop') continue;
+    pushUnique(lines, asLine(type, firstText(memory)));
+  }
+  return lines;
+}
+
+function openLoopLines(memoryResults = [], structuredSummary = {}) {
+  const lines = [];
+  for (const result of memoryResults || []) {
+    if (!result || result.action !== 'promote') continue;
+    const memory = result.memory || result.record || result.candidate || {};
+    const type = memoryTypeOf(memory);
+    if (type !== 'open_loop') continue;
+    const owner = normalizeText(memory.owner || memory.payload?.owner);
+    pushUnique(lines, asLine(type, firstText(memory), owner ? `（owner: ${owner}）` : ''));
+  }
+  if (lines.length === 0) {
+    addStructuredItems(lines, structuredSummary, type => type === 'open_loop');
+  }
+  return lines;
+}
+
+function inactiveLines(memoryResults = [], extraInactive = []) {
+  const lines = [];
+  for (const result of memoryResults || []) {
+    if (!result || result.action === 'promote') continue;
+    const candidate = result.candidate || result.memory || result.record || {};
+    const text = firstText(candidate);
+    const reason = normalizeText(result.reason);
+    const action = normalizeText(result.action || 'skipped');
+    if (text || reason) {
+      pushUnique(lines, `${action}：${text || '未命名候選'}${reason ? `（${reason}）` : ''}`);
+    }
+  }
+  for (const item of extraInactive || []) {
+    const text = firstText(item);
+    const status = normalizeText(item.status || item.action || 'inactive');
+    const reason = normalizeText(item.reason || item.obsoleteReason || item.obsolete_reason);
+    if (text || reason) {
+      pushUnique(lines, `${status}：${text || '未命名記憶'}${reason ? `（${reason}）` : ''}`);
+    }
+  }
+  return lines;
+}
+
+function linesOrNone(lines) {
+  if (!lines || lines.length === 0) return '無';
+  return lines.map(line => `- ${line}`).join('\n');
+}
+
+function buildAuditLines(input = {}) {
+  const finalization = input.finalization || {};
+  const memoryResult = input.memoryResult || {};
+  const audit = input.audit || {};
+  const pairs = [
+    ['sessionId', audit.sessionId || input.sessionId],
+    ['finalizationId', audit.finalizationId || finalization.id],
+    ['handoffId', audit.handoffId || input.handoffId],
+    ['transcriptHash', audit.transcriptHash || input.transcriptHash],
+    ['promoted', memoryResult.promoted],
+    ['quarantined', memoryResult.quarantined],
+    ['skipped', memoryResult.skipped],
+    ['policyVersion', audit.policyVersion || input.policyVersion],
+    ['schemaVersion', audit.schemaVersion || input.schemaVersion],
+  ].filter(([, value]) => value !== undefined && value !== null && value !== '');
+  return pairs.map(([key, value]) => `${key}: ${value}`);
+}
+
+function collectRemembered(input = {}) {
+  const structuredSummary = input.structuredSummary || input.summary?.structuredSummary || {};
+  const memoryResults = input.memoryResults || [];
+  const lines = promotedMemoryLines(memoryResults);
+  if (lines.length === 0) {
+    addStructuredItems(lines, structuredSummary, type => type !== 'open_loop');
+  }
+  return lines;
+}
+
+function buildCarryForwardLines(input = {}) {
+  const lines = [];
+  for (const line of input.openLoops || openLoopLines(input.memoryResults, input.structuredSummary || input.summary?.structuredSummary || {})) {
+    pushUnique(lines, line);
+  }
+  const next = normalizeText(input.next || input.metadata?.handoff?.next);
+  if (next && next !== '無') pushUnique(lines, `下一步：${next}`);
+  return lines;
+}
+
+function buildFinalizationReview(input = {}, opts = {}) {
+  const summary = input.summary || {};
+  const structuredSummary = input.structuredSummary || summary.structuredSummary || {};
+  const summaryText = input.summaryText || summary.summaryText || input.overview || '';
+  const statusLine = truncate(input.currentStatus || summaryText || input.title || '已完成本段 finalization。');
+  const remembered = collectRemembered({ ...input, structuredSummary });
+  const openLoops = openLoopLines(input.memoryResults || [], structuredSummary);
+  const inactive = inactiveLines(input.memoryResults || [], input.inactive || []);
+  const carryForward = buildCarryForwardLines({ ...input, structuredSummary, openLoops });
+  const omit = [];
+  for (const item of opts.omit || input.omit || DEFAULT_OMIT) pushUnique(omit, item);
+  const heading = opts.preview ? '準備整理進 DB：' : '已整理進 DB：';
+  const lines = [
+    heading,
+    `目前狀態：\n${linesOrNone([statusLine])}`,
+    `已記住：\n${linesOrNone(remembered)}`,
+    `未完成：\n${linesOrNone(openLoops)}`,
+    `已作廢或隔離：\n${linesOrNone(inactive)}`,
+    `下一段只需要帶：\n${linesOrNone(carryForward)}`,
+    `不要帶：\n${linesOrNone(omit)}`,
+  ];
+  if (opts.includeAudit === true) {
+    lines.push(`Audit：\n${linesOrNone(buildAuditLines(input))}`);
+  }
+  return `${lines.join('\n\n')}\n`;
+}
+
+function buildSessionStartContext(records = [], opts = {}) {
+  const asOf = opts.asOf ? Date.parse(opts.asOf) : null;
+  const limit = Math.max(1, Math.min(50, opts.limit || 12));
+  const maxChars = Math.max(120, opts.maxChars || 1800);
+  const active = [];
+  for (const [index, record] of (records || []).entries()) {
+    const status = record.status || 'candidate';
+    const visible = record.visibleInBootstrap ?? record.visible_in_bootstrap;
+    if (status !== 'active' || visible !== true) continue;
+    if (Number.isFinite(asOf)) {
+      const validFrom = Date.parse(record.validFrom || record.valid_from || '');
+      const validTo = Date.parse(record.validTo || record.valid_to || '');
+      const staleAfter = Date.parse(record.staleAfter || record.stale_after || '');
+      if (Number.isFinite(validFrom) && validFrom > asOf) continue;
+      if (Number.isFinite(validTo) && validTo <= asOf) continue;
+      if (Number.isFinite(staleAfter) && staleAfter <= asOf) continue;
+    }
+    active.push({ record, index });
+  }
+
+  active.sort((a, b) => {
+    const aType = SESSION_START_TYPE_PRIORITY[memoryTypeOf(a.record)] ?? 99;
+    const bType = SESSION_START_TYPE_PRIORITY[memoryTypeOf(b.record)] ?? 99;
+    if (aType !== bType) return aType - bType;
+
+    const aAuth = AUTHORITY_PRIORITY[a.record.authority] ?? 99;
+    const bAuth = AUTHORITY_PRIORITY[b.record.authority] ?? 99;
+    if (aAuth !== bAuth) return aAuth - bAuth;
+
+    const aAccepted = Date.parse(a.record.acceptedAt || a.record.accepted_at || '') || 0;
+    const bAccepted = Date.parse(b.record.acceptedAt || b.record.accepted_at || '') || 0;
+    if (aAccepted !== bAccepted) return bAccepted - aAccepted;
+    return a.index - b.index;
+  });
+
+  const lines = [];
+  for (const { record } of active.slice(0, limit)) {
+    const type = memoryTypeOf(record);
+    pushUnique(lines, asLine(type, firstText(record)));
+  }
+  let selected = lines;
+  let text = `下一段只需要帶：\n${linesOrNone(selected)}\n`;
+  while (text.length > maxChars && selected.length > 1) {
+    selected = selected.slice(0, -1);
+    text = `下一段只需要帶：\n${linesOrNone(selected)}\n`;
+  }
+  return text;
+}
+
+module.exports = {
+  buildFinalizationReview,
+  buildSessionStartContext,
+};

package/core/insights.js

@@ -26,65 +26,20 @@ const DEFAULT_RECALL_WEIGHTS = Object.freeze({
   recency: 0.10,
 });
 
+const DEFAULT_DEDUP = Object.freeze({
+  mode: 'off',
+  cosineThreshold: 0.88,
+  closeBandFrom: 0.85,
+});
+
+const VALID_DEDUP_MODES = new Set(['off', 'shadow', 'enforce']);
+
 // Recency linear decay horizon — an insight is treated as "fully recent" at
 // creation (age=0) and "zero recency" at age >= recencyWindowDays. Beyond,
 // recency contribution is clamped to 0 rather than going negative. Configurable
 // via createAquifer({ insights: { recencyWindowDays } }).
 const DEFAULT_RECENCY_WINDOW_DAYS = 90;
 
-const LEADING_PUNCT_RE = /^[\s\-_.,;:!?'"()\[\]{}@#]+/;
-const TRAILING_PUNCT_RE = /[\s\-_.,;:!?'"()\[\]{}@#]+$/;
-
-function _normalizeText(input) {
-  if (typeof input !== 'string' || !input) return '';
-  let s = input.normalize('NFKC');
-  s = s.toLowerCase();
-  s = s.replace(/\s+/g, ' ');
-  s = s.replace(LEADING_PUNCT_RE, '');
-  s = s.replace(TRAILING_PUNCT_RE, '');
-  return s;
-}
-
-function normalizeCanonicalClaim(text) {
-  return _normalizeText(text);
-}
-
-function normalizeBody(text) {
-  return _normalizeText(text);
-}
-
-function normalizeEntitySet(entities) {
-  if (!entities || !Array.isArray(entities)) return '';
-  const { normalizeEntityName } = require('./entity');
-  const normalized = entities
-    .map(e => normalizeEntityName(e))
-    .filter(Boolean);
-  const deduped = [...new Set(normalized)];
-  deduped.sort();
-  return deduped.join('|');
-}
-
-function defaultCanonicalKey({ tenantId, agentId, type, canonicalClaim, entities }) {
-  const normClaim = normalizeCanonicalClaim(canonicalClaim);
-  const normEntities = normalizeEntitySet(entities);
-  const input = `${tenantId || ''}|${agentId || ''}|${type || ''}|${normClaim}|${normEntities}`;
-  return crypto.createHash('sha256').update(input).digest('hex');
-}
-
-function defaultIdempotencyKey({
-  tenantId, agentId, type, title, body, sourceSessionIds, evidenceWindow,
-}) {
-  const sorted = (sourceSessionIds || []).slice().sort().join('|');
-  const winFrom = evidenceWindow && evidenceWindow.from ? new Date(evidenceWindow.from).toISOString() : '';
-  const winTo = evidenceWindow && evidenceWindow.to ? new Date(evidenceWindow.to).toISOString() : '';
-  // Hash must include body + window so legitimate revisions (same sessions but
-  // tightened body, or extended window) get a new key and replace the old row
-  // via supersede, not get swallowed as a duplicate.
-  return crypto.createHash('sha256')
-    .update(`${tenantId}|${agentId}|${type}|${title}|${body || ''}|${sorted}|${winFrom}|${winTo}`)
-    .digest('hex');
-}
-
 // ---------------------------------------------------------------------------
 // Canonical identity helpers (Phase 2 C1)
 //
@@ -160,6 +115,94 @@ function vecToPgLiteral(v) {
   return `[${v.join(',')}]`;
 }
 
+function truncate(input, limit) {
+  if (typeof input !== 'string') return '';
+  if (!Number.isFinite(limit) || limit < 0) return '';
+  return input.length <= limit ? input : input.slice(0, limit);
+}
+
+function truncateNormalized(input, limit) {
+  return truncate(normalizeBody(input), limit);
+}
+
+function resolveDedupConfig(dedup, embedFn) {
+  let resolved;
+  if (dedup === true) {
+    resolved = { ...DEFAULT_DEDUP, mode: 'enforce' };
+  } else if (dedup === false || dedup === undefined) {
+    resolved = { ...DEFAULT_DEDUP };
+  } else if (dedup && typeof dedup === 'object') {
+    resolved = { ...DEFAULT_DEDUP, ...dedup };
+  } else {
+    resolved = { ...DEFAULT_DEDUP };
+  }
+
+  const rawMode = typeof resolved.mode === 'string' ? resolved.mode.trim().toLowerCase() : resolved.mode;
+  if (!VALID_DEDUP_MODES.has(rawMode)) {
+    console.warn(`[aquifer] insights dedup: invalid mode ${JSON.stringify(resolved.mode)}; coercing to 'off'`);
+    resolved.mode = 'off';
+  } else {
+    resolved.mode = rawMode;
+  }
+
+  const envMode = process.env.AQUIFER_INSIGHTS_DEDUP_MODE;
+  if (typeof envMode === 'string') {
+    const normalizedEnvMode = envMode.trim().toLowerCase();
+    if (VALID_DEDUP_MODES.has(normalizedEnvMode)) {
+      resolved.mode = normalizedEnvMode;
+    }
+  }
+
+  // Reject non-numeric sentinels (null, bool, objects) BEFORE Number()
+  // coerces them to 0 — 0 would silently become a "merge everything"
+  // threshold in enforce mode.
+  let cosineThreshold;
+  if (resolved.cosineThreshold === null || resolved.cosineThreshold === undefined
+      || typeof resolved.cosineThreshold === 'boolean') {
+    console.warn(`[aquifer] insights dedup: invalid cosineThreshold ${JSON.stringify(resolved.cosineThreshold)}; defaulting to 0.88`);
+    cosineThreshold = DEFAULT_DEDUP.cosineThreshold;
+  } else {
+    cosineThreshold = Number(resolved.cosineThreshold);
+    if (!Number.isFinite(cosineThreshold)) {
+      console.warn('[aquifer] insights dedup: invalid cosineThreshold; defaulting to 0.88');
+      cosineThreshold = DEFAULT_DEDUP.cosineThreshold;
+    } else if (cosineThreshold < 0.75 || cosineThreshold > 0.95) {
+      const clamped = Math.max(0, Math.min(1, cosineThreshold));
+      console.warn(`[aquifer] insights dedup: cosineThreshold ${cosineThreshold} outside recommended [0.75,0.95]; using ${clamped}`);
+      cosineThreshold = (cosineThreshold >= 0 && cosineThreshold <= 1) ? cosineThreshold : clamped;
+    }
+  }
+  resolved.cosineThreshold = cosineThreshold;
+
+  let closeBandFrom;
+  if (resolved.closeBandFrom === null || resolved.closeBandFrom === undefined
+      || typeof resolved.closeBandFrom === 'boolean') {
+    console.warn(`[aquifer] insights dedup: invalid closeBandFrom ${JSON.stringify(resolved.closeBandFrom)}; defaulting to 0.85`);
+    closeBandFrom = DEFAULT_DEDUP.closeBandFrom;
+  } else {
+    closeBandFrom = Number(resolved.closeBandFrom);
+    if (!Number.isFinite(closeBandFrom)) {
+      console.warn('[aquifer] insights dedup: invalid closeBandFrom; defaulting to 0.85');
+      closeBandFrom = DEFAULT_DEDUP.closeBandFrom;
+    }
+  }
+  if (closeBandFrom >= resolved.cosineThreshold) {
+    const adjusted = Math.max(0, resolved.cosineThreshold - 0.03);
+    console.warn(`[aquifer] insights dedup: closeBandFrom ${closeBandFrom} must be below cosineThreshold ${resolved.cosineThreshold}; using ${adjusted}`);
+    closeBandFrom = adjusted;
+  }
+  resolved.closeBandFrom = closeBandFrom;
+
+  if (resolved.mode !== 'off') {
+    console.log(`[aquifer] insights dedup: mode=${resolved.mode} threshold=${resolved.cosineThreshold} close_band_from=${resolved.closeBandFrom}`);
+    if (!embedFn) {
+      console.warn('[aquifer] insights dedup: embedFn unavailable; semantic dedup disabled at runtime');
+    }
+  }
+
+  return Object.freeze(resolved);
+}
+
 function mapRow(row) {
   if (!row) return null;
   return {
@@ -184,7 +227,7 @@ function mapRow(row) {
   };
 }
 
-function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights, recencyWindowDays }) {
+function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights, recencyWindowDays, dedup }) {
   if (!pool) throw new Error('createInsights: pool is required');
   if (!schema) throw new Error('createInsights: schema is required');
 
@@ -192,6 +235,24 @@ function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights,
   const recencyWindow = Number.isFinite(recencyWindowDays) && recencyWindowDays > 0
     ? recencyWindowDays : DEFAULT_RECENCY_WINDOW_DAYS;
   const tbl = `${schema}.insights`;
+  const dedupConfig = resolveDedupConfig(dedup, embedFn);
+
+  if (dedupConfig.mode !== 'off') {
+    pool.query(
+      `SELECT count(*)::int AS n FROM ${tbl}
+        WHERE canonical_key_v2 IS NULL AND status = 'active'`
+    ).then(r => {
+      const n = r && r.rows && r.rows[0] ? Number(r.rows[0].n) : 0;
+      if (n > 0) {
+        console.warn(
+          `[aquifer] insights: ${n} active rows with canonical_key_v2 IS NULL. `
+          + 'Run scripts/backfill-canonical-key.js to include them in canonical dedup.'
+        );
+      }
+    }).catch(() => {
+      // non-fatal
+    });
+  }
 
   // -------------------------------------------------------------------------
   // commitInsight
@@ -283,9 +344,101 @@ function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights,
         toSupersede = Number(activeRow.id);
       }
 
-      // Optional embedding.
       let embedding = null;
-      if (embedFn) {
+      let embeddingReady = false;
+
+      if (dedupConfig.mode !== 'off' && !toSupersede && embedFn) {
+        // Embed the incoming title+body once. If this throws, the label
+        // is genuinely 'embed_failed' — the candidate SELECT never ran.
+        let embedFailed = false;
+        try {
+          const v = await embedFn([`${title}\n\n${body}`]);
+          if (Array.isArray(v) && Array.isArray(v[0])) {
+            embedding = vecToPgLiteral(v[0]);
+          }
+          embeddingReady = true;
+        } catch {
+          embedFailed = true;
+          embeddingReady = true;
+          metadata = { ...metadata, dedupSkipped: 'embed_failed' };
+        }
+
+        if (!embedFailed && embedding) {
+          // Candidate lookup. If this throws (DB error), let it bubble
+          // to the outer commitInsight try/catch → AQ_INTERNAL. Do NOT
+          // mislabel it as embed_failed.
+          const semanticLookup = await pool.query(
+            `SELECT *, 1.0 - (embedding <=> $4::vector) AS cos_sim
+               FROM ${tbl}
+              WHERE tenant_id = $1
+                AND agent_id = $2
+                AND insight_type = $3
+                AND status = 'active'
+                AND embedding IS NOT NULL
+              ORDER BY embedding <=> $4::vector
+              LIMIT 1`,
+            [tenantId, agentId, type, embedding]
+          );
+
+          if (semanticLookup.rowCount > 0) {
+            const candidate = semanticLookup.rows[0];
+            const cosine = Number(candidate.cos_sim);
+
+            if (cosine >= dedupConfig.cosineThreshold) {
+              const candidateUpper = parseUpperFromRange(candidate.evidence_window);
+              const isStaleReplay = candidateUpper
+                && new Date(toIso).getTime() < candidateUpper.getTime();
+
+              if (dedupConfig.mode === 'enforce') {
+                // Enforce path: stale-replay returns the candidate as
+                // duplicate; otherwise supersede.
+                if (isStaleReplay) {
+                  return ok({ insight: mapRow(candidate), duplicate: true });
+                }
+                toSupersede = Number(candidate.id);
+                metadata = {
+                  ...metadata,
+                  dedupVia: 'semantic',
+                  dedupCandidate: { id: Number(candidate.id), cosine },
+                };
+              } else {
+                // Shadow path: always insert the new row, always record
+                // shadowMatch metadata. staleReplay flag tells reviewers
+                // the enforce-mode twin would have returned duplicate
+                // instead of superseding.
+                metadata = {
+                  ...metadata,
+                  shadowMatch: {
+                    candidateId: Number(candidate.id),
+                    cosine,
+                    threshold: dedupConfig.cosineThreshold,
+                    candidateTitle: truncate(candidate.title, 200),
+                    candidateBody: truncateNormalized(candidate.body, 200),
+                    wouldSupersede: !isStaleReplay,
+                    staleReplay: Boolean(isStaleReplay),
+                    ranAt: new Date().toISOString(),
+                  },
+                };
+              }
+            } else if (cosine >= dedupConfig.closeBandFrom) {
+              metadata = {
+                ...metadata,
+                dedupNear: {
+                  candidateId: Number(candidate.id),
+                  cosine,
+                  threshold: dedupConfig.cosineThreshold,
+                  closeBandFrom: dedupConfig.closeBandFrom,
+                  candidateTitle: truncate(candidate.title, 200),
+                  candidateBody: truncateNormalized(candidate.body, 200),
+                },
+              };
+            }
+          }
+        }
+      }
+
+      // Optional embedding.
+      if (embedFn && !embeddingReady) {
         try {
           const v = await embedFn([`${title}\n\n${body}`]);
           if (Array.isArray(v) && Array.isArray(v[0])) embedding = vecToPgLiteral(v[0]);
@@ -485,13 +638,12 @@ function createInsights({ pool, schema, defaultTenantId, embedFn, recallWeights,
     recallInsights,
     markStale,
     supersede,
-    _internal: { defaultIdempotencyKey, vecToPgLiteral, mapRow, weights },
+    _internal: { vecToPgLiteral, mapRow, weights, dedup: dedupConfig },
   };
 }
 
 module.exports = {
   createInsights,
-  defaultIdempotencyKey,
   defaultCanonicalKey,
   normalizeCanonicalClaim,
   normalizeBody,