@geraldmaron/construct 1.0.24 → 1.1.1

package/lib/document-extract.mjs

@@ -107,6 +107,11 @@ function run(command, args) {
 }
 
 function commandExists(command) {
+  // Test seam: a comma-separated deny-list forces a command to read as absent,
+  // so the cross-platform fallbacks (e.g. RTF stripping without macOS textutil)
+  // are reachable on any host.
+  const denied = (process.env.CONSTRUCT_EXTRACT_NO_COMMANDS || '').split(',').map((s) => s.trim()).filter(Boolean);
+  if (denied.includes(command)) return false;
   const checker = process.platform === 'win32' ? 'where' : 'which';
   return spawnSync(checker, [command], { stdio: 'ignore' }).status === 0;
 }
@@ -216,16 +221,38 @@ function extractZipDocument(filePath, extension) {
   return { text, method: 'zip-xml', droppedInfo, structured: null };
 }
 
-function extractRichText(filePath) {
-  if (!commandExists('textutil')) {
-    throw new Error('textutil not available for rich-text extraction');
+// Minimal RTF-to-text: strip control words, groups, and escapes. RTF is a
+// text format, so this works everywhere; it is the cross-platform fallback when
+// textutil (macOS only) is absent. Lossy on complex documents, but never throws.
+export function rtfToText(raw) {
+  let s = String(raw);
+  s = s.replace(/\{(?:\\\*)?\\(?:fonttbl|colortbl|stylesheet|info|pict|object)[\s\S]*?\}/gi, '');
+  s = s.replace(/\\u(-?\d+)\s?\??/g, (_, n) => (Number(n) >= 0 ? String.fromCharCode(Number(n)) : ''));
+  s = s.replace(/\\'([0-9a-fA-F]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
+  s = s.replace(/\\par[d]?\b/g, '\n').replace(/\\line\b/g, '\n').replace(/\\tab\b/g, '\t');
+  s = s.replace(/\\[a-zA-Z]+-?\d* ?/g, '');
+  s = s.replace(/\\([{}\\])/g, '$1');
+  return s.replace(/[{}]/g, '');
+}
+
+function extractRichText(filePath, extension) {
+  if (commandExists('textutil')) {
+    return {
+      text: normalizeText(run('textutil', ['-convert', 'txt', '-stdout', filePath])),
+      method: 'textutil',
+      droppedInfo: [],
+      structured: null,
+    };
   }
-  return {
-    text: normalizeText(run('textutil', ['-convert', 'txt', '-stdout', filePath])),
-    method: 'textutil',
-    droppedInfo: [],
-    structured: null,
-  };
+  if (extension === '.rtf') {
+    return {
+      text: normalizeText(rtfToText(readFileSync(filePath, 'utf8'))),
+      method: 'rtf-strip',
+      droppedInfo: [],
+      structured: null,
+    };
+  }
+  throw new Error('textutil not available for rich-text extraction');
 }
 
 function extractPdfWithPdftotext(filePath) {
@@ -660,7 +687,7 @@ export function extractDocumentText(filePath, { maxChars = null } = {}) {
   } else if (ZIP_DOCUMENT_EXTS.has(extension)) {
     extracted = extractZipDocument(resolvedPath, extension);
   } else if (RICH_TEXT_EXTS.has(extension)) {
-    extracted = extractRichText(resolvedPath);
+    extracted = extractRichText(resolvedPath, extension);
   } else if (MDLS_DOCUMENT_EXTS.has(extension) && process.platform === 'darwin' && commandExists('mdls')) {
     extracted = { text: extractWithMdls(resolvedPath), method: 'mdls', droppedInfo: [], structured: null };
   } else if (EMAIL_DOCUMENT_EXTS.has(extension)) {
@@ -671,6 +698,13 @@ export function extractDocumentText(filePath, { maxChars = null } = {}) {
     throw new Error(`Unsupported document type: ${extension || 'unknown'}`);
   }
 
+  return buildExtractionResult(resolvedPath, extension, extracted, maxChars);
+}
+
+// Shared envelope construction so the sync, async, and Node-native paths emit
+// an identical result shape (callers route on extractionMethod and droppedInfo).
+
+function buildExtractionResult(resolvedPath, extension, extracted, maxChars) {
   const text = extracted.text;
   const limit = Number.isFinite(Number(maxChars)) && Number(maxChars) > 0
     ? Math.min(Number(maxChars), 200_000)
@@ -687,13 +721,54 @@ export function extractDocumentText(filePath, { maxChars = null } = {}) {
     droppedInfo: extracted.droppedInfo ?? [],
     structured: extracted.structured ?? null,
   };
+
   // Surface eml-specific metadata so callers (intake, ingest) can route
   // attachment filenames and oversize-skip markers without re-parsing.
+
   if (extracted.attachments) result.attachments = extracted.attachments;
   if (extracted.skipped) result.skipped = extracted.skipped;
   return result;
 }
 
+// Node-native extraction for the everyday adapter path: unpdf (PDF) and mammoth
+// (DOCX) are optional pure-JS deps, so plain documents ingest with no Python venv
+// and no system binary. Missing dep, an empty parse, or any other type delegates
+// to the sync system-binary extractor — the choice degrades, never fails hard.
+
+async function extractWithUnpdf(filePath) {
+  const { extractText, getDocumentProxy } = await import('unpdf');
+  const proxy = await getDocumentProxy(new Uint8Array(readFileSync(filePath)));
+  const { text } = await extractText(proxy, { mergePages: true });
+  return normalizeText(typeof text === 'string' ? text : (Array.isArray(text) ? text.join('\n') : ''));
+}
+
+async function extractWithMammoth(filePath) {
+  const mammoth = (await import('mammoth')).default;
+  const { value } = await mammoth.extractRawText({ path: filePath });
+  return normalizeText(value || '');
+}
+
+export async function extractDocumentTextNodeNative(filePath, { maxChars = null } = {}) {
+  const resolvedPath = resolve(filePath);
+  if (!existsSync(resolvedPath)) throw new Error(`File not found: ${resolvedPath}`);
+  const extension = extname(resolvedPath).toLowerCase();
+
+  const nodeBackend = extension === '.pdf' ? { fn: extractWithUnpdf, method: 'unpdf' }
+    : extension === '.docx' ? { fn: extractWithMammoth, method: 'mammoth' }
+      : null;
+
+  if (nodeBackend) {
+    try {
+      const text = await nodeBackend.fn(resolvedPath);
+      if (text) {
+        return buildExtractionResult(resolvedPath, extension, { text, method: nodeBackend.method, droppedInfo: [], structured: null }, maxChars);
+      }
+    } catch { /* optional dep absent or parse failure — fall through to the sync extractor */ }
+  }
+
+  return extractDocumentText(resolvedPath, { maxChars });
+}
+
 // Async high-fidelity extraction path. Routes PDF/Office/HTML through the
 // docling Python sidecar (layout-aware markdown) and audio/video through
 // whisper.cpp (Metal-accelerated on macOS). Text/transcript/calendar/email

package/lib/document-ingest.mjs

@@ -8,14 +8,15 @@
  * and records the selected strategy/model in the returned `ingestion` block.
  */
 import { existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from 'node:fs';
-import { basename, dirname, extname, isAbsolute, join, parse, relative, resolve } from 'node:path';
-import { extractDocumentText, extractDocumentTextAsync, extractDocumentMetadata, isExtractableDocumentPath } from './document-extract.mjs';
+import { basename, dirname, extname, isAbsolute, join, parse, relative, resolve, sep } from 'node:path';
+import { extractDocumentText, extractDocumentTextAsync, extractDocumentTextNodeNative, extractDocumentMetadata, isExtractableDocumentPath } from './document-extract.mjs';
 import { syncFileStateToSql } from './storage/sync.mjs';
 import { stampFrontmatter } from './doc-stamp.mjs';
 import { KNOWLEDGE_ROOT, KNOWLEDGE_SUBDIRS } from './knowledge/layout.mjs';
 import { loadProjectConfig } from './config/project-config.mjs';
 import { resolveIngestStrategy, INGEST_STRATEGIES, INGEST_ORCHESTRATION_STRATEGIES } from './ingest/strategy.mjs';
 import { extractViaProvider } from './ingest/provider-extract.mjs';
+import { extractViaDoclingRemote } from './ingest/docling-remote.mjs';
 
 const DEFAULT_TARGET_DIR = '.cx/knowledge/internal';
 
@@ -56,6 +57,31 @@ function nextAvailablePath(targetPath) {
   }
 }
 
+// docling embeds figures as base64 `data:` URIs in the markdown so no pixel data
+// is lost in transit. Write each to an assets/ directory beside the output file
+// and rewrite the reference to a relative path, so the markdown stays small and
+// the images are real files an editor or viewer can open — replacing the bare
+// `<!-- image -->` placeholder docling emits by default.
+
+export function externalizeEmbeddedImages(markdown, { mdPath }) {
+  if (!markdown || !markdown.includes('data:image/')) return { markdown, assets: [] };
+  const assetsRel = `assets/${basename(mdPath, '.md')}`;
+  const assetsDir = join(dirname(mdPath), assetsRel.split('/').join(sep));
+  const assets = [];
+  let index = 0;
+  const re = /!\[([^\]]*)\]\(\s*data:image\/([a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+?)\s*\)/g;
+  const out = markdown.replace(re, (_match, alt, type, b64) => {
+    index += 1;
+    const ext = type.toLowerCase() === 'jpeg' ? 'jpg' : type.toLowerCase().replace(/[^a-z0-9]/g, '') || 'png';
+    const fileName = `image-${index}.${ext}`;
+    if (!existsSync(assetsDir)) mkdirSync(assetsDir, { recursive: true });
+    writeFileSync(join(assetsDir, fileName), Buffer.from(b64.replace(/\s+/g, ''), 'base64'));
+    assets.push(join(assetsDir, fileName));
+    return `![${alt || `image ${index}`}](${assetsRel}/${fileName})`;
+  });
+  return { markdown: out, assets };
+}
+
 function renderMarkdown({ sourcePath, extractedAt, title, extractionMethod, characters, truncated, text, outputPath, cwd, metadata }) {
   const relSource = relative(cwd, sourcePath) || basename(sourcePath);
   const relOutput = relative(cwd, outputPath) || basename(outputPath);
@@ -126,17 +152,79 @@ function resolveOutputPath(sourcePath, { cwd, outputPath, outputDir, target }) {
   return join(resolvedDir, `${basename(sourcePath)}.md`);
 }
 
+// The high-fidelity (docling) path provisions a Python venv and downloads ML
+// models on first use, then runs layout inference on the document — any of which
+// can stall (a large PDF, a slow model download) or fail. Without a bound the
+// caller hangs: the CLI never returns, and the ingest_document MCP tool blocks
+// until the client times out and surfaces an opaque error. Bound the whole
+// docling attempt and fall back to the legacy extractor so ingest always returns
+// a usable result, with the fallback recorded in droppedInfo. Override the bound
+// with CONSTRUCT_DOCLING_TIMEOUT_MS (0 disables the timeout).
+
+const DOCLING_TIMEOUT_MS = (() => {
+  const raw = Number(process.env.CONSTRUCT_DOCLING_TIMEOUT_MS);
+  return Number.isFinite(raw) && raw >= 0 ? raw : 600_000;
+})();
+
+export function withTimeout(promise, ms, onTimeoutMessage) {
+  if (!ms) return promise;
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(Object.assign(new Error(onTimeoutMessage), { code: 'DOCLING_TIMEOUT' })), ms);
+    promise.then((v) => { clearTimeout(timer); resolve(v); }, (e) => { clearTimeout(timer); reject(e); });
+  });
+}
+
+/**
+ * Run the high-fidelity (docling) extractor with a bound, falling back to the
+ * legacy extractor on timeout or any failure so a caller never hangs. The
+ * extractor functions are injectable for testing; production uses the real
+ * docling-async and legacy-sync extractors.
+ */
+export async function extractWithDoclingFallback(sourcePath, {
+  maxChars = null,
+  timeoutMs = DOCLING_TIMEOUT_MS,
+  asyncExtract = extractDocumentTextAsync,
+  syncExtract = extractDocumentText,
+} = {}) {
+  try {
+    return await withTimeout(
+      asyncExtract(sourcePath, { maxChars }),
+      timeoutMs,
+      `docling extraction exceeded ${Math.round(timeoutMs / 1000)}s`,
+    );
+  } catch (err) {
+    const legacy = syncExtract(sourcePath, { maxChars });
+    legacy.droppedInfo = [
+      ...(legacy.droppedInfo ?? []),
+      {
+        kind: 'docling-fallback',
+        count: 1,
+        reason: `High-fidelity (docling) extraction failed or timed out (${err.message}); used the legacy extractor. Re-run after \`construct install --with-docling\`, or pass --legacy-extractor to skip docling.`,
+        recoverable: true,
+      },
+    ];
+    return legacy;
+  }
+}
+
+// The default adapter path prefers the Node-native extractor (unpdf/mammoth) so
+// plain PDF/DOCX ingest needs no Python venv; it self-degrades to the sync
+// system-binary extractor when the optional dep is absent. High-fidelity stays
+// on docling, reserved for scanned/layout-critical and audio/video.
+
 function extractViaAdapter(sourcePath, { highFidelity, maxChars }) {
   return highFidelity
-    ? extractDocumentTextAsync(sourcePath, { maxChars })
-    : extractDocumentText(sourcePath, { maxChars });
+    ? extractWithDoclingFallback(sourcePath, { maxChars })
+    : extractDocumentTextNodeNative(sourcePath, { maxChars });
 }
 
 async function extractWithStrategy(sourcePath, { strategy, fallback, model, provider, highFidelity, maxChars, env }) {
-  const primary = strategy === 'provider' ? 'provider' : 'adapter';
+  const primary = strategy === 'provider' || strategy === 'docling-remote' ? strategy : 'adapter';
   const run = (mode) => (mode === 'provider'
     ? extractViaProvider({ filePath: sourcePath, model, provider, maxChars, env })
-    : extractViaAdapter(sourcePath, { highFidelity, maxChars }));
+    : mode === 'docling-remote'
+      ? extractViaDoclingRemote({ filePath: sourcePath, maxChars, env })
+      : extractViaAdapter(sourcePath, { highFidelity, maxChars }));
 
   try {
     const extracted = await run(primary);
@@ -207,6 +295,7 @@ export async function ingestDocuments(inputPaths, {
     const finalPath = nextAvailablePath(targetPath);
     const extractedAt = new Date().toISOString();
     const title = metadata?.title || formatTitle(sourcePath);
+    const { markdown: bodyText, assets: imageAssets } = externalizeEmbeddedImages(extracted.text, { mdPath: finalPath });
     const markdown = renderMarkdown({
       sourcePath,
       extractedAt,
@@ -214,7 +303,7 @@ export async function ingestDocuments(inputPaths, {
       extractionMethod: extracted.extractionMethod,
       characters: extracted.characters,
       truncated: extracted.truncated,
-      text: extracted.text,
+      text: bodyText,
       outputPath: finalPath,
       cwd,
       metadata,
@@ -229,6 +318,7 @@ export async function ingestDocuments(inputPaths, {
       characters: extracted.characters,
       droppedInfo: extracted.droppedInfo ?? [],
       structured: extracted.structured ?? null,
+      images: imageAssets.map((p) => relative(cwd, p)),
       metadata: { title, authors: metadata?.authors || [], dates: metadata?.dates || {} },
     });
   }

package/lib/embed/roadmap.mjs

@@ -202,6 +202,12 @@ function normalizeStatus(raw) {
   return 'planned';
 }
 
+// Normalize the volatile "Last updated" stamp so two renders can be compared on
+// content alone.
+function stripTimestamp(markdown) {
+  return String(markdown || '').replace(/^> Last updated: .*$/m, '> Last updated:');
+}
+
 /**
  * Render entries back to roadmap markdown.
  */
@@ -324,7 +330,16 @@ export async function generateRoadmap({ targetPath, snapshot, roles }) {
   // Render
   const markdown = renderRoadmap(entries, { roles, sources });
 
-  // Write
+  // Skip the write when only the generated "Last updated" stamp would change —
+  // otherwise every render rewrites a tracked file with no semantic diff and
+  // leaves the working tree perpetually dirty. The timestamp then reflects when
+  // the roadmap content last actually moved, not when generation last ran.
+  let existingRaw = null;
+  try { existingRaw = readFileSync(roadmapPath, 'utf8'); } catch { existingRaw = null; }
+  if (existingRaw !== null && stripTimestamp(existingRaw) === stripTimestamp(markdown)) {
+    return { path: roadmapPath, updatedAt, itemCount: entries.length, isNew: false, unchanged: true };
+  }
+
   mkdirSync(join(targetPath, 'docs'), { recursive: true });
   writeFileSync(roadmapPath, markdown, 'utf8');
 

package/lib/embed/semantic.mjs

@@ -3,8 +3,10 @@
  *
  * Generates embeddings via @huggingface/transformers, caches them to disk,
  * computes cosine similarity, and clusters related signals by topic.
- * All local — no external API calls. Default model is all-MiniLM-L6-v2
- * (384-dimensional vectors, ~50MB on disk, fast inference).
+ * Inference is local and offline (allowRemoteModels=false): the model loads
+ * from local cache only and returns null when no cached weights are present,
+ * never reaching the network. Default model is all-MiniLM-L6-v2 (384-dimensional
+ * vectors, ~50MB on disk, fast inference).
  *
  * Storage:
  *   ~/.cx/cache/embeddings/<sha256>.json — single embedding vector
@@ -37,6 +39,7 @@ async function getEmbedder() {
   try {
     const { pipeline, env: hfEnv } = await import('@huggingface/transformers');
     hfEnv.allowLocalModels = true;
+    hfEnv.allowRemoteModels = false;
     hfEnv.useBrowserCache = false;
     embedder = await pipeline('feature-extraction', MODEL_NAME, {
       quantized: true,

package/lib/engine/consolidate.mjs

@@ -8,9 +8,18 @@
  * wires in) — when no summariser is provided, the representative observation's
  * own summary is kept verbatim.
  *
+ * Supersede (the mem0/Letta decision layer): clustering alone left every
+ * near-duplicate live, so search returned N restatements of the same fact. The
+ * supersede pass keeps only the highest-salience member of a tight cluster live
+ * and archives the rest behind a `supersededBy` pointer — salience (the
+ * observation's confidence) decides the winner, the newest breaks a tie. The
+ * winner becomes the cluster representative, so the kept insight points at the
+ * most valuable member rather than the lexically-first one.
+ *
  * Outputs:
  *   .cx/observations/consolidated.json    list of insight clusters
  *   .cx/observations/archive/<id>.json    observations demoted to cold archive
+ *                                         (superseded members carry supersededBy)
  *
  * The pass is idempotent: re-running it on a stable corpus produces the same
  * cluster set. It is safe to schedule as a cron / launchd job, run in the
@@ -24,6 +33,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { cosineSimilarity } from '../storage/embeddings.mjs';
+import { detectContradiction } from './contradiction.mjs';
 
 const OBS_DIR = '.cx/observations';
 const ARCHIVE_DIR = '.cx/observations/archive';
@@ -35,6 +45,19 @@ const DEFAULTS = {
   similarityThreshold: 0.95,
   archiveAfterDays: 60,
   archiveBelowConfidence: 0.5,
+  // Supersede only collapses a member that is a true restatement of the winner,
+  // a stricter bar than the cluster-forming threshold: cluster-adjacent is not
+  // the same as duplicate.
+  supersedeDuplicates: true,
+  supersedeThreshold: 0.97,
+  // Contradiction sits in the band between same-subject and duplicate: low
+  // enough that the flipped claim drops cosine out of the duplicate range, high
+  // enough that the two are still about the same thing. The O(n²) scan is
+  // bounded by contradictionScanMax so a large store degrades to a no-op rather
+  // than a stall.
+  detectContradictions: true,
+  contradictionMinSimilarity: 0.75,
+  contradictionScanMax: 1500,
   maxStored: 5000,
   // Archive retention: keep at most this many archived files, and delete
   // anything older than archiveRetainDays. Both bounds apply.
@@ -56,12 +79,27 @@ function readObservation(rootDir, id) {
   return readJsonOrEmpty(filePath, null);
 }
 
-function archiveObservation(rootDir, id) {
+function archiveObservation(rootDir, id, supersededBy = null, reason = null) {
   const src = path.join(rootDir, OBS_DIR, `${id}.json`);
   if (!fs.existsSync(src)) return false;
   const archiveDir = path.join(rootDir, ARCHIVE_DIR);
   fs.mkdirSync(archiveDir, { recursive: true });
   const dest = path.join(archiveDir, `${id}.json`);
+
+  // A superseded member records what replaced it and why before leaving the
+  // live store, so the archive stays auditable and a future undo knows both the
+  // winner and whether it was a restatement or a contradiction.
+  if (supersededBy) {
+    const record = readJsonOrEmpty(src, null);
+    if (record) {
+      record.supersededBy = supersededBy;
+      record.supersededAt = new Date().toISOString();
+      if (reason) record.supersededReason = reason;
+      fs.writeFileSync(dest, JSON.stringify(record, null, 2) + '\n');
+      fs.unlinkSync(src);
+      return true;
+    }
+  }
   fs.renameSync(src, dest);
   return true;
 }
@@ -71,6 +109,77 @@ function daysAgo(dateStr) {
   return (Date.now() - new Date(dateStr).getTime()) / (24 * 60 * 60 * 1000);
 }
 
+// The winner of a near-duplicate cluster is the member worth keeping live:
+// highest salience (confidence) first, then the most recent restatement, then a
+// stable id tiebreak so the choice is deterministic across runs.
+function rankForSupersede(a, b) {
+  const conf = (b.confidence || 0) - (a.confidence || 0);
+  if (conf !== 0) return conf;
+  const at = a.createdAt ? new Date(a.createdAt).getTime() : 0;
+  const bt = b.createdAt ? new Date(b.createdAt).getTime() : 0;
+  if (bt !== at) return bt - at;
+  return String(a.id).localeCompare(String(b.id));
+}
+
+// A contradiction resolves the opposite way from a restatement: the world
+// changed, so the most recent claim wins regardless of salience; salience and a
+// stable id only break a same-timestamp tie.
+function rankByRecency(a, b) {
+  const at = a.createdAt ? new Date(a.createdAt).getTime() : 0;
+  const bt = b.createdAt ? new Date(b.createdAt).getTime() : 0;
+  if (bt !== at) return bt - at;
+  const conf = (b.confidence || 0) - (a.confidence || 0);
+  if (conf !== 0) return conf;
+  return String(a.id).localeCompare(String(b.id));
+}
+
+/**
+ * Scan a record set for same-subject contradictions and archive the older of
+ * each contradicting pair behind a supersededBy pointer with reason
+ * 'contradiction'. Mutates nothing; returns the loser ids and the supersede
+ * entries. Bounded by scanMax — above it the O(n²) scan is skipped wholesale so
+ * a large store never stalls consolidation.
+ */
+async function scanContradictions(rootDir, records, settings) {
+  const superseded = [];
+  const losers = new Set();
+  if (records.length > settings.contradictionScanMax) {
+    return { superseded, losers, skipped: true };
+  }
+
+  const ordered = [...records].sort(rankByRecency);
+  for (let i = 0; i < ordered.length; i++) {
+    const newer = ordered[i];
+    if (losers.has(newer.id)) continue;
+    for (let j = i + 1; j < ordered.length; j++) {
+      const older = ordered[j];
+      if (losers.has(older.id)) continue;
+      if (!newer.embedding?.length || !older.embedding?.length) continue;
+
+      const sim = cosineSimilarity(newer.embedding, older.embedding);
+      const sameSubject = sim >= settings.contradictionMinSimilarity && sim < settings.supersedeThreshold;
+      if (!sameSubject) continue;
+
+      // The heuristic runs first and for free; the optional judge is consulted
+      // only on its misses (the value-swap case it abstains on). Awaiting covers
+      // both a sync local judge and a future async one.
+      const heuristic = detectContradiction(newer.summary, older.summary);
+      let contradicts = heuristic.contradicts;
+      if (!contradicts && settings.contradictionJudge?.judge) {
+        try { contradicts = !!((await settings.contradictionJudge.judge(newer, older))?.contradicts); }
+        catch { contradicts = false; }
+      }
+      if (!contradicts) continue;
+
+      if (archiveObservation(rootDir, older.id, newer.id, 'contradiction')) {
+        losers.add(older.id);
+        superseded.push({ id: older.id, supersededBy: newer.id, reason: 'contradiction' });
+      }
+    }
+  }
+  return { superseded, losers, skipped: false };
+}
+
 /**
  * Greedy single-pass clustering. Each observation is added to the existing
  * cluster whose centroid has highest cosine similarity, provided that
@@ -145,9 +254,15 @@ async function buildSummary(cluster, summariser) {
  * @param {number} [opts.similarityThreshold=0.95]
  * @param {number} [opts.archiveAfterDays=60]
  * @param {number} [opts.archiveBelowConfidence=0.5]
+ * @param {boolean} [opts.supersedeDuplicates=true] - archive tight restatements, keeping the highest-salience member live
+ * @param {number} [opts.supersedeThreshold=0.97] - cosine bar a member must clear against the winner to be superseded
+ * @param {boolean} [opts.detectContradictions=true] - archive the older of a same-subject contradicting pair (newest wins)
+ * @param {number} [opts.contradictionMinSimilarity=0.75] - cosine floor for two observations to count as same-subject
+ * @param {number} [opts.contradictionScanMax=1500] - skip the O(n²) contradiction scan above this many live records
+ * @param {object} [opts.contradictionJudge] - optional plugin with `judge(a, b) -> { contradicts }` for the value-swap case the heuristic misses
  * @param {number} [opts.maxStored=5000]
  * @param {object} [opts.summariser] - Compressor-shaped plugin
- * @returns {Promise<{ clustersBefore: number, clusters: number, archived: string[], stored: number }>}
+ * @returns {Promise<{ clustersBefore: number, clusters: number, archived: string[], superseded: Array<{id: string, supersededBy: string, reason: string}>, contradictionScanSkipped: boolean, stored: number }>}
  */
 export async function consolidate(rootDir, opts = {}) {
   const settings = { ...DEFAULTS, ...opts };
@@ -177,7 +292,7 @@ export async function consolidate(rootDir, opts = {}) {
   const clustersBefore = records.length;
 
   const archived = [];
-  const liveRecords = records.filter((r) => {
+  let liveRecords = records.filter((r) => {
     const old = daysAgo(r.createdAt) > settings.archiveAfterDays;
     const lowConfidence = r.confidence < settings.archiveBelowConfidence;
     if (old && lowConfidence) {
@@ -187,8 +302,47 @@ export async function consolidate(rootDir, opts = {}) {
     return true;
   });
 
+  const superseded = [];
+
+  // Contradiction runs before clustering: a flipped claim sits below the
+  // duplicate threshold, so it would never cluster with what it contradicts and
+  // both stale and current statements would stay live. The older loser leaves
+  // the set so it is not clustered or returned by search.
+  let contradictionScanSkipped = false;
+  if (settings.detectContradictions) {
+    const result = await scanContradictions(rootDir, liveRecords, settings);
+    contradictionScanSkipped = result.skipped;
+    for (const id of result.losers) archived.push(id);
+    superseded.push(...result.superseded);
+    if (result.losers.size > 0) {
+      liveRecords = liveRecords.filter((r) => !result.losers.has(r.id));
+    }
+  }
+
   const clusters = greedyCluster(liveRecords, settings.similarityThreshold);
 
+  // Supersede tight restatements: rank each multi-member cluster by salience so
+  // the winner leads (it becomes the representative), then archive any member
+  // that duplicates the winner closely enough to be the same fact.
+  if (settings.supersedeDuplicates) {
+    for (const cluster of clusters) {
+      if (cluster.members.length < 2) continue;
+      cluster.members.sort(rankForSupersede);
+      const winner = cluster.members[0];
+      cluster.supersededIds = [];
+      for (const loser of cluster.members.slice(1)) {
+        const tight = winner.embedding?.length && loser.embedding?.length &&
+          cosineSimilarity(winner.embedding, loser.embedding) >= settings.supersedeThreshold;
+        if (!tight) continue;
+        if (archiveObservation(rootDir, loser.id, winner.id, 'restatement')) {
+          archived.push(loser.id);
+          superseded.push({ id: loser.id, supersededBy: winner.id, reason: 'restatement' });
+          cluster.supersededIds.push(loser.id);
+        }
+      }
+    }
+  }
+
   const consolidated = [];
   for (const cluster of clusters) {
     const representative = cluster.members[0];
@@ -206,6 +360,7 @@ export async function consolidate(rootDir, opts = {}) {
       category: representative.category,
       tags: [...new Set(cluster.members.flatMap((m) => m.tags || []))].slice(0, 10),
       memberIds: cluster.members.map((m) => m.id),
+      supersededIds: cluster.supersededIds || [],
       avgConfidence:
         cluster.members.reduce((acc, m) => acc + (m.confidence || 0), 0) /
         Math.max(cluster.members.length, 1),
@@ -247,6 +402,8 @@ export async function consolidate(rootDir, opts = {}) {
     clustersBefore,
     clusters: consolidated.length,
     archived,
+    superseded,
+    contradictionScanSkipped,
     archivePruned,
     stored: consolidated.length,
   };

package/lib/engine/contradiction-judge.mjs

@@ -0,0 +1,71 @@
+/**
+ * lib/engine/contradiction-judge.mjs — optional LLM second opinion for the
+ * consolidation contradiction pass.
+ *
+ * The negation-polarity heuristic (lib/engine/contradiction.mjs) catches a
+ * flipped claim but abstains on a value swap with no negation cue — "auth uses
+ * RS256" vs "auth uses HS256" share every word yet disagree. Resolving that
+ * needs to know the two values are mutually exclusive, which is semantic
+ * judgment. This factory builds the judge consolidation consults only when the
+ * heuristic says "no".
+ *
+ * Offline-first: the judge is backed by a local Ollama model and the factory
+ * returns null when Ollama is not running or has no model, so consolidation
+ * degrades to heuristic-only with no provider and no key required. The status
+ * probe and the runner are injectable so the parse logic is testable without a
+ * live model.
+ */
+
+import { checkOllamaStatus, testModel } from '../ollama-manager.mjs';
+
+function buildPrompt(a, b) {
+  const textA = (a?.summary || a?.content || '').slice(0, 400);
+  const textB = (b?.summary || b?.content || '').slice(0, 400);
+  return [
+    'Two short notes describe the same subject. Decide whether they directly',
+    'contradict — assert mutually exclusive facts (e.g. different algorithms,',
+    'opposite states, incompatible values). A restatement or an added detail is',
+    'NOT a contradiction. Answer with only YES or NO.',
+    '',
+    `Note A: ${textA}`,
+    `Note B: ${textB}`,
+    'Answer:',
+  ].join('\n');
+}
+
+function parseVerdict(response) {
+  const match = String(response || '').toLowerCase().match(/\b(yes|no)\b/);
+  return { contradicts: !!match && match[1] === 'yes' };
+}
+
+/**
+ * Build a contradiction judge, or null when no local model is available.
+ *
+ * @param {object} [opts]
+ * @param {() => {running?: boolean, models?: Array<{name?: string, model?: string}>}} [opts.statusFn]
+ * @param {(model: string, prompt: string) => {success?: boolean, response?: string}} [opts.runFn]
+ * @param {string} [opts.model] explicit model override (else env, else first available)
+ * @returns {{judge: (a: object, b: object) => {contradicts: boolean}} | null}
+ */
+export function createContradictionJudge({ statusFn = checkOllamaStatus, runFn = testModel, model } = {}) {
+  const status = statusFn();
+  if (!status?.running || !status.models?.length) return null;
+
+  const chosen = model || process.env.CONSTRUCT_CONTRADICTION_JUDGE_MODEL ||
+    status.models[0]?.name || status.models[0]?.model;
+  if (!chosen) return null;
+
+  return {
+    judge(a, b) {
+      try {
+        const result = runFn(chosen, buildPrompt(a, b));
+        if (!result?.success) return { contradicts: false };
+        return parseVerdict(result.response);
+      } catch {
+        return { contradicts: false };
+      }
+    },
+  };
+}
+
+export const __testing = { buildPrompt, parseVerdict };