@geraldmaron/construct 1.0.14 → 1.0.15

package/lib/contracts/validate.mjs

@@ -19,6 +19,8 @@
 import { existsSync, readFileSync } from 'node:fs';
 import { join, dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { POSTCONDITIONS, validateBinaryPostconditions } from '../specialists/postconditions.mjs';
+import { logViolation } from './violation-log.mjs';
 
 const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
 const CONTRACTS_PATH = join(REPO_ROOT, 'specialists', 'contracts.json');
@@ -145,28 +147,62 @@ export function findContract({ producer, consumer, id, contractsPath = CONTRACTS
  *   { ok: true, contract }
  *   { ok: false, status: 'BLOCKED_CONTRACT', errors[], contract }
  *
- * Enforcement defaults to the value of process.env.CONSTRUCT_CONTRACT_ENFORCEMENT
- * ('warn' or 'block'), and may be overridden by the caller. In warn mode the
- * result includes errors but ok stays true so a session is not blocked.
+ * Enforcement defaults to `block` (CISA Secure-by-Design pledge —
+ * strong-default configurations). Callers can pass `enforcement: 'warn'`
+ * explicitly per-call when they genuinely need advisory mode.
+ *
+ * Binary postconditions are enforced when the producer has a rule table in
+ * `lib/specialists/postconditions.mjs`. The check is self-enforcing: if a
+ * producer has rules and the caller does not pass a `packet`, the call is
+ * itself a contract violation (MCP best-practice 2025 — no silent skip on
+ * tool-argument omission).
  */
 export function validateHandoff({
   producer,
   consumer,
   id,
   artifact,
+  packet,
   contractsPath = CONTRACTS_PATH,
   repoRoot = REPO_ROOT,
-  enforcement = process.env.CONSTRUCT_CONTRACT_ENFORCEMENT || 'warn',
+  enforcement = 'block',
 }) {
+  const errors = [];
+  const postconditionFailures = [];
+
+  // Binary postconditions are producer-bound invariants that fire regardless
+  // of whether the producer→consumer contract is registered in contracts.json.
+  // A producer with rules cannot have its packet validation skipped by
+  // claiming an unregistered consumer.
+  const hasBinaryRules = Array.isArray(POSTCONDITIONS[producer]) && POSTCONDITIONS[producer].length > 0;
+  if (hasBinaryRules) {
+    if (packet === undefined || packet === null) {
+      errors.push(`producer '${producer}' has binary postconditions; validateHandoff must be called with a packet argument`);
+    } else {
+      const binary = validateBinaryPostconditions(producer, packet);
+      for (const failure of binary.failures) {
+        errors.push(`[${failure.id}] ${failure.reason}`);
+        postconditionFailures.push(failure);
+      }
+    }
+  }
+
   const contract = findContract({ producer, consumer, id, contractsPath });
   if (!contract) {
+    if (errors.length > 0) {
+      logViolation(`${producer}->${consumer}`, 'output', errors, packet ?? artifact, {
+        verdict: postconditionFailures.length > 0 ? 'BLOCKED_CONTRACT' : 'CONTRACT_VIOLATION',
+        postconditionFailures,
+      });
+      return enforcement === 'block'
+        ? { ok: false, status: 'BLOCKED_CONTRACT', errors: [`no contract found for ${producer}→${consumer}${id ? ` id=${id}` : ''}`, ...errors], contract: null }
+        : { ok: true, warnings: [`no contract found for ${producer}→${consumer}`, ...errors], contract: null };
+    }
     return enforcement === 'block'
       ? { ok: false, status: 'BLOCKED_CONTRACT', errors: [`no contract found for ${producer}→${consumer}${id ? ` id=${id}` : ''}`], contract: null }
       : { ok: true, warnings: [`no contract found for ${producer}→${consumer}`], contract: null };
   }
 
-  const errors = [];
-
   // A handoff carries a producer's output, which is the consumer's input. The
   // input contract is what the consumer expects to receive; check mustContain
   // against that first. Output.schema validation only applies when the artifact
@@ -193,18 +229,26 @@ export function validateHandoff({
     }
   }
 
-  // Postcondition checks run against the artifact file on disk when the
-  // handoff envelope declares `artifactPath`. Resolved relative to repoRoot.
+  // Disk-artifact postconditions: frontmatter, sections, citations.
   const declaredArtifactPath = artifact?.artifactPath || artifact?.output?.artifactPath;
   if (declaredArtifactPath) {
     const absPath = isAbsolutePath(declaredArtifactPath)
       ? declaredArtifactPath
       : join(repoRoot, declaredArtifactPath);
-    const pcErrors = validatePostconditions({ contract, artifactPath: absPath });
+    const pcErrors = validateArtifactPostconditions({ contract, artifactPath: absPath });
     for (const e of pcErrors) errors.push(e);
   }
 
   if (errors.length === 0) return { ok: true, contract };
+
+  // Persist to the chain-hashed violation log so `construct doctor` and
+  // forensic replay see the failure. Best-effort; logging never throws.
+  const verdict = postconditionFailures.length > 0 ? 'BLOCKED_CONTRACT' : 'CONTRACT_VIOLATION';
+  logViolation(contract.id, 'output', errors, packet ?? artifact, {
+    verdict,
+    postconditionFailures,
+  });
+
   if (enforcement === 'block') return { ok: false, status: 'BLOCKED_CONTRACT', errors, contract };
   return { ok: true, warnings: errors, contract };
 }
@@ -272,7 +316,7 @@ function claimsHaveCitations(body) {
  * disk. Postconditions can be strings (descriptive, ignored here) or objects
  * with { id, check, field?, section? }. Returns an array of error strings.
  */
-export function validatePostconditions({ contract, artifactPath }) {
+export function validateArtifactPostconditions({ contract, artifactPath }) {
   if (!contract?.postconditions?.length) return [];
   if (!artifactPath || !existsSync(artifactPath)) {
     const structured = contract.postconditions.filter((p) => typeof p === 'object');

package/lib/contracts/violation-log.mjs

@@ -0,0 +1,154 @@
+/**
+ * lib/contracts/violation-log.mjs — tamper-evident append-only log for
+ * specialist-contract violations.
+ *
+ * Each record carries:
+ *   ts                  — ISO timestamp
+ *   sequence            — monotonic per-log counter, gap-detects truncation
+ *   prev_line_hash      — sha256 of the prior line, gap-detects modification
+ *   agent               — last-known active agent (from ~/.cx/last-agent.json)
+ *   contractId          — the contract that fired the violation
+ *   direction           — 'input' | 'output'
+ *   missing             — field names that failed shape validation
+ *   packet_keys         — Object.keys(packet) when packet is an object
+ *   verdict             — 'CONTRACT_VIOLATION' (default) or 'BLOCKED_CONTRACT'
+ *                         (binary postcondition failures)
+ *   postconditionFailures — [{id, reason}] when verdict is BLOCKED_CONTRACT
+ *
+ * The hash chain spans rotation: when the active file is empty (post-rotate),
+ * the tail is read from the most recent segment (gzipped or plain) via
+ * readLastLineAcrossSegments. The sequence counter follows the same path.
+ *
+ * Single owner of the file. All readers go through recentViolations or
+ * verifyChain; all writers go through logViolation.
+ */
+
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { homedir } from 'node:os';
+import { createHash } from 'node:crypto';
+import { appendBounded, readLastLineAcrossSegments } from '../logging/rotate.mjs';
+import { resolveProjectScopedPath } from '../project-root.mjs';
+
+const CX_DIR = join(homedir(), '.cx');
+const LAST_AGENT = join(CX_DIR, 'last-agent.json');
+
+// contract-violations are PROJECT-SCOPED: resolves to
+// <project>/.cx/contract-violations.jsonl when inside a project, falling
+// back to ~/.cx for standalone invocations. Resolved on every call so
+// cwd/HOME changes inside the same process (tests, harness reuse) route
+// correctly.
+
+function logFile() {
+  return resolveProjectScopedPath('contract-violations.jsonl', { ensureDir: false });
+}
+
+function sha256(input) { return createHash('sha256').update(input).digest('hex'); }
+
+function readLastAgent() {
+  try { return JSON.parse(readFileSync(LAST_AGENT, 'utf8'))?.agent || 'construct'; }
+  catch { return 'construct'; }
+}
+
+function readTailRecord() {
+  const file = logFile();
+  const lastLine = readLastLineAcrossSegments(file);
+  if (!lastLine) return null;
+  try { return JSON.parse(lastLine); }
+  catch { return null; }
+}
+
+function readPrevLineHash() {
+  const file = logFile();
+  const lastLine = readLastLineAcrossSegments(file);
+  return lastLine ? sha256(lastLine) : null;
+}
+
+function nextSequence() {
+  const tail = readTailRecord();
+  const prior = Number.isInteger(tail?.sequence) ? tail.sequence : 0;
+  return prior + 1;
+}
+
+/**
+ * Append a violation record. Best-effort: file I/O failures are swallowed
+ * so logging never crashes the caller.
+ */
+export function logViolation(contractId, direction, missing, packet, extra = {}) {
+  try {
+    const file = logFile();
+    mkdirSync(dirname(file), { recursive: true });
+    const record = {
+      ts: new Date().toISOString(),
+      sequence: nextSequence(),
+      agent: readLastAgent(),
+      contractId,
+      direction,
+      missing,
+      packet_keys: packet && typeof packet === 'object' ? Object.keys(packet) : null,
+      prev_line_hash: readPrevLineHash(),
+      ...extra,
+    };
+    appendBounded('contract-violations', file, JSON.stringify(record) + '\n');
+  } catch { /* logging is best-effort */ }
+}
+
+/**
+ * Read recent violations from the active log segment. Returns oldest-first.
+ * Rotated segments are intentionally not included — this is the doctor
+ * surface, scoped to the active window.
+ */
+export function recentViolations({ windowMs = 24 * 60 * 60 * 1000 } = {}) {
+  const file = logFile();
+  if (!existsSync(file)) return [];
+  try {
+    const cutoff = Date.now() - windowMs;
+    return readFileSync(file, 'utf8')
+      .trim()
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => { try { return JSON.parse(line); } catch { return null; } })
+      .filter((r) => r && new Date(r.ts).getTime() >= cutoff);
+  } catch { return []; }
+}
+
+/**
+ * Walk the active log segment, asserting:
+ *   - each record's prev_line_hash matches sha256 of the prior line
+ *   - sequence numbers are monotonic with no gaps
+ *
+ * Returns { ok, brokenAt? } where brokenAt is { index, reason } on first
+ * failure. An empty / missing log is `{ ok: true }` (nothing to break).
+ */
+export function verifyChain() {
+  const file = logFile();
+  if (!existsSync(file)) return { ok: true };
+  try {
+    const lines = readFileSync(file, 'utf8').trim().split('\n').filter(Boolean);
+    let priorLine = null;
+    let priorSeq = null;
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      let record;
+      try { record = JSON.parse(line); }
+      catch { return { ok: false, brokenAt: { index: i, reason: 'unparseable JSON' } }; }
+
+      if (priorLine !== null) {
+        const expectedHash = sha256(priorLine);
+        if (record.prev_line_hash !== expectedHash) {
+          return { ok: false, brokenAt: { index: i, reason: 'prev_line_hash mismatch' } };
+        }
+      }
+      if (Number.isInteger(record.sequence)) {
+        if (priorSeq !== null && record.sequence !== priorSeq + 1) {
+          return { ok: false, brokenAt: { index: i, reason: `sequence gap (expected ${priorSeq + 1}, got ${record.sequence})` } };
+        }
+        priorSeq = record.sequence;
+      }
+      priorLine = line;
+    }
+    return { ok: true };
+  } catch (err) {
+    return { ok: false, brokenAt: { index: -1, reason: err.message } };
+  }
+}

package/lib/document-extract/docling-client.mjs

@@ -0,0 +1,114 @@
+/**
+ * lib/document-extract/docling-client.mjs — Node side of the docling sidecar.
+ *
+ * Spawns one long-lived Python process per Node session, framed by newline-
+ * delimited JSON over stdin/stdout. Avoids per-call uv/venv warm-up
+ * (~2s otherwise) and keeps the docling model resident.
+ *
+ * Public API:
+ *   extractViaDocling(filePath) → { markdown, metadata, droppedInfo }
+ *   shutdownDoclingSidecar()    → idempotent process exit
+ *
+ * The first call provisions the venv on demand via ensureDoclingVenv().
+ * Subsequent calls reuse the running sidecar.
+ */
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ensureDoclingVenv } from '../runtime/uv-bootstrap.mjs';
+
+const SIDECAR_SCRIPT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), 'docling-sidecar.py');
+const REQUEST_TIMEOUT_MS = 300_000;
+
+let activeSidecar = null;
+let requestSeq = 0;
+
+function spawnSidecar({ runtimeDir } = {}) {
+  const { pythonBin } = ensureDoclingVenv({ runtimeDir });
+  const child = spawn(pythonBin, [SIDECAR_SCRIPT], {
+    stdio: ['pipe', 'pipe', 'pipe'],
+    env: { ...process.env, PYTHONUNBUFFERED: '1' },
+  });
+
+  const pending = new Map();
+  let stdoutBuffer = '';
+  let stderrBuffer = '';
+  let exitReason = null;
+
+  child.stdout.setEncoding('utf8');
+  child.stderr.setEncoding('utf8');
+
+  child.stdout.on('data', (chunk) => {
+    stdoutBuffer += chunk;
+    let newlineIdx;
+    while ((newlineIdx = stdoutBuffer.indexOf('\n')) !== -1) {
+      const line = stdoutBuffer.slice(0, newlineIdx);
+      stdoutBuffer = stdoutBuffer.slice(newlineIdx + 1);
+      if (!line.trim()) continue;
+      let message;
+      try { message = JSON.parse(line); }
+      catch { continue; }
+      const id = message.id;
+      const handler = pending.get(id);
+      if (!handler) continue;
+      pending.delete(id);
+      clearTimeout(handler.timer);
+      if (message.error) handler.reject(Object.assign(new Error(message.error.message || 'docling sidecar error'), { code: message.error.code, trace: message.error.trace }));
+      else handler.resolve(message.result);
+    }
+  });
+
+  child.stderr.on('data', (chunk) => { stderrBuffer += chunk; });
+
+  child.on('exit', (code) => {
+    exitReason = `exit code ${code}`;
+    for (const [, handler] of pending) {
+      clearTimeout(handler.timer);
+      handler.reject(new Error(`docling sidecar exited (${exitReason}); stderr: ${stderrBuffer.slice(-500)}`));
+    }
+    pending.clear();
+    if (activeSidecar === sidecar) activeSidecar = null;
+  });
+
+  function send(method, params = {}) {
+    if (child.killed || child.exitCode !== null) {
+      return Promise.reject(new Error(`docling sidecar not running: ${exitReason || 'killed'}`));
+    }
+    const id = ++requestSeq;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        pending.delete(id);
+        reject(new Error(`docling sidecar timeout after ${REQUEST_TIMEOUT_MS}ms (method=${method})`));
+      }, REQUEST_TIMEOUT_MS);
+      pending.set(id, { resolve, reject, timer });
+      child.stdin.write(JSON.stringify({ id, method, params }) + '\n');
+    });
+  }
+
+  const sidecar = { send, child, get stderr() { return stderrBuffer; } };
+  return sidecar;
+}
+
+function getSidecar() {
+  if (activeSidecar && activeSidecar.child.exitCode === null && !activeSidecar.child.killed) return activeSidecar;
+  activeSidecar = spawnSidecar();
+  return activeSidecar;
+}
+
+export async function extractViaDocling(filePath) {
+  const sidecar = getSidecar();
+  const result = await sidecar.send('extract', { path: path.resolve(filePath) });
+  return result;
+}
+
+export async function shutdownDoclingSidecar() {
+  if (!activeSidecar) return;
+  const sidecar = activeSidecar;
+  activeSidecar = null;
+  try { await sidecar.send('shutdown'); } catch { /* sidecar already gone */ }
+  if (sidecar.child.exitCode === null && !sidecar.child.killed) {
+    sidecar.child.kill('SIGTERM');
+  }
+}
+
+process.on('exit', () => { if (activeSidecar) activeSidecar.child.kill('SIGTERM'); });

package/lib/document-extract/docling-sidecar.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""
+lib/document-extract/docling-sidecar.py — long-lived JSON-RPC wrapper around docling.
+
+Protocol: newline-delimited JSON over stdin/stdout. Each request is
+{"id": <int>, "method": <str>, "params": <obj>}. Each response is
+{"id": <int>, "result": <obj>} or {"id": <int>, "error": {"code": <str>, "message": <str>}}.
+
+Methods:
+  - ping            → {"ok": true, "doclingVersion": "<x.y.z>"}
+  - extract {path}  → {"markdown": "...", "metadata": {...}, "droppedInfo": [...]}
+  - shutdown        → {"ok": true}; process exits after acknowledgement
+
+Best-practice notes (2026-06):
+  - One sidecar per Node session, kept warm to avoid uv/venv startup (~2s).
+  - Stdio JSON-RPC is the leanest Python↔Node IPC for non-LLM use cases;
+    MCP rides the same transport but adds protocol overhead unsuitable
+    for the parser sidecar.
+  - Drops are surfaced explicitly (kind/count/reason/recoverable) so info
+    loss is observable to the CLI, not silent.
+"""
+import json
+import sys
+import traceback
+from pathlib import Path
+
+try:
+    from docling.document_converter import DocumentConverter
+    from importlib.metadata import version as _pkg_version
+    DOCLING_VERSION = _pkg_version("docling")
+except ImportError as exc:
+    sys.stderr.write(json.dumps({"fatal": "docling-import-failed", "detail": str(exc)}) + "\n")
+    sys.exit(2)
+
+
+_converter = None
+
+
+def get_converter():
+    global _converter
+    if _converter is None:
+        _converter = DocumentConverter()
+    return _converter
+
+
+def extract(params):
+    path_raw = params.get("path")
+    if not path_raw:
+        raise ValueError("missing 'path' parameter")
+    path = Path(path_raw)
+    if not path.exists():
+        raise FileNotFoundError(f"file not found: {path}")
+
+    result = get_converter().convert(str(path))
+    doc = result.document
+    markdown = doc.export_to_markdown()
+
+    metadata = {
+        "format": result.input.format.value if hasattr(result.input, "format") else None,
+        "pageCount": len(doc.pages) if hasattr(doc, "pages") and doc.pages is not None else None,
+        "doclingVersion": DOCLING_VERSION,
+        "sourcePath": str(path),
+    }
+
+    dropped_info = []
+    try:
+        page_count = metadata.get("pageCount") or 0
+        if page_count and len(markdown) / max(page_count, 1) < 50:
+            dropped_info.append({
+                "kind": "low-text-yield",
+                "count": page_count,
+                "reason": "Extracted text density below 50 chars/page suggests image-heavy or scanned content; OCR may have partial coverage.",
+                "recoverable": True,
+            })
+    except Exception:
+        pass
+
+    return {
+        "markdown": markdown,
+        "metadata": metadata,
+        "droppedInfo": dropped_info,
+    }
+
+
+def handle(request):
+    method = request.get("method")
+    params = request.get("params") or {}
+    if method == "ping":
+        return {"ok": True, "doclingVersion": DOCLING_VERSION}
+    if method == "extract":
+        return extract(params)
+    if method == "shutdown":
+        return {"ok": True}
+    raise ValueError(f"unknown method: {method}")
+
+
+def main():
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        request_id = None
+        try:
+            request = json.loads(line)
+            request_id = request.get("id")
+            result = handle(request)
+            sys.stdout.write(json.dumps({"id": request_id, "result": result}) + "\n")
+            sys.stdout.flush()
+            if request.get("method") == "shutdown":
+                return
+        except Exception as exc:
+            error = {
+                "code": type(exc).__name__,
+                "message": str(exc),
+                "trace": traceback.format_exc() if not isinstance(exc, (ValueError, FileNotFoundError)) else None,
+            }
+            sys.stdout.write(json.dumps({"id": request_id, "error": error}) + "\n")
+            sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    main()

package/lib/document-extract/whisper-client.mjs

@@ -0,0 +1,79 @@
+/**
+ * lib/document-extract/whisper-client.mjs — invoke whisper.cpp on audio/video.
+ *
+ * Returns `{ markdown, metadata, droppedInfo }` matching the shape returned
+ * by docling-client for symmetry with the document-extract dispatcher.
+ *
+ * Strategy: run whisper-cli with the source file → text transcript. The
+ * transcript is wrapped as a fenced "## Transcript" section in markdown so
+ * the format-converter pipeline downstream gets a uniform shape.
+ *
+ * 2026-06 notes:
+ *   - whisper.cpp + Metal hits ~10× real-time on M-series for large-v3.
+ *     Default model is `base.en` for speed; override via
+ *     CONSTRUCT_WHISPER_MODEL for non-English or higher-quality runs.
+ *   - Non-audio formats are rejected up front so the docling sidecar stays
+ *     the single owner of PDF/Office/HTML parsing.
+ */
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+import os from 'node:os';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { ensureWhisperRuntime } from '../runtime/whisper-bootstrap.mjs';
+
+const CLI_TIMEOUT_MS = 600_000;
+
+function runWhisperCli({ binary, modelPath, sourcePath, outputBase }) {
+  return new Promise((resolve, reject) => {
+    const args = [
+      '-m', modelPath,
+      '-f', sourcePath,
+      '-otxt',
+      '-of', outputBase,
+      '-l', 'auto',
+      '-nt',
+    ];
+    const child = spawn(binary, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (c) => { stdout += c; });
+    child.stderr.on('data', (c) => { stderr += c; });
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      reject(new Error(`whisper-cli timeout after ${CLI_TIMEOUT_MS}ms`));
+    }, CLI_TIMEOUT_MS);
+    child.on('exit', (code) => {
+      clearTimeout(timer);
+      if (code !== 0) reject(new Error(`whisper-cli exited ${code}: ${stderr.slice(-500)}`));
+      else resolve({ stdout, stderr });
+    });
+    child.on('error', (err) => { clearTimeout(timer); reject(err); });
+  });
+}
+
+export async function extractViaWhisper(filePath, { runtimeDir, model } = {}) {
+  const absolutePath = path.resolve(filePath);
+  const { binary, modelPath, model: resolvedModel } = await ensureWhisperRuntime({ runtimeDir, model });
+  const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'construct-whisper-'));
+  const outputBase = path.join(tmpDir, 'transcript');
+  try {
+    const { stderr } = await runWhisperCli({ binary, modelPath, sourcePath: absolutePath, outputBase });
+    const transcript = readFileSync(`${outputBase}.txt`, 'utf8').trim();
+    const detectedLanguage = stderr.match(/auto-detected language: (\w+)/)?.[1] || null;
+    const markdown = `## Transcript\n\n${transcript}\n`;
+    return {
+      markdown,
+      metadata: {
+        sourcePath: absolutePath,
+        whisperModel: resolvedModel,
+        binarySource: binary,
+        detectedLanguage,
+      },
+      droppedInfo: transcript.length === 0
+        ? [{ kind: 'empty-transcript', count: 1, reason: 'whisper-cli returned no text — silent file, unsupported codec, or model mismatch.', recoverable: true }]
+        : [],
+    };
+  } finally {
+    try { rmSync(tmpDir, { recursive: true, force: true }); } catch { /* tmp cleanup best-effort */ }
+  }
+}

package/lib/document-extract.mjs

@@ -1,15 +1,19 @@
 /**
  * lib/document-extract.mjs — shared local document extraction for retrieval and MCP reads.
  *
- * Handles plain-text formats directly, XML/HTML via lightweight tag stripping,
- * Office zip containers via unzip, and PDFs via pdftotext or macOS-native fallbacks.
- * Also provides extractDocumentMetadata() for structured metadata (title, authors,
- * dates, links) from source document frontmatter and content.
+ * Two extraction paths:
+ *   - Async (preferred): `extractDocumentTextAsync` routes PDF/Office/HTML
+ *     through the docling Python sidecar and audio/video through whisper.cpp.
+ *     Returns { text, markdown, metadata, droppedInfo, ... }.
+ *   - Sync (legacy): `extractDocumentText` keeps regex/CLI extractors.
+ *     PDF/Office output is lower fidelity (tables flattened, formulas
+ *     dropped, scanned pages return empty). Audio/video throws
+ *     { code: 'ASR_REQUIRED' } on this path.
  *
- * All paths now return { text, structured, droppedInfo, method, ... } where
- * droppedInfo is an array of { kind, count, reason, recoverable } objects
- * describing information that was silently dropped during extraction.
- * Audio/video files throw a typed { code: 'ASR_REQUIRED' } error.
+ * `extractDocumentMetadata` returns frontmatter + title/authors/dates.
+ *
+ * All paths return droppedInfo as { kind, count, reason, recoverable }[] so
+ * info loss is observable rather than silent.
  */
 import { existsSync, readFileSync, statSync } from 'node:fs';
 import { basename, extname, resolve } from 'node:path';
@@ -689,3 +693,64 @@ export function extractDocumentText(filePath, { maxChars = null } = {}) {
   if (extracted.skipped) result.skipped = extracted.skipped;
   return result;
 }
+
+// Async high-fidelity extraction path. Routes PDF/Office/HTML through the
+// docling Python sidecar (layout-aware markdown) and audio/video through
+// whisper.cpp (Metal-accelerated on macOS). Text/transcript/calendar/email
+// paths reuse the sync extraction since regex-stripping is the right tool
+// for those formats. Returns the same { text, droppedInfo, ... } shape as
+// the sync extractor plus a `markdown` field for paths that produce it.
+//
+// 2026-06 best practice: this is the path new callers should use. The sync
+// extractDocumentText is preserved for backwards compatibility with
+// pre-async callers; PDF/Office output via that path is lower fidelity
+// (tables flattened, formulas dropped) and emits a droppedInfo notice.
+
+export async function extractDocumentTextAsync(filePath, { maxChars = null } = {}) {
+  const resolvedPath = resolve(filePath);
+  if (!existsSync(resolvedPath)) throw new Error(`File not found: ${resolvedPath}`);
+  const extension = extname(resolvedPath).toLowerCase();
+  if (!EXTRACTABLE_DOCUMENT_EXTS.has(extension)) {
+    throw new Error(`Unsupported document type: ${extension || 'unknown'}`);
+  }
+
+  if (AUDIO_VIDEO_EXTS.has(extension)) {
+    const { extractViaWhisper } = await import('./document-extract/whisper-client.mjs');
+    const out = await extractViaWhisper(resolvedPath);
+    return finalizeAsyncResult({ resolvedPath, extension, extractionMethod: 'whisper', extracted: out, maxChars });
+  }
+
+  const doclingFormats = new Set([
+    '.pdf',
+    ...ZIP_DOCUMENT_EXTS,
+    ...RICH_TEXT_EXTS,
+    ...MDLS_DOCUMENT_EXTS,
+  ]);
+  if (doclingFormats.has(extension)) {
+    const { extractViaDocling } = await import('./document-extract/docling-client.mjs');
+    const out = await extractViaDocling(resolvedPath);
+    return finalizeAsyncResult({ resolvedPath, extension, extractionMethod: 'docling', extracted: out, maxChars });
+  }
+
+  return extractDocumentText(resolvedPath, { maxChars });
+}
+
+function finalizeAsyncResult({ resolvedPath, extension, extractionMethod, extracted, maxChars }) {
+  const text = extracted.markdown ?? extracted.text ?? '';
+  const limit = Number.isFinite(Number(maxChars)) && Number(maxChars) > 0
+    ? Math.min(Number(maxChars), 200_000)
+    : null;
+  const truncated = limit !== null && text.length > limit;
+  return {
+    filePath: resolvedPath,
+    extension,
+    extractionMethod,
+    text: truncated ? `${text.slice(0, limit)}\n` : text,
+    markdown: extracted.markdown ?? null,
+    metadata: extracted.metadata ?? null,
+    truncated,
+    characters: text.length,
+    droppedInfo: extracted.droppedInfo ?? [],
+    structured: extracted.structured ?? null,
+  };
+}