@geraldmaron/construct 1.0.24 → 1.1.0

package/lib/bridges/copilot-proxy.mjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+/**
+ * lib/bridges/copilot-proxy.mjs — Minimal GitHub Copilot to OpenAI-compatible bridge.
+ *
+ * Translates standard OpenAI /v1/chat/completions requests to the Copilot API
+ * using the user's active `gh` CLI session.
+ */
+import http from 'node:http';
+import { execSync } from 'node:child_process';
+
+const PORT = parseInt(process.argv.find(a => a.startsWith('--port='))?.split('=')[1] || '5174', 10);
+
+let cachedSession = null;
+
+async function getCopilotToken() {
+  if (cachedSession && cachedSession.expires_at > (Date.now() / 1000) + 60) {
+    return cachedSession.token;
+  }
+
+  try {
+    const ghToken = execSync('gh auth token', { encoding: 'utf8' }).trim();
+    const res = await fetch('https://api.github.com/copilot_internal/v2/token', {
+      headers: {
+        'Authorization': `Bearer ${ghToken}`,
+        'Accept': 'application/json',
+      }
+    });
+
+    if (!res.ok) throw new Error(`Failed to get session token: ${res.statusText}`);
+    
+    cachedSession = await res.json();
+    return cachedSession.token;
+  } catch (err) {
+    console.error('Error fetching Copilot token:', err.message);
+    return null;
+  }
+}
+
+const server = http.createServer(async (req, res) => {
+  // CORS
+  res.setHeader('Access-Control-Allow-Origin', '*');
+  res.setHeader('Access-Control-Allow-Methods', 'POST, GET, OPTIONS');
+  res.setHeader('Access-Control-Allow-Headers', '*');
+
+  if (req.method === 'OPTIONS') {
+    res.writeHead(204);
+    res.end();
+    return;
+  }
+
+  if (req.url === '/v1/chat/completions' && req.method === 'POST') {
+    const token = await getCopilotToken();
+    if (!token) {
+      res.writeHead(401, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: 'Failed to authenticate with GitHub Copilot via gh CLI.' }));
+      return;
+    }
+
+    let body = '';
+    req.on('data', chunk => { body += chunk; });
+    req.on('end', async () => {
+      try {
+        const payload = JSON.parse(body);
+        
+        // Map common model names to Copilot equivalents if needed
+        // For now, we pass them through or default to gpt-4o
+        const model = payload.model?.includes('gpt-4o') ? 'gpt-4o' : 
+                      payload.model?.includes('claude-3.5-sonnet') ? 'claude-3.5-sonnet' : 
+                      'gpt-4o';
+
+        const copilotRes = await fetch('https://api.githubcopilot.com/chat/completions', {
+          method: 'POST',
+          headers: {
+            'Authorization': `Bearer ${token}`,
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            'X-Github-Api-Version': '2023-07-07',
+            'Editor-Version': 'vscode/1.90.0', // Spoof VS Code for compatibility
+          },
+          body: JSON.stringify({
+            ...payload,
+            model
+          })
+        });
+
+        res.writeHead(copilotRes.status, {
+          'Content-Type': copilotRes.headers.get('Content-Type'),
+        });
+
+        if (payload.stream) {
+           const reader = copilotRes.body.getReader();
+           while (true) {
+             const { done, value } = await reader.read();
+             if (done) break;
+             res.write(value);
+           }
+           res.end();
+        } else {
+          const data = await copilotRes.text();
+          res.end(data);
+        }
+      } catch (err) {
+        res.writeHead(500, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: err.message }));
+      }
+    });
+    return;
+  }
+
+  res.writeHead(404);
+  res.end('Not Found');
+});
+
+server.listen(PORT, '127.0.0.1', () => {
+  console.log(`Copilot Bridge listening on http://127.0.0.1:${PORT}`);
+});

package/lib/cli-commands.mjs

@@ -61,13 +61,14 @@ export const CLI_COMMANDS = [
     category: 'Core',
     core: true,
     description: 'Machine setup (scoped per ADR-0029): --scope=project|user|both, default project',
-    usage: 'construct install [--scope=project|user|both] [--yes] [--no-docker] [--no-launch-agent] [--reconfigure]',
+    usage: 'construct install [--scope=project|user|both] [--yes] [--no-docker] [--no-launch-agent] [--reconfigure] [--with-docling]',
     options: [
       { flag: '--scope=<s>',      desc: 'project (default, no-op + guidance) | user (writes ~/.construct/, MCP, ~/.claude/* via consent) | both' },
       { flag: '--yes',            desc: 'Apply defaults without prompts (only meaningful with --scope=user|both)' },
       { flag: '--no-docker',      desc: 'Skip Docker-based service setup (local Postgres)' },
       { flag: '--no-launch-agent', desc: 'Skip background macOS LaunchAgent registration' },
       { flag: '--reconfigure',    desc: 'Re-prompt for service consent, ignoring cached answers' },
+      { flag: '--with-docling',   desc: 'Eagerly provision the docling document-extraction venv now (heavy, ~10 min; else lazy on first ingest)' },
     ],
   },
   {
@@ -936,6 +937,9 @@ export const CLI_COMMANDS = [
   { name: 'lint:contracts',    category: 'Internal', core: false, internal: true, description: 'Internal lint: specialist contracts', usage: 'construct lint:contracts' },
   { name: 'lint:research',     category: 'Internal', core: false, internal: true, description: 'Internal lint: research artifacts', usage: 'construct lint:research' },
   { name: 'lint:templates',    category: 'Internal', core: false, internal: true, description: 'Internal lint: shipped templates', usage: 'construct lint:templates' },
+  { name: 'lint:prompts',      category: 'Internal', core: false, internal: true, description: 'Internal lint: specialist prompt frontmatter + sections', usage: 'construct lint:prompts' },
+  { name: 'specialist',        category: 'Internal', core: false, internal: true, description: 'Maintainer tool: scaffold, edit, and lint specialist prompts', usage: 'construct specialist <create|edit|lint>' },
+  { name: 'registry:status',   category: 'Internal', core: false, internal: true, description: 'Dev: capability-matrix registry inspector', usage: 'construct registry:status' },
   { name: 'evaluator:rubrics', category: 'Internal', core: false, internal: true, description: 'Internal: list registered evaluator rubrics', usage: 'construct evaluator:rubrics' },
   { name: 'activation:status', category: 'Internal', core: false, internal: true, description: 'Internal: agent activation telemetry', usage: 'construct activation:status' },
   { name: 'prune',             category: 'Internal', core: false, internal: true, description: 'Internal: prune ephemeral storage entries', usage: 'construct prune' },

package/lib/comment-lint.mjs

@@ -86,7 +86,7 @@ function isConstructSelfRepo(rootDir) {
 
 function requiresHeader(rel) {
   const ext = path.extname(rel);
-  if (['.yaml', '.yml', '.json', '.jsonl', '.toml'].includes(ext)) return { required: false, type: null };
+  if (['.yaml', '.yml', '.json', '.jsonl', '.toml', '.txt'].includes(ext)) return { required: false, type: null };
   const jsMatch = JS_HEADER_GLOBS.some(r => r.test(rel));
   const mdMatch = MD_HEADER_GLOBS.some(r => r.test(rel));
   // Exclude static assets under lib/server/static from requiring headers

package/lib/config/schema.mjs

@@ -24,7 +24,7 @@ export const DEFAULT_PROFILE_ID = 'rnd';
 
 export const SURFACES = ['claude', 'opencode', 'codex', 'copilot', 'vscode', 'cursor'];
 
-export const INGEST_STRATEGIES = ['adapter', 'provider'];
+export const INGEST_STRATEGIES = ['adapter', 'provider', 'docling-remote'];
 export const INGEST_FALLBACKS = ['none', 'provider', 'adapter'];
 export const INGEST_ORCHESTRATIONS = ['prompt-only', 'orchestrated'];
 

package/lib/document-extract/docling-client.mjs

@@ -21,10 +21,11 @@ const SIDECAR_SCRIPT = path.resolve(path.dirname(fileURLToPath(import.meta.url))
 const REQUEST_TIMEOUT_MS = 300_000;
 
 let activeSidecar = null;
+let sidecarStarting = null;
 let requestSeq = 0;
 
-function spawnSidecar({ runtimeDir } = {}) {
-  const { pythonBin } = ensureDoclingVenv({ runtimeDir });
+async function spawnSidecar({ runtimeDir } = {}) {
+  const { pythonBin } = await ensureDoclingVenv({ runtimeDir });
   const child = spawn(pythonBin, [SIDECAR_SCRIPT], {
     stdio: ['pipe', 'pipe', 'pipe'],
     env: { ...process.env, PYTHONUNBUFFERED: '1' },
@@ -89,14 +90,23 @@ function spawnSidecar({ runtimeDir } = {}) {
   return sidecar;
 }
 
-function getSidecar() {
+// Provisioning is now async, so two concurrent extracts could both spawn a
+// sidecar. Cache the in-flight start promise so the first call wins and the rest
+// await it; once resolved, the running sidecar is reused directly.
+
+async function getSidecar() {
   if (activeSidecar && activeSidecar.child.exitCode === null && !activeSidecar.child.killed) return activeSidecar;
-  activeSidecar = spawnSidecar();
-  return activeSidecar;
+  if (!sidecarStarting) {
+    sidecarStarting = spawnSidecar().then(
+      (s) => { activeSidecar = s; sidecarStarting = null; return s; },
+      (e) => { sidecarStarting = null; throw e; },
+    );
+  }
+  return sidecarStarting;
 }
 
 export async function extractViaDocling(filePath) {
-  const sidecar = getSidecar();
+  const sidecar = await getSidecar();
   const result = await sidecar.send('extract', { path: path.resolve(filePath) });
   return result;
 }

package/lib/document-extract/docling-sidecar.py

@@ -36,10 +36,40 @@ except ImportError as exc:
 _converter = None
 
 
+# By default docling emits a bare `<!-- image -->` marker and discards pixel data,
+# so embedded figures are lost. Enable picture-image generation on the PDF
+# pipeline and export markdown with images embedded as base64 data URIs; the Node
+# side externalizes those into an assets/ directory. The configuration is wrapped
+# so a docling API change degrades to the default converter instead of crashing.
+
+def build_converter():
+    try:
+        from docling.datamodel.base_models import InputFormat
+        from docling.datamodel.pipeline_options import PdfPipelineOptions
+        from docling.document_converter import PdfFormatOption
+
+        pdf_options = PdfPipelineOptions()
+        pdf_options.generate_picture_images = True
+        pdf_options.images_scale = 2.0
+        return DocumentConverter(
+            format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=pdf_options)}
+        )
+    except Exception:
+        return DocumentConverter()
+
+
+def export_markdown(doc):
+    try:
+        from docling_core.types.doc import ImageRefMode
+        return doc.export_to_markdown(image_mode=ImageRefMode.EMBEDDED)
+    except Exception:
+        return doc.export_to_markdown()
+
+
 def get_converter():
     global _converter
     if _converter is None:
-        _converter = DocumentConverter()
+        _converter = build_converter()
     return _converter
 
 
@@ -53,7 +83,7 @@ def extract(params):
 
     result = get_converter().convert(str(path))
     doc = result.document
-    markdown = doc.export_to_markdown()
+    markdown = export_markdown(doc)
 
     metadata = {
         "format": result.input.format.value if hasattr(result.input, "format") else None,

package/lib/document-extract.mjs

@@ -107,6 +107,11 @@ function run(command, args) {
 }
 
 function commandExists(command) {
+  // Test seam: a comma-separated deny-list forces a command to read as absent,
+  // so the cross-platform fallbacks (e.g. RTF stripping without macOS textutil)
+  // are reachable on any host.
+  const denied = (process.env.CONSTRUCT_EXTRACT_NO_COMMANDS || '').split(',').map((s) => s.trim()).filter(Boolean);
+  if (denied.includes(command)) return false;
   const checker = process.platform === 'win32' ? 'where' : 'which';
   return spawnSync(checker, [command], { stdio: 'ignore' }).status === 0;
 }
@@ -216,16 +221,38 @@ function extractZipDocument(filePath, extension) {
   return { text, method: 'zip-xml', droppedInfo, structured: null };
 }
 
-function extractRichText(filePath) {
-  if (!commandExists('textutil')) {
-    throw new Error('textutil not available for rich-text extraction');
+// Minimal RTF-to-text: strip control words, groups, and escapes. RTF is a
+// text format, so this works everywhere; it is the cross-platform fallback when
+// textutil (macOS only) is absent. Lossy on complex documents, but never throws.
+export function rtfToText(raw) {
+  let s = String(raw);
+  s = s.replace(/\{(?:\\\*)?\\(?:fonttbl|colortbl|stylesheet|info|pict|object)[\s\S]*?\}/gi, '');
+  s = s.replace(/\\u(-?\d+)\s?\??/g, (_, n) => (Number(n) >= 0 ? String.fromCharCode(Number(n)) : ''));
+  s = s.replace(/\\'([0-9a-fA-F]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
+  s = s.replace(/\\par[d]?\b/g, '\n').replace(/\\line\b/g, '\n').replace(/\\tab\b/g, '\t');
+  s = s.replace(/\\[a-zA-Z]+-?\d* ?/g, '');
+  s = s.replace(/\\([{}\\])/g, '$1');
+  return s.replace(/[{}]/g, '');
+}
+
+function extractRichText(filePath, extension) {
+  if (commandExists('textutil')) {
+    return {
+      text: normalizeText(run('textutil', ['-convert', 'txt', '-stdout', filePath])),
+      method: 'textutil',
+      droppedInfo: [],
+      structured: null,
+    };
   }
-  return {
-    text: normalizeText(run('textutil', ['-convert', 'txt', '-stdout', filePath])),
-    method: 'textutil',
-    droppedInfo: [],
-    structured: null,
-  };
+  if (extension === '.rtf') {
+    return {
+      text: normalizeText(rtfToText(readFileSync(filePath, 'utf8'))),
+      method: 'rtf-strip',
+      droppedInfo: [],
+      structured: null,
+    };
+  }
+  throw new Error('textutil not available for rich-text extraction');
 }
 
 function extractPdfWithPdftotext(filePath) {
@@ -660,7 +687,7 @@ export function extractDocumentText(filePath, { maxChars = null } = {}) {
   } else if (ZIP_DOCUMENT_EXTS.has(extension)) {
     extracted = extractZipDocument(resolvedPath, extension);
   } else if (RICH_TEXT_EXTS.has(extension)) {
-    extracted = extractRichText(resolvedPath);
+    extracted = extractRichText(resolvedPath, extension);
   } else if (MDLS_DOCUMENT_EXTS.has(extension) && process.platform === 'darwin' && commandExists('mdls')) {
     extracted = { text: extractWithMdls(resolvedPath), method: 'mdls', droppedInfo: [], structured: null };
   } else if (EMAIL_DOCUMENT_EXTS.has(extension)) {

package/lib/document-ingest.mjs

@@ -8,7 +8,7 @@
  * and records the selected strategy/model in the returned `ingestion` block.
  */
 import { existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from 'node:fs';
-import { basename, dirname, extname, isAbsolute, join, parse, relative, resolve } from 'node:path';
+import { basename, dirname, extname, isAbsolute, join, parse, relative, resolve, sep } from 'node:path';
 import { extractDocumentText, extractDocumentTextAsync, extractDocumentMetadata, isExtractableDocumentPath } from './document-extract.mjs';
 import { syncFileStateToSql } from './storage/sync.mjs';
 import { stampFrontmatter } from './doc-stamp.mjs';
@@ -16,6 +16,7 @@ import { KNOWLEDGE_ROOT, KNOWLEDGE_SUBDIRS } from './knowledge/layout.mjs';
 import { loadProjectConfig } from './config/project-config.mjs';
 import { resolveIngestStrategy, INGEST_STRATEGIES, INGEST_ORCHESTRATION_STRATEGIES } from './ingest/strategy.mjs';
 import { extractViaProvider } from './ingest/provider-extract.mjs';
+import { extractViaDoclingRemote } from './ingest/docling-remote.mjs';
 
 const DEFAULT_TARGET_DIR = '.cx/knowledge/internal';
 
@@ -56,6 +57,31 @@ function nextAvailablePath(targetPath) {
   }
 }
 
+// docling embeds figures as base64 `data:` URIs in the markdown so no pixel data
+// is lost in transit. Write each to an assets/ directory beside the output file
+// and rewrite the reference to a relative path, so the markdown stays small and
+// the images are real files an editor or viewer can open — replacing the bare
+// `<!-- image -->` placeholder docling emits by default.
+
+export function externalizeEmbeddedImages(markdown, { mdPath }) {
+  if (!markdown || !markdown.includes('data:image/')) return { markdown, assets: [] };
+  const assetsRel = `assets/${basename(mdPath, '.md')}`;
+  const assetsDir = join(dirname(mdPath), assetsRel.split('/').join(sep));
+  const assets = [];
+  let index = 0;
+  const re = /!\[([^\]]*)\]\(\s*data:image\/([a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+?)\s*\)/g;
+  const out = markdown.replace(re, (_match, alt, type, b64) => {
+    index += 1;
+    const ext = type.toLowerCase() === 'jpeg' ? 'jpg' : type.toLowerCase().replace(/[^a-z0-9]/g, '') || 'png';
+    const fileName = `image-${index}.${ext}`;
+    if (!existsSync(assetsDir)) mkdirSync(assetsDir, { recursive: true });
+    writeFileSync(join(assetsDir, fileName), Buffer.from(b64.replace(/\s+/g, ''), 'base64'));
+    assets.push(join(assetsDir, fileName));
+    return `![${alt || `image ${index}`}](${assetsRel}/${fileName})`;
+  });
+  return { markdown: out, assets };
+}
+
 function renderMarkdown({ sourcePath, extractedAt, title, extractionMethod, characters, truncated, text, outputPath, cwd, metadata }) {
   const relSource = relative(cwd, sourcePath) || basename(sourcePath);
   const relOutput = relative(cwd, outputPath) || basename(outputPath);
@@ -126,17 +152,74 @@ function resolveOutputPath(sourcePath, { cwd, outputPath, outputDir, target }) {
   return join(resolvedDir, `${basename(sourcePath)}.md`);
 }
 
+// The high-fidelity (docling) path provisions a Python venv and downloads ML
+// models on first use, then runs layout inference on the document — any of which
+// can stall (a large PDF, a slow model download) or fail. Without a bound the
+// caller hangs: the CLI never returns, and the ingest_document MCP tool blocks
+// until the client times out and surfaces an opaque error. Bound the whole
+// docling attempt and fall back to the legacy extractor so ingest always returns
+// a usable result, with the fallback recorded in droppedInfo. Override the bound
+// with CONSTRUCT_DOCLING_TIMEOUT_MS (0 disables the timeout).
+
+const DOCLING_TIMEOUT_MS = (() => {
+  const raw = Number(process.env.CONSTRUCT_DOCLING_TIMEOUT_MS);
+  return Number.isFinite(raw) && raw >= 0 ? raw : 600_000;
+})();
+
+export function withTimeout(promise, ms, onTimeoutMessage) {
+  if (!ms) return promise;
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(Object.assign(new Error(onTimeoutMessage), { code: 'DOCLING_TIMEOUT' })), ms);
+    promise.then((v) => { clearTimeout(timer); resolve(v); }, (e) => { clearTimeout(timer); reject(e); });
+  });
+}
+
+/**
+ * Run the high-fidelity (docling) extractor with a bound, falling back to the
+ * legacy extractor on timeout or any failure so a caller never hangs. The
+ * extractor functions are injectable for testing; production uses the real
+ * docling-async and legacy-sync extractors.
+ */
+export async function extractWithDoclingFallback(sourcePath, {
+  maxChars = null,
+  timeoutMs = DOCLING_TIMEOUT_MS,
+  asyncExtract = extractDocumentTextAsync,
+  syncExtract = extractDocumentText,
+} = {}) {
+  try {
+    return await withTimeout(
+      asyncExtract(sourcePath, { maxChars }),
+      timeoutMs,
+      `docling extraction exceeded ${Math.round(timeoutMs / 1000)}s`,
+    );
+  } catch (err) {
+    const legacy = syncExtract(sourcePath, { maxChars });
+    legacy.droppedInfo = [
+      ...(legacy.droppedInfo ?? []),
+      {
+        kind: 'docling-fallback',
+        count: 1,
+        reason: `High-fidelity (docling) extraction failed or timed out (${err.message}); used the legacy extractor. Re-run after \`construct install --with-docling\`, or pass --legacy-extractor to skip docling.`,
+        recoverable: true,
+      },
+    ];
+    return legacy;
+  }
+}
+
 function extractViaAdapter(sourcePath, { highFidelity, maxChars }) {
   return highFidelity
-    ? extractDocumentTextAsync(sourcePath, { maxChars })
+    ? extractWithDoclingFallback(sourcePath, { maxChars })
     : extractDocumentText(sourcePath, { maxChars });
 }
 
 async function extractWithStrategy(sourcePath, { strategy, fallback, model, provider, highFidelity, maxChars, env }) {
-  const primary = strategy === 'provider' ? 'provider' : 'adapter';
+  const primary = strategy === 'provider' || strategy === 'docling-remote' ? strategy : 'adapter';
   const run = (mode) => (mode === 'provider'
     ? extractViaProvider({ filePath: sourcePath, model, provider, maxChars, env })
-    : extractViaAdapter(sourcePath, { highFidelity, maxChars }));
+    : mode === 'docling-remote'
+      ? extractViaDoclingRemote({ filePath: sourcePath, maxChars, env })
+      : extractViaAdapter(sourcePath, { highFidelity, maxChars }));
 
   try {
     const extracted = await run(primary);
@@ -207,6 +290,7 @@ export async function ingestDocuments(inputPaths, {
     const finalPath = nextAvailablePath(targetPath);
     const extractedAt = new Date().toISOString();
     const title = metadata?.title || formatTitle(sourcePath);
+    const { markdown: bodyText, assets: imageAssets } = externalizeEmbeddedImages(extracted.text, { mdPath: finalPath });
     const markdown = renderMarkdown({
       sourcePath,
       extractedAt,
@@ -214,7 +298,7 @@ export async function ingestDocuments(inputPaths, {
       extractionMethod: extracted.extractionMethod,
       characters: extracted.characters,
       truncated: extracted.truncated,
-      text: extracted.text,
+      text: bodyText,
       outputPath: finalPath,
       cwd,
       metadata,
@@ -229,6 +313,7 @@ export async function ingestDocuments(inputPaths, {
       characters: extracted.characters,
       droppedInfo: extracted.droppedInfo ?? [],
       structured: extracted.structured ?? null,
+      images: imageAssets.map((p) => relative(cwd, p)),
       metadata: { title, authors: metadata?.authors || [], dates: metadata?.dates || {} },
     });
   }

package/lib/embed/roadmap.mjs

@@ -202,6 +202,12 @@ function normalizeStatus(raw) {
   return 'planned';
 }
 
+// Normalize the volatile "Last updated" stamp so two renders can be compared on
+// content alone.
+function stripTimestamp(markdown) {
+  return String(markdown || '').replace(/^> Last updated: .*$/m, '> Last updated:');
+}
+
 /**
  * Render entries back to roadmap markdown.
  */
@@ -324,7 +330,16 @@ export async function generateRoadmap({ targetPath, snapshot, roles }) {
   // Render
   const markdown = renderRoadmap(entries, { roles, sources });
 
-  // Write
+  // Skip the write when only the generated "Last updated" stamp would change —
+  // otherwise every render rewrites a tracked file with no semantic diff and
+  // leaves the working tree perpetually dirty. The timestamp then reflects when
+  // the roadmap content last actually moved, not when generation last ran.
+  let existingRaw = null;
+  try { existingRaw = readFileSync(roadmapPath, 'utf8'); } catch { existingRaw = null; }
+  if (existingRaw !== null && stripTimestamp(existingRaw) === stripTimestamp(markdown)) {
+    return { path: roadmapPath, updatedAt, itemCount: entries.length, isNew: false, unchanged: true };
+  }
+
   mkdirSync(join(targetPath, 'docs'), { recursive: true });
   writeFileSync(roadmapPath, markdown, 'utf8');