@geraldmaron/construct 1.0.0 → 1.0.2

package/lib/env-config.mjs

@@ -1,7 +1,9 @@
 /**
- * lib/env-config.mjs — <one-line purpose>
+ * lib/env-config.mjs — Read and write Construct environment configuration files.
  *
- * <2–6 line summary: what it does, who calls it, key side effects.>
+ * Provides helpers to load ~/.cx/env, merge with process.env, and persist
+ * key/value pairs back to disk. Backs setup, model-router, and the MCP
+ * server when they resolve API keys and model overrides without leaking secrets.
  */
 import fs from 'node:fs';
 import os from 'node:os';
@@ -46,8 +48,71 @@ export function writeEnvValues(filePath, values = {}) {
   fs.writeFileSync(filePath, `${content}\n`, 'utf8');
 }
 
-export function loadConstructEnv({ rootDir, homeDir = os.homedir(), env = process.env } = {}) {
+function cleanEnvValue(value) {
+  return String(value ?? '').trim();
+}
+
+export function resolveDatabaseUrl(env = process.env) {
+  const explicit = cleanEnvValue(env.DATABASE_URL);
+  if (explicit) return explicit;
+
+  const host = cleanEnvValue(env.DB_HOST);
+  const name = cleanEnvValue(env.DB_NAME);
+  const user = cleanEnvValue(env.DB_USER);
+  const password = cleanEnvValue(env.DB_PASSWORD);
+  const port = cleanEnvValue(env.DB_PORT) || '5432';
+
+  if (!host || !name || !user || !password) return '';
+
+  return `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}:${port}/${name}`;
+}
+
+// Keys that are expected to be set in process.env and should not trigger
+// shadow warnings (e.g. standard Node/shell vars, CI variables).
+const ENV_SHADOW_IGNORE = new Set([
+  'PATH', 'HOME', 'USER', 'SHELL', 'TERM', 'PWD', 'OLDPWD', 'LOGNAME',
+  'TMPDIR', 'LANG', 'LC_ALL', 'LC_CTYPE', 'TZ', 'SHLVL', '_',
+  'NODE_ENV', 'NODE_PATH', 'NODE_OPTIONS',
+  'npm_lifecycle_event', 'npm_package_name', 'npm_package_version',
+]);
+
+/**
+ * Keys that configure Construct services. When process.env provides one
+ * of these and a .env file also sets it to a *different* value, a one-time
+ * warning fires so stale exported vars are not silently authoritative.
+ */
+const ENV_SHADOW_WARN = new Set([
+  'OPENROUTER_API_KEY', 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY',
+  'DASHBOARD_PORT', 'MEMORY_PORT', 'BRIDGE_PORT',
+  'CONSTRUCT_INSTANCE_ID',
+  'CONSTRUCT_DEPLOYMENT_MODE',
+]);
+
+export function loadConstructEnv({ rootDir, homeDir = os.homedir(), env = process.env, warn = true } = {}) {
   const rootEnv = rootDir ? parseEnvFile(path.join(rootDir, '.env')) : {};
   const userEnv = parseEnvFile(getUserEnvPath(homeDir));
-  return { ...rootEnv, ...userEnv, ...env };
+  const fileEnv = { ...rootEnv, ...userEnv };
+
+  if (warn) {
+    for (const key of Object.keys(env)) {
+      if (ENV_SHADOW_IGNORE.has(key)) continue;
+      if (!ENV_SHADOW_WARN.has(key)) continue;
+      if (key in fileEnv && fileEnv[key] !== env[key]) {
+        process.stderr.write(
+          `[construct] WARNING: process.env.${key} (${env[key].slice(0, 6)}…) shadows ` +
+          `config.env value (${fileEnv[key].slice(0, 6)}…). ` +
+          `Unset the shell variable or update ~/.construct/config.env.\n`,
+        );
+      }
+    }
+  }
+
+  // process.env is the base; config.env (file) values override shell values
+  // so that credentials set in the dashboard always win over stale shell exports.
+  const merged = { ...env, ...fileEnv };
+  const databaseUrl = resolveDatabaseUrl(merged);
+  if (databaseUrl && !cleanEnvValue(merged.DATABASE_URL)) {
+    merged.DATABASE_URL = databaseUrl;
+  }
+  return merged;
 }

package/lib/evals/retrieval-bench.mjs

@@ -0,0 +1,159 @@
+/**
+ * lib/evals/retrieval-bench.mjs — retrieval benchmark runner with structured metrics.
+ *
+ * Drives a retrieval function against a set of fixture queries with
+ * expected document ids, then summarizes recall@K, precision@K, MRR,
+ * latency p50/p95, fallback usage, and empty-result rate. Designed to
+ * gate CI on retrieval regressions: pass a stable fixture set, fail
+ * the run when any metric drops below threshold.
+ *
+ * Fixture shape:
+ *   {
+ *     name, query, expectedIds: string[], filters?: object,
+ *     k?: number, mustNotInclude?: string[]
+ *   }
+ *
+ * Result shape:
+ *   {
+ *     summary: { recallAt5, precisionAt5, mrr, p50LatencyMs, p95LatencyMs,
+ *                emptyRate, fallbackRate, queries: N, passed: N, regressed: [...] },
+ *     perQuery: [{ name, recall, precision, reciprocalRank, latencyMs, fallback }]
+ *   }
+ */
+
+function percentile(values, p) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
+  return sorted[idx];
+}
+
+function uniq(values) {
+  return [...new Set(values)];
+}
+
+function recallForQuery(actualIds, expectedIds) {
+  if (expectedIds.length === 0) return 1;
+  const hit = expectedIds.filter((id) => actualIds.includes(id)).length;
+  return hit / expectedIds.length;
+}
+
+function precisionForQuery(actualIds, expectedIds) {
+  if (actualIds.length === 0) return 0;
+  const hit = actualIds.filter((id) => expectedIds.includes(id)).length;
+  return hit / actualIds.length;
+}
+
+function mrrForQuery(actualIds, expectedIds) {
+  for (let i = 0; i < actualIds.length; i++) {
+    if (expectedIds.includes(actualIds[i])) return 1 / (i + 1);
+  }
+  return 0;
+}
+
+/**
+ * Run a benchmark suite.
+ *
+ * @param {object} opts
+ * @param {Array} opts.fixtures
+ * @param {Function} opts.search — async (query, options) → { ids, fallback?, results? }
+ * @param {object} [opts.thresholds] — optional regression gates
+ * @returns {Promise<{summary, perQuery}>}
+ */
+export async function runRetrievalBench({ fixtures, search, thresholds = {} } = {}) {
+  if (!Array.isArray(fixtures) || fixtures.length === 0) {
+    throw new Error('runRetrievalBench: at least one fixture is required');
+  }
+  if (typeof search !== 'function') {
+    throw new Error('runRetrievalBench: search function is required');
+  }
+
+  const perQuery = [];
+  const latencies = [];
+  let emptyCount = 0;
+  let fallbackCount = 0;
+
+  for (const fixture of fixtures) {
+    const k = fixture.k || 5;
+    const started = Date.now();
+    const response = await search(fixture.query, { filters: fixture.filters, k });
+    const latency = Date.now() - started;
+    latencies.push(latency);
+
+    const ids = uniq(response.ids || (response.results || []).map((r) => r.id || r.source_path || r.path)).slice(0, k);
+    const fallback = Boolean(response.fallback);
+    if (fallback) fallbackCount += 1;
+    if (ids.length === 0) emptyCount += 1;
+
+    const recall = recallForQuery(ids, fixture.expectedIds || []);
+    const precision = precisionForQuery(ids, fixture.expectedIds || []);
+    const reciprocalRank = mrrForQuery(ids, fixture.expectedIds || []);
+    const mustNotMatched = (fixture.mustNotInclude || []).some((id) => ids.includes(id));
+
+    perQuery.push({
+      name: fixture.name || fixture.query.slice(0, 40),
+      recall,
+      precision,
+      reciprocalRank,
+      latencyMs: latency,
+      fallback,
+      mustNotMatched,
+      returnedIds: ids,
+    });
+  }
+
+  const n = perQuery.length;
+  const avg = (selector) => perQuery.reduce((acc, q) => acc + selector(q), 0) / n;
+  const summary = {
+    queries: n,
+    recallAt5: avg((q) => q.recall),
+    precisionAt5: avg((q) => q.precision),
+    mrr: avg((q) => q.reciprocalRank),
+    p50LatencyMs: percentile(latencies, 50),
+    p95LatencyMs: percentile(latencies, 95),
+    emptyRate: emptyCount / n,
+    fallbackRate: fallbackCount / n,
+    regressed: [],
+  };
+
+  if (thresholds.minRecallAt5 !== undefined && summary.recallAt5 < thresholds.minRecallAt5) {
+    summary.regressed.push({ metric: 'recallAt5', actual: summary.recallAt5, expected: thresholds.minRecallAt5 });
+  }
+  if (thresholds.minMrr !== undefined && summary.mrr < thresholds.minMrr) {
+    summary.regressed.push({ metric: 'mrr', actual: summary.mrr, expected: thresholds.minMrr });
+  }
+  if (thresholds.maxP95LatencyMs !== undefined && summary.p95LatencyMs > thresholds.maxP95LatencyMs) {
+    summary.regressed.push({ metric: 'p95LatencyMs', actual: summary.p95LatencyMs, expected: thresholds.maxP95LatencyMs });
+  }
+  if (perQuery.some((q) => q.mustNotMatched)) {
+    summary.regressed.push({ metric: 'mustNotInclude', actual: perQuery.filter((q) => q.mustNotMatched).map((q) => q.name) });
+  }
+
+  summary.passed = n - summary.regressed.length;
+
+  return { summary, perQuery };
+}
+
+export function formatBenchSummary({ summary, perQuery }) {
+  const lines = [];
+  lines.push(`Retrieval benchmark — ${summary.queries} queries`);
+  lines.push(`  recall@5     ${summary.recallAt5.toFixed(3)}`);
+  lines.push(`  precision@5  ${summary.precisionAt5.toFixed(3)}`);
+  lines.push(`  MRR          ${summary.mrr.toFixed(3)}`);
+  lines.push(`  p50 latency  ${summary.p50LatencyMs}ms`);
+  lines.push(`  p95 latency  ${summary.p95LatencyMs}ms`);
+  lines.push(`  empty rate   ${(summary.emptyRate * 100).toFixed(1)}%`);
+  lines.push(`  fallback     ${(summary.fallbackRate * 100).toFixed(1)}%`);
+  if (summary.regressed.length > 0) {
+    lines.push('  Regressed:');
+    for (const r of summary.regressed) lines.push(`    - ${r.metric}: ${JSON.stringify(r.actual)} (expected ${JSON.stringify(r.expected)})`);
+  }
+  if (perQuery && perQuery.length > 0) {
+    lines.push('');
+    lines.push('Per query:');
+    for (const q of perQuery) {
+      lines.push(`  ${q.name.padEnd(40)} recall=${q.recall.toFixed(2)} mrr=${q.reciprocalRank.toFixed(2)} ${q.latencyMs}ms${q.fallback ? ' [fallback]' : ''}`);
+    }
+  }
+  return lines.join('\n');
+}

package/lib/evaluator-optimizer.mjs

@@ -0,0 +1,317 @@
+/**
+ * lib/evaluator-optimizer.mjs — Iterative refinement loop for document authoring.
+ *
+ * Implements the evaluator-optimizer pattern from Anthropic's best practices:
+ * 1. Generator creates a draft
+ * 2. Evaluator scores against rubric (0.0 - 1.0)
+ * 3. If score < threshold, generate revision with feedback
+ * 4. Repeat until score >= threshold or max iterations reached
+ *
+ * Usage:
+ *   const result = await runEvaluatorOptimizer({
+ *     request: 'Write a PRD for feature X',
+ *     docType: 'prd',
+ *     threshold: 0.7,
+ *     maxIterations: 3,
+ *   });
+ *
+ * Returns:
+ *   {
+ *     content: string,
+ *     iterations: number,
+ *     finalScore: number,
+ *     scores: [0.6, 0.75, 0.82],
+ *     feedback: [...],
+ *     reachedThreshold: boolean,
+ *   }
+ */
+
+import { getTemplate } from './templates.mjs';
+
+const DEFAULT_THRESHOLD = 0.7;
+const DEFAULT_MAX_ITERATIONS = 3;
+
+const RUBRICS = {
+  prd: {
+    name: 'PRD Quality Rubric',
+    criteria: [
+      { id: 'summary', name: 'Summary', description: 'One paragraph that captures what, who, why', weight: 0.1 },
+      { id: 'problem', name: 'Problem Statement', description: 'Clear user pain, not solution-speak', weight: 0.15 },
+      { id: 'goals', name: 'Goals', description: '3-5 outcome-focused goals, not activities', weight: 0.1 },
+      { id: 'scope', name: 'Scope', description: 'In/out of scope clearly bounded', weight: 0.1 },
+      { id: 'phases', name: 'Phases', description: 'Each phase independently shippable', weight: 0.15 },
+      { id: 'requirements', name: 'Requirements', description: 'FRs/NFRs with acceptance criteria', weight: 0.2 },
+      { id: 'metrics', name: 'Success Metrics', description: 'Leading and lagging indicators', weight: 0.1 },
+      { id: 'risks', name: 'Risks', description: 'Likelihood/impact with mitigations', weight: 0.1 },
+    ],
+  },
+  adr: {
+    name: 'ADR Quality Rubric',
+    criteria: [
+      { id: 'title', name: 'Title', description: 'Clear, imperative format', weight: 0.05 },
+      { id: 'status', name: 'Status', description: 'Proposed/Accepted/Deprecated with date', weight: 0.05 },
+      { id: 'context', name: 'Context', description: 'Forces, constraints, assumptions', weight: 0.2 },
+      { id: 'decision', name: 'Decision', description: 'Clear statement of what was decided', weight: 0.25 },
+      { id: 'alternatives', name: 'Alternatives', description: 'Rejected options with rationale', weight: 0.2 },
+      { id: 'consequences', name: 'Consequences', description: 'Positive, negative, neutral outcomes', weight: 0.15 },
+      { id: 'compliance', name: 'Compliance', description: 'Links to related ADRs, PRDs', weight: 0.1 },
+    ],
+  },
+  rfc: {
+    name: 'RFC Quality Rubric',
+    criteria: [
+      { id: 'summary', name: 'Summary', description: 'Executive summary for busy readers', weight: 0.1 },
+      { id: 'motivation', name: 'Motivation', description: 'Why this matters now', weight: 0.15 },
+      { id: 'proposal', name: 'Proposal', description: 'Clear, specific technical change', weight: 0.25 },
+      { id: 'alternatives', name: 'Alternatives Considered', description: 'Other approaches and why not', weight: 0.15 },
+      { id: 'implementation', name: 'Implementation Plan', description: 'Phases, milestones, owners', weight: 0.15 },
+      { id: 'risks', name: 'Risks & Mitigations', description: 'What could go wrong', weight: 0.1 },
+      { id: 'success', name: 'Success Criteria', description: 'How we know this worked', weight: 0.1 },
+    ],
+  },
+  runbook: {
+    name: 'Runbook Quality Rubric',
+    criteria: [
+      { id: 'trigger', name: 'Trigger', description: 'When to run this runbook', weight: 0.1 },
+      { id: 'steps', name: 'Steps', description: 'Numbered, executable steps', weight: 0.3 },
+      { id: 'verification', name: 'Verification', description: 'How to confirm it worked', weight: 0.2 },
+      { id: 'rollback', name: 'Rollback', description: 'How to undo if needed', weight: 0.15 },
+      { id: 'escalation', name: 'Escalation', description: 'Who to call if stuck', weight: 0.1 },
+      { id: 'references', name: 'References', description: 'Related docs, runbooks, alerts', weight: 0.15 },
+    ],
+  },
+};
+
+/**
+ * Evaluate a document against its rubric.
+ * Returns a score (0.0 - 1.0) and detailed feedback.
+ */
+export async function evaluateDocument({ content, docType, evaluatorFn = null }) {
+  const rubric = RUBRICS[docType];
+  if (!rubric) {
+    throw new Error(`Unknown document type: ${docType}. Available: ${Object.keys(RUBRICS).join(', ')}`);
+  }
+  
+  // If custom evaluator function provided, use it
+  if (evaluatorFn) {
+    return await evaluatorFn({ content, docType, rubric });
+  }
+  
+  // Default: deterministic rubric-based evaluation
+  // In production, this would call an LLM evaluator
+  // For now, we use structural checks as a proxy
+  
+  const scores = [];
+  const feedback = [];
+  
+  for (const criterion of rubric.criteria) {
+    // Check if the criterion section exists in the content
+    const sectionPatterns = getSectionPatterns(docType, criterion.id);
+    let score = 0;
+    let found = false;
+    
+    for (const pattern of sectionPatterns) {
+      if (content.toLowerCase().includes(pattern.toLowerCase())) {
+        found = true;
+        break;
+      }
+    }
+    
+    if (found) {
+      // Check for quality indicators
+      const qualityIndicators = getQualityIndicators(criterion.id);
+      let qualityScore = 0.5; // Base score for having the section
+      
+      for (const indicator of qualityIndicators) {
+        if (content.toLowerCase().includes(indicator.toLowerCase())) {
+          qualityScore += 0.1;
+        }
+      }
+      
+      score = Math.min(1.0, qualityScore);
+    }
+    
+    scores.push(score * criterion.weight);
+    feedback.push({
+      criterion: criterion.name,
+      score,
+      maxScore: 1.0,
+      weight: criterion.weight,
+      comment: found ? `Section present, quality: ${(score * 100).toFixed(0)}%` : 'Section missing',
+    });
+  }
+  
+  const finalScore = scores.reduce((a, b) => a + b, 0);
+  
+  return {
+    score: finalScore,
+    feedback,
+    rubric: rubric.name,
+    docType,
+    evaluatedAt: new Date().toISOString(),
+  };
+}
+
+function getSectionPatterns(docType, criterionId) {
+  const patterns = {
+    prd: {
+      summary: ['## summary', '## overview', '## executive summary'],
+      problem: ['## problem', '## problem statement', '## background'],
+      goals: ['## goals', '## objectives'],
+      scope: ['## in scope', '## out of scope', '## scope'],
+      phases: ['## phase', '## phases'],
+      requirements: ['functional', 'non-functional', 'FR-', 'NFR-', 'acceptance'],
+      metrics: ['## success metric', '## metrics', 'kpi'],
+      risks: ['## risk', '## risks', 'mitigation'],
+    },
+    adr: {
+      title: ['# adr:', '## title'],
+      status: ['## status', 'status:'],
+      context: ['## context', '## background', '## forces'],
+      decision: ['## decision', '## proposed solution'],
+      alternatives: ['## alternatives', '## alternative', 'rejected'],
+      consequences: ['## consequences', '## implications'],
+      compliance: ['## compliance', '## related', 'links to'],
+    },
+    rfc: {
+      summary: ['## summary', '## abstract', '## overview'],
+      motivation: ['## motivation', '## why', '## rationale'],
+      proposal: ['## proposal', '## design', '## solution'],
+      alternatives: ['## alternatives', '## alternative approaches'],
+      implementation: ['## implementation', '## plan', '## timeline'],
+      risks: ['## risks', '## concerns'],
+      success: ['## success criteria', '## how we measure'],
+    },
+    runbook: {
+      trigger: ['## trigger', '## when to run', '## purpose'],
+      steps: ['## steps', '## procedure', '## instructions'],
+      verification: ['## verification', '## confirm', '## validate'],
+      rollback: ['## rollback', '## undo', '## revert'],
+      escalation: ['## escalation', '## contacts', '## who to call'],
+      references: ['## references', '## related', '## see also'],
+    },
+  };
+  
+  return patterns[docType]?.[criterionId] || [];
+}
+
+function getQualityIndicators(criterionId) {
+  const indicators = {
+    summary: ['one paragraph', 'in brief', 'tl;dr'],
+    problem: ['user', 'customer', 'pain', 'currently unable'],
+    goals: ['reduce', 'increase', 'eliminate', 'from', 'to'],
+    scope: ['explicitly', 'deferred', 'not included'],
+    phases: ['phase 1', 'phase 2', 'phase 3', 'independently shippable'],
+    requirements: ['must', 'should', 'acceptance:', 'falsifiable'],
+    metrics: ['leading', 'lagging', 'baseline', 'target'],
+    risks: ['likelihood', 'impact', 'mitigation', 'low', 'medium', 'high'],
+  };
+  
+  return indicators[criterionId] || [];
+}
+
+/**
+ * Generate revision feedback based on evaluation results.
+ */
+export function generateRevisionFeedback(evaluationResult) {
+  const improvements = [];
+  
+  for (const item of evaluationResult.feedback) {
+    if (item.score < 0.7) {
+      improvements.push(`- **${item.criterion}** (score: ${(item.score * 100).toFixed(0)}%): ${item.comment}`);
+    }
+  }
+  
+  if (improvements.length === 0) {
+    return 'Document meets all quality criteria. No revisions needed.';
+  }
+  
+  return `Please revise the document to address the following:\n\n${improvements.join('\n')}`;
+}
+
+/**
+ * Run the full evaluator-optimizer loop.
+ */
+export async function runEvaluatorOptimizer({
+  request,
+  docType,
+  threshold = DEFAULT_THRESHOLD,
+  maxIterations = DEFAULT_MAX_ITERATIONS,
+  generatorFn,
+  evaluatorFn,
+}) {
+  if (!generatorFn) {
+    throw new Error('generatorFn is required — provide a function that generates document drafts');
+  }
+  
+  const results = {
+    content: null,
+    iterations: 0,
+    finalScore: 0,
+    scores: [],
+    feedback: [],
+    reachedThreshold: false,
+  };
+  
+  let currentContent = null;
+  let iterationFeedback = null;
+  
+  for (let iteration = 1; iteration <= maxIterations; iteration++) {
+    results.iterations = iteration;
+    
+    // Generate draft (or revision)
+    currentContent = await generatorFn({
+      request,
+      docType,
+      iteration,
+      previousFeedback: iterationFeedback,
+    });
+    
+    // Evaluate
+    const evaluation = await evaluateDocument({
+      content: currentContent,
+      docType,
+      evaluatorFn,
+    });
+    
+    results.scores.push(evaluation.score);
+    results.feedback.push(evaluation.feedback);
+    results.finalScore = evaluation.score;
+    
+    if (evaluation.score >= threshold) {
+      results.reachedThreshold = true;
+      results.content = currentContent;
+      break;
+    }
+    
+    // Generate feedback for next iteration
+    iterationFeedback = generateRevisionFeedback(evaluation);
+  }
+  
+  results.content = currentContent;
+  
+  return results;
+}
+
+/**
+ * Get available rubrics for a document type.
+ */
+export function getRubric(docType) {
+  return RUBRICS[docType] || null;
+}
+
+/**
+ * List all available rubrics.
+ */
+export function listRubrics() {
+  return Object.keys(RUBRICS).map(type => ({
+    docType: type,
+    name: RUBRICS[type].name,
+    criteriaCount: RUBRICS[type].criteria.length,
+    criteria: RUBRICS[type].criteria.map(c => ({
+      id: c.id,
+      name: c.name,
+      weight: c.weight,
+    })),
+  }));
+}