@geraldmaron/construct 1.0.0 → 1.0.2

package/lib/telemetry/intent-verifications.mjs

@@ -0,0 +1,86 @@
+/**
+ * lib/telemetry/intent-verifications.mjs — offline log of LLM-vs-keyword
+ * routing agreement.
+ *
+ * Each entry pins the keyword classifier's verdict alongside the fast-tier
+ * LLM's verdict so disagreements can be analyzed without gating dispatch
+ * on the LLM round-trip. A future tuning pass can replay this log to find
+ * flavors with low LLM agreement (candidates for keyword rewording or a
+ * confidence-thresholded re-introduction of inline verification).
+ *
+ * Disable with `CONSTRUCT_INTENT_VERIFY=off` to skip the background call
+ * entirely; the dispatch path itself is unaffected either way.
+ */
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+export const DEFAULT_LOG_PATH = path.join(os.homedir(), '.cx', 'intent-verifications.jsonl');
+
+/**
+ * Fire-and-forget append of one verification event.
+ *
+ * @param {object} event
+ * @param {string} event.request — the user request that hit the classifier
+ * @param {string} event.specialist — cx-* slug the keyword classifier picked
+ * @param {string} event.flavor — flavor the keyword classifier matched
+ * @param {boolean} event.keywordVerdict — always true at the call site (the
+ *   classifier matched; this is the baseline we're comparing against)
+ * @param {boolean} event.llmVerdict — what the LLM judge returned
+ * @param {boolean} event.agreed — convenience: keywordVerdict === llmVerdict
+ * @param {number} event.confidence — 0-1 from the LLM judge
+ * @param {string} event.reason — LLM's one-sentence justification
+ * @param {string} event.source — 'llm' | 'cache' | 'fallback' | 'disabled' | 'no-flavor'
+ * @param {number} event.latencyMs — wall-clock of the LLM call
+ * @param {object} [opts]
+ * @param {string} [opts.logPath]
+ */
+export function logIntentVerification(event, opts = {}) {
+  if (!event) return;
+  const logPath = opts.logPath || DEFAULT_LOG_PATH;
+  const entry = {
+    ts: new Date().toISOString(),
+    requestExcerpt: String(event.request || '').slice(0, 200),
+    specialist: event.specialist,
+    flavor: event.flavor,
+    keywordVerdict: event.keywordVerdict !== false,
+    llmVerdict: Boolean(event.llmVerdict),
+    agreed: event.agreed === undefined
+      ? (event.keywordVerdict !== false) === Boolean(event.llmVerdict)
+      : Boolean(event.agreed),
+    confidence: typeof event.confidence === 'number' ? event.confidence : null,
+    reason: String(event.reason || '').slice(0, 240),
+    source: event.source || 'unknown',
+    latencyMs: typeof event.latencyMs === 'number' ? event.latencyMs : null,
+  };
+
+  try {
+    fs.mkdirSync(path.dirname(logPath), { recursive: true });
+    fs.appendFileSync(logPath, JSON.stringify(entry) + '\n', 'utf8');
+  } catch {
+    // Telemetry must never throw — the routing path doesn't await us anyway.
+  }
+}
+
+/**
+ * Aggregate a verifications log into agreement stats per (specialist, flavor).
+ * Useful for the audit CLI and tests.
+ */
+export function summarizeIntentVerifications({ logPath = DEFAULT_LOG_PATH } = {}) {
+  if (!fs.existsSync(logPath)) return { totalEvents: 0, byFlavor: {} };
+  const lines = fs.readFileSync(logPath, 'utf8').split('\n').filter(Boolean);
+  const byFlavor = {};
+  for (const line of lines) {
+    let entry;
+    try { entry = JSON.parse(line); } catch { continue; }
+    if (!entry?.specialist || !entry?.flavor) continue;
+    const key = `${entry.specialist}/${entry.flavor}`;
+    const slot = byFlavor[key] ||= { matches: 0, agreed: 0, disagreed: 0, lastSeen: null };
+    slot.matches += 1;
+    if (entry.agreed) slot.agreed += 1;
+    else slot.disagreed += 1;
+    if (entry.ts && (!slot.lastSeen || entry.ts > slot.lastSeen)) slot.lastSeen = entry.ts;
+  }
+  return { totalEvents: lines.length, byFlavor };
+}

package/lib/telemetry/llm-judge.mjs

@@ -0,0 +1,350 @@
+/**
+ * lib/telemetry/llm-judge.mjs — LLM-as-a-judge evaluation system.
+ *
+ * Automatically evaluates agent work by:
+ * 1. Sampling traces that lack scores
+ * 2. Using an LLM to score quality based on the task and output
+ * 3. Recording scores back to the telemetry backend for continuous improvement
+ *
+ * Evaluation criteria includes:
+ * - Task completion accuracy (did it solve the problem?)
+ * - Code quality (if applicable)
+ * - Adherence to requirements
+ * - Best practices followed
+ * - Documentation completeness
+ * - Error handling robustness
+ */
+
+import { homedir } from 'node:os';
+import { addObservation } from '../observation-store.mjs';
+import { loadConstructEnv } from '../env-config.mjs';
+
+const CONF_ENV = loadConstructEnv({ warn: false });
+
+function telemetryHeaders() {
+  const key = CONF_ENV.CONSTRUCT_TELEMETRY_PUBLIC_KEY ?? process.env.CONSTRUCT_TELEMETRY_PUBLIC_KEY;
+  const secret = CONF_ENV.CONSTRUCT_TELEMETRY_SECRET_KEY ?? process.env.CONSTRUCT_TELEMETRY_SECRET_KEY;
+  if (!key || !secret) throw new Error('CONSTRUCT_TELEMETRY_PUBLIC_KEY and CONSTRUCT_TELEMETRY_SECRET_KEY must be set.');
+  return {
+    Authorization: `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`,
+    'Content-Type': 'application/json',
+  };
+}
+
+function telemetryBaseUrl() {
+  return (CONF_ENV.CONSTRUCT_TELEMETRY_URL ?? process.env.CONSTRUCT_TELEMETRY_URL ?? '').replace(/\/$/, '');
+}
+
+/**
+ * Fetch recent traces that have no scores yet
+ */
+async function fetchUnscoredTraces({ url, headers, limit = 50, fetchImpl = globalThis.fetch }) {
+  // First get recent traces
+  const recentRes = await fetchImpl(`${url}/api/public/traces?limit=${limit * 2}&orderBy=timestamp.desc`, { headers });
+  if (!recentRes.ok) throw new Error(`Telemetry traces fetch ${recentRes.status}`);
+  const recentJson = await recentRes.json().catch(() => ({}));
+  const recentTraces = Array.isArray(recentJson.data) ? recentJson.data : [];
+  
+  // Get scores to filter out already-scored traces
+  const scoresRes = await fetchImpl(`${url}/api/public/scores?name=quality&limit=100`, { headers });
+  if (!scoresRes.ok) throw new Error(`Telemetry scores fetch ${scoresRes.status}`);
+  const scoresJson = await scoresRes.json().catch(() => ({}));
+  const scoredTraceIds = new Set((Array.isArray(scoresJson.data) ? scoresJson.data : [])
+    .map(s => s.traceId)
+    .filter(Boolean));
+  
+  // Filter to traces without scores
+  const unscored = recentTraces
+    .filter(trace => !scoredTraceIds.has(trace.id))
+    .slice(0, limit);
+  
+  return unscored;
+}
+
+/**
+ * Build evaluation prompt for the LLM judge
+ */
+function buildEvaluationPrompt(trace, metadata = {}) {
+  const agentName = trace?.metadata?.agentName || trace?.name || 'unknown';
+  const task = typeof trace.input === 'string' ? trace.input : JSON.stringify(trace.input || '');
+  const output = typeof trace.output === 'string' ? trace.output : JSON.stringify(trace.output || '');
+  
+  const isCodeTask = task.toLowerCase().includes('code') || 
+                    task.toLowerCase().includes('implement') ||
+                    task.toLowerCase().includes('fix') ||
+                    task.toLowerCase().includes('write');
+  
+  const criteria = isCodeTask ? `
+Rate this code implementation on a scale of 0.0 (complete failure) to 1.0 (perfect):
+1. **Task Completion (0.3 weight)**: Did it solve the exact problem stated?
+2. **Code Quality (0.25 weight)**: Clean, readable, follows best practices?
+3. **Requirements Adherence (0.2 weight)**: Followed all specifications/constraints?
+4. **Robustness (0.15 weight)**: Handles edge cases, proper error handling?
+5. **Documentation (0.1 weight)**: Comments, clarity, maintainability?
+` : `
+Rate this agent work on a scale of 0.0 (complete failure) to 1.0 (perfect):
+1. **Task Completion (0.4 weight)**: Did it solve the exact problem stated?
+2. **Requirements Adherence (0.3 weight)**: Followed all specifications/constraints?
+3. **Clarity & Quality (0.2 weight)**: Clear, well-structured, professional?
+4. **Thoroughness (0.1 weight)**: Complete, covers all aspects needed?
+`;
+
+  const responseFormat = `Return ONLY a JSON object with:
+{
+  "score": 0.85,  // numeric score 0.0-1.0
+  "comment": "Brief explanation of the score",
+  "category": "quality",
+  "breakdown": {
+    "task_completion": 0.9,
+    "code_quality": 0.8,
+    "requirements_adherence": 0.9,
+    "robustness": 0.7,
+    "documentation": 0.8
+  }
+}`;
+
+  return {
+    system: `You are an expert quality evaluator for AI agent work. Evaluate the work objectively based on the criteria.`,
+    user: `# Agent Task
+Agent: ${agentName}
+Task: ${task}
+
+# Agent Output
+${output}
+
+# Evaluation Criteria
+${criteria}
+
+# Instructions
+${responseFormat}
+
+# Additional Context
+${metadata?.workCategory ? `Work category: ${metadata.workCategory}` : ''}
+${metadata?.routeIntent ? `Intent: ${metadata.routeIntent}` : ''}
+${trace?.metadata?.goal ? `Goal: ${trace.metadata.goal}` : ''}`,
+  };
+}
+
+/**
+ * Call LLM for evaluation
+ */
+async function callLLMJudge(prompt, { model = 'claude-3-5-sonnet-20241022', apiKey, baseUrl = 'https://api.anthropic.com' }) {
+  const headers = {
+    'x-api-key': apiKey,
+    'anthropic-version': '2023-06-01',
+    'content-type': 'application/json',
+  };
+  
+  const body = {
+    model,
+    max_tokens: 1000,
+    messages: [
+      { role: 'system', content: prompt.system },
+      { role: 'user', content: prompt.user },
+    ],
+    temperature: 0.1,
+  };
+  
+  const res = await fetch(`${baseUrl}/v1/messages`, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(body),
+  });
+  
+  if (!res.ok) {
+    const errorText = await res.text().catch(() => '');
+    throw new Error(`LLM API error ${res.status}: ${errorText.slice(0, 200)}`);
+  }
+  
+  const data = await res.json();
+  const content = data.content?.[0]?.text || '';
+  
+  // Parse JSON from response
+  try {
+    const jsonMatch = content.match(/\{[\s\S]*\}/);
+    if (jsonMatch) {
+      return JSON.parse(jsonMatch[0]);
+    }
+    // Fallback: try to extract score
+    const scoreMatch = content.match(/"score":\s*([0-9.]+)/);
+    const commentMatch = content.match(/"comment":\s*"([^"]+)"/);
+    return {
+      score: scoreMatch ? parseFloat(scoreMatch[1]) : 0.5,
+      comment: commentMatch ? commentMatch[1] : 'Could not parse evaluation',
+      category: 'quality',
+    };
+  } catch (err) {
+    return {
+      score: 0.5,
+      comment: `Failed to parse LLM response: ${err.message}`,
+      category: 'quality',
+    };
+  }
+}
+
+/**
+ * Score a trace using LLM judge
+ */
+async function scoreTraceWithLLM(trace, { llmApiKey, llmModel, fetchImpl = globalThis.fetch }) {
+  if (!llmApiKey) {
+    throw new Error('LLM API key required for judge evaluations');
+  }
+  
+  const prompt = buildEvaluationPrompt(trace, trace.metadata);
+  const evaluation = await callLLMJudge(prompt, {
+    model: llmModel || 'claude-3-5-sonnet-20241022',
+    apiKey: llmApiKey,
+  });
+  
+  return {
+    traceId: trace.id,
+    score: evaluation.score,
+    comment: evaluation.comment,
+    breakdown: evaluation.breakdown,
+    metadata: {
+      evaluatedAt: new Date().toISOString(),
+      evaluatorModel: llmModel || 'claude-3-5-sonnet-20241022',
+      source: 'llm-judge',
+    },
+  };
+}
+
+/**
+ * Post score to telemetry backend
+ */
+async function postScoreToTelemetry({ url, headers, scoreData, fetchImpl = globalThis.fetch }) {
+  const body = {
+    id: crypto.randomUUID(),
+    traceId: scoreData.traceId,
+    name: 'quality',
+    value: scoreData.score,
+    dataType: 'NUMERIC',
+    comment: scoreData.comment,
+    metadata: scoreData.metadata,
+  };
+  
+  const res = await fetchImpl(`${url}/api/public/scores`, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(body),
+  });
+  
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`Telemetry score post ${res.status}: ${text.slice(0, 200)}`);
+  }
+  
+  return res.json().catch(() => ({}));
+}
+
+/**
+ * Record evaluation observation for learning
+ */
+function recordEvaluationObservation(scoreData, trace) {
+  const rootDir = homedir();
+  const agentName = trace?.metadata?.agentName || trace?.name || 'construct';
+  
+  addObservation(rootDir, {
+    role: 'llm-judge',
+    category: 'evaluation',
+    summary: `LLM judge scored ${agentName} work at ${scoreData.score.toFixed(2)}: ${scoreData.comment.slice(0, 100)}`,
+    content: `Trace: ${scoreData.traceId}
+Score: ${scoreData.score}
+Comment: ${scoreData.comment}
+Agent: ${agentName}
+Task: ${typeof trace.input === 'string' ? trace.input.slice(0, 500) : 'N/A'}
+Evaluator: ${scoreData.metadata.evaluatorModel}
+Breakdown: ${JSON.stringify(scoreData.breakdown || {}, null, 2)}`,
+    tags: ['llm-judge', 'quality-score', agentName, 'auto-evaluated'],
+    confidence: 0.9,
+    source: 'llm-judge',
+  });
+}
+
+/**
+ * Main entrypoint: Run LLM judge evaluations on unscored traces
+ */
+export async function runLLMJudgeEvaluations({
+  publicKey = CONF_ENV.CONSTRUCT_TELEMETRY_PUBLIC_KEY ?? process.env.CONSTRUCT_TELEMETRY_PUBLIC_KEY,
+  secretKey = CONF_ENV.CONSTRUCT_TELEMETRY_SECRET_KEY ?? process.env.CONSTRUCT_TELEMETRY_SECRET_KEY,
+  llmApiKey = CONF_ENV.ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_API_KEY,
+  llmModel = 'claude-3-5-sonnet-20241022',
+  limit = 10,
+  bestEffort = true,
+  fetchImpl = globalThis.fetch,
+} = {}) {
+  if (!publicKey || !secretKey) {
+    return { evaluated: 0, errors: ['Telemetry credentials not configured'] };
+  }
+  
+  if (!llmApiKey) {
+    return { evaluated: 0, errors: ['LLM API key (ANTHROPIC_API_KEY) required for judge evaluations'] };
+  }
+  
+  const url = telemetryBaseUrl();
+  const headers = telemetryHeaders();
+  const errors = [];
+  let evaluated = 0;
+  
+  try {
+    // Get unscored traces
+    const unscoredTraces = await fetchUnscoredTraces({ url, headers, limit, fetchImpl });
+    
+    for (const trace of unscoredTraces) {
+      try {
+        // Score with LLM
+        const scoreData = await scoreTraceWithLLM(trace, { llmApiKey, llmModel, fetchImpl });
+        
+        // Post score to telemetry backend
+        await postScoreToTelemetry({ url, headers, scoreData, fetchImpl });
+        
+        // Record observation for learning
+        recordEvaluationObservation(scoreData, trace);
+        
+        evaluated++;
+        
+      } catch (err) {
+        if (bestEffort) {
+          errors.push(`${trace?.id}: ${err.message}`);
+        } else {
+          throw err;
+        }
+      }
+    }
+    
+  } catch (err) {
+    if (bestEffort) {
+      errors.push(`Initialization error: ${err.message}`);
+    } else {
+      throw err;
+    }
+  }
+  
+  return { evaluated, errors };
+}
+
+/**
+ * CLI entrypoint
+ */
+export async function runLLMJudgeCli(args = []) {
+  const limitArg = args.find((a) => a.startsWith('--limit='));
+  const limit = limitArg ? parseInt(limitArg.split('=')[1], 10) || 10 : 10;
+  
+  const modelArg = args.find((a) => a.startsWith('--model='));
+  const model = modelArg ? modelArg.split('=')[1] : 'claude-3-5-sonnet-20241022';
+  
+  process.stdout.write(`Running LLM judge evaluations (limit: ${limit}, model: ${model})…\n`);
+  
+  const result = await runLLMJudgeEvaluations({
+    limit,
+    llmModel: model,
+    bestEffort: false,
+  });
+  
+  process.stdout.write(`Done. evaluated=${result.evaluated}${result.errors.length ? ` errors=${result.errors.length}` : ''}\n`);
+  if (result.errors.length) {
+    for (const e of result.errors) process.stderr.write(`  ${e}\n`);
+  }
+  
+  return result;
+}