@principles/pd-cli 1.152.12 → 1.152.14

package/src/services/quality-scorecard/data-extractor.ts

@@ -1,150 +0,0 @@
-/**
- * PRI-361 — Data Extractor (I/O layer in pd-cli)
- *
- * Reads trajectory.db and event logs. Uses runtime validation
- * from @principles/core — no `as RawPainEvent[]` casts.
- */
-
-import type { PainEpisode } from '@principles/core/quality-scorecard';
-import {
-  validatePainRow,
-  validateEvolutionRow,
-  validatePrincipleEventRow,
-  validateGateRow,
-  sanitize,
-  truncate,
-} from '@principles/core/quality-scorecard';
-import { readdirSync, readFileSync } from 'fs';
-import { join } from 'path';
-
-export interface ExtractionResult {
-  episodes: PainEpisode[];
-  stats: { total: number; dateRange: { from: string; to: string } };
-}
-
-export async function extractEpisodes(
-  dbPath: string,
-  options: { minScore?: number; limit?: number } = {}
-): Promise<ExtractionResult> {
-  const Database = (await import('better-sqlite3')).default;
-  const db = new Database(dbPath, { readonly: true });
-
-  const minScore = options.minScore ?? 0;
-  const limit = options.limit ?? 0;
-
-  try {
-    // Fetch pain events with parameterized query
-    let painQuery = 'SELECT id, session_id, source, score, reason, severity, created_at FROM pain_events WHERE 1=1';
-    const params: (string | number)[] = [];
-    if (minScore > 0) {
-      painQuery += ' AND score >= ?';
-      params.push(minScore);
-    }
-    painQuery += ' ORDER BY created_at DESC';
-    if (limit > 0) {
-      painQuery += ' LIMIT ?';
-      params.push(limit);
-    }
-
-    const rawPainRows = db.prepare(painQuery).all(...params);
-    // Validate each row — no unsafe cast
-    const painEvents = rawPainRows.map(validatePainRow).filter((r): r is NonNullable<typeof r> => r !== null);
-
-    // Fetch evolution tasks
-    const rawEvoRows = db.prepare('SELECT task_id, score, status, resolution, created_at FROM evolution_tasks').all();
-    const evoTasks = rawEvoRows.map(validateEvolutionRow).filter((r): r is NonNullable<typeof r> => r !== null);
-
-    // Fetch principle events
-    const rawPeRows = db.prepare('SELECT principle_id, event_type, created_at FROM principle_events').all();
-    const prEvents = rawPeRows.map(validatePrincipleEventRow).filter((r): r is NonNullable<typeof r> => r !== null);
-
-    // Gate blocks count per session — validate each row
-    const rawGateRows = db.prepare('SELECT session_id, COUNT(*) as cnt FROM gate_blocks GROUP BY session_id').all();
-    const gateRows = rawGateRows.map(validateGateRow).filter((r): r is NonNullable<typeof r> => r !== null);
-    const gateBlockMap = new Map(gateRows.map(g => [g.session_id, g.cnt]));
-
-    // Deduplicate by sanitized reason
-    const seen = new Set<string>();
-    const episodes: PainEpisode[] = [];
-
-    for (const pe of painEvents) {
-      const dedupKey = sanitize(pe.reason).substring(0, 80);
-      if (seen.has(dedupKey)) continue;
-      seen.add(dedupKey);
-
-      const peTime = new Date(pe.created_at).getTime();
-      const linkedTask = evoTasks.find(t => {
-        const tTime = new Date(t.created_at).getTime();
-        return Math.abs(tTime - peTime) < 3600000 && Math.abs(t.score - pe.score) <= 10;
-      });
-
-      const linkedPrinciples = prEvents
-        .filter(e => {
-          if (!e.principle_id) return false;
-          const eTime = new Date(e.created_at).getTime();
-          return Math.abs(eTime - peTime) < 7200000;
-        })
-        .map(e => e.principle_id)
-        .filter((v): v is string => v !== null)
-        .filter((v, i, a) => a.indexOf(v) === i);
-
-      episodes.push({
-        episodeId: `EP-${pe.id}`,
-        summary: truncate(sanitize(pe.reason)),
-        source: pe.source,
-        score: pe.score,
-        severity: pe.severity,
-        createdAt: pe.created_at,
-        evolutionTaskResolution: linkedTask?.resolution ?? null,
-        linkedPrinciples,
-        gateBlockCount: gateBlockMap.get(pe.session_id) ?? 0,
-      });
-    }
-
-    const dates = painEvents.map(e => e.created_at).sort();
-    return {
-      episodes,
-      stats: {
-        total: painEvents.length,
-        dateRange: {
-          from: dates[0] ?? new Date().toISOString(),
-          to: dates[dates.length - 1] ?? new Date().toISOString(),
-        },
-      },
-    };
-  } finally {
-    db.close();
-  }
-}
-
-export interface LogStats {
-  totalEvents: number;
-  painSignalCount: number;
-  degradedReasons: string[];
-}
-
-export function extractLogStats(logsDir: string): LogStats {
-  const stats: LogStats = { totalEvents: 0, painSignalCount: 0, degradedReasons: [] };
-
-  try {
-    const files = readdirSync(logsDir).filter(f => f.endsWith('.jsonl'));
-    for (const file of files) {
-      const lines = readFileSync(join(logsDir, file), 'utf-8').split('\n').filter(Boolean);
-      for (const line of lines) {
-        try {
-          const ev = JSON.parse(line) as Record<string, unknown>;
-          stats.totalEvents++;
-          if (ev.type === 'pain_signal') stats.painSignalCount++;
-        } catch (parseErr: unknown) {
-          const msg = parseErr instanceof Error ? parseErr.message : String(parseErr);
-          stats.degradedReasons.push(`jsonl-parse-fail:${file}:${msg}`);
-        }
-      }
-    }
-  } catch (dirErr: unknown) {
-    const msg = dirErr instanceof Error ? dirErr.message : String(dirErr);
-    stats.degradedReasons.push(`logs-dir-unreadable:${msg}`);
-  }
-
-  return stats;
-}

package/src/services/quality-scorecard/local-evaluator.ts

@@ -1,142 +0,0 @@
-/**
- * PRI-361 — Local Evaluator (I/O layer in pd-cli)
- *
- * Calls LM Studio for advisory scoring. Uses core validation
- * to parse LLM responses — no unsafe casts.
- */
-
-import type {
-  PainEpisode,
-  LocalEvaluation,
-  RubricDimension,
-  RubricScore,
-} from '@principles/core/quality-scorecard';
-import {
-  RUBRIC_LABELS,
-  RUBRIC_PROMPTS,
-  RUBRIC_DIMENSIONS as DIMS,
-  meetsMvpThreshold,
-  sumScores,
-  validateLlmScoreResponse,
-  extractJsonFromLlmResponse,
-} from '@principles/core/quality-scorecard';
-
-function buildEvaluationPrompt(episode: PainEpisode): string {
-  const dimensions = DIMS.map(d => `${d} (${RUBRIC_LABELS[d]}): ${RUBRIC_PROMPTS[d]}`).join('\n');
-
-  return `You are a quality evaluator for an AI agent's pain-signal -> diagnosis -> principle pipeline.
-
-## Task
-Evaluate this pain episode on a 7-dimension rubric. Each dimension scores 0 (fail), 1 (partial), or 2 (pass).
-
-## Pain Episode
-- ID: ${episode.episodeId}
-- Source: ${episode.source}
-- Pain Score: ${episode.score}
-- Severity: ${episode.severity}
-- Summary: ${episode.summary}
-- Created: ${episode.createdAt}
-- Evolution Task Resolution: ${episode.evolutionTaskResolution ?? 'none'}
-- Linked Principles: ${episode.linkedPrinciples.length > 0 ? episode.linkedPrinciples.join(', ') : 'none'}
-- Gate Blocks: ${episode.gateBlockCount}
-
-## Rubric Dimensions
-${dimensions}
-
-## Additional Checks
-- Is the language consistent (not mixing Chinese and English incoherently)?
-- Is the diagnosis/principle overly abstract (no concrete actionable guidance)?
-- Does it fabricate non-existent evidence, axioms, or references?
-
-## Output Format (STRICT JSON)
-Respond with ONLY a JSON object:
-{
-  "scores": { "G1": 0-2, "G2": 0-2, "G3": 0-2, "G4": 0-2, "G5": 0-2, "G6": 0-2, "G7": 0-2 },
-  "rationales": { "G1": "...", "G2": "...", "G3": "...", "G4": "...", "G5": "...", "G6": "...", "G7": "..." },
-  "flags": ["list of issues found"]
-}
-
-Do NOT output anything other than this JSON object.`;
-}
-
-export interface LocalEvaluatorConfig {
-  baseUrl: string;
-  model: string;
-}
-
-export async function evaluateWithLocalModel(
-  episode: PainEpisode,
-  config: LocalEvaluatorConfig,
-  log: (msg: string) => void
-): Promise<LocalEvaluation> {
-  const prompt = buildEvaluationPrompt(episode);
-  const url = `${config.baseUrl.replace(/\/+$/, '')}/chat/completions`;
-
-  try {
-    const resp = await fetch(url, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({
-        model: config.model,
-        messages: [
-          { role: 'system', content: 'You are a precise JSON-output quality evaluator. Output only valid JSON.' },
-          { role: 'user', content: prompt },
-        ],
-        temperature: 0.1,
-        max_tokens: 2000,
-      }),
-      signal: AbortSignal.timeout(120_000),
-    });
-
-    if (!resp.ok) {
-      throw new Error(`LM Studio request failed: ${resp.status}`);
-    }
-
-    const data = (await resp.json()) as { choices: { message: { content: string } }[] };
-    const content = data.choices?.[0]?.message?.content ?? '';
-
-    const parsed = extractJsonFromLlmResponse(content);
-    if (parsed === null) {
-      throw new Error(`LM Studio returned non-JSON response`);
-    }
-
-    const { scores, rationales, flags } = validateLlmScoreResponse(parsed);
-    const totalScore = sumScores(scores);
-
-    return {
-      model: config.model,
-      dimensionScores: scores,
-      dimensionRationales: rationales,
-      totalScore,
-      maxScore: 14,
-      mvpMet: meetsMvpThreshold(scores),
-      flags: flags,
-    };
-  } catch (err: unknown) {
-    const msg = err instanceof Error ? err.message : String(err);
-    log(`Evaluation error for ${episode.episodeId}: ${msg}`);
-    const zeroScores = Object.fromEntries(DIMS.map(d => [d, 0])) as Record<RubricDimension, RubricScore>;
-    return {
-      model: config.model,
-      dimensionScores: zeroScores,
-      dimensionRationales: Object.fromEntries(DIMS.map(d => [d, `Evaluation failed: ${msg}`])) as Record<RubricDimension, string>,
-      totalScore: 0,
-      maxScore: 14,
-      mvpMet: false,
-      flags: ['evaluation_error'],
-    };
-  }
-}
-
-export async function checkLmStudioAvailable(baseUrl: string): Promise<{ available: boolean; models: string[]; error?: string }> {
-  try {
-    const url = `${baseUrl.replace(/\/+$/, '')}/models`;
-    const resp = await fetch(url, { signal: AbortSignal.timeout(5000) });
-    if (!resp.ok) return { available: false, models: [], error: `HTTP ${resp.status}` };
-    const data = (await resp.json()) as { data: { id: string }[] };
-    const models = (data.data || []).map((m) => m.id);
-    return { available: true, models };
-  } catch (err: unknown) {
-    return { available: false, models: [], error: err instanceof Error ? err.message : String(err) };
-  }
-}

package/src/services/quality-scorecard/strong-model-gate.ts

@@ -1,202 +0,0 @@
-/**
- * PRI-361 — Strong Model Adjudication Gate (I/O layer in pd-cli)
- *
- * Calls cloud model for adjudication. Uses core validation
- * to parse responses — no unsafe casts.
- */
-
-import type {
-  PainEpisode,
-  LocalEvaluation,
-  StrongModelAdjudication,
-  AdjudicationStatus,
-} from '@principles/core/quality-scorecard';
-import {
-  RUBRIC_LABELS,
-  RUBRIC_DIMENSIONS as DIMS,
-  meetsMvpThreshold,
-  validateAdjudicationResponse,
-  extractJsonFromLlmResponse,
-} from '@principles/core/quality-scorecard';
-
-function buildAdjudicationPrompt(
-  episode: PainEpisode,
-  localEval: LocalEvaluation
-): string {
-  const localScores = DIMS.map(d =>
-    `- ${d} (${RUBRIC_LABELS[d]}): ${localEval.dimensionScores[d]}/2 — ${localEval.dimensionRationales[d]}`
-  ).join('\n');
-
-  return `You are a senior quality adjudicator for an AI agent evolution pipeline.
-Your job is to independently re-evaluate a pain episode that was first scored by a local (smaller) model.
-You must provide your own scores — do NOT simply copy the local model's scores.
-
-## Pain Episode
-- ID: ${episode.episodeId}
-- Source: ${episode.source}
-- Pain Score: ${episode.score}
-- Severity: ${episode.severity}
-- Summary: ${episode.summary}
-- Evolution Task Resolution: ${episode.evolutionTaskResolution ?? 'none'}
-- Linked Principles: ${episode.linkedPrinciples.length > 0 ? episode.linkedPrinciples.join(', ') : 'none'}
-
-## Local Model Scores (${localEval.model})
-${localScores}
-Flags: ${localEval.flags.length > 0 ? localEval.flags.join(', ') : 'none'}
-
-## Your Task
-1. Independently score each dimension (0/1/2) based on the evidence.
-2. Check for: language inconsistency, over-abstraction, fabricated evidence.
-3. If your scores differ from the local model by >=2 points on any dimension, explain why.
-4. Give a final verdict: pass, fail, or needs-review.
-
-## Output Format (STRICT JSON)
-{
-  "scores": { "G1": 0-2, "G2": 0-2, "G3": 0-2, "G4": 0-2, "G5": 0-2, "G6": 0-2, "G7": 0-2 },
-  "rationale": "Overall assessment...",
-  "verdict": "pass" | "fail" | "needs-review"
-}
-
-Do NOT output anything other than this JSON object.`;
-}
-
-/**
- * CWE-918 (SSRF) mitigation for operator-supplied LLM API base URLs.
- *
- * Threat model: OPENAI_BASE_URL is explicitly configured by the Owner /
- * operator in the environment or Runtime Profile — it is trusted operator
- * configuration, NOT untrusted remote input. Local and private-network
- * OpenAI-compatible endpoints (llama.cpp, LM Studio, local gateways,
- * intranet model servers) are legitimate PD runtime targets and must keep
- * working.
- *
- * What stays blocked:
- * - non-http(s) schemes (file:, javascript:, data:, ...)
- * - malformed / unparseable URLs
- * - credentials embedded in the URL (secrets leak into logs/errors)
- *
- * Callers must not allow the host to be rewritten from untrusted input
- * after this validation; the endpoint is derived from this URL only.
- */
-export function assertSafeLlmBaseUrl(rawBaseUrl: string): URL {
-  let url: URL;
-  try {
-    url = new URL(rawBaseUrl);
-  } catch {
-    throw new Error(`Invalid OPENAI_BASE_URL: "${rawBaseUrl}" is not a valid URL`);
-  }
-  if (url.protocol !== 'https:' && url.protocol !== 'http:') {
-    throw new Error(`Invalid OPENAI_BASE_URL: protocol must be http or https, got "${url.protocol}"`);
-  }
-  if (url.username || url.password) {
-    throw new Error(`Invalid OPENAI_BASE_URL: credentials must not be embedded in the URL`);
-  }
-  return url;
-}
-
-export async function adjudicate(
-  episode: PainEpisode,
-  localEval: LocalEvaluation,
-  config: { modelId: string; log: (msg: string) => void }
-): Promise<StrongModelAdjudication> {
-  const { modelId: strongModelId, log } = config;
-  const prompt = buildAdjudicationPrompt(episode, localEval);
-  // CWE-918 (SSRF): validate the operator-supplied base URL before any
-  // network request — http(s) only, no embedded credentials. Local and
-  // private OpenAI-compatible endpoints remain valid (trusted operator
-  // configuration; llama.cpp / LM Studio / local gateways are supported).
-  const baseUrl = assertSafeLlmBaseUrl(process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1');
-  const apiKey = process.env.OPENAI_API_KEY;
-
-  if (!apiKey) {
-    return {
-      model: strongModelId,
-      adjudicationStatus: 'needs-review',
-      confirmedScores: null,
-      confirmedMvpMet: null,
-      rationale: 'OPENAI_API_KEY not set — cannot run strong-model adjudication',
-      nextAction: 'Set OPENAI_API_KEY and re-run with --strong-model',
-    };
-  }
-
-  try {
-    // Build the endpoint on the validated URL: append the chat completions
-    // path to the base URL's (possibly empty) path component.
-    const endpoint = new URL(baseUrl);
-    endpoint.pathname = endpoint.pathname.replace(/\/+$/, '') + '/chat/completions';
-    const resp = await fetch(endpoint.toString(), {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        Authorization: `Bearer ${apiKey}`,
-      },
-      body: JSON.stringify({
-        model: strongModelId,
-        messages: [
-          { role: 'system', content: 'You are a precise JSON-output quality adjudicator. Output only valid JSON.' },
-          { role: 'user', content: prompt },
-        ],
-        temperature: 0.1,
-        max_tokens: 2000,
-      }),
-      signal: AbortSignal.timeout(120_000),
-    });
-
-    if (!resp.ok) {
-      throw new Error(`Strong model request failed: ${resp.status}`);
-    }
-
-    const data = (await resp.json()) as { choices: { message: { content: string } }[] };
-    const content = data.choices?.[0]?.message?.content ?? '';
-    const parsed = extractJsonFromLlmResponse(content);
-    if (parsed === null) {
-      throw new Error('Strong model returned non-JSON');
-    }
-
-    const validated = validateAdjudicationResponse(parsed);
-    const { scores, verdict } = validated;
-
-    return {
-      model: strongModelId,
-      adjudicationStatus: verdict,
-      confirmedScores: scores,
-      confirmedMvpMet: meetsMvpThreshold(scores),
-      rationale: validated.rationale,
-      nextAction: null,
-    };
-  } catch (err: unknown) {
-    const msg = err instanceof Error ? err.message : String(err);
-    log(`Adjudication error: ${msg}`);
-    return {
-      model: strongModelId,
-      adjudicationStatus: 'needs-review',
-      confirmedScores: null,
-      confirmedMvpMet: null,
-      rationale: `Adjudication failed: ${msg}`,
-      nextAction: 'Retry with strong model or manually review',
-    };
-  }
-}
-
-export function skippedAdjudication(reason: string): StrongModelAdjudication {
-  return {
-    model: 'none',
-    adjudicationStatus: 'skipped',
-    confirmedScores: null,
-    confirmedMvpMet: null,
-    rationale: reason,
-    nextAction: 'Configure and run strong-model adjudication for final quality verdict',
-  };
-}
-
-export function determineFinalLabel(
-  localEval: LocalEvaluation,
-  adjudication: StrongModelAdjudication | null
-): AdjudicationStatus {
-  if (!adjudication || adjudication.adjudicationStatus === 'skipped') {
-    if (localEval.mvpMet && localEval.totalScore >= 12) return 'local-pass';
-    if (localEval.totalScore <= 6) return 'local-fail';
-    return 'needs-review';
-  }
-  return adjudication.adjudicationStatus;
-}