@ngockhoale/ukit 3.4.47 → 3.4.49

package/scripts/bench/decision-runtime-baseline.mjs

@@ -1,427 +0,0 @@
-#!/usr/bin/env node
-// TASK-004 — DR-01b decision-runtime baseline runner (SPEC §5 FR-006, §8, §10).
-//
-// Replays the versioned synthetic corpus (docs/AI_HANDOFF/benchmark/corpus.yaml)
-// through the TASK-003 metric module and writes a schema-valid JSON report:
-//   {version, env{sha,package,node,os,arch,fs}, runs,
-//    scenarios[{id,verdict,wallMs,wakes,eligible,quality,tokens}],
-//    summary{externalWakeRate,qualityScore,p50,p95}}
-//
-// The baseline replays deterministic inline commands only — no model/agent
-// wake is ever made — so `wakes` is measured as 0 and `tokens` is 'UNKNOWN'
-// (no provider data source exists; never 0 or an estimate, metric-spec §5).
-//
-// Exit codes: 0 = ran to completion (verdicts are data), 2 = malformed
-// corpus/args or harness crash. Per-step timeout default 30s.
-
-import { spawn, execSync } from 'node:child_process';
-import fs from 'node:fs';
-import os from 'node:os';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import yaml from 'yaml';
-
-import {
-  UNKNOWN,
-  computeExternalWakeRate,
-  computeQualityScore,
-  summarizeRuns,
-} from './decision-runtime-metrics.mjs';
-
-const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
-const DEFAULT_RUNS = 9;
-const DEFAULT_TIMEOUT_MS = 30_000;
-const ORACLE_TYPES = ['exit-code', 'output-match', 'file-exists'];
-
-const USAGE = `Usage: node scripts/bench/decision-runtime-baseline.mjs --corpus <file.yaml> --out <file.json> [options]
-
-Replays the DR-01b synthetic corpus and writes a schema-valid JSON report
-(SPEC §5 FR-006). Verdicts are data: a failing scenario does not fail the run.
-
-Options:
-  --corpus <file.yaml>   Corpus path (required)
-  --out <file.json>      Report output path (required)
-  --runs <n>             Repetitions per scenario (default ${DEFAULT_RUNS})
-  --timeout <ms>         Per-step timeout (default ${DEFAULT_TIMEOUT_MS})
-  --help                 Show this help
-
-Exit codes: 0 = completed, 2 = malformed corpus/args or crash.`;
-
-// ---------- args ----------
-
-function parseArgs(argv) {
-  const opts = { runs: DEFAULT_RUNS, timeoutMs: DEFAULT_TIMEOUT_MS };
-  for (let i = 0; i < argv.length; i += 1) {
-    const arg = argv[i];
-    if (arg === '--help' || arg === '-h') return { help: true };
-    const takeValue = () => {
-      i += 1;
-      if (i >= argv.length) throw new Error(`Missing value for ${arg}`);
-      return argv[i];
-    };
-    if (arg === '--corpus') opts.corpus = takeValue();
-    else if (arg === '--out') opts.out = takeValue();
-    else if (arg === '--runs') opts.runs = Number(takeValue());
-    else if (arg === '--timeout') opts.timeoutMs = Number(takeValue());
-    else throw new Error(`Unknown argument: ${arg}`);
-  }
-  return opts;
-}
-
-// ---------- corpus loading + validation ----------
-
-function isNonEmptyString(value) {
-  return typeof value === 'string' && value.trim() !== '';
-}
-
-function isNonNegativeInt(value) {
-  return typeof value === 'number' && Number.isInteger(value) && value >= 0;
-}
-
-/** Validate a parsed corpus doc; returns problem strings (empty = valid). */
-function validateCorpus(doc) {
-  const problems = [];
-  if (!doc || typeof doc !== 'object' || Array.isArray(doc)) {
-    return ['corpus is not an object'];
-  }
-  if (doc.version !== 1) problems.push('missing or invalid `version` (must be 1)');
-  const scenarios = doc.scenarios;
-  if (!Array.isArray(scenarios) || scenarios.length === 0) {
-    problems.push('`scenarios` must be a non-empty array');
-    return problems;
-  }
-  const seen = new Set();
-  scenarios.forEach((sc, idx) => {
-    const where = `scenarios[${idx}]`;
-    if (!sc || typeof sc !== 'object') {
-      problems.push(`${where}: not an object`);
-      return;
-    }
-    const label = isNonEmptyString(sc.id) ? `scenario ${sc.id}` : where;
-    if (!isNonEmptyString(sc.id)) problems.push(`${where}: missing required field \`id\``);
-    else if (seen.has(sc.id)) problems.push(`duplicate scenario id: ${sc.id}`);
-    else seen.add(sc.id);
-    if (!isNonEmptyString(sc.class)) problems.push(`${label}: missing required field \`class\``);
-    if (!isNonEmptyString(sc.description)) problems.push(`${label}: missing required field \`description\``);
-    if (!Array.isArray(sc.steps) || sc.steps.length === 0) {
-      problems.push(`${label}: \`steps\` must be a non-empty array`);
-    } else {
-      sc.steps.forEach((step, sIdx) => {
-        if (!step || typeof step !== 'object' || !isNonEmptyString(step.run)) {
-          problems.push(`${label} steps[${sIdx}]: missing required field \`run\``);
-        } else if (step.expectExit !== undefined && !Number.isInteger(step.expectExit)) {
-          problems.push(`${label} steps[${sIdx}]: \`expectExit\` must be an integer`);
-        }
-      });
-    }
-    const oracle = sc.oracle;
-    if (!oracle || typeof oracle !== 'object' || !ORACLE_TYPES.includes(oracle.type)) {
-      problems.push(`${label}: \`oracle.type\` must be one of ${ORACLE_TYPES.join('|')}`);
-    } else if (oracle.type === 'exit-code' && !Number.isInteger(oracle.expect)) {
-      problems.push(`${label}: exit-code oracle requires integer \`expect\``);
-    } else if (oracle.type === 'output-match' && !isNonEmptyString(oracle.pattern)) {
-      problems.push(`${label}: output-match oracle requires \`pattern\``);
-    } else if (oracle.type === 'file-exists' && !isNonEmptyString(oracle.path)) {
-      problems.push(`${label}: file-exists oracle requires \`path\``);
-    }
-    if (!isNonNegativeInt(sc.eligibleTransitions)) {
-      problems.push(`${label}: missing required field \`eligibleTransitions\` (non-negative integer)`);
-    }
-    if (!isNonNegativeInt(sc.expectedExternalWakes)) {
-      problems.push(`${label}: missing required field \`expectedExternalWakes\` (non-negative integer)`);
-    }
-  });
-  return problems;
-}
-
-function loadCorpus(corpusPath) {
-  if (!fs.existsSync(corpusPath)) {
-    throw new Error(`corpus not found: ${corpusPath}`);
-  }
-  let doc;
-  try {
-    doc = yaml.parse(fs.readFileSync(corpusPath, 'utf8'));
-  } catch (error) {
-    throw new Error(`corpus YAML parse error: ${error.message}`);
-  }
-  const problems = validateCorpus(doc);
-  if (problems.length > 0) {
-    throw new Error(`malformed corpus:\n  - ${problems.join('\n  - ')}`);
-  }
-  return doc;
-}
-
-// ---------- step execution ----------
-
-const SHELL = process.platform === 'win32'
-  ? { cmd: 'cmd.exe', args: ['/d', '/s', '/c'] }
-  : { cmd: '/bin/sh', args: ['-c'] };
-
-/** Run one inline step command; captures output for output-match oracles. */
-function runStep(run, { cwd, timeoutMs }) {
-  return new Promise((resolve) => {
-    const started = Date.now();
-    let timedOut = false;
-    let spawnError = null;
-    let output = '';
-    let child;
-    try {
-      // Bounded child: `timeout` arms the spawn-level deadline (SIGTERM) and
-      // the watchdog timer below escalates to SIGKILL — a wedged step can
-      // never hang the runner.
-      child = spawn(SHELL.cmd, [...SHELL.args, run], {
-        cwd,
-        env: process.env,
-        stdio: ['ignore', 'pipe', 'pipe'],
-        timeout: timeoutMs,
-      });
-    } catch (error) {
-      resolve({ exitCode: -1, durationMs: Date.now() - started, timedOut: false, error: String(error?.message ?? error), output: '' });
-      return;
-    }
-    // Watchdog: escalates the spawn timeout's SIGTERM to SIGKILL so a step
-    // that ignores termination still dies at the deadline.
-    const timer = setTimeout(() => {
-      timedOut = true;
-      child.kill('SIGKILL');
-    }, timeoutMs);
-    child.stdout.on('data', (chunk) => { output += chunk; });
-    child.stderr.on('data', (chunk) => { output += chunk; });
-    child.on('error', (error) => { spawnError = error; });
-    child.on('close', (code) => {
-      clearTimeout(timer);
-      resolve({
-        exitCode: timedOut || code == null ? -1 : code,
-        durationMs: Date.now() - started,
-        timedOut,
-        error: spawnError ? String(spawnError.message ?? spawnError) : null,
-        output,
-      });
-    });
-  });
-}
-
-// ---------- oracle ----------
-
-function checkOracle(oracle, { lastExit, output, cwd }) {
-  switch (oracle.type) {
-    case 'exit-code':
-      return lastExit === oracle.expect;
-    case 'output-match':
-      return output.includes(oracle.pattern);
-    case 'file-exists':
-      return fs.existsSync(path.resolve(cwd, oracle.path));
-    default:
-      return false;
-  }
-}
-
-// ---------- scenario replay ----------
-
-/**
- * Replay one scenario `runs` times. Each run executes steps in order and stops
- * at the first step whose exit code differs from `expectExit` (default 0) or
- * that times out — a crash mid-operation means later declared-eligible
- * transitions are never reached (coverage gap, metric-spec §1).
- */
-async function replayScenario(scenario, { runs, timeoutMs, workDir }) {
-  const durations = [];
-  let oraclePasses = 0;
-  let forbiddenFailures = 0;
-  let reachedTransitions = 0;
-  let stepMismatches = 0;
-  let stepTimeouts = 0;
-
-  for (let r = 0; r < runs; r += 1) {
-    const runDir = path.join(workDir, `run-${r}`);
-    fs.mkdirSync(runDir, { recursive: true });
-    const started = Date.now();
-    let lastExit = -1;
-    let output = '';
-    let stepsOk = true;
-    let reached = 0;
-
-    for (const step of scenario.steps) {
-      const expected = step.expectExit ?? 0;
-      const result = await runStep(step.run, { cwd: runDir, timeoutMs });
-      lastExit = result.exitCode;
-      output += result.output;
-      if (result.timedOut) stepTimeouts += 1;
-      if (result.exitCode !== expected || result.timedOut || result.error) {
-        stepsOk = false;
-        stepMismatches += 1;
-        break;
-      }
-      reached += 1;
-    }
-
-    durations.push(Date.now() - started);
-    reachedTransitions += Math.min(reached, scenario.eligibleTransitions);
-
-    const oraclePass = checkOracle(scenario.oracle, { lastExit, output, cwd: runDir });
-    if (oraclePass) oraclePasses += 1;
-    // Forbidden failure (metric-spec §2): every step completed as declared
-    // (exit-code success) while the oracle rejects the run — false completion.
-    if (stepsOk && !oraclePass) forbiddenFailures += 1;
-  }
-
-  const quality = computeQualityScore({
-    satisfied: oraclePasses,
-    eligible: runs,
-    forbiddenFailures,
-  });
-  const { p50, p95 } = summarizeRuns(durations);
-  const coverageGap = Math.max(0, scenario.eligibleTransitions * runs - reachedTransitions);
-
-  return {
-    entry: {
-      id: scenario.id,
-      class: scenario.class,
-      verdict: quality.verdict,
-      wallMs: { n: runs, p50, p95 },
-      wakes: 0,
-      eligible: reachedTransitions,
-      coverageGap,
-      quality: {
-        score: quality.score,
-        satisfied: oraclePasses,
-        eligible: runs,
-        forbiddenFailures,
-      },
-      tokens: UNKNOWN,
-      stepMismatches,
-      stepTimeouts,
-    },
-    durations,
-  };
-}
-
-// ---------- env ----------
-
-function detectFsType(dir) {
-  try {
-    if (typeof fs.statfsSync === 'function') {
-      const stats = fs.statfsSync(dir);
-      if (stats && typeof stats.type === 'number') {
-        return `0x${stats.type.toString(16)}`;
-      }
-    }
-  } catch { /* fall through */ }
-  return 'local';
-}
-
-function collectEnv() {
-  const env = {
-    sha: 'unknown',
-    package: 'unknown',
-    node: process.version,
-    os: process.platform,
-    arch: process.arch,
-    fs: detectFsType(REPO_ROOT),
-  };
-  try {
-    env.sha = execSync('git rev-parse HEAD', { cwd: REPO_ROOT, encoding: 'utf8', timeout: 10_000 }).trim();
-  } catch { /* leave 'unknown' */ }
-  try {
-    env.package = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')).version ?? 'unknown';
-  } catch { /* leave 'unknown' */ }
-  return env;
-}
-
-// ---------- main ----------
-
-async function main() {
-  let opts;
-  try {
-    opts = parseArgs(process.argv.slice(2));
-  } catch (error) {
-    console.error(`[decision-runtime-baseline] ${error.message}\n\n${USAGE}`);
-    process.exit(2);
-  }
-  if (opts.help) {
-    console.log(USAGE);
-    process.exit(0);
-  }
-  if (!opts.corpus || !opts.out) {
-    console.error(`[decision-runtime-baseline] --corpus and --out are required.\n\n${USAGE}`);
-    process.exit(2);
-  }
-  if (!Number.isInteger(opts.runs) || opts.runs < 1) {
-    console.error('[decision-runtime-baseline] --runs must be a positive integer.');
-    process.exit(2);
-  }
-  if (!Number.isFinite(opts.timeoutMs) || opts.timeoutMs < 1) {
-    console.error('[decision-runtime-baseline] --timeout must be a positive number of ms.');
-    process.exit(2);
-  }
-
-  let corpus;
-  try {
-    corpus = loadCorpus(path.resolve(opts.corpus));
-  } catch (error) {
-    console.error(`[decision-runtime-baseline] ${error.message}`);
-    process.exit(2);
-  }
-
-  const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ukit-dr-baseline-'));
-  const scenarios = [];
-  const allDurations = [];
-  for (const scenario of corpus.scenarios) {
-    const workDir = path.join(tmpRoot, scenario.id);
-    fs.mkdirSync(workDir, { recursive: true });
-    const { entry, durations } = await replayScenario(scenario, {
-      runs: opts.runs,
-      timeoutMs: opts.timeoutMs,
-      workDir,
-    });
-    scenarios.push(entry);
-    allDurations.push(...durations);
-  }
-
-  const totals = scenarios.reduce((acc, sc) => ({
-    wakes: acc.wakes + sc.wakes,
-    eligible: acc.eligible + sc.eligible,
-    satisfied: acc.satisfied + sc.quality.satisfied,
-    qualityEligible: acc.qualityEligible + sc.quality.eligible,
-    forbiddenFailures: acc.forbiddenFailures + sc.quality.forbiddenFailures,
-  }), { wakes: 0, eligible: 0, satisfied: 0, qualityEligible: 0, forbiddenFailures: 0 });
-
-  const wallSummary = summarizeRuns(allDurations);
-  const summaryQuality = computeQualityScore({
-    satisfied: totals.satisfied,
-    eligible: totals.qualityEligible,
-    forbiddenFailures: totals.forbiddenFailures,
-  });
-
-  const report = {
-    version: 1,
-    env: collectEnv(),
-    corpus: { path: path.resolve(opts.corpus), version: corpus.version, scenarios: corpus.scenarios.length },
-    runs: opts.runs,
-    scenarios,
-    summary: {
-      externalWakeRate: computeExternalWakeRate({
-        externalWakes: totals.wakes,
-        eligibleTransitions: totals.eligible,
-      }),
-      qualityScore: summaryQuality.score,
-      qualityVerdict: summaryQuality.verdict,
-      p50: wallSummary.p50,
-      p95: wallSummary.p95,
-    },
-  };
-
-  const outPath = path.resolve(opts.out);
-  fs.mkdirSync(path.dirname(outPath), { recursive: true });
-  fs.writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`);
-
-  console.log(`[decision-runtime-baseline] report written: ${outPath}`);
-  console.log(`[decision-runtime-baseline] scenarios=${scenarios.length} runs=${opts.runs} `
-    + `externalWakeRate=${report.summary.externalWakeRate} quality=${report.summary.qualityScore}`);
-  process.exit(0);
-}
-
-main().catch((error) => {
-  console.error(`[decision-runtime-baseline] harness crash: ${error?.stack ?? error}`);
-  process.exit(2);
-});