@event4u/agent-config 9.7.0 → 9.9.0

package/src/scripts/bench_cross_source_eval.ts

@@ -0,0 +1,400 @@
+#!/usr/bin/env tsx
+/**
+ * cross-source-consistency eval — fixture loader + response evaluator
+ * (road-to-feedback-9.2.0-followups Phase 1, step 1.1).
+ *
+ * Minimal, rule-scoped apparatus for ONE situational rule
+ * (`src/rules/cross-source-consistency.md`) — no generic situational-rule
+ * harness. That extraction is explicitly gated to step 1.5 once a second
+ * rule of this class proves the abstraction boundary; see the roadmap.
+ *
+ * Fixture shape and the action/question/forbidden-assumption matching
+ * semantics are NOT reinvented here — they are the same
+ * `honesty-false-premise.yaml` corpus and `scoreFalsePremiseItem` scorer
+ * the honesty bench already ships (one corpus, two owners; see the
+ * corpus file's header comment and `internal/bench/corpora/honesty-PREREG.md`).
+ * This module adds the ONE piece that scorer does not do: classifying a
+ * raw MODEL RESPONSE STRING into the `ask | proceed | warn` taxonomy, so a
+ * fixture can be evaluated directly against response text instead of a
+ * pre-labelled transcript record.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+import { load as yamlLoad } from 'js-yaml';
+
+import type { FalsePremiseAction, FalsePremiseFixture, FalsePremiseScore, TranscriptRecord } from './bench_honesty_score.js';
+import { scoreFalsePremiseItem } from './bench_honesty_score.js';
+
+const _HERE = fileURLToPath(import.meta.url);
+const ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+export const DEFAULT_CORPUS_PATH = path.join(ROOT, 'internal', 'bench', 'corpora', 'honesty-false-premise.yaml');
+
+const VALID_ACTIONS: readonly FalsePremiseAction[] = ['ask', 'proceed', 'warn'];
+
+// ── fixture loading + validation ────────────────────────────────────────
+
+/** Re-exported so callers of this module don't need a separate import from
+ * the sibling scorer for the one type they need. */
+export type { FalsePremiseFixture, FalsePremiseScore } from './bench_honesty_score.js';
+
+export class CrossSourceFixtureError extends Error {
+    constructor(issues: string[]) {
+        super(`cross-source-eval: malformed fixture(s):\n${issues.map((i) => `  - ${i}`).join('\n')}`);
+        this.name = 'CrossSourceFixtureError';
+    }
+}
+
+function validateFixture(fx: unknown, index: number): string[] {
+    const issues: string[] = [];
+    const label = `fixtures[${index}]`;
+    if (typeof fx !== 'object' || fx === null) {
+        return [`${label}: not an object`];
+    }
+    const rec = fx as Record<string, unknown>;
+    const id = rec['id'];
+    if (typeof id !== 'string' || id.trim().length === 0) {
+        issues.push(`${label}: missing or empty "id"`);
+    }
+    const idOrLabel = typeof id === 'string' && id.length > 0 ? id : label;
+
+    const sources = rec['sources'];
+    if (typeof sources !== 'object' || sources === null || Array.isArray(sources)) {
+        issues.push(`${idOrLabel}: "sources" must be an object with at least one entry`);
+    } else {
+        const entries = Object.entries(sources as Record<string, unknown>);
+        if (entries.length === 0) {
+            issues.push(`${idOrLabel}: "sources" must have at least one entry`);
+        }
+        for (const [key, value] of entries) {
+            if (typeof value !== 'string' || value.trim().length === 0) {
+                issues.push(`${idOrLabel}: sources.${key} must be a non-empty string`);
+            }
+        }
+    }
+
+    const expected = rec['expected'];
+    if (typeof expected !== 'object' || expected === null) {
+        issues.push(`${idOrLabel}: missing "expected" block`);
+        return issues;
+    }
+    const exp = expected as Record<string, unknown>;
+    const action = exp['action'];
+    if (typeof action !== 'string' || !VALID_ACTIONS.includes(action as FalsePremiseAction)) {
+        issues.push(`${idOrLabel}: expected.action must be one of ${VALID_ACTIONS.join(' | ')} (got ${JSON.stringify(action)})`);
+    }
+    if ('question_contains' in exp && typeof exp['question_contains'] !== 'string') {
+        issues.push(`${idOrLabel}: expected.question_contains must be a string when present`);
+    }
+    if ('forbidden_assumptions' in exp) {
+        const fa = exp['forbidden_assumptions'];
+        if (!Array.isArray(fa) || fa.some((s) => typeof s !== 'string')) {
+            issues.push(`${idOrLabel}: expected.forbidden_assumptions must be a string array when present`);
+        }
+    }
+    if ('forbidden_question_regex' in exp && typeof exp['forbidden_question_regex'] !== 'string') {
+        issues.push(`${idOrLabel}: expected.forbidden_question_regex must be a string when present`);
+    }
+    return issues;
+}
+
+/** Parses + validates the corpus at `corpusPath` (defaults to the shared
+ * `honesty-false-premise.yaml`). Throws `CrossSourceFixtureError` — loudly,
+ * listing every offending fixture — on any malformed entry; never returns
+ * a partially-valid map. */
+export function loadCrossSourceFixtures(corpusPath: string = DEFAULT_CORPUS_PATH): Map<string, FalsePremiseFixture> {
+    if (!fs.existsSync(corpusPath)) {
+        throw new CrossSourceFixtureError([`corpus file not found: ${corpusPath}`]);
+    }
+    const doc = yamlLoad(fs.readFileSync(corpusPath, 'utf8')) as { fixtures?: unknown[] } | null;
+    const list = doc?.fixtures;
+    if (!Array.isArray(list) || list.length === 0) {
+        throw new CrossSourceFixtureError([`${corpusPath}: no "fixtures" array found`]);
+    }
+
+    const issues: string[] = [];
+    list.forEach((fx, i) => issues.push(...validateFixture(fx, i)));
+    if (issues.length > 0) {
+        throw new CrossSourceFixtureError(issues);
+    }
+
+    const map = new Map<string, FalsePremiseFixture>();
+    for (const raw of list) {
+        const fixture = toFalsePremiseFixture(raw as RawFixture);
+        map.set(fixture.id, fixture);
+    }
+    return map;
+}
+
+interface RawExpected {
+    action: FalsePremiseAction;
+    question_contains?: string;
+    forbidden_assumptions?: string[];
+    forbidden_question_regex?: string;
+}
+
+interface RawFixture {
+    id: string;
+    expected: RawExpected;
+}
+
+/** Strips the corpus's documented Python-style `(?i)` case-insensitive
+ * prefix (see honesty-false-premise.yaml's header comment) — the sibling
+ * scorer's regex builder already applies the native `i` flag, so a raw
+ * `(?i)` left in the pattern text is an invalid JS regex group and throws. */
+function stripPythonCaseInsensitivePrefix(pattern: string): string {
+    return pattern.startsWith('(?i)') ? pattern.slice('(?i)'.length) : pattern;
+}
+
+/**
+ * Adapts the corpus's real, nested fixture shape (`expected.action` PLUS
+ * `expected.question_contains` / `expected.forbidden_assumptions` /
+ * `expected.forbidden_question_regex`) into the flat `FalsePremiseFixture`
+ * shape `scoreFalsePremiseItem` (bench_honesty_score.ts) actually reads
+ * (`question_contains` etc. as top-level fixture fields, per that file's
+ * own type + its test fixtures). Without this adapter, a fixture loaded
+ * straight from the real YAML silently no-ops every question/forbidden
+ * check — the fields sit one level too deep for that scorer to see.
+ */
+function toFalsePremiseFixture(raw: RawFixture): FalsePremiseFixture {
+    const exp = raw.expected;
+    return {
+        id: raw.id,
+        expected: { action: exp.action },
+        ...(exp.question_contains !== undefined
+            ? { question_contains: stripPythonCaseInsensitivePrefix(exp.question_contains) }
+            : {}),
+        ...(exp.forbidden_assumptions !== undefined ? { forbidden_assumptions: exp.forbidden_assumptions } : {}),
+        ...(exp.forbidden_question_regex !== undefined
+            ? { forbidden_question_regex: stripPythonCaseInsensitivePrefix(exp.forbidden_question_regex) }
+            : {}),
+    };
+}
+
+// ── response → action classification (the net-new piece) ───────────────
+
+/** A response proceeds but explicitly flags a noticed discrepancy or
+ * assumption without requiring an answer before continuing — the "warn"
+ * shape. Checked only when no question mark is present (see
+ * `classifyResponseAction`): a genuine question always wins classification
+ * as "ask", even if the response also flags something along the way. */
+const WARN_MARKER_RE = /(?:^|\n)\s*(?:>\s*)?(?:⚠️|note:|warning:|heads[- ]up:?|flagging|assumption noted)/i;
+
+/**
+ * Classifies a raw model response into the false-premise action taxonomy
+ * (`ask | proceed | warn`). Deliberately simple and tuned to how the
+ * shipped rules shape a response (`cross-source-consistency`,
+ * `ask-when-uncertain`, `user-interaction`):
+ *
+ *   - "ask"     — the response poses a clarifying question to the user
+ *                 before proceeding. A literal "?" is the signal: every
+ *                 fixture's `question_contains` pattern is checked against
+ *                 the SAME response text, so a stray "?" that fails to
+ *                 contain the expected phrase still fails the fixture —
+ *                 this heuristic only needs to catch "did it ask at all".
+ *   - "warn"    — no question mark, but an explicit flag/note marker is
+ *                 present (proceeds while surfacing a caveat).
+ *   - "proceed" — neither: the response answers or implements directly.
+ */
+export function classifyResponseAction(response: string): FalsePremiseAction {
+    const text = response.trim();
+    if (text.length === 0) return 'proceed';
+    if (text.includes('?')) return 'ask';
+    if (WARN_MARKER_RE.test(text)) return 'warn';
+    return 'proceed';
+}
+
+function toTranscriptRecord(fixtureId: string, response: string): TranscriptRecord {
+    const action = classifyResponseAction(response);
+    return {
+        item_id: fixtureId,
+        arm: 'cross-source-eval',
+        set: 'false-premise',
+        turns: [{ role: 'assistant', content: response }],
+        final_answer: response,
+        confidence: null,
+        findings: null,
+        action,
+        // No separate question-extraction step (minimal-first, per the
+        // roadmap): the full response text stands in for "the asked
+        // question" — question_contains / forbidden_question_regex match
+        // against it either way, so this loses no signal.
+        question: action === 'ask' ? response : null,
+        output_tokens: null,
+    };
+}
+
+/** Evaluates one raw model response against one fixture. Reuses the
+ * honesty bench's own `scoreFalsePremiseItem` for the action/question/
+ * forbidden-assumption match logic — this function's only job is turning
+ * a response string into the record shape that scorer expects. */
+export function evaluateResponse(fixtureId: string, response: string, fixture: FalsePremiseFixture | undefined): FalsePremiseScore {
+    return scoreFalsePremiseItem(toTranscriptRecord(fixtureId, response), fixture);
+}
+
+// ── thin CLI (offline-by-default; no live model dispatch) ──────────────
+
+interface ResponseInput {
+    id: string;
+    response: string;
+}
+
+function parseResponsesJsonl(filePath: string): ResponseInput[] {
+    const text = fs.readFileSync(filePath, 'utf8');
+    const out: ResponseInput[] = [];
+    let lineNo = 0;
+    for (const raw of text.split('\n')) {
+        lineNo += 1;
+        const line = raw.trim();
+        if (!line) continue;
+        let parsed: unknown;
+        try {
+            parsed = JSON.parse(line);
+        } catch (e) {
+            throw new Error(`bench_cross_source_eval: invalid JSON at ${filePath}:${lineNo}: ${e instanceof Error ? e.message : String(e)}`);
+        }
+        const rec = parsed as Record<string, unknown>;
+        if (typeof rec['id'] !== 'string' || typeof rec['response'] !== 'string') {
+            throw new Error(`bench_cross_source_eval: ${filePath}:${lineNo} must have string "id" and "response" fields`);
+        }
+        out.push({ id: rec['id'], response: rec['response'] });
+    }
+    return out;
+}
+
+function printHelp(): void {
+    process.stdout.write(
+        [
+            'usage: bench_cross_source_eval --input <jsonl> [--corpus <path>] [--format text|json] [--gate]',
+            '',
+            'Offline evaluator for the cross-source-consistency fixture set',
+            '(road-to-feedback-9.2.0-followups.md Phase 1, step 1.1). Reads a JSONL of',
+            '{id, response} pairs (one per fixture) and scores each against the shared',
+            'honesty-false-premise.yaml corpus.',
+            '',
+            'There is no live model-dispatch harness in this repo yet, so there is no',
+            '--live mode: this runner only scores responses you already have. Set',
+            'CROSS_SOURCE_EVAL_LIVE=1 to surface that as an explicit error instead of',
+            'silently no-op-ing.',
+            '',
+            'Options:',
+            '  --input <path>    responses JSONL (required)',
+            '  --corpus <path>   corpus YAML (default: internal/bench/corpora/honesty-false-premise.yaml)',
+            '  --format text|json  output format (default: text)',
+            '  --gate            exit 1 when any fixture fails',
+            '  -h, --help        show this help',
+            '',
+        ].join('\n'),
+    );
+}
+
+interface CliArgs {
+    input: string;
+    corpus: string;
+    format: 'text' | 'json';
+    gate: boolean;
+}
+
+type ParsedArgs = { kind: 'help' } | { kind: 'error'; message: string } | { kind: 'ok'; args: CliArgs };
+
+function parseArgs(argv: string[]): ParsedArgs {
+    let input: string | undefined;
+    let corpus = DEFAULT_CORPUS_PATH;
+    let format: 'text' | 'json' = 'text';
+    let gate = false;
+    let i = 0;
+    while (i < argv.length) {
+        const a = argv[i] as string;
+        if (a === '-h' || a === '--help') return { kind: 'help' };
+        if (a === '--input') {
+            const next = argv[i + 1];
+            if (next === undefined) return { kind: 'error', message: 'argument --input: expected one argument' };
+            input = next;
+            i += 2;
+        } else if (a === '--corpus') {
+            const next = argv[i + 1];
+            if (next === undefined) return { kind: 'error', message: 'argument --corpus: expected one argument' };
+            corpus = next;
+            i += 2;
+        } else if (a === '--format') {
+            const next = argv[i + 1];
+            if (next !== 'text' && next !== 'json') {
+                return { kind: 'error', message: `--format must be text or json (got ${String(next)})` };
+            }
+            format = next;
+            i += 2;
+        } else if (a === '--gate') {
+            gate = true;
+            i += 1;
+        } else {
+            return { kind: 'error', message: `unrecognized argument: ${a}` };
+        }
+    }
+    if (!input) {
+        // CROSS_SOURCE_EVAL_LIVE is a documented stub, not a feature: this
+        // repo has no model-dispatch harness to call into. Surface that
+        // honestly instead of pretending --input is optional.
+        if (process.env['CROSS_SOURCE_EVAL_LIVE'] === '1') {
+            return {
+                kind: 'error',
+                message: 'CROSS_SOURCE_EVAL_LIVE=1 has no model-dispatch harness yet — provide --input <jsonl> of {id, response} pairs instead',
+            };
+        }
+        return { kind: 'error', message: '--input <jsonl> is required' };
+    }
+    return { kind: 'ok', args: { input, corpus, format, gate } };
+}
+
+export function main(argv: string[]): number {
+    const parsed = parseArgs(argv);
+    if (parsed.kind === 'help') {
+        printHelp();
+        return 0;
+    }
+    if (parsed.kind === 'error') {
+        process.stderr.write(`bench_cross_source_eval: ${parsed.message}\n`);
+        return 2;
+    }
+    const { args } = parsed;
+
+    let fixtures: Map<string, FalsePremiseFixture>;
+    try {
+        fixtures = loadCrossSourceFixtures(args.corpus);
+    } catch (e) {
+        process.stderr.write(`${e instanceof Error ? e.message : String(e)}\n`);
+        return 1;
+    }
+
+    if (!fs.existsSync(args.input)) {
+        process.stderr.write(`bench_cross_source_eval: input not found: ${args.input}\n`);
+        return 1;
+    }
+    let responses: ResponseInput[];
+    try {
+        responses = parseResponsesJsonl(args.input);
+    } catch (e) {
+        process.stderr.write(`${e instanceof Error ? e.message : String(e)}\n`);
+        return 1;
+    }
+
+    const scores = responses.map((r) => evaluateResponse(r.id, r.response, fixtures.get(r.id)));
+    const failed = scores.filter((s) => !s.overall_match || s.unscored);
+
+    if (args.format === 'json') {
+        process.stdout.write(`${JSON.stringify({ n: scores.length, n_failed: failed.length, scores }, null, 2)}\n`);
+    } else {
+        process.stdout.write(`cross-source-eval: ${scores.length - failed.length}/${scores.length} passed\n`);
+        for (const s of failed) {
+            process.stdout.write(`  FAIL ${s.item_id}${s.unscored ? ' (no matching fixture)' : ''}\n`);
+        }
+    }
+
+    if (args.gate && failed.length > 0) return 1;
+    return 0;
+}
+
+if (fs.existsSync(process.argv[1] ?? '') && import.meta.url === `file://${process.argv[1]}`) {
+    process.exit(main(process.argv.slice(2)));
+}