@hmharness/evaluation 0.8.0

package/dist/evaluators.d.ts

@@ -0,0 +1,36 @@
+import type { Evaluator } from './types.ts';
+export interface TextAssertionInput {
+    output: string;
+    /** ALL substrings must appear (case-insensitive). */
+    expect?: string[];
+    /** Output must equal this (trimmed). */
+    expectExact?: string;
+    /** Output must match this regex. */
+    expectRegex?: string;
+    /** NONE of these may appear. */
+    expectNone?: string[];
+    /** At least ONE must appear. */
+    expectAny?: string[];
+}
+export declare const textAssertionEvaluator: Evaluator<TextAssertionInput>;
+export interface CommandInput {
+    /** Executable to run (NO shell string - argv array style only, per shellgate doctrine). */
+    command: string;
+    args?: string[];
+    cwd?: string;
+    timeoutMs?: number;
+    /** Substrings expected in stdout when the command is considered successful. */
+    expectOut?: string[];
+    /** Forbid these in stdout+stderr (failure markers). */
+    forbidOut?: string[];
+}
+export declare const commandEvaluator: Evaluator<CommandInput>;
+export interface LlmJudgeInput {
+    task: string;
+    output: string;
+    criteria: string[];
+    /** Injected provider call - tests substitute; production passes kernel chat(). */
+    call: (system: string, user: string) => Promise<string>;
+}
+export declare const llmJudgeEvaluator: Evaluator<LlmJudgeInput>;
+export declare const allEvaluators: Evaluator<never>[];

package/dist/evaluators.js

@@ -0,0 +1,125 @@
+/**
+ * @hmharness/evaluation - concrete evaluators (hard evidence first).
+ */
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { scoreFromEvidence } from "./types.js";
+const execCb = promisify(execFile);
+function textEvidence(input) {
+    const evidence = [];
+    const failures = [];
+    const out = input.output ?? '';
+    const lower = out.toLowerCase();
+    if (input.expectExact !== undefined) {
+        const ok = out.trim() === input.expectExact.trim();
+        evidence.push({ kind: 'static', detail: `exact match ${ok ? 'hit' : 'miss'}`, passed: ok });
+        if (!ok)
+            failures.push({ reason: `exact mismatch: got "${out.trim().slice(0, 80)}"` });
+    }
+    if (input.expectRegex) {
+        let ok = false;
+        try {
+            ok = new RegExp(input.expectRegex).test(out);
+        }
+        catch {
+            failures.push({ reason: `invalid expectRegex: ${input.expectRegex.slice(0, 60)}` });
+        }
+        evidence.push({ kind: 'static', detail: `regex ${ok ? 'matched' : 'no match'}`, passed: ok });
+        if (!ok)
+            failures.push({ reason: `regex did not match: ${input.expectRegex.slice(0, 60)}` });
+    }
+    if (input.expect?.length) {
+        const missing = input.expect.filter((e) => !lower.includes(e.toLowerCase()));
+        evidence.push({ kind: 'static', detail: `all-substrings ${input.expect.length - missing.length}/${input.expect.length}`, passed: missing.length === 0 });
+        if (missing.length)
+            failures.push({ reason: `missing substrings: ${missing.join(' && ').slice(0, 120)}` });
+    }
+    if (input.expectNone?.length) {
+        const leaked = input.expectNone.filter((e) => lower.includes(e.toLowerCase()));
+        evidence.push({ kind: 'static', detail: `forbidden-substrings clean: ${leaked.length === 0}`, passed: leaked.length === 0 });
+        if (leaked.length)
+            failures.push({ reason: `forbidden substrings present: ${leaked.join(' && ').slice(0, 120)}` });
+    }
+    if (input.expectAny?.length) {
+        const hit = input.expectAny.find((e) => lower.includes(e.toLowerCase()));
+        evidence.push({ kind: 'static', detail: `any-substring ${hit ? `hit: ${hit.slice(0, 40)}` : 'none'}`, passed: Boolean(hit) });
+        if (!hit)
+            failures.push({ reason: `none of the any-of substrings appeared` });
+    }
+    return { evidence, failures };
+}
+export const textAssertionEvaluator = {
+    id: 'text-assertion',
+    version: '1.0.0',
+    evidenceKind: 'static',
+    description: 'Structured text assertions: exact / regex / all-substrings / forbidden / any-of (the bench gate semantics, as an Evaluator).',
+    async evaluate(input) {
+        const t0 = Date.now();
+        const { evidence, failures } = textEvidence(input);
+        const { score, passed } = scoreFromEvidence(evidence, failures);
+        return { score, passed, evidence, failures, evaluatorId: this.id, evaluatorVersion: this.version, durationMs: Date.now() - t0 };
+    },
+};
+export const commandEvaluator = {
+    id: 'command-exit',
+    version: '1.0.0',
+    evidenceKind: 'build',
+    description: 'Run a command (execFile, no shell) and treat exit code + output as hard evidence - the build/test tier of the ladder.',
+    async evaluate(input) {
+        const t0 = Date.now();
+        const evidence = [];
+        const failures = [];
+        try {
+            const r = await execCb(input.command, input.args ?? [], { cwd: input.cwd, timeout: input.timeoutMs ?? 120_000, windowsHide: true, maxBuffer: 4 * 1024 * 1024 });
+            const out = String(r.stdout ?? '') + String(r.stderr ?? '');
+            evidence.push({ kind: 'build', detail: `exit 0 in ${Date.now() - t0}ms; output head: ${out.replace(/\s+/g, ' ').slice(0, 120)}`, passed: true });
+            if (input.expectOut?.length) {
+                const missing = input.expectOut.filter((e) => !out.toLowerCase().includes(e.toLowerCase()));
+                evidence.push({ kind: 'build', detail: `expected output ${input.expectOut.length - missing.length}/${input.expectOut.length}`, passed: missing.length === 0 });
+                if (missing.length)
+                    failures.push({ reason: `expected output missing: ${missing.join(' && ')}` });
+            }
+            if (input.forbidOut?.length) {
+                const leaked = input.forbidOut.filter((e) => out.toLowerCase().includes(e.toLowerCase()));
+                evidence.push({ kind: 'build', detail: `forbidden markers clean: ${leaked.length === 0}`, passed: leaked.length === 0 });
+                if (leaked.length)
+                    failures.push({ reason: `forbidden markers present: ${leaked.join(' && ')}` });
+            }
+        }
+        catch (err) {
+            const e = err;
+            const out = String(e.stdout ?? '') + String(e.stderr ?? '');
+            evidence.push({ kind: 'build', detail: `exit ${e.code ?? '?'}${e.killed ? ' (timed out)' : ''}; output head: ${out.replace(/\s+/g, ' ').slice(0, 120)}`, passed: false });
+            failures.push({ reason: `command failed with exit ${e.code ?? '?'}: ${out.replace(/\s+/g, ' ').slice(0, 140)}` });
+        }
+        const { score, passed } = scoreFromEvidence(evidence, failures);
+        return { score, passed, evidence, failures, evaluatorId: this.id, evaluatorVersion: this.version, durationMs: Date.now() - t0 };
+    },
+};
+export const llmJudgeEvaluator = {
+    id: 'llm-judge',
+    version: '1.0.0',
+    evidenceKind: 'llmJudge',
+    description: 'LLM judge - LAST resort on the evidence ladder. Its verdict alone can never mark a run fully passed (score hard-capped at 0.7).',
+    async evaluate(input) {
+        const t0 = Date.now();
+        const evidence = [];
+        const failures = [];
+        try {
+            const verdict = await input.call('You are an independent evaluator. Judge ONLY from the task, the output, and the criteria. Never trust the agent\'s self-assessment. Reply with exactly PASS or FAIL on the first line, then one short reason.', `Task: ${input.task.slice(0, 500)}\nCriteria:\n${input.criteria.map((c) => `- ${c}`).join('\n')}\nOutput:\n${input.output.slice(0, 4000)}`);
+            const pass = /^\s*PASS\b/i.test(verdict);
+            evidence.push({ kind: 'llmJudge', detail: verdict.replace(/\s+/g, ' ').slice(0, 160), passed: pass });
+            if (!pass)
+                failures.push({ reason: `judge: ${verdict.replace(/\s+/g, ' ').slice(0, 140)}` });
+        }
+        catch (err) {
+            failures.push({ reason: `judge call failed: ${String(err).slice(0, 120)}` });
+            evidence.push({ kind: 'llmJudge', detail: 'judge unavailable', passed: false });
+        }
+        const { score, passed } = scoreFromEvidence(evidence, failures);
+        // judge-only pass is capped by scoreFromEvidence (0.7) and can still be
+        // 'passed' for advisory purposes; promotion gates must combine with harder evidence
+        return { score, passed, evidence, failures, evaluatorId: this.id, evaluatorVersion: this.version, durationMs: Date.now() - t0 };
+    },
+};
+export const allEvaluators = [textAssertionEvaluator, commandEvaluator, llmJudgeEvaluator];

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from './types.ts';
+export * from './evaluators.ts';
+export * from './runner.ts';

package/dist/index.js

@@ -0,0 +1,3 @@
+export * from "./types.js";
+export * from "./evaluators.js";
+export * from "./runner.js";

package/dist/runner.d.ts

@@ -0,0 +1,35 @@
+/**
+ * @hmharness/evaluation - trajectory linkage + bench bridge (M2).
+ *
+ * evaluateRun(): judge a finished run from its TRAJECTORY RECORD - the
+ * outcome, tool completion ratio and error observations - never from the
+ * agent's word for itself. The verdict is appended back as judge.completed
+ * events on a linked judge-run, so every promotion decision is auditable.
+ * runBenchCase(): the existing bench gate semantics through the Evaluator
+ * contract, so evolution and evaluation share one assertion core.
+ */
+import { type Trajectory } from '@hmharness/observability';
+import { type TextAssertionInput, type EvaluationResult } from './index.ts';
+/** Score a run from its recorded trajectory metrics (hard evidence only). */
+export declare function evaluateTrajectory(trajectory: Trajectory): EvaluationResult;
+/** Evaluate a completed run from its trajectory and record the verdict back
+ *  onto a linked judge-run (part of the auditable record). */
+export declare function evaluateRun(home: string, runId: string, _assertions?: TextAssertionInput): Promise<EvaluationResult & {
+    runId: string;
+}>;
+/** Bench gate semantics through the Evaluator contract (shared assertion core). */
+export declare function runBenchCase(c: {
+    name: string;
+    prompt: string;
+    output: string;
+    expect?: string[];
+    expectExact?: string;
+    expectRegex?: string;
+    expectNone?: string[];
+    expectAny?: string[];
+}): Promise<{
+    name: string;
+    pass: boolean;
+    detail: string;
+    score: number;
+}>;

package/dist/runner.js

@@ -0,0 +1,67 @@
+/**
+ * @hmharness/evaluation - trajectory linkage + bench bridge (M2).
+ *
+ * evaluateRun(): judge a finished run from its TRAJECTORY RECORD - the
+ * outcome, tool completion ratio and error observations - never from the
+ * agent's word for itself. The verdict is appended back as judge.completed
+ * events on a linked judge-run, so every promotion decision is auditable.
+ * runBenchCase(): the existing bench gate semantics through the Evaluator
+ * contract, so evolution and evaluation share one assertion core.
+ */
+import { jsonlTrajectoryStore, createTrajectoryRecorder } from '@hmharness/observability';
+import { textAssertionEvaluator } from "./index.js";
+/** Score a run from its recorded trajectory metrics (hard evidence only). */
+export function evaluateTrajectory(trajectory) {
+    const t0 = Date.now();
+    const evidence = [];
+    const failures = [];
+    const outcome = trajectory.outcome;
+    if (outcome) {
+        evidence.push({ kind: 'runtime', detail: `outcome ${outcome.success ? 'success' : 'failed'} (${outcome.reason ?? '?'})`, passed: outcome.success });
+        if (!outcome.success)
+            failures.push({ reason: `run outcome: ${outcome.reason ?? 'failed'}${outcome.error ? ` - ${outcome.error}` : ''}` });
+    }
+    else {
+        failures.push({ reason: 'run has no outcome recorded (unfinished?)' });
+    }
+    const toolDone = trajectory.events.filter((e) => e.type === 'tool.completed');
+    const toolFailed = toolDone.filter((e) => e.payload?.isError === true);
+    if (toolDone.length > 0) {
+        const ratio = (toolDone.length - toolFailed.length) / toolDone.length;
+        evidence.push({ kind: 'runtime', detail: `tool completion ${toolDone.length - toolFailed.length}/${toolDone.length}`, passed: ratio >= 0.5 });
+        if (ratio < 0.5)
+            failures.push({ reason: `majority of tool calls failed (${toolFailed.length}/${toolDone.length})` });
+    }
+    const errors = trajectory.events.filter((e) => e.type === 'error.observed').length;
+    evidence.push({ kind: 'runtime', detail: `${errors} error observation(s)`, passed: errors === 0 });
+    const score = failures.length === 0 ? 1 : Math.max(0, 1 - failures.length * 0.34);
+    return {
+        score, passed: failures.length === 0, evidence, failures,
+        evaluatorId: 'trajectory-metrics', evaluatorVersion: '1.0.0', durationMs: Date.now() - t0,
+    };
+}
+/** Evaluate a completed run from its trajectory and record the verdict back
+ *  onto a linked judge-run (part of the auditable record). */
+export async function evaluateRun(home, runId, _assertions) {
+    const store = jsonlTrajectoryStore(home);
+    const trajectory = await store.getRun(runId);
+    const result = evaluateTrajectory(trajectory);
+    const rec = createTrajectoryRecorder(home, { task: `(judge) ${trajectory.task.slice(0, 80)}`, cwd: trajectory.cwd });
+    rec.emit('judge.started', 'judge', { subjectRunId: runId, evaluator: 'trajectory-metrics' });
+    rec.emit('judge.completed', 'judge', {
+        subjectRunId: runId, evaluator: 'trajectory-metrics',
+        passed: result.passed, score: result.score, failures: result.failures.map((f) => f.reason),
+    });
+    rec.finish({ success: result.passed, reason: 'final' });
+    return { ...result, runId };
+}
+/** Bench gate semantics through the Evaluator contract (shared assertion core). */
+export async function runBenchCase(c) {
+    void textAssertionEvaluator; // assertion core shared via evaluators.ts
+    const r = await textAssertionEvaluator.evaluate({
+        output: c.output,
+        expect: c.expect, expectExact: c.expectExact, expectRegex: c.expectRegex,
+        expectNone: c.expectNone, expectAny: c.expectAny,
+    });
+    return { name: c.name, pass: r.passed, detail: r.failures.map((f) => f.reason).join('; ') || 'ok', score: r.score };
+}

package/dist/types.d.ts

@@ -0,0 +1,55 @@
+/**
+ * @hmharness/evaluation - the Evaluator contract (V2 blueprint M2).
+ *
+ * Evidence outranks narrative. The evidence ladder (ADR-0001 rule 4):
+ * a build failure is FACT, an LLM opinion is a guess with good manners.
+ * Every evaluation carries machine-checkable evidence references so a
+ * promotion decision can be audited without re-running anything.
+ */
+/** Where a piece of evidence sits on the trust ladder - LOWER number = harder. */
+export declare const EVIDENCE_RANK: {
+    readonly build: 1;
+    readonly tests: 2;
+    readonly static: 3;
+    readonly runtime: 4;
+    readonly deviceLogs: 5;
+    readonly screenshot: 6;
+    readonly llmJudge: 7;
+    readonly selfReport: 8;
+};
+export type EvidenceKind = keyof typeof EVIDENCE_RANK;
+export interface Evidence {
+    kind: EvidenceKind;
+    /** What was observed, verbatim where possible (bounded). */
+    detail: string;
+    passed?: boolean;
+}
+export interface Failure {
+    reason: string;
+    evidence?: Evidence;
+}
+export interface EvaluationResult<E = Evidence> {
+    /** 0..1 */
+    score: number;
+    passed: boolean;
+    evidence: E[];
+    failures: Failure[];
+    evaluatorId: string;
+    evaluatorVersion: string;
+    durationMs: number;
+    /** The run this evaluation judged, when tied to a trajectory. */
+    runId?: string;
+}
+export interface Evaluator<Input = unknown> {
+    id: string;
+    version: string;
+    /** Where this evaluator's evidence sits on the trust ladder. */
+    evidenceKind: EvidenceKind;
+    description: string;
+    evaluate(input: Input): Promise<EvaluationResult>;
+}
+/** Shared scoring helper: all-or-nothing evidence -> 1 or 0; partial allowed. */
+export declare function scoreFromEvidence(evidence: Evidence[], failures: Failure[]): {
+    score: number;
+    passed: boolean;
+};

package/dist/types.js

@@ -0,0 +1,29 @@
+/**
+ * @hmharness/evaluation - the Evaluator contract (V2 blueprint M2).
+ *
+ * Evidence outranks narrative. The evidence ladder (ADR-0001 rule 4):
+ * a build failure is FACT, an LLM opinion is a guess with good manners.
+ * Every evaluation carries machine-checkable evidence references so a
+ * promotion decision can be audited without re-running anything.
+ */
+/** Where a piece of evidence sits on the trust ladder - LOWER number = harder. */
+export const EVIDENCE_RANK = {
+    build: 1,
+    tests: 2,
+    static: 3,
+    runtime: 4,
+    deviceLogs: 5,
+    screenshot: 6,
+    llmJudge: 7,
+    selfReport: 8,
+};
+/** Shared scoring helper: all-or-nothing evidence -> 1 or 0; partial allowed. */
+export function scoreFromEvidence(evidence, failures) {
+    const ranked = [...evidence].sort((a, b) => EVIDENCE_RANK[a.kind] - EVIDENCE_RANK[b.kind]);
+    const passing = ranked.filter((e) => e.passed !== false).length;
+    // an llmJudge/selfReport-only pass is never a full pass (evidence-first rule)
+    const weakestPass = ranked.length > 0 ? EVIDENCE_RANK[ranked[ranked.length - 1].kind] : EVIDENCE_RANK.selfReport;
+    const hardCap = weakestPass >= EVIDENCE_RANK.llmJudge ? 0.7 : 1;
+    const ratio = ranked.length > 0 ? passing / ranked.length : 0;
+    return { score: Math.min(hardCap, ratio), passed: failures.length === 0 && ratio === 1 };
+}

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@hmharness/evaluation",
+  "version": "0.8.0",
+  "description": "hmharness evaluation: the Evaluator/Judge contract (V2 blueprint M2). Hard evidence outranks LLM judgment - build results, exit codes, exact/regex assertions first; the LLM judge is a last resort and is labeled as such. Evaluations attach to trajectories (judge.completed events).",
+  "type": "module",
+  "main": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json"
+  },
+  "dependencies": {
+    "@hmharness/kernel": "0.8.0",
+    "@hmharness/observability": "0.7.0"
+  },
+  "files": [
+    "dist"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/swsgbl/hmharness.git"
+  },
+  "engines": {
+    "node": ">=22"
+  }
+}