akm-cli 0.6.1 → 0.7.0-rc1

package/dist/tests/bench/workflow-evaluator.js

@@ -0,0 +1,557 @@
+/**
+ * akm-bench workflow compliance evaluator (issue #256).
+ *
+ * Consumes a normalized `WorkflowTraceResult` (from `./workflow-trace`, issue
+ * #254) plus a declarative `WorkflowSpec` (from `./workflow-spec`, issue #255)
+ * and produces one `WorkflowCheckResult` per (run, spec) describing how well
+ * the run satisfied the spec.
+ *
+ * Design rules:
+ *   - Pure: never mutates the input trace, spec, run, or task metadata, and
+ *     never throws on malformed input. Bad input yields a `harness_error`
+ *     status with a structured violation.
+ *   - Bounded: caps total emitted violations per (run, spec) at
+ *     `MAX_VIOLATIONS_PER_CHECK` (32) so a pathological run cannot OOM
+ *     downstream reporters.
+ *   - Stable contract: `schemaVersion: 1` on the result envelope. Only
+ *     additive changes to violation codes / new optional fields are allowed
+ *     without a major bump.
+ *
+ * Spec semantics (from #255 fixtures):
+ *   - `required_sequence[i].event` — event must appear at least once.
+ *   - `required_sequence[i].before = X` — first occurrence of `event` must
+ *     precede first occurrence of `X`.
+ *   - `required_sequence[i].after  = X` — first occurrence of `event` must
+ *     follow first occurrence of `X` (if `X` is absent, the constraint is
+ *     vacuous; the `after`-event's required-event check captures missing-X).
+ *   - `required_sequence[i].min_count = N` — event count must be ≥ N.
+ *   - `required_sequence[i].polarity` — at least one matching feedback event
+ *     must carry that polarity (resolved from `metadata.polarity` on the
+ *     trace event when present, else inferred from feedback args).
+ *   - `required_sequence[i].ref_must_equal = "gold_ref"` — the matching
+ *     event's `assetRef` must equal `taskMetadata.gold_ref`.
+ *   - `required_sequence[i].required_if = <flag>` — step is only required
+ *     when `taskMetadata.flags[<flag>]` is true. Unknown flags default to
+ *     `false`, i.e. the step is treated as optional.
+ *   - `forbidden[i].event` — event must NOT appear.
+ *   - `forbidden[i].before = X` — event must NOT appear before X.
+ *   - `forbidden[i].after  = X` — event must NOT appear after X.
+ *   - `forbidden[i].polarity` — feedback event with that polarity must NOT
+ *     appear.
+ *
+ * Verifier-aware checks (acceptance criteria):
+ *   - `feedback polarity` — when `verifierFailed = true`, any recorded
+ *     `akm_feedback` must include a negative-polarity event; emitting only
+ *     positive feedback raises `wrong_feedback_polarity`.
+ *   - `gold_ref` — when `taskMetadata.goldRef` is set, the trace must contain
+ *     an `akm_show` whose `assetRef` matches `goldRef`. Loading other refs
+ *     instead raises `irrelevant_asset_loaded`.
+ */
+import { specApplies } from "./workflow-spec";
+/* ─── Caps (documented contract) ──────────────────────────────────────────── */
+/** Hard cap on emitted violations per (run, spec). Prevents runaway reports. */
+export const MAX_VIOLATIONS_PER_CHECK = 32;
+/* ─── Top-level entry points ──────────────────────────────────────────────── */
+/**
+ * Evaluate one run-trace against one spec. Pure; never throws; never mutates
+ * inputs.
+ *
+ * Callers who want applicability filtering should use
+ * `evaluateRunAgainstAllSpecs` — this entry point evaluates unconditionally
+ * and assumes the caller already decided the spec applies. (It still returns
+ * `harness_error` if inputs are obviously malformed.)
+ */
+export function evaluateRunAgainstSpec(trace, spec, run, task = {}) {
+    // Harness validation: malformed inputs short-circuit with a structured violation.
+    const harnessError = validateInputs(trace, spec, run);
+    if (harnessError)
+        return harnessError;
+    // Past validateInputs we know trace & spec are well-formed.
+    const t = trace;
+    const s = spec;
+    const events = Array.isArray(t.events) ? t.events.filter(isUsableEvent) : [];
+    const violations = [];
+    const flags = task.flags ?? {};
+    // ── Required-sequence checks ──────────────────────────────────────────────
+    let requiredTotal = 0;
+    let requiredPassed = 0;
+    for (const step of s.required_sequence) {
+        if (step.required_if && !flags[step.required_if]) {
+            // Conditional step skipped.
+            continue;
+        }
+        requiredTotal += 1;
+        const ok = checkRequiredStep(step, events, task, violations);
+        if (ok)
+            requiredPassed += 1;
+    }
+    // ── Forbidden checks ──────────────────────────────────────────────────────
+    let forbiddenTotal = 0;
+    let forbiddenPassed = 0;
+    for (const step of s.forbidden ?? []) {
+        forbiddenTotal += 1;
+        const ok = checkForbiddenStep(step, events, violations);
+        if (ok)
+            forbiddenPassed += 1;
+    }
+    // ── Verifier-aware feedback polarity check ────────────────────────────────
+    const verifierFailed = run.verifierFailed ?? run.outcome === "fail";
+    const feedbackEvents = events.filter((e) => e.type === "akm_feedback");
+    const polarityCheckOk = checkVerifierPolarity(verifierFailed, feedbackEvents, violations);
+    // ── Gold-ref check (when task has goldRef and spec expects gold loading) ──
+    const goldAssetLoaded = checkGoldRef(task.goldRef, events, s, violations);
+    // ── Score & status ────────────────────────────────────────────────────────
+    const w = s.scoring;
+    const requiredFraction = requiredTotal === 0 ? 1 : requiredPassed / requiredTotal;
+    const forbiddenFraction = forbiddenTotal === 0 ? 1 : forbiddenPassed / forbiddenTotal;
+    // Evidence quality: feedback-polarity match + gold-ref match + verifier_run present.
+    const evidenceSignals = [polarityCheckOk ? 1 : 0];
+    if (task.goldRef !== undefined)
+        evidenceSignals.push(goldAssetLoaded ? 1 : 0);
+    if (events.some((e) => e.type === "verifier_run"))
+        evidenceSignals.push(1);
+    else
+        evidenceSignals.push(0);
+    const evidenceFraction = evidenceSignals.length === 0 ? 1 : average(evidenceSignals);
+    let score = w.required_steps_weight * requiredFraction +
+        w.forbidden_steps_weight * forbiddenFraction +
+        w.evidence_quality_weight * evidenceFraction;
+    score = clampUnit(score);
+    // Cap violations after computing score so the score reflects ALL findings.
+    const cappedViolations = capViolations(violations);
+    let status;
+    if (requiredTotal === 0) {
+        // No required steps applied (e.g. all required_if guards were false).
+        // Treat as pass when no violations were found, partial otherwise.
+        status = cappedViolations.length === 0 ? "pass" : "partial";
+    }
+    else if (requiredPassed === requiredTotal && cappedViolations.length === 0) {
+        status = "pass";
+    }
+    else if (requiredPassed === 0) {
+        status = "fail";
+    }
+    else {
+        status = "partial";
+    }
+    const evidence = {
+        matchedEvents: countMatchingEvents(events, s),
+        feedbackRecorded: feedbackEvents.length > 0,
+        goldAssetLoaded,
+        traceTruncated: t.truncated === true,
+    };
+    return {
+        schemaVersion: 1,
+        workflowId: s.id,
+        taskId: run.taskId,
+        arm: run.arm,
+        seed: run.seed,
+        ...(typeof run.outcome === "string" ? { taskOutcome: run.outcome } : {}),
+        status,
+        score,
+        requiredPassed,
+        requiredTotal,
+        violations: cappedViolations,
+        evidence,
+    };
+}
+/**
+ * Evaluate one trace against many specs, applying `applies_to` filters. Specs
+ * whose filter excludes the run produce a `not_applicable` result with empty
+ * violations and `score = 0`; this preserves a stable 1-result-per-spec shape
+ * for downstream reporters.
+ */
+export function evaluateRunAgainstAllSpecs(trace, specs, run, task = {}) {
+    if (!Array.isArray(specs))
+        return [];
+    const out = [];
+    for (const spec of specs) {
+        if (!spec || typeof spec !== "object")
+            continue;
+        const applicable = specApplies(spec, {
+            arm: run.arm,
+            taskId: run.taskId,
+            outcome: run.outcome,
+            hasGoldRef: task.goldRef !== undefined,
+            repeatedFailures: run.repeatedFailures,
+        });
+        if (!applicable) {
+            out.push(makeNotApplicable(spec, run));
+            continue;
+        }
+        out.push(evaluateRunAgainstSpec(trace, spec, run, task));
+    }
+    return out;
+}
+/* ─── Required-step check ─────────────────────────────────────────────────── */
+function checkRequiredStep(step, events, task, violations) {
+    const matchingForStep = events.filter((e) => e.type === step.event);
+    // 1) Polarity-aware filter: a polarity-tagged step only counts feedback
+    //    events with the right polarity.
+    const matching = step.polarity !== undefined ? matchingForStep.filter((e) => eventPolarity(e) === step.polarity) : matchingForStep;
+    // 2) Presence + min_count.
+    const minCount = step.min_count ?? 1;
+    if (matching.length < minCount) {
+        pushViolation(violations, {
+            code: "missing_required_event",
+            message: step.polarity !== undefined
+                ? `expected ≥${minCount} ${step.event} event(s) with polarity=${step.polarity}, observed ${matching.length}`
+                : `expected ≥${minCount} ${step.event} event(s), observed ${matching.length}`,
+            expected: step.polarity ? `${step.event}(${step.polarity}) x${minCount}` : `${step.event} x${minCount}`,
+            observed: `${matching.length}`,
+        });
+        return false;
+    }
+    // 3) ref_must_equal — match a matching event to the task field.
+    if (step.ref_must_equal !== undefined) {
+        const wantedRef = resolveRefField(step.ref_must_equal, task);
+        if (wantedRef === undefined) {
+            pushViolation(violations, {
+                code: "missing_evidence",
+                message: `step expects ref_must_equal=${step.ref_must_equal} but task does not declare it`,
+                expected: step.ref_must_equal,
+            });
+            return false;
+        }
+        const refMatch = matching.find((e) => e.assetRef === wantedRef);
+        if (!refMatch) {
+            const observedRef = matching.find((e) => typeof e.assetRef === "string")?.assetRef;
+            pushViolation(violations, {
+                code: "irrelevant_asset_loaded",
+                message: `${step.event} did not load ${step.ref_must_equal}=${wantedRef}`,
+                expected: wantedRef,
+                observed: observedRef,
+            });
+            return false;
+        }
+    }
+    // 4) before/after order checks. We compare FIRST occurrence (by event id).
+    const firstThis = firstId(matching);
+    if (step.before !== undefined && firstThis !== undefined) {
+        const firstOther = firstId(events.filter((e) => e.type === step.before));
+        if (firstOther !== undefined && firstThis >= firstOther) {
+            pushViolation(violations, {
+                code: "wrong_order",
+                message: `expected ${step.event} before ${step.before}, but ${step.before} occurred first`,
+                expected: `${step.event} < ${step.before}`,
+                observed: `${step.before} < ${step.event}`,
+            });
+            return false;
+        }
+    }
+    if (step.after !== undefined && firstThis !== undefined) {
+        const firstOther = firstId(events.filter((e) => e.type === step.after));
+        if (firstOther !== undefined && firstThis <= firstOther) {
+            pushViolation(violations, {
+                code: "wrong_order",
+                message: `expected ${step.event} after ${step.after}, but ${step.event} occurred first`,
+                expected: `${step.after} < ${step.event}`,
+                observed: `${step.event} < ${step.after}`,
+            });
+            return false;
+        }
+    }
+    return true;
+}
+/* ─── Forbidden-step check ────────────────────────────────────────────────── */
+function checkForbiddenStep(step, events, violations) {
+    const matchingForType = events.filter((e) => e.type === step.event);
+    const matching = step.polarity !== undefined ? matchingForType.filter((e) => eventPolarity(e) === step.polarity) : matchingForType;
+    if (matching.length === 0)
+        return true;
+    // Pure forbidden (no before/after): event must be absent.
+    if (step.before === undefined && step.after === undefined) {
+        pushViolation(violations, {
+            code: "forbidden_event",
+            message: step.polarity !== undefined
+                ? `forbidden event ${step.event}(polarity=${step.polarity}) appeared`
+                : `forbidden event ${step.event} appeared`,
+            expected: "absent",
+            observed: step.polarity ? `${step.event}(${step.polarity})` : step.event,
+        });
+        return false;
+    }
+    // Conditional forbidden: forbidden when ordered relative to a guard.
+    if (step.before !== undefined) {
+        const firstGuard = firstId(events.filter((e) => e.type === step.before));
+        const firstThis = firstId(matching);
+        if (firstGuard === undefined) {
+            // Guard absent — `before X` cannot be violated.
+            return true;
+        }
+        if (firstThis !== undefined && firstThis < firstGuard) {
+            pushViolation(violations, {
+                code: classifyForbiddenCode(step),
+                message: `${step.event} occurred before ${step.before}`,
+                expected: `${step.event} after ${step.before}`,
+                observed: `${step.event} before ${step.before}`,
+            });
+            return false;
+        }
+        return true;
+    }
+    if (step.after !== undefined) {
+        const firstGuard = firstId(events.filter((e) => e.type === step.after));
+        const firstThis = firstId(matching);
+        if (firstGuard === undefined)
+            return true;
+        if (firstThis !== undefined && firstThis > firstGuard) {
+            pushViolation(violations, {
+                code: classifyForbiddenCode(step),
+                message: `${step.event} occurred after ${step.after}`,
+                expected: `${step.event} before ${step.after}`,
+                observed: `${step.event} after ${step.after}`,
+            });
+            return false;
+        }
+        return true;
+    }
+    return true;
+}
+/** Pick a more specific violation code for ordering-flavoured forbidden rules. */
+function classifyForbiddenCode(step) {
+    // Domain-specific specialisations for diagnostic clarity.
+    if (step.event === "akm_reflect" && step.before === "akm_feedback") {
+        return "reflection_without_failure";
+    }
+    if (step.event === "akm_proposal_accept") {
+        // Accepting a proposal before propose/verifier/test = unvalidated acceptance.
+        return "proposal_accepted_without_validation";
+    }
+    if (step.event === "akm_feedback" && step.before === "verifier_run") {
+        return "late_feedback";
+    }
+    return "forbidden_event";
+}
+/* ─── Verifier-aware feedback polarity ────────────────────────────────────── */
+/**
+ * When the verifier failed, any recorded `akm_feedback` must include at least
+ * one negative-polarity event. Mirror: when the verifier passed, recorded
+ * feedback should include a positive-polarity event. Returns true when the
+ * polarity matches (or there is no feedback at all — that's the
+ * `feedback-after-use` spec's job to flag).
+ */
+function checkVerifierPolarity(verifierFailed, feedbackEvents, violations) {
+    if (feedbackEvents.length === 0)
+        return true; // separate spec covers missing feedback.
+    if (verifierFailed) {
+        const hasNegative = feedbackEvents.some((e) => eventPolarity(e) === "negative");
+        if (!hasNegative) {
+            pushViolation(violations, {
+                code: "wrong_feedback_polarity",
+                message: "verifier failed but agent recorded no negative feedback",
+                expected: "negative",
+                observed: feedbackEvents.map((e) => eventPolarity(e) ?? "unknown").join(","),
+            });
+            return false;
+        }
+        return true;
+    }
+    // Verifier passed. Positive feedback expected when feedback was recorded.
+    const hasPositive = feedbackEvents.some((e) => eventPolarity(e) === "positive");
+    if (!hasPositive) {
+        pushViolation(violations, {
+            code: "wrong_feedback_polarity",
+            message: "verifier passed but agent recorded no positive feedback",
+            expected: "positive",
+            observed: feedbackEvents.map((e) => eventPolarity(e) ?? "unknown").join(","),
+        });
+        return false;
+    }
+    return true;
+}
+/* ─── Gold-ref check ──────────────────────────────────────────────────────── */
+function checkGoldRef(goldRef, events, spec, violations) {
+    if (goldRef === undefined)
+        return false; // not loaded, but also not required.
+    const showEvents = events.filter((e) => e.type === "akm_show");
+    const matched = showEvents.some((e) => e.assetRef === goldRef);
+    if (matched)
+        return true;
+    // Don't double-count if the spec already had a `ref_must_equal: gold_ref`
+    // step that flagged this — check by scanning existing violations.
+    const alreadyFlagged = violations.some((v) => v.code === "irrelevant_asset_loaded" && v.expected === goldRef);
+    if (alreadyFlagged)
+        return false;
+    // Only emit a top-level gold-ref violation when the spec at least mentions
+    // gold loading — e.g. has a step with `ref_must_equal` OR declares its own
+    // `gold_ref`. Otherwise this would noisy-fire on every unrelated spec.
+    const specCaresAboutGold = spec.gold_ref !== undefined || spec.required_sequence.some((step) => step.ref_must_equal !== undefined);
+    if (!specCaresAboutGold)
+        return false;
+    const observed = showEvents.find((e) => typeof e.assetRef === "string")?.assetRef;
+    pushViolation(violations, {
+        code: "irrelevant_asset_loaded",
+        message: `gold asset ${goldRef} was never loaded via akm_show`,
+        expected: goldRef,
+        observed,
+    });
+    return false;
+}
+/* ─── Helpers ─────────────────────────────────────────────────────────────── */
+function isUsableEvent(e) {
+    if (!e || typeof e !== "object")
+        return false;
+    const ev = e;
+    return typeof ev.type === "string" && typeof ev.id === "number";
+}
+function firstId(events) {
+    if (events.length === 0)
+        return undefined;
+    let min = events[0].id;
+    for (let i = 1; i < events.length; i += 1) {
+        if (events[i].id < min)
+            min = events[i].id;
+    }
+    return min;
+}
+/**
+ * Recover an event's feedback polarity. The trace contract does NOT carry a
+ * top-level `polarity` field, so we probe known shapes:
+ *   - args contain a token like `+1` / `-1` / `positive` / `negative`.
+ *   - command-style: `akm feedback +1 skill:foo`.
+ * Returns `undefined` when polarity cannot be determined.
+ */
+function eventPolarity(ev) {
+    // args-based detection.
+    if (Array.isArray(ev.args)) {
+        for (const a of ev.args) {
+            if (a === "+1" || a === "positive" || a === "+")
+                return "positive";
+            if (a === "-1" || a === "negative" || a === "-")
+                return "negative";
+        }
+    }
+    // command-string fallback (rare, but cheap).
+    if (typeof ev.command === "string") {
+        if (/(?:^|\s)(?:\+1|positive)(?:\s|$)/.test(ev.command))
+            return "positive";
+        if (/(?:^|\s)(?:-1|negative)(?:\s|$)/.test(ev.command))
+            return "negative";
+    }
+    return undefined;
+}
+function resolveRefField(field, task) {
+    if (field === "gold_ref")
+        return task.goldRef;
+    return undefined;
+}
+function pushViolation(out, v) {
+    if (out.length >= MAX_VIOLATIONS_PER_CHECK)
+        return;
+    out.push(v);
+}
+function capViolations(violations) {
+    if (violations.length <= MAX_VIOLATIONS_PER_CHECK)
+        return violations;
+    return violations.slice(0, MAX_VIOLATIONS_PER_CHECK);
+}
+function countMatchingEvents(events, spec) {
+    const vocab = new Set();
+    for (const step of spec.required_sequence) {
+        vocab.add(step.event);
+        if (step.before)
+            vocab.add(step.before);
+        if (step.after)
+            vocab.add(step.after);
+    }
+    for (const step of spec.forbidden ?? []) {
+        vocab.add(step.event);
+    }
+    let n = 0;
+    for (const ev of events)
+        if (vocab.has(ev.type))
+            n += 1;
+    return n;
+}
+function average(xs) {
+    let sum = 0;
+    for (const x of xs)
+        sum += x;
+    return sum / xs.length;
+}
+function clampUnit(x) {
+    if (!Number.isFinite(x))
+        return 0;
+    if (x < 0)
+        return 0;
+    if (x > 1)
+        return 1;
+    return x;
+}
+function makeNotApplicable(spec, run) {
+    return {
+        schemaVersion: 1,
+        workflowId: spec.id,
+        taskId: run.taskId,
+        arm: run.arm,
+        seed: run.seed,
+        ...(typeof run.outcome === "string" ? { taskOutcome: run.outcome } : {}),
+        status: "not_applicable",
+        score: 0,
+        requiredPassed: 0,
+        requiredTotal: 0,
+        violations: [],
+        evidence: {
+            matchedEvents: 0,
+            feedbackRecorded: false,
+            goldAssetLoaded: false,
+            traceTruncated: false,
+        },
+    };
+}
+function validateInputs(trace, spec, run) {
+    if (!run || typeof run !== "object" || typeof run.taskId !== "string" || typeof run.arm !== "string") {
+        return harnessError(spec, run, "run context missing arm/taskId");
+    }
+    if (!trace || typeof trace !== "object") {
+        return harnessError(spec, run, "trace is missing or not an object");
+    }
+    if (!Array.isArray(trace.events)) {
+        return harnessError(spec, run, "trace.events is not an array");
+    }
+    if (!spec || typeof spec !== "object") {
+        return harnessError(spec, run, "spec is missing or not an object");
+    }
+    const s = spec;
+    if (typeof s.id !== "string" || !Array.isArray(s.required_sequence) || !s.scoring) {
+        return harnessError(spec, run, "spec is malformed (missing id/required_sequence/scoring)");
+    }
+    return undefined;
+}
+function harnessError(spec, run, reason) {
+    const safeRun = run && typeof run === "object"
+        ? {
+            arm: typeof run.arm === "string" ? run.arm : "unknown",
+            taskId: typeof run.taskId === "string" ? run.taskId : "unknown",
+            seed: typeof run.seed === "number" ? run.seed : -1,
+        }
+        : { arm: "unknown", taskId: "unknown", seed: -1 };
+    return {
+        schemaVersion: 1,
+        workflowId: spec && typeof spec === "object" && typeof spec.id === "string"
+            ? spec.id
+            : "unknown",
+        taskId: safeRun.taskId,
+        arm: safeRun.arm,
+        seed: safeRun.seed,
+        ...(typeof run?.outcome === "string" ? { taskOutcome: run.outcome } : {}),
+        status: "harness_error",
+        score: 0,
+        requiredPassed: 0,
+        requiredTotal: 0,
+        violations: [
+            {
+                code: "missing_evidence",
+                message: `harness error: ${reason}`,
+                observed: reason,
+            },
+        ],
+        evidence: {
+            matchedEvents: 0,
+            feedbackRecorded: false,
+            goldAssetLoaded: false,
+            traceTruncated: false,
+        },
+    };
+}