@skill-harness/core 0.5.0 → 0.7.0

package/dist/trace-gates.js

@@ -0,0 +1,519 @@
+export const PREDICATE_KEYS = ["equals", "contains", "starts_with", "ends_with", "matches", "exists", "any"];
+/**
+ * Recognize the known subagent argument shapes.
+ *
+ * Three are supported because three exist in the wild; anything else yields an
+ * empty list, and the scenario should use plain `require_calls` instead. It
+ * deliberately does NOT guess: inventing an `agent` from an unrecognized shape
+ * would produce a confident assertion about a field nobody wrote.
+ */
+export function normalizeSubagentCall(args) {
+    const one = (v) => {
+        if (v === null || typeof v !== "object" || Array.isArray(v))
+            return null;
+        const o = v;
+        const agent = typeof o.agent === "string" ? o.agent : typeof o.name === "string" ? o.name : undefined;
+        if (agent === undefined)
+            return null;
+        const task = typeof o.task === "string" ? o.task : typeof o.prompt === "string" ? o.prompt : "";
+        return { agent, task };
+    };
+    // Parallel: { tasks: [ {agent, task}, … ] }
+    if (Array.isArray(args.tasks))
+        return args.tasks.map(one).filter((x) => x !== null);
+    // Chain: { chain: [ {agent, task}, … ] }
+    if (Array.isArray(args.chain))
+        return args.chain.map(one).filter((x) => x !== null);
+    // Single: { agent, task }
+    const single = one(args);
+    return single ? [single] : [];
+}
+/**
+ * Evaluate every assertion. All of them run even after the first failure — a
+ * scorecard that reports one problem per run makes the author re-run to find the
+ * second, and re-running is the expensive thing this whole layer exists to avoid.
+ *
+ * (One deliberate exception, marked inline in the `require_subagents` loop: the
+ * three sub-questions there are reported separately, and a later one is skipped
+ * when an earlier one already established there is nothing to ask it about.)
+ */
+export function evaluateTraceGates(assert, trace) {
+    const assertions = [];
+    for (const req of assert.require_calls ?? []) {
+        const matched = trace.tool_calls.filter((c) => c.name === req.tool && argsMatch(c, req.args));
+        const min = req.count?.min ?? 1;
+        const max = req.count?.max;
+        const described = describeArgs(req.args);
+        if (matched.length < min) {
+            assertions.push({
+                kind: "require_call",
+                status: "FAIL",
+                detail: `expected at least ${min} call(s) to \`${req.tool}\`${described}, saw ${matched.length}${nearMiss(trace, req)}`,
+            });
+        }
+        else if (max !== undefined && matched.length > max) {
+            assertions.push({
+                kind: "require_call",
+                status: "FAIL",
+                detail: `expected at most ${max} call(s) to \`${req.tool}\`${described}, saw ${matched.length}`,
+            });
+        }
+        else {
+            assertions.push({
+                kind: "require_call",
+                status: "PASS",
+                detail: `\`${req.tool}\`${described} called ${matched.length} time(s)`,
+            });
+        }
+    }
+    for (const req of assert.require_subagents ?? []) {
+        // Three independent questions, reported separately, because they send the
+        // author to three different places: selection (did it delegate at all, and to
+        // the right agent), handoff-completeness (did the task carry what the child
+        // needs), and handoff-leakage (did it carry something it must not).
+        const invocations = trace.tool_calls
+            .filter((c) => c.name === req.tool)
+            .flatMap((c) => normalizeSubagentCall(c.args));
+        const matched = invocations.filter((i) => i.agent === req.agent);
+        const min = req.count?.min ?? 1;
+        const max = req.count?.max;
+        if (matched.length < min || (max !== undefined && matched.length > max)) {
+            const bound = matched.length < min ? `at least ${min}` : `at most ${max}`;
+            const seen = invocations.length === 0
+                ? `no \`${req.tool}\` invocation was recorded`
+                : `saw agents: ${[...new Set(invocations.map((i) => i.agent))].join(", ")}`;
+            assertions.push({
+                kind: "require_subagent",
+                status: "FAIL",
+                detail: `expected ${bound} delegation(s) to \`${req.agent}\` via \`${req.tool}\`, saw ${matched.length} (${seen})`,
+            });
+            // Handoff assertions are meaningless with nothing to inspect, and reporting
+            // them as failures too would triple one root cause.
+            continue;
+        }
+        assertions.push({
+            kind: "require_subagent",
+            status: "PASS",
+            detail: `delegated to \`${req.agent}\` ${matched.length} time(s) via \`${req.tool}\``,
+        });
+        for (const needle of req.task_contains ?? []) {
+            const ok = matched.some((i) => i.task.includes(needle));
+            assertions.push({
+                kind: "require_subagent",
+                status: ok ? "PASS" : "FAIL",
+                detail: ok
+                    ? `handoff to \`${req.agent}\` carried ${JSON.stringify(needle)}`
+                    : `handoff to \`${req.agent}\` omitted required context ${JSON.stringify(needle)}`,
+            });
+        }
+        for (const needle of req.task_excludes ?? []) {
+            const leaked = matched.filter((i) => i.task.includes(needle));
+            // Same asymmetry as `forbid_calls`: a leak check that ran over a truncated
+            // or redacted task has not established that nothing leaked.
+            const lost = leaked.length === 0 && matched.some((i) => valueWasLost(i.task));
+            assertions.push({
+                kind: "require_subagent",
+                status: lost ? "ERROR" : leaked.length === 0 ? "PASS" : "FAIL",
+                detail: lost
+                    ? `leak check on the handoff to \`${req.agent}\` could not be run — the task text was redacted or truncated before the trace was written`
+                    : leaked.length === 0
+                        ? `handoff to \`${req.agent}\` did not carry ${JSON.stringify(needle)}`
+                        : `handoff to \`${req.agent}\` leaked forbidden content ${JSON.stringify(needle)}`,
+            });
+        }
+    }
+    for (const forbid of assert.forbid_calls ?? []) {
+        const hits = trace.tool_calls.filter((c) => c.name === forbid.tool && argsMatch(c, forbid.args));
+        // A "not called" verdict is only trustworthy if the arguments it searched
+        // were intact. Where redaction destroyed one, the honest answer is that the
+        // assertion could not be checked.
+        const lost = [...new Set(trace.tool_calls.filter((c) => c.name === forbid.tool).flatMap((c) => lostArgs(c, forbid.args)))];
+        if (hits.length === 0 && lost.length > 0) {
+            assertions.push({
+                kind: "forbid_call",
+                status: "ERROR",
+                detail: `\`${forbid.tool}\`${describeArgs(forbid.args)} could not be checked — ${lost.map((k) => `\`${k}\``).join(", ")} was redacted or truncated before the trace was written`,
+            });
+            continue;
+        }
+        assertions.push(hits.length === 0
+            ? { kind: "forbid_call", status: "PASS", detail: `\`${forbid.tool}\`${describeArgs(forbid.args)} not called` }
+            : {
+                kind: "forbid_call",
+                status: "FAIL",
+                detail: `\`${forbid.tool}\`${describeArgs(forbid.args)} called ${hits.length} time(s) — forbidden`,
+            });
+    }
+    for (const pattern of assert.unchanged_paths ?? []) {
+        // `null` is "we never looked", and it must not be graded. The whole tri-state
+        // exists so this branch can be written: an unobserved workspace produces
+        // ERROR, which blocks the ship, rather than the vacuous PASS an empty list
+        // used to produce.
+        if (trace.changed_paths === null) {
+            assertions.push({
+                kind: "unchanged_path",
+                status: "ERROR",
+                detail: `\`${pattern}\` could not be checked — the workspace was never observed`,
+            });
+            continue;
+        }
+        const changed = trace.changed_paths.filter((p) => matchesGlob(pattern, p));
+        assertions.push(changed.length === 0
+            ? { kind: "unchanged_path", status: "PASS", detail: `\`${pattern}\` unchanged` }
+            : { kind: "unchanged_path", status: "FAIL", detail: `\`${pattern}\` changed: ${changed.join(", ")}` });
+    }
+    return {
+        // ERROR outranks FAIL: "the evidence is missing" must never be reported as
+        // "the assertion held", and it must not be softened into a plain failure
+        // either — the two call for different fixes.
+        status: assertions.some((a) => a.status === "ERROR")
+            ? "ERROR"
+            : assertions.some((a) => a.status === "FAIL")
+                ? "FAIL"
+                : "PASS",
+        assertions,
+    };
+}
+/**
+ * When a required call is missing, say whether the tool was called at all.
+ *
+ * "expected Agent(agent=plan), saw 0" and "…, saw 0 (Agent called 1x with
+ * different arguments)" send the author to completely different places.
+ */
+function nearMiss(trace, req) {
+    if (!req.args)
+        return "";
+    const byName = trace.tool_calls.filter((c) => c.name === req.tool);
+    if (byName.length === 0)
+        return ` (\`${req.tool}\` was never called)`;
+    return ` (\`${req.tool}\` called ${byName.length}x, but with different arguments)`;
+}
+function describeArgs(args) {
+    if (!args || Object.keys(args).length === 0)
+        return "";
+    const parts = Object.entries(args).map(([k, p]) => {
+        const [op] = PREDICATE_KEYS.filter((key) => p[key] !== undefined);
+        return op ? `${k} ${op} ${JSON.stringify(p[op])}` : k;
+    });
+    return ` (${parts.join(", ")})`;
+}
+function argsMatch(call, args) {
+    if (!args)
+        return true;
+    return Object.entries(args).every(([key, predicate]) => testPredicate(call.args[key], predicate));
+}
+/**
+ * Apply one predicate to one value.
+ *
+ * Multiple operators on the same field are ANDed. An unknown operator can never
+ * reach here — `parseTraceAssert` rejects it at load time, so a typo'd operator
+ * is a spec error rather than an assertion that silently passes.
+ */
+/**
+ * Did redaction destroy the value this predicate needs to read?
+ *
+ * Trace arguments are redacted, truncated and depth-bounded before they are
+ * persisted — necessary, since they reach disk. But the gate then evaluates
+ * predicates against that lossy projection, and the two failure directions are
+ * not symmetric:
+ *
+ * - `require_calls` degrades SAFELY: a needle that redaction removed simply is
+ *   not found, and the assertion FAILS. Over-strict, never over-permissive.
+ * - `forbid_calls` and `task_excludes` degrade DANGEROUSLY: the predicate cannot
+ *   match, so the forbidden thing is reported as absent. `forbid_calls` on
+ *   `{ authorization: { contains: "Bearer" } }` could never fire, because the
+ *   value is always `[redacted]` by the time the gate sees it.
+ *
+ * So the negative assertions ask this first, and report ERROR — "could not be
+ * checked" — instead of a PASS they have not earned.
+ */
+function valueWasLost(value) {
+    if (typeof value === "string") {
+        return value === "[redacted]" || value === "[nested]" || value.includes("… [truncated ");
+    }
+    if (Array.isArray(value))
+        return value.some(valueWasLost);
+    if (value && typeof value === "object")
+        return Object.values(value).some(valueWasLost);
+    return false;
+}
+/** The arg names a predicate set reads whose values redaction has destroyed. */
+function lostArgs(call, args) {
+    if (!args)
+        return [];
+    return Object.keys(args).filter((key) => valueWasLost(call.args[key]));
+}
+export function testPredicate(value, p) {
+    if (p.exists !== undefined) {
+        if (p.exists !== (value !== undefined && value !== null))
+            return false;
+        // `exists: false` is satisfied and nothing else can be tested on an absent value.
+        if (p.exists === false)
+            return true;
+    }
+    if (p.equals !== undefined && !deepEqual(value, p.equals))
+        return false;
+    if (p.contains !== undefined && !asString(value).includes(p.contains))
+        return false;
+    if (p.starts_with !== undefined && !asString(value).startsWith(p.starts_with))
+        return false;
+    if (p.ends_with !== undefined && !asString(value).endsWith(p.ends_with))
+        return false;
+    if (p.matches !== undefined) {
+        let re;
+        try {
+            re = new RegExp(p.matches);
+        }
+        catch {
+            return false; // unreachable via parseTraceAssert, which compiles it first
+        }
+        if (!re.test(asString(value)))
+            return false;
+    }
+    if (p.any !== undefined) {
+        if (!Array.isArray(value))
+            return false;
+        if (!value.some((v) => testPredicate(v, p.any)))
+            return false;
+    }
+    return true;
+}
+/** Stringify for text operators without inventing a match on an absent value. */
+function asString(v) {
+    if (typeof v === "string")
+        return v;
+    if (v === undefined || v === null)
+        return "";
+    return JSON.stringify(v) ?? "";
+}
+function deepEqual(a, b) {
+    if (a === b)
+        return true;
+    if (typeof a !== typeof b || a === null || b === null)
+        return false;
+    if (typeof a !== "object")
+        return false;
+    return JSON.stringify(a) === JSON.stringify(b);
+}
+/**
+ * Minimal glob over workspace-relative paths: `**` any depth, `*` one segment.
+ *
+ * Paths are normalized to forward slashes and stripped of a leading `./` first,
+ * so `./src/a.ts` and `src/a.ts` are the same path — otherwise an assertion
+ * would pass or fail on how the runner happened to spell it.
+ */
+export function matchesGlob(pattern, path) {
+    const p = normalizePath(path);
+    const pat = normalizePath(pattern);
+    if (pat === p)
+        return true;
+    const escaped = pat
+        .replace(/[.+^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*\*\//g, "SLASHSTAR")
+        .replace(/\*\*/g, "GLOBSTAR")
+        .replace(/\*/g, "[^/]*")
+        .replace(/SLASHSTAR/g, "(?:.*/)?")
+        .replace(/GLOBSTAR/g, ".*");
+    return new RegExp(`^${escaped}$`).test(p);
+}
+function normalizePath(p) {
+    return p.replace(/\\/g, "/").replace(/^\.\//, "");
+}
+// ---------------------------------------------------------------------------
+// Parsing / validation
+// ---------------------------------------------------------------------------
+/**
+ * Validate an `assert.trace` block from a spec.
+ *
+ * Strict on purpose: an unknown key is an error, not something ignored. A
+ * silently-ignored `forbid_call` (singular, say) would read in review as a gate
+ * that is protecting something while asserting nothing at all — the worst
+ * possible failure for a safety check.
+ */
+export function parseTraceAssert(raw, ctx) {
+    if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
+        throw new Error(`${ctx}: \`assert.trace\` must be a mapping`);
+    }
+    const obj = raw;
+    const allowed = new Set(["require_calls", "require_subagents", "forbid_calls", "unchanged_paths"]);
+    for (const key of Object.keys(obj)) {
+        if (!allowed.has(key)) {
+            throw new Error(`${ctx}: unknown \`assert.trace\` key \`${key}\` (allowed: ${[...allowed].join(", ")})`);
+        }
+    }
+    const out = {};
+    if (obj.require_calls !== undefined) {
+        out.require_calls = asArray(obj.require_calls, `${ctx}: \`require_calls\``).map((item, i) => {
+            const entry = asObject(item, `${ctx}: \`require_calls[${i}]\``);
+            const tool = requireToolName(entry.tool, `${ctx}: \`require_calls[${i}]\``);
+            const req = { tool };
+            if (entry.count !== undefined)
+                req.count = parseCount(entry.count, `${ctx}: \`require_calls[${i}].count\``);
+            if (entry.args !== undefined)
+                req.args = parseArgs(entry.args, `${ctx}: \`require_calls[${i}].args\``);
+            for (const key of Object.keys(entry)) {
+                if (!["tool", "count", "args"].includes(key)) {
+                    throw new Error(`${ctx}: unknown key \`${key}\` in \`require_calls[${i}]\``);
+                }
+            }
+            return req;
+        });
+    }
+    if (obj.require_subagents !== undefined) {
+        out.require_subagents = asArray(obj.require_subagents, `${ctx}: \`require_subagents\``).map((item, i) => {
+            const where = `${ctx}: \`require_subagents[${i}]\``;
+            const entry = asObject(item, where);
+            for (const key of Object.keys(entry)) {
+                if (!["tool", "agent", "count", "task_contains", "task_excludes"].includes(key)) {
+                    throw new Error(`${ctx}: unknown key \`${key}\` in \`require_subagents[${i}]\``);
+                }
+            }
+            const sub = {
+                tool: requireToolName(entry.tool, where),
+                agent: requireNonEmpty(entry.agent, `${where}: \`agent\``),
+            };
+            if (entry.count !== undefined)
+                sub.count = parseCount(entry.count, `${where}.count`);
+            if (entry.task_contains !== undefined)
+                sub.task_contains = parseNeedles(entry.task_contains, `${where}.task_contains`);
+            if (entry.task_excludes !== undefined)
+                sub.task_excludes = parseNeedles(entry.task_excludes, `${where}.task_excludes`);
+            return sub;
+        });
+    }
+    if (obj.forbid_calls !== undefined) {
+        out.forbid_calls = asArray(obj.forbid_calls, `${ctx}: \`forbid_calls\``).map((item, i) => {
+            // A bare string is the common case — `forbid_calls: [write]`.
+            if (typeof item === "string")
+                return { tool: item };
+            const entry = asObject(item, `${ctx}: \`forbid_calls[${i}]\``);
+            const forbid = { tool: requireToolName(entry.tool, `${ctx}: \`forbid_calls[${i}]\``) };
+            if (entry.args !== undefined)
+                forbid.args = parseArgs(entry.args, `${ctx}: \`forbid_calls[${i}].args\``);
+            for (const key of Object.keys(entry)) {
+                if (!["tool", "args"].includes(key)) {
+                    throw new Error(`${ctx}: unknown key \`${key}\` in \`forbid_calls[${i}]\``);
+                }
+            }
+            return forbid;
+        });
+    }
+    if (obj.unchanged_paths !== undefined) {
+        const paths = asArray(obj.unchanged_paths, `${ctx}: \`unchanged_paths\``);
+        out.unchanged_paths = paths.map((p, i) => {
+            if (typeof p !== "string" || p.trim() === "") {
+                throw new Error(`${ctx}: \`unchanged_paths[${i}]\` must be a non-empty string`);
+            }
+            return p;
+        });
+    }
+    if (!out.require_calls && !out.require_subagents && !out.forbid_calls && !out.unchanged_paths) {
+        throw new Error(`${ctx}: \`assert.trace\` declares no assertions — remove it or add one`);
+    }
+    return out;
+}
+function requireNonEmpty(v, ctx) {
+    if (typeof v !== "string" || v.trim() === "")
+        throw new Error(`${ctx} must be a non-empty string`);
+    return v;
+}
+/** Needles must be non-empty: an empty one matches everything, so the check could never fail. */
+function parseNeedles(raw, ctx) {
+    return asArray(raw, ctx).map((n, i) => {
+        if (typeof n !== "string" || n === "")
+            throw new Error(`${ctx}[${i}] must be a non-empty string`);
+        return n;
+    });
+}
+function requireToolName(v, ctx) {
+    if (typeof v !== "string" || v.trim() === "")
+        throw new Error(`${ctx}: needs a non-empty \`tool\` name`);
+    return v;
+}
+function asArray(v, ctx) {
+    if (!Array.isArray(v) || v.length === 0)
+        throw new Error(`${ctx} must be a non-empty list`);
+    return v;
+}
+function asObject(v, ctx) {
+    if (v === null || typeof v !== "object" || Array.isArray(v))
+        throw new Error(`${ctx} must be a mapping`);
+    return v;
+}
+function parseCount(raw, ctx) {
+    const obj = asObject(raw, ctx);
+    const out = {};
+    for (const key of Object.keys(obj)) {
+        if (key !== "min" && key !== "max")
+            throw new Error(`${ctx}: unknown key \`${key}\` (allowed: min, max)`);
+    }
+    for (const key of ["min", "max"]) {
+        if (obj[key] === undefined)
+            continue;
+        const n = obj[key];
+        if (typeof n !== "number" || !Number.isInteger(n) || n < 0) {
+            throw new Error(`${ctx}: \`${key}\` must be a non-negative integer`);
+        }
+        out[key] = n;
+    }
+    if (out.min !== undefined && out.max !== undefined && out.min > out.max) {
+        throw new Error(`${ctx}: min (${out.min}) exceeds max (${out.max}) — nothing can satisfy it`);
+    }
+    return out;
+}
+function parseArgs(raw, ctx) {
+    const obj = asObject(raw, ctx);
+    const out = {};
+    for (const [key, value] of Object.entries(obj)) {
+        out[key] = parsePredicate(value, `${ctx}.${key}`);
+    }
+    return out;
+}
+function parsePredicate(raw, ctx) {
+    // `agent: plan` is shorthand for `agent: { equals: plan }` — the common case
+    // should not require the author to know the operator vocabulary.
+    if (typeof raw === "string" || typeof raw === "number" || typeof raw === "boolean") {
+        return { equals: raw };
+    }
+    const obj = asObject(raw, ctx);
+    const out = {};
+    for (const [key, value] of Object.entries(obj)) {
+        if (!PREDICATE_KEYS.includes(key)) {
+            throw new Error(`${ctx}: unknown operator \`${key}\` (allowed: ${PREDICATE_KEYS.join(", ")})`);
+        }
+        if (key === "matches") {
+            if (typeof value !== "string")
+                throw new Error(`${ctx}: \`matches\` must be a string pattern`);
+            try {
+                new RegExp(value);
+            }
+            catch (e) {
+                throw new Error(`${ctx}: \`matches\` is not a valid regular expression: ${e instanceof Error ? e.message : e}`);
+            }
+            out.matches = value;
+            continue;
+        }
+        if (key === "exists") {
+            if (typeof value !== "boolean")
+                throw new Error(`${ctx}: \`exists\` must be true or false`);
+            out.exists = value;
+            continue;
+        }
+        if (key === "any") {
+            out.any = parsePredicate(value, `${ctx}.any`);
+            continue;
+        }
+        if (key === "equals") {
+            out.equals = value;
+            continue;
+        }
+        // contains / starts_with / ends_with
+        if (typeof value !== "string")
+            throw new Error(`${ctx}: \`${key}\` must be a string`);
+        out[key] = value;
+    }
+    if (Object.keys(out).length === 0)
+        throw new Error(`${ctx}: predicate declares no operator`);
+    return out;
+}
+//# sourceMappingURL=trace-gates.js.map

package/dist/trends.d.ts

@@ -38,6 +38,34 @@ export interface TrendData {
     }[];
     models: TrendModel[];
 }
+/** One model tag's scored run history in ONE delivery mode, chronologically ascending. */
+export interface ScoredRunGroup {
+    tag: string;
+    mode: string;
+    model: string;
+    runs: ResultsFile[];
+    /** Runs in this tag whose results.yaml could not be parsed (per tag, not per mode). */
+    skipped: number;
+}
+/**
+ * Walk `<skillDir>/tests/results/` and group every SCORED run by model tag × delivery
+ * mode, chronologically (timestamp-slug dir names sort correctly).
+ *
+ * The single history reader: `collectTrends` renders it, `collectStability` derives
+ * run-over-run flips from it. Two walkers over the same tree is how "which runs count"
+ * drifts — the mistake that had force runs excluded from scoring in seven places at
+ * once (see SCORED_MODES).
+ *
+ * Red runs are excluded: a baseline has no grade, and pairing it with anything would
+ * compare a skill-off run to a skill-on one. Green and force are never pooled into one
+ * group — placement moves verdicts, so a green run and a force run of the same scenario
+ * are two measurements, not two samples.
+ *
+ * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic write) is
+ * logged via `console.warn`, skipped, and counted in `skipped` — never thrown, because
+ * one torn file must not take down a whole read-only view.
+ */
+export declare function collectScoredRuns(skillDir: string): ScoredRunGroup[];
 /**
  * Per model-tag, read the full run history (not just the latest) from
  * <skillDir>/tests/results/, chronologically (timestamp-slug dir names sort