@skill-harness/core 0.5.0 → 0.7.0

package/dist/affected.d.ts

@@ -0,0 +1,88 @@
+import type { Scenario } from "./spec.js";
+/**
+ * Which scenarios could plausibly be affected by a change.
+ *
+ * The governing asymmetry: **an under-inclusive set is dangerous and an
+ * over-inclusive one is merely expensive.** Missing a regression means shipping
+ * it; running extra scenarios costs tokens. So every ambiguity resolves toward
+ * selecting more, and anything the mapping cannot explain selects everything.
+ *
+ * An affected run is always partial and can never report SHIP. It is an
+ * iteration tool — a full run is still what clears a skill for publishing.
+ */
+export type SelectionReason = {
+    kind: "covers";
+    detail: string;
+} | {
+    kind: "critical";
+} | {
+    kind: "under-pressure";
+} | {
+    kind: "stimulus-changed";
+    detail: string;
+} | {
+    kind: "unmapped-change";
+    detail: string;
+} | {
+    kind: "no-covers-declared";
+};
+export interface SelectedScenario {
+    id: string;
+    reasons: SelectionReason[];
+}
+export interface AffectedResult {
+    selected: SelectedScenario[];
+    /** True when the mapping could not be trusted and everything was selected. */
+    conservative: boolean;
+    /** Human-readable account of why, when `conservative`. */
+    conservativeReason: string | null;
+    /** Files in the diff whose changed lines map to no covered section. A file nothing `covers` at all is not listed here — it selects everything (see `isInstructionText`) or is skipped as source. */
+    unmappedFiles: string[];
+}
+export interface DiffHunk {
+    file: string;
+    /**
+     * 1-based first changed line in the NEW file.
+     *
+     * For a pure deletion git emits `@@ -2,3 +1,0 @@`, so this is the line the
+     * deletion sits AFTER — 1 in that example, not 0. `selectAffected` maps
+     * sections from the real value; "fixing" it to 0 to match an earlier version
+     * of this comment would break deletion→section mapping.
+     */
+    start: number;
+    /** Number of changed lines; 0 for a pure deletion at `start`. */
+    count: number;
+}
+/**
+ * Parse `git diff --unified=0` hunk headers.
+ *
+ * `--unified=0` matters: with context lines the hunk range covers unchanged text,
+ * and a change at the top of one section would be attributed to the section above
+ * it too. Over-selection is the safe direction, but only when it is *reasoned*
+ * over-selection rather than an artefact of a flag.
+ */
+export declare function parseDiffHunks(diff: string): DiffHunk[];
+/** Files the diff touched (added, modified or deleted). */
+export declare function parseDiffFiles(diff: string): string[];
+export interface AffectedOptions {
+    scenarios: Scenario[];
+    /** Dir `covers` and fixture paths resolve against. */
+    specDir: string;
+    /** Unified diff text (`git diff --unified=0 <base>`). */
+    diff: string;
+    /** Repo root the diff paths are relative to. */
+    repoRoot: string;
+}
+/** Run `git diff --unified=0 <base>` in a repo. Empty string when git fails. */
+export declare function gitDiff(repoRoot: string, base: string): Promise<string>;
+/**
+ * Select the scenarios a change could affect.
+ *
+ * Always unions in every critical and every B-series scenario, whatever the diff
+ * said. Those are the ship gates: if the mapping is wrong — and a mapping built
+ * from author-written labels can be — the scenarios that decide releases are the
+ * worst possible ones to skip.
+ */
+export declare function selectAffected(opts: AffectedOptions): AffectedResult;
+/** One line per selected scenario, naming every reason it was picked. */
+export declare function formatAffected(result: AffectedResult, total: number): string;

package/dist/affected.js

@@ -0,0 +1,222 @@
+import { existsSync, readFileSync } from "node:fs";
+import { resolve, dirname, relative } from "node:path";
+import { parseSections, sectionAtLine, parseCoversRef } from "./instruction-coverage.js";
+import { exec } from "./util/exec.js";
+/**
+ * Parse `git diff --unified=0` hunk headers.
+ *
+ * `--unified=0` matters: with context lines the hunk range covers unchanged text,
+ * and a change at the top of one section would be attributed to the section above
+ * it too. Over-selection is the safe direction, but only when it is *reasoned*
+ * over-selection rather than an artefact of a flag.
+ */
+export function parseDiffHunks(diff) {
+    const hunks = [];
+    let file = null;
+    for (const line of diff.split("\n")) {
+        if (line.startsWith("+++ ")) {
+            const p = line.slice(4).trim();
+            file = p === "/dev/null" ? null : p.replace(/^b\//, "");
+            continue;
+        }
+        if (!line.startsWith("@@") || file === null)
+            continue;
+        const m = /@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/.exec(line);
+        if (!m)
+            continue;
+        hunks.push({ file, start: Number(m[1]), count: m[2] === undefined ? 1 : Number(m[2]) });
+    }
+    return hunks;
+}
+/** Files the diff touched (added, modified or deleted). */
+export function parseDiffFiles(diff) {
+    const files = new Set();
+    for (const line of diff.split("\n")) {
+        const m = /^diff --git a\/(.+?) b\/(.+)$/.exec(line);
+        if (m) {
+            files.add(m[1]);
+            files.add(m[2]);
+        }
+    }
+    return [...files];
+}
+/** Run `git diff --unified=0 <base>` in a repo. Empty string when git fails. */
+export async function gitDiff(repoRoot, base) {
+    const r = await exec("git", ["diff", "--unified=0", base], { cwd: repoRoot, timeoutMs: 60_000 });
+    if (r.code !== 0)
+        throw new Error(`git diff --unified=0 ${base} failed: ${r.stderr.trim() || `exit ${r.code}`}`);
+    return r.stdout;
+}
+/**
+ * Select the scenarios a change could affect.
+ *
+ * Always unions in every critical and every B-series scenario, whatever the diff
+ * said. Those are the ship gates: if the mapping is wrong — and a mapping built
+ * from author-written labels can be — the scenarios that decide releases are the
+ * worst possible ones to skip.
+ */
+export function selectAffected(opts) {
+    const { scenarios, specDir, diff, repoRoot } = opts;
+    const reasons = new Map();
+    const add = (id, reason) => {
+        const list = reasons.get(id) ?? [];
+        list.push(reason);
+        reasons.set(id, list);
+    };
+    const selectAll = (why) => {
+        for (const s of scenarios)
+            if (!reasons.has(s.id))
+                add(s.id, { kind: "unmapped-change", detail: why });
+        return {
+            selected: [...reasons.entries()].map(([id, rs]) => ({ id, reasons: rs })),
+            conservative: true,
+            conservativeReason: why,
+            unmappedFiles: [],
+        };
+    };
+    // The ship gates, unconditionally.
+    for (const s of scenarios) {
+        if (s.critical)
+            add(s.id, { kind: "critical" });
+        if (/^B/i.test(s.id))
+            add(s.id, { kind: "under-pressure" });
+    }
+    const hunks = parseDiffHunks(diff);
+    const changedFiles = parseDiffFiles(diff);
+    // A scenario with no `covers` cannot be excluded by a coverage mapping — there
+    // is nothing to consult. Selecting it is the only honest answer.
+    for (const s of scenarios) {
+        if (!s.covers || s.covers.length === 0)
+            add(s.id, { kind: "no-covers-declared" });
+    }
+    // Stimulus files: a changed fixture, post-test, agent file or extension changes
+    // what the scenario RUNS, regardless of any instruction text.
+    const stimulusFiles = (s) => {
+        const files = [];
+        if (s.fixture)
+            files.push(s.fixture);
+        if (s.assert?.post_test)
+            files.push(s.assert.post_test);
+        if (s.systemPromptFile)
+            files.push(s.systemPromptFile);
+        for (const e of s.extensions ?? [])
+            files.push(e);
+        return files;
+    };
+    const changedAbs = new Set(changedFiles.map((f) => resolve(repoRoot, f)));
+    for (const s of scenarios) {
+        for (const f of stimulusFiles(s)) {
+            const abs = resolve(specDir, f);
+            // Directory-ish match too: a fixture is a tree, and a change anywhere in it
+            // counts.
+            const hit = [...changedAbs].some((c) => c === abs || c.startsWith(`${abs}/`));
+            if (hit)
+                add(s.id, { kind: "stimulus-changed", detail: f });
+        }
+    }
+    // Reverse the covers map: changed line → section → scenarios.
+    const sectionsFor = new Map();
+    const load = (abs) => {
+        if (sectionsFor.has(abs))
+            return sectionsFor.get(abs);
+        const parsed = existsSync(abs) ? parseSections(readFileSync(abs, "utf8")) : null;
+        sectionsFor.set(abs, parsed);
+        return parsed;
+    };
+    const coversIndex = new Map(); // "abs#slug" | "abs" -> scenario ids
+    for (const s of scenarios) {
+        for (const raw of s.covers ?? []) {
+            const ref = parseCoversRef(raw);
+            const abs = resolve(specDir, ref.file);
+            const key = ref.slug === undefined ? abs : `${abs}#${ref.slug}`;
+            coversIndex.set(key, [...(coversIndex.get(key) ?? []), s.id]);
+        }
+    }
+    // The skill's own directory, minus its `tests/` subtree. Markdown here is
+    // instruction text the model reads; markdown elsewhere in the repo is not.
+    const skillRoot = dirname(specDir);
+    const isInstructionText = (abs) => abs.startsWith(`${skillRoot}/`) && !abs.startsWith(`${specDir}/`) && /\.(?:md|markdown)$/i.test(abs);
+    const unmappedFiles = new Set();
+    for (const hunk of hunks) {
+        const abs = resolve(repoRoot, hunk.file);
+        // Only instruction files participate; a changed source file is not a section.
+        const referenced = [...coversIndex.keys()].some((k) => k === abs || k.startsWith(`${abs}#`));
+        if (!referenced) {
+            // "No `covers` mentions it" is not the same as "it cannot matter". For
+            // instruction text inside the skill, it is the opposite: every scenario
+            // reads the skill, and nobody claimed this prose — so the mapping has no
+            // basis for ruling it out. `continue` here deselected scenarios the edit
+            // could well have broken, and did not even reach `unmappedFiles`, so the
+            // output said nothing about it. Under-inclusion is the one failure this
+            // module exists to avoid.
+            if (isInstructionText(abs)) {
+                return selectAll(`${relative(repoRoot, abs) || hunk.file} is instruction text that no scenario \`covers\` — the mapping cannot rule it out`);
+            }
+            continue;
+        }
+        const sections = load(abs);
+        if (sections === null) {
+            // The file is referenced but gone — a rename or delete. Nothing can be
+            // mapped, and guessing would be worse than admitting it.
+            return selectAll(`${hunk.file} is referenced by \`covers\` but is not readable — it may have been renamed or deleted`);
+        }
+        for (const id of coversIndex.get(abs) ?? [])
+            add(id, { kind: "covers", detail: `${hunk.file} (whole file)` });
+        const lines = hunk.count === 0 ? [hunk.start] : Array.from({ length: hunk.count }, (_, i) => hunk.start + i);
+        let mappedAny = false;
+        for (const line of lines) {
+            const section = sectionAtLine(sections, line);
+            if (!section)
+                continue; // preamble before the first heading
+            const ids = coversIndex.get(`${abs}#${section.slug}`) ?? [];
+            for (const id of ids)
+                add(id, { kind: "covers", detail: `${hunk.file}#${section.slug}` });
+            if (ids.length > 0)
+                mappedAny = true;
+        }
+        if (!mappedAny && (coversIndex.get(abs) ?? []).length === 0)
+            unmappedFiles.add(hunk.file);
+    }
+    // A wholesale rewrite defeats line mapping: every line looks changed, and the
+    // sections that "match" are an artefact of the rewrite's shape.
+    const rewritten = hunks.filter((h) => h.count > 200);
+    if (rewritten.length > 0) {
+        return selectAll(`${rewritten[0].file} changed by ${rewritten[0].count} lines in one hunk — too large to map to sections reliably`);
+    }
+    return {
+        selected: [...reasons.entries()].map(([id, rs]) => ({ id, reasons: rs })),
+        conservative: false,
+        conservativeReason: null,
+        unmappedFiles: [...unmappedFiles],
+    };
+}
+/** One line per selected scenario, naming every reason it was picked. */
+export function formatAffected(result, total) {
+    const out = [];
+    if (result.conservative) {
+        out.push(`selecting ALL ${total} scenario(s): ${result.conservativeReason}`);
+    }
+    else {
+        out.push(`selected ${result.selected.length}/${total} scenario(s):`);
+    }
+    for (const s of [...result.selected].sort((a, b) => a.id.localeCompare(b.id))) {
+        out.push(`  ${s.id}  ${s.reasons.map(describe).join(", ")}`);
+    }
+    if (result.unmappedFiles.length) {
+        out.push(`  note: changes in ${result.unmappedFiles.join(", ")} map to no covered section`);
+    }
+    out.push("");
+    out.push("an affected run is partial and never reports SHIP — a full run still gates a release");
+    return out.join("\n");
+}
+function describe(r) {
+    switch (r.kind) {
+        case "covers": return `covers ${r.detail}`;
+        case "critical": return "critical (always run)";
+        case "under-pressure": return "B-series (always run)";
+        case "stimulus-changed": return `stimulus changed: ${r.detail}`;
+        case "unmapped-change": return `conservative: ${r.detail}`;
+        case "no-covers-declared": return "declares no `covers` — cannot be ruled out";
+    }
+}
+//# sourceMappingURL=affected.js.map

package/dist/capture-trace-types.d.ts

@@ -0,0 +1,228 @@
+/**
+ * Data contracts for the pi-native regression capture program.
+ *
+ * Types only — no behavior. Phase 0 of
+ * `docs/superpowers/plans/2026-08-07-pi-native-regression-capture-program.md`
+ * fixes the shapes before any parser, capture UI, or gate evaluator is written,
+ * because both shapes get persisted and a persisted shape is expensive to move.
+ *
+ * Both carry an explicit version. The pi event stream they derive from is not a
+ * stable public contract (measured against pi 0.83.0; see
+ * `packages/adapters/test/fixtures/pi-json/README.md`), so a reader must be able
+ * to tell which pi produced an artifact and refuse one it does not understand
+ * rather than silently misreading it.
+ */
+import type { ModelRef, RunMode } from "./adapters/types.js";
+export declare const EXECUTION_TRACE_VERSION = 2;
+/**
+ * Tool arguments after sanitization.
+ *
+ * `unknown` rather than `any`: an assertion evaluator must narrow before
+ * comparing, and the safe-DSL operators (equals/contains/matches/…) are the only
+ * things allowed to inspect these.
+ */
+export type SanitizedArgs = Record<string, unknown>;
+/**
+ * One tool call, correlated across pi's `tool_execution_start` /
+ * `tool_execution_end` pair.
+ *
+ * Correlation is by `toolCallId` and nothing else. Measured: pi executes tool
+ * calls batched in one assistant message CONCURRENTLY, and emits their `end`
+ * events in completion order, not issue order — three `bash` calls sleeping
+ * 6s/1s/3s started SLOW,FAST,MID and ended FAST,MID,SLOW. Any parser that pairs
+ * by position is wrong.
+ */
+export interface TraceToolCall {
+    /** pi's `toolCallId`. Unique within a trace; the only sound correlation key. */
+    id: string;
+    /** Registered tool name, e.g. `read`, `bash`, `Agent`. */
+    name: string;
+    /** Sanitized call arguments. Secrets and oversized values are redacted. */
+    args: SanitizedArgs;
+    /** 0-based position in ISSUE order (the order pi emitted `tool_execution_start`). */
+    issueIndex: number;
+    /** 0-based position in COMPLETION order. Differs from `issueIndex` under parallelism. */
+    completionIndex: number;
+    /** pi's `isError` on the `tool_execution_end` event. */
+    isError: boolean;
+    /**
+     * Bounded metadata about the result body — never the body itself.
+     *
+     * Full tool results are not persisted: they routinely contain file contents,
+     * command output, and absolute paths (a failing `read` embeds the full path in
+     * its error string). A gate that needs to assert on content asserts on the
+     * workspace instead.
+     */
+    result: TraceResultMeta;
+}
+export interface TraceResultMeta {
+    /** Byte length of the serialized result content. */
+    bytes: number;
+    /** SHA-256 of the serialized result content, so identity is checkable without the body. */
+    sha256: string;
+    /**
+     * The tool's own structured `details`, when it returned any.
+     *
+     * Measured: a value a tool returns as `details` survives verbatim into
+     * `tool_execution_end.result.details`. That is the one stable structured
+     * channel an extension can expose deliberately — unlike prose in `content`,
+     * which is formatting, not contract. Retained only when it is small and free
+     * of redaction hits; absent otherwise.
+     */
+    details?: Record<string, unknown>;
+}
+/**
+ * A trace of one subject execution — one `pi` invocation.
+ *
+ * A multi-turn scenario produces ONE TRACE PER TURN, because each `pi` call
+ * emits an independent stream containing only that invocation's messages (the
+ * conversation carries in the session dir, not in the event stream). This
+ * mirrors the loop the adapter already runs.
+ */
+export interface ExecutionTraceV1 {
+    trace_version: typeof EXECUTION_TRACE_VERSION;
+    /** `pi --version` at capture time. Null when it could not be determined. */
+    pi_version: string | null;
+    /** The model under test. */
+    subject: ModelRef;
+    scenario_id: string;
+    mode: RunMode;
+    /** 0-based repetition index within a run. */
+    rep: number;
+    /** 0-based turn index; always 0 for a single-turn scenario. */
+    turn: number;
+    /**
+     * The assistant's final answer text for this invocation.
+     *
+     * Deliberately the FINAL assistant message only, not every assistant text
+     * block. Measured: pi's print mode emits exactly this, and it is what today's
+     * adapter hands the judge — concatenating interim text blocks would feed the
+     * judge narration ("Let me read that file…") that the current transcript has
+     * never contained, changing grades on scenarios nobody edited.
+     */
+    final_text: string;
+    /** Every tool call in this invocation, in issue order. */
+    tool_calls: TraceToolCall[];
+    /**
+     * Workspace paths whose content changed during the invocation, relative to the
+     * workspace root. Evidence for `unchanged_paths` assertions. Bounded to the
+     * isolated workspace: writes outside it are not observable and never claimed.
+     *
+     * **Tri-state, and the third state is the point.** `null` means the workspace
+     * was never observed — there was none, or the snapshot could not be taken.
+     * `[]` means observed, and nothing changed. Collapsing the two made a safety
+     * gate report green from evidence that does not exist: a run whose observation
+     * failed recorded `objective: ERROR` honestly, and `regate` then read the saved
+     * `[]` and re-graded it to PASS, re-stamping the source hash so `lint` called
+     * the result current.
+     *
+     * Version 2 exists for exactly this widening: a v1 reader must decline a v2
+     * trace rather than read `null` as empty.
+     */
+    changed_paths: string[] | null;
+    /**
+     * Reported token cost of this invocation, when pi provided it.
+     *
+     * pi's `usage.cost` carries real per-message costs. Recorded for spend
+     * disclosure, never for grading.
+     */
+    cost_usd: number | null;
+    /** SHA-256 over the deterministic serialization of this trace, minus this field. */
+    trace_sha256?: string;
+}
+export declare const CAPTURE_SCHEMA_VERSION = 1;
+/** Why a conversation was captured. */
+export type CaptureClassification = "failure" | "good_example";
+/** Where a capture is in the human review pipeline. */
+export type CapturePromotionStatus = "pending" | "promoted" | "rejected";
+/** What the human confirmed as responsible for the behavior. */
+export interface CaptureTarget {
+    kind: "skill" | "subagent";
+    /** Path relative to the skills root or repo root — never absolute. */
+    path: string;
+    /** SHA-256 of the target's content at capture time, so drift is detectable. */
+    content_sha256: string;
+}
+/**
+ * Provenance for a capture.
+ *
+ * Hashed, not absolute: an absolute session path identifies a machine and a
+ * user, and the capture file is meant to be committed. The hash is enough to
+ * recognize the same session again locally, which is all provenance needs to do.
+ */
+export interface CaptureProvenance {
+    session_sha256: string;
+    /** Indices of the selected turn range within the active branch, inclusive. */
+    turn_range: {
+        start: number;
+        end: number;
+    };
+    subject?: ModelRef;
+    git_commit?: string;
+    /** True when the working tree was dirty at capture time. */
+    git_dirty?: boolean;
+}
+/**
+ * A reviewed conversation awaiting promotion into a scenario.
+ *
+ * Lives under `<skill>/tests/captures/`, NOT in `specification.yaml`. A pending
+ * capture is not a test: putting it in the spec would drag it into ship-bar
+ * totals, staleness, lift, and stability, and the alternative — a `draft: true`
+ * flag every runner, scorer and linter has to honor — is a state that only has
+ * to be forgotten once to either corrupt a grade or silently drop a real
+ * scenario from a release run.
+ */
+export interface CaptureCaseV1 {
+    capture_schema: typeof CAPTURE_SCHEMA_VERSION;
+    id: string;
+    created: string;
+    classification: CaptureClassification;
+    /** Sanitized user turns only. Assistant prose is evidence, never an oracle. */
+    turns: string[];
+    /** Human-written, in their own words. Required — a capture with no expectation is not reviewable. */
+    expected_behavior: string;
+    /** Editable draft checklist. Offline-derived by default; LLM drafting is opt-in and costs tokens. */
+    checklist: string[];
+    target: CaptureTarget;
+    provenance: CaptureProvenance;
+    status: CapturePromotionStatus;
+    /** Set once promoted: the scenario id appended to `specification.yaml`. */
+    scenario_id?: string;
+    /**
+     * `covers` refs for this pending case, so `coverage` can park it against the
+     * instructions it is about before anyone promotes it.
+     *
+     * Derived from `target` — the author has already been made to choose which
+     * instructions are responsible, and asking again in different words would be
+     * asking the same question twice. File granularity, not `#section`: the target
+     * choice attributes a file, and inventing a section would be a guess the
+     * session cannot support.
+     *
+     * Written relative to the SPEC dir (`<skill>/tests`), matching how a scenario's
+     * own `covers` resolve. `coverage` read this field from the start; nothing ever
+     * wrote it, so every pending case parked against nothing and the feature was
+     * inert.
+     */
+    covers?: string[];
+}
+/**
+ * Local-only evidence sidecar for a capture.
+ *
+ * Written to `<skill>/tests/captures/.local/` and git-ignored by default. It may
+ * hold a sanitized assistant excerpt and tool-call summaries to help a human
+ * review the capture, and must NEVER hold hidden thinking, complete tool-result
+ * bodies, or the effective system prompt.
+ *
+ * pi's stream carries thinking in `message_end`, `turn_end` AND `agent_end`, so
+ * dropping it is an explicit filter at every one of those, not a side effect of
+ * reading the convenient field.
+ */
+export interface CaptureEvidenceV1 {
+    capture_id: string;
+    /** Sanitized, truncated excerpt of the assistant text of every selected turn, joined. */
+    assistant_excerpt: string;
+    /** Tool name, error state and redacted arguments — no bodies. */
+    tool_calls: Array<Pick<TraceToolCall, "name" | "isError"> & {
+        args: SanitizedArgs;
+    }>;
+}

package/dist/capture-trace-types.js

@@ -0,0 +1,23 @@
+/**
+ * Data contracts for the pi-native regression capture program.
+ *
+ * Types only — no behavior. Phase 0 of
+ * `docs/superpowers/plans/2026-08-07-pi-native-regression-capture-program.md`
+ * fixes the shapes before any parser, capture UI, or gate evaluator is written,
+ * because both shapes get persisted and a persisted shape is expensive to move.
+ *
+ * Both carry an explicit version. The pi event stream they derive from is not a
+ * stable public contract (measured against pi 0.83.0; see
+ * `packages/adapters/test/fixtures/pi-json/README.md`), so a reader must be able
+ * to tell which pi produced an artifact and refuse one it does not understand
+ * rather than silently misreading it.
+ */
+// ---------------------------------------------------------------------------
+// Execution trace
+// ---------------------------------------------------------------------------
+export const EXECUTION_TRACE_VERSION = 2;
+// ---------------------------------------------------------------------------
+// Capture case
+// ---------------------------------------------------------------------------
+export const CAPTURE_SCHEMA_VERSION = 1;
+//# sourceMappingURL=capture-trace-types.js.map