@skill-harness/core 0.16.0 → 0.18.0

package/schemas/specification-v1.schema.json

@@ -4,197 +4,134 @@
   "title": "skill-harness specification v1",
   "type": "object",
   "additionalProperties": false,
-  "required": ["skill", "judge_persona", "ship_bar", "scenarios"],
+  "required": [
+    "skill",
+    "judge_persona",
+    "ship_bar",
+    "scenarios"
+  ],
   "properties": {
-    "schema": { "const": 1 },
-    "skill": { "type": "string", "minLength": 1 },
-    "judge_persona": { "type": "string", "minLength": 1 },
-    "ship_bar": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["total", "min_pass"],
-      "properties": {
-        "total": { "type": "integer", "minimum": 1 },
-        "min_pass": { "type": "integer", "minimum": 1 },
-        "no_critical_fail": { "type": "boolean" }
-      }
+    "schema": {
+      "const": 1
     },
-    "critical": { "type": "array", "items": { "type": "string" }, "uniqueItems": true },
-    "adjudication": { "type": "object" },
-    "scenarios": { "type": "array", "items": { "$ref": "#/$defs/scenario" } }
-  },
-  "$defs": {
-    "stringList": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "uniqueItems": true },
-    "predicate": {
-      "oneOf": [
-        { "type": ["string", "number", "boolean"] },
-        {
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "equals": {},
-            "contains": { "type": "string" },
-            "starts_with": { "type": "string" },
-            "ends_with": { "type": "string" },
-            "matches": { "type": "string" },
-            "exists": { "type": "boolean" },
-            "any": { "$ref": "#/$defs/predicate" }
-          },
-          "minProperties": 1
-        }
-      ]
-    },
-    "count": {
-      "type": "object",
-      "additionalProperties": false,
-      "properties": {
-        "min": { "type": "integer", "minimum": 0 },
-        "max": { "type": "integer", "minimum": 0 }
-      }
+    "skill": {
+      "type": "string",
+      "minLength": 1
     },
-    "selector": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["event"],
-      "properties": {
-        "event": { "type": "string", "minLength": 1 },
-        "where": { "type": "object", "additionalProperties": { "$ref": "#/$defs/predicate" } },
-        "select": { "enum": ["first", "last", "all"] }
-      }
+    "judge_persona": {
+      "type": "string",
+      "minLength": 1
     },
-    "requiredSelector": {
+    "ship_bar": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["event"],
+      "required": [
+        "total",
+        "min_pass"
+      ],
       "properties": {
-        "event": { "type": "string", "minLength": 1 },
-        "where": { "type": "object", "additionalProperties": { "$ref": "#/$defs/predicate" } },
-        "select": { "enum": ["first", "last", "all"] },
-        "count": { "$ref": "#/$defs/count" }
+        "total": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "min_pass": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "no_critical_fail": {
+          "type": "boolean"
+        }
       }
     },
-    "correlation": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["left", "right"],
-      "properties": {
-        "left": { "$ref": "#/$defs/selector" },
-        "right": { "$ref": "#/$defs/selector" },
-        "same": { "$ref": "#/$defs/stringList" },
-        "different": { "$ref": "#/$defs/stringList" },
-        "order": { "enum": ["before", "after"] }
+    "critical": {
+      "type": "array",
+      "items": {
+        "type": "string"
       },
-      "anyOf": [{ "required": ["same"] }, { "required": ["different"] }, { "required": ["order"] }]
-    },
-    "freshness": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["subject", "after"],
-      "properties": {
-        "subject": { "$ref": "#/$defs/selector" },
-        "after": { "type": "array", "items": { "$ref": "#/$defs/selector" }, "minItems": 1 },
-        "same": { "$ref": "#/$defs/stringList" }
-      }
-    },
-    "unique": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["events", "fields"],
-      "properties": {
-        "events": { "$ref": "#/$defs/selector" },
-        "fields": { "$ref": "#/$defs/stringList" }
-      }
+      "uniqueItems": true
     },
-    "forbidAfter": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["anchor", "forbidden"],
-      "properties": {
-        "anchor": { "$ref": "#/$defs/selector" },
-        "forbidden": { "type": "array", "items": { "$ref": "#/$defs/selector" }, "minItems": 1 },
-        "same": { "$ref": "#/$defs/stringList" },
-        "anchor_optional": { "type": "boolean" }
+    "scenarios": {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/scenario"
       }
-    },
-    "approvalAssertion": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["grant", "use"],
-      "properties": {
-        "grant": { "$ref": "#/$defs/selector" },
-        "use": { "$ref": "#/$defs/selector" },
-        "same": { "$ref": "#/$defs/stringList" },
-        "scopes": { "$ref": "#/$defs/stringList" },
-        "sources": { "$ref": "#/$defs/stringList" },
-        "unexpired": { "type": "boolean" },
-        "max_uses": { "type": "integer", "minimum": 1 }
-      }
-    },
-    "coverageAssertion": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["requirements"],
-      "properties": {
-        "requirements": { "$ref": "#/$defs/stringList" },
-        "events": { "$ref": "#/$defs/selector" }
-      }
-    },
-    "trajectory": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["version"],
-      "properties": {
-        "version": { "const": "1.0" },
-        "require": { "type": "array", "items": { "$ref": "#/$defs/requiredSelector" }, "minItems": 1 },
-        "forbid": { "type": "array", "items": { "$ref": "#/$defs/selector" }, "minItems": 1 },
-        "ordered": { "type": "array", "items": { "type": "array", "items": { "$ref": "#/$defs/selector" }, "minItems": 1 }, "minItems": 1 },
-        "correlate": { "type": "array", "items": { "$ref": "#/$defs/correlation" }, "minItems": 1 },
-        "freshness": { "type": "array", "items": { "$ref": "#/$defs/freshness" }, "minItems": 1 },
-        "unique": { "type": "array", "items": { "$ref": "#/$defs/unique" }, "minItems": 1 },
-        "forbid_after": { "type": "array", "items": { "$ref": "#/$defs/forbidAfter" }, "minItems": 1 },
-        "approvals": { "type": "array", "items": { "$ref": "#/$defs/approvalAssertion" }, "minItems": 1 },
-        "coverage": { "type": "array", "items": { "$ref": "#/$defs/coverageAssertion" }, "minItems": 1 }
-      },
-      "minProperties": 2
-    },
+    }
+  },
+  "$defs": {
     "scenario": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["id", "title", "turns", "checklist"],
+      "required": [
+        "id",
+        "title",
+        "turns",
+        "checklist"
+      ],
       "properties": {
-        "id": { "type": "string", "minLength": 1 },
-        "title": { "type": "string", "minLength": 1 },
-        "critical": { "type": "boolean" },
-        "mode": { "enum": ["inline", "seeded"] },
-        "fixture": { "type": "string" },
-        "turns": { "type": "array", "items": { "type": "string" }, "minItems": 1 },
-        "checklist": { "type": "array", "items": { "type": "string" }, "minItems": 1 },
-        "covers": { "type": "array", "items": { "type": "string" } },
-        "reps": { "type": "integer", "minimum": 1 },
-        "pass_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
-        "system_prompt_file": { "type": "string" },
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "title": {
+          "type": "string",
+          "minLength": 1
+        },
+        "critical": {
+          "type": "boolean"
+        },
+        "mode": {
+          "enum": [
+            "inline",
+            "seeded"
+          ]
+        },
+        "fixture": {
+          "type": "string"
+        },
+        "turns": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "minItems": 1
+        },
+        "checklist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "minItems": 1
+        },
+        "covers": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "reps": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "pass_threshold": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "system_prompt_file": {
+          "type": "string"
+        },
         "env": {
           "type": "object",
           "additionalProperties": false,
           "properties": {
             "workspace": {},
-            "remote": { "type": "boolean" },
-            "extensions": { "type": "array", "items": { "type": "string" } },
-            "event_sources": {
+            "remote": {
+              "type": "boolean"
+            },
+            "extensions": {
               "type": "array",
               "items": {
-                "type": "object",
-                "additionalProperties": false,
-                "required": ["adapter", "path"],
-                "properties": {
-                  "adapter": { "enum": ["normalized-v1", "principal-assurance-v1", "pi-daddy-v1", "pi-daddy-ledger-v3"] },
-                  "path": {
-                    "type": "string",
-                    "minLength": 1,
-                    "pattern": "^(?!/)(?![A-Za-z]:[\\\\/])(?!.*\\\\)(?!.*//)(?!.*(?:^|/)\\.\\.?(?:/|$)).+$"
-                  },
-                  "required": { "type": "boolean" }
-                }
+                "type": "string"
               }
             }
           }
@@ -203,12 +140,29 @@
           "type": "object",
           "additionalProperties": false,
           "properties": {
-            "vitest": { "type": "boolean" },
-            "diff_contains": { "type": "array", "items": { "type": "string", "minLength": 1 } },
-            "diff_excludes": { "type": "array", "items": { "type": "string", "minLength": 1 } },
-            "post_test": { "type": "string" },
-            "trace": { "type": "object" },
-            "trajectory": { "$ref": "#/$defs/trajectory" }
+            "vitest": {
+              "type": "boolean"
+            },
+            "diff_contains": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
+            "diff_excludes": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            },
+            "post_test": {
+              "type": "string"
+            },
+            "trace": {
+              "type": "object"
+            }
           }
         }
       }

package/dist/adjudication.d.ts

@@ -1,219 +0,0 @@
-import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
-import type { ScenarioResult, AdjudicationResult, Judgment, ResultsFile, CriterionVote } from "./results.js";
-import type { Scenario, ShipBar, Spec } from "./spec.js";
-import type { Verdict } from "./score.js";
-/**
- * Confidence-aware adjudication: decide which judged cells are untrustworthy
- * enough to be worth asking again, collapse the resulting votes, and project the
- * outcome into the fields the existing scorer already understands.
- *
- * Two rules govern everything here.
- *
- * **Spend is never implicit.** A spec may declare triggers, but only an explicit
- * `--auto-rejudge` (or a confirmed extension dialog) authorizes a second call.
- * `plan(...)` is a pure function that returns what WOULD be spent; nothing in
- * this module calls a judge.
- *
- * **Unresolved disagreement must not resolve itself.** When two judges disagree
- * and no third is authorized, the cell projects to `suspect: true` — which the
- * existing ship bar already treats as blocking. That is deliberate reuse rather
- * than a parallel gate: a second ship rule could drift out of step with the
- * first, and a disagreement that quietly became a PASS is the exact failure this
- * feature exists to prevent.
- */
-export type TriggerKind = "ambiguous" | "contradictory" | "non_unanimous" | "ship_deciding";
-/** Maximum judgments per cell, first wave included. Hard cap, not a default. */
-export declare const MAX_JUDGMENTS = 3;
-export interface CellState {
-    id: string;
-    /** First-wave verdict for the cell (already aggregated over reps). */
-    verdict: Verdict;
-    reason: string;
-    /** The judge misfired, or the cell is otherwise untrustworthy. */
-    suspect: boolean;
-    /** Per-rep verdicts, when the cell ran with reps. Empty for a single-rep cell. */
-    repVerdicts?: Verdict[];
-    criteria?: CriterionVote[];
-    /** Delivery prerequisite for schema-3 cells; non-PASS cells are never judged. */
-    deliveryStatus?: "PASS" | "NOT-MEASURED" | "ERROR";
-    /** The actual selected repetition's primary vote; aggregate fields remain for planning only. */
-    primaryJudgment?: Judgment;
-}
-export interface TriggerDecision {
-    id: string;
-    triggers: TriggerKind[];
-}
-export interface PlanInput {
-    cells: CellState[];
-    scenarios: Scenario[];
-    shipBar: ShipBar;
-    critical: string[];
-    /** Which trigger classes are enabled. Defaults to all four. */
-    enabled?: TriggerKind[];
-    /** A tie-break judge exists, so a third call is permitted on disagreement. */
-    tieBreakAvailable: boolean;
-}
-export interface AdjudicationPlan {
-    decisions: TriggerDecision[];
-    /** Cells that will be re-judged at least once. */
-    triggered: string[];
-    /**
-     * Triggered cells whose VERDICT this plan provably cannot settle.
-     *
-     * Their first-wave judgment misfired, so it is not a clean vote (see
-     * `collapseJudgments`). With no tie-break judge, one extra call reaches at most
-     * one clean vote and a collapse needs two: the cell returns `unresolved` and
-     * stays `suspect: true` whatever the second judge says.
-     *
-     * They are still re-judged, and deliberately so — the second opinion is the
-     * evidence an author reads to resolve the misfire by hand, which is the only
-     * way these cells ever get resolved. What was missing was saying so: the
-     * preflight offered them alongside cells a call could actually settle, so the
-     * buyer could not tell which was which. This list is that disclosure.
-     */
-    needsTieBreak: string[];
-    /**
-     * Exact upper bound on ADDITIONAL judge calls this plan can make.
-     *
-     * Deliberately a call count, not a cost estimate. The default judge runs on a
-     * Claude subscription and reports no per-call usage back to the harness, so a
-     * dollar figure there would be invented. Count is the only honest unit.
-     */
-    maxAdditionalCalls: number;
-}
-/**
- * Classify triggers and compute the spend ceiling. Pure — calls nothing.
- *
- * Triggers are computed from the COMPLETE first wave, never incrementally. A
- * `ship_deciding` cell can only be identified once every other cell's verdict is
- * known, and computing it mid-wave would make the trigger set depend on the order
- * scenarios happened to finish in.
- */
-export declare function planAdjudication(input: PlanInput): AdjudicationPlan;
-export type AdjudicationState = "confirmed" | "tie_broken" | "unresolved";
-/**
- * Collapse the judgments recorded for one cell.
- *
- * A judgment is a CLEAN vote only if it is a plain PASS or FAIL from a judge that
- * did not misfire. Ambiguous, suspect and unreadable judgments are recorded in
- * full but never counted — a malformed answer is not evidence, and letting one
- * cast a deciding vote would launder exactly the unreliability being measured.
- */
-export declare function collapseJudgments(judgments: Judgment[], trigger: TriggerKind): AdjudicationResult;
-/**
- * Project an adjudication onto the compatibility fields the scorer reads.
- *
- * `unresolved` becomes `suspect: true`, which the existing ship bar already
- * blocks on. The raw judgments survive on the result either way — an author
- * resolving this needs to see what each judge actually said, not just the
- * collapsed answer.
- */
-export declare function boundAdjudicationToRepetitions(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): AdjudicationResult;
-export declare function projectAdjudication(result: ScenarioResult, adj: AdjudicationResult): ScenarioResult;
-/** Apply the one-transcript bound and label it only when the bound changed the panel result. */
-export declare function projectAdjudicationForScenario(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): ScenarioResult;
-/** Ask one judge about one cell's saved transcript. Supplied by the caller. */
-export type RejudgeFn = (id: string, judge: ModelRef) => Promise<{
-    verdict: Verdict;
-    reason: string;
-    suspect: boolean;
-    criteria?: CriterionVote[];
-}>;
-export interface RunAdjudicationOptions {
-    plan: AdjudicationPlan;
-    cells: CellState[];
-    /** The judge that produced the first wave — recorded as judgment 1. */
-    primaryJudge: ModelRef;
-    secondaryJudge: ModelRef;
-    /** Absent means a disagreement stays unresolved rather than being tie-broken. */
-    tieBreakJudge?: ModelRef;
-    rejudge: RejudgeFn;
-    log?: (msg: string) => void;
-}
-export interface RunAdjudicationResult {
-    /** Per triggered cell, the collapsed adjudication. Cells that did not trigger are absent. */
-    byId: Map<string, AdjudicationResult>;
-    /** Judge calls actually made. Never exceeds `plan.maxAdditionalCalls`. */
-    callsMade: number;
-}
-/**
- * Execute an adjudication plan.
- *
- * ONE implementation, called from `run`, `grade` and the review server's rejudge
- * path. The plan's three collapse policies were the most likely thing to be
- * reimplemented three times and to disagree — a cell that shipped from the CLI
- * and blocked in the browser would be worse than no feature.
- *
- * The tie-break call is made only when the secondary genuinely disagreed. A third
- * opinion on a settled cell is spend with no decision attached to it.
- */
-export declare function runAdjudication(opts: RunAdjudicationOptions): Promise<RunAdjudicationResult>;
-export interface AdjudicateRunOptions {
-    runDir: string;
-    spec: Spec;
-    adapter: HarnessAdapter;
-    /** The first-wave results this adjudication refines. */
-    results: ResultsFile;
-    primaryJudge: ModelRef;
-    secondaryJudge: ModelRef;
-    tieBreakJudge?: ModelRef;
-    specDir: string;
-    now: () => string;
-    log?: (msg: string) => void;
-}
-/**
- * Adjudicate a completed run: plan, disclose the ceiling, execute, persist.
- *
- * Re-judges the SAME saved transcripts — no subject re-run, so the model's
- * behavior is held fixed and any movement is the judge. That is what makes this
- * a measurement of judge reliability rather than another sample of the model.
- */
-export declare function adjudicateRun(opts: AdjudicateRunOptions): Promise<ResultsFile>;
-/**
- * Build the plan's input cells from a run.
- *
- * EXPORTED so the CLI, the review server and the pi extension all price the work
- * exactly as `adjudicateRun` will perform it. When the preflight built cells
- * without `repVerdicts` and the executor built them with, the browser could show
- * "no cell triggered" and then spend on non-unanimous cells nobody was told
- * about — inverting the one invariant this feature actually promises.
- */
-export declare function cellsFromResults(runDir: string, results: ResultsFile): CellState[];
-/**
- * Resolve and validate the extra judges.
- *
- * Every configured judge passes the SAME two gates the primary does: the metered
- * refusal and the judge≠subject check. A secondary judge is still a judge — a
- * feature that multiplies judge calls is the last place to let one slip past the
- * policy that exists because a default once billed a corpus by accident.
- *
- * Returns null when adjudication was not authorized, so callers can treat "not
- * enabled" and "enabled with no triggers" as different things.
- */
-export declare function resolveAdjudicationJudges(opts: {
-    enabled: boolean;
-    primary: ModelRef;
-    secondaryToken?: string;
-    tieBreakToken?: string;
-    /**
-     * The run's recorded subject, as a raw token.
-     *
-     * A TOKEN rather than a parsed ref, and parsed only after the `enabled` check:
-     * an eagerly-parsed argument threw on any run whose recorded model is not
-     * `provider:model` — which killed the whole regrade over a provenance oddity in
-     * a field only used for a warning.
-     */
-    subjectToken: string;
-    parseRef: (token: string) => ModelRef;
-    assertAllowed: (judge: ModelRef, source: string) => void;
-    resemblesSubject: (judge: ModelRef, subject: ModelRef) => boolean;
-    warn: (msg: string) => void;
-}): {
-    secondary: ModelRef;
-    tieBreak?: ModelRef;
-} | null;
-/** Human-readable preflight line. Counts, never dollars — see `maxAdditionalCalls`. */
-export declare function formatAdjudicationPlan(plan: AdjudicationPlan, judges: {
-    secondary: ModelRef;
-    tieBreak?: ModelRef;
-}): string;