@skill-harness/core 0.11.0 → 0.13.0

package/dist/factory-calibration.d.ts

@@ -0,0 +1,63 @@
+export interface CalibrationComponent {
+    kind: "detector" | "adjudicator";
+    id: string;
+    version: string;
+    population: string;
+}
+export type CalibrationKind = "positive" | "approval" | "rejection";
+export type CalibrationSplit = "calibration" | "heldout" | "tuning";
+export interface CalibrationPrediction {
+    id: string;
+    incidentId: string;
+    component: CalibrationComponent;
+    kind: CalibrationKind;
+    split: CalibrationSplit;
+}
+/** Independently established host context, not fields accepted from predictions or panel agreement. */
+export interface CalibrationOutcome extends CalibrationPrediction {
+    correct: boolean;
+}
+export interface CalibrationReport {
+    component: CalibrationComponent;
+    kind: CalibrationKind;
+    split: Exclude<CalibrationSplit, "tuning">;
+    correct: number;
+    incorrect: number;
+    resolved: number;
+    unresolved: number;
+    conflicted: number;
+    totalIncidents: number;
+    precision: number | null;
+    interval: {
+        lower: number;
+        upper: number;
+    } | null;
+    advisoryOnly: true;
+}
+/** Version/population/conditioning-specific statistics; no policy, grants, routing or adoption is changed. */
+export declare function calibratePredictions(predictions: readonly CalibrationPrediction[], context?: {
+    outcomes: readonly CalibrationOutcome[];
+}): {
+    version: "factory-calibration-v1";
+    reports: CalibrationReport[];
+    excludedTuning: number;
+    referenceBasis: "independently-supplied-host-context";
+    advisoryOnly: true;
+};
+export interface ExposurePolicy {
+    id: string;
+    component: CalibrationComponent;
+    kind: CalibrationKind;
+    split: "calibration" | "heldout";
+    minimumResolved: number;
+    minimumLowerBound: number;
+    attentionRemaining: number;
+    expiresAt: number;
+    retireBelowUpperBound?: number;
+}
+/** A selected pre-established host policy may produce advice; this never applies it or expands authority. */
+export declare function recommendExposure(report: CalibrationReport, policy: ExposurePolicy | null, now: number): {
+    mode: "silent" | "ask" | "retire";
+    reason: string;
+    advisoryOnly: true;
+};

package/dist/factory-calibration.js

@@ -0,0 +1,115 @@
+const computedReports = new WeakSet();
+const text = (x) => typeof x === "string" && x.length > 0 && x.length <= 512 && !/[\u0000-\u001f\u007f]/.test(x);
+function closed(x, fields) {
+    return x !== null && typeof x === "object" && !Array.isArray(x) && Object.keys(x).length === fields.length && Object.keys(x).every(k => fields.includes(k));
+}
+function valid(x, outcome = false) {
+    return closed(x, outcome ? ["id", "incidentId", "component", "kind", "split", "correct"] : ["id", "incidentId", "component", "kind", "split"])
+        && text(x.id) && text(x.incidentId) && closed(x.component, ["kind", "id", "version", "population"])
+        && [x.component.id, x.component.version, x.component.population].every(text)
+        && ["calibration", "heldout", "tuning"].includes(x.split)
+        && (x.component.kind === "detector" ? x.kind === "positive" : x.component.kind === "adjudicator" && ["approval", "rejection"].includes(x.kind))
+        && (!outcome || typeof x.correct === "boolean");
+}
+const key = (x) => JSON.stringify([x.component.kind, x.component.id, x.component.version, x.component.population, x.kind, x.split]);
+function interval(correct, count) {
+    const z = 1.96, z2 = z * z, p = correct / count, denominator = 1 + z2 / count;
+    const center = (p + z2 / (2 * count)) / denominator;
+    const half = z * Math.sqrt(p * (1 - p) / count + z2 / (4 * count * count)) / denominator;
+    return { lower: Math.max(0, center - half), upper: Math.min(1, center + half) };
+}
+/** Version/population/conditioning-specific statistics; no policy, grants, routing or adoption is changed. */
+export function calibratePredictions(predictions, context = { outcomes: [] }) {
+    if (predictions.length > 10000 || context.outcomes.length > 10000 || predictions.some(p => !valid(p)) || context.outcomes.some(r => !valid(r, true)))
+        throw new Error("invalid calibration input");
+    const byId = new Map();
+    for (const p of predictions) {
+        const previous = byId.get(p.id);
+        if (previous && (key(previous) !== key(p) || previous.incidentId !== p.incidentId))
+            throw new Error("prediction identity conflict");
+        byId.set(p.id, p);
+    }
+    const groups = new Map();
+    let excludedTuning = 0;
+    for (const p of byId.values()) {
+        if (p.split === "tuning") {
+            excludedTuning++;
+            continue;
+        }
+        const k = key(p);
+        groups.set(k, [...(groups.get(k) ?? []), p]);
+    }
+    const referenceIds = new Map();
+    const conflictingIds = new Set();
+    for (const r of context.outcomes) {
+        const signature = JSON.stringify([key(r), r.incidentId, r.correct]);
+        if (referenceIds.has(r.id) && referenceIds.get(r.id) !== signature)
+            conflictingIds.add(r.id);
+        referenceIds.set(r.id, signature);
+    }
+    const labels = new Map();
+    const conflictedIncidents = new Set();
+    for (const r of context.outcomes) {
+        const k = JSON.stringify([key(r), r.incidentId]);
+        if (conflictingIds.has(r.id)) {
+            conflictedIncidents.add(k);
+            continue;
+        }
+        const values = labels.get(k) ?? new Set();
+        values.add(r.correct);
+        labels.set(k, values);
+    }
+    const reports = [...groups].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([groupKey, group]) => {
+        const incidents = new Set(group.map(p => p.incidentId));
+        let correct = 0, incorrect = 0, unresolved = 0, conflicted = 0;
+        for (const incident of incidents) {
+            const incidentKey = JSON.stringify([groupKey, incident]);
+            const outcomes = labels.get(incidentKey) ?? new Set();
+            if (outcomes.size !== 1 || conflictedIncidents.has(incidentKey)) {
+                unresolved++;
+                if (outcomes.size > 1 || conflictedIncidents.has(incidentKey))
+                    conflicted++;
+            }
+            else if (outcomes.has(true))
+                correct++;
+            else
+                incorrect++;
+        }
+        const resolved = correct + incorrect;
+        return { component: { ...group[0].component }, kind: group[0].kind, split: group[0].split,
+            correct, incorrect, resolved, unresolved, conflicted, totalIncidents: incidents.size,
+            precision: resolved ? correct / resolved : null, interval: resolved ? interval(correct, resolved) : null, advisoryOnly: true };
+    });
+    for (const report of reports) {
+        Object.freeze(report.component);
+        if (report.interval)
+            Object.freeze(report.interval);
+        Object.freeze(report);
+        computedReports.add(report);
+    }
+    return { version: "factory-calibration-v1", reports, excludedTuning, referenceBasis: "independently-supplied-host-context", advisoryOnly: true };
+}
+/** A selected pre-established host policy may produce advice; this never applies it or expands authority. */
+export function recommendExposure(report, policy, now) {
+    if (!computedReports.has(report))
+        throw new Error("recompute calibration from predictions and independent reference context before evaluating exposure");
+    const result = (mode, reason) => ({ mode, reason, advisoryOnly: true });
+    if (!policy)
+        return result("silent", "policy-unavailable");
+    if (!text(policy.id) || !Number.isSafeInteger(policy.minimumResolved) || policy.minimumResolved < 1
+        || !Number.isSafeInteger(policy.attentionRemaining) || policy.attentionRemaining < 0
+        || !Number.isFinite(policy.minimumLowerBound) || policy.minimumLowerBound < 0 || policy.minimumLowerBound > 1
+        || !Number.isFinite(policy.expiresAt) || !Number.isFinite(now)
+        || (policy.retireBelowUpperBound !== undefined && (!Number.isFinite(policy.retireBelowUpperBound) || policy.retireBelowUpperBound < 0 || policy.retireBelowUpperBound > 1)))
+        throw new Error("invalid exposure policy");
+    if (key(policy) !== key(report) || now >= policy.expiresAt)
+        return result("silent", "policy-inapplicable-or-expired");
+    if (!report.interval || report.resolved < policy.minimumResolved)
+        return result("silent", "insufficient-independent-evidence");
+    if (policy.retireBelowUpperBound !== undefined && report.interval.upper < policy.retireBelowUpperBound)
+        return result("retire", "below-selected-reliability-policy");
+    if (report.interval.lower < policy.minimumLowerBound || policy.attentionRemaining === 0)
+        return result("silent", "confidence-or-attention-bound");
+    return result("ask", "selected-evidence-and-attention-policy-met");
+}
+//# sourceMappingURL=factory-calibration.js.map

package/dist/grade.d.ts

@@ -1,6 +1,7 @@
 import type { Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
 import type { Verdict } from "./score.js";
+import { type CriterionVote } from "./results.js";
 export interface JudgePromptInput {
     skill: string;
     persona: string;
@@ -36,6 +37,7 @@ export declare function parseVerdict(out: string): ParsedVerdict;
 export declare function judgeResemblesSubject(judge: ModelRef, subject: ModelRef): boolean;
 export interface GradeResult extends ParsedVerdict {
     raw: string;
+    criteria: CriterionVote[];
     /** Judge misfire: the overall verdict disagrees with AND(per-item grades). Recorded, never auto-passed; blocks SHIP until re-judged or overridden. */
     suspect: boolean;
 }

package/dist/grade.js

@@ -1,4 +1,5 @@
 import { createWorkspace } from "./workspace.js";
+import { parseCriterionVotes } from "./results.js";
 /** The heading runSeeded writes above the staged diff. Gating on this exact string is what keeps the guidance honest — see below. */
 export const STAGED_DIFF_HEADING = "=== STAGED DIFF ===";
 /**
@@ -133,7 +134,7 @@ export async function gradeTranscript(adapter, judge, prompt, cwd) {
             parsed.reason = `judge unparseable: ${snippet}`;
     }
     const suspect = detectMisfire(raw, parsed.verdict);
-    return { ...parsed, raw, suspect };
+    return { ...parsed, raw, suspect, criteria: parseCriterionVotes(raw) };
 }
 /**
  * Grade a transcript in a fresh, isolated, throwaway workspace — never the

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+export * from "./trusted-host-supervision.js";
 export * from "./spec.js";
 export * from "./discover.js";
 export * from "./run.js";
@@ -23,12 +24,24 @@ export * from "./util/env.js";
 export * from "./scaffold.js";
 export * from "./version.js";
 export * from "./defaults.js";
+export * from "./command-cost.js";
 export * from "./judge-policy.js";
+export * from "./judge-agreement.js";
 export * from "./regate.js";
+export * from "./screen.js";
+export * from "./prompt-normalization.js";
 export * from "./downgrade.js";
 export * from "./canary.js";
 export * from "./stability.js";
 export * from "./capture-trace-types.js";
+export * from "./work-capture.js";
+export * from "./work-signals.js";
+export * from "./work-capture-schema.js";
+export * from "./factory-calibration.js";
+export * from "./adoption.js";
+export * from "./investigation.js";
+export * from "./intervention.js";
+export * from "./intervention-results.js";
 export * from "./spec-write.js";
 export * from "./capture.js";
 export * from "./execution-trace.js";
@@ -37,7 +50,15 @@ export * from "./trajectory-gates.js";
 export * from "./instruction-coverage.js";
 export * from "./affected.js";
 export * from "./adjudication.js";
+export * from "./vote-panel.js";
+export * from "./qualification-panels.js";
+export * from "./qualification-panel-store.js";
+export * from "./qualification-validation.js";
 export * from "./comparison.js";
 export * from "./sandbox.js";
 export * from "./provider-failure.js";
 export * from "./arms.js";
+export * from "./qualification-config.js";
+export * from "./qualification-oauth-directory.js";
+export * from "./qualification-store.js";
+export * from "./qualification-runner.js";

package/dist/index.js

@@ -1,3 +1,4 @@
+export * from "./trusted-host-supervision.js";
 export * from "./spec.js";
 export * from "./discover.js";
 export * from "./run.js";
@@ -23,12 +24,24 @@ export * from "./util/env.js";
 export * from "./scaffold.js";
 export * from "./version.js";
 export * from "./defaults.js";
+export * from "./command-cost.js";
 export * from "./judge-policy.js";
+export * from "./judge-agreement.js";
 export * from "./regate.js";
+export * from "./screen.js";
+export * from "./prompt-normalization.js";
 export * from "./downgrade.js";
 export * from "./canary.js";
 export * from "./stability.js";
 export * from "./capture-trace-types.js";
+export * from "./work-capture.js";
+export * from "./work-signals.js";
+export * from "./work-capture-schema.js";
+export * from "./factory-calibration.js";
+export * from "./adoption.js";
+export * from "./investigation.js";
+export * from "./intervention.js";
+export * from "./intervention-results.js";
 export * from "./spec-write.js";
 export * from "./capture.js";
 export * from "./execution-trace.js";
@@ -37,8 +50,16 @@ export * from "./trajectory-gates.js";
 export * from "./instruction-coverage.js";
 export * from "./affected.js";
 export * from "./adjudication.js";
+export * from "./vote-panel.js";
+export * from "./qualification-panels.js";
+export * from "./qualification-panel-store.js";
+export * from "./qualification-validation.js";
 export * from "./comparison.js";
 export * from "./sandbox.js";
 export * from "./provider-failure.js";
 export * from "./arms.js";
+export * from "./qualification-config.js";
+export * from "./qualification-oauth-directory.js";
+export * from "./qualification-store.js";
+export * from "./qualification-runner.js";
 //# sourceMappingURL=index.js.map

package/dist/intervention-results.d.ts

@@ -0,0 +1,6 @@
+import type { InterventionManifest, InterventionEvidence } from "./intervention.js";
+/** Normalize existing validated schema3 records, retaining the complete source votes alongside the summary. */
+export declare function interventionEvidenceFromResults(manifest: InterventionManifest, armId: string, raw: unknown, outputBytes: ReadonlyMap<string, Uint8Array>): {
+    evidence: InterventionEvidence;
+    sourceResults: import("./results.js").ResultsFile;
+};

package/dist/intervention-results.js

@@ -0,0 +1,52 @@
+import { createHash } from "node:crypto";
+import { validateResults } from "./results.js";
+import { aggregateMetrics } from "./comparison.js";
+/** Normalize existing validated schema3 records, retaining the complete source votes alongside the summary. */
+export function interventionEvidenceFromResults(manifest, armId, raw, outputBytes) {
+    const sourceResults = validateResults(raw), arm = manifest.arms.find(a => a.id === armId);
+    if (!arm || sourceResults.model !== arm.configuration.model)
+        throw new Error("intervention result model mismatch");
+    if (sourceResults.schema !== 3 || sourceResults.partial || sourceResults.mode !== manifest.common.mode
+        || `${sourceResults.judge.provider}:${sourceResults.judge.model}` !== manifest.judge)
+        throw new Error("incomplete or incompatible intervention results");
+    if (sourceResults.source_hashes?.["intervention:manifest"] !== manifest.id || sourceResults.source_hashes?.["intervention:inputs"] !== manifest.inputDigest)
+        throw new Error("recorded frozen intervention binding missing; never stamp historical results");
+    if (sourceResults.scenarios.length !== manifest.cases.length || sourceResults.scenarios.some(s => !manifest.cases.some(c => c.id === s.id)))
+        throw new Error("intervention scenario plan mismatch");
+    const cells = [], artifacts = new Set();
+    for (const spec of manifest.cases) {
+        const scenario = sourceResults.scenarios.find(s => s.id === spec.id);
+        if (scenario.criterion_count !== spec.criteria || (scenario.reps ?? 1) !== spec.reps || (scenario.pass_threshold ?? 1) !== spec.threshold)
+            throw new Error("intervention repetition/criterion policy mismatch");
+        for (const panel of scenario.rep_judgments ?? []) {
+            const delivery = sourceResults.subject_invocations.find(o => o.scenario_id === spec.id && o.repetition === panel.repetition)?.prompt?.status;
+            if (!delivery || !panel.objective)
+                throw new Error("intervention objective/delivery evidence missing");
+            const objective = panel.objective.status;
+            const bytes = outputBytes.get(`${spec.id}:${panel.repetition}`);
+            if (bytes !== undefined && (!(bytes instanceof Uint8Array) || bytes.byteLength > 8 * 1024 * 1024))
+                throw new Error("intervention output exceeds bounds");
+            const artifactSha256 = bytes ? createHash("sha256").update(bytes).digest("hex") : null;
+            if (artifactSha256)
+                artifacts.add(artifactSha256);
+            let criteria = [], suspect = false;
+            if (delivery === "PASS" && objective === "PASS") {
+                if (panel.judgments.length !== 1)
+                    throw new Error("multi-judge normalization not yet supported; retain original panel without selecting a convenient vote");
+                const judgment = panel.judgments[0];
+                if (`${judgment.judge.provider}:${judgment.judge.model}` !== manifest.judge)
+                    throw new Error("intervention cell judge mismatch");
+                criteria = [...(judgment.criteria ?? [])].sort((a, b) => a.index - b.index).map(c => c.verdict === "ERROR" ? "UNKNOWN" : c.verdict);
+                suspect = judgment.suspect || scenario.suspect;
+            }
+            cells.push({ caseId: spec.id, repetition: panel.repetition, delivery, objective, criteria, suspect, artifactSha256 });
+        }
+    }
+    const metrics = aggregateMetrics(sourceResults.scenarios);
+    const cost = manifest.resourceMetric === "wall_ms" ? (sourceResults.scenarios.every(s => s.metrics) ? metrics.wall_time_ms : null)
+        : manifest.resourceMetric === "tool_calls" ? metrics.tool_calls
+            : metrics.cost_source === "provider-reported" && metrics.subject_metrics_reps === metrics.total_reps ? metrics.subject_cost_usd : null;
+    const evidence = { armId, inputDigest: manifest.inputDigest, artifactDigests: [...artifacts].sort(), cells, cost, costUnit: manifest.resourceMetric };
+    return { evidence, sourceResults };
+}
+//# sourceMappingURL=intervention-results.js.map

package/dist/intervention.d.ts

@@ -0,0 +1,123 @@
+/** Plain, bounded JSON encoding for retained intervention inputs; not semantic validation. */
+export declare function interventionCanonicalJson(value: unknown): string;
+export type InterventionAxis = "model" | "effort" | "skill" | "prompt" | "configuration";
+export interface InterventionDraft {
+    family: "intervention";
+    investigationSha256: string;
+    resourceMetric: "usd" | "wall_ms" | "tool_calls";
+    axes: InterventionAxis[];
+    common: {
+        mode: "green" | "force";
+        scenarioSha256: string;
+        rubricSha256: string;
+        fixtureSha256: string;
+        heldoutSha256: string;
+        harnessSha256: string;
+        judgePolicySha256: string;
+    };
+    proposer: string;
+    judge: string;
+    cases: Array<{
+        id: string;
+        criteria: number;
+        reps: number;
+        threshold: number;
+        critical: boolean;
+    }>;
+    arms: Array<{
+        id: string;
+        configuration: Record<InterventionAxis, string>;
+    }>;
+}
+export interface InterventionManifest extends InterventionDraft {
+    version: "intervention-comparison-v1";
+    id: string;
+    inputDigest: string;
+    changedAxes: InterventionAxis[];
+    deterministicSampling: false;
+}
+export declare function freezeIntervention(input: InterventionDraft): InterventionManifest;
+export interface InterventionCell {
+    caseId: string;
+    repetition: number;
+    delivery: "PASS" | "NOT-MEASURED" | "ERROR";
+    objective: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";
+    criteria: Array<"PASS" | "FAIL" | "UNKNOWN">;
+    suspect: boolean;
+    artifactSha256: string | null;
+}
+export interface InterventionEvidence {
+    armId: string;
+    inputDigest: string;
+    artifactDigests: string[];
+    cells: InterventionCell[];
+    cost: number | null;
+    costUnit: string;
+}
+export declare const interventionEvidenceDigest: (value: InterventionEvidence) => string;
+export interface InterventionQualification {
+    manifestId: string;
+    proposer: {
+        requested: string;
+        canonical: string;
+    };
+    judge: {
+        requested: string;
+        canonical: string;
+    };
+    subjects: Record<string, {
+        requested: string;
+        canonical: string;
+    }>;
+    evidenceDigests: Record<string, string>;
+    /** Actual independently retained output bytes, not an availability flag supplied by an arm. */
+    artifacts: ReadonlyMap<string, Uint8Array>;
+}
+export interface InterventionAssessment {
+    manifestId: string;
+    complete: boolean;
+    arms: Array<{
+        armId: string;
+        state: "MEASURED" | "FAILED" | "MISSING" | "ERROR" | "NOT-MEASURED" | "UNRESOLVED";
+        eligible: boolean;
+        artifactDigests: string[];
+        cost: number | null;
+        costUnit: string;
+    }>;
+    cheapestEligible: string | null;
+    routingDefault: null;
+    adoptionAuthorized: false;
+}
+/** Pre-spend role gate: consumes independently resolved host facts, never guesses aliases or calls a provider. */
+export declare function assertInterventionRoles(manifest: InterventionManifest, qualification: InterventionQualification | null): asserts qualification is InterventionQualification;
+/** Inspects separately host-qualified evidence; does not obtain qualification or execute a model. */
+export declare function assessIntervention(manifest: InterventionManifest, evidence: readonly InterventionEvidence[], qualification: InterventionQualification | null): InterventionAssessment;
+export type BlindQualityChoice = {
+    kind: "one" | "tie" | "none" | "insufficient";
+    labels: string[];
+};
+/** Only this view omits identity/cost; arbitrary artifact content can still leak clues and must be reviewed. */
+export declare function createBlindComparison(manifest: InterventionManifest, assessment: InterventionAssessment, fixtureOrPersistedSeed?: string): Readonly<{
+    view: () => {
+        version: string;
+        cards: {
+            label: string;
+            artifactDigests: string[];
+        }[];
+        limitations: string[];
+    };
+    readArtifact(label: string, hash: string): Buffer<ArrayBuffer>;
+    choose(input: BlindQualityChoice): BlindQualityChoice;
+    reveal(): {
+        manifestId: string;
+        choice: BlindQualityChoice;
+        arms: {
+            label: string;
+            armId: string;
+            configuration: Record<InterventionAxis, string>;
+            cost: number | null;
+            costUnit: string;
+        }[];
+        routingDefault: null;
+    };
+}>;