@skill-harness/core 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +1 -2
  2. package/dist/adapters/types.d.ts +0 -12
  3. package/dist/capture-trace-types.d.ts +0 -96
  4. package/dist/capture-trace-types.js +0 -4
  5. package/dist/command-cost.d.ts +1 -1
  6. package/dist/command-cost.js +0 -5
  7. package/dist/execution-trace.js +1 -1
  8. package/dist/index.d.ts +3 -23
  9. package/dist/index.js +3 -23
  10. package/dist/instruction-coverage.d.ts +0 -7
  11. package/dist/instruction-coverage.js +5 -7
  12. package/dist/journal.d.ts +0 -8
  13. package/dist/metrics.d.ts +18 -0
  14. package/dist/metrics.js +28 -0
  15. package/dist/redaction.d.ts +6 -0
  16. package/dist/redaction.js +55 -0
  17. package/dist/regate.js +6 -71
  18. package/dist/regrade.d.ts +1 -3
  19. package/dist/regrade.js +1 -21
  20. package/dist/report.d.ts +1 -15
  21. package/dist/report.js +1 -13
  22. package/dist/reps.d.ts +1 -2
  23. package/dist/reps.js +0 -2
  24. package/dist/results.d.ts +0 -9
  25. package/dist/results.js +2 -19
  26. package/dist/run.js +11 -72
  27. package/dist/seeded.d.ts +1 -7
  28. package/dist/seeded.js +5 -11
  29. package/dist/sources.js +4 -9
  30. package/dist/spec.d.ts +0 -13
  31. package/dist/spec.js +6 -48
  32. package/dist/trajectory-events.d.ts +58 -0
  33. package/dist/trajectory-events.js +119 -0
  34. package/package.json +1 -1
  35. package/schemas/results-v2.schema.json +376 -75
  36. package/schemas/specification-v1.schema.json +128 -174
  37. package/dist/adjudication.d.ts +0 -219
  38. package/dist/adjudication.js +0 -422
  39. package/dist/adoption.d.ts +0 -128
  40. package/dist/adoption.js +0 -102
  41. package/dist/affected.d.ts +0 -88
  42. package/dist/affected.js +0 -222
  43. package/dist/capture.d.ts +0 -193
  44. package/dist/capture.js +0 -344
  45. package/dist/comparison.d.ts +0 -97
  46. package/dist/comparison.js +0 -189
  47. package/dist/factory-calibration.d.ts +0 -63
  48. package/dist/factory-calibration.js +0 -115
  49. package/dist/intervention-results.d.ts +0 -6
  50. package/dist/intervention-results.js +0 -52
  51. package/dist/intervention.d.ts +0 -123
  52. package/dist/intervention.js +0 -218
  53. package/dist/investigation.d.ts +0 -94
  54. package/dist/investigation.js +0 -148
  55. package/dist/judge-agreement.d.ts +0 -19
  56. package/dist/judge-agreement.js +0 -31
  57. package/dist/qualification-capture.d.ts +0 -44
  58. package/dist/qualification-capture.js +0 -163
  59. package/dist/qualification-config.d.ts +0 -233
  60. package/dist/qualification-config.js +0 -694
  61. package/dist/qualification-lock.d.ts +0 -15
  62. package/dist/qualification-lock.js +0 -87
  63. package/dist/qualification-oauth-directory.d.ts +0 -79
  64. package/dist/qualification-oauth-directory.js +0 -444
  65. package/dist/qualification-panel-store.d.ts +0 -14
  66. package/dist/qualification-panel-store.js +0 -188
  67. package/dist/qualification-panels.d.ts +0 -99
  68. package/dist/qualification-panels.js +0 -91
  69. package/dist/qualification-process.d.ts +0 -13
  70. package/dist/qualification-process.js +0 -190
  71. package/dist/qualification-runner.d.ts +0 -203
  72. package/dist/qualification-runner.js +0 -2037
  73. package/dist/qualification-store.d.ts +0 -187
  74. package/dist/qualification-store.js +0 -833
  75. package/dist/qualification-validation.d.ts +0 -12
  76. package/dist/qualification-validation.js +0 -8
  77. package/dist/screen.d.ts +0 -31
  78. package/dist/screen.js +0 -105
  79. package/dist/trajectory-gates.d.ts +0 -137
  80. package/dist/trajectory-gates.js +0 -576
  81. package/dist/trusted-host-supervision.d.ts +0 -25
  82. package/dist/trusted-host-supervision.js +0 -94
  83. package/dist/work-capture-schema.d.ts +0 -38
  84. package/dist/work-capture-schema.js +0 -33
  85. package/dist/work-capture.d.ts +0 -43
  86. package/dist/work-capture.js +0 -81
  87. package/dist/work-signals.d.ts +0 -55
  88. package/dist/work-signals.js +0 -71
  89. package/schemas/qualification-cell-result-v1.schema.json +0 -32
  90. package/schemas/qualification-config-v1.schema.json +0 -244
  91. package/schemas/qualification-invocation-request-v1.schema.json +0 -59
  92. package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
  93. package/schemas/trajectory-event-v1.schema.json +0 -80
@@ -1,63 +0,0 @@
1
- export interface CalibrationComponent {
2
- kind: "detector" | "adjudicator";
3
- id: string;
4
- version: string;
5
- population: string;
6
- }
7
- export type CalibrationKind = "positive" | "approval" | "rejection";
8
- export type CalibrationSplit = "calibration" | "heldout" | "tuning";
9
- export interface CalibrationPrediction {
10
- id: string;
11
- incidentId: string;
12
- component: CalibrationComponent;
13
- kind: CalibrationKind;
14
- split: CalibrationSplit;
15
- }
16
- /** Independently established host context, not fields accepted from predictions or panel agreement. */
17
- export interface CalibrationOutcome extends CalibrationPrediction {
18
- correct: boolean;
19
- }
20
- export interface CalibrationReport {
21
- component: CalibrationComponent;
22
- kind: CalibrationKind;
23
- split: Exclude<CalibrationSplit, "tuning">;
24
- correct: number;
25
- incorrect: number;
26
- resolved: number;
27
- unresolved: number;
28
- conflicted: number;
29
- totalIncidents: number;
30
- precision: number | null;
31
- interval: {
32
- lower: number;
33
- upper: number;
34
- } | null;
35
- advisoryOnly: true;
36
- }
37
- /** Version/population/conditioning-specific statistics; no policy, grants, routing or adoption is changed. */
38
- export declare function calibratePredictions(predictions: readonly CalibrationPrediction[], context?: {
39
- outcomes: readonly CalibrationOutcome[];
40
- }): {
41
- version: "factory-calibration-v1";
42
- reports: CalibrationReport[];
43
- excludedTuning: number;
44
- referenceBasis: "independently-supplied-host-context";
45
- advisoryOnly: true;
46
- };
47
- export interface ExposurePolicy {
48
- id: string;
49
- component: CalibrationComponent;
50
- kind: CalibrationKind;
51
- split: "calibration" | "heldout";
52
- minimumResolved: number;
53
- minimumLowerBound: number;
54
- attentionRemaining: number;
55
- expiresAt: number;
56
- retireBelowUpperBound?: number;
57
- }
58
- /** A selected pre-established host policy may produce advice; this never applies it or expands authority. */
59
- export declare function recommendExposure(report: CalibrationReport, policy: ExposurePolicy | null, now: number): {
60
- mode: "silent" | "ask" | "retire";
61
- reason: string;
62
- advisoryOnly: true;
63
- };
@@ -1,115 +0,0 @@
1
- const computedReports = new WeakSet();
2
- const text = (x) => typeof x === "string" && x.length > 0 && x.length <= 512 && !/[\u0000-\u001f\u007f]/.test(x);
3
- function closed(x, fields) {
4
- return x !== null && typeof x === "object" && !Array.isArray(x) && Object.keys(x).length === fields.length && Object.keys(x).every(k => fields.includes(k));
5
- }
6
- function valid(x, outcome = false) {
7
- return closed(x, outcome ? ["id", "incidentId", "component", "kind", "split", "correct"] : ["id", "incidentId", "component", "kind", "split"])
8
- && text(x.id) && text(x.incidentId) && closed(x.component, ["kind", "id", "version", "population"])
9
- && [x.component.id, x.component.version, x.component.population].every(text)
10
- && ["calibration", "heldout", "tuning"].includes(x.split)
11
- && (x.component.kind === "detector" ? x.kind === "positive" : x.component.kind === "adjudicator" && ["approval", "rejection"].includes(x.kind))
12
- && (!outcome || typeof x.correct === "boolean");
13
- }
14
- const key = (x) => JSON.stringify([x.component.kind, x.component.id, x.component.version, x.component.population, x.kind, x.split]);
15
- function interval(correct, count) {
16
- const z = 1.96, z2 = z * z, p = correct / count, denominator = 1 + z2 / count;
17
- const center = (p + z2 / (2 * count)) / denominator;
18
- const half = z * Math.sqrt(p * (1 - p) / count + z2 / (4 * count * count)) / denominator;
19
- return { lower: Math.max(0, center - half), upper: Math.min(1, center + half) };
20
- }
21
- /** Version/population/conditioning-specific statistics; no policy, grants, routing or adoption is changed. */
22
- export function calibratePredictions(predictions, context = { outcomes: [] }) {
23
- if (predictions.length > 10000 || context.outcomes.length > 10000 || predictions.some(p => !valid(p)) || context.outcomes.some(r => !valid(r, true)))
24
- throw new Error("invalid calibration input");
25
- const byId = new Map();
26
- for (const p of predictions) {
27
- const previous = byId.get(p.id);
28
- if (previous && (key(previous) !== key(p) || previous.incidentId !== p.incidentId))
29
- throw new Error("prediction identity conflict");
30
- byId.set(p.id, p);
31
- }
32
- const groups = new Map();
33
- let excludedTuning = 0;
34
- for (const p of byId.values()) {
35
- if (p.split === "tuning") {
36
- excludedTuning++;
37
- continue;
38
- }
39
- const k = key(p);
40
- groups.set(k, [...(groups.get(k) ?? []), p]);
41
- }
42
- const referenceIds = new Map();
43
- const conflictingIds = new Set();
44
- for (const r of context.outcomes) {
45
- const signature = JSON.stringify([key(r), r.incidentId, r.correct]);
46
- if (referenceIds.has(r.id) && referenceIds.get(r.id) !== signature)
47
- conflictingIds.add(r.id);
48
- referenceIds.set(r.id, signature);
49
- }
50
- const labels = new Map();
51
- const conflictedIncidents = new Set();
52
- for (const r of context.outcomes) {
53
- const k = JSON.stringify([key(r), r.incidentId]);
54
- if (conflictingIds.has(r.id)) {
55
- conflictedIncidents.add(k);
56
- continue;
57
- }
58
- const values = labels.get(k) ?? new Set();
59
- values.add(r.correct);
60
- labels.set(k, values);
61
- }
62
- const reports = [...groups].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([groupKey, group]) => {
63
- const incidents = new Set(group.map(p => p.incidentId));
64
- let correct = 0, incorrect = 0, unresolved = 0, conflicted = 0;
65
- for (const incident of incidents) {
66
- const incidentKey = JSON.stringify([groupKey, incident]);
67
- const outcomes = labels.get(incidentKey) ?? new Set();
68
- if (outcomes.size !== 1 || conflictedIncidents.has(incidentKey)) {
69
- unresolved++;
70
- if (outcomes.size > 1 || conflictedIncidents.has(incidentKey))
71
- conflicted++;
72
- }
73
- else if (outcomes.has(true))
74
- correct++;
75
- else
76
- incorrect++;
77
- }
78
- const resolved = correct + incorrect;
79
- return { component: { ...group[0].component }, kind: group[0].kind, split: group[0].split,
80
- correct, incorrect, resolved, unresolved, conflicted, totalIncidents: incidents.size,
81
- precision: resolved ? correct / resolved : null, interval: resolved ? interval(correct, resolved) : null, advisoryOnly: true };
82
- });
83
- for (const report of reports) {
84
- Object.freeze(report.component);
85
- if (report.interval)
86
- Object.freeze(report.interval);
87
- Object.freeze(report);
88
- computedReports.add(report);
89
- }
90
- return { version: "factory-calibration-v1", reports, excludedTuning, referenceBasis: "independently-supplied-host-context", advisoryOnly: true };
91
- }
92
- /** A selected pre-established host policy may produce advice; this never applies it or expands authority. */
93
- export function recommendExposure(report, policy, now) {
94
- if (!computedReports.has(report))
95
- throw new Error("recompute calibration from predictions and independent reference context before evaluating exposure");
96
- const result = (mode, reason) => ({ mode, reason, advisoryOnly: true });
97
- if (!policy)
98
- return result("silent", "policy-unavailable");
99
- if (!text(policy.id) || !Number.isSafeInteger(policy.minimumResolved) || policy.minimumResolved < 1
100
- || !Number.isSafeInteger(policy.attentionRemaining) || policy.attentionRemaining < 0
101
- || !Number.isFinite(policy.minimumLowerBound) || policy.minimumLowerBound < 0 || policy.minimumLowerBound > 1
102
- || !Number.isFinite(policy.expiresAt) || !Number.isFinite(now)
103
- || (policy.retireBelowUpperBound !== undefined && (!Number.isFinite(policy.retireBelowUpperBound) || policy.retireBelowUpperBound < 0 || policy.retireBelowUpperBound > 1)))
104
- throw new Error("invalid exposure policy");
105
- if (key(policy) !== key(report) || now >= policy.expiresAt)
106
- return result("silent", "policy-inapplicable-or-expired");
107
- if (!report.interval || report.resolved < policy.minimumResolved)
108
- return result("silent", "insufficient-independent-evidence");
109
- if (policy.retireBelowUpperBound !== undefined && report.interval.upper < policy.retireBelowUpperBound)
110
- return result("retire", "below-selected-reliability-policy");
111
- if (report.interval.lower < policy.minimumLowerBound || policy.attentionRemaining === 0)
112
- return result("silent", "confidence-or-attention-bound");
113
- return result("ask", "selected-evidence-and-attention-policy-met");
114
- }
115
- //# sourceMappingURL=factory-calibration.js.map
@@ -1,6 +0,0 @@
1
- import type { InterventionManifest, InterventionEvidence } from "./intervention.js";
2
- /** Normalize existing validated schema3 records, retaining the complete source votes alongside the summary. */
3
- export declare function interventionEvidenceFromResults(manifest: InterventionManifest, armId: string, raw: unknown, outputBytes: ReadonlyMap<string, Uint8Array>): {
4
- evidence: InterventionEvidence;
5
- sourceResults: import("./results.js").ResultsFile;
6
- };
@@ -1,52 +0,0 @@
1
- import { createHash } from "node:crypto";
2
- import { validateResults } from "./results.js";
3
- import { aggregateMetrics } from "./comparison.js";
4
- /** Normalize existing validated schema3 records, retaining the complete source votes alongside the summary. */
5
- export function interventionEvidenceFromResults(manifest, armId, raw, outputBytes) {
6
- const sourceResults = validateResults(raw), arm = manifest.arms.find(a => a.id === armId);
7
- if (!arm || sourceResults.model !== arm.configuration.model)
8
- throw new Error("intervention result model mismatch");
9
- if (sourceResults.schema !== 3 || sourceResults.partial || sourceResults.mode !== manifest.common.mode
10
- || `${sourceResults.judge.provider}:${sourceResults.judge.model}` !== manifest.judge)
11
- throw new Error("incomplete or incompatible intervention results");
12
- if (sourceResults.source_hashes?.["intervention:manifest"] !== manifest.id || sourceResults.source_hashes?.["intervention:inputs"] !== manifest.inputDigest)
13
- throw new Error("recorded frozen intervention binding missing; never stamp historical results");
14
- if (sourceResults.scenarios.length !== manifest.cases.length || sourceResults.scenarios.some(s => !manifest.cases.some(c => c.id === s.id)))
15
- throw new Error("intervention scenario plan mismatch");
16
- const cells = [], artifacts = new Set();
17
- for (const spec of manifest.cases) {
18
- const scenario = sourceResults.scenarios.find(s => s.id === spec.id);
19
- if (scenario.criterion_count !== spec.criteria || (scenario.reps ?? 1) !== spec.reps || (scenario.pass_threshold ?? 1) !== spec.threshold)
20
- throw new Error("intervention repetition/criterion policy mismatch");
21
- for (const panel of scenario.rep_judgments ?? []) {
22
- const delivery = sourceResults.subject_invocations.find(o => o.scenario_id === spec.id && o.repetition === panel.repetition)?.prompt?.status;
23
- if (!delivery || !panel.objective)
24
- throw new Error("intervention objective/delivery evidence missing");
25
- const objective = panel.objective.status;
26
- const bytes = outputBytes.get(`${spec.id}:${panel.repetition}`);
27
- if (bytes !== undefined && (!(bytes instanceof Uint8Array) || bytes.byteLength > 8 * 1024 * 1024))
28
- throw new Error("intervention output exceeds bounds");
29
- const artifactSha256 = bytes ? createHash("sha256").update(bytes).digest("hex") : null;
30
- if (artifactSha256)
31
- artifacts.add(artifactSha256);
32
- let criteria = [], suspect = false;
33
- if (delivery === "PASS" && objective === "PASS") {
34
- if (panel.judgments.length !== 1)
35
- throw new Error("multi-judge normalization not yet supported; retain original panel without selecting a convenient vote");
36
- const judgment = panel.judgments[0];
37
- if (`${judgment.judge.provider}:${judgment.judge.model}` !== manifest.judge)
38
- throw new Error("intervention cell judge mismatch");
39
- criteria = [...(judgment.criteria ?? [])].sort((a, b) => a.index - b.index).map(c => c.verdict === "ERROR" ? "UNKNOWN" : c.verdict);
40
- suspect = judgment.suspect || scenario.suspect;
41
- }
42
- cells.push({ caseId: spec.id, repetition: panel.repetition, delivery, objective, criteria, suspect, artifactSha256 });
43
- }
44
- }
45
- const metrics = aggregateMetrics(sourceResults.scenarios);
46
- const cost = manifest.resourceMetric === "wall_ms" ? (sourceResults.scenarios.every(s => s.metrics) ? metrics.wall_time_ms : null)
47
- : manifest.resourceMetric === "tool_calls" ? metrics.tool_calls
48
- : metrics.cost_source === "provider-reported" && metrics.subject_metrics_reps === metrics.total_reps ? metrics.subject_cost_usd : null;
49
- const evidence = { armId, inputDigest: manifest.inputDigest, artifactDigests: [...artifacts].sort(), cells, cost, costUnit: manifest.resourceMetric };
50
- return { evidence, sourceResults };
51
- }
52
- //# sourceMappingURL=intervention-results.js.map
@@ -1,123 +0,0 @@
1
- /** Plain, bounded JSON encoding for retained intervention inputs; not semantic validation. */
2
- export declare function interventionCanonicalJson(value: unknown): string;
3
- export type InterventionAxis = "model" | "effort" | "skill" | "prompt" | "configuration";
4
- export interface InterventionDraft {
5
- family: "intervention";
6
- investigationSha256: string;
7
- resourceMetric: "usd" | "wall_ms" | "tool_calls";
8
- axes: InterventionAxis[];
9
- common: {
10
- mode: "green" | "force";
11
- scenarioSha256: string;
12
- rubricSha256: string;
13
- fixtureSha256: string;
14
- heldoutSha256: string;
15
- harnessSha256: string;
16
- judgePolicySha256: string;
17
- };
18
- proposer: string;
19
- judge: string;
20
- cases: Array<{
21
- id: string;
22
- criteria: number;
23
- reps: number;
24
- threshold: number;
25
- critical: boolean;
26
- }>;
27
- arms: Array<{
28
- id: string;
29
- configuration: Record<InterventionAxis, string>;
30
- }>;
31
- }
32
- export interface InterventionManifest extends InterventionDraft {
33
- version: "intervention-comparison-v1";
34
- id: string;
35
- inputDigest: string;
36
- changedAxes: InterventionAxis[];
37
- deterministicSampling: false;
38
- }
39
- export declare function freezeIntervention(input: InterventionDraft): InterventionManifest;
40
- export interface InterventionCell {
41
- caseId: string;
42
- repetition: number;
43
- delivery: "PASS" | "NOT-MEASURED" | "ERROR";
44
- objective: "PASS" | "FAIL" | "ERROR" | "NOT-MEASURED";
45
- criteria: Array<"PASS" | "FAIL" | "UNKNOWN">;
46
- suspect: boolean;
47
- artifactSha256: string | null;
48
- }
49
- export interface InterventionEvidence {
50
- armId: string;
51
- inputDigest: string;
52
- artifactDigests: string[];
53
- cells: InterventionCell[];
54
- cost: number | null;
55
- costUnit: string;
56
- }
57
- export declare const interventionEvidenceDigest: (value: InterventionEvidence) => string;
58
- export interface InterventionQualification {
59
- manifestId: string;
60
- proposer: {
61
- requested: string;
62
- canonical: string;
63
- };
64
- judge: {
65
- requested: string;
66
- canonical: string;
67
- };
68
- subjects: Record<string, {
69
- requested: string;
70
- canonical: string;
71
- }>;
72
- evidenceDigests: Record<string, string>;
73
- /** Actual independently retained output bytes, not an availability flag supplied by an arm. */
74
- artifacts: ReadonlyMap<string, Uint8Array>;
75
- }
76
- export interface InterventionAssessment {
77
- manifestId: string;
78
- complete: boolean;
79
- arms: Array<{
80
- armId: string;
81
- state: "MEASURED" | "FAILED" | "MISSING" | "ERROR" | "NOT-MEASURED" | "UNRESOLVED";
82
- eligible: boolean;
83
- artifactDigests: string[];
84
- cost: number | null;
85
- costUnit: string;
86
- }>;
87
- cheapestEligible: string | null;
88
- routingDefault: null;
89
- adoptionAuthorized: false;
90
- }
91
- /** Pre-spend role gate: consumes independently resolved host facts, never guesses aliases or calls a provider. */
92
- export declare function assertInterventionRoles(manifest: InterventionManifest, qualification: InterventionQualification | null): asserts qualification is InterventionQualification;
93
- /** Inspects separately host-qualified evidence; does not obtain qualification or execute a model. */
94
- export declare function assessIntervention(manifest: InterventionManifest, evidence: readonly InterventionEvidence[], qualification: InterventionQualification | null): InterventionAssessment;
95
- export type BlindQualityChoice = {
96
- kind: "one" | "tie" | "none" | "insufficient";
97
- labels: string[];
98
- };
99
- /** Only this view omits identity/cost; arbitrary artifact content can still leak clues and must be reviewed. */
100
- export declare function createBlindComparison(manifest: InterventionManifest, assessment: InterventionAssessment, fixtureOrPersistedSeed?: string): Readonly<{
101
- view: () => {
102
- version: string;
103
- cards: {
104
- label: string;
105
- artifactDigests: string[];
106
- }[];
107
- limitations: string[];
108
- };
109
- readArtifact(label: string, hash: string): Buffer<ArrayBuffer>;
110
- choose(input: BlindQualityChoice): BlindQualityChoice;
111
- reveal(): {
112
- manifestId: string;
113
- choice: BlindQualityChoice;
114
- arms: {
115
- label: string;
116
- armId: string;
117
- configuration: Record<InterventionAxis, string>;
118
- cost: number | null;
119
- costUnit: string;
120
- }[];
121
- routingDefault: null;
122
- };
123
- }>;
@@ -1,218 +0,0 @@
1
- import { createHash, randomBytes } from "node:crypto";
2
- const SHA = /^[a-f0-9]{64}$/;
3
- const text = (x, max = 512) => typeof x === "string" && x.length > 0 && x.length <= max && !/[\u0000-\u001f\u007f]/.test(x);
4
- const closed = (x, names) => !!x && typeof x === "object" && !Array.isArray(x) && Object.keys(x).sort().join() === [...names].sort().join();
5
- function canonical(x, depth = 0) {
6
- if (depth > 16)
7
- throw new Error("intervention data exceeds depth");
8
- if (x === null || typeof x === "string" || typeof x === "boolean" || (typeof x === "number" && Number.isFinite(x)))
9
- return JSON.stringify(x);
10
- if (!x || typeof x !== "object" || (!Array.isArray(x) && ![Object.prototype, null].includes(Object.getPrototypeOf(x))))
11
- throw new Error("plain intervention JSON required");
12
- const d = Object.getOwnPropertyDescriptors(x);
13
- if (Array.isArray(x)) {
14
- if (x.length > 4096 || Reflect.ownKeys(x).length !== x.length + 1)
15
- throw new Error("bounded dense array required");
16
- return `[${Array.from({ length: x.length }, (_, i) => { if (!d[i] || !Object.hasOwn(d[i], "value"))
17
- throw new Error("plain array required"); return canonical(d[i].value, depth + 1); }).join(",")}]`;
18
- }
19
- return `{${Reflect.ownKeys(d).sort((a, b) => String(a) < String(b) ? -1 : 1).map(k => { if (typeof k !== "string" || !d[k].enumerable || !Object.hasOwn(d[k], "value"))
20
- throw new Error("plain object required"); return `${JSON.stringify(k)}:${canonical(d[k].value, depth + 1)}`; }).join(",")}}`;
21
- }
22
- /** Plain, bounded JSON encoding for retained intervention inputs; not semantic validation. */
23
- export function interventionCanonicalJson(value) {
24
- const bytes = canonical(value);
25
- if (Buffer.byteLength(bytes) > 2 * 1024 * 1024)
26
- throw new Error("intervention data exceeds byte bound");
27
- return bytes;
28
- }
29
- const digest = (value) => createHash("sha256").update(interventionCanonicalJson(value)).digest("hex");
30
- function frozen(value) { if (value && typeof value === "object") {
31
- Object.values(value).forEach(frozen);
32
- Object.freeze(value);
33
- } return value; }
34
- const clone = (value) => JSON.parse(canonical(value));
35
- const axes = ["model", "effort", "skill", "prompt", "configuration"];
36
- export function freezeIntervention(input) {
37
- const d = clone(input);
38
- if (!closed(d, ["family", "investigationSha256", "resourceMetric", "axes", "common", "proposer", "judge", "cases", "arms"]) || d.family !== "intervention" || !SHA.test(d.investigationSha256)
39
- || !["usd", "wall_ms", "tool_calls"].includes(d.resourceMetric)
40
- || !Array.isArray(d.axes) || !d.axes.length || new Set(d.axes).size !== d.axes.length || d.axes.some(a => !axes.includes(a))
41
- || !closed(d.common, ["mode", "scenarioSha256", "rubricSha256", "fixtureSha256", "heldoutSha256", "harnessSha256", "judgePolicySha256"])
42
- || !["green", "force"].includes(d.common.mode) || Object.entries(d.common).some(([k, v]) => k !== "mode" && !SHA.test(v))
43
- || !text(d.proposer) || !text(d.judge) || !Array.isArray(d.arms) || d.arms.length < 2 || d.arms.length > 8
44
- || !Array.isArray(d.cases) || !d.cases.length || d.cases.length > 128)
45
- throw new Error("invalid intervention manifest");
46
- for (const arm of d.arms)
47
- if (!closed(arm, ["id", "configuration"]) || !text(arm.id, 128) || !closed(arm.configuration, axes)
48
- || !text(arm.configuration.model) || !text(arm.configuration.effort) || ["skill", "prompt", "configuration"].some(k => !SHA.test(arm.configuration[k])))
49
- throw new Error("invalid intervention arm");
50
- if (new Set(d.arms.map(a => a.id)).size !== d.arms.length || new Set(d.cases.map(c => c.id)).size !== d.cases.length)
51
- throw new Error("duplicate intervention identity");
52
- for (const c of d.cases)
53
- if (!closed(c, ["id", "criteria", "reps", "threshold", "critical"]) || !text(c.id, 128)
54
- || !Number.isSafeInteger(c.criteria) || c.criteria < 1 || c.criteria > 128 || !Number.isSafeInteger(c.reps) || c.reps < 1 || c.reps > 20
55
- || !Number.isFinite(c.threshold) || c.threshold <= 0 || c.threshold > 1 || typeof c.critical !== "boolean"
56
- || ((c.critical || c.id.startsWith("B")) && c.threshold !== 1))
57
- throw new Error("invalid frozen case policy");
58
- const changedAxes = axes.filter(axis => d.arms.some(a => a.configuration[axis] !== d.arms[0].configuration[axis]));
59
- if (changedAxes.some(axis => !d.axes.includes(axis)))
60
- throw new Error("undeclared intervention axis");
61
- const body = { ...d, version: "intervention-comparison-v1", inputDigest: digest({ common: d.common, cases: d.cases }), changedAxes, deterministicSampling: false };
62
- return frozen({ ...body, id: digest(body) });
63
- }
64
- export const interventionEvidenceDigest = (value) => digest(value);
65
- const assessments = new WeakSet();
66
- const assessmentArtifacts = new WeakMap();
67
- function manifestValid(m) {
68
- const detached = clone(m);
69
- const { version, id: _id, inputDigest: _inputs, changedAxes: _axes, deterministicSampling, ...draft } = detached;
70
- if (version !== "intervention-comparison-v1" || deterministicSampling !== false || canonical(freezeIntervention(draft)) !== canonical(detached))
71
- throw new Error("frozen intervention changed");
72
- }
73
- /** Pre-spend role gate: consumes independently resolved host facts, never guesses aliases or calls a provider. */
74
- export function assertInterventionRoles(manifest, qualification) {
75
- manifestValid(manifest);
76
- if (!qualification || !qualification.proposer || !qualification.judge || !qualification.subjects || qualification.manifestId !== manifest.id || qualification.proposer.requested !== manifest.proposer || qualification.judge.requested !== manifest.judge
77
- || !text(qualification.proposer.canonical) || !text(qualification.judge.canonical))
78
- throw new Error("independent role qualification required");
79
- const reserved = new Set([qualification.proposer.canonical, qualification.judge.canonical]);
80
- if (reserved.size !== 2 || manifest.arms.some(a => !Object.hasOwn(qualification.subjects, a.id) || qualification.subjects[a.id].requested !== a.configuration.model
81
- || !text(qualification.subjects[a.id].canonical) || reserved.has(qualification.subjects[a.id].canonical)))
82
- throw new Error("role identity conflict or unresolved role");
83
- }
84
- /** Inspects separately host-qualified evidence; does not obtain qualification or execute a model. */
85
- export function assessIntervention(manifest, evidence, qualification) {
86
- assertInterventionRoles(manifest, qualification);
87
- if (evidence.length > manifest.arms.length || new Set(evidence.map(e => e.armId)).size !== evidence.length || evidence.some(e => !manifest.arms.some(a => a.id === e.armId)))
88
- throw new Error("unexpected intervention evidence identity");
89
- const retained = new Map();
90
- let retainedBytes = 0;
91
- const arms = manifest.arms.map(arm => {
92
- const e = evidence.find(e => e.armId === arm.id);
93
- let admitted = false;
94
- const result = (state) => ({ armId: arm.id, state, eligible: state === "MEASURED", artifactDigests: admitted ? [...e.artifactDigests] : [], cost: admitted ? e.cost : null, costUnit: admitted ? e.costUnit : "unknown" });
95
- if (!e)
96
- return result("MISSING");
97
- if (qualification.evidenceDigests[arm.id] !== interventionEvidenceDigest(e) || !closed(e, ["armId", "inputDigest", "artifactDigests", "cells", "cost", "costUnit"])
98
- || e.inputDigest !== manifest.inputDigest || !Array.isArray(e.artifactDigests) || e.artifactDigests.length > 4096 || e.artifactDigests.some(h => !SHA.test(h))
99
- || !(e.cost === null || (Number.isFinite(e.cost) && e.cost >= 0)) || e.costUnit !== manifest.resourceMetric || !Array.isArray(e.cells)
100
- || new Set(e.artifactDigests).size !== e.artifactDigests.length || e.cells.length !== manifest.cases.reduce((n, c) => n + c.reps, 0))
101
- return result("ERROR");
102
- admitted = true;
103
- const cells = new Map();
104
- let error = false, unmeasured = false, unresolved = false, missingOutput = false;
105
- for (const c of e.cells) {
106
- const spec = manifest.cases.find(s => s.id === c.caseId), id = `${c.caseId}:${c.repetition}`;
107
- if (!closed(c, ["caseId", "repetition", "delivery", "objective", "criteria", "suspect", "artifactSha256"]) || !spec || !Number.isSafeInteger(c.repetition) || c.repetition < 0 || c.repetition >= spec.reps || cells.has(id)
108
- || !["PASS", "NOT-MEASURED", "ERROR"].includes(c.delivery) || !["PASS", "FAIL", "ERROR", "NOT-MEASURED"].includes(c.objective) || !Array.isArray(c.criteria) || typeof c.suspect !== "boolean") {
109
- error = true;
110
- continue;
111
- }
112
- cells.set(id, c);
113
- if (c.artifactSha256 !== null && (typeof c.artifactSha256 !== "string" || !SHA.test(c.artifactSha256) || !e.artifactDigests.includes(c.artifactSha256)))
114
- error = true;
115
- if (c.delivery === "PASS" && c.objective === "PASS" && c.artifactSha256 === null)
116
- missingOutput = true;
117
- if (c.delivery !== "PASS" || c.objective !== "PASS") {
118
- if (c.criteria.length)
119
- error = true;
120
- if (c.delivery === "NOT-MEASURED")
121
- unmeasured = true;
122
- if (c.delivery === "ERROR" || c.objective === "ERROR" || (c.objective === "NOT-MEASURED" && c.delivery !== "NOT-MEASURED"))
123
- error = true;
124
- }
125
- else if (c.criteria.length !== spec.criteria || c.criteria.some(v => !["PASS", "FAIL", "UNKNOWN"].includes(v)))
126
- error = true;
127
- else if (c.suspect || c.criteria.includes("UNKNOWN"))
128
- unresolved = true;
129
- }
130
- if (error)
131
- return result("ERROR");
132
- if (unmeasured)
133
- return result("NOT-MEASURED");
134
- if (unresolved)
135
- return result("UNRESOLVED");
136
- if (missingOutput)
137
- return result("MISSING");
138
- const passed = manifest.cases.every(spec => Array.from(cells.values()).filter(c => c.caseId === spec.id && c.objective === "PASS" && c.criteria.every(v => v === "PASS")).length / spec.reps >= spec.threshold);
139
- if (!passed)
140
- return result("FAILED");
141
- if (!e.artifactDigests.length)
142
- return result("MISSING");
143
- for (const hash of e.artifactDigests) {
144
- const bytes = qualification.artifacts?.get(hash);
145
- if (bytes === undefined)
146
- return result("MISSING");
147
- if (!(bytes instanceof Uint8Array) || bytes.byteLength > 8 * 1024 * 1024)
148
- return result("ERROR");
149
- const stable = Buffer.from(bytes);
150
- if (createHash("sha256").update(stable).digest("hex") !== hash)
151
- return result("ERROR");
152
- if (!retained.has(hash)) {
153
- if (retainedBytes + bytes.byteLength > 64 * 1024 * 1024)
154
- return result("ERROR");
155
- retained.set(hash, stable);
156
- retainedBytes += stable.byteLength;
157
- }
158
- }
159
- return result("MEASURED");
160
- });
161
- const eligible = arms.filter(a => a.eligible), knownCosts = eligible.length && eligible.every(a => a.cost !== null && a.costUnit === eligible[0].costUnit);
162
- const cheapestEligible = knownCosts ? [...eligible].sort((a, b) => a.cost - b.cost || (a.armId < b.armId ? -1 : 1))[0].armId : null;
163
- const result = frozen({ manifestId: manifest.id, complete: arms.every(a => a.state === "MEASURED" || a.state === "FAILED"), arms, cheapestEligible, routingDefault: null, adoptionAuthorized: false });
164
- assessments.add(result);
165
- assessmentArtifacts.set(result, retained);
166
- return result;
167
- }
168
- /** Only this view omits identity/cost; arbitrary artifact content can still leak clues and must be reviewed. */
169
- export function createBlindComparison(manifest, assessment, fixtureOrPersistedSeed) {
170
- manifestValid(manifest);
171
- if (!assessments.has(assessment) || assessment.manifestId !== manifest.id || !assessment.complete)
172
- throw new Error("complete recomputed intervention assessment required");
173
- const seed = fixtureOrPersistedSeed ?? randomBytes(32).toString("hex");
174
- if (!SHA.test(seed))
175
- throw new Error("invalid private blinding seed");
176
- const assessmentId = digest(assessment);
177
- const eligible = assessment.arms.filter(a => a.eligible).sort((a, b) => {
178
- const left = digest([seed, manifest.id, assessmentId, a.armId]), right = digest([seed, manifest.id, assessmentId, b.armId]);
179
- return left < right ? -1 : left > right ? 1 : 0;
180
- });
181
- const mapping = new Map(eligible.map((a, i) => [`variant-${digest([seed, manifest.id, assessmentId, i]).slice(0, 16)}`, a.armId]));
182
- if (mapping.size !== eligible.length)
183
- throw new Error("opaque label collision");
184
- const cards = [...mapping].map(([label, armId]) => ({ label, artifactDigests: assessment.arms.find(a => a.armId === armId).artifactDigests }));
185
- let choice = null, revealed = false;
186
- return Object.freeze({
187
- view: () => frozen({ version: "blind-intervention-view-v1", cards, limitations: ["artifact-content-may-disclose-identity"] }),
188
- readArtifact(label, hash) {
189
- const armId = mapping.get(label), arm = assessment.arms.find(a => a.armId === armId);
190
- if (!arm || !arm.artifactDigests.includes(hash))
191
- throw new Error("artifact outside blind view");
192
- const bytes = assessmentArtifacts.get(assessment)?.get(hash);
193
- if (!bytes)
194
- throw new Error("retained blind artifact missing");
195
- return Buffer.from(bytes);
196
- },
197
- choose(input) {
198
- const value = clone(input);
199
- if (!closed(value, ["kind", "labels"]) || !["one", "tie", "none", "insufficient"].includes(value.kind) || !Array.isArray(value.labels)
200
- || new Set(value.labels).size !== value.labels.length || value.labels.some(l => !mapping.has(l))
201
- || (value.kind === "one" ? value.labels.length !== 1 : value.kind === "tie" ? value.labels.length < 2 : value.labels.length !== 0))
202
- throw new Error("invalid blind quality choice");
203
- value.labels.sort();
204
- if (revealed && canonical(choice) !== canonical(value))
205
- throw new Error("quality choice locked after reveal");
206
- choice = frozen(value);
207
- return choice;
208
- },
209
- reveal() {
210
- if (!choice)
211
- throw new Error("quality choice required before reveal");
212
- revealed = true;
213
- return frozen({ manifestId: manifest.id, choice, arms: [...mapping].map(([label, armId]) => ({ label, armId, configuration: manifest.arms.find(a => a.id === armId).configuration,
214
- cost: assessment.arms.find(a => a.armId === armId).cost, costUnit: assessment.arms.find(a => a.armId === armId).costUnit })), routingDefault: null });
215
- },
216
- });
217
- }
218
- //# sourceMappingURL=intervention.js.map