@skill-harness/core 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/adapters/types.d.ts +0 -12
- package/dist/capture-trace-types.d.ts +0 -96
- package/dist/capture-trace-types.js +0 -4
- package/dist/command-cost.d.ts +1 -1
- package/dist/command-cost.js +0 -5
- package/dist/execution-trace.js +1 -1
- package/dist/index.d.ts +3 -23
- package/dist/index.js +3 -23
- package/dist/instruction-coverage.d.ts +0 -7
- package/dist/instruction-coverage.js +5 -7
- package/dist/journal.d.ts +0 -8
- package/dist/metrics.d.ts +18 -0
- package/dist/metrics.js +28 -0
- package/dist/redaction.d.ts +6 -0
- package/dist/redaction.js +55 -0
- package/dist/regate.js +6 -71
- package/dist/regrade.d.ts +1 -3
- package/dist/regrade.js +1 -21
- package/dist/report.d.ts +1 -15
- package/dist/report.js +1 -13
- package/dist/reps.d.ts +1 -2
- package/dist/reps.js +0 -2
- package/dist/results.d.ts +0 -9
- package/dist/results.js +2 -19
- package/dist/run.js +11 -72
- package/dist/seeded.d.ts +1 -7
- package/dist/seeded.js +5 -11
- package/dist/sources.js +4 -9
- package/dist/spec.d.ts +0 -13
- package/dist/spec.js +6 -48
- package/dist/trajectory-events.d.ts +58 -0
- package/dist/trajectory-events.js +119 -0
- package/package.json +1 -1
- package/schemas/results-v2.schema.json +376 -75
- package/schemas/specification-v1.schema.json +128 -174
- package/dist/adjudication.d.ts +0 -219
- package/dist/adjudication.js +0 -422
- package/dist/adoption.d.ts +0 -128
- package/dist/adoption.js +0 -102
- package/dist/affected.d.ts +0 -88
- package/dist/affected.js +0 -222
- package/dist/capture.d.ts +0 -193
- package/dist/capture.js +0 -344
- package/dist/comparison.d.ts +0 -97
- package/dist/comparison.js +0 -189
- package/dist/factory-calibration.d.ts +0 -63
- package/dist/factory-calibration.js +0 -115
- package/dist/intervention-results.d.ts +0 -6
- package/dist/intervention-results.js +0 -52
- package/dist/intervention.d.ts +0 -123
- package/dist/intervention.js +0 -218
- package/dist/investigation.d.ts +0 -94
- package/dist/investigation.js +0 -148
- package/dist/judge-agreement.d.ts +0 -19
- package/dist/judge-agreement.js +0 -31
- package/dist/qualification-capture.d.ts +0 -44
- package/dist/qualification-capture.js +0 -163
- package/dist/qualification-config.d.ts +0 -233
- package/dist/qualification-config.js +0 -694
- package/dist/qualification-lock.d.ts +0 -15
- package/dist/qualification-lock.js +0 -87
- package/dist/qualification-oauth-directory.d.ts +0 -79
- package/dist/qualification-oauth-directory.js +0 -444
- package/dist/qualification-panel-store.d.ts +0 -14
- package/dist/qualification-panel-store.js +0 -188
- package/dist/qualification-panels.d.ts +0 -99
- package/dist/qualification-panels.js +0 -91
- package/dist/qualification-process.d.ts +0 -13
- package/dist/qualification-process.js +0 -190
- package/dist/qualification-runner.d.ts +0 -203
- package/dist/qualification-runner.js +0 -2037
- package/dist/qualification-store.d.ts +0 -187
- package/dist/qualification-store.js +0 -833
- package/dist/qualification-validation.d.ts +0 -12
- package/dist/qualification-validation.js +0 -8
- package/dist/screen.d.ts +0 -31
- package/dist/screen.js +0 -105
- package/dist/trajectory-gates.d.ts +0 -137
- package/dist/trajectory-gates.js +0 -576
- package/dist/trusted-host-supervision.d.ts +0 -25
- package/dist/trusted-host-supervision.js +0 -94
- package/dist/work-capture-schema.d.ts +0 -38
- package/dist/work-capture-schema.js +0 -33
- package/dist/work-capture.d.ts +0 -43
- package/dist/work-capture.js +0 -81
- package/dist/work-signals.d.ts +0 -55
- package/dist/work-signals.js +0 -71
- package/schemas/qualification-cell-result-v1.schema.json +0 -32
- package/schemas/qualification-config-v1.schema.json +0 -244
- package/schemas/qualification-invocation-request-v1.schema.json +0 -59
- package/schemas/qualification-judge-panel-result-v1.schema.json +0 -42
- package/schemas/trajectory-event-v1.schema.json +0 -80
package/dist/investigation.d.ts
DELETED
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
/** Bounded operator envelope; duplicate keys and over-expanded/cyclic objects are refused before dispatch. */
|
|
2
|
-
export declare function parseLearningRequest(text: string): unknown;
|
|
3
|
-
export interface HypothesisProposal {
|
|
4
|
-
archiveSnapshot: string;
|
|
5
|
-
caseIds: string[];
|
|
6
|
-
population: string;
|
|
7
|
-
intervention: string;
|
|
8
|
-
alternatives: string[];
|
|
9
|
-
prediction: string;
|
|
10
|
-
downside: string;
|
|
11
|
-
disproof: string;
|
|
12
|
-
rollback: string;
|
|
13
|
-
limits: {
|
|
14
|
-
subjectCalls: number;
|
|
15
|
-
judgeCalls: number;
|
|
16
|
-
wallMs: number;
|
|
17
|
-
};
|
|
18
|
-
effectProfile: string | null;
|
|
19
|
-
}
|
|
20
|
-
export interface Hypothesis {
|
|
21
|
-
version: "factory-hypothesis-v1";
|
|
22
|
-
id: string;
|
|
23
|
-
proposal: HypothesisProposal;
|
|
24
|
-
status: "proposed";
|
|
25
|
-
}
|
|
26
|
-
export declare function buildHypothesis(input: HypothesisProposal): Hypothesis;
|
|
27
|
-
export interface InvestigationAuthority {
|
|
28
|
-
id: string;
|
|
29
|
-
investigations: readonly string[];
|
|
30
|
-
promotions: readonly string[];
|
|
31
|
-
}
|
|
32
|
-
/** Independent host declarations, not approval fields on a proposal and never adoption/activation authority. */
|
|
33
|
-
export declare function authorizeInvestigation(h: Hypothesis, authority: InvestigationAuthority | null): {
|
|
34
|
-
version: string;
|
|
35
|
-
hypothesisId: string;
|
|
36
|
-
authorityId: string;
|
|
37
|
-
scope: "investigation-only";
|
|
38
|
-
};
|
|
39
|
-
export interface InvestigationScenarioPreview {
|
|
40
|
-
version: "investigation-scenario-preview-v1";
|
|
41
|
-
hypothesisId: string;
|
|
42
|
-
specPath: string;
|
|
43
|
-
baseSha256: string;
|
|
44
|
-
afterSha256: string;
|
|
45
|
-
scenario: Record<string, unknown>;
|
|
46
|
-
block: string;
|
|
47
|
-
digest: string;
|
|
48
|
-
}
|
|
49
|
-
export declare function previewInvestigationScenario(h: Hypothesis, specPath: string, scenario: Record<string, unknown>): InvestigationScenarioPreview;
|
|
50
|
-
/** Exact preview consent includes destination, existing spec and complete scenario bytes. No automatic sanitization claim. */
|
|
51
|
-
export declare function applyInvestigationScenario(h: Hypothesis, preview: InvestigationScenarioPreview, authority: InvestigationAuthority | null): {
|
|
52
|
-
scenarioId: string;
|
|
53
|
-
replayed: boolean;
|
|
54
|
-
sha256: string;
|
|
55
|
-
scope: "scenario-only";
|
|
56
|
-
};
|
|
57
|
-
export interface FrozenInvestigationInputs {
|
|
58
|
-
specSha256: string;
|
|
59
|
-
rubricSha256: string;
|
|
60
|
-
judgePolicySha256: string;
|
|
61
|
-
heldoutSha256: string;
|
|
62
|
-
configurationSha256: string;
|
|
63
|
-
}
|
|
64
|
-
export declare function freezeInvestigation(h: Hypothesis, inputs: FrozenInvestigationInputs, authority: InvestigationAuthority | null): {
|
|
65
|
-
digest: string;
|
|
66
|
-
version: string;
|
|
67
|
-
hypothesisId: string;
|
|
68
|
-
authorityId: string;
|
|
69
|
-
inputs: FrozenInvestigationInputs;
|
|
70
|
-
executionReady: false;
|
|
71
|
-
};
|
|
72
|
-
export declare function assertFrozenInvestigation(frozen: ReturnType<typeof freezeInvestigation>, current: FrozenInvestigationInputs): void;
|
|
73
|
-
export interface WeeklyInvestigationInput {
|
|
74
|
-
week: string;
|
|
75
|
-
population: string;
|
|
76
|
-
policyDigest: string;
|
|
77
|
-
archiveSnapshot: string;
|
|
78
|
-
eligibleCaseIds: string[];
|
|
79
|
-
maxCases: number;
|
|
80
|
-
}
|
|
81
|
-
export interface WeeklyInvestigationSelection {
|
|
82
|
-
version: "weekly-investigation-selection-v1";
|
|
83
|
-
id: string;
|
|
84
|
-
key: string;
|
|
85
|
-
inputDigest: string;
|
|
86
|
-
week: string;
|
|
87
|
-
population: string;
|
|
88
|
-
policyDigest: string;
|
|
89
|
-
archiveSnapshot: string;
|
|
90
|
-
selectedCaseIds: string[];
|
|
91
|
-
sampling: "exploratory-sha256-order-v1";
|
|
92
|
-
}
|
|
93
|
-
/** The host persists these immutable selections; this function neither schedules a process nor spends tokens. */
|
|
94
|
-
export declare function selectWeeklyInvestigation(history: readonly WeeklyInvestigationSelection[], input: WeeklyInvestigationInput): WeeklyInvestigationSelection;
|
package/dist/investigation.js
DELETED
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
|
-
import yaml from "js-yaml";
|
|
3
|
-
import { readFileSync, realpathSync } from "node:fs";
|
|
4
|
-
import { appendScenario, renderScenarioBlock, specSha256 } from "./spec-write.js";
|
|
5
|
-
import { parseSpec } from "./spec.js";
|
|
6
|
-
const SHA = /^[a-f0-9]{64}$/;
|
|
7
|
-
const keys = (x, expected) => Object.keys(x).sort().join() === [...expected].sort().join();
|
|
8
|
-
const text = (x, max = 4000) => typeof x === "string" && x.length > 0 && x.length <= max;
|
|
9
|
-
function json(input) {
|
|
10
|
-
let nodes = 0;
|
|
11
|
-
const visit = (x, depth) => {
|
|
12
|
-
if (++nodes > 4096 || depth > 16)
|
|
13
|
-
throw new Error("investigation JSON exceeds bounds");
|
|
14
|
-
if (x === null || typeof x === "boolean" || typeof x === "string" || (typeof x === "number" && Number.isFinite(x)))
|
|
15
|
-
return x;
|
|
16
|
-
if (!x || typeof x !== "object" || (!Array.isArray(x) && ![Object.prototype, null].includes(Object.getPrototypeOf(x))))
|
|
17
|
-
throw new Error("plain investigation JSON required");
|
|
18
|
-
const descriptors = Object.getOwnPropertyDescriptors(x);
|
|
19
|
-
if (Array.isArray(x)) {
|
|
20
|
-
if (Object.getPrototypeOf(x) !== Array.prototype || x.length > 256 || Reflect.ownKeys(x).length !== x.length + 1)
|
|
21
|
-
throw new Error("bounded dense JSON array required");
|
|
22
|
-
return Array.from({ length: x.length }, (_, i) => { const d = descriptors[String(i)]; if (!d || !('value' in d))
|
|
23
|
-
throw new Error("plain JSON array required"); return visit(d.value, depth + 1); });
|
|
24
|
-
}
|
|
25
|
-
const out = {};
|
|
26
|
-
for (const key of Reflect.ownKeys(descriptors).sort((a, b) => String(a) < String(b) ? -1 : 1)) {
|
|
27
|
-
const d = descriptors[key];
|
|
28
|
-
if (typeof key !== "string" || !d.enumerable || !('value' in d))
|
|
29
|
-
throw new Error("plain JSON properties required");
|
|
30
|
-
Object.defineProperty(out, key, { value: visit(d.value, depth + 1), enumerable: true });
|
|
31
|
-
}
|
|
32
|
-
return out;
|
|
33
|
-
};
|
|
34
|
-
const result = visit(input, 0);
|
|
35
|
-
if (Buffer.byteLength(JSON.stringify(result)) > 65536)
|
|
36
|
-
throw new Error("investigation JSON exceeds byte bound");
|
|
37
|
-
return result;
|
|
38
|
-
}
|
|
39
|
-
/** Bounded operator envelope; duplicate keys and over-expanded/cyclic objects are refused before dispatch. */
|
|
40
|
-
export function parseLearningRequest(text) {
|
|
41
|
-
if (typeof text !== "string" || Buffer.byteLength(text) > 65536)
|
|
42
|
-
throw new Error("learning request exceeds bound");
|
|
43
|
-
let value;
|
|
44
|
-
try {
|
|
45
|
-
value = yaml.load(text, { schema: yaml.JSON_SCHEMA, json: false });
|
|
46
|
-
}
|
|
47
|
-
catch {
|
|
48
|
-
throw new Error("invalid learning request");
|
|
49
|
-
}
|
|
50
|
-
return json(value);
|
|
51
|
-
}
|
|
52
|
-
const digest = (x) => createHash("sha256").update(JSON.stringify(json(x))).digest("hex");
|
|
53
|
-
function freeze(value) { if (value && typeof value === "object") {
|
|
54
|
-
Object.values(value).forEach(freeze);
|
|
55
|
-
Object.freeze(value);
|
|
56
|
-
} return value; }
|
|
57
|
-
export function buildHypothesis(input) {
|
|
58
|
-
const p = json(input);
|
|
59
|
-
if (!keys(p, ["archiveSnapshot", "caseIds", "population", "intervention", "alternatives", "prediction", "downside", "disproof", "rollback", "limits", "effectProfile"])
|
|
60
|
-
|| !SHA.test(p.archiveSnapshot) || !Array.isArray(p.caseIds) || !p.caseIds.length || p.caseIds.some(id => typeof id !== "string" || !SHA.test(id))
|
|
61
|
-
|| ![p.population, p.intervention, p.prediction, p.downside, p.disproof, p.rollback].every(s => text(s))
|
|
62
|
-
|| !Array.isArray(p.alternatives) || !p.alternatives.length || p.alternatives.some(a => !text(a))
|
|
63
|
-
|| !p.limits || !keys(p.limits, ["subjectCalls", "judgeCalls", "wallMs"])
|
|
64
|
-
|| !Object.values(p.limits).every(n => Number.isSafeInteger(n) && n >= 0 && n <= 1000000000)
|
|
65
|
-
|| !(p.effectProfile === null || text(p.effectProfile, 512)))
|
|
66
|
-
throw new Error("invalid bounded hypothesis");
|
|
67
|
-
return freeze({ version: "factory-hypothesis-v1", id: digest({ version: "factory-hypothesis-v1", proposal: p }), proposal: p, status: "proposed" });
|
|
68
|
-
}
|
|
69
|
-
function authorize(h, authority) {
|
|
70
|
-
const value = json(h);
|
|
71
|
-
if (!keys(value, ["version", "id", "proposal", "status"]) || value.version !== "factory-hypothesis-v1" || value.status !== "proposed" || buildHypothesis(value.proposal).id !== value.id)
|
|
72
|
-
throw new Error("hypothesis identity changed");
|
|
73
|
-
if (!authority || !text(authority.id, 512) || !Array.isArray(authority.investigations) || !Array.isArray(authority.promotions)
|
|
74
|
-
|| authority.investigations.length > 256 || authority.promotions.length > 256
|
|
75
|
-
|| [...authority.investigations, ...authority.promotions].some(id => typeof id !== "string" || !SHA.test(id))
|
|
76
|
-
|| !authority.investigations.includes(value.id))
|
|
77
|
-
throw new Error("independent investigation authority required");
|
|
78
|
-
return authority;
|
|
79
|
-
}
|
|
80
|
-
/** Independent host declarations, not approval fields on a proposal and never adoption/activation authority. */
|
|
81
|
-
export function authorizeInvestigation(h, authority) {
|
|
82
|
-
const selected = authorize(h, authority);
|
|
83
|
-
return freeze({ version: "investigation-approval-v1", hypothesisId: h.id, authorityId: selected.id, scope: "investigation-only" });
|
|
84
|
-
}
|
|
85
|
-
export function previewInvestigationScenario(h, specPath, scenario) {
|
|
86
|
-
if (buildHypothesis(h.proposal).id !== h.id)
|
|
87
|
-
throw new Error("hypothesis identity changed");
|
|
88
|
-
const path = realpathSync(specPath), current = readFileSync(path, "utf8"), safe = json(scenario), block = renderScenarioBlock(safe);
|
|
89
|
-
parseSpec(current + block, path);
|
|
90
|
-
const value = { version: "investigation-scenario-preview-v1", hypothesisId: h.id, specPath: path, baseSha256: specSha256(current), afterSha256: specSha256(current + block), scenario: safe, block };
|
|
91
|
-
return freeze({ ...value, digest: digest(value) });
|
|
92
|
-
}
|
|
93
|
-
/** Exact preview consent includes destination, existing spec and complete scenario bytes. No automatic sanitization claim. */
|
|
94
|
-
export function applyInvestigationScenario(h, preview, authority) {
|
|
95
|
-
const selected = authorize(h, authority), copy = json(preview), { digest: claimed, ...body } = copy;
|
|
96
|
-
if (!keys(copy, ["version", "hypothesisId", "specPath", "baseSha256", "afterSha256", "scenario", "block", "digest"])
|
|
97
|
-
|| body.version !== "investigation-scenario-preview-v1" || body.hypothesisId !== h.id || digest(body) !== claimed
|
|
98
|
-
|| renderScenarioBlock(body.scenario) !== body.block || !selected.promotions.includes(claimed))
|
|
99
|
-
throw new Error("exact privacy/promotion preview authority required");
|
|
100
|
-
if (realpathSync(body.specPath) !== body.specPath)
|
|
101
|
-
throw new Error("promotion destination changed");
|
|
102
|
-
const current = readFileSync(body.specPath, "utf8");
|
|
103
|
-
if (specSha256(current) === body.afterSha256)
|
|
104
|
-
return { scenarioId: String(body.scenario.id), replayed: true, sha256: body.afterSha256, scope: "scenario-only" };
|
|
105
|
-
const result = appendScenario({ specPath: body.specPath, scenario: body.scenario, baseSha256: body.baseSha256 });
|
|
106
|
-
return { scenarioId: result.id, replayed: false, sha256: result.sha256, scope: "scenario-only" };
|
|
107
|
-
}
|
|
108
|
-
function frozenInputs(input) {
|
|
109
|
-
const value = json(input);
|
|
110
|
-
if (!keys(value, ["specSha256", "rubricSha256", "judgePolicySha256", "heldoutSha256", "configurationSha256"]) || Object.values(value).some(v => typeof v !== "string" || !SHA.test(v)))
|
|
111
|
-
throw new Error("invalid frozen evaluation inputs");
|
|
112
|
-
return value;
|
|
113
|
-
}
|
|
114
|
-
export function freezeInvestigation(h, inputs, authority) {
|
|
115
|
-
const approved = authorizeInvestigation(h, authority), value = { version: "frozen-investigation-v1", hypothesisId: h.id, authorityId: approved.authorityId, inputs: frozenInputs(inputs), executionReady: false };
|
|
116
|
-
return freeze({ ...value, digest: digest(value) });
|
|
117
|
-
}
|
|
118
|
-
export function assertFrozenInvestigation(frozen, current) {
|
|
119
|
-
const { digest: recorded, ...value } = json(frozen);
|
|
120
|
-
if (!keys(value, ["version", "hypothesisId", "authorityId", "inputs", "executionReady"]) || !SHA.test(value.hypothesisId) || !text(value.authorityId, 512)
|
|
121
|
-
|| value.version !== "frozen-investigation-v1" || value.executionReady !== false || digest(value) !== recorded || digest(value.inputs) !== digest(frozenInputs(current)))
|
|
122
|
-
throw new Error("frozen investigation changed or invalid");
|
|
123
|
-
}
|
|
124
|
-
/** The host persists these immutable selections; this function neither schedules a process nor spends tokens. */
|
|
125
|
-
export function selectWeeklyInvestigation(history, input) {
|
|
126
|
-
const value = json(input);
|
|
127
|
-
if (!keys(value, ["week", "population", "policyDigest", "archiveSnapshot", "eligibleCaseIds", "maxCases"])
|
|
128
|
-
|| !/^\d{4}-W(?:0[1-9]|[1-4]\d|5[0-3])$/.test(value.week) || !text(value.population, 512)
|
|
129
|
-
|| !SHA.test(value.policyDigest) || !SHA.test(value.archiveSnapshot) || !Array.isArray(value.eligibleCaseIds)
|
|
130
|
-
|| !value.eligibleCaseIds.length || value.eligibleCaseIds.some(id => typeof id !== "string" || !SHA.test(id))
|
|
131
|
-
|| !Number.isSafeInteger(value.maxCases) || value.maxCases < 1 || value.maxCases > 32 || history.length > 4096)
|
|
132
|
-
throw new Error("invalid weekly investigation selection");
|
|
133
|
-
const eligible = [...new Set(value.eligibleCaseIds)].sort();
|
|
134
|
-
const key = digest({ week: value.week, population: value.population, policyDigest: value.policyDigest });
|
|
135
|
-
const inputDigest = digest({ ...value, eligibleCaseIds: eligible });
|
|
136
|
-
const selectedCaseIds = eligible.sort((a, b) => { const ka = digest([key, value.archiveSnapshot, a]), kb = digest([key, value.archiveSnapshot, b]); return ka < kb ? -1 : ka > kb ? 1 : 0; }).slice(0, value.maxCases);
|
|
137
|
-
const body = { version: "weekly-investigation-selection-v1", key, inputDigest, week: value.week, population: value.population,
|
|
138
|
-
policyDigest: value.policyDigest, archiveSnapshot: value.archiveSnapshot, selectedCaseIds, sampling: "exploratory-sha256-order-v1" };
|
|
139
|
-
const result = freeze({ ...body, id: digest(body) });
|
|
140
|
-
const previous = history.filter(record => record.key === key);
|
|
141
|
-
for (const record of previous) {
|
|
142
|
-
const { id, ...old } = json(record);
|
|
143
|
-
if (digest(old) !== id || id !== result.id)
|
|
144
|
-
throw new Error("weekly investigation already frozen or history invalid");
|
|
145
|
-
}
|
|
146
|
-
return previous.length ? freeze(json(previous[0])) : result;
|
|
147
|
-
}
|
|
148
|
-
//# sourceMappingURL=investigation.js.map
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import type { ResultsFile } from "./results.js";
|
|
2
|
-
import type { Verdict } from "./score.js";
|
|
3
|
-
export interface JudgeAgreementCell {
|
|
4
|
-
scenario: string;
|
|
5
|
-
status: "agree" | "disagree" | "error";
|
|
6
|
-
judges: string[];
|
|
7
|
-
verdicts: Verdict[];
|
|
8
|
-
detail: string;
|
|
9
|
-
}
|
|
10
|
-
export interface JudgeAgreementReport {
|
|
11
|
-
cells: JudgeAgreementCell[];
|
|
12
|
-
agree: number;
|
|
13
|
-
disagree: number;
|
|
14
|
-
error: number;
|
|
15
|
-
comparable: number;
|
|
16
|
-
rate: number | null;
|
|
17
|
-
}
|
|
18
|
-
/** Pure/offline projection of judgments already persisted in results.yaml. */
|
|
19
|
-
export declare function judgeAgreement(results: ResultsFile): JudgeAgreementReport;
|
package/dist/judge-agreement.js
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
/** Pure/offline projection of judgments already persisted in results.yaml. */
|
|
2
|
-
export function judgeAgreement(results) {
|
|
3
|
-
const cells = results.scenarios.map((scenario) => {
|
|
4
|
-
const judgments = scenario.judge_history ?? scenario.adjudication?.judgments ?? [];
|
|
5
|
-
const distinct = new Map();
|
|
6
|
-
for (const judgment of judgments) {
|
|
7
|
-
const key = `${judgment.judge.provider}:${judgment.judge.model}`;
|
|
8
|
-
distinct.delete(key); // insertion order now reflects each judge's latest persisted vote
|
|
9
|
-
distinct.set(key, judgment);
|
|
10
|
-
}
|
|
11
|
-
const pair = [...distinct.entries()].slice(-2);
|
|
12
|
-
if (pair.length < 2 || pair.some(([, judgment]) => judgment.suspect || judgment.verdict === "ERROR" || judgment.verdict === "NOT-MEASURED" || judgment.verdict === "JUDGE-AMBIGUOUS")) {
|
|
13
|
-
return {
|
|
14
|
-
scenario: scenario.id, status: "error", judges: pair.map(([judge]) => judge),
|
|
15
|
-
verdicts: pair.map(([, judgment]) => judgment.verdict),
|
|
16
|
-
detail: pair.length < 2 ? "fewer than two distinct persisted judges" : "one or more judgments is not a clean vote",
|
|
17
|
-
};
|
|
18
|
-
}
|
|
19
|
-
const status = pair[0][1].verdict === pair[1][1].verdict ? "agree" : "disagree";
|
|
20
|
-
return {
|
|
21
|
-
scenario: scenario.id, status, judges: pair.map(([judge]) => judge),
|
|
22
|
-
verdicts: pair.map(([, judgment]) => judgment.verdict), detail: `${pair[0][1].verdict} vs ${pair[1][1].verdict}`,
|
|
23
|
-
};
|
|
24
|
-
});
|
|
25
|
-
const agree = cells.filter((cell) => cell.status === "agree").length;
|
|
26
|
-
const disagree = cells.filter((cell) => cell.status === "disagree").length;
|
|
27
|
-
const error = cells.filter((cell) => cell.status === "error").length;
|
|
28
|
-
const comparable = agree + disagree;
|
|
29
|
-
return { cells, agree, disagree, error, comparable, rate: comparable ? agree / comparable : null };
|
|
30
|
-
}
|
|
31
|
-
//# sourceMappingURL=judge-agreement.js.map
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
export interface QualificationOutputReceipt {
|
|
2
|
-
path: string;
|
|
3
|
-
total_bytes: number;
|
|
4
|
-
captured_bytes: number;
|
|
5
|
-
truncated: boolean;
|
|
6
|
-
sha256: string;
|
|
7
|
-
}
|
|
8
|
-
export interface QualificationBoundedCapture {
|
|
9
|
-
path: string;
|
|
10
|
-
total_bytes: number;
|
|
11
|
-
captured_bytes: number;
|
|
12
|
-
truncated: boolean;
|
|
13
|
-
write(chunk: Buffer): void;
|
|
14
|
-
close(): void;
|
|
15
|
-
}
|
|
16
|
-
export interface QualificationArtifactAttestation {
|
|
17
|
-
ok: boolean;
|
|
18
|
-
actual: {
|
|
19
|
-
provider: string;
|
|
20
|
-
model: string;
|
|
21
|
-
} | null;
|
|
22
|
-
fallback: boolean;
|
|
23
|
-
refused: boolean;
|
|
24
|
-
error: string | null;
|
|
25
|
-
}
|
|
26
|
-
export declare function redactQualificationOutput(text: string, secretValues?: readonly string[]): string;
|
|
27
|
-
export declare function openQualificationBoundedCapture(path: string, limit: number, redact: (text: string) => string): QualificationBoundedCapture;
|
|
28
|
-
export declare function qualificationOutputReceipt(spoolRoot: string, path: string, capture: QualificationBoundedCapture): QualificationOutputReceipt;
|
|
29
|
-
export declare function verifyQualificationOutputReceipt(spoolRoot: string, receipt: QualificationOutputReceipt): void;
|
|
30
|
-
export declare function attestQualificationPiJsonl(text: string, requested: {
|
|
31
|
-
provider: string;
|
|
32
|
-
model: string;
|
|
33
|
-
}): QualificationArtifactAttestation;
|
|
34
|
-
export declare function spawnQualificationCapture(command: string, argv: string[], options: {
|
|
35
|
-
cwd: string;
|
|
36
|
-
env: NodeJS.ProcessEnv;
|
|
37
|
-
timeout_ms: number;
|
|
38
|
-
output_limit_bytes: number;
|
|
39
|
-
}): Promise<{
|
|
40
|
-
stdout: string;
|
|
41
|
-
stderr: string;
|
|
42
|
-
code: number | null;
|
|
43
|
-
timed_out: boolean;
|
|
44
|
-
}>;
|
|
@@ -1,163 +0,0 @@
|
|
|
1
|
-
import { spawn } from "node:child_process";
|
|
2
|
-
import { closeSync, constants, fstatSync, fsyncSync, openSync, readFileSync, writeSync } from "node:fs";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
import { StringDecoder } from "node:string_decoder";
|
|
5
|
-
import { redactText } from "./capture.js";
|
|
6
|
-
import { qualificationSha256 } from "./qualification-config.js";
|
|
7
|
-
export function redactQualificationOutput(text, secretValues = []) {
|
|
8
|
-
let output = text;
|
|
9
|
-
for (const secret of [...new Set(secretValues)].sort((left, right) => right.length - left.length)) {
|
|
10
|
-
if (secret.length >= 8)
|
|
11
|
-
output = output.split(secret).join("[REDACTED credential]");
|
|
12
|
-
}
|
|
13
|
-
output = output
|
|
14
|
-
.replace(/\b([A-Z][A-Z0-9_]*(?:API_KEY|ACCESS_KEY|SECRET_KEY|AUTH_TOKEN|BEARER_TOKEN|SESSION_TOKEN))\s*[:=]\s*[^\s,;"']+/gi, "$1=[REDACTED credential]")
|
|
15
|
-
.replace(/\bBearer\s+[A-Za-z0-9._~+\/-]{8,}/gi, "Bearer [REDACTED credential]")
|
|
16
|
-
.replace(/\b(?:sk|sess|key|token)-[A-Za-z0-9._-]{8,}/gi, "[REDACTED credential]")
|
|
17
|
-
.replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, "[REDACTED credential]");
|
|
18
|
-
return redactText(output);
|
|
19
|
-
}
|
|
20
|
-
export function openQualificationBoundedCapture(path, limit, redact) {
|
|
21
|
-
const fd = openSync(path, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
|
|
22
|
-
const decoder = new StringDecoder("utf8");
|
|
23
|
-
let pending = "";
|
|
24
|
-
let acceptedRawBytes = 0;
|
|
25
|
-
let closed = false;
|
|
26
|
-
const persist = (text) => {
|
|
27
|
-
if (!text)
|
|
28
|
-
return;
|
|
29
|
-
const sanitized = Buffer.from(redact(text), "utf8");
|
|
30
|
-
const remaining = Math.max(0, limit - state.captured_bytes);
|
|
31
|
-
const part = sanitized.subarray(0, remaining);
|
|
32
|
-
if (part.length > 0) {
|
|
33
|
-
writeSync(fd, part);
|
|
34
|
-
state.captured_bytes += part.length;
|
|
35
|
-
fsyncSync(fd);
|
|
36
|
-
}
|
|
37
|
-
if (sanitized.length > remaining)
|
|
38
|
-
state.truncated = true;
|
|
39
|
-
};
|
|
40
|
-
const flushLines = () => {
|
|
41
|
-
let newline = pending.indexOf("\n");
|
|
42
|
-
while (newline >= 0) {
|
|
43
|
-
persist(pending.slice(0, newline + 1));
|
|
44
|
-
pending = pending.slice(newline + 1);
|
|
45
|
-
newline = pending.indexOf("\n");
|
|
46
|
-
}
|
|
47
|
-
};
|
|
48
|
-
const state = {
|
|
49
|
-
path, total_bytes: 0, captured_bytes: 0, truncated: false,
|
|
50
|
-
write(chunk) {
|
|
51
|
-
state.total_bytes += chunk.length;
|
|
52
|
-
const remainingRaw = Math.max(0, limit - acceptedRawBytes);
|
|
53
|
-
const accepted = chunk.subarray(0, remainingRaw);
|
|
54
|
-
acceptedRawBytes += accepted.length;
|
|
55
|
-
if (accepted.length > 0) {
|
|
56
|
-
pending += decoder.write(accepted);
|
|
57
|
-
flushLines();
|
|
58
|
-
}
|
|
59
|
-
if (chunk.length > remainingRaw)
|
|
60
|
-
state.truncated = true;
|
|
61
|
-
},
|
|
62
|
-
close() {
|
|
63
|
-
if (closed)
|
|
64
|
-
return;
|
|
65
|
-
pending += decoder.end();
|
|
66
|
-
// If the raw limit split a line, discard that incomplete tail rather than
|
|
67
|
-
// persisting a credential prefix that cannot be matched as a whole value.
|
|
68
|
-
if (!state.truncated)
|
|
69
|
-
persist(pending);
|
|
70
|
-
pending = "";
|
|
71
|
-
fsyncSync(fd);
|
|
72
|
-
closeSync(fd);
|
|
73
|
-
closed = true;
|
|
74
|
-
},
|
|
75
|
-
};
|
|
76
|
-
return state;
|
|
77
|
-
}
|
|
78
|
-
export function qualificationOutputReceipt(spoolRoot, path, capture) {
|
|
79
|
-
const bytes = readFileSync(path);
|
|
80
|
-
return {
|
|
81
|
-
path: path.slice(spoolRoot.length + 1),
|
|
82
|
-
total_bytes: capture.total_bytes,
|
|
83
|
-
captured_bytes: bytes.length,
|
|
84
|
-
truncated: capture.truncated,
|
|
85
|
-
sha256: qualificationSha256(bytes),
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
export function verifyQualificationOutputReceipt(spoolRoot, receipt) {
|
|
89
|
-
const path = join(spoolRoot, receipt.path);
|
|
90
|
-
let fd;
|
|
91
|
-
try {
|
|
92
|
-
fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
93
|
-
const stat = fstatSync(fd);
|
|
94
|
-
if (!stat.isFile())
|
|
95
|
-
throw new Error(`qualification output ${receipt.path} must be a regular non-symlink file`);
|
|
96
|
-
const bytes = readFileSync(fd);
|
|
97
|
-
if (bytes.length !== receipt.captured_bytes || qualificationSha256(bytes) !== receipt.sha256) {
|
|
98
|
-
throw new Error(`qualification output ${receipt.path} digest/size mismatch`);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
finally {
|
|
102
|
-
if (fd !== undefined)
|
|
103
|
-
closeSync(fd);
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
export function attestQualificationPiJsonl(text, requested) {
|
|
107
|
-
const records = [];
|
|
108
|
-
try {
|
|
109
|
-
for (const line of text.split("\n")) {
|
|
110
|
-
if (!line.trim())
|
|
111
|
-
continue;
|
|
112
|
-
const value = JSON.parse(line);
|
|
113
|
-
if (!plainObject(value))
|
|
114
|
-
throw new Error("event is not an object");
|
|
115
|
-
records.push(value);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
catch {
|
|
119
|
-
return { ok: false, actual: null, fallback: false, refused: false, error: "artifact is not valid JSONL" };
|
|
120
|
-
}
|
|
121
|
-
const fallback = records.some((record) => record.type === "provider_fallback" || record.type === "model_fallback" || Object.hasOwn(record, "fallback_provider") || Object.hasOwn(record, "fallback_model"));
|
|
122
|
-
const messages = records
|
|
123
|
-
.filter((record) => record.type === "message_end" && plainObject(record.message) && record.message.role === "assistant")
|
|
124
|
-
.map((record) => record.message);
|
|
125
|
-
if (messages.length === 0)
|
|
126
|
-
return { ok: false, actual: null, fallback, refused: false, error: "artifact has no authoritative assistant provider/model identity" };
|
|
127
|
-
const identities = messages.map((message) => ({ provider: stringValue(message.provider), model: stringValue(message.model) }));
|
|
128
|
-
if (identities.some((identity) => !identity.provider || !identity.model))
|
|
129
|
-
return { ok: false, actual: null, fallback, refused: false, error: "artifact assistant identity is missing provider or model" };
|
|
130
|
-
const actual = { provider: identities.at(-1).provider, model: identities.at(-1).model };
|
|
131
|
-
if (fallback)
|
|
132
|
-
return { ok: false, actual, fallback: true, refused: false, error: "artifact reports provider/model fallback" };
|
|
133
|
-
if (identities.some((identity) => identity.provider !== requested.provider))
|
|
134
|
-
return { ok: false, actual, fallback: false, refused: false, error: `provider substitution: requested ${requested.provider}, observed ${actual.provider}` };
|
|
135
|
-
if (identities.some((identity) => identity.model !== requested.model))
|
|
136
|
-
return { ok: false, actual, fallback: false, refused: false, error: `model substitution: requested ${requested.model}, observed ${actual.model}` };
|
|
137
|
-
const terminal = messages.at(-1);
|
|
138
|
-
const errorMessage = stringValue(terminal.errorMessage) ?? "";
|
|
139
|
-
const refused = terminal.stopReason === "error" && /(?:refus|not supported|not available|subscription|usage limit|unauthor)/i.test(errorMessage);
|
|
140
|
-
if (terminal.stopReason === "error" && !refused)
|
|
141
|
-
return { ok: false, actual, fallback: false, refused: false, error: errorMessage || "assistant ended with an error" };
|
|
142
|
-
return { ok: true, actual, fallback: false, refused, error: refused ? errorMessage : null };
|
|
143
|
-
}
|
|
144
|
-
export async function spawnQualificationCapture(command, argv, options) {
|
|
145
|
-
return new Promise((resolve, reject) => {
|
|
146
|
-
const child = spawn(command, argv, { cwd: options.cwd, env: options.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
147
|
-
let stdout = Buffer.alloc(0), stderr = Buffer.alloc(0), timedOut = false;
|
|
148
|
-
const append = (current, chunk) => Buffer.concat([current, chunk]).subarray(0, options.output_limit_bytes);
|
|
149
|
-
child.stdout.on("data", (chunk) => { stdout = append(stdout, chunk); });
|
|
150
|
-
child.stderr.on("data", (chunk) => { stderr = append(stderr, chunk); });
|
|
151
|
-
const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, options.timeout_ms);
|
|
152
|
-
child.once("error", (error) => { clearTimeout(timer); reject(error); });
|
|
153
|
-
child.once("close", (code) => { clearTimeout(timer); resolve({ stdout: stdout.toString("utf8"), stderr: stderr.toString("utf8"), code, timed_out: timedOut }); });
|
|
154
|
-
});
|
|
155
|
-
}
|
|
156
|
-
function plainObject(value) {
|
|
157
|
-
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
158
|
-
return false;
|
|
159
|
-
const prototype = Object.getPrototypeOf(value);
|
|
160
|
-
return prototype === Object.prototype || prototype === null;
|
|
161
|
-
}
|
|
162
|
-
function stringValue(value) { return typeof value === "string" ? value : undefined; }
|
|
163
|
-
//# sourceMappingURL=qualification-capture.js.map
|