@skill-harness/cli 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/archive-learning.d.ts +3 -0
- package/dist/archive-learning.js +153 -0
- package/dist/archive.d.ts +3 -0
- package/dist/archive.js +52 -0
- package/dist/cli.d.ts +0 -1
- package/dist/cli.js +6 -16
- package/package.json +3 -3
- package/dist/mutation-catalogue.d.ts +0 -8
- package/dist/mutation-catalogue.js +0 -95
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { constants, openSync, readSync, fstatSync, closeSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute } from 'node:path';
|
|
3
|
+
import { createArchiveAccess, openArchiveAccess, readFixedOrderFacts, createWeeklyInvestigation, openWeeklyInvestigation, createTrustLifecycle, openTrustLifecycle } from '@skill-harness/adapters';
|
|
4
|
+
import { parseLearningRequest } from '@skill-harness/core';
|
|
5
|
+
/** Explicit operator requests, not a scheduler or model route. The inert host cannot invoke this CLI. */
|
|
6
|
+
export function cmdArchiveLearning(args) {
|
|
7
|
+
if (args._.length !== 1 || Object.keys(args.flags).sort().join() !== 'request,state' || typeof args.flags.state !== 'string' || !isAbsolute(args.flags.state) || typeof args.flags.request !== 'string')
|
|
8
|
+
throw Error('archive weekly|trust|access requires --state /absolute/directory --request file');
|
|
9
|
+
const fd = openSync(args.flags.request, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
10
|
+
let text;
|
|
11
|
+
try {
|
|
12
|
+
const stat = fstatSync(fd);
|
|
13
|
+
if (!stat.isFile() || stat.size > 65536)
|
|
14
|
+
throw Error('bounded regular learning request required');
|
|
15
|
+
const bytes = Buffer.alloc(65537);
|
|
16
|
+
let n = 0;
|
|
17
|
+
while (n < bytes.length) {
|
|
18
|
+
const k = readSync(fd, bytes, n, bytes.length - n, n);
|
|
19
|
+
if (!k)
|
|
20
|
+
break;
|
|
21
|
+
n += k;
|
|
22
|
+
}
|
|
23
|
+
if (n > 65536)
|
|
24
|
+
throw Error('learning request exceeds bound');
|
|
25
|
+
text = new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, n));
|
|
26
|
+
}
|
|
27
|
+
finally {
|
|
28
|
+
closeSync(fd);
|
|
29
|
+
}
|
|
30
|
+
const request = parseLearningRequest(text);
|
|
31
|
+
if (Object.keys(request).sort().join() !== 'authority,input,operation')
|
|
32
|
+
throw Error('closed learning request required');
|
|
33
|
+
const input = request.input, authority = request.authority, state = args.flags.state;
|
|
34
|
+
let result;
|
|
35
|
+
if (args._[0] === 'access') {
|
|
36
|
+
if (request.operation === 'create')
|
|
37
|
+
result = createArchiveAccess(state, input, authority).inspect();
|
|
38
|
+
else {
|
|
39
|
+
const access = openArchiveAccess(state);
|
|
40
|
+
switch (request.operation) {
|
|
41
|
+
case 'inspect':
|
|
42
|
+
result = access.inspect();
|
|
43
|
+
break;
|
|
44
|
+
case 'preview-consent':
|
|
45
|
+
result = access.previewConsent(input);
|
|
46
|
+
break;
|
|
47
|
+
case 'consent': {
|
|
48
|
+
const p = input;
|
|
49
|
+
result = access.consent(p.grant, authority, p.now);
|
|
50
|
+
break;
|
|
51
|
+
}
|
|
52
|
+
case 'preview-revocation':
|
|
53
|
+
result = access.previewRevocation(input);
|
|
54
|
+
break;
|
|
55
|
+
case 'revoke': {
|
|
56
|
+
const p = input;
|
|
57
|
+
access.revoke(p.id, authority, p.now);
|
|
58
|
+
result = { recorded: true };
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
case 'facts': {
|
|
62
|
+
const p = input;
|
|
63
|
+
result = readFixedOrderFacts(access.inspect().policy.archiveRoot, p.manifestId, { directory: state, purpose: p.purpose }, p.expected, p.now);
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
default: throw Error('unsupported access operation; raw read/export is not a CLI operation');
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
else if (args._[0] === 'weekly') {
|
|
71
|
+
if (request.operation === 'create') {
|
|
72
|
+
result = createWeeklyInvestigation(state, input).inspect();
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
const job = openWeeklyInvestigation(state);
|
|
76
|
+
switch (request.operation) {
|
|
77
|
+
case 'inspect':
|
|
78
|
+
result = job.inspect();
|
|
79
|
+
break;
|
|
80
|
+
case 'run':
|
|
81
|
+
result = job.run(request.input);
|
|
82
|
+
break;
|
|
83
|
+
case 'approve':
|
|
84
|
+
result = job.approve(authority);
|
|
85
|
+
break;
|
|
86
|
+
case 'preview': {
|
|
87
|
+
const p = request.input;
|
|
88
|
+
result = job.preview(p.specPath, p.scenario);
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
case 'promote':
|
|
92
|
+
result = job.promote(input, authority);
|
|
93
|
+
break;
|
|
94
|
+
case 'freeze':
|
|
95
|
+
result = job.freeze(input, authority);
|
|
96
|
+
break;
|
|
97
|
+
case 'preview-edit':
|
|
98
|
+
result = job.previewEdit(input);
|
|
99
|
+
break;
|
|
100
|
+
case 'edit':
|
|
101
|
+
result = job.edit(input, authority);
|
|
102
|
+
break;
|
|
103
|
+
case 'evaluation':
|
|
104
|
+
result = job.evaluation();
|
|
105
|
+
break;
|
|
106
|
+
case 'preview-screen':
|
|
107
|
+
result = job.previewScreen(input);
|
|
108
|
+
break;
|
|
109
|
+
case 'screen':
|
|
110
|
+
result = job.screen(input, authority);
|
|
111
|
+
break;
|
|
112
|
+
default: throw Error('unsupported weekly operation');
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
else if (args._[0] === 'trust') {
|
|
117
|
+
if (request.operation === 'create') {
|
|
118
|
+
result = createTrustLifecycle(state, input, authority).inspect(0);
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
const job = openTrustLifecycle(state);
|
|
122
|
+
switch (request.operation) {
|
|
123
|
+
case 'inspect':
|
|
124
|
+
result = job.inspect(input);
|
|
125
|
+
break;
|
|
126
|
+
case 'predict':
|
|
127
|
+
job.predict(input);
|
|
128
|
+
result = { recorded: true };
|
|
129
|
+
break;
|
|
130
|
+
case 'correct': {
|
|
131
|
+
const p = request.input;
|
|
132
|
+
job.correct(p.prior, p.prediction, p.reason);
|
|
133
|
+
result = { recorded: true };
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
case 'outcome':
|
|
137
|
+
job.outcome(input, authority);
|
|
138
|
+
result = { recorded: true };
|
|
139
|
+
break;
|
|
140
|
+
case 'expose': {
|
|
141
|
+
const p = request.input;
|
|
142
|
+
result = job.expose(p.id, p.now);
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
default: throw Error('unsupported trust operation');
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
else
|
|
150
|
+
throw Error('unsupported learning family');
|
|
151
|
+
console.log(JSON.stringify(result, null, 2));
|
|
152
|
+
}
|
|
153
|
+
//# sourceMappingURL=archive-learning.js.map
|
package/dist/archive.js
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { ingestPolicySource, inspectPolicyCheckpoint, observeArchiveSource } from "@skill-harness/adapters";
|
|
2
|
+
import { cmdArchiveLearning } from "./archive-learning.js";
|
|
3
|
+
/** Explicit file ingestion/metadata inspection only; never launches a worker or publishes content. */
|
|
4
|
+
export async function cmdArchive(args) {
|
|
5
|
+
const operation = args._[0];
|
|
6
|
+
if (operation === "weekly" || operation === "trust" || operation === "access") {
|
|
7
|
+
cmdArchiveLearning(args);
|
|
8
|
+
return;
|
|
9
|
+
}
|
|
10
|
+
if (operation === "watch") {
|
|
11
|
+
if (args._.length !== 1 || Object.keys(args.flags).some(key => !["policy", "source", "previous", "interval-ms", "max-polls"].includes(key)))
|
|
12
|
+
throw new Error("unsupported archive watch option");
|
|
13
|
+
const required = (key) => { const value = args.flags[key]; if (typeof value !== "string" || !value)
|
|
14
|
+
throw new Error(`archive watch requires --${key}`); return value; };
|
|
15
|
+
const signal = new AbortController(), abort = () => signal.abort();
|
|
16
|
+
process.on("SIGINT", abort);
|
|
17
|
+
process.on("SIGTERM", abort);
|
|
18
|
+
try {
|
|
19
|
+
const result = await observeArchiveSource({ policyPath: required("policy"), sourceId: required("source"),
|
|
20
|
+
previousCheckpointId: args.flags.previous === undefined ? undefined : required("previous"),
|
|
21
|
+
intervalMs: args.flags["interval-ms"] === undefined ? 1000 : Number(required("interval-ms")), maxPolls: Number(required("max-polls")), signal: signal.signal,
|
|
22
|
+
onObservation: observation => console.log(JSON.stringify({ checkpointId: observation.checkpointId, policySha256: observation.policySha256, syntax: observation.syntax })),
|
|
23
|
+
});
|
|
24
|
+
console.log(JSON.stringify(result));
|
|
25
|
+
}
|
|
26
|
+
finally {
|
|
27
|
+
process.removeListener("SIGINT", abort);
|
|
28
|
+
process.removeListener("SIGTERM", abort);
|
|
29
|
+
}
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
if (args._.length !== 1 || !["ingest", "inspect"].includes(operation)) {
|
|
33
|
+
throw new Error("usage: archive ingest|inspect --policy <file> --source <id> [--previous <checkpoint> | --checkpoint <id>]");
|
|
34
|
+
}
|
|
35
|
+
const allowed = operation === "ingest" ? ["policy", "source", "previous"] : ["policy", "source", "checkpoint"];
|
|
36
|
+
for (const key of Object.keys(args.flags))
|
|
37
|
+
if (!allowed.includes(key))
|
|
38
|
+
throw new Error(`unsupported archive option: ${key}`);
|
|
39
|
+
const required = (key) => {
|
|
40
|
+
const value = args.flags[key];
|
|
41
|
+
if (typeof value !== "string" || !value)
|
|
42
|
+
throw new Error(`archive requires --${key}`);
|
|
43
|
+
return value;
|
|
44
|
+
};
|
|
45
|
+
const policy = required("policy"), source = required("source");
|
|
46
|
+
const result = operation === "ingest"
|
|
47
|
+
? ingestPolicySource(policy, source, args.flags.previous === undefined ? undefined : required("previous"))
|
|
48
|
+
: inspectPolicyCheckpoint(policy, source, required("checkpoint"));
|
|
49
|
+
// Successful storage/reporting is not successful work: missing/partial fields are never promoted to acceptance.
|
|
50
|
+
console.log(JSON.stringify(result, null, 2));
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=archive.js.map
|
package/dist/cli.d.ts
CHANGED
|
@@ -31,7 +31,6 @@ export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter):
|
|
|
31
31
|
* than reconcile two numbers in prose.
|
|
32
32
|
*/
|
|
33
33
|
export declare function cmdJudgeAgreement(args: Args): void;
|
|
34
|
-
export declare function cmdMutationTest(): Promise<void>;
|
|
35
34
|
/**
|
|
36
35
|
* Re-evaluate needle gates against the saved staged diffs — no subject call. A rep
|
|
37
36
|
* whose gate verdict flips from fail to pass was never judged and costs one judge call.
|
package/dist/cli.js
CHANGED
|
@@ -3,12 +3,12 @@ import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync
|
|
|
3
3
|
import { load as yamlLoad } from "js-yaml";
|
|
4
4
|
import { basename, dirname, join, resolve, relative } from "node:path";
|
|
5
5
|
import { tmpdir } from "node:os";
|
|
6
|
-
import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, judgeAgreement, assertNotDowngraded, downgradeWarning, isScoredMode,
|
|
6
|
+
import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, resolveAdjudicationJudges, adjudicateRun, judgeResemblesSubject, computeCoverage, formatCoverage, selectAffected, formatAffected, gitDiff, exec, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, judgeAgreement, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, screenResults, formatScreen, } from "@skill-harness/core";
|
|
7
7
|
import { getAdapter } from "@skill-harness/adapters";
|
|
8
8
|
import { serveReview } from "./serve.js";
|
|
9
|
-
import { runSelfScreeningMutationCases } from "./mutation-catalogue.js";
|
|
10
9
|
import { runCompareCommand } from "./compare.js";
|
|
11
10
|
import { cmdQualification } from "./qualification.js";
|
|
11
|
+
import { cmdArchive } from "./archive.js";
|
|
12
12
|
const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
|
|
13
13
|
// The judge default lives in core (`defaultJudge()`), which resolves
|
|
14
14
|
// SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
|
|
@@ -404,18 +404,6 @@ export function cmdJudgeAgreement(args) {
|
|
|
404
404
|
console.log(`\njudge agreement: ${report.agree} agree / ${report.disagree} disagree / ${report.error} error; ${report.rate === null ? "n/a" : `${(report.rate * 100).toFixed(1)}%`} across ${report.comparable} comparable scenario(s).`);
|
|
405
405
|
console.log("offline report only; no model or judge calls.");
|
|
406
406
|
}
|
|
407
|
-
export async function cmdMutationTest() {
|
|
408
|
-
const trajectory = runTrajectoryMutationSelfTest();
|
|
409
|
-
const report = { baseline: trajectory.baseline, cases: [...trajectory.cases, ...await runSelfScreeningMutationCases()] };
|
|
410
|
-
console.log(`mutation self-test: baseline ${report.baseline}`);
|
|
411
|
-
for (const test of report.cases) {
|
|
412
|
-
console.log(` ${test.detected ? "✓" : "✗"} ${test.id}: ${test.status} — ${test.detail}`);
|
|
413
|
-
}
|
|
414
|
-
const missed = report.cases.filter((test) => !test.detected);
|
|
415
|
-
console.log(`\n${report.cases.length - missed.length}/${report.cases.length} mutations detected; no model or judge calls.`);
|
|
416
|
-
if (missed.length)
|
|
417
|
-
process.exitCode = 1;
|
|
418
|
-
}
|
|
419
407
|
async function cmdRescore(args) {
|
|
420
408
|
const runDirs = args._;
|
|
421
409
|
if (runDirs.length === 0)
|
|
@@ -906,7 +894,9 @@ export function help() {
|
|
|
906
894
|
[--auto-rejudge] [--secondary-judge p:m] [--tie-break-judge p:m]
|
|
907
895
|
ask again about untrustworthy cells (ambiguous / contradictory / non-unanimous /
|
|
908
896
|
ship-deciding). OFF by default; prints the exact MAX extra call count first.
|
|
909
|
-
|
|
897
|
+
archive ingest|inspect|watch --policy file --source id explicit external ingestion/metadata (${free("archive")})
|
|
898
|
+
archive weekly|trust|access --state /private/dir --request file durable learning/consent lifecycle (${free("archive")})
|
|
899
|
+
watch requires --max-polls N; optional --interval-ms N and --previous checkpoint
|
|
910
900
|
judge-agreement <run-dir> compare two distinct persisted judge votes per scenario (${free("judge-agreement")})
|
|
911
901
|
rescore <run-dir>... re-score saved reps vs current spec thresholds (${free("rescore")})
|
|
912
902
|
regate <run-dir>... [--judge prov:model] re-evaluate saved gates (no subject call; judges fail→pass reps)
|
|
@@ -943,7 +933,7 @@ export async function main(argv) {
|
|
|
943
933
|
case "run": return cmdRun(args);
|
|
944
934
|
case "compare": return cmdCompare(args);
|
|
945
935
|
case "grade": return cmdGrade(args);
|
|
946
|
-
case "
|
|
936
|
+
case "archive": return cmdArchive(args);
|
|
947
937
|
case "judge-agreement": return cmdJudgeAgreement(args);
|
|
948
938
|
case "rescore": return cmdRescore(args);
|
|
949
939
|
case "regate": return cmdRegate(args);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skill-harness/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
"prepack": "node ../../scripts/release-pack.mjs --guard"
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@skill-harness/core": "0.
|
|
48
|
-
"@skill-harness/adapters": "0.
|
|
47
|
+
"@skill-harness/core": "0.14.0",
|
|
48
|
+
"@skill-harness/adapters": "0.14.0"
|
|
49
49
|
}
|
|
50
50
|
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
export interface PermanentMutationCase {
|
|
2
|
-
id: string;
|
|
3
|
-
detected: boolean;
|
|
4
|
-
status: "FAIL" | "ERROR";
|
|
5
|
-
detail: string;
|
|
6
|
-
}
|
|
7
|
-
/** Permanent non-trajectory mutation cases exercised by the free mutation-test command. */
|
|
8
|
-
export declare function runSelfScreeningMutationCases(): Promise<PermanentMutationCase[]>;
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { tmpdir } from "node:os";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
import { parseSpec, planAdjudication, runSkillModel, screenResults, validateResults, } from "@skill-harness/core";
|
|
5
|
-
import { authenticatePromptObservation, bindPromptObservation, normalizePromptPayload, observeProviderPayload, promptCaptureIsTrusted, } from "@skill-harness/adapters";
|
|
6
|
-
const h = "a".repeat(64);
|
|
7
|
-
const deliveryAssertion = (status = "PASS") => ({ kind: "skill_delivered", status, detail: status });
|
|
8
|
-
function validV3() {
|
|
9
|
-
return {
|
|
10
|
-
schema: 3, skill: "mutation", harness: "pi", model: "fake:m", judge: { provider: "fake", model: "j" },
|
|
11
|
-
timestamp: "t", label: null, mode: "force", effective_grade: { passed: 1, total: 1, pct: 100, letter: "A", ship: true, note: "" },
|
|
12
|
-
subject_invocations: [{ scenario_id: "A1", repetition: 0, prompt: { capture_version: "prompt-provenance-v1", request_index: 0, raw_sha256: h, normalized_sha256: h, normalization_rule: "cwd-line-v1", bytes: 1, contract_sha256: h, contract_bytes: 1, contract_occurrences: 1, mechanism: "append-system-prompt", status: "PASS" } }],
|
|
13
|
-
scenarios: [{ id: "A1", criterion_count: 1, judge_verdict: "PASS", judge_reason: "ok", suspect: false, override: null, note: "", objective: { status: "PASS", assertions: [deliveryAssertion()] }, rep_judgments: [{ repetition: 0, recorded_verdict: "PASS", objective: { status: "PASS", assertions: [deliveryAssertion()] }, judgments: [{ ordinal: 1, judge: { provider: "fake", model: "j" }, verdict: "PASS", reason: "ok", suspect: false, criteria: [{ index: 1, verdict: "PASS", reason: "ok" }] }] }] }],
|
|
14
|
-
};
|
|
15
|
-
}
|
|
16
|
-
function catches(id, mutate, detail) {
|
|
17
|
-
const result = structuredClone(validV3());
|
|
18
|
-
mutate(result);
|
|
19
|
-
let detected = false;
|
|
20
|
-
try {
|
|
21
|
-
validateResults(result);
|
|
22
|
-
}
|
|
23
|
-
catch {
|
|
24
|
-
detected = true;
|
|
25
|
-
}
|
|
26
|
-
return { id, detected, status: detected ? "FAIL" : "ERROR", detail };
|
|
27
|
-
}
|
|
28
|
-
function screenFile(id, verdicts) {
|
|
29
|
-
const result = validV3();
|
|
30
|
-
result.scenarios[0].id = id;
|
|
31
|
-
result.scenarios[0].reps = verdicts.length;
|
|
32
|
-
result.subject_invocations = verdicts.map((_, repetition) => ({ ...structuredClone(result.subject_invocations[0]), scenario_id: id, repetition }));
|
|
33
|
-
result.scenarios[0].rep_judgments = verdicts.map((verdict, repetition) => ({ repetition, recorded_verdict: verdict, objective: { status: "PASS", assertions: [deliveryAssertion()] }, judgments: [{ ordinal: 1, judge: { provider: "fake", model: "j" }, verdict, reason: verdict, suspect: false, criteria: [{ index: 1, verdict, reason: verdict }] }] }));
|
|
34
|
-
return result;
|
|
35
|
-
}
|
|
36
|
-
async function objectiveSuppressesJudge() {
|
|
37
|
-
const dir = mkdtempSync(join(tmpdir(), "skill-harness-mutation-delivery-"));
|
|
38
|
-
let judgeCalls = 0;
|
|
39
|
-
try {
|
|
40
|
-
mkdirSync(join(dir, "tests"), { recursive: true });
|
|
41
|
-
writeFileSync(join(dir, "SKILL.md"), "---\nname: mutation\ndescription: fixture\n---\n\n## Contract\nDo it.\n", "utf8");
|
|
42
|
-
const specPath = join(dir, "tests/specification.yaml");
|
|
43
|
-
writeFileSync(specPath, "skill: mutation\njudge_persona: strict\nship_bar: { total: 1, min_pass: 1, no_critical_fail: true }\ncritical: []\nscenarios:\n - id: A1\n title: delivery\n turns: [hi]\n checklist: [does it]\n", "utf8");
|
|
44
|
-
const spec = parseSpec(readFileSync(specPath, "utf8"), specPath);
|
|
45
|
-
const adapter = {
|
|
46
|
-
name: "pi", observesPrompts: true, available: async () => true,
|
|
47
|
-
run: async (req) => { req.onPromptObservation?.({ capture_version: "prompt-provenance-v1", request_index: 0, raw_sha256: h, normalized_sha256: h, normalization_rule: "cwd-line-v1", bytes: 1, contract_sha256: h, contract_bytes: 1, contract_occurrences: 0, mechanism: "append-system-prompt", status: "NOT-MEASURED" }); return ">>> USER:\nhi\n\n<<< ASSISTANT:\nanswer\n"; },
|
|
48
|
-
judge: async () => { judgeCalls++; return "VERDICT: PASS\nREASON: wrong"; },
|
|
49
|
-
};
|
|
50
|
-
const out = await runSkillModel({ spec, skillDir: dir, specPath, adapter, model: { provider: "fake", model: "m" }, modelToken: "fake:m", judge: { provider: "fake-judge", model: "j" }, mode: "force", timestamp: "2026-09-04T01:00:00.000Z" });
|
|
51
|
-
const detected = judgeCalls === 0 && out.results.scenarios[0].judge_verdict === "NOT-MEASURED" && out.results.effective_grade.total === 0;
|
|
52
|
-
return { id: "delivery-objective-suppresses-judge", detected, status: detected ? "FAIL" : "ERROR", detail: "undelivered subject must never reach the judge or efficacy denominator" };
|
|
53
|
-
}
|
|
54
|
-
finally {
|
|
55
|
-
rmSync(dir, { recursive: true, force: true });
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
/** Permanent non-trajectory mutation cases exercised by the free mutation-test command. */
|
|
59
|
-
export async function runSelfScreeningMutationCases() {
|
|
60
|
-
const contract = "## Contract\nDo it\n";
|
|
61
|
-
const one = observeProviderPayload({ instructions: contract }, contract, "append-system-prompt", 0);
|
|
62
|
-
const forged = { ...one, contract_sha256: "f".repeat(64) };
|
|
63
|
-
const ceiling = screenFile("C", ["PASS", "PASS", "PASS", "PASS", "FAIL"]);
|
|
64
|
-
for (const observation of ceiling.subject_invocations)
|
|
65
|
-
observation.prompt = { ...observation.prompt, mechanism: "none", contract_occurrences: 0, status: "PASS" };
|
|
66
|
-
const undelivered = screenFile("U", ["PASS", "PASS"]);
|
|
67
|
-
undelivered.subject_invocations[1].prompt = { ...undelivered.subject_invocations[1].prompt, contract_occurrences: 0, status: "NOT-MEASURED" };
|
|
68
|
-
const suspect = screenFile("S", ["PASS"]);
|
|
69
|
-
suspect.scenarios[0].rep_judgments[0].judgments[0].suspect = true;
|
|
70
|
-
const cases = [
|
|
71
|
-
catches("schema-v3-missing-observation-rejected", r => { delete r.subject_invocations; }, "schema v3 must reject missing delivery evidence"),
|
|
72
|
-
catches("schema-v3-panel-divergence-rejected", r => { r.scenarios[0].rep_judgments[0].recorded_verdict = "FAIL"; }, "stored panel result must equal recomputed clean votes"),
|
|
73
|
-
catches("schema-v3-missing-criterion-rejected", r => { r.scenarios[0].rep_judgments[0].judgments[0].criteria.pop(); }, "criterion omissions must remain explicit"),
|
|
74
|
-
catches("schema-v3-per-repetition-objective-rejected", r => { delete r.scenarios[0].rep_judgments[0].objective; }, "aggregate objective cannot hide a repetition"),
|
|
75
|
-
catches("schema-v3-top-level-verdict-rejected", r => { const panel = r.scenarios[0].rep_judgments[0]; panel.judgments[0].verdict = "FAIL"; panel.judgments[0].criteria = panel.judgments[0].criteria.map(vote => ({ ...vote, verdict: "FAIL" })); panel.recorded_verdict = "FAIL"; }, "top-level verdict must match retained repetition evidence"),
|
|
76
|
-
catches("schema-v3-unsupported-pass-rejected", r => { r.scenarios[0].rep_judgments[0].judgments = []; }, "behavioral PASS requires a retained clean judgment"),
|
|
77
|
-
catches("schema-v3-adjudication-state-rejected", r => { const primary = r.scenarios[0].rep_judgments[0].judgments[0]; r.scenarios[0].adjudication = { repetition: 0, trigger: "ship_deciding", state: "unresolved", judgments: [primary, { ...structuredClone(primary), ordinal: 2 }] }; }, "adjudication state and verdict must match clean votes"),
|
|
78
|
-
{ id: "delivery-zero-not-measured", detected: observeProviderPayload({ instructions: "none" }, contract, "append-system-prompt", 0).status === "NOT-MEASURED", status: "FAIL", detail: "zero delivery is not product failure" },
|
|
79
|
-
{ id: "delivery-duplicate-not-measured", detected: observeProviderPayload({ instructions: contract + contract }, contract, "append-system-prompt", 0).status === "NOT-MEASURED", status: "FAIL", detail: "duplicate delivery is not valid measurement" },
|
|
80
|
-
{ id: "delivery-unobservable-error", detected: observeProviderPayload({}, contract, "append-system-prompt", 0).status === "ERROR", status: "FAIL", detail: "unsupported payload is instrument error" },
|
|
81
|
-
await objectiveSuppressesJudge(),
|
|
82
|
-
{ id: "delivery-error-suppresses-adjudication", detected: planAdjudication({ cells: [{ id: "A1", verdict: "ERROR", reason: "instrument", suspect: false, deliveryStatus: "ERROR" }], scenarios: [{ id: "A1", title: "A1", critical: false, mode: "inline", turns: ["x"], checklist: ["y"], workspace: "none", remote: false }], shipBar: { total: 1, min_pass: 1, no_critical_fail: true }, critical: [], tieBreakAvailable: true }).maxAdditionalCalls === 0, status: "FAIL", detail: "instrument delivery ERROR cannot authorize adjudication spend" },
|
|
83
|
-
{ id: "observer-normalization-scope", detected: JSON.stringify(normalizePromptPayload({ instructions: "Keep /a\nCurrent working directory: /a\nKeep /a" }, "cwd-line-v1")) === JSON.stringify({ instructions: "Keep /a\nCurrent working directory:<normalized>\nKeep /a" }), status: "FAIL", detail: "normalization changes only the cwd line" },
|
|
84
|
-
{ id: "observer-contract-binding", detected: bindPromptObservation(authenticatePromptObservation(forged, "key"), contract, "append-system-prompt", 0, "key").status === "ERROR", status: "FAIL", detail: "child contract identity cannot replace parent binding" },
|
|
85
|
-
{ id: "observer-mac-tamper-rejected", detected: bindPromptObservation({ ...authenticatePromptObservation(one, "key"), mac: "0".repeat(64) }, contract, "append-system-prompt", 0, "key").status === "ERROR", status: "FAIL", detail: "complete child observation requires parent authentication" },
|
|
86
|
-
{ id: "observer-extension-forgery-rejected", detected: !promptCaptureIsTrusted(1), status: "FAIL", detail: "same-process subject extensions cannot produce trusted provenance" },
|
|
87
|
-
{ id: "observer-argv-wiring", detected: promptCaptureIsTrusted(0) && !promptCaptureIsTrusted(1), status: "FAIL", detail: "observer is enabled only on the authenticated extension-free path" },
|
|
88
|
-
{ id: "screen-ceiling-boundary", detected: screenResults([ceiling]).scenarios[0].classification === "CEILING", status: "FAIL", detail: "four of five control passes is CEILING" },
|
|
89
|
-
{ id: "screen-undelivered-filter", detected: (() => { const row = screenResults([undelivered]).scenarios[0]; return row.treatment.n === 1 && row.not_measured === 1; })(), status: "FAIL", detail: "undelivered reps have a bucket outside efficacy" },
|
|
90
|
-
{ id: "screen-suspect-filter", detected: screenResults([suspect]).scenarios[0].control.n === 0, status: "FAIL", detail: "suspect-only panels do not enter rates" },
|
|
91
|
-
{ id: "screen-unsupported-pass-filter", detected: (() => { const unsupported = screenFile("P", ["PASS"]); unsupported.scenarios[0].rep_judgments[0].judgments = []; return screenResults([unsupported]).scenarios[0].control.n === 0; })(), status: "FAIL", detail: "recorded PASS without a clean vote does not enter efficacy" },
|
|
92
|
-
];
|
|
93
|
-
return cases.map(test => ({ ...test, status: test.detected ? "FAIL" : "ERROR" }));
|
|
94
|
-
}
|
|
95
|
-
//# sourceMappingURL=mutation-catalogue.js.map
|