@skill-harness/cli 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -13
- package/assets/report.template.html +1 -43
- package/dist/cli.d.ts +0 -8
- package/dist/cli.js +16 -237
- package/dist/serve.js +2 -82
- package/package.json +4 -6
- package/dist/archive-learning.d.ts +0 -3
- package/dist/archive-learning.js +0 -153
- package/dist/archive.d.ts +0 -3
- package/dist/archive.js +0 -52
- package/dist/compare.d.ts +0 -26
- package/dist/compare.js +0 -232
- package/dist/learning.d.ts +0 -26
- package/dist/learning.js +0 -656
- package/dist/qualification.d.ts +0 -7
- package/dist/qualification.js +0 -210
- package/docs/PRODUCT-GUIDE.md +0 -133
- package/docs/STATUS.md +0 -45
package/dist/archive-learning.js
DELETED
|
@@ -1,153 +0,0 @@
|
|
|
1
|
-
import { constants, openSync, readSync, fstatSync, closeSync } from 'node:fs';
|
|
2
|
-
import { isAbsolute } from 'node:path';
|
|
3
|
-
import { createArchiveAccess, openArchiveAccess, readFixedOrderFacts, createWeeklyInvestigation, openWeeklyInvestigation, createTrustLifecycle, openTrustLifecycle } from '@skill-harness/adapters';
|
|
4
|
-
import { parseLearningRequest } from '@skill-harness/core';
|
|
5
|
-
/** Explicit operator requests, not a scheduler or model route. The inert host cannot invoke this CLI. */
|
|
6
|
-
export function cmdArchiveLearning(args) {
|
|
7
|
-
if (args._.length !== 1 || Object.keys(args.flags).sort().join() !== 'request,state' || typeof args.flags.state !== 'string' || !isAbsolute(args.flags.state) || typeof args.flags.request !== 'string')
|
|
8
|
-
throw Error('archive weekly|trust|access requires --state /absolute/directory --request file');
|
|
9
|
-
const fd = openSync(args.flags.request, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
10
|
-
let text;
|
|
11
|
-
try {
|
|
12
|
-
const stat = fstatSync(fd);
|
|
13
|
-
if (!stat.isFile() || stat.size > 65536)
|
|
14
|
-
throw Error('bounded regular learning request required');
|
|
15
|
-
const bytes = Buffer.alloc(65537);
|
|
16
|
-
let n = 0;
|
|
17
|
-
while (n < bytes.length) {
|
|
18
|
-
const k = readSync(fd, bytes, n, bytes.length - n, n);
|
|
19
|
-
if (!k)
|
|
20
|
-
break;
|
|
21
|
-
n += k;
|
|
22
|
-
}
|
|
23
|
-
if (n > 65536)
|
|
24
|
-
throw Error('learning request exceeds bound');
|
|
25
|
-
text = new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, n));
|
|
26
|
-
}
|
|
27
|
-
finally {
|
|
28
|
-
closeSync(fd);
|
|
29
|
-
}
|
|
30
|
-
const request = parseLearningRequest(text);
|
|
31
|
-
if (Object.keys(request).sort().join() !== 'authority,input,operation')
|
|
32
|
-
throw Error('closed learning request required');
|
|
33
|
-
const input = request.input, authority = request.authority, state = args.flags.state;
|
|
34
|
-
let result;
|
|
35
|
-
if (args._[0] === 'access') {
|
|
36
|
-
if (request.operation === 'create')
|
|
37
|
-
result = createArchiveAccess(state, input, authority).inspect();
|
|
38
|
-
else {
|
|
39
|
-
const access = openArchiveAccess(state);
|
|
40
|
-
switch (request.operation) {
|
|
41
|
-
case 'inspect':
|
|
42
|
-
result = access.inspect();
|
|
43
|
-
break;
|
|
44
|
-
case 'preview-consent':
|
|
45
|
-
result = access.previewConsent(input);
|
|
46
|
-
break;
|
|
47
|
-
case 'consent': {
|
|
48
|
-
const p = input;
|
|
49
|
-
result = access.consent(p.grant, authority, p.now);
|
|
50
|
-
break;
|
|
51
|
-
}
|
|
52
|
-
case 'preview-revocation':
|
|
53
|
-
result = access.previewRevocation(input);
|
|
54
|
-
break;
|
|
55
|
-
case 'revoke': {
|
|
56
|
-
const p = input;
|
|
57
|
-
access.revoke(p.id, authority, p.now);
|
|
58
|
-
result = { recorded: true };
|
|
59
|
-
break;
|
|
60
|
-
}
|
|
61
|
-
case 'facts': {
|
|
62
|
-
const p = input;
|
|
63
|
-
result = readFixedOrderFacts(access.inspect().policy.archiveRoot, p.manifestId, { directory: state, purpose: p.purpose }, p.expected, p.now);
|
|
64
|
-
break;
|
|
65
|
-
}
|
|
66
|
-
default: throw Error('unsupported access operation; raw read/export is not a CLI operation');
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
else if (args._[0] === 'weekly') {
|
|
71
|
-
if (request.operation === 'create') {
|
|
72
|
-
result = createWeeklyInvestigation(state, input).inspect();
|
|
73
|
-
}
|
|
74
|
-
else {
|
|
75
|
-
const job = openWeeklyInvestigation(state);
|
|
76
|
-
switch (request.operation) {
|
|
77
|
-
case 'inspect':
|
|
78
|
-
result = job.inspect();
|
|
79
|
-
break;
|
|
80
|
-
case 'run':
|
|
81
|
-
result = job.run(request.input);
|
|
82
|
-
break;
|
|
83
|
-
case 'approve':
|
|
84
|
-
result = job.approve(authority);
|
|
85
|
-
break;
|
|
86
|
-
case 'preview': {
|
|
87
|
-
const p = request.input;
|
|
88
|
-
result = job.preview(p.specPath, p.scenario);
|
|
89
|
-
break;
|
|
90
|
-
}
|
|
91
|
-
case 'promote':
|
|
92
|
-
result = job.promote(input, authority);
|
|
93
|
-
break;
|
|
94
|
-
case 'freeze':
|
|
95
|
-
result = job.freeze(input, authority);
|
|
96
|
-
break;
|
|
97
|
-
case 'preview-edit':
|
|
98
|
-
result = job.previewEdit(input);
|
|
99
|
-
break;
|
|
100
|
-
case 'edit':
|
|
101
|
-
result = job.edit(input, authority);
|
|
102
|
-
break;
|
|
103
|
-
case 'evaluation':
|
|
104
|
-
result = job.evaluation();
|
|
105
|
-
break;
|
|
106
|
-
case 'preview-screen':
|
|
107
|
-
result = job.previewScreen(input);
|
|
108
|
-
break;
|
|
109
|
-
case 'screen':
|
|
110
|
-
result = job.screen(input, authority);
|
|
111
|
-
break;
|
|
112
|
-
default: throw Error('unsupported weekly operation');
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
else if (args._[0] === 'trust') {
|
|
117
|
-
if (request.operation === 'create') {
|
|
118
|
-
result = createTrustLifecycle(state, input, authority).inspect(0);
|
|
119
|
-
}
|
|
120
|
-
else {
|
|
121
|
-
const job = openTrustLifecycle(state);
|
|
122
|
-
switch (request.operation) {
|
|
123
|
-
case 'inspect':
|
|
124
|
-
result = job.inspect(input);
|
|
125
|
-
break;
|
|
126
|
-
case 'predict':
|
|
127
|
-
job.predict(input);
|
|
128
|
-
result = { recorded: true };
|
|
129
|
-
break;
|
|
130
|
-
case 'correct': {
|
|
131
|
-
const p = request.input;
|
|
132
|
-
job.correct(p.prior, p.prediction, p.reason);
|
|
133
|
-
result = { recorded: true };
|
|
134
|
-
break;
|
|
135
|
-
}
|
|
136
|
-
case 'outcome':
|
|
137
|
-
job.outcome(input, authority);
|
|
138
|
-
result = { recorded: true };
|
|
139
|
-
break;
|
|
140
|
-
case 'expose': {
|
|
141
|
-
const p = request.input;
|
|
142
|
-
result = job.expose(p.id, p.now);
|
|
143
|
-
break;
|
|
144
|
-
}
|
|
145
|
-
default: throw Error('unsupported trust operation');
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
else
|
|
150
|
-
throw Error('unsupported learning family');
|
|
151
|
-
console.log(JSON.stringify(result, null, 2));
|
|
152
|
-
}
|
|
153
|
-
//# sourceMappingURL=archive-learning.js.map
|
package/dist/archive.d.ts
DELETED
package/dist/archive.js
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import { ingestPolicySource, inspectPolicyCheckpoint, observeArchiveSource } from "@skill-harness/adapters";
|
|
2
|
-
import { cmdArchiveLearning } from "./archive-learning.js";
|
|
3
|
-
/** Explicit file ingestion/metadata inspection only; never launches a worker or publishes content. */
|
|
4
|
-
export async function cmdArchive(args) {
|
|
5
|
-
const operation = args._[0];
|
|
6
|
-
if (operation === "weekly" || operation === "trust" || operation === "access") {
|
|
7
|
-
cmdArchiveLearning(args);
|
|
8
|
-
return;
|
|
9
|
-
}
|
|
10
|
-
if (operation === "watch") {
|
|
11
|
-
if (args._.length !== 1 || Object.keys(args.flags).some(key => !["policy", "source", "previous", "interval-ms", "max-polls"].includes(key)))
|
|
12
|
-
throw new Error("unsupported archive watch option");
|
|
13
|
-
const required = (key) => { const value = args.flags[key]; if (typeof value !== "string" || !value)
|
|
14
|
-
throw new Error(`archive watch requires --${key}`); return value; };
|
|
15
|
-
const signal = new AbortController(), abort = () => signal.abort();
|
|
16
|
-
process.on("SIGINT", abort);
|
|
17
|
-
process.on("SIGTERM", abort);
|
|
18
|
-
try {
|
|
19
|
-
const result = await observeArchiveSource({ policyPath: required("policy"), sourceId: required("source"),
|
|
20
|
-
previousCheckpointId: args.flags.previous === undefined ? undefined : required("previous"),
|
|
21
|
-
intervalMs: args.flags["interval-ms"] === undefined ? 1000 : Number(required("interval-ms")), maxPolls: Number(required("max-polls")), signal: signal.signal,
|
|
22
|
-
onObservation: observation => console.log(JSON.stringify({ checkpointId: observation.checkpointId, policySha256: observation.policySha256, syntax: observation.syntax })),
|
|
23
|
-
});
|
|
24
|
-
console.log(JSON.stringify(result));
|
|
25
|
-
}
|
|
26
|
-
finally {
|
|
27
|
-
process.removeListener("SIGINT", abort);
|
|
28
|
-
process.removeListener("SIGTERM", abort);
|
|
29
|
-
}
|
|
30
|
-
return;
|
|
31
|
-
}
|
|
32
|
-
if (args._.length !== 1 || !["ingest", "inspect"].includes(operation)) {
|
|
33
|
-
throw new Error("usage: archive ingest|inspect --policy <file> --source <id> [--previous <checkpoint> | --checkpoint <id>]");
|
|
34
|
-
}
|
|
35
|
-
const allowed = operation === "ingest" ? ["policy", "source", "previous"] : ["policy", "source", "checkpoint"];
|
|
36
|
-
for (const key of Object.keys(args.flags))
|
|
37
|
-
if (!allowed.includes(key))
|
|
38
|
-
throw new Error(`unsupported archive option: ${key}`);
|
|
39
|
-
const required = (key) => {
|
|
40
|
-
const value = args.flags[key];
|
|
41
|
-
if (typeof value !== "string" || !value)
|
|
42
|
-
throw new Error(`archive requires --${key}`);
|
|
43
|
-
return value;
|
|
44
|
-
};
|
|
45
|
-
const policy = required("policy"), source = required("source");
|
|
46
|
-
const result = operation === "ingest"
|
|
47
|
-
? ingestPolicySource(policy, source, args.flags.previous === undefined ? undefined : required("previous"))
|
|
48
|
-
: inspectPolicyCheckpoint(policy, source, required("checkpoint"));
|
|
49
|
-
// Successful storage/reporting is not successful work: missing/partial fields are never promoted to acceptance.
|
|
50
|
-
console.log(JSON.stringify(result, null, 2));
|
|
51
|
-
}
|
|
52
|
-
//# sourceMappingURL=archive.js.map
|
package/dist/compare.d.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { buildComparison, type ComparisonThresholds, type HarnessAdapter, type RunMode } from "@skill-harness/core";
|
|
2
|
-
export interface CompareCommandOptions {
|
|
3
|
-
target: string;
|
|
4
|
-
reference: string;
|
|
5
|
-
candidateRoot: string;
|
|
6
|
-
models: string[];
|
|
7
|
-
judgeToken: string;
|
|
8
|
-
mode: RunMode;
|
|
9
|
-
reps: number;
|
|
10
|
-
passThreshold: number;
|
|
11
|
-
parallel: number;
|
|
12
|
-
only?: string[];
|
|
13
|
-
canary?: boolean;
|
|
14
|
-
output?: string;
|
|
15
|
-
thresholds?: ComparisonThresholds;
|
|
16
|
-
adapter: HarnessAdapter;
|
|
17
|
-
now: () => string;
|
|
18
|
-
log?: (line: string) => void;
|
|
19
|
-
}
|
|
20
|
-
export interface CompareCommandResult {
|
|
21
|
-
outputDir: string;
|
|
22
|
-
reports: ReturnType<typeof buildComparison>[];
|
|
23
|
-
exitCode: 0 | 1 | 2;
|
|
24
|
-
}
|
|
25
|
-
/** Execute the same immutable plan against reference and candidate snapshots. */
|
|
26
|
-
export declare function runCompareCommand(options: CompareCommandOptions): Promise<CompareCommandResult>;
|
package/dist/compare.js
DELETED
|
@@ -1,232 +0,0 @@
|
|
|
1
|
-
import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { createHash } from "node:crypto";
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
5
|
-
import { execFileSync } from "node:child_process";
|
|
6
|
-
import { createRequire } from "node:module";
|
|
7
|
-
import { fileURLToPath } from "node:url";
|
|
8
|
-
import { dump as yamlDump } from "js-yaml";
|
|
9
|
-
import { buildComparison, comparisonExitCode, discover, fileSha256, FIXTURE_PREFIX, formatComparison, HARNESS_VERSION, loadSpec, modelSlug, parseModelRef, resolveSkill, runSkillModel, SKILL_KEY, SKILL_PROMPT_KEY, sourceHashes, } from "@skill-harness/core";
|
|
10
|
-
/** Execute the same immutable plan against reference and candidate snapshots. */
|
|
11
|
-
export async function runCompareCommand(options) {
|
|
12
|
-
const log = options.log ?? console.log;
|
|
13
|
-
const temp = mkdtempSync(join(tmpdir(), "skill-harness-compare-"));
|
|
14
|
-
try {
|
|
15
|
-
const candidate = snapshotExisting(options.candidateRoot, join(temp, "candidate"));
|
|
16
|
-
const reference = existsSync(options.reference)
|
|
17
|
-
? snapshotExisting(options.reference, join(temp, "reference"))
|
|
18
|
-
: snapshotGitRef(options.candidateRoot, options.reference, join(temp, "reference"));
|
|
19
|
-
const candidateSkills = options.target === "all"
|
|
20
|
-
? discover(candidate.skillsRoot).filter((skill) => skill.hasSpec)
|
|
21
|
-
: [resolveSkill(candidate.skillsRoot, options.target)];
|
|
22
|
-
const referenceSkills = new Map((options.target === "all"
|
|
23
|
-
? discover(reference.skillsRoot).filter((skill) => skill.hasSpec)
|
|
24
|
-
: [resolveSkill(reference.skillsRoot, options.target)])
|
|
25
|
-
.map((skill) => [skill.name, skill]));
|
|
26
|
-
if (!candidateSkills.length)
|
|
27
|
-
throw new Error(`no candidate skills with specs under ${options.candidateRoot}`);
|
|
28
|
-
const outputDir = resolve(options.output ?? join(options.candidateRoot, ".skill-harness", "comparisons", timestampSlug(options.now())));
|
|
29
|
-
mkdirSync(outputDir, { recursive: true });
|
|
30
|
-
const reports = [];
|
|
31
|
-
const fixtures = (sources) => Object.fromEntries(Object.entries(sources).filter(([key]) => key.startsWith(FIXTURE_PREFIX)));
|
|
32
|
-
const inputs = (sources) => Object.fromEntries(Object.entries(sources).filter(([key]) => key !== SKILL_KEY && key !== SKILL_PROMPT_KEY));
|
|
33
|
-
// Materialize and validate the ENTIRE skill × model plan before the first
|
|
34
|
-
// paid call. `compare all` must not spend on skill 1 and only then discover
|
|
35
|
-
// that skill 7 is not a valid pair.
|
|
36
|
-
const plans = candidateSkills.map((candidateSkill) => {
|
|
37
|
-
const referenceSkill = referenceSkills.get(candidateSkill.name);
|
|
38
|
-
if (!referenceSkill)
|
|
39
|
-
throw new Error(`reference has no matching skill \`${candidateSkill.name}\``);
|
|
40
|
-
const candidateSpec = loadSpec(candidateSkill.specPath);
|
|
41
|
-
const referenceSpec = loadSpec(referenceSkill.specPath);
|
|
42
|
-
const candidateIds = candidateSpec.scenarios.map((scenario) => scenario.id).sort();
|
|
43
|
-
const referenceIds = referenceSpec.scenarios.map((scenario) => scenario.id).sort();
|
|
44
|
-
if (JSON.stringify(candidateIds) !== JSON.stringify(referenceIds))
|
|
45
|
-
throw new Error(`${candidateSkill.name}: reference/candidate scenario IDs differ`);
|
|
46
|
-
const refSpecDigest = fileSha256(referenceSkill.specPath);
|
|
47
|
-
const candSpecDigest = fileSha256(candidateSkill.specPath);
|
|
48
|
-
if (!refSpecDigest || !candSpecDigest || refSpecDigest !== candSpecDigest) {
|
|
49
|
-
throw new Error(`${candidateSkill.name}: reference/candidate specification.yaml differs — paired comparison changes only the skill under test`);
|
|
50
|
-
}
|
|
51
|
-
const refSources = sourceHashes({ skillDir: referenceSkill.dir, specDir: dirname(referenceSkill.specPath), scenarios: referenceSpec.scenarios, judgePersona: referenceSpec.judge_persona });
|
|
52
|
-
const candSources = sourceHashes({ skillDir: candidateSkill.dir, specDir: dirname(candidateSkill.specPath), scenarios: candidateSpec.scenarios, judgePersona: candidateSpec.judge_persona });
|
|
53
|
-
if (JSON.stringify(Object.entries(inputs(refSources)).sort()) !== JSON.stringify(Object.entries(inputs(candSources)).sort())) {
|
|
54
|
-
throw new Error(`${candidateSkill.name}: reference/candidate fixtures or other test inputs differ — refusing before model calls`);
|
|
55
|
-
}
|
|
56
|
-
return { candidateSkill, referenceSkill, candidateSpec, referenceSpec, refSpecDigest, candSpecDigest, refSources, candSources };
|
|
57
|
-
});
|
|
58
|
-
const modelPlans = options.models.map((token) => ({ token, model: parseModelRef(token) }));
|
|
59
|
-
const judge = parseModelRef(options.judgeToken);
|
|
60
|
-
for (const { candidateSkill, referenceSkill, candidateSpec, referenceSpec, refSpecDigest, candSpecDigest, refSources, candSources } of plans) {
|
|
61
|
-
for (const { token: modelToken, model } of modelPlans) {
|
|
62
|
-
log(`\n▶ compare ${candidateSkill.name} · ${modelToken} · reference then candidate`);
|
|
63
|
-
const timestamp = options.now();
|
|
64
|
-
const referenceRun = await runSkillModel({
|
|
65
|
-
spec: referenceSpec,
|
|
66
|
-
skillDir: referenceSkill.dir,
|
|
67
|
-
specPath: referenceSkill.specPath,
|
|
68
|
-
adapter: options.adapter,
|
|
69
|
-
model,
|
|
70
|
-
modelToken,
|
|
71
|
-
judge,
|
|
72
|
-
mode: options.mode,
|
|
73
|
-
timestamp,
|
|
74
|
-
label: "compare-reference",
|
|
75
|
-
concurrency: options.parallel,
|
|
76
|
-
reps: options.reps,
|
|
77
|
-
passThreshold: options.passThreshold,
|
|
78
|
-
only: options.only,
|
|
79
|
-
canary: options.canary,
|
|
80
|
-
onProgress: (line) => log(` reference ${line}`),
|
|
81
|
-
});
|
|
82
|
-
const candidateRun = await runSkillModel({
|
|
83
|
-
spec: candidateSpec,
|
|
84
|
-
skillDir: candidateSkill.dir,
|
|
85
|
-
specPath: candidateSkill.specPath,
|
|
86
|
-
adapter: options.adapter,
|
|
87
|
-
model,
|
|
88
|
-
modelToken,
|
|
89
|
-
judge,
|
|
90
|
-
mode: options.mode,
|
|
91
|
-
timestamp,
|
|
92
|
-
label: "compare-candidate",
|
|
93
|
-
concurrency: options.parallel,
|
|
94
|
-
reps: options.reps,
|
|
95
|
-
passThreshold: options.passThreshold,
|
|
96
|
-
only: options.only,
|
|
97
|
-
canary: options.canary,
|
|
98
|
-
onProgress: (line) => log(` candidate ${line}`),
|
|
99
|
-
});
|
|
100
|
-
const digests = {
|
|
101
|
-
reference: {
|
|
102
|
-
skill: mustDigest(join(referenceSkill.dir, "SKILL.md")),
|
|
103
|
-
spec: refSpecDigest,
|
|
104
|
-
fixtures: fixtures(refSources),
|
|
105
|
-
inputs: inputs(refSources),
|
|
106
|
-
},
|
|
107
|
-
candidate: {
|
|
108
|
-
skill: mustDigest(join(candidateSkill.dir, "SKILL.md")),
|
|
109
|
-
spec: candSpecDigest,
|
|
110
|
-
fixtures: fixtures(candSources),
|
|
111
|
-
inputs: inputs(candSources),
|
|
112
|
-
},
|
|
113
|
-
harness: runtimeHarnessDigest(),
|
|
114
|
-
model: sha256(modelToken),
|
|
115
|
-
judge: sha256(options.judgeToken),
|
|
116
|
-
environment: {
|
|
117
|
-
node: process.version,
|
|
118
|
-
platform: process.platform,
|
|
119
|
-
arch: process.arch,
|
|
120
|
-
harness_version: HARNESS_VERSION,
|
|
121
|
-
harness_cli_reference: referenceRun.results.harness_cli_version ?? "unavailable",
|
|
122
|
-
harness_cli_candidate: candidateRun.results.harness_cli_version ?? "unavailable",
|
|
123
|
-
},
|
|
124
|
-
};
|
|
125
|
-
const report = buildComparison({
|
|
126
|
-
skill: candidateSpec.skill,
|
|
127
|
-
model: modelToken,
|
|
128
|
-
mode: options.mode,
|
|
129
|
-
reps: options.reps,
|
|
130
|
-
judge: options.judgeToken,
|
|
131
|
-
reference: referenceRun.results,
|
|
132
|
-
candidate: candidateRun.results,
|
|
133
|
-
critical: candidateSpec.critical,
|
|
134
|
-
digests,
|
|
135
|
-
partial: Boolean(options.only?.length),
|
|
136
|
-
thresholds: options.thresholds,
|
|
137
|
-
});
|
|
138
|
-
reports.push(report);
|
|
139
|
-
const cellDir = join(outputDir, candidateSkill.name, modelSlug(model));
|
|
140
|
-
mkdirSync(cellDir, { recursive: true });
|
|
141
|
-
cpSync(referenceRun.runDir, join(cellDir, "reference"), { recursive: true });
|
|
142
|
-
cpSync(candidateRun.runDir, join(cellDir, "candidate"), { recursive: true });
|
|
143
|
-
writeFileSync(join(cellDir, "comparison.yaml"), yamlDump(report, { lineWidth: 120 }), "utf8");
|
|
144
|
-
writeFileSync(join(cellDir, "comparison.txt"), `${formatComparison(report)}\n`, "utf8");
|
|
145
|
-
log(`\n${formatComparison(report)}\n artifacts: ${cellDir}`);
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
const exitCode = reports.reduce((highest, report) => Math.max(highest, comparisonExitCode(report)), 0);
|
|
149
|
-
return { outputDir, reports, exitCode };
|
|
150
|
-
}
|
|
151
|
-
finally {
|
|
152
|
-
rmSync(temp, { recursive: true, force: true });
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
function snapshotExisting(skillsRoot, destination) {
|
|
156
|
-
const absolute = realpathSync(resolve(skillsRoot));
|
|
157
|
-
const repo = gitRoot(absolute);
|
|
158
|
-
const sourceRoot = repo ? realpathSync(repo) : absolute;
|
|
159
|
-
const inside = relativeInside(sourceRoot, absolute);
|
|
160
|
-
copyTree(sourceRoot, destination);
|
|
161
|
-
return { skillsRoot: join(destination, inside) };
|
|
162
|
-
}
|
|
163
|
-
function snapshotGitRef(candidateSkillsRoot, ref, destination) {
|
|
164
|
-
const absolute = realpathSync(resolve(candidateSkillsRoot));
|
|
165
|
-
const rawRepo = gitRoot(absolute);
|
|
166
|
-
const repo = rawRepo ? realpathSync(rawRepo) : null;
|
|
167
|
-
if (!repo)
|
|
168
|
-
throw new Error(`--reference ${ref} is not a directory and --candidate is not inside a git repository`);
|
|
169
|
-
execFileSync("git", ["clone", "--quiet", "--no-checkout", repo, destination], { stdio: ["ignore", "ignore", "pipe"] });
|
|
170
|
-
try {
|
|
171
|
-
execFileSync("git", ["checkout", "--quiet", "--detach", ref], { cwd: destination, stdio: ["ignore", "ignore", "pipe"] });
|
|
172
|
-
}
|
|
173
|
-
catch (error) {
|
|
174
|
-
throw new Error(`cannot materialize reference git ref \`${ref}\`: ${error instanceof Error ? error.message : String(error)}`);
|
|
175
|
-
}
|
|
176
|
-
return { skillsRoot: join(destination, relativeInside(repo, absolute)) };
|
|
177
|
-
}
|
|
178
|
-
function relativeInside(root, target) {
|
|
179
|
-
const rel = relative(root, target);
|
|
180
|
-
if (rel === ".." || rel.startsWith(`..${sep}`) || resolve(root, rel) !== target) {
|
|
181
|
-
throw new Error(`skills root ${target} is outside snapshot root ${root}`);
|
|
182
|
-
}
|
|
183
|
-
return rel;
|
|
184
|
-
}
|
|
185
|
-
function gitRoot(path) {
|
|
186
|
-
try {
|
|
187
|
-
return execFileSync("git", ["-C", path, "rev-parse", "--show-toplevel"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
|
|
188
|
-
}
|
|
189
|
-
catch {
|
|
190
|
-
return null;
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
function copyTree(source, destination) {
|
|
194
|
-
cpSync(source, destination, {
|
|
195
|
-
recursive: true,
|
|
196
|
-
filter: (path) => {
|
|
197
|
-
const rel = relative(source, path).replace(/\\/g, "/");
|
|
198
|
-
if (!rel)
|
|
199
|
-
return true;
|
|
200
|
-
return !rel.split("/").some((part) => part === ".git" || part === "node_modules" || part === ".skill-harness") && !/(^|\/)tests\/results(\/|$)/.test(rel);
|
|
201
|
-
},
|
|
202
|
-
});
|
|
203
|
-
}
|
|
204
|
-
function mustDigest(path) { const digest = fileSha256(path); if (!digest)
|
|
205
|
-
throw new Error(`cannot digest ${path}`); return digest; }
|
|
206
|
-
function sha256(value) { return createHash("sha256").update(value).digest("hex"); }
|
|
207
|
-
function timestampSlug(value) { return value.replace(/[:.]/g, "-"); }
|
|
208
|
-
function runtimeHarnessDigest() {
|
|
209
|
-
const hash = createHash("sha256").update(`skill-harness/${HARNESS_VERSION}\0`);
|
|
210
|
-
const require = createRequire(import.meta.url);
|
|
211
|
-
const cliDir = dirname(fileURLToPath(import.meta.url));
|
|
212
|
-
const roots = [
|
|
213
|
-
cliDir,
|
|
214
|
-
join(dirname(cliDir), "assets"),
|
|
215
|
-
dirname(require.resolve("@skill-harness/core")),
|
|
216
|
-
dirname(require.resolve("@skill-harness/adapters")),
|
|
217
|
-
].filter((root, index, all) => existsSync(root) && all.indexOf(root) === index);
|
|
218
|
-
if (!roots.length)
|
|
219
|
-
throw new Error("cannot locate running harness artifacts for comparison digest");
|
|
220
|
-
roots.forEach((root, rootIndex) => {
|
|
221
|
-
for (const file of filesUnder(root).sort()) {
|
|
222
|
-
hash.update(`${rootIndex}/${relative(root, file).replace(/\\/g, "/")}`).update("\0").update(readFileSync(file)).update("\0");
|
|
223
|
-
}
|
|
224
|
-
});
|
|
225
|
-
return hash.digest("hex");
|
|
226
|
-
}
|
|
227
|
-
function filesUnder(path) {
|
|
228
|
-
if (statSync(path).isFile())
|
|
229
|
-
return [path];
|
|
230
|
-
return readdirSync(path, { withFileTypes: true }).flatMap((entry) => entry.isDirectory() ? filesUnder(join(path, entry.name)) : entry.isFile() ? [join(path, entry.name)] : []);
|
|
231
|
-
}
|
|
232
|
-
//# sourceMappingURL=compare.js.map
|
package/dist/learning.d.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { openLearningWorkspace } from '@skill-harness/adapters';
|
|
2
|
-
export interface LearningUI {
|
|
3
|
-
select(title: string, choices: string[]): Promise<string | undefined>;
|
|
4
|
-
input(title: string, initial?: string): Promise<string | undefined>;
|
|
5
|
-
editor(title: string, text: string): Promise<string | undefined>;
|
|
6
|
-
confirm(title: string, detail: string): Promise<boolean>;
|
|
7
|
-
notify(text: string, level?: 'info' | 'warning' | 'error'): void;
|
|
8
|
-
}
|
|
9
|
-
export declare const LEARNING_HELP = "Learning \u2014 retained evidence, no model calls\n learning open guided review (interactive terminal)\n learning init --archive DIR --author NAME [--scope-item N] --confirm\n [--scope TEXT --population TEXT only for a comparison-only archive without retained cases]\n learning status [--json] readiness; never automatic exposure\n learning import list retained batches/comparisons\n learning import --item N --name NAME --title TEXT --scope-note TEXT --confirm\n learning cases NAME [--offset N]\n learning evidence NAME --item N [--evidence N] inspect only the selected case's retained sources\n learning case NAME --item N --disposition confirmed_defect|expected_behavior|exemplar|uncertain|skip --note TEXT --confirm\n learning comparisons\n learning review NAME guided complete-output review in a terminal\n learning artifact NAME --variant A [--artifact N]\n learning choose NAME --kind one|tie|none|insufficient [--variants A,B] --full-review --note TEXT --confirm\n learning reveal NAME requires durable full-scope quality receipt\n learning decide NAME --disposition adopt|reject|defer --note TEXT --confirm\n learning hypothesis NAME --name SLUG --intervention TEXT --prediction TEXT --disproof TEXT --downside TEXT --rollback TEXT --alternative TEXT --confirm\n learning link NAME --cases BATCH --item N --hypothesis SLUG --confirm\n learning trust sample FILE --name SLUG --title TEXT [--split calibration|heldout|tuning] --confirm\n learning trust setup --cases NAME [--detector ID] [--split calibration|heldout|tuning] [--max-unflagged N]\n [--attention N --minimum-resolved N --minimum-lower-bound T --expires DAYS] --confirm\n learning trust status\n learning trust label --cases NAME --item N --reference-author NAME --confirm\n learning trust unflagged --item N --miss yes|no --evidence FILE --reference-author NAME --note TEXT --confirm\n learning adoption NAME scope/decision/readiness; activation is producer-owned\n learning outcome NAME inspect observed outcomes\n learning outcome NAME --result success|confirmed-defect|unknown --artifact FILE --original-requirement FILE --current-requirement FILE --evidence FILE --reference-author NAME --note TEXT [--accepted-artifact FILE --acceptance-evidence FILE] --confirm\n learning guide | current installed portable guide / requirement register\nAll commands accept --state DIR (default .skill-harness/learning), --json for structured output.\n--confirm authorizes only the displayed local write, never models, acceptance or registry activation.\nNo raw manifest authoring. Import selects actual retained inputs. Exit zero is command completion, not quality.\nRegistry activation/rollback use /grants learning in interactive Pi, with independent authority.";
|
|
10
|
-
/** Escape terminal control/format characters in untrusted retained display text. Exact archive
|
|
11
|
-
* bytes never change; output honestly identifies this as a display representation. */
|
|
12
|
-
export declare function learningDisplay(text: string): string;
|
|
13
|
-
export declare function formatLearningStatus(view: ReturnType<ReturnType<typeof openLearningWorkspace>['inspect']>): string;
|
|
14
|
-
export declare function formatLearningTrust(trust: Record<string, unknown>): string;
|
|
15
|
-
export interface LearningCommandOptions {
|
|
16
|
-
cwd?: string;
|
|
17
|
-
write?: (text: string) => void;
|
|
18
|
-
ui?: LearningUI;
|
|
19
|
-
}
|
|
20
|
-
export declare function runLearningCommand(argv: string[], options?: LearningCommandOptions): Promise<unknown>;
|
|
21
|
-
export declare function reviewLearningComparison(directory: string, name: string, ui: LearningUI): Promise<void>;
|
|
22
|
-
export declare function runLearningWizard(options: {
|
|
23
|
-
cwd: string;
|
|
24
|
-
directory: string;
|
|
25
|
-
ui: LearningUI;
|
|
26
|
-
}): Promise<void>;
|