@skill-harness/core 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/types.d.ts +6 -0
- package/dist/discover.d.ts +5 -1
- package/dist/discover.js +9 -1
- package/dist/grade.d.ts +14 -1
- package/dist/grade.js +74 -11
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/journal.d.ts +14 -0
- package/dist/lift.d.ts +69 -0
- package/dist/lift.js +163 -0
- package/dist/lint.d.ts +1 -1
- package/dist/lint.js +168 -4
- package/dist/regrade.d.ts +6 -0
- package/dist/regrade.js +25 -3
- package/dist/report.d.ts +14 -0
- package/dist/report.js +16 -1
- package/dist/rescore.d.ts +34 -0
- package/dist/rescore.js +66 -0
- package/dist/results.d.ts +41 -9
- package/dist/results.js +58 -14
- package/dist/run.d.ts +25 -1
- package/dist/run.js +115 -22
- package/dist/scaffold.d.ts +28 -0
- package/dist/scaffold.js +188 -0
- package/dist/score.d.ts +5 -1
- package/dist/seeded.d.ts +83 -3
- package/dist/seeded.js +315 -15
- package/dist/sources.d.ts +115 -0
- package/dist/sources.js +253 -0
- package/dist/spec.d.ts +17 -0
- package/dist/spec.js +63 -0
- package/dist/util/env.d.ts +32 -0
- package/dist/util/env.js +67 -0
- package/dist/workspace.d.ts +33 -1
- package/dist/workspace.js +125 -6
- package/package.json +1 -1
package/dist/sources.js
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFileSync, readdirSync } from "node:fs";
|
|
3
|
+
import { isAbsolute, join, resolve } from "node:path";
|
|
4
|
+
/**
|
|
5
|
+
* What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
|
|
6
|
+
* lint can prove a published result still describes the current inputs.
|
|
7
|
+
*
|
|
8
|
+
* A run measures more than the skill text. It measures the SKILL.md, any agent
|
|
9
|
+
* file, **the scenario definition itself** (turns, checklist, gates) and **the
|
|
10
|
+
* fixture** the scenario starts from. Hashing only the first two — which is what
|
|
11
|
+
* shipped through 0.2.1 — meant editing a checklist or swapping a fixture left
|
|
12
|
+
* every published result looking current.
|
|
13
|
+
*
|
|
14
|
+
* ## Key scheme
|
|
15
|
+
*
|
|
16
|
+
* | key | means |
|
|
17
|
+
* |---|---|
|
|
18
|
+
* | `SKILL.md` | the skill text, resolved against the skill dir |
|
|
19
|
+
* | `scenario:<id>` | the semantic content of one scenario in the spec |
|
|
20
|
+
* | `fixture:<path>` | every file under one fixture dir |
|
|
21
|
+
* | anything else | a file path resolved against the spec's dir (`system_prompt_file`) |
|
|
22
|
+
*
|
|
23
|
+
* Separation from bare-path keys is conventional, not guaranteed: `scenario:A1`
|
|
24
|
+
* is a legal POSIX filename, so nothing stops someone naming an agent file that.
|
|
25
|
+
* In practice `system_prompt_file` and `post_test` values are ordinary relative
|
|
26
|
+
* paths, and a `<name>:` prefix is reserved for this scheme. Old results carrying
|
|
27
|
+
* only bare-path keys keep resolving exactly as before.
|
|
28
|
+
*
|
|
29
|
+
* ## Why per-scenario, not one hash of specification.yaml
|
|
30
|
+
*
|
|
31
|
+
* Hashing the whole spec file would mark **every** historical run stale the
|
|
32
|
+
* moment a spec grows by one scenario — the precise noise `lint`'s scenario-set
|
|
33
|
+
* check already exists to prevent ("a spec reshape must not consistency-flag
|
|
34
|
+
* every historical run"). A per-scenario digest says what actually changed:
|
|
35
|
+
* editing A1's checklist marks A1 stale and leaves A2 alone; appending a new
|
|
36
|
+
* scenario marks nothing stale, because nothing already measured changed.
|
|
37
|
+
*
|
|
38
|
+
* It also ignores formatting: the digest is built from the *parsed* scenario, so
|
|
39
|
+
* reindenting the YAML or reordering scenarios is correctly a no-op, while
|
|
40
|
+
* changing a single checklist word is correctly a change.
|
|
41
|
+
*/
|
|
42
|
+
export const SCENARIO_PREFIX = "scenario:";
|
|
43
|
+
export const FIXTURE_PREFIX = "fixture:";
|
|
44
|
+
/**
|
|
45
|
+
* Recorded in place of a hash when a source existed but could not be read.
|
|
46
|
+
*
|
|
47
|
+
* Omitting it instead — which is what an early version did — is the worst
|
|
48
|
+
* available option: `lint` only ever iterates the keys a run recorded, so a
|
|
49
|
+
* source dropped at record time is never compared again for the life of that
|
|
50
|
+
* result. A fixture briefly unreadable during a run could then be replaced
|
|
51
|
+
* wholesale and `lint` would still report 0 findings, which is verbatim the miss
|
|
52
|
+
* this module was written to close.
|
|
53
|
+
*
|
|
54
|
+
* Not valid sha256 hex, so it can never equal a real digest and always surfaces.
|
|
55
|
+
*/
|
|
56
|
+
export const UNREADABLE = "unreadable";
|
|
57
|
+
/** sha256 of a file, or null when it doesn't exist / isn't readable. */
|
|
58
|
+
export function fileSha256(path) {
|
|
59
|
+
try {
|
|
60
|
+
return createHash("sha256").update(readFileSync(path)).digest("hex");
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Stable sha256 over a directory tree: every file's relative path (POSIX-slashed,
|
|
68
|
+
* sorted) and contents. Null if the directory is missing or unreadable.
|
|
69
|
+
*
|
|
70
|
+
* Sorting is what makes it stable — readdir order is filesystem-dependent, so an
|
|
71
|
+
* unsorted walk would produce different digests for identical trees on different
|
|
72
|
+
* machines and turn CI into a staleness alarm. Paths are hashed alongside
|
|
73
|
+
* contents so that renaming a fixture file is a change, and separators are
|
|
74
|
+
* normalised so a Linux-recorded hash still matches on Windows.
|
|
75
|
+
*/
|
|
76
|
+
export function dirSha256(dir) {
|
|
77
|
+
let files;
|
|
78
|
+
try {
|
|
79
|
+
files = walk(dir).sort();
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
const h = createHash("sha256");
|
|
85
|
+
for (const rel of files) {
|
|
86
|
+
h.update(rel);
|
|
87
|
+
h.update("\0");
|
|
88
|
+
try {
|
|
89
|
+
h.update(readFileSync(join(dir, rel)));
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return null; // a file that vanished mid-walk makes the whole digest untrustworthy
|
|
93
|
+
}
|
|
94
|
+
h.update("\0");
|
|
95
|
+
}
|
|
96
|
+
return h.digest("hex");
|
|
97
|
+
}
|
|
98
|
+
/** Relative POSIX paths of every file under `dir`, recursively. Throws if `dir` is unreadable. */
|
|
99
|
+
function walk(dir, prefix = "") {
|
|
100
|
+
const out = [];
|
|
101
|
+
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
|
102
|
+
const rel = prefix ? `${prefix}/${e.name}` : e.name;
|
|
103
|
+
if (e.isDirectory()) {
|
|
104
|
+
out.push(...walk(join(dir, e.name), rel));
|
|
105
|
+
}
|
|
106
|
+
else if (e.isFile()) {
|
|
107
|
+
out.push(rel);
|
|
108
|
+
}
|
|
109
|
+
// symlinks/sockets/etc. are deliberately skipped: a fixture is plain files,
|
|
110
|
+
// and following links could hash something outside the fixture entirely.
|
|
111
|
+
}
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* A scenario's semantic digest: everything that changes what the scenario
|
|
116
|
+
* measures, and nothing that doesn't.
|
|
117
|
+
*
|
|
118
|
+
* Built from the parsed scenario rather than its YAML text, so formatting is
|
|
119
|
+
* irrelevant. `critical` is included because it changes whether the scenario can
|
|
120
|
+
* block a ship; `title` is included because it is what a reader of the scorecard
|
|
121
|
+
* believes was tested.
|
|
122
|
+
*/
|
|
123
|
+
export function scenarioDigest(s) {
|
|
124
|
+
// Destructured, with the remainder pinned to `Record<string, never>`, so that
|
|
125
|
+
// adding a field to `Scenario` or `SeededAssert` FAILS THE BUILD here instead of
|
|
126
|
+
// silently escaping the digest. A field nobody remembered to add is a permanent
|
|
127
|
+
// staleness blind spot — edit it and every published result still looks current,
|
|
128
|
+
// which is precisely the bug this module exists to kill. This PR added two
|
|
129
|
+
// SeededAssert fields and remembered both; that only proves the discipline works
|
|
130
|
+
// while someone is looking, so the compiler now does the looking.
|
|
131
|
+
const { id, title, critical, mode, turns, checklist, fixture, assert, workspace, remote, systemPromptFile, reps, passThreshold, ...restScenario } = s;
|
|
132
|
+
const _scenarioExhaustive = restScenario;
|
|
133
|
+
void _scenarioExhaustive;
|
|
134
|
+
const { vitest, diff_contains, diff_excludes, post_test, ...restAssert } = assert ?? {};
|
|
135
|
+
const _assertExhaustive = restAssert;
|
|
136
|
+
void _assertExhaustive;
|
|
137
|
+
const canonical = JSON.stringify([
|
|
138
|
+
id,
|
|
139
|
+
title,
|
|
140
|
+
critical,
|
|
141
|
+
mode,
|
|
142
|
+
turns,
|
|
143
|
+
checklist,
|
|
144
|
+
fixture ?? null,
|
|
145
|
+
assert ? [vitest ?? null, diff_contains ?? null, diff_excludes ?? null, post_test ?? null] : null,
|
|
146
|
+
workspace,
|
|
147
|
+
remote,
|
|
148
|
+
systemPromptFile ?? null,
|
|
149
|
+
reps ?? null,
|
|
150
|
+
passThreshold ?? null,
|
|
151
|
+
]);
|
|
152
|
+
return createHash("sha256").update(canonical).digest("hex");
|
|
153
|
+
}
|
|
154
|
+
/** Absolute path a fixture key refers to. Fixtures resolve against the spec's dir, like workspace.ts. */
|
|
155
|
+
function fixtureAbs(specDir, fixture) {
|
|
156
|
+
return isAbsolute(fixture) ? fixture : resolve(specDir, fixture);
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* The fixture path a scenario actually runs in, or undefined.
|
|
160
|
+
*
|
|
161
|
+
* The EFFECTIVE workspace fixture, which is not always `scenario.fixture`: an
|
|
162
|
+
* inline scenario with `env.workspace: fixture:PATH` sets `workspace.fixture`
|
|
163
|
+
* and leaves `scenario.fixture` unset. Exported and shared with lint, which
|
|
164
|
+
* needs the identical rule — a second copy of this expression is how the hashed
|
|
165
|
+
* set and the checked set drift apart.
|
|
166
|
+
*/
|
|
167
|
+
export function effectiveFixture(s) {
|
|
168
|
+
return typeof s.workspace === "object" && s.workspace !== null ? s.workspace.fixture : undefined;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Hash every source this run measures: SKILL.md, each distinct
|
|
172
|
+
* `system_prompt_file`, each scenario's definition, and each distinct fixture
|
|
173
|
+
* tree. Entries that can't be read are omitted — a missing source is lint's
|
|
174
|
+
* problem to report, not run's to crash on.
|
|
175
|
+
*/
|
|
176
|
+
export function sourceHashes(ctx) {
|
|
177
|
+
const hashes = {};
|
|
178
|
+
// UNREADABLE rather than omission on every branch below: a source we failed to
|
|
179
|
+
// hash must stay visible to lint, not vanish from the record. See UNREADABLE.
|
|
180
|
+
hashes["SKILL.md"] = fileSha256(resolve(ctx.skillDir, "SKILL.md")) ?? UNREADABLE;
|
|
181
|
+
for (const s of ctx.scenarios) {
|
|
182
|
+
hashes[SCENARIO_PREFIX + s.id] = scenarioDigest(s);
|
|
183
|
+
if (s.systemPromptFile && !(s.systemPromptFile in hashes)) {
|
|
184
|
+
hashes[s.systemPromptFile] = fileSha256(resolve(ctx.specDir, s.systemPromptFile)) ?? UNREADABLE;
|
|
185
|
+
}
|
|
186
|
+
// The post-test IS the gate on a post_test scenario, and it lives outside the
|
|
187
|
+
// fixture tree by convention (`fixture: fixtures/A1`, `post_test: post/A1.test.ts`),
|
|
188
|
+
// so neither the fixture digest nor the scenario digest — which holds only the
|
|
189
|
+
// path string — covers its contents. Tightening an assertion in it changes what
|
|
190
|
+
// the scorecard measured; without this it would change nothing lint can see.
|
|
191
|
+
const pt = s.assert?.post_test;
|
|
192
|
+
if (pt && !(pt in hashes)) {
|
|
193
|
+
hashes[pt] = fileSha256(isAbsolute(pt) ? pt : resolve(ctx.specDir, pt)) ?? UNREADABLE;
|
|
194
|
+
}
|
|
195
|
+
const fx = effectiveFixture(s);
|
|
196
|
+
if (fx && !(FIXTURE_PREFIX + fx in hashes)) {
|
|
197
|
+
hashes[FIXTURE_PREFIX + fx] = dirSha256(fixtureAbs(ctx.specDir, fx)) ?? UNREADABLE;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return hashes;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* The current hash for a recorded key, or null when the source is gone.
|
|
204
|
+
*
|
|
205
|
+
* `undefined` is distinct from `null` and means "not comparable": the key names a
|
|
206
|
+
* scenario the spec no longer has. That is a spec *reshape*, not staleness — the
|
|
207
|
+
* same stance lint's scenario-set check already takes — so the caller stays quiet
|
|
208
|
+
* rather than reporting a removed scenario as a stale measurement.
|
|
209
|
+
*
|
|
210
|
+
* Sharing this resolver with `sourceHashes` is what keeps recording and checking
|
|
211
|
+
* from drifting: a new key kind is defined once, for both sides.
|
|
212
|
+
*/
|
|
213
|
+
export function currentHashFor(key, ctx) {
|
|
214
|
+
if (key === "SKILL.md")
|
|
215
|
+
return fileSha256(resolve(ctx.skillDir, "SKILL.md"));
|
|
216
|
+
if (key.startsWith(SCENARIO_PREFIX)) {
|
|
217
|
+
const id = key.slice(SCENARIO_PREFIX.length);
|
|
218
|
+
const s = ctx.scenarios.find((x) => x.id === id);
|
|
219
|
+
return s ? scenarioDigest(s) : undefined; // removed → reshape, not stale
|
|
220
|
+
}
|
|
221
|
+
if (key.startsWith(FIXTURE_PREFIX)) {
|
|
222
|
+
return dirSha256(fixtureAbs(ctx.specDir, key.slice(FIXTURE_PREFIX.length)));
|
|
223
|
+
}
|
|
224
|
+
// A prefixed key this version doesn't know is written by a NEWER skill-harness.
|
|
225
|
+
// Falling through to the path branch would resolve `agent:foo` as a filename,
|
|
226
|
+
// find nothing, and report a confident "agent:foo no longer exists" — a wrong
|
|
227
|
+
// finding about a source that is fine. Not comparable is the honest answer.
|
|
228
|
+
if (/^[a-z][a-z0-9-]*:/.test(key))
|
|
229
|
+
return undefined;
|
|
230
|
+
return fileSha256(resolve(ctx.specDir, key)); // system_prompt_file, post_test
|
|
231
|
+
}
|
|
232
|
+
/** Human label for a recorded key, used in lint messages. */
|
|
233
|
+
export function describeSourceKey(key) {
|
|
234
|
+
if (key.startsWith(SCENARIO_PREFIX))
|
|
235
|
+
return `scenario \`${key.slice(SCENARIO_PREFIX.length)}\``;
|
|
236
|
+
if (key.startsWith(FIXTURE_PREFIX))
|
|
237
|
+
return `fixture \`${key.slice(FIXTURE_PREFIX.length)}\``;
|
|
238
|
+
return key;
|
|
239
|
+
}
|
|
240
|
+
/** The scenario id a key belongs to, for per-scenario lint findings. Undefined for skill-wide keys. */
|
|
241
|
+
export function scenarioIdForKey(key, scenarios) {
|
|
242
|
+
if (key.startsWith(SCENARIO_PREFIX))
|
|
243
|
+
return key.slice(SCENARIO_PREFIX.length);
|
|
244
|
+
if (key.startsWith(FIXTURE_PREFIX)) {
|
|
245
|
+
const fx = key.slice(FIXTURE_PREFIX.length);
|
|
246
|
+
const owners = scenarios.filter((s) => effectiveFixture(s) === fx);
|
|
247
|
+
// Only attribute a shared fixture to a scenario when exactly one owns it —
|
|
248
|
+
// naming an arbitrary one of several would misdirect the re-run.
|
|
249
|
+
return owners.length === 1 ? owners[0].id : undefined;
|
|
250
|
+
}
|
|
251
|
+
return undefined;
|
|
252
|
+
}
|
|
253
|
+
//# sourceMappingURL=sources.js.map
|
package/dist/spec.d.ts
CHANGED
|
@@ -3,6 +3,21 @@ export type ScenarioMode = "inline" | "seeded";
|
|
|
3
3
|
export interface SeededAssert {
|
|
4
4
|
vitest?: boolean;
|
|
5
5
|
diff_contains?: string[];
|
|
6
|
+
/**
|
|
7
|
+
* Needles that must NOT appear in the staged diff. The negative twin of
|
|
8
|
+
* diff_contains: it makes a scope-discipline requirement ("fix sliceRange, do
|
|
9
|
+
* not touch lastIndex") objective instead of something the judge has to infer
|
|
10
|
+
* from the model's prose.
|
|
11
|
+
*/
|
|
12
|
+
diff_excludes?: string[];
|
|
13
|
+
/**
|
|
14
|
+
* A test file copied into the workspace AFTER the agent finishes, then run.
|
|
15
|
+
* The model never sees it, so it cannot write code shaped to pass it — this
|
|
16
|
+
* checks the behavior the task actually required. Orthogonal to `vitest`,
|
|
17
|
+
* which runs the model's OWN tests and therefore grades a claim the model
|
|
18
|
+
* gets to make about itself.
|
|
19
|
+
*/
|
|
20
|
+
post_test?: string;
|
|
6
21
|
}
|
|
7
22
|
export interface Scenario {
|
|
8
23
|
id: string;
|
|
@@ -14,6 +29,8 @@ export interface Scenario {
|
|
|
14
29
|
fixture?: string;
|
|
15
30
|
assert?: SeededAssert;
|
|
16
31
|
workspace: WorkspaceKind;
|
|
32
|
+
remote: boolean;
|
|
33
|
+
systemPromptFile?: string;
|
|
17
34
|
reps?: number;
|
|
18
35
|
passThreshold?: number;
|
|
19
36
|
}
|
package/dist/spec.js
CHANGED
|
@@ -54,6 +54,24 @@ function resolveWorkspace(env, mode, fixture, id, file) {
|
|
|
54
54
|
}
|
|
55
55
|
throw new SpecError(`scenario \`${id}\` env.workspace must be none | empty-git | fixture:<path>`, file);
|
|
56
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Resolve `env.remote`. A remote needs a repo to attach to, so it is only meaningful
|
|
59
|
+
* with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
|
|
60
|
+
* not something to silently ignore.
|
|
61
|
+
*/
|
|
62
|
+
function resolveRemote(env, workspace, id, file) {
|
|
63
|
+
const raw = env && typeof env === "object" ? env.remote : undefined;
|
|
64
|
+
if (raw === undefined)
|
|
65
|
+
return false;
|
|
66
|
+
if (typeof raw !== "boolean") {
|
|
67
|
+
throw new SpecError(`scenario \`${id}\` env.remote must be true or false`, file);
|
|
68
|
+
}
|
|
69
|
+
if (raw && workspace === "none") {
|
|
70
|
+
throw new SpecError(`scenario \`${id}\` sets env.remote but has no repo to attach it to — ` +
|
|
71
|
+
`use env.workspace: empty-git or fixture:<path>`, file);
|
|
72
|
+
}
|
|
73
|
+
return raw;
|
|
74
|
+
}
|
|
57
75
|
/** Parse + validate a specification.yaml from its raw text. `file` is used in error messages. */
|
|
58
76
|
export function parseSpec(text, file) {
|
|
59
77
|
let doc;
|
|
@@ -124,6 +142,7 @@ export function parseSpec(text, file) {
|
|
|
124
142
|
turns: s.turns,
|
|
125
143
|
checklist: s.checklist,
|
|
126
144
|
workspace: "none",
|
|
145
|
+
remote: false,
|
|
127
146
|
};
|
|
128
147
|
if (mode === "seeded") {
|
|
129
148
|
if (typeof s.fixture !== "string" || s.fixture.length === 0) {
|
|
@@ -139,12 +158,56 @@ export function parseSpec(text, file) {
|
|
|
139
158
|
if (!isStringArray(a.diff_contains)) {
|
|
140
159
|
throw new SpecError(`seeded scenario \`${id}\` \`assert.diff_contains\` must be strings`, file);
|
|
141
160
|
}
|
|
161
|
+
if (a.diff_contains.some((n) => n === "")) {
|
|
162
|
+
// Every string contains "", so an empty positive needle makes the gate
|
|
163
|
+
// pass on ANY diff, including an empty one. This is the more dangerous
|
|
164
|
+
// twin of the diff_excludes check below: that one fails forever and gets
|
|
165
|
+
// investigated, this one passes forever and nobody ever looks.
|
|
166
|
+
throw new SpecError(`seeded scenario \`${id}\` \`assert.diff_contains\` contains an empty string — it would match every diff, so the gate could never fail`, file);
|
|
167
|
+
}
|
|
142
168
|
assertObj.diff_contains = a.diff_contains;
|
|
143
169
|
}
|
|
170
|
+
if (a.diff_excludes !== undefined) {
|
|
171
|
+
if (!isStringArray(a.diff_excludes)) {
|
|
172
|
+
throw new SpecError(`seeded scenario \`${id}\` \`assert.diff_excludes\` must be strings`, file);
|
|
173
|
+
}
|
|
174
|
+
if (a.diff_excludes.some((n) => n === "")) {
|
|
175
|
+
// "" is in every string, so the gate could never pass — and the failure
|
|
176
|
+
// would read as a mysterious diff problem rather than a spec typo.
|
|
177
|
+
throw new SpecError(`seeded scenario \`${id}\` \`assert.diff_excludes\` contains an empty string — it would match every diff`, file);
|
|
178
|
+
}
|
|
179
|
+
assertObj.diff_excludes = a.diff_excludes;
|
|
180
|
+
}
|
|
181
|
+
// A needle required AND forbidden can never pass. Catching it here turns a
|
|
182
|
+
// scenario that always fails for an invisible reason into an authoring error.
|
|
183
|
+
const both = (assertObj.diff_contains ?? []).filter((n) => (assertObj.diff_excludes ?? []).includes(n));
|
|
184
|
+
if (both.length > 0) {
|
|
185
|
+
throw new SpecError(`seeded scenario \`${id}\` lists ${both.map((n) => JSON.stringify(n)).join(", ")} in both ` +
|
|
186
|
+
`\`assert.diff_contains\` and \`assert.diff_excludes\` — the gate could never pass`, file);
|
|
187
|
+
}
|
|
188
|
+
if (a.post_test !== undefined) {
|
|
189
|
+
if (typeof a.post_test !== "string" || !a.post_test.trim()) {
|
|
190
|
+
throw new SpecError(`seeded scenario \`${id}\` \`assert.post_test\` must be a non-empty path`, file);
|
|
191
|
+
}
|
|
192
|
+
assertObj.post_test = a.post_test.trim();
|
|
193
|
+
}
|
|
144
194
|
scenario.assert = assertObj;
|
|
145
195
|
}
|
|
146
196
|
}
|
|
147
197
|
scenario.workspace = resolveWorkspace(s.env, mode, scenario.fixture, id, file);
|
|
198
|
+
scenario.remote = resolveRemote(s.env, scenario.workspace, id, file);
|
|
199
|
+
if (s.system_prompt_file !== undefined) {
|
|
200
|
+
if (typeof s.system_prompt_file !== "string" || !s.system_prompt_file.trim()) {
|
|
201
|
+
throw new SpecError(`scenario \`${id}\` \`system_prompt_file\` must be a non-empty string`, file);
|
|
202
|
+
}
|
|
203
|
+
// A subagent has no turn two. Testing an agent file across multiple turns would
|
|
204
|
+
// measure conversation armor the single-shot contract deliberately drops.
|
|
205
|
+
if (scenario.turns.length !== 1) {
|
|
206
|
+
throw new SpecError(`scenario \`${id}\` uses system_prompt_file, so it must have exactly one turn ` +
|
|
207
|
+
`(got ${scenario.turns.length}) — an agent definition is single-shot by contract`, file);
|
|
208
|
+
}
|
|
209
|
+
scenario.systemPromptFile = s.system_prompt_file.trim();
|
|
210
|
+
}
|
|
148
211
|
if (s.reps !== undefined) {
|
|
149
212
|
if (typeof s.reps !== "number" || !Number.isInteger(s.reps) || s.reps < 1) {
|
|
150
213
|
throw new SpecError(`scenario \`${id}\` \`reps\` must be a positive integer`, file);
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Environment-variable resolution for the `SKILL_HARNESS_*` namespace.
|
|
3
|
+
*
|
|
4
|
+
* The tuning vars predate the skill-check → skill-harness rename and shipped in
|
|
5
|
+
* 0.1.x under a `SKILL_CHECK_` prefix. Anyone who set one of those in a shell
|
|
6
|
+
* profile or CI config must keep working, so every name is resolved new-first
|
|
7
|
+
* with a legacy fallback and a one-time notice pointing at the new spelling.
|
|
8
|
+
*
|
|
9
|
+
* Call these with the *suffix* only (`"PI_TIMEOUT_MS"`), never a full name — the
|
|
10
|
+
* prefixes are this module's business, which is what keeps ROADMAP rule 5 (no
|
|
11
|
+
* new `SKILL_CHECK_*` names) enforceable by grepping for the prefix.
|
|
12
|
+
*/
|
|
13
|
+
/** Test seam: clears the once-per-suffix warning memo. */
|
|
14
|
+
export declare function __resetEnvWarnings(): void;
|
|
15
|
+
/**
|
|
16
|
+
* Resolve `SKILL_HARNESS_<suffix>`, falling back to `SKILL_CHECK_<suffix>`.
|
|
17
|
+
*
|
|
18
|
+
* An empty value counts as unset: exporting a var as `""` is a common way to
|
|
19
|
+
* neutralize it in CI, and honoring it as "set" would make the legacy fallback
|
|
20
|
+
* unreachable for exactly those users.
|
|
21
|
+
*/
|
|
22
|
+
export declare function readEnv(suffix: string): string | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* Resolve a positive-integer var, e.g. a timeout in ms.
|
|
25
|
+
*
|
|
26
|
+
* A set-but-unparseable value warns and yields `fallback` rather than passing
|
|
27
|
+
* `NaN` down: the pre-rename `Number(process.env.X ?? default)` turned a typo
|
|
28
|
+
* into a NaN timeout, which silently disabled the timeout it was feeding.
|
|
29
|
+
*/
|
|
30
|
+
export declare function envNum(suffix: string, fallback: number): number;
|
|
31
|
+
/** Resolve a boolean var: any non-empty value is on. */
|
|
32
|
+
export declare function envFlag(suffix: string): boolean;
|
package/dist/util/env.js
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Environment-variable resolution for the `SKILL_HARNESS_*` namespace.
|
|
3
|
+
*
|
|
4
|
+
* The tuning vars predate the skill-check → skill-harness rename and shipped in
|
|
5
|
+
* 0.1.x under a `SKILL_CHECK_` prefix. Anyone who set one of those in a shell
|
|
6
|
+
* profile or CI config must keep working, so every name is resolved new-first
|
|
7
|
+
* with a legacy fallback and a one-time notice pointing at the new spelling.
|
|
8
|
+
*
|
|
9
|
+
* Call these with the *suffix* only (`"PI_TIMEOUT_MS"`), never a full name — the
|
|
10
|
+
* prefixes are this module's business, which is what keeps ROADMAP rule 5 (no
|
|
11
|
+
* new `SKILL_CHECK_*` names) enforceable by grepping for the prefix.
|
|
12
|
+
*/
|
|
13
|
+
const NEW_PREFIX = "SKILL_HARNESS_";
|
|
14
|
+
const LEGACY_PREFIX = "SKILL_CHECK_";
|
|
15
|
+
/** Suffixes already warned about, so a module-scope read doesn't nag per call. */
|
|
16
|
+
const warned = new Set();
|
|
17
|
+
/** Test seam: clears the once-per-suffix warning memo. */
|
|
18
|
+
export function __resetEnvWarnings() {
|
|
19
|
+
warned.clear();
|
|
20
|
+
}
|
|
21
|
+
function warnOnce(key, message) {
|
|
22
|
+
if (warned.has(key))
|
|
23
|
+
return;
|
|
24
|
+
warned.add(key);
|
|
25
|
+
process.stderr.write(`skill-harness: ${message}\n`);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Resolve `SKILL_HARNESS_<suffix>`, falling back to `SKILL_CHECK_<suffix>`.
|
|
29
|
+
*
|
|
30
|
+
* An empty value counts as unset: exporting a var as `""` is a common way to
|
|
31
|
+
* neutralize it in CI, and honoring it as "set" would make the legacy fallback
|
|
32
|
+
* unreachable for exactly those users.
|
|
33
|
+
*/
|
|
34
|
+
export function readEnv(suffix) {
|
|
35
|
+
const fresh = process.env[NEW_PREFIX + suffix];
|
|
36
|
+
if (fresh)
|
|
37
|
+
return fresh;
|
|
38
|
+
const legacy = process.env[LEGACY_PREFIX + suffix];
|
|
39
|
+
if (legacy) {
|
|
40
|
+
warnOnce(`legacy:${suffix}`, `${LEGACY_PREFIX}${suffix} is the pre-rename name and still honored; rename it to ${NEW_PREFIX}${suffix}.`);
|
|
41
|
+
return legacy;
|
|
42
|
+
}
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Resolve a positive-integer var, e.g. a timeout in ms.
|
|
47
|
+
*
|
|
48
|
+
* A set-but-unparseable value warns and yields `fallback` rather than passing
|
|
49
|
+
* `NaN` down: the pre-rename `Number(process.env.X ?? default)` turned a typo
|
|
50
|
+
* into a NaN timeout, which silently disabled the timeout it was feeding.
|
|
51
|
+
*/
|
|
52
|
+
export function envNum(suffix, fallback) {
|
|
53
|
+
const raw = readEnv(suffix);
|
|
54
|
+
if (raw === undefined)
|
|
55
|
+
return fallback;
|
|
56
|
+
const n = Number(raw);
|
|
57
|
+
if (!Number.isFinite(n) || n <= 0) {
|
|
58
|
+
warnOnce(`malformed:${suffix}`, `${NEW_PREFIX}${suffix}=${JSON.stringify(raw)} is not a positive number; using ${fallback}.`);
|
|
59
|
+
return fallback;
|
|
60
|
+
}
|
|
61
|
+
return n;
|
|
62
|
+
}
|
|
63
|
+
/** Resolve a boolean var: any non-empty value is on. */
|
|
64
|
+
export function envFlag(suffix) {
|
|
65
|
+
return readEnv(suffix) !== undefined;
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=env.js.map
|
package/dist/workspace.d.ts
CHANGED
|
@@ -6,12 +6,44 @@ export interface Workspace {
|
|
|
6
6
|
cwd: string;
|
|
7
7
|
cleanup(): void;
|
|
8
8
|
}
|
|
9
|
+
export declare const MARKERS: string[];
|
|
10
|
+
/**
|
|
11
|
+
* Top-level directories in a fixture that claim to be markers but aren't one.
|
|
12
|
+
*
|
|
13
|
+
* A misspelled marker (`_uncommited/`) used to be copied into the baseline commit as a
|
|
14
|
+
* literal directory: the tree came up clean, the scenario silently measured the opposite
|
|
15
|
+
* of its intent, and nothing reported it. Any top-level `_name/` is therefore treated as
|
|
16
|
+
* a marker claim and must be a real one.
|
|
17
|
+
*
|
|
18
|
+
* The predicate is `_` followed by a LETTER, so `__tests__` and `__pycache__` — ordinary
|
|
19
|
+
* directories a fixture may legitimately contain — are not marker claims, and neither are
|
|
20
|
+
* oddities like `_2fa/` or `_-tmp/`, which no marker could plausibly be mistaken for.
|
|
21
|
+
* Nested `pkg/_staged/` is likewise ordinary content: markers are top-level only.
|
|
22
|
+
*
|
|
23
|
+
* Exported so `lint` can report the same set this module refuses to run. If the two
|
|
24
|
+
* disagreed, lint would hand out a clean bill of health for a fixture the runtime then
|
|
25
|
+
* rejects — which is worse than not checking at all.
|
|
26
|
+
*/
|
|
27
|
+
export declare function unknownMarkerDirs(src: string): string[];
|
|
28
|
+
/**
|
|
29
|
+
* The known marker a misspelling was probably reaching for, or null.
|
|
30
|
+
*
|
|
31
|
+
* Case-insensitive within an edit distance of 2 — enough for `_uncommited` (one
|
|
32
|
+
* dropped `t`), `_Staged` (case) and `_uncommmitted` (doubled letter), while
|
|
33
|
+
* `_fixtures` or `_helpers` correctly suggest nothing rather than a confident
|
|
34
|
+
* wrong guess.
|
|
35
|
+
*/
|
|
36
|
+
export declare function suggestMarker(name: string): string | null;
|
|
9
37
|
/**
|
|
10
38
|
* Create an isolated temp-dir working directory for one scenario. `none` is an
|
|
11
39
|
* empty dir (no git); `empty-git` initialises a clean repo; `{ fixture }` copies
|
|
12
40
|
* the fixture (relative paths resolve against `specDir`) then initialises a repo
|
|
13
|
-
* with a baseline commit.
|
|
41
|
+
* with a baseline commit. A fixture may carry top-level `_staged/` and
|
|
42
|
+
* `_uncommitted/` subdirectories, applied after that commit to start the scenario
|
|
43
|
+
* from a dirty tree. `opts.remote` additionally wires a local bare `origin`.
|
|
44
|
+
* Child processes run here, never in the user's home.
|
|
14
45
|
*/
|
|
15
46
|
export declare function createWorkspace(kind: WorkspaceKind, opts: {
|
|
16
47
|
specDir: string;
|
|
48
|
+
remote?: boolean;
|
|
17
49
|
}): Workspace;
|