@uinaf/skillcheck 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +11 -3
- package/dist/grok-provider.js +5 -0
- package/dist/scenario.js +7 -2
- package/docs/scenarios.md +3 -1
- package/docs/usage.md +3 -3
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -235,7 +235,7 @@ function cmdSweep(argv) {
|
|
|
235
235
|
for (const dir of discoverScenarios(root)) {
|
|
236
236
|
const name = runNameFor(dir, opts.harness);
|
|
237
237
|
const resultPath = path.join(resultsDir, `${name}.json`);
|
|
238
|
-
if (!all && fs.existsSync(resultPath)) {
|
|
238
|
+
if (!all && fs.existsSync(resultPath) && !fs.existsSync(attemptPath(resultPath))) {
|
|
239
239
|
skipped++;
|
|
240
240
|
console.log(`SKIP ${name} (results exist; use --all to rerun)`);
|
|
241
241
|
continue;
|
|
@@ -256,13 +256,21 @@ function cmdSweep(argv) {
|
|
|
256
256
|
process.exit(errored > 0 ? 2 : failed > 0 ? 1 : 0);
|
|
257
257
|
}
|
|
258
258
|
function resultIdentity(file) {
|
|
259
|
+
const decode = (part) => {
|
|
260
|
+
if (!part.startsWith("~v2~")) return part;
|
|
261
|
+
try {
|
|
262
|
+
return decodeURIComponent(part.slice(4));
|
|
263
|
+
} catch {
|
|
264
|
+
return part;
|
|
265
|
+
}
|
|
266
|
+
};
|
|
259
267
|
const base = file.replace(/\.json$/, "");
|
|
260
268
|
const suffix = base.match(/--(codex|grok|cursor)$/);
|
|
261
269
|
const harness = suffix === null ? "claude" : suffix[1];
|
|
262
270
|
const [skill, ...rest] = base.replace(/--(codex|grok|cursor)$/, "").split("--");
|
|
263
271
|
return {
|
|
264
|
-
skill,
|
|
265
|
-
scenario: rest.join("--"),
|
|
272
|
+
skill: decode(skill),
|
|
273
|
+
scenario: decode(rest.join("--")),
|
|
266
274
|
harness
|
|
267
275
|
};
|
|
268
276
|
}
|
package/dist/grok-provider.js
CHANGED
|
@@ -113,6 +113,11 @@ var GrokProvider = class {
|
|
|
113
113
|
const abort = () => stop("grok run aborted");
|
|
114
114
|
callOptions?.abortSignal?.addEventListener("abort", abort, { once: true });
|
|
115
115
|
if (callOptions?.abortSignal?.aborted) abort();
|
|
116
|
+
child.on("exit", () => {
|
|
117
|
+
if (process.platform !== "win32" && child.pid !== void 0) try {
|
|
118
|
+
process.kill(-child.pid, "SIGKILL");
|
|
119
|
+
} catch {}
|
|
120
|
+
});
|
|
116
121
|
child.on("close", (code, signal) => {
|
|
117
122
|
clearTimeout(timer);
|
|
118
123
|
callOptions?.abortSignal?.removeEventListener("abort", abort);
|
package/dist/scenario.js
CHANGED
|
@@ -32,10 +32,15 @@ function loadScenario(scenarioDir) {
|
|
|
32
32
|
criteria
|
|
33
33
|
};
|
|
34
34
|
}
|
|
35
|
+
function encodeRunNamePart(part) {
|
|
36
|
+
if (!part.includes("--") && !part.startsWith("-") && !part.endsWith("-") && !part.startsWith("~v2~")) return part;
|
|
37
|
+
return `~v2~${encodeURIComponent(part).replaceAll("-", "%2D").replaceAll("~", "%7E")}`;
|
|
38
|
+
}
|
|
35
39
|
function runNameFor(scenarioDir, harness) {
|
|
36
40
|
const m = path.resolve(scenarioDir).match(/skills\/([^/]+)\/evals\/([^/]+)$/);
|
|
37
41
|
if (!m) throw new Error(`not a scenario dir (want .../skills/<skill>/evals/<scenario>): ${scenarioDir}`);
|
|
38
|
-
|
|
42
|
+
const name = `${encodeRunNamePart(m[1])}--${encodeRunNamePart(m[2])}`;
|
|
43
|
+
return harness === "claude" ? name : `${name}--${harness}`;
|
|
39
44
|
}
|
|
40
45
|
function frontmatterRange(text) {
|
|
41
46
|
const lines = text.split("\n");
|
|
@@ -258,4 +263,4 @@ function generateRun(scenarioDir, opts, paths) {
|
|
|
258
263
|
};
|
|
259
264
|
}
|
|
260
265
|
//#endregion
|
|
261
|
-
export { buildConfig, generateRun, isHiddenSkill, loadScenario, materialize, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag };
|
|
266
|
+
export { buildConfig, encodeRunNamePart, generateRun, isHiddenSkill, loadScenario, materialize, requiredEvalPackages, resolvePackageDir, runNameFor, sdkNodeModulesDir, stripHiddenFlag };
|
package/docs/scenarios.md
CHANGED
|
@@ -9,7 +9,9 @@ A scenario is two files in a frozen location:
|
|
|
9
9
|
|
|
10
10
|
The path is the identity: `<skill>--<scenario>` names the run, the result file,
|
|
11
11
|
and the scorecard entry. Codex and Grok results gain `--codex` and `--grok`
|
|
12
|
-
suffixes, so harnesses can hold results side by side.
|
|
12
|
+
suffixes, so harnesses can hold results side by side. Double hyphens and
|
|
13
|
+
leading or trailing hyphens in a skill or scenario name are escaped in the run
|
|
14
|
+
and result filename; the scorecard keeps the original name.
|
|
13
15
|
A directory missing either file is not discovered.
|
|
14
16
|
|
|
15
17
|
## task.md
|
package/docs/usage.md
CHANGED
|
@@ -77,7 +77,7 @@ provider-qualified judge; the Anthropic judge does not take one.
|
|
|
77
77
|
## Sweep
|
|
78
78
|
|
|
79
79
|
```sh
|
|
80
|
-
skillcheck sweep #
|
|
80
|
+
skillcheck sweep # scenarios without completed results
|
|
81
81
|
skillcheck sweep --all # rerun everything
|
|
82
82
|
```
|
|
83
83
|
|
|
@@ -91,8 +91,8 @@ within one scenario, not across them.
|
|
|
91
91
|
One known failure mode: judge calls through a gateway can drop at the transport
|
|
92
92
|
layer ([uinaf/zebroid-infra#44](https://github.com/uinaf/zebroid-infra/issues/44)).
|
|
93
93
|
That surfaces as an ERROR with no usable result, not as a graded FAIL, and the
|
|
94
|
-
mitigation is a rerun. `sweep` without `--all` resumes, so a rerun
|
|
95
|
-
|
|
94
|
+
mitigation is a rerun. `sweep` without `--all` resumes, so a rerun picks up
|
|
95
|
+
missing or ungraded results.
|
|
96
96
|
|
|
97
97
|
## Summarize
|
|
98
98
|
|