@sx4im/skillcheck 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -38
- package/dist/bin/skillcheck.js +2 -2
- package/dist/bin/skillcheck.js.map +1 -1
- package/dist/src/adapters/nvidia-nim.js +5 -2
- package/dist/src/adapters/nvidia-nim.js.map +1 -1
- package/dist/src/cli.d.ts +8 -0
- package/dist/src/cli.js +69 -16
- package/dist/src/cli.js.map +1 -1
- package/dist/src/env.js +29 -13
- package/dist/src/env.js.map +1 -1
- package/dist/src/eval.d.ts +1 -0
- package/dist/src/eval.js +7 -4
- package/dist/src/eval.js.map +1 -1
- package/dist/src/generate.js +3 -1
- package/dist/src/generate.js.map +1 -1
- package/dist/src/grade.js +7 -2
- package/dist/src/grade.js.map +1 -1
- package/dist/src/run.js +7 -2
- package/dist/src/run.js.map +1 -1
- package/dist/src/ui.d.ts +12 -0
- package/dist/src/ui.js +310 -0
- package/dist/src/ui.js.map +1 -0
- package/docs/skillcheck-cloud.md +100 -0
- package/examples/nvidia-proxy/README.md +19 -0
- package/examples/nvidia-proxy/server.mjs +96 -0
- package/package.json +4 -2
package/README.md
CHANGED
|
@@ -8,39 +8,42 @@ Measure whether an agent skill actually improves task performance.
|
|
|
8
8
|
|
|
9
9
|
- M0-M4 gates have passed and are recorded in `BUILD-LOG.md`.
|
|
10
10
|
- M5 completed a 20-skill launch corpus in `results/launch/20260605T110514Z-qwen-next`.
|
|
11
|
-
-
|
|
12
|
-
- Do not publish to npm until `RELEASE-CHECKLIST.md` is complete.
|
|
11
|
+
- Published on npm as `@sx4im/skillcheck`.
|
|
13
12
|
|
|
14
|
-
##
|
|
15
|
-
|
|
16
|
-
After npm publication, users can run the CLI with one command and no global install:
|
|
13
|
+
## One-Line Install
|
|
17
14
|
|
|
18
15
|
```bash
|
|
19
|
-
|
|
16
|
+
npm install -g @sx4im/skillcheck
|
|
20
17
|
```
|
|
21
18
|
|
|
22
|
-
|
|
19
|
+
Requires Node.js 20 or newer.
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
23
22
|
|
|
24
23
|
```bash
|
|
25
|
-
|
|
26
|
-
skillcheck --help
|
|
24
|
+
skillcheck
|
|
27
25
|
```
|
|
28
26
|
|
|
29
|
-
|
|
27
|
+
This opens the interactive picker. Select `SKILL.md`, `AGENTS.md`, `CLAUDE.md`, `.cursorrules`, or a folder containing one.
|
|
28
|
+
|
|
29
|
+
You can also pass a path directly:
|
|
30
30
|
|
|
31
31
|
```bash
|
|
32
|
-
|
|
32
|
+
skillcheck check path/to/SKILL.md
|
|
33
|
+
skillcheck path/to/skill-folder
|
|
33
34
|
```
|
|
34
35
|
|
|
35
|
-
|
|
36
|
+
The quick run shows a blue/white result card in the terminal. It does not write a results folder unless you explicitly pass `--output`.
|
|
37
|
+
|
|
38
|
+
## Cloud Setup
|
|
39
|
+
|
|
40
|
+
For shared/public usage, point the CLI at your Skillcheck Cloud endpoint:
|
|
36
41
|
|
|
37
42
|
```bash
|
|
38
|
-
|
|
43
|
+
export SKILLCHECK_API_URL=https://api.yourdomain.com/v1
|
|
39
44
|
```
|
|
40
45
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
Requires Node.js 20 or newer.
|
|
46
|
+
See `docs/skillcheck-cloud.md` for the proxy and dashboard plan.
|
|
44
47
|
|
|
45
48
|
## Local Development
|
|
46
49
|
|
|
@@ -50,40 +53,27 @@ npm run build
|
|
|
50
53
|
node dist/bin/skillcheck.js --help
|
|
51
54
|
```
|
|
52
55
|
|
|
53
|
-
|
|
56
|
+
## Commands
|
|
57
|
+
|
|
58
|
+
Friendly skill check:
|
|
54
59
|
|
|
55
60
|
```bash
|
|
56
|
-
|
|
61
|
+
skillcheck check path/to/SKILL.md
|
|
57
62
|
```
|
|
58
63
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
Copy `.env.example` to `.env` and set `NVIDIA_API_KEY`.
|
|
62
|
-
|
|
63
|
-
Required for live NVIDIA NIM calls:
|
|
64
|
+
You can also omit `check` when the argument is an existing path:
|
|
64
65
|
|
|
65
66
|
```bash
|
|
66
|
-
|
|
67
|
-
NVIDIA_BASE_URL=https://integrate.api.nvidia.com/v1
|
|
68
|
-
NVIDIA_GENERATOR_MODEL=qwen/qwen3-next-80b-a3b-instruct
|
|
69
|
-
NVIDIA_GRADER_MODEL=qwen/qwen3-next-80b-a3b-instruct
|
|
70
|
-
NVIDIA_RUNNER_MODEL=qwen/qwen3-next-80b-a3b-instruct
|
|
67
|
+
skillcheck path/to/skill-folder
|
|
71
68
|
```
|
|
72
69
|
|
|
73
|
-
|
|
70
|
+
Run a stronger check:
|
|
74
71
|
|
|
75
72
|
```bash
|
|
76
|
-
|
|
77
|
-
NVIDIA_REQUEST_DELAY_MS=5000
|
|
78
|
-
NVIDIA_MAX_ATTEMPTS=8
|
|
79
|
-
NVIDIA_MAX_RETRY_DELAY_MS=60000
|
|
73
|
+
skillcheck check path/to/SKILL.md --tasks 10 --trials 3
|
|
80
74
|
```
|
|
81
75
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
## Commands
|
|
85
|
-
|
|
86
|
-
Evaluate one skill:
|
|
76
|
+
Advanced raw JSON evaluator:
|
|
87
77
|
|
|
88
78
|
```bash
|
|
89
79
|
skillcheck eval path/to/SKILL.md --tasks 10 --trials 3 --output results/my-run.json
|
package/dist/bin/skillcheck.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { main } from '../src/cli.js';
|
|
3
|
+
import { formatFatalError } from '../src/ui.js';
|
|
3
4
|
main(process.argv).catch((error) => {
|
|
4
|
-
|
|
5
|
-
console.error(message);
|
|
5
|
+
console.error(formatFatalError(error));
|
|
6
6
|
process.exitCode = 1;
|
|
7
7
|
});
|
|
8
8
|
//# sourceMappingURL=skillcheck.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skillcheck.js","sourceRoot":"","sources":["../../packages/cli/bin/skillcheck.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"skillcheck.js","sourceRoot":"","sources":["../../packages/cli/bin/skillcheck.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC1C,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
@@ -64,9 +64,12 @@ function getRetryAfterMs(error) {
|
|
|
64
64
|
return undefined;
|
|
65
65
|
}
|
|
66
66
|
function describeRetry(error, attempt, waitMs, maxAttempts) {
|
|
67
|
+
if (process.env.SKILLCHECK_DEBUG_RETRIES !== '1') {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
67
70
|
const status = getStatus(error);
|
|
68
71
|
const label = status === undefined ? 'connection' : `status ${status}`;
|
|
69
|
-
console.error(`[
|
|
72
|
+
console.error(`[skillcheck] retry ${attempt + 1}/${maxAttempts} after ${label}; waiting ${waitMs} ms`);
|
|
70
73
|
}
|
|
71
74
|
export class NvidiaNimClient {
|
|
72
75
|
static requestQueue = Promise.resolve();
|
|
@@ -132,7 +135,7 @@ export class NvidiaNimClient {
|
|
|
132
135
|
const content = extractMessageText(message);
|
|
133
136
|
if (content === undefined) {
|
|
134
137
|
const keys = message ? Object.keys(message).sort().join(',') : 'none';
|
|
135
|
-
throw new Error(`
|
|
138
|
+
throw new Error(`Skillcheck Cloud response did not include text content; message keys: ${keys}`);
|
|
136
139
|
}
|
|
137
140
|
return {
|
|
138
141
|
content,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nvidia-nim.js","sourceRoot":"","sources":["../../../packages/cli/src/adapters/nvidia-nim.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAuB5B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAC;gBAC/C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAgF,CAAC;IAChG,OAAO,CACL,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;QAClC,kBAAkB,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACrE,MAAM,MAAM,GAAI,KAA8B,CAAC,MAAM,CAAC;QACtD,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,MAAM,UAAU,GACd,OAAO,YAAY,OAAO;QACxB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC5B,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,aAAa,IAAI,OAAO;YAC3E,CAAC,CAAE,OAAuC,CAAC,aAAa,CAAC;YACzD,CAAC,CAAC,SAAS,CAAC;IAElB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,MAAc,EAAE,WAAmB;IACzF,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"nvidia-nim.js","sourceRoot":"","sources":["../../../packages/cli/src/adapters/nvidia-nim.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAuB5B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAC;gBAC/C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAgF,CAAC;IAChG,OAAO,CACL,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;QAClC,kBAAkB,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACrE,MAAM,MAAM,GAAI,KAA8B,CAAC,MAAM,CAAC;QACtD,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,MAAM,UAAU,GACd,OAAO,YAAY,OAAO;QACxB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC5B,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,aAAa,IAAI,OAAO;YAC3E,CAAC,CAAE,OAAuC,CAAC,aAAa,CAAC;YACzD,CAAC,CAAC,SAAS,CAAC;IAElB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,MAAc,EAAE,WAAmB;IACzF,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG,EAAE,CAAC;QACjD,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,GAAG,CAAC,IAAI,WAAW,UAAU,KAAK,aAAa,MAAM,KAAK,CAAC,CAAC;AACzG,CAAC;AAED,MAAM,OAAO,eAAe;IAClB,MAAM,CAAC,YAAY,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IACvD,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;IAEhB,MAAM,CAAS;IACf,cAAc,CAAS;IACvB,WAAW,CAAS;IACpB,eAAe,CAAS;IAEzC,YAAY,MAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,CAAC,SAAS;SAC1B,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAI,SAA2B;QAC/D,MAAM,eAAe,GAAG,eAAe,CAAC,YAAY,CAAC;QACrD,IAAI,qBAAqB,GAAe,GAAG,EAAE,CAAC,SAAS,CAAC;QACxD,eAAe,CAAC,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC3D,qBAAqB,GAAG,OAAO,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,CAAC;QACtB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,aAAa,CAAC;gBAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;gBAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,eAAe,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3C,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,qBAAqB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAA0B;QACvC,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,OAAO,CAAC,kBAAkB;oBAC1C,CAAC,CAAC;wBACE,oBAAoB,EAAE,OAAO,CAAC,kBAAkB;qBACjD;oBACH,CAAC,CAAC,SAAS,CAAC;gBACd,MAAM,WAAW,GAAG;oBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,UAAU,EAAE,OAAO,CAAC,SAAS;oBAC7B,eAAe,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS;oBACtF,oBAAoB,EAAE,OAAO,CAAC,kBAAkB;oBAChD,UAAU,EAAE,SAAS;oBACrB,MAAM,EAAE,KAAK;iBACd,CAAC;gBACF,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,CACjD,CAAmB,CAAC;gBAErB,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC7C,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;oBACtE,MAAM,IAAI,KAAK,CAAC,yEAAyE,IAAI,EAAE,CAAC,CAAC;gBACnG,CAAC;gBAED,OAAO;oBACL,OAAO;oBACP,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,KAAK,EAAE;wBACL,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa;wBAC3C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB;wBACnD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY;qBAC1C;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACzE,IAAI,CAAC,SAAS,IAAI,OAAO,KAAK,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;oBACnD,MAAM;gBACR,CAAC;gBAED,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;gBAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;gBAC/C,MAAM,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,WAAW,GAAG,CAAC,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC;gBACnG,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxD,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC9E,CAAC"}
|
package/dist/src/cli.d.ts
CHANGED
|
@@ -1 +1,9 @@
|
|
|
1
|
+
import { type EvalOptions } from './eval.js';
|
|
2
|
+
interface CheckOptions {
|
|
3
|
+
evalOptions: EvalOptions;
|
|
4
|
+
json: boolean;
|
|
5
|
+
output?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function parseCheckOptions(argv: string[], inputIndex?: number): CheckOptions;
|
|
1
8
|
export declare function main(argv: string[]): Promise<void>;
|
|
9
|
+
export {};
|
package/dist/src/cli.js
CHANGED
|
@@ -1,23 +1,13 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
1
2
|
import { NvidiaNimClient } from './adapters/nvidia-nim.js';
|
|
2
3
|
import { runCorpus } from './corpus.js';
|
|
3
4
|
import { evalSkill } from './eval.js';
|
|
4
5
|
import { runM0Gate } from './m0/run.js';
|
|
5
6
|
import { runRot } from './rot.js';
|
|
7
|
+
import { formatResultCard, printHelpUi, printBanner, selectSkillPath, startProgress, validateSkillInput } from './ui.js';
|
|
6
8
|
import { verifyResult } from './verify.js';
|
|
7
9
|
function printHelp() {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
Usage:
|
|
11
|
-
skillcheck m0
|
|
12
|
-
skillcheck eval <path> [--tasks N] [--trials K] [--output file.json] [--task-suite file.json]
|
|
13
|
-
[--runner model] [--grader model] [--generator model]
|
|
14
|
-
skillcheck verify <result.json> [--sample n]
|
|
15
|
-
skillcheck corpus run --corpus corpus.json [--results dir] [--tasks N] [--trials K]
|
|
16
|
-
[--concurrency N] [--runner model] [--limit N]
|
|
17
|
-
skillcheck rot [--results dir] [--output file.json] [--model model] [--corpus corpus.yaml]
|
|
18
|
-
[--tasks N] [--trials K]
|
|
19
|
-
|
|
20
|
-
M0 is the hardcoded spike. eval is the M1 forced-injection evaluator.`);
|
|
10
|
+
printHelpUi();
|
|
21
11
|
}
|
|
22
12
|
function readOption(argv, name) {
|
|
23
13
|
const index = argv.indexOf(name);
|
|
@@ -52,8 +42,11 @@ function readOptionalNumberOption(argv, name) {
|
|
|
52
42
|
}
|
|
53
43
|
return parsed;
|
|
54
44
|
}
|
|
55
|
-
function
|
|
56
|
-
|
|
45
|
+
function hasFlag(argv, name) {
|
|
46
|
+
return argv.includes(name);
|
|
47
|
+
}
|
|
48
|
+
function parseEvalOptions(argv, inputIndex = 3) {
|
|
49
|
+
const inputPath = argv[inputIndex];
|
|
57
50
|
if (!inputPath || inputPath.startsWith('--')) {
|
|
58
51
|
throw new Error('Usage: skillcheck eval <path> [--tasks N] [--trials K] [--output file.json]');
|
|
59
52
|
}
|
|
@@ -73,6 +66,29 @@ function parseEvalOptions(argv) {
|
|
|
73
66
|
taskSuite: readOption(argv, '--task-suite')
|
|
74
67
|
};
|
|
75
68
|
}
|
|
69
|
+
export function parseCheckOptions(argv, inputIndex = 3) {
|
|
70
|
+
const inputPath = argv[inputIndex];
|
|
71
|
+
if (!inputPath || inputPath.startsWith('--')) {
|
|
72
|
+
throw new Error('Usage: skillcheck check <path-to-skill-file-or-folder>\nExample: skillcheck check ./SKILL.md');
|
|
73
|
+
}
|
|
74
|
+
const output = readOption(argv, '--output');
|
|
75
|
+
return {
|
|
76
|
+
evalOptions: {
|
|
77
|
+
inputPath,
|
|
78
|
+
output,
|
|
79
|
+
tasks: readNumberOption(argv, '--tasks', 3),
|
|
80
|
+
trials: readNumberOption(argv, '--trials', 2),
|
|
81
|
+
mode: 'forced',
|
|
82
|
+
runner: readOption(argv, '--runner'),
|
|
83
|
+
grader: readOption(argv, '--grader'),
|
|
84
|
+
generator: readOption(argv, '--generator'),
|
|
85
|
+
taskSuite: readOption(argv, '--task-suite'),
|
|
86
|
+
saveArtifacts: Boolean(output)
|
|
87
|
+
},
|
|
88
|
+
json: hasFlag(argv, '--json'),
|
|
89
|
+
output
|
|
90
|
+
};
|
|
91
|
+
}
|
|
76
92
|
function parseRotOptions(argv) {
|
|
77
93
|
return {
|
|
78
94
|
resultsDir: readOption(argv, '--results') ?? 'results',
|
|
@@ -98,18 +114,51 @@ function parseCorpusRunOptions(argv) {
|
|
|
98
114
|
limit: readOptionalNumberOption(argv, '--limit')
|
|
99
115
|
};
|
|
100
116
|
}
|
|
117
|
+
async function runCheck(options, showBanner = true) {
|
|
118
|
+
if (showBanner && !options.json) {
|
|
119
|
+
printBanner();
|
|
120
|
+
}
|
|
121
|
+
await validateSkillInput(options.evalOptions.inputPath);
|
|
122
|
+
const progress = options.json ? undefined : startProgress();
|
|
123
|
+
let result;
|
|
124
|
+
try {
|
|
125
|
+
result = await evalSkill(options.evalOptions);
|
|
126
|
+
progress?.finish();
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
progress?.fail();
|
|
130
|
+
throw error;
|
|
131
|
+
}
|
|
132
|
+
if (options.json) {
|
|
133
|
+
console.log(JSON.stringify(result, null, 2));
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
console.log(formatResultCard(result, options.output));
|
|
137
|
+
}
|
|
138
|
+
async function runInteractiveCheck() {
|
|
139
|
+
const selectedPath = await selectSkillPath();
|
|
140
|
+
await runCheck(parseCheckOptions(['node', 'skillcheck', 'check', selectedPath]), false);
|
|
141
|
+
}
|
|
101
142
|
export async function main(argv) {
|
|
102
143
|
const command = argv[2];
|
|
103
|
-
if (
|
|
144
|
+
if (command === '--help' || command === '-h') {
|
|
104
145
|
printHelp();
|
|
105
146
|
return;
|
|
106
147
|
}
|
|
148
|
+
if (!command) {
|
|
149
|
+
await runInteractiveCheck();
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
107
152
|
if (command === 'm0') {
|
|
108
153
|
const report = await runM0Gate((config) => new NvidiaNimClient(config));
|
|
109
154
|
console.log(JSON.stringify(report, null, 2));
|
|
110
155
|
process.exitCode = report.passed ? 0 : 1;
|
|
111
156
|
return;
|
|
112
157
|
}
|
|
158
|
+
if (command === 'check') {
|
|
159
|
+
await runCheck(parseCheckOptions(argv));
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
113
162
|
if (command === 'eval') {
|
|
114
163
|
const result = await evalSkill(parseEvalOptions(argv));
|
|
115
164
|
console.log(JSON.stringify(result, null, 2));
|
|
@@ -141,6 +190,10 @@ export async function main(argv) {
|
|
|
141
190
|
console.log(JSON.stringify(result, null, 2));
|
|
142
191
|
return;
|
|
143
192
|
}
|
|
193
|
+
if (!command.startsWith('-') && existsSync(command)) {
|
|
194
|
+
await runCheck(parseCheckOptions(['node', 'skillcheck', 'check', ...argv.slice(2)]));
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
144
197
|
throw new Error(`Unknown command: ${command}`);
|
|
145
198
|
}
|
|
146
199
|
//# sourceMappingURL=cli.js.map
|
package/dist/src/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../packages/cli/src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAyB,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAoB,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,MAAM,EAAmB,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,SAAS;IAChB,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../packages/cli/src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAyB,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAoB,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,MAAM,EAAmB,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AACzH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,SAAS;IAChB,WAAW,EAAE,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,IAAc,EAAE,IAAY;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,IAAY,EAAE,QAAgB;IACtE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAc,EAAE,IAAY;IAC5D,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,IAAc,EAAE,IAAY;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,UAAU,GAAG,CAAC;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAC;IACpD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,SAAS;QACT,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7C,IAAI;QACJ,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC;QAC1C,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC;KAC5C,CAAC;AACJ,CAAC;AAQD,MAAM,UAAU,iBAAiB,CAAC,IAAc,EAAE,UAAU,GAAG,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CACb,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC5C,OAAO;QACL,WAAW,EAAE;YACX,SAAS;YACT,MAAM;YACN,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAC3C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAC7C,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;YACpC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;YACpC,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC;YAC1C,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC;YAC3C,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC;SAC/B;QACD,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;QAC7B,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,IAAc;IACrC,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,SAAS;QACtD,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,yBAAyB;QACjE,KAAK,EAAE,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC;QAClC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAc;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,OAAO;QACL,MAAM;QACN,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,gBAAgB;QAC5D,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7C,WAAW,EAAE,gBAAgB,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;QACvD,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,wBAAwB,CAAC,IAAI,EAAE,SAAS,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAAqB,EAAE,UAAU,GAAG,IAAI;IAC9D,IAAI,UAAU,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChC,WAAW,EAAE,CAAC;IAChB,CAAC;IACD,MAAM,kBAAkB,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IAC5D,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,EAAE,IAAI,EAAE,CAAC;QACjB,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,KAAK,UAAU,mBAAmB;IAChC,MAAM,YAAY,GAAG,MAAM,eAAe,EAAE,CAAC;IAC7C,MAAM,QAAQ,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC1F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAc;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAExB,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QAC7C,SAAS,EAAE,CAAC;QACZ,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,mBAAmB,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,MAAM,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,UAAU;YACV,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;SAC9C,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,MAAM,QAAQ,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,OAAO;IACT,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;AACjD,CAAC"}
|
package/dist/src/env.js
CHANGED
|
@@ -1,39 +1,55 @@
|
|
|
1
1
|
import dotenv from 'dotenv';
|
|
2
2
|
dotenv.config();
|
|
3
|
-
|
|
3
|
+
const DEFAULT_MODEL = 'qwen/qwen3-next-80b-a3b-instruct';
|
|
4
|
+
function readEnv(name) {
|
|
4
5
|
const value = process.env[name]?.trim();
|
|
5
|
-
|
|
6
|
-
|
|
6
|
+
return value || undefined;
|
|
7
|
+
}
|
|
8
|
+
function resolveApiKey(proxyUrl) {
|
|
9
|
+
const nvidiaApiKey = readEnv('NVIDIA_API_KEY');
|
|
10
|
+
if (nvidiaApiKey) {
|
|
11
|
+
return nvidiaApiKey;
|
|
12
|
+
}
|
|
13
|
+
const proxyApiKey = readEnv('SKILLCHECK_TOKEN') ?? readEnv('SKILLCHECK_API_KEY');
|
|
14
|
+
if (proxyUrl) {
|
|
15
|
+
return proxyApiKey ?? 'skillcheck-cloud';
|
|
7
16
|
}
|
|
8
|
-
|
|
17
|
+
throw new Error('Skillcheck Cloud is not connected for this workspace. Set SKILLCHECK_API_URL or try again later.');
|
|
18
|
+
}
|
|
19
|
+
function resolveModel(role) {
|
|
20
|
+
return (readEnv(`NVIDIA_${role}_MODEL`) ??
|
|
21
|
+
readEnv(`SKILLCHECK_${role}_MODEL`) ??
|
|
22
|
+
readEnv('SKILLCHECK_MODEL') ??
|
|
23
|
+
DEFAULT_MODEL);
|
|
9
24
|
}
|
|
10
25
|
export function loadNvidiaConfig() {
|
|
11
26
|
const timeoutMs = Number(process.env.NVIDIA_TIMEOUT_MS?.trim() || 120000);
|
|
12
27
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
13
|
-
throw new Error('
|
|
28
|
+
throw new Error('Request timeout must be a positive number when set');
|
|
14
29
|
}
|
|
15
30
|
const requestDelayMs = Number(process.env.NVIDIA_REQUEST_DELAY_MS?.trim() || 5000);
|
|
16
31
|
if (!Number.isFinite(requestDelayMs) || requestDelayMs < 0) {
|
|
17
|
-
throw new Error('
|
|
32
|
+
throw new Error('Request delay must be a non-negative number when set');
|
|
18
33
|
}
|
|
19
34
|
const maxAttempts = Number(process.env.NVIDIA_MAX_ATTEMPTS?.trim() || 8);
|
|
20
35
|
if (!Number.isInteger(maxAttempts) || maxAttempts <= 0) {
|
|
21
|
-
throw new Error('
|
|
36
|
+
throw new Error('Retry attempt budget must be a positive integer when set');
|
|
22
37
|
}
|
|
23
38
|
const maxRetryDelayMs = Number(process.env.NVIDIA_MAX_RETRY_DELAY_MS?.trim() || 60000);
|
|
24
39
|
if (!Number.isFinite(maxRetryDelayMs) || maxRetryDelayMs <= 0) {
|
|
25
|
-
throw new Error('
|
|
40
|
+
throw new Error('Retry delay cap must be a positive number when set');
|
|
26
41
|
}
|
|
42
|
+
const proxyUrl = readEnv('SKILLCHECK_API_URL');
|
|
27
43
|
return {
|
|
28
|
-
apiKey:
|
|
29
|
-
baseUrl:
|
|
44
|
+
apiKey: resolveApiKey(proxyUrl),
|
|
45
|
+
baseUrl: proxyUrl ?? readEnv('NVIDIA_BASE_URL') ?? 'https://integrate.api.nvidia.com/v1',
|
|
30
46
|
timeoutMs,
|
|
31
47
|
requestDelayMs,
|
|
32
48
|
maxAttempts,
|
|
33
49
|
maxRetryDelayMs,
|
|
34
|
-
generatorModel:
|
|
35
|
-
graderModel:
|
|
36
|
-
runnerModel:
|
|
50
|
+
generatorModel: resolveModel('GENERATOR'),
|
|
51
|
+
graderModel: resolveModel('GRADER'),
|
|
52
|
+
runnerModel: resolveModel('RUNNER')
|
|
37
53
|
};
|
|
38
54
|
}
|
|
39
55
|
//# sourceMappingURL=env.js.map
|
package/dist/src/env.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../packages/cli/src/env.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../packages/cli/src/env.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,aAAa,GAAG,kCAAkC,CAAC;AAczD,SAAS,OAAO,CAAC,IAAY;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;IACxC,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED,SAAS,aAAa,CAAC,QAA4B;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC/C,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjF,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,WAAW,IAAI,kBAAkB,CAAC;IAC3C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,kGAAkG,CAAC,CAAC;AACtH,CAAC;AAED,SAAS,YAAY,CAAC,IAAuC;IAC3D,OAAO,CACL,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC;QAC/B,OAAO,CAAC,cAAc,IAAI,QAAQ,CAAC;QACnC,OAAO,CAAC,kBAAkB,CAAC;QAC3B,aAAa,CACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,CAAC;IAC1E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;IACnF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,CAAC;IACvF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAE/C,OAAO;QACL,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC;QAC/B,OAAO,EAAE,QAAQ,IAAI,OAAO,CAAC,iBAAiB,CAAC,IAAI,qCAAqC;QACxF,SAAS;QACT,cAAc;QACd,WAAW;QACX,eAAe;QACf,cAAc,EAAE,YAAY,CAAC,WAAW,CAAC;QACzC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC;QACnC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC;KACpC,CAAC;AACJ,CAAC"}
|
package/dist/src/eval.d.ts
CHANGED
package/dist/src/eval.js
CHANGED
|
@@ -84,17 +84,20 @@ function parseTaskSuite(text) {
|
|
|
84
84
|
});
|
|
85
85
|
}
|
|
86
86
|
export async function evalSkill(options) {
|
|
87
|
+
const skill = await normalizeSkill(options.inputPath);
|
|
87
88
|
const baseConfig = loadNvidiaConfig();
|
|
88
89
|
const config = applyModelOverrides(baseConfig, options);
|
|
89
90
|
const client = new NvidiaNimClient(config);
|
|
90
91
|
const cache = new JsonCache();
|
|
91
|
-
const skill = await normalizeSkill(options.inputPath);
|
|
92
92
|
const tasks = options.taskSuite
|
|
93
93
|
? parseTaskSuite(await readFile(options.taskSuite, 'utf8')).slice(0, options.tasks)
|
|
94
94
|
: await generateTasks({ domain: skill.domain, count: options.tasks }, config, client, cache);
|
|
95
95
|
const taskSuiteHash = hashJson({ skill: skill.versionHash, tasks });
|
|
96
|
-
const
|
|
97
|
-
|
|
96
|
+
const shouldSaveArtifacts = options.saveArtifacts ?? true;
|
|
97
|
+
const taskSuitePath = options.taskSuite ?? (shouldSaveArtifacts ? `results/tasks/${taskSuiteHash}.json` : undefined);
|
|
98
|
+
if (!options.taskSuite && taskSuitePath) {
|
|
99
|
+
await writeJson(taskSuitePath, tasks);
|
|
100
|
+
}
|
|
98
101
|
const outputs = await runTrials(skill, tasks, options.trials, config, client, cache);
|
|
99
102
|
const graded = await gradeOutputs(tasks, outputs, config, client, cache);
|
|
100
103
|
const score = scorePairedObservations(pairedObservations(graded));
|
|
@@ -134,7 +137,7 @@ export async function evalSkill(options) {
|
|
|
134
137
|
},
|
|
135
138
|
tasks: breakdowns,
|
|
136
139
|
reproducibility: {
|
|
137
|
-
task_suite_path: taskSuitePath,
|
|
140
|
+
...(taskSuitePath ? { task_suite_path: taskSuitePath } : {}),
|
|
138
141
|
transcript_hashes: graded.map((item) => item.transcriptHash)
|
|
139
142
|
},
|
|
140
143
|
history: [
|
package/dist/src/eval.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../packages/cli/src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAqB,MAAM,UAAU,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../packages/cli/src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAqB,MAAM,UAAU,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,YAAY,CAAC;AAiB7E,SAAS,mBAAmB,CAAC,MAAoB,EAAE,OAAoB;IACrE,OAAO;QACL,GAAG,MAAM;QACT,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,cAAc,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAsB;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC;YAC9B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAyB,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAsB,EAAE,MAAsB;IACpE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QACrE,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,cAAc,EAAE,IAAI,CAAC,aAAa;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,eAAe,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;YAChF,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;SAC7E,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACjG,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,IAAI,QAAQ,CAAC;QAC5E,IAAI,aAAa,KAAK,QAAQ,IAAI,aAAa,KAAK,eAAe,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,aAAa;YACb,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;SACxC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAoB;IAClD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAC;IAE9B,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS;QAC7B,CAAC,CAAC,cAAc,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;QACnF,CAAC,CAAC,MAAM,aAAa,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC/F,MAAM,aAAa,GAAG,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,MAAM,mBAAmB,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;IAC1D,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,iBAAiB,aAAa,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACrH,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,aAAa,EAAE,CAAC;QACxC,MAAM,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,uBAAuB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACjD,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5G,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACxG,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,gBAAgB,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChH,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEtD,MAAM,MAAM,GAAG;QACb,KAAK,EAAE;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,UAAU;YAC/C,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB;QACD,MAAM,EAAE;YACN,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,cAAc,EAAE,MAAM,CAAC,WAAW;YAClC,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,cAAc,EAAE,MAAM,CAAC,WAAW;YAClC,eAAe,EAAE,MAAM,CAAC,cAAc;YACtC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,GAAG;YAChB,IAAI,EAAE,kBAAkB;SACzB;QACD,MAAM,EAAE;YACN,SAAS,EAAE,KAAK,CAAC,QAAQ;YACzB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,eAAe,EAAE,KAAK,CAAC,aAAa;YACpC,aAAa,EAAE,KAAK,CAAC,WAAW;YAChC,cAAc,EAAE,aAAa;YAC7B,mBAAmB,EAAE,gBAAgB;SACtC;QACD,KAAK,EAAE,UAAU;QACjB,eAAe,EAAE;YACf,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC;SAC7D;QACD,OAAO,EAAE;YACP;gBACE,cAAc,EAAE,MAAM,CAAC,WAAW;gBAClC,QAAQ,EAAE,OAAO;gBACjB,SAAS,EAAE,KAAK,CAAC,QAAQ;gBACzB,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB;SACF;QACD,QAAQ,EAAE,OAAO;KAClB,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/src/generate.js
CHANGED
|
@@ -86,7 +86,9 @@ export async function generateTasks(input, config, client, cache) {
|
|
|
86
86
|
}
|
|
87
87
|
catch (error) {
|
|
88
88
|
lastError = error;
|
|
89
|
-
|
|
89
|
+
if (process.env.SKILLCHECK_DEBUG === '1') {
|
|
90
|
+
console.error(`[skillcheck] generator returned invalid JSON on attempt ${attempt}/3`);
|
|
91
|
+
}
|
|
90
92
|
}
|
|
91
93
|
}
|
|
92
94
|
throw lastError instanceof Error ? lastError : new Error('Generator did not return valid tasks after retries');
|
package/dist/src/generate.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../packages/cli/src/generate.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,WAAW,GAAG,UAAU,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;IACnD,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7C,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,KAAa;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,mBAAmB,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,iCAAiC,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM;YACN,aAAa,EAAE,QAAQ;YACvB,SAAS;SACV,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAA0B,EAC1B,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACvC,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,WAAW,EACX,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE,OAAO,EAAE,EAClF,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;YACd,KAAK,EAAE,MAAM,CAAC,cAAc;YAC5B,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;YACf,cAAc,EAAE,aAAa;YAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;YACvC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,qIAAqI;iBAC/I;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,qBAAqB,KAAK,CAAC,MAAM,gBAAgB,cAAc,kPAAkP;iBAC3T;aACF;SACF,CAAC,CACL,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,aAAa,CAAC,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;YACtF,OAAO,aAAa,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtH,GAAG,IAAI;gBACP,EAAE,EAAE,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;aAC7C,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../packages/cli/src/generate.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,WAAW,GAAG,UAAU,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;IACnD,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7C,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,KAAa;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,mBAAmB,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,iCAAiC,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM;YACN,aAAa,EAAE,QAAQ;YACvB,SAAS;SACV,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAA0B,EAC1B,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACvC,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,WAAW,EACX,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE,OAAO,EAAE,EAClF,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;YACd,KAAK,EAAE,MAAM,CAAC,cAAc;YAC5B,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;YACf,cAAc,EAAE,aAAa;YAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;YACvC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,qIAAqI;iBAC/I;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,qBAAqB,KAAK,CAAC,MAAM,gBAAgB,cAAc,kPAAkP;iBAC3T;aACF;SACF,CAAC,CACL,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,aAAa,CAAC,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;YACtF,OAAO,aAAa,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtH,GAAG,IAAI;gBACP,EAAE,EAAE,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;aAC7C,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,EAAE,CAAC;gBACzC,OAAO,CAAC,KAAK,CAAC,2DAA2D,OAAO,IAAI,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACjH,CAAC"}
|
package/dist/src/grade.js
CHANGED
|
@@ -41,6 +41,7 @@ export async function gradeOutputs(tasks, outputs, config, client, cache) {
|
|
|
41
41
|
const taskById = new Map(tasks.map((task) => [task.id, task]));
|
|
42
42
|
const shuffled = seededShuffle(outputs, hashJson(outputs.map((output) => output.transcriptHash)));
|
|
43
43
|
const graded = new Map();
|
|
44
|
+
const debug = process.env.SKILLCHECK_DEBUG === '1';
|
|
44
45
|
for (const output of shuffled) {
|
|
45
46
|
const task = taskById.get(output.taskId);
|
|
46
47
|
if (!task) {
|
|
@@ -54,7 +55,9 @@ export async function gradeOutputs(tasks, outputs, config, client, cache) {
|
|
|
54
55
|
});
|
|
55
56
|
continue;
|
|
56
57
|
}
|
|
57
|
-
|
|
58
|
+
if (debug) {
|
|
59
|
+
console.error(`[skillcheck] grade ${output.taskId} trial ${output.trial}`);
|
|
60
|
+
}
|
|
58
61
|
let grade;
|
|
59
62
|
let lastError;
|
|
60
63
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
@@ -88,7 +91,9 @@ export async function gradeOutputs(tasks, outputs, config, client, cache) {
|
|
|
88
91
|
}
|
|
89
92
|
catch (error) {
|
|
90
93
|
lastError = error;
|
|
91
|
-
|
|
94
|
+
if (debug) {
|
|
95
|
+
console.error(`[skillcheck] grader returned invalid JSON on attempt ${attempt}/3`);
|
|
96
|
+
}
|
|
92
97
|
}
|
|
93
98
|
}
|
|
94
99
|
if (!grade) {
|
package/dist/src/grade.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IAEnD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;gBAChC,GAAG,MAAM;gBACT,GAAG,KAAK;aACT,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,KAA+B,CAAC;QACpC,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,QAAQ,EACR;gBACE,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,CAAC;gBAChB,cAAc,EAAE,aAAa;gBAC7B,OAAO;aACR,EACD,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,GAAG;gBACd,cAAc,EAAE,aAAa;gBAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACvC,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EACL,qMAAqM;qBACxM;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,uBAAuB,IAAI,CAAC,SAAS,yBAAyB,MAAM,CAAC,MAAM,0MAA0M;qBAC/R;iBACF;aACF,CAAC,CACL,CAAC;YAEF,IAAI,CAAC;gBACH,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACrC,MAAM;YACR,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,KAAK,CAAC,wDAAwD,OAAO,IAAI,CAAC,CAAC;gBACrF,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAChH,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;YAChC,GAAG,MAAM;YACT,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,GAAG;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/src/run.js
CHANGED
|
@@ -34,11 +34,16 @@ async function runOne(skill, task, trial, arm, config, client, cache) {
|
|
|
34
34
|
}
|
|
35
35
|
export async function runTrials(skill, tasks, trials, config, client, cache) {
|
|
36
36
|
const outputs = [];
|
|
37
|
+
const debug = process.env.SKILLCHECK_DEBUG === '1';
|
|
37
38
|
for (const task of tasks) {
|
|
38
39
|
for (let trial = 1; trial <= trials; trial += 1) {
|
|
39
|
-
|
|
40
|
+
if (debug) {
|
|
41
|
+
console.error(`[skillcheck] run ${task.id} trial ${trial}/${trials} with_skill`);
|
|
42
|
+
}
|
|
40
43
|
outputs.push(await runOne(skill, task, trial, 'with_skill', config, client, cache));
|
|
41
|
-
|
|
44
|
+
if (debug) {
|
|
45
|
+
console.error(`[skillcheck] run ${task.id} trial ${trial}/${trials} no_skill`);
|
|
46
|
+
}
|
|
42
47
|
outputs.push(await runOne(skill, task, trial, 'no_skill', config, client, cache));
|
|
43
48
|
}
|
|
44
49
|
}
|
package/dist/src/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAC/H,MAAM,CAAC,QAAQ,CAAC;QACd,KAAK,EAAE,MAAM,CAAC,WAAW;QACzB,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,IAAI;KAChB,CAAC,CACH,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAEjH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,IAAI,CAAC;QACvD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;QAC7C,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAsB,EACtB,KAAsB,EACtB,MAAc,EACd,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAC/H,MAAM,CAAC,QAAQ,CAAC;QACd,KAAK,EAAE,MAAM,CAAC,WAAW;QACzB,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,IAAI;KAChB,CAAC,CACH,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAEjH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,IAAI,CAAC;QACvD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;QAC7C,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAsB,EACtB,KAAsB,EACtB,MAAc,EACd,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,aAAa,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,WAAW,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/src/ui.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare function printBanner(): void;
|
|
2
|
+
export declare function printHelpUi(): void;
|
|
3
|
+
export declare function supportedSkillFilesText(): string;
|
|
4
|
+
export declare function validateSkillInput(inputPath: string): Promise<string>;
|
|
5
|
+
export declare function selectSkillPath(startDir?: string): Promise<string>;
|
|
6
|
+
export declare function startProgress(): {
|
|
7
|
+
finish: () => void;
|
|
8
|
+
fail: () => void;
|
|
9
|
+
};
|
|
10
|
+
export declare function formatResultCard(result: unknown, outputPath?: string): string;
|
|
11
|
+
export declare function sanitizeCliError(error: unknown): string;
|
|
12
|
+
export declare function formatFatalError(error: unknown): string;
|
package/dist/src/ui.js
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import { readdir, stat } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import process from 'node:process';
|
|
4
|
+
import readline from 'node:readline';
|
|
5
|
+
const SUPPORTED_SKILL_FILES = ['SKILL.md', 'AGENTS.md', 'CLAUDE.md', '.cursorrules'];
|
|
6
|
+
const useColor = () => process.stdout.isTTY && process.env.NO_COLOR !== '1';
|
|
7
|
+
const color = (code, value) => (useColor() ? `\x1b[${code}m${value}\x1b[0m` : value);
|
|
8
|
+
const blue = (value) => color('38;5;33', value);
|
|
9
|
+
const deepBlue = (value) => color('1;38;5;18', value);
|
|
10
|
+
const white = (value) => color('1;37', value);
|
|
11
|
+
const dim = (value) => color('2', value);
|
|
12
|
+
const red = (value) => color('1;31', value);
|
|
13
|
+
const green = (value) => color('1;32', value);
|
|
14
|
+
const yellow = (value) => color('1;33', value);
|
|
15
|
+
export function printBanner() {
|
|
16
|
+
console.log(deepBlue(` ____ _ _ _ _ _ _
|
|
17
|
+
/ ___|| | _(_) | | ___| |__ ___ ___| | __
|
|
18
|
+
\\___ \\| |/ / | | |/ __| '_ \\ / _ \\/ __| |/ /
|
|
19
|
+
___) | <| | | | (__| | | | __/ (__| <
|
|
20
|
+
|____/|_|\\_\\_|_|_|\\___|_| |_|\\___|\\___|_|\\_\\`));
|
|
21
|
+
console.log(white(' Drop a skill file. Get a verdict.\n'));
|
|
22
|
+
}
|
|
23
|
+
export function printHelpUi() {
|
|
24
|
+
printBanner();
|
|
25
|
+
console.log(`${blue('Quick start')}`);
|
|
26
|
+
console.log(` ${white('skillcheck')} ${blue('check')} path/to/SKILL.md`);
|
|
27
|
+
console.log(` ${white('skillcheck')} path/to/skill-folder\n`);
|
|
28
|
+
console.log(`${blue('Supported inputs')}`);
|
|
29
|
+
console.log(` ${SUPPORTED_SKILL_FILES.join(', ')} or a folder containing one\n`);
|
|
30
|
+
console.log(`${blue('Commands')}`);
|
|
31
|
+
console.log(` skillcheck check <path> [--tasks N] [--trials K] [--output file.json] [--json]`);
|
|
32
|
+
console.log(` skillcheck eval <path> [--tasks N] [--trials K] [--output file.json]`);
|
|
33
|
+
console.log(` skillcheck verify <result.json> [--sample n]`);
|
|
34
|
+
console.log(` skillcheck corpus run --corpus corpus.json [--results dir]`);
|
|
35
|
+
console.log(` skillcheck rot [--results dir] [--output file.json]\n`);
|
|
36
|
+
}
|
|
37
|
+
export function supportedSkillFilesText() {
|
|
38
|
+
return SUPPORTED_SKILL_FILES.join(', ');
|
|
39
|
+
}
|
|
40
|
+
async function fileExists(filePath) {
|
|
41
|
+
try {
|
|
42
|
+
const stats = await stat(filePath);
|
|
43
|
+
return stats.isFile();
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
if (error.code === 'ENOENT') {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async function directoryHasSkillFile(dirPath) {
|
|
53
|
+
for (const fileName of SUPPORTED_SKILL_FILES) {
|
|
54
|
+
if (await fileExists(path.join(dirPath, fileName))) {
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
function isSupportedSkillFile(filePath) {
|
|
61
|
+
return SUPPORTED_SKILL_FILES.includes(path.basename(filePath));
|
|
62
|
+
}
|
|
63
|
+
export async function validateSkillInput(inputPath) {
|
|
64
|
+
try {
|
|
65
|
+
const resolved = path.resolve(inputPath);
|
|
66
|
+
const stats = await stat(resolved);
|
|
67
|
+
if (stats.isDirectory() && (await directoryHasSkillFile(resolved))) {
|
|
68
|
+
return inputPath;
|
|
69
|
+
}
|
|
70
|
+
if (stats.isFile() && isSupportedSkillFile(resolved)) {
|
|
71
|
+
return inputPath;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (error.code !== 'ENOENT') {
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
throw new Error(`This path does not exist. Please give me a path to ${supportedSkillFilesText()}, or a folder containing one.`);
|
|
79
|
+
}
|
|
80
|
+
throw new Error(`That is not a skill file. Please give me a path to ${supportedSkillFilesText()}, or a folder containing one.`);
|
|
81
|
+
}
|
|
82
|
+
async function listPickerEntries(currentDir) {
|
|
83
|
+
const entries = await readdir(currentDir, { withFileTypes: true });
|
|
84
|
+
const parent = path.dirname(currentDir);
|
|
85
|
+
const mapped = await Promise.all(entries
|
|
86
|
+
.filter((entry) => entry.name !== 'node_modules' && entry.name !== '.git')
|
|
87
|
+
.map(async (entry) => {
|
|
88
|
+
const fullPath = path.join(currentDir, entry.name);
|
|
89
|
+
if (entry.isDirectory()) {
|
|
90
|
+
const runnable = await directoryHasSkillFile(fullPath);
|
|
91
|
+
return {
|
|
92
|
+
label: `${entry.name}/`,
|
|
93
|
+
fullPath,
|
|
94
|
+
kind: 'directory',
|
|
95
|
+
runnable,
|
|
96
|
+
note: runnable ? 'skill folder' : 'folder'
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
const runnable = entry.isFile() && isSupportedSkillFile(fullPath);
|
|
100
|
+
return {
|
|
101
|
+
label: entry.name,
|
|
102
|
+
fullPath,
|
|
103
|
+
kind: 'file',
|
|
104
|
+
runnable,
|
|
105
|
+
note: runnable ? 'skill file' : 'not a skill'
|
|
106
|
+
};
|
|
107
|
+
}));
|
|
108
|
+
return [
|
|
109
|
+
{
|
|
110
|
+
label: '../',
|
|
111
|
+
fullPath: parent,
|
|
112
|
+
kind: 'parent',
|
|
113
|
+
runnable: false,
|
|
114
|
+
note: 'back'
|
|
115
|
+
},
|
|
116
|
+
...mapped.sort((a, b) => {
|
|
117
|
+
if (a.kind !== b.kind) {
|
|
118
|
+
return a.kind === 'directory' ? -1 : 1;
|
|
119
|
+
}
|
|
120
|
+
if (a.runnable !== b.runnable) {
|
|
121
|
+
return a.runnable ? -1 : 1;
|
|
122
|
+
}
|
|
123
|
+
return a.label.localeCompare(b.label);
|
|
124
|
+
})
|
|
125
|
+
];
|
|
126
|
+
}
|
|
127
|
+
function visibleWindow(items, selected, size) {
|
|
128
|
+
const offset = Math.max(0, Math.min(selected - Math.floor(size / 2), Math.max(0, items.length - size)));
|
|
129
|
+
return { items: items.slice(offset, offset + size), offset };
|
|
130
|
+
}
|
|
131
|
+
function renderPicker(currentDir, entries, selected, message) {
|
|
132
|
+
process.stdout.write('\x1b[2J\x1b[H');
|
|
133
|
+
printBanner();
|
|
134
|
+
console.log(`${blue('Select a skill file or folder')}`);
|
|
135
|
+
console.log(`${dim('Current folder:')} ${currentDir}`);
|
|
136
|
+
console.log(dim('Use arrows, Enter to open/select, q to quit.\n'));
|
|
137
|
+
if (message) {
|
|
138
|
+
console.log(`${red(message)}\n`);
|
|
139
|
+
}
|
|
140
|
+
const window = visibleWindow(entries, selected, 12);
|
|
141
|
+
for (const [index, entry] of window.items.entries()) {
|
|
142
|
+
const realIndex = window.offset + index;
|
|
143
|
+
const pointer = realIndex === selected ? blue('>') : ' ';
|
|
144
|
+
const label = realIndex === selected ? white(entry.label) : entry.runnable ? blue(entry.label) : entry.label;
|
|
145
|
+
const note = entry.runnable ? blue(entry.note) : dim(entry.note);
|
|
146
|
+
console.log(`${pointer} ${label.padEnd(34)} ${note}`);
|
|
147
|
+
}
|
|
148
|
+
console.log('');
|
|
149
|
+
}
|
|
150
|
+
function readKey() {
|
|
151
|
+
return new Promise((resolve) => {
|
|
152
|
+
process.stdin.once('keypress', (input, key) => resolve({ input, key }));
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
export async function selectSkillPath(startDir = process.cwd()) {
|
|
156
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
|
157
|
+
throw new Error(`Please give me a path to ${supportedSkillFilesText()}, or a folder containing one.`);
|
|
158
|
+
}
|
|
159
|
+
readline.emitKeypressEvents(process.stdin);
|
|
160
|
+
const wasRaw = process.stdin.isRaw === true;
|
|
161
|
+
process.stdin.setRawMode(true);
|
|
162
|
+
let currentDir = path.resolve(startDir);
|
|
163
|
+
let selected = 0;
|
|
164
|
+
let message;
|
|
165
|
+
try {
|
|
166
|
+
for (;;) {
|
|
167
|
+
const entries = await listPickerEntries(currentDir);
|
|
168
|
+
selected = Math.max(0, Math.min(selected, entries.length - 1));
|
|
169
|
+
renderPicker(currentDir, entries, selected, message);
|
|
170
|
+
message = undefined;
|
|
171
|
+
const { input, key } = await readKey();
|
|
172
|
+
if ((key.ctrl && key.name === 'c') || input === 'q') {
|
|
173
|
+
throw new Error('Selection cancelled.');
|
|
174
|
+
}
|
|
175
|
+
if (key.name === 'up') {
|
|
176
|
+
selected = selected <= 0 ? entries.length - 1 : selected - 1;
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (key.name === 'down') {
|
|
180
|
+
selected = selected >= entries.length - 1 ? 0 : selected + 1;
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
if (key.name !== 'return') {
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
const entry = entries[selected];
|
|
187
|
+
if (!entry) {
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
if (entry.kind === 'parent') {
|
|
191
|
+
currentDir = entry.fullPath;
|
|
192
|
+
selected = 0;
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
if (entry.kind === 'directory' && !entry.runnable) {
|
|
196
|
+
currentDir = entry.fullPath;
|
|
197
|
+
selected = 0;
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
if (entry.runnable) {
|
|
201
|
+
process.stdout.write('\x1b[2J\x1b[H');
|
|
202
|
+
return entry.fullPath;
|
|
203
|
+
}
|
|
204
|
+
message = `That is not a skill file. Select ${supportedSkillFilesText()}, or a folder containing one.`;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
process.stdin.setRawMode(wasRaw);
|
|
209
|
+
process.stdin.pause();
|
|
210
|
+
process.stdout.write('\x1b[?25h');
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
function progressLine(percent) {
|
|
214
|
+
const width = 32;
|
|
215
|
+
const filled = Math.round((width * percent) / 100);
|
|
216
|
+
const bar = `${'#'.repeat(filled)}${'-'.repeat(width - filled)}`;
|
|
217
|
+
return `${blue('Analyzing skill')} ${white(`[${bar}]`)} ${String(percent).padStart(3, ' ')}%`;
|
|
218
|
+
}
|
|
219
|
+
export function startProgress() {
|
|
220
|
+
if (!process.stdout.isTTY) {
|
|
221
|
+
return { finish: () => undefined, fail: () => undefined };
|
|
222
|
+
}
|
|
223
|
+
let percent = 0;
|
|
224
|
+
const render = () => {
|
|
225
|
+
process.stdout.write(`\r\x1b[2K${progressLine(percent)}`);
|
|
226
|
+
};
|
|
227
|
+
const timer = setInterval(() => {
|
|
228
|
+
percent = Math.min(94, percent + 2);
|
|
229
|
+
render();
|
|
230
|
+
}, 400);
|
|
231
|
+
process.stdout.write('\x1b[?25l');
|
|
232
|
+
render();
|
|
233
|
+
return {
|
|
234
|
+
finish: () => {
|
|
235
|
+
clearInterval(timer);
|
|
236
|
+
percent = 100;
|
|
237
|
+
render();
|
|
238
|
+
process.stdout.write('\n\x1b[?25h');
|
|
239
|
+
},
|
|
240
|
+
fail: () => {
|
|
241
|
+
clearInterval(timer);
|
|
242
|
+
process.stdout.write('\r\x1b[2K\x1b[?25h');
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
function asRecord(value) {
|
|
247
|
+
return typeof value === 'object' && value !== null ? value : {};
|
|
248
|
+
}
|
|
249
|
+
function formatPercent(value) {
|
|
250
|
+
return typeof value === 'number' ? `${(value * 100).toFixed(1)}%` : 'n/a';
|
|
251
|
+
}
|
|
252
|
+
function formatNumber(value) {
|
|
253
|
+
return typeof value === 'number' ? value.toFixed(1) : 'n/a';
|
|
254
|
+
}
|
|
255
|
+
function verdictColor(verdict) {
|
|
256
|
+
if (verdict === 'helps') {
|
|
257
|
+
return green(verdict.toUpperCase());
|
|
258
|
+
}
|
|
259
|
+
if (verdict === 'harms') {
|
|
260
|
+
return red(verdict.toUpperCase());
|
|
261
|
+
}
|
|
262
|
+
return yellow(verdict.toUpperCase());
|
|
263
|
+
}
|
|
264
|
+
function boxLine(label, value) {
|
|
265
|
+
return `${blue('|')} ${white(label.padEnd(15))} ${value}`;
|
|
266
|
+
}
|
|
267
|
+
export function formatResultCard(result, outputPath) {
|
|
268
|
+
const root = asRecord(result);
|
|
269
|
+
const skill = asRecord(root.skill);
|
|
270
|
+
const config = asRecord(root.config);
|
|
271
|
+
const score = asRecord(root.result);
|
|
272
|
+
const ci = Array.isArray(score.ci_pp) ? score.ci_pp : [];
|
|
273
|
+
const ciText = typeof ci[0] === 'number' && typeof ci[1] === 'number'
|
|
274
|
+
? `[${ci[0].toFixed(1)}, ${ci[1].toFixed(1)}] pp`
|
|
275
|
+
: 'n/a';
|
|
276
|
+
const verdict = String(score.verdict ?? 'unknown');
|
|
277
|
+
const lines = [
|
|
278
|
+
blue('+-----------------------------------------------+'),
|
|
279
|
+
`${blue('|')} ${white('SKILLCHECK RESULT')}`,
|
|
280
|
+
blue('+-----------------------------------------------+'),
|
|
281
|
+
boxLine('Skill', String(skill.name ?? 'unknown')),
|
|
282
|
+
boxLine('Verdict', verdictColor(verdict)),
|
|
283
|
+
boxLine('Effect', `${formatNumber(score.effect_pp)} pp`),
|
|
284
|
+
boxLine('Confidence', ciText),
|
|
285
|
+
boxLine('With skill', formatPercent(score.with_skill_pass)),
|
|
286
|
+
boxLine('Without skill', formatPercent(score.no_skill_pass)),
|
|
287
|
+
boxLine('Token cost', `${String(score.token_overhead ?? 'n/a')} tokens`),
|
|
288
|
+
boxLine('Run size', `${String(config.tasks ?? 'n/a')} tasks x ${String(config.trials ?? 'n/a')} trials`)
|
|
289
|
+
];
|
|
290
|
+
if (outputPath) {
|
|
291
|
+
lines.push(boxLine('Saved JSON', outputPath));
|
|
292
|
+
}
|
|
293
|
+
lines.push(blue('+-----------------------------------------------+'));
|
|
294
|
+
return lines.join('\n');
|
|
295
|
+
}
|
|
296
|
+
export function sanitizeCliError(error) {
|
|
297
|
+
const raw = error instanceof Error ? error.message : String(error);
|
|
298
|
+
if (/api[_ -]?key|credential|unauthorized|authentication|401/i.test(raw)) {
|
|
299
|
+
return 'Skillcheck Cloud is not connected for this workspace. Please try again later or contact the workspace owner.';
|
|
300
|
+
}
|
|
301
|
+
return raw
|
|
302
|
+
.replace(/NVIDIA NIM/gi, 'Skillcheck Cloud')
|
|
303
|
+
.replace(/NVIDIA_API_KEY/gi, 'Skillcheck Cloud')
|
|
304
|
+
.replace(/NVIDIA/gi, 'Skillcheck Cloud')
|
|
305
|
+
.replace(/API[_ -]?KEY/gi, 'credential');
|
|
306
|
+
}
|
|
307
|
+
export function formatFatalError(error) {
|
|
308
|
+
return `${red('Skillcheck stopped')}\n${sanitizeCliError(error)}`;
|
|
309
|
+
}
|
|
310
|
+
//# sourceMappingURL=ui.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ui.js","sourceRoot":"","sources":["../../packages/cli/src/ui.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,QAAQ,MAAM,eAAe,CAAC;AAErC,MAAM,qBAAqB,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;AACrF,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC;AAC5E,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,KAAa,EAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAC7G,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAChE,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;AACtE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC9D,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;AACzD,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC5D,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC9D,MAAM,MAAM,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAgB/D,MAAM,UAAU,WAAW;IACzB,OAAO,CAAC,GAAG,CACT,QAAQ,CAAC;;;;qDAIwC,CAAC,CACnD,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,WAAW,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,YAAY,CAAC,yBAAyB,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,KAAK,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,kFAAkF,CAAC,CAAC;IAChG,OAAO,CAAC,GAAG,CAAC,wEAAwE,CAAC,CAAC;IACtF,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,uBAAuB;IACrC,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,OAAe;IAClD,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC7C,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB;IACxD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACnE,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrD,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,sDAAsD,uBAAuB,EAAE,+BAA+B,CAAC,CAAC;IAClI,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,sDAAsD,uBAAuB,EAAE,+BAA+B,CAAC,CAAC;AAClI,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,UAAkB;IACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,OAAO;SACJ,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;SACzE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAwB,EAAE;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YACvD,OAAO;gBACL,KAAK,EAAE,GAAG,KAAK,CAAC,IAAI,GAAG;gBACvB,QAAQ;gBACR,IAAI,EAAE,WAAW;gBACjB,QAAQ;gBACR,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ;aAC3C,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAClE,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,QAAQ;YACR,IAAI,EAAE,MAAM;YACZ,QAAQ;YACR,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa;SAC9C,CAAC;IACJ,CAAC,CAAC,CACL,CAAC;IAEF,OAAO;QACL;YACE,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK;YACf,IAAI,EAAE,MAAM;SACb;QACD,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACtB,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBACtB,OAAO,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC9B,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;YACD,OAAO,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxC,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB,EAAE,IAAY;IAClE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACxG,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;AAC/D,CAAC;AAED,SAAS,YAAY,CAAC,UAAkB,EAAE,OAAsB,EAAE,QAAgB,EAAE,OAAgB;IAClG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACtC,WAAW,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,+BAA+B,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,iBAAiB,CAAC,IAAI,UAAU,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC,CAAC;IAEnE,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IACpD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC;QACxC,MAAM,OAAO,GAAG,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACzD,MAAM,KAAK,GAAG,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7G,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjE,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,OAAO;IACd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,KAAa,EAAE,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,EAAE;IAC5D,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,4BAA4B,uBAAuB,EAAE,+BAA+B,CAAC,CAAC;IACxG,CAAC;IAED,QAAQ,CAAC,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC;IAC5C,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAE/B,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAA2B,CAAC;IAEhC,IAAI,CAAC;QACH,SAAS,CAAC;YACR,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,UAAU,CAAC,CAAC;YACpD,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/D,YAAY,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,OAAO,GAAG,SAAS,CAAC;YAEpB,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,OAAO,EAAE,CAAC;YACvC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBACtB,QAAQ,GAAG,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAC7D,SAAS;YACX,CAAC;YACD,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxB,QAAQ,GAAG,QAAQ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAC7D,SAAS;YACX,CAAC;YACD,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC5B,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,GAAG,CAAC,CAAC;gBACb,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClD,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,GAAG,CAAC,CAAC;gBACb,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACtC,OAAO,KAAK,CAAC,QAAQ,CAAC;YACxB,CAAC;YACD,OAAO,GAAG,oCAAoC,uBAAuB,EAAE,+BAA+B,CAAC;QACzG,CAAC;IACH,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC;IACjE,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC;IAC5D,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5D,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QACpC,MAAM,EAAE,CAAC;IACX,CAAC,EAAE,GAAG,CAAC,CAAC;IAER,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAClC,MAAM,EAAE,CAAC;IAET,OAAO;QACL,MAAM,EAAE,GAAG,EAAE;YACX,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,GAAG,GAAG,CAAC;YACd,MAAM,EAAE,CAAC;YACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,EAAE,GAAG,EAAE;YACT,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC7C,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/F,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AAC5E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9D,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,OAAO,CAAC,KAAa,EAAE,KAAa;IAC3C,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAe,EAAE,UAAmB;IACnE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,MAAM,GACV,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ;QACpD,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QACjD,CAAC,CAAC,KAAK,CAAC;IACZ,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,SAAS,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG;QACZ,IAAI,CAAC,mDAAmD,CAAC;QACzD,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,EAAE;QAC5C,IAAI,CAAC,mDAAmD,CAAC;QACzD,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;QACjD,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,QAAQ,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC;QACxD,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC;QAC7B,OAAO,CAAC,YAAY,EAAE,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAC3D,OAAO,CAAC,eAAe,EAAE,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5D,OAAO,CAAC,YAAY,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,SAAS,CAAC;QACxE,OAAO,CAAC,UAAU,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC;KACzG,CAAC;IACF,IAAI,UAAU,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC,CAAC;IACtE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnE,IAAI,0DAA0D,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACzE,OAAO,8GAA8G,CAAC;IACxH,CAAC;IACD,OAAO,GAAG;SACP,OAAO,CAAC,cAAc,EAAE,kBAAkB,CAAC;SAC3C,OAAO,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;SAC/C,OAAO,CAAC,UAAU,EAAE,kBAAkB,CAAC;SACvC,OAAO,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,OAAO,GAAG,GAAG,CAAC,oBAAoB,CAAC,KAAK,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;AACpE,CAAC"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# Skillcheck Cloud setup
|
|
2
|
+
|
|
3
|
+
Use this when you want users to install the CLI and run checks without configuring model-provider secrets locally.
|
|
4
|
+
|
|
5
|
+
## Architecture
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
skillcheck CLI
|
|
9
|
+
-> Skillcheck Cloud API
|
|
10
|
+
-> model provider
|
|
11
|
+
|
|
12
|
+
Dashboard
|
|
13
|
+
-> user signs up
|
|
14
|
+
-> user creates a Skillcheck token
|
|
15
|
+
-> token is stored hashed in your database
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The CLI only needs:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
export SKILLCHECK_API_URL=https://api.yourdomain.com/v1
|
|
22
|
+
export SKILLCHECK_TOKEN=sk_live_...
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
If you want public free trials, the proxy can allow anonymous requests with strict rate limits and no token.
|
|
26
|
+
|
|
27
|
+
## API contract
|
|
28
|
+
|
|
29
|
+
The CLI expects an OpenAI-compatible endpoint:
|
|
30
|
+
|
|
31
|
+
```http
|
|
32
|
+
POST /v1/chat/completions
|
|
33
|
+
Authorization: Bearer <skillcheck-token>
|
|
34
|
+
Content-Type: application/json
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Response should match OpenAI chat completions enough for the `openai` Node SDK:
|
|
38
|
+
|
|
39
|
+
```json
|
|
40
|
+
{
|
|
41
|
+
"id": "chatcmpl_...",
|
|
42
|
+
"object": "chat.completion",
|
|
43
|
+
"created": 1780000000,
|
|
44
|
+
"model": "your-model",
|
|
45
|
+
"choices": [
|
|
46
|
+
{
|
|
47
|
+
"index": 0,
|
|
48
|
+
"message": {
|
|
49
|
+
"role": "assistant",
|
|
50
|
+
"content": "..."
|
|
51
|
+
},
|
|
52
|
+
"finish_reason": "stop"
|
|
53
|
+
}
|
|
54
|
+
],
|
|
55
|
+
"usage": {
|
|
56
|
+
"prompt_tokens": 1,
|
|
57
|
+
"completion_tokens": 1,
|
|
58
|
+
"total_tokens": 2
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Minimal proxy
|
|
64
|
+
|
|
65
|
+
The repo includes a tiny Node proxy in `examples/nvidia-proxy/`. It is useful for testing the `SKILLCHECK_API_URL` flow before building the dashboard.
|
|
66
|
+
|
|
67
|
+
Run it on a server:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
export NVIDIA_API_KEY=...
|
|
71
|
+
export SKILLCHECK_PROXY_TOKEN=dev-token
|
|
72
|
+
node examples/nvidia-proxy/server.mjs
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Point the CLI at it:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
export SKILLCHECK_API_URL=https://your-proxy.example.com/v1
|
|
79
|
+
export SKILLCHECK_TOKEN=dev-token
|
|
80
|
+
skillcheck check path/to/SKILL.md
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Dashboard requirements
|
|
84
|
+
|
|
85
|
+
- `users`: id, email, password/session provider, created_at.
|
|
86
|
+
- `tokens`: id, user_id, token_hash, prefix, created_at, last_used_at, revoked_at.
|
|
87
|
+
- `usage_events`: user_id, token_id, request_id, model, prompt_tokens, completion_tokens, created_at.
|
|
88
|
+
- Rate limit by token and IP.
|
|
89
|
+
- Store model-provider secrets only on the server.
|
|
90
|
+
- Never expose upstream provider secrets to the browser or CLI.
|
|
91
|
+
|
|
92
|
+
## First production path
|
|
93
|
+
|
|
94
|
+
1. Deploy the proxy API at `https://api.yourdomain.com/v1`.
|
|
95
|
+
2. Add dashboard auth with GitHub or email login.
|
|
96
|
+
3. Add “Create token” in the dashboard and show the token once.
|
|
97
|
+
4. Hash tokens before storing them.
|
|
98
|
+
5. Verify `Authorization: Bearer <token>` in the proxy.
|
|
99
|
+
6. Add rate limits and usage logging.
|
|
100
|
+
7. Ship the CLI with docs telling users to set `SKILLCHECK_API_URL` and `SKILLCHECK_TOKEN`.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# skillcheck NVIDIA proxy
|
|
2
|
+
|
|
3
|
+
This is the safe way to let CLI users run `skillcheck` without seeing your NVIDIA key.
|
|
4
|
+
|
|
5
|
+
Run the proxy on a server:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
export NVIDIA_API_KEY=...
|
|
9
|
+
node examples/nvidia-proxy/server.mjs
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Point the CLI at the proxy:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
export SKILLCHECK_API_URL=https://your-proxy.example.com/v1
|
|
16
|
+
skillcheck check path/to/SKILL.md
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Do not publish `NVIDIA_API_KEY` inside the npm package. If the proxy is public, put it behind rate limiting, quotas, or authentication before sharing it broadly.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createServer } from 'node:http';
|
|
3
|
+
|
|
4
|
+
const port = Number(process.env.PORT || 8787);
|
|
5
|
+
const nvidiaApiKey = process.env.NVIDIA_API_KEY?.trim();
|
|
6
|
+
const nvidiaBaseUrl = process.env.NVIDIA_BASE_URL?.trim() || 'https://integrate.api.nvidia.com/v1';
|
|
7
|
+
const proxyToken = process.env.SKILLCHECK_PROXY_TOKEN?.trim();
|
|
8
|
+
const maxBodyBytes = Number(process.env.MAX_BODY_BYTES || 5_000_000);
|
|
9
|
+
|
|
10
|
+
if (!nvidiaApiKey) {
|
|
11
|
+
throw new Error('Missing NVIDIA_API_KEY. Set it on the server, never inside the npm CLI.');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function writeJson(res, status, body) {
|
|
15
|
+
res.writeHead(status, {
|
|
16
|
+
'access-control-allow-headers': 'authorization, content-type',
|
|
17
|
+
'access-control-allow-methods': 'POST, OPTIONS',
|
|
18
|
+
'access-control-allow-origin': '*',
|
|
19
|
+
'content-type': 'application/json'
|
|
20
|
+
});
|
|
21
|
+
res.end(`${JSON.stringify(body)}\n`);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function readBody(req) {
|
|
25
|
+
return new Promise((resolve, reject) => {
|
|
26
|
+
const chunks = [];
|
|
27
|
+
let size = 0;
|
|
28
|
+
req.on('data', (chunk) => {
|
|
29
|
+
size += chunk.length;
|
|
30
|
+
if (size > maxBodyBytes) {
|
|
31
|
+
reject(new Error('Request body is too large'));
|
|
32
|
+
req.destroy();
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
chunks.push(chunk);
|
|
36
|
+
});
|
|
37
|
+
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
|
38
|
+
req.on('error', reject);
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function isAuthorized(req) {
|
|
43
|
+
if (!proxyToken) {
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
return req.headers.authorization === `Bearer ${proxyToken}`;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const server = createServer(async (req, res) => {
|
|
50
|
+
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
|
51
|
+
|
|
52
|
+
if (req.method === 'OPTIONS') {
|
|
53
|
+
writeJson(res, 204, {});
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (req.method === 'GET' && url.pathname === '/health') {
|
|
58
|
+
writeJson(res, 200, { ok: true });
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (req.method !== 'POST' || url.pathname !== '/v1/chat/completions') {
|
|
63
|
+
writeJson(res, 404, { error: 'Not found' });
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (!isAuthorized(req)) {
|
|
68
|
+
writeJson(res, 401, { error: 'Unauthorized' });
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
try {
|
|
73
|
+
const body = await readBody(req);
|
|
74
|
+
const upstream = await fetch(`${nvidiaBaseUrl}/chat/completions`, {
|
|
75
|
+
method: 'POST',
|
|
76
|
+
headers: {
|
|
77
|
+
authorization: `Bearer ${nvidiaApiKey}`,
|
|
78
|
+
'content-type': 'application/json'
|
|
79
|
+
},
|
|
80
|
+
body
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
res.writeHead(upstream.status, {
|
|
84
|
+
'access-control-allow-origin': '*',
|
|
85
|
+
'content-type': upstream.headers.get('content-type') || 'application/json'
|
|
86
|
+
});
|
|
87
|
+
res.end(await upstream.text());
|
|
88
|
+
} catch (error) {
|
|
89
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
90
|
+
writeJson(res, 500, { error: message });
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
server.listen(port, () => {
|
|
95
|
+
console.log(`skillcheck NVIDIA proxy listening on http://localhost:${port}`);
|
|
96
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sx4im/skillcheck",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Measure whether agent skills improve task performance.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -28,11 +28,13 @@
|
|
|
28
28
|
},
|
|
29
29
|
"files": [
|
|
30
30
|
"dist",
|
|
31
|
+
"docs",
|
|
32
|
+
"examples",
|
|
31
33
|
"README.md",
|
|
32
34
|
"METHODOLOGY.md"
|
|
33
35
|
],
|
|
34
36
|
"scripts": {
|
|
35
|
-
"build": "rm -rf dist && tsc -p tsconfig.json",
|
|
37
|
+
"build": "rm -rf dist && tsc -p tsconfig.json && chmod +x dist/bin/skillcheck.js",
|
|
36
38
|
"m0": "tsx packages/cli/bin/skillcheck.ts m0",
|
|
37
39
|
"site:build": "next build packages/site",
|
|
38
40
|
"test": "vitest run",
|