@sx4im/skillcheck 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -49
- package/dist/bin/skillcheck.js +2 -2
- package/dist/bin/skillcheck.js.map +1 -1
- package/dist/src/adapters/nvidia-nim.js +5 -2
- package/dist/src/adapters/nvidia-nim.js.map +1 -1
- package/dist/src/cli.d.ts +1 -3
- package/dist/src/cli.js +30 -67
- package/dist/src/cli.js.map +1 -1
- package/dist/src/env.js +29 -13
- package/dist/src/env.js.map +1 -1
- package/dist/src/eval.d.ts +1 -0
- package/dist/src/eval.js +7 -4
- package/dist/src/eval.js.map +1 -1
- package/dist/src/generate.js +3 -1
- package/dist/src/generate.js.map +1 -1
- package/dist/src/grade.js +7 -2
- package/dist/src/grade.js.map +1 -1
- package/dist/src/run.js +7 -2
- package/dist/src/run.js.map +1 -1
- package/dist/src/ui.d.ts +12 -0
- package/dist/src/ui.js +310 -0
- package/dist/src/ui.js.map +1 -0
- package/docs/skillcheck-cloud.md +100 -0
- package/examples/nvidia-proxy/README.md +19 -0
- package/examples/nvidia-proxy/server.mjs +96 -0
- package/package.json +4 -2
package/README.md
CHANGED
|
@@ -8,46 +8,42 @@ Measure whether an agent skill actually improves task performance.
|
|
|
8
8
|
|
|
9
9
|
- M0-M4 gates have passed and are recorded in `BUILD-LOG.md`.
|
|
10
10
|
- M5 completed a 20-skill launch corpus in `results/launch/20260605T110514Z-qwen-next`.
|
|
11
|
-
-
|
|
12
|
-
- Do not publish to npm until `RELEASE-CHECKLIST.md` is complete.
|
|
11
|
+
- Published on npm as `@sx4im/skillcheck`.
|
|
13
12
|
|
|
14
13
|
## One-Line Install
|
|
15
14
|
|
|
16
|
-
After npm publication, install the CLI globally with one command:
|
|
17
|
-
|
|
18
15
|
```bash
|
|
19
16
|
npm install -g @sx4im/skillcheck
|
|
20
17
|
```
|
|
21
18
|
|
|
22
|
-
|
|
19
|
+
Requires Node.js 20 or newer.
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
23
22
|
|
|
24
23
|
```bash
|
|
25
|
-
skillcheck
|
|
24
|
+
skillcheck
|
|
26
25
|
```
|
|
27
26
|
|
|
28
|
-
|
|
27
|
+
This opens the interactive picker. Select `SKILL.md`, `AGENTS.md`, `CLAUDE.md`, `.cursorrules`, or a folder containing one.
|
|
28
|
+
|
|
29
|
+
You can also pass a path directly:
|
|
29
30
|
|
|
30
31
|
```bash
|
|
31
32
|
skillcheck check path/to/SKILL.md
|
|
33
|
+
skillcheck path/to/skill-folder
|
|
32
34
|
```
|
|
33
35
|
|
|
34
|
-
|
|
36
|
+
The quick run shows a blue/white result card in the terminal. It does not write a results folder unless you explicitly pass `--output`.
|
|
35
37
|
|
|
36
|
-
|
|
38
|
+
## Cloud Setup
|
|
37
39
|
|
|
38
|
-
|
|
39
|
-
npm install -g git+ssh://git@github.com/sx4im/skillcheck.git
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
Remove the pre-publish GitHub install with:
|
|
40
|
+
For shared/public usage, point the CLI at your Skillcheck Cloud endpoint:
|
|
43
41
|
|
|
44
42
|
```bash
|
|
45
|
-
|
|
43
|
+
export SKILLCHECK_API_URL=https://api.yourdomain.com/v1
|
|
46
44
|
```
|
|
47
45
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
Requires Node.js 20 or newer.
|
|
46
|
+
See `docs/skillcheck-cloud.md` for the proxy and dashboard plan.
|
|
51
47
|
|
|
52
48
|
## Local Development
|
|
53
49
|
|
|
@@ -57,37 +53,6 @@ npm run build
|
|
|
57
53
|
node dist/bin/skillcheck.js --help
|
|
58
54
|
```
|
|
59
55
|
|
|
60
|
-
You can also run the compiled CLI directly:
|
|
61
|
-
|
|
62
|
-
```bash
|
|
63
|
-
node dist/bin/skillcheck.js eval fixtures/m2/strong-skill/SKILL.md --task-suite fixtures/m2/deterministic-tasks.json --tasks 3 --trials 3 --output /tmp/skillcheck-result.json
|
|
64
|
-
```
|
|
65
|
-
|
|
66
|
-
## Environment
|
|
67
|
-
|
|
68
|
-
Copy `.env.example` to `.env` and set `NVIDIA_API_KEY`.
|
|
69
|
-
|
|
70
|
-
Required for live NVIDIA NIM calls:
|
|
71
|
-
|
|
72
|
-
```bash
|
|
73
|
-
NVIDIA_API_KEY=...
|
|
74
|
-
NVIDIA_BASE_URL=https://integrate.api.nvidia.com/v1
|
|
75
|
-
NVIDIA_GENERATOR_MODEL=qwen/qwen3-next-80b-a3b-instruct
|
|
76
|
-
NVIDIA_GRADER_MODEL=qwen/qwen3-next-80b-a3b-instruct
|
|
77
|
-
NVIDIA_RUNNER_MODEL=qwen/qwen3-next-80b-a3b-instruct
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
Recommended launch-run provider controls:
|
|
81
|
-
|
|
82
|
-
```bash
|
|
83
|
-
NVIDIA_TIMEOUT_MS=120000
|
|
84
|
-
NVIDIA_REQUEST_DELAY_MS=5000
|
|
85
|
-
NVIDIA_MAX_ATTEMPTS=8
|
|
86
|
-
NVIDIA_MAX_RETRY_DELAY_MS=60000
|
|
87
|
-
```
|
|
88
|
-
|
|
89
|
-
The final M5 run used `--concurrency 1` and a 5 second process-wide request delay. The completed run had 75 recovered NVIDIA connection retries and no `429` rate-limit lines.
|
|
90
|
-
|
|
91
56
|
## Commands
|
|
92
57
|
|
|
93
58
|
Friendly skill check:
|
package/dist/bin/skillcheck.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { main } from '../src/cli.js';
|
|
3
|
+
import { formatFatalError } from '../src/ui.js';
|
|
3
4
|
main(process.argv).catch((error) => {
|
|
4
|
-
|
|
5
|
-
console.error(message);
|
|
5
|
+
console.error(formatFatalError(error));
|
|
6
6
|
process.exitCode = 1;
|
|
7
7
|
});
|
|
8
8
|
//# sourceMappingURL=skillcheck.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skillcheck.js","sourceRoot":"","sources":["../../packages/cli/bin/skillcheck.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"skillcheck.js","sourceRoot":"","sources":["../../packages/cli/bin/skillcheck.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC1C,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
@@ -64,9 +64,12 @@ function getRetryAfterMs(error) {
|
|
|
64
64
|
return undefined;
|
|
65
65
|
}
|
|
66
66
|
function describeRetry(error, attempt, waitMs, maxAttempts) {
|
|
67
|
+
if (process.env.SKILLCHECK_DEBUG_RETRIES !== '1') {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
67
70
|
const status = getStatus(error);
|
|
68
71
|
const label = status === undefined ? 'connection' : `status ${status}`;
|
|
69
|
-
console.error(`[
|
|
72
|
+
console.error(`[skillcheck] retry ${attempt + 1}/${maxAttempts} after ${label}; waiting ${waitMs} ms`);
|
|
70
73
|
}
|
|
71
74
|
export class NvidiaNimClient {
|
|
72
75
|
static requestQueue = Promise.resolve();
|
|
@@ -132,7 +135,7 @@ export class NvidiaNimClient {
|
|
|
132
135
|
const content = extractMessageText(message);
|
|
133
136
|
if (content === undefined) {
|
|
134
137
|
const keys = message ? Object.keys(message).sort().join(',') : 'none';
|
|
135
|
-
throw new Error(`
|
|
138
|
+
throw new Error(`Skillcheck Cloud response did not include text content; message keys: ${keys}`);
|
|
136
139
|
}
|
|
137
140
|
return {
|
|
138
141
|
content,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nvidia-nim.js","sourceRoot":"","sources":["../../../packages/cli/src/adapters/nvidia-nim.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAuB5B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAC;gBAC/C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAgF,CAAC;IAChG,OAAO,CACL,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;QAClC,kBAAkB,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACrE,MAAM,MAAM,GAAI,KAA8B,CAAC,MAAM,CAAC;QACtD,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,MAAM,UAAU,GACd,OAAO,YAAY,OAAO;QACxB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC5B,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,aAAa,IAAI,OAAO;YAC3E,CAAC,CAAE,OAAuC,CAAC,aAAa,CAAC;YACzD,CAAC,CAAC,SAAS,CAAC;IAElB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,MAAc,EAAE,WAAmB;IACzF,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"nvidia-nim.js","sourceRoot":"","sources":["../../../packages/cli/src/adapters/nvidia-nim.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAuB5B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAC;gBAC/C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAgF,CAAC;IAChG,OAAO,CACL,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;QAClC,kBAAkB,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACrE,MAAM,MAAM,GAAI,KAA8B,CAAC,MAAM,CAAC;QACtD,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,MAAM,UAAU,GACd,OAAO,YAAY,OAAO;QACxB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC5B,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,aAAa,IAAI,OAAO;YAC3E,CAAC,CAAE,OAAuC,CAAC,aAAa,CAAC;YACzD,CAAC,CAAC,SAAS,CAAC;IAElB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,MAAc,EAAE,WAAmB;IACzF,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG,EAAE,CAAC;QACjD,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,GAAG,CAAC,IAAI,WAAW,UAAU,KAAK,aAAa,MAAM,KAAK,CAAC,CAAC;AACzG,CAAC;AAED,MAAM,OAAO,eAAe;IAClB,MAAM,CAAC,YAAY,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IACvD,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;IAEhB,MAAM,CAAS;IACf,cAAc,CAAS;IACvB,WAAW,CAAS;IACpB,eAAe,CAAS;IAEzC,YAAY,MAAoB;QAC9B,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,CAAC,SAAS;SAC1B,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAI,SAA2B;QAC/D,MAAM,eAAe,GAAG,eAAe,CAAC,YAAY,CAAC;QACrD,IAAI,qBAAqB,GAAe,GAAG,EAAE,CAAC,SAAS,CAAC;QACxD,eAAe,CAAC,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC3D,qBAAqB,GAAG,OAAO,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,CAAC;QACtB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,aAAa,CAAC;gBAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;gBAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,eAAe,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3C,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,qBAAqB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAA0B;QACvC,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,OAAO,CAAC,kBAAkB;oBAC1C,CAAC,CAAC;wBACE,oBAAoB,EAAE,OAAO,CAAC,kBAAkB;qBACjD;oBACH,CAAC,CAAC,SAAS,CAAC;gBACd,MAAM,WAAW,GAAG;oBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,UAAU,EAAE,OAAO,CAAC,SAAS;oBAC7B,eAAe,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS;oBACtF,oBAAoB,EAAE,OAAO,CAAC,kBAAkB;oBAChD,UAAU,EAAE,SAAS;oBACrB,MAAM,EAAE,KAAK;iBACd,CAAC;gBACF,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,CACjD,CAAmB,CAAC;gBAErB,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC7C,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;oBACtE,MAAM,IAAI,KAAK,CAAC,yEAAyE,IAAI,EAAE,CAAC,CAAC;gBACnG,CAAC;gBAED,OAAO;oBACL,OAAO;oBACP,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,KAAK,EAAE;wBACL,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa;wBAC3C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB;wBACnD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY;qBAC1C;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACzE,IAAI,CAAC,SAAS,IAAI,OAAO,KAAK,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;oBACnD,MAAM;gBACR,CAAC;gBAED,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;gBAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;gBAC/C,MAAM,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,WAAW,GAAG,CAAC,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC;gBACnG,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxD,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC9E,CAAC"}
|
package/dist/src/cli.d.ts
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import { type EvalOptions } from './eval.js';
|
|
2
|
-
export declare function defaultCheckOutputPath(inputPath: string, now?: Date): string;
|
|
3
2
|
interface CheckOptions {
|
|
4
3
|
evalOptions: EvalOptions;
|
|
5
4
|
json: boolean;
|
|
6
|
-
output
|
|
5
|
+
output?: string;
|
|
7
6
|
}
|
|
8
7
|
export declare function parseCheckOptions(argv: string[], inputIndex?: number): CheckOptions;
|
|
9
|
-
export declare function formatCheckSummary(result: unknown, outputPath: string): string;
|
|
10
8
|
export declare function main(argv: string[]): Promise<void>;
|
|
11
9
|
export {};
|
package/dist/src/cli.js
CHANGED
|
@@ -1,31 +1,13 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
|
-
import path from 'node:path';
|
|
3
2
|
import { NvidiaNimClient } from './adapters/nvidia-nim.js';
|
|
4
3
|
import { runCorpus } from './corpus.js';
|
|
5
4
|
import { evalSkill } from './eval.js';
|
|
6
5
|
import { runM0Gate } from './m0/run.js';
|
|
7
6
|
import { runRot } from './rot.js';
|
|
7
|
+
import { formatResultCard, printHelpUi, printBanner, selectSkillPath, startProgress, validateSkillInput } from './ui.js';
|
|
8
8
|
import { verifyResult } from './verify.js';
|
|
9
9
|
function printHelp() {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
Quick start:
|
|
13
|
-
skillcheck check <path-to-SKILL.md-or-folder>
|
|
14
|
-
skillcheck <path-to-SKILL.md-or-folder>
|
|
15
|
-
|
|
16
|
-
Usage:
|
|
17
|
-
skillcheck check <path> [--tasks N] [--trials K] [--output file.json] [--json]
|
|
18
|
-
skillcheck m0
|
|
19
|
-
skillcheck eval <path> [--tasks N] [--trials K] [--output file.json] [--task-suite file.json]
|
|
20
|
-
[--runner model] [--grader model] [--generator model]
|
|
21
|
-
skillcheck verify <result.json> [--sample n]
|
|
22
|
-
skillcheck corpus run --corpus corpus.json [--results dir] [--tasks N] [--trials K]
|
|
23
|
-
[--concurrency N] [--runner model] [--limit N]
|
|
24
|
-
skillcheck rot [--results dir] [--output file.json] [--model model] [--corpus corpus.yaml]
|
|
25
|
-
[--tasks N] [--trials K]
|
|
26
|
-
|
|
27
|
-
Drop a SKILL.md, AGENTS.md, CLAUDE.md, .cursorrules file, or a folder containing one.
|
|
28
|
-
check is the friendly quick run. eval is the raw reproducible JSON evaluator.`);
|
|
10
|
+
printHelpUi();
|
|
29
11
|
}
|
|
30
12
|
function readOption(argv, name) {
|
|
31
13
|
const index = argv.indexOf(name);
|
|
@@ -63,19 +45,6 @@ function readOptionalNumberOption(argv, name) {
|
|
|
63
45
|
function hasFlag(argv, name) {
|
|
64
46
|
return argv.includes(name);
|
|
65
47
|
}
|
|
66
|
-
function slugify(value) {
|
|
67
|
-
const slug = value
|
|
68
|
-
.toLowerCase()
|
|
69
|
-
.replace(/[^a-z0-9]+/g, '-')
|
|
70
|
-
.replace(/^-+|-+$/g, '');
|
|
71
|
-
return slug || 'skill';
|
|
72
|
-
}
|
|
73
|
-
export function defaultCheckOutputPath(inputPath, now = new Date()) {
|
|
74
|
-
const parsed = path.parse(path.resolve(inputPath));
|
|
75
|
-
const name = parsed.name || path.basename(parsed.dir) || 'skill';
|
|
76
|
-
const timestamp = now.toISOString().replace(/[:.]/g, '-');
|
|
77
|
-
return path.join('skillcheck-results', `${slugify(name)}-${timestamp}.json`);
|
|
78
|
-
}
|
|
79
48
|
function parseEvalOptions(argv, inputIndex = 3) {
|
|
80
49
|
const inputPath = argv[inputIndex];
|
|
81
50
|
if (!inputPath || inputPath.startsWith('--')) {
|
|
@@ -102,7 +71,7 @@ export function parseCheckOptions(argv, inputIndex = 3) {
|
|
|
102
71
|
if (!inputPath || inputPath.startsWith('--')) {
|
|
103
72
|
throw new Error('Usage: skillcheck check <path-to-skill-file-or-folder>\nExample: skillcheck check ./SKILL.md');
|
|
104
73
|
}
|
|
105
|
-
const output = readOption(argv, '--output')
|
|
74
|
+
const output = readOption(argv, '--output');
|
|
106
75
|
return {
|
|
107
76
|
evalOptions: {
|
|
108
77
|
inputPath,
|
|
@@ -113,7 +82,8 @@ export function parseCheckOptions(argv, inputIndex = 3) {
|
|
|
113
82
|
runner: readOption(argv, '--runner'),
|
|
114
83
|
grader: readOption(argv, '--grader'),
|
|
115
84
|
generator: readOption(argv, '--generator'),
|
|
116
|
-
taskSuite: readOption(argv, '--task-suite')
|
|
85
|
+
taskSuite: readOption(argv, '--task-suite'),
|
|
86
|
+
saveArtifacts: Boolean(output)
|
|
117
87
|
},
|
|
118
88
|
json: hasFlag(argv, '--json'),
|
|
119
89
|
output
|
|
@@ -144,48 +114,41 @@ function parseCorpusRunOptions(argv) {
|
|
|
144
114
|
limit: readOptionalNumberOption(argv, '--limit')
|
|
145
115
|
};
|
|
146
116
|
}
|
|
147
|
-
function
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
const ciText = typeof ci[0] === 'number' && typeof ci[1] === 'number'
|
|
163
|
-
? `[${ci[0].toFixed(1)}, ${ci[1].toFixed(1)}] pp`
|
|
164
|
-
: 'n/a';
|
|
165
|
-
return `Skillcheck result
|
|
166
|
-
Skill: ${String(skill.name ?? 'unknown')}
|
|
167
|
-
Verdict: ${String(score.verdict ?? 'unknown')}
|
|
168
|
-
Effect: ${formatNumber(score.effect_pp)} pp, 95% CI ${ciText}
|
|
169
|
-
Pass rate: with skill ${formatPercent(score.with_skill_pass)} vs without skill ${formatPercent(score.no_skill_pass)}
|
|
170
|
-
Token overhead: ${String(score.token_overhead ?? 'n/a')} tokens
|
|
171
|
-
Tasks/trials: ${String(config.tasks ?? 'n/a')} tasks x ${String(config.trials ?? 'n/a')} trials
|
|
172
|
-
Saved JSON: ${outputPath}`;
|
|
173
|
-
}
|
|
174
|
-
async function runCheck(options) {
|
|
175
|
-
console.error(`Checking ${options.evalOptions.inputPath} (${options.evalOptions.tasks} tasks x ${options.evalOptions.trials} trials)...`);
|
|
176
|
-
const result = await evalSkill(options.evalOptions);
|
|
117
|
+
async function runCheck(options, showBanner = true) {
|
|
118
|
+
if (showBanner && !options.json) {
|
|
119
|
+
printBanner();
|
|
120
|
+
}
|
|
121
|
+
await validateSkillInput(options.evalOptions.inputPath);
|
|
122
|
+
const progress = options.json ? undefined : startProgress();
|
|
123
|
+
let result;
|
|
124
|
+
try {
|
|
125
|
+
result = await evalSkill(options.evalOptions);
|
|
126
|
+
progress?.finish();
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
progress?.fail();
|
|
130
|
+
throw error;
|
|
131
|
+
}
|
|
177
132
|
if (options.json) {
|
|
178
133
|
console.log(JSON.stringify(result, null, 2));
|
|
179
134
|
return;
|
|
180
135
|
}
|
|
181
|
-
console.log(
|
|
136
|
+
console.log(formatResultCard(result, options.output));
|
|
137
|
+
}
|
|
138
|
+
async function runInteractiveCheck() {
|
|
139
|
+
const selectedPath = await selectSkillPath();
|
|
140
|
+
await runCheck(parseCheckOptions(['node', 'skillcheck', 'check', selectedPath]), false);
|
|
182
141
|
}
|
|
183
142
|
export async function main(argv) {
|
|
184
143
|
const command = argv[2];
|
|
185
|
-
if (
|
|
144
|
+
if (command === '--help' || command === '-h') {
|
|
186
145
|
printHelp();
|
|
187
146
|
return;
|
|
188
147
|
}
|
|
148
|
+
if (!command) {
|
|
149
|
+
await runInteractiveCheck();
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
189
152
|
if (command === 'm0') {
|
|
190
153
|
const report = await runM0Gate((config) => new NvidiaNimClient(config));
|
|
191
154
|
console.log(JSON.stringify(report, null, 2));
|
package/dist/src/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../packages/cli/src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../packages/cli/src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAyB,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAoB,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,MAAM,EAAmB,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AACzH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,SAAS;IAChB,WAAW,EAAE,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,IAAc,EAAE,IAAY;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,IAAY,EAAE,QAAgB;IACtE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAc,EAAE,IAAY;IAC5D,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,IAAc,EAAE,IAAY;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,UAAU,GAAG,CAAC;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAC;IACpD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,SAAS;QACT,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7C,IAAI;QACJ,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC;QAC1C,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC;KAC5C,CAAC;AACJ,CAAC;AAQD,MAAM,UAAU,iBAAiB,CAAC,IAAc,EAAE,UAAU,GAAG,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CACb,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC5C,OAAO;QACL,WAAW,EAAE;YACX,SAAS;YACT,MAAM;YACN,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAC3C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAC7C,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;YACpC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;YACpC,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC;YAC1C,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC;YAC3C,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC;SAC/B;QACD,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;QAC7B,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,IAAc;IACrC,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,SAAS;QACtD,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,yBAAyB;QACjE,KAAK,EAAE,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC;QAClC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAc;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,OAAO;QACL,MAAM;QACN,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,gBAAgB;QAC5D,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7C,WAAW,EAAE,gBAAgB,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;QACvD,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,wBAAwB,CAAC,IAAI,EAAE,SAAS,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAAqB,EAAE,UAAU,GAAG,IAAI;IAC9D,IAAI,UAAU,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChC,WAAW,EAAE,CAAC;IAChB,CAAC;IACD,MAAM,kBAAkB,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IAC5D,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,EAAE,IAAI,EAAE,CAAC;QACjB,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,KAAK,UAAU,mBAAmB;IAChC,MAAM,YAAY,GAAG,MAAM,eAAe,EAAE,CAAC;IAC7C,MAAM,QAAQ,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC1F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAc;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAExB,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QAC7C,SAAS,EAAE,CAAC;QACZ,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,mBAAmB,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,MAAM,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,UAAU;YACV,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;SAC9C,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,MAAM,QAAQ,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,OAAO;IACT,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;AACjD,CAAC"}
|
package/dist/src/env.js
CHANGED
|
@@ -1,39 +1,55 @@
|
|
|
1
1
|
import dotenv from 'dotenv';
|
|
2
2
|
dotenv.config();
|
|
3
|
-
|
|
3
|
+
const DEFAULT_MODEL = 'qwen/qwen3-next-80b-a3b-instruct';
|
|
4
|
+
function readEnv(name) {
|
|
4
5
|
const value = process.env[name]?.trim();
|
|
5
|
-
|
|
6
|
-
|
|
6
|
+
return value || undefined;
|
|
7
|
+
}
|
|
8
|
+
function resolveApiKey(proxyUrl) {
|
|
9
|
+
const nvidiaApiKey = readEnv('NVIDIA_API_KEY');
|
|
10
|
+
if (nvidiaApiKey) {
|
|
11
|
+
return nvidiaApiKey;
|
|
12
|
+
}
|
|
13
|
+
const proxyApiKey = readEnv('SKILLCHECK_TOKEN') ?? readEnv('SKILLCHECK_API_KEY');
|
|
14
|
+
if (proxyUrl) {
|
|
15
|
+
return proxyApiKey ?? 'skillcheck-cloud';
|
|
7
16
|
}
|
|
8
|
-
|
|
17
|
+
throw new Error('Skillcheck Cloud is not connected for this workspace. Set SKILLCHECK_API_URL or try again later.');
|
|
18
|
+
}
|
|
19
|
+
function resolveModel(role) {
|
|
20
|
+
return (readEnv(`NVIDIA_${role}_MODEL`) ??
|
|
21
|
+
readEnv(`SKILLCHECK_${role}_MODEL`) ??
|
|
22
|
+
readEnv('SKILLCHECK_MODEL') ??
|
|
23
|
+
DEFAULT_MODEL);
|
|
9
24
|
}
|
|
10
25
|
export function loadNvidiaConfig() {
|
|
11
26
|
const timeoutMs = Number(process.env.NVIDIA_TIMEOUT_MS?.trim() || 120000);
|
|
12
27
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
13
|
-
throw new Error('
|
|
28
|
+
throw new Error('Request timeout must be a positive number when set');
|
|
14
29
|
}
|
|
15
30
|
const requestDelayMs = Number(process.env.NVIDIA_REQUEST_DELAY_MS?.trim() || 5000);
|
|
16
31
|
if (!Number.isFinite(requestDelayMs) || requestDelayMs < 0) {
|
|
17
|
-
throw new Error('
|
|
32
|
+
throw new Error('Request delay must be a non-negative number when set');
|
|
18
33
|
}
|
|
19
34
|
const maxAttempts = Number(process.env.NVIDIA_MAX_ATTEMPTS?.trim() || 8);
|
|
20
35
|
if (!Number.isInteger(maxAttempts) || maxAttempts <= 0) {
|
|
21
|
-
throw new Error('
|
|
36
|
+
throw new Error('Retry attempt budget must be a positive integer when set');
|
|
22
37
|
}
|
|
23
38
|
const maxRetryDelayMs = Number(process.env.NVIDIA_MAX_RETRY_DELAY_MS?.trim() || 60000);
|
|
24
39
|
if (!Number.isFinite(maxRetryDelayMs) || maxRetryDelayMs <= 0) {
|
|
25
|
-
throw new Error('
|
|
40
|
+
throw new Error('Retry delay cap must be a positive number when set');
|
|
26
41
|
}
|
|
42
|
+
const proxyUrl = readEnv('SKILLCHECK_API_URL');
|
|
27
43
|
return {
|
|
28
|
-
apiKey:
|
|
29
|
-
baseUrl:
|
|
44
|
+
apiKey: resolveApiKey(proxyUrl),
|
|
45
|
+
baseUrl: proxyUrl ?? readEnv('NVIDIA_BASE_URL') ?? 'https://integrate.api.nvidia.com/v1',
|
|
30
46
|
timeoutMs,
|
|
31
47
|
requestDelayMs,
|
|
32
48
|
maxAttempts,
|
|
33
49
|
maxRetryDelayMs,
|
|
34
|
-
generatorModel:
|
|
35
|
-
graderModel:
|
|
36
|
-
runnerModel:
|
|
50
|
+
generatorModel: resolveModel('GENERATOR'),
|
|
51
|
+
graderModel: resolveModel('GRADER'),
|
|
52
|
+
runnerModel: resolveModel('RUNNER')
|
|
37
53
|
};
|
|
38
54
|
}
|
|
39
55
|
//# sourceMappingURL=env.js.map
|
package/dist/src/env.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../packages/cli/src/env.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../packages/cli/src/env.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,aAAa,GAAG,kCAAkC,CAAC;AAczD,SAAS,OAAO,CAAC,IAAY;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;IACxC,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED,SAAS,aAAa,CAAC,QAA4B;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC/C,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjF,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,WAAW,IAAI,kBAAkB,CAAC;IAC3C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,kGAAkG,CAAC,CAAC;AACtH,CAAC;AAED,SAAS,YAAY,CAAC,IAAuC;IAC3D,OAAO,CACL,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC;QAC/B,OAAO,CAAC,cAAc,IAAI,QAAQ,CAAC;QACnC,OAAO,CAAC,kBAAkB,CAAC;QAC3B,aAAa,CACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,CAAC;IAC1E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;IACnF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,CAAC;IACvF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAE/C,OAAO;QACL,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC;QAC/B,OAAO,EAAE,QAAQ,IAAI,OAAO,CAAC,iBAAiB,CAAC,IAAI,qCAAqC;QACxF,SAAS;QACT,cAAc;QACd,WAAW;QACX,eAAe;QACf,cAAc,EAAE,YAAY,CAAC,WAAW,CAAC;QACzC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC;QACnC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC;KACpC,CAAC;AACJ,CAAC"}
|
package/dist/src/eval.d.ts
CHANGED
package/dist/src/eval.js
CHANGED
|
@@ -84,17 +84,20 @@ function parseTaskSuite(text) {
|
|
|
84
84
|
});
|
|
85
85
|
}
|
|
86
86
|
export async function evalSkill(options) {
|
|
87
|
+
const skill = await normalizeSkill(options.inputPath);
|
|
87
88
|
const baseConfig = loadNvidiaConfig();
|
|
88
89
|
const config = applyModelOverrides(baseConfig, options);
|
|
89
90
|
const client = new NvidiaNimClient(config);
|
|
90
91
|
const cache = new JsonCache();
|
|
91
|
-
const skill = await normalizeSkill(options.inputPath);
|
|
92
92
|
const tasks = options.taskSuite
|
|
93
93
|
? parseTaskSuite(await readFile(options.taskSuite, 'utf8')).slice(0, options.tasks)
|
|
94
94
|
: await generateTasks({ domain: skill.domain, count: options.tasks }, config, client, cache);
|
|
95
95
|
const taskSuiteHash = hashJson({ skill: skill.versionHash, tasks });
|
|
96
|
-
const
|
|
97
|
-
|
|
96
|
+
const shouldSaveArtifacts = options.saveArtifacts ?? true;
|
|
97
|
+
const taskSuitePath = options.taskSuite ?? (shouldSaveArtifacts ? `results/tasks/${taskSuiteHash}.json` : undefined);
|
|
98
|
+
if (!options.taskSuite && taskSuitePath) {
|
|
99
|
+
await writeJson(taskSuitePath, tasks);
|
|
100
|
+
}
|
|
98
101
|
const outputs = await runTrials(skill, tasks, options.trials, config, client, cache);
|
|
99
102
|
const graded = await gradeOutputs(tasks, outputs, config, client, cache);
|
|
100
103
|
const score = scorePairedObservations(pairedObservations(graded));
|
|
@@ -134,7 +137,7 @@ export async function evalSkill(options) {
|
|
|
134
137
|
},
|
|
135
138
|
tasks: breakdowns,
|
|
136
139
|
reproducibility: {
|
|
137
|
-
task_suite_path: taskSuitePath,
|
|
140
|
+
...(taskSuitePath ? { task_suite_path: taskSuitePath } : {}),
|
|
138
141
|
transcript_hashes: graded.map((item) => item.transcriptHash)
|
|
139
142
|
},
|
|
140
143
|
history: [
|
package/dist/src/eval.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../packages/cli/src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAqB,MAAM,UAAU,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../packages/cli/src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAqB,MAAM,UAAU,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,YAAY,CAAC;AAiB7E,SAAS,mBAAmB,CAAC,MAAoB,EAAE,OAAoB;IACrE,OAAO;QACL,GAAG,MAAM;QACT,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,cAAc,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAsB;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC;YAC9B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAyB,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAsB,EAAE,MAAsB;IACpE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QACrE,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,cAAc,EAAE,IAAI,CAAC,aAAa;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,eAAe,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;YAChF,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;SAC7E,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACjG,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,IAAI,QAAQ,CAAC;QAC5E,IAAI,aAAa,KAAK,QAAQ,IAAI,aAAa,KAAK,eAAe,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,aAAa;YACb,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;SACxC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAoB;IAClD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAC;IAE9B,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS;QAC7B,CAAC,CAAC,cAAc,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;QACnF,CAAC,CAAC,MAAM,aAAa,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC/F,MAAM,aAAa,GAAG,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,MAAM,mBAAmB,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;IAC1D,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,iBAAiB,aAAa,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACrH,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,aAAa,EAAE,CAAC;QACxC,MAAM,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,uBAAuB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACjD,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5G,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACxG,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,gBAAgB,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChH,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEtD,MAAM,MAAM,GAAG;QACb,KAAK,EAAE;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,UAAU;YAC/C,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB;QACD,MAAM,EAAE;YACN,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,cAAc,EAAE,MAAM,CAAC,WAAW;YAClC,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,cAAc,EAAE,MAAM,CAAC,WAAW;YAClC,eAAe,EAAE,MAAM,CAAC,cAAc;YACtC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,GAAG;YAChB,IAAI,EAAE,kBAAkB;SACzB;QACD,MAAM,EAAE;YACN,SAAS,EAAE,KAAK,CAAC,QAAQ;YACzB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,eAAe,EAAE,KAAK,CAAC,aAAa;YACpC,aAAa,EAAE,KAAK,CAAC,WAAW;YAChC,cAAc,EAAE,aAAa;YAC7B,mBAAmB,EAAE,gBAAgB;SACtC;QACD,KAAK,EAAE,UAAU;QACjB,eAAe,EAAE;YACf,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC;SAC7D;QACD,OAAO,EAAE;YACP;gBACE,cAAc,EAAE,MAAM,CAAC,WAAW;gBAClC,QAAQ,EAAE,OAAO;gBACjB,SAAS,EAAE,KAAK,CAAC,QAAQ;gBACzB,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB;SACF;QACD,QAAQ,EAAE,OAAO;KAClB,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/src/generate.js
CHANGED
|
@@ -86,7 +86,9 @@ export async function generateTasks(input, config, client, cache) {
|
|
|
86
86
|
}
|
|
87
87
|
catch (error) {
|
|
88
88
|
lastError = error;
|
|
89
|
-
|
|
89
|
+
if (process.env.SKILLCHECK_DEBUG === '1') {
|
|
90
|
+
console.error(`[skillcheck] generator returned invalid JSON on attempt ${attempt}/3`);
|
|
91
|
+
}
|
|
90
92
|
}
|
|
91
93
|
}
|
|
92
94
|
throw lastError instanceof Error ? lastError : new Error('Generator did not return valid tasks after retries');
|
package/dist/src/generate.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../packages/cli/src/generate.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,WAAW,GAAG,UAAU,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;IACnD,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7C,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,KAAa;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,mBAAmB,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,iCAAiC,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM;YACN,aAAa,EAAE,QAAQ;YACvB,SAAS;SACV,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAA0B,EAC1B,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACvC,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,WAAW,EACX,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE,OAAO,EAAE,EAClF,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;YACd,KAAK,EAAE,MAAM,CAAC,cAAc;YAC5B,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;YACf,cAAc,EAAE,aAAa;YAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;YACvC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,qIAAqI;iBAC/I;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,qBAAqB,KAAK,CAAC,MAAM,gBAAgB,cAAc,kPAAkP;iBAC3T;aACF;SACF,CAAC,CACL,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,aAAa,CAAC,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;YACtF,OAAO,aAAa,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtH,GAAG,IAAI;gBACP,EAAE,EAAE,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;aAC7C,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../packages/cli/src/generate.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,WAAW,GAAG,UAAU,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;IACnD,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7C,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,KAAa;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,mBAAmB,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,iCAAiC,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM;YACN,aAAa,EAAE,QAAQ;YACvB,SAAS;SACV,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAA0B,EAC1B,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACvC,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,WAAW,EACX,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE,OAAO,EAAE,EAClF,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;YACd,KAAK,EAAE,MAAM,CAAC,cAAc;YAC5B,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;YACf,cAAc,EAAE,aAAa;YAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;YACvC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,qIAAqI;iBAC/I;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,qBAAqB,KAAK,CAAC,MAAM,gBAAgB,cAAc,kPAAkP;iBAC3T;aACF;SACF,CAAC,CACL,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,aAAa,CAAC,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;YACtF,OAAO,aAAa,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtH,GAAG,IAAI;gBACP,EAAE,EAAE,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;aAC7C,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,EAAE,CAAC;gBACzC,OAAO,CAAC,KAAK,CAAC,2DAA2D,OAAO,IAAI,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACjH,CAAC"}
|
package/dist/src/grade.js
CHANGED
|
@@ -41,6 +41,7 @@ export async function gradeOutputs(tasks, outputs, config, client, cache) {
|
|
|
41
41
|
const taskById = new Map(tasks.map((task) => [task.id, task]));
|
|
42
42
|
const shuffled = seededShuffle(outputs, hashJson(outputs.map((output) => output.transcriptHash)));
|
|
43
43
|
const graded = new Map();
|
|
44
|
+
const debug = process.env.SKILLCHECK_DEBUG === '1';
|
|
44
45
|
for (const output of shuffled) {
|
|
45
46
|
const task = taskById.get(output.taskId);
|
|
46
47
|
if (!task) {
|
|
@@ -54,7 +55,9 @@ export async function gradeOutputs(tasks, outputs, config, client, cache) {
|
|
|
54
55
|
});
|
|
55
56
|
continue;
|
|
56
57
|
}
|
|
57
|
-
|
|
58
|
+
if (debug) {
|
|
59
|
+
console.error(`[skillcheck] grade ${output.taskId} trial ${output.trial}`);
|
|
60
|
+
}
|
|
58
61
|
let grade;
|
|
59
62
|
let lastError;
|
|
60
63
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
@@ -88,7 +91,9 @@ export async function gradeOutputs(tasks, outputs, config, client, cache) {
|
|
|
88
91
|
}
|
|
89
92
|
catch (error) {
|
|
90
93
|
lastError = error;
|
|
91
|
-
|
|
94
|
+
if (debug) {
|
|
95
|
+
console.error(`[skillcheck] grader returned invalid JSON on attempt ${attempt}/3`);
|
|
96
|
+
}
|
|
92
97
|
}
|
|
93
98
|
}
|
|
94
99
|
if (!grade) {
|
package/dist/src/grade.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IAEnD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;gBAChC,GAAG,MAAM;gBACT,GAAG,KAAK;aACT,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,KAA+B,CAAC;QACpC,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,QAAQ,EACR;gBACE,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,CAAC;gBAChB,cAAc,EAAE,aAAa;gBAC7B,OAAO;aACR,EACD,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,GAAG;gBACd,cAAc,EAAE,aAAa;gBAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACvC,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EACL,qMAAqM;qBACxM;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,uBAAuB,IAAI,CAAC,SAAS,yBAAyB,MAAM,CAAC,MAAM,0MAA0M;qBAC/R;iBACF;aACF,CAAC,CACL,CAAC;YAEF,IAAI,CAAC;gBACH,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACrC,MAAM;YACR,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,KAAK,CAAC,wDAAwD,OAAO,IAAI,CAAC,CAAC;gBACrF,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAChH,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;YAChC,GAAG,MAAM;YACT,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,GAAG;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/src/run.js
CHANGED
|
@@ -34,11 +34,16 @@ async function runOne(skill, task, trial, arm, config, client, cache) {
|
|
|
34
34
|
}
|
|
35
35
|
export async function runTrials(skill, tasks, trials, config, client, cache) {
|
|
36
36
|
const outputs = [];
|
|
37
|
+
const debug = process.env.SKILLCHECK_DEBUG === '1';
|
|
37
38
|
for (const task of tasks) {
|
|
38
39
|
for (let trial = 1; trial <= trials; trial += 1) {
|
|
39
|
-
|
|
40
|
+
if (debug) {
|
|
41
|
+
console.error(`[skillcheck] run ${task.id} trial ${trial}/${trials} with_skill`);
|
|
42
|
+
}
|
|
40
43
|
outputs.push(await runOne(skill, task, trial, 'with_skill', config, client, cache));
|
|
41
|
-
|
|
44
|
+
if (debug) {
|
|
45
|
+
console.error(`[skillcheck] run ${task.id} trial ${trial}/${trials} no_skill`);
|
|
46
|
+
}
|
|
42
47
|
outputs.push(await runOne(skill, task, trial, 'no_skill', config, client, cache));
|
|
43
48
|
}
|
|
44
49
|
}
|
package/dist/src/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAC/H,MAAM,CAAC,QAAQ,CAAC;QACd,KAAK,EAAE,MAAM,CAAC,WAAW;QACzB,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,IAAI;KAChB,CAAC,CACH,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAEjH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,IAAI,CAAC;QACvD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;QAC7C,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAsB,EACtB,KAAsB,EACtB,MAAc,EACd,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAC/H,MAAM,CAAC,QAAQ,CAAC;QACd,KAAK,EAAE,MAAM,CAAC,WAAW;QACzB,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,IAAI;KAChB,CAAC,CACH,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAEjH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,IAAI,CAAC;QACvD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;QAC7C,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAsB,EACtB,KAAsB,EACtB,MAAc,EACd,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,aAAa,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,WAAW,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/src/ui.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare function printBanner(): void;
|
|
2
|
+
export declare function printHelpUi(): void;
|
|
3
|
+
export declare function supportedSkillFilesText(): string;
|
|
4
|
+
export declare function validateSkillInput(inputPath: string): Promise<string>;
|
|
5
|
+
export declare function selectSkillPath(startDir?: string): Promise<string>;
|
|
6
|
+
export declare function startProgress(): {
|
|
7
|
+
finish: () => void;
|
|
8
|
+
fail: () => void;
|
|
9
|
+
};
|
|
10
|
+
export declare function formatResultCard(result: unknown, outputPath?: string): string;
|
|
11
|
+
export declare function sanitizeCliError(error: unknown): string;
|
|
12
|
+
export declare function formatFatalError(error: unknown): string;
|
package/dist/src/ui.js
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import { readdir, stat } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import process from 'node:process';
|
|
4
|
+
import readline from 'node:readline';
|
|
5
|
+
const SUPPORTED_SKILL_FILES = ['SKILL.md', 'AGENTS.md', 'CLAUDE.md', '.cursorrules'];
|
|
6
|
+
const useColor = () => process.stdout.isTTY && process.env.NO_COLOR !== '1';
|
|
7
|
+
const color = (code, value) => (useColor() ? `\x1b[${code}m${value}\x1b[0m` : value);
|
|
8
|
+
const blue = (value) => color('38;5;33', value);
|
|
9
|
+
const deepBlue = (value) => color('1;38;5;18', value);
|
|
10
|
+
const white = (value) => color('1;37', value);
|
|
11
|
+
const dim = (value) => color('2', value);
|
|
12
|
+
const red = (value) => color('1;31', value);
|
|
13
|
+
const green = (value) => color('1;32', value);
|
|
14
|
+
const yellow = (value) => color('1;33', value);
|
|
15
|
+
export function printBanner() {
|
|
16
|
+
console.log(deepBlue(` ____ _ _ _ _ _ _
|
|
17
|
+
/ ___|| | _(_) | | ___| |__ ___ ___| | __
|
|
18
|
+
\\___ \\| |/ / | | |/ __| '_ \\ / _ \\/ __| |/ /
|
|
19
|
+
___) | <| | | | (__| | | | __/ (__| <
|
|
20
|
+
|____/|_|\\_\\_|_|_|\\___|_| |_|\\___|\\___|_|\\_\\`));
|
|
21
|
+
console.log(white(' Drop a skill file. Get a verdict.\n'));
|
|
22
|
+
}
|
|
23
|
+
export function printHelpUi() {
|
|
24
|
+
printBanner();
|
|
25
|
+
console.log(`${blue('Quick start')}`);
|
|
26
|
+
console.log(` ${white('skillcheck')} ${blue('check')} path/to/SKILL.md`);
|
|
27
|
+
console.log(` ${white('skillcheck')} path/to/skill-folder\n`);
|
|
28
|
+
console.log(`${blue('Supported inputs')}`);
|
|
29
|
+
console.log(` ${SUPPORTED_SKILL_FILES.join(', ')} or a folder containing one\n`);
|
|
30
|
+
console.log(`${blue('Commands')}`);
|
|
31
|
+
console.log(` skillcheck check <path> [--tasks N] [--trials K] [--output file.json] [--json]`);
|
|
32
|
+
console.log(` skillcheck eval <path> [--tasks N] [--trials K] [--output file.json]`);
|
|
33
|
+
console.log(` skillcheck verify <result.json> [--sample n]`);
|
|
34
|
+
console.log(` skillcheck corpus run --corpus corpus.json [--results dir]`);
|
|
35
|
+
console.log(` skillcheck rot [--results dir] [--output file.json]\n`);
|
|
36
|
+
}
|
|
37
|
+
export function supportedSkillFilesText() {
|
|
38
|
+
return SUPPORTED_SKILL_FILES.join(', ');
|
|
39
|
+
}
|
|
40
|
+
async function fileExists(filePath) {
|
|
41
|
+
try {
|
|
42
|
+
const stats = await stat(filePath);
|
|
43
|
+
return stats.isFile();
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
if (error.code === 'ENOENT') {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async function directoryHasSkillFile(dirPath) {
|
|
53
|
+
for (const fileName of SUPPORTED_SKILL_FILES) {
|
|
54
|
+
if (await fileExists(path.join(dirPath, fileName))) {
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
function isSupportedSkillFile(filePath) {
|
|
61
|
+
return SUPPORTED_SKILL_FILES.includes(path.basename(filePath));
|
|
62
|
+
}
|
|
63
|
+
export async function validateSkillInput(inputPath) {
|
|
64
|
+
try {
|
|
65
|
+
const resolved = path.resolve(inputPath);
|
|
66
|
+
const stats = await stat(resolved);
|
|
67
|
+
if (stats.isDirectory() && (await directoryHasSkillFile(resolved))) {
|
|
68
|
+
return inputPath;
|
|
69
|
+
}
|
|
70
|
+
if (stats.isFile() && isSupportedSkillFile(resolved)) {
|
|
71
|
+
return inputPath;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (error.code !== 'ENOENT') {
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
throw new Error(`This path does not exist. Please give me a path to ${supportedSkillFilesText()}, or a folder containing one.`);
|
|
79
|
+
}
|
|
80
|
+
throw new Error(`That is not a skill file. Please give me a path to ${supportedSkillFilesText()}, or a folder containing one.`);
|
|
81
|
+
}
|
|
82
|
+
async function listPickerEntries(currentDir) {
|
|
83
|
+
const entries = await readdir(currentDir, { withFileTypes: true });
|
|
84
|
+
const parent = path.dirname(currentDir);
|
|
85
|
+
const mapped = await Promise.all(entries
|
|
86
|
+
.filter((entry) => entry.name !== 'node_modules' && entry.name !== '.git')
|
|
87
|
+
.map(async (entry) => {
|
|
88
|
+
const fullPath = path.join(currentDir, entry.name);
|
|
89
|
+
if (entry.isDirectory()) {
|
|
90
|
+
const runnable = await directoryHasSkillFile(fullPath);
|
|
91
|
+
return {
|
|
92
|
+
label: `${entry.name}/`,
|
|
93
|
+
fullPath,
|
|
94
|
+
kind: 'directory',
|
|
95
|
+
runnable,
|
|
96
|
+
note: runnable ? 'skill folder' : 'folder'
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
const runnable = entry.isFile() && isSupportedSkillFile(fullPath);
|
|
100
|
+
return {
|
|
101
|
+
label: entry.name,
|
|
102
|
+
fullPath,
|
|
103
|
+
kind: 'file',
|
|
104
|
+
runnable,
|
|
105
|
+
note: runnable ? 'skill file' : 'not a skill'
|
|
106
|
+
};
|
|
107
|
+
}));
|
|
108
|
+
return [
|
|
109
|
+
{
|
|
110
|
+
label: '../',
|
|
111
|
+
fullPath: parent,
|
|
112
|
+
kind: 'parent',
|
|
113
|
+
runnable: false,
|
|
114
|
+
note: 'back'
|
|
115
|
+
},
|
|
116
|
+
...mapped.sort((a, b) => {
|
|
117
|
+
if (a.kind !== b.kind) {
|
|
118
|
+
return a.kind === 'directory' ? -1 : 1;
|
|
119
|
+
}
|
|
120
|
+
if (a.runnable !== b.runnable) {
|
|
121
|
+
return a.runnable ? -1 : 1;
|
|
122
|
+
}
|
|
123
|
+
return a.label.localeCompare(b.label);
|
|
124
|
+
})
|
|
125
|
+
];
|
|
126
|
+
}
|
|
127
|
+
function visibleWindow(items, selected, size) {
|
|
128
|
+
const offset = Math.max(0, Math.min(selected - Math.floor(size / 2), Math.max(0, items.length - size)));
|
|
129
|
+
return { items: items.slice(offset, offset + size), offset };
|
|
130
|
+
}
|
|
131
|
+
function renderPicker(currentDir, entries, selected, message) {
|
|
132
|
+
process.stdout.write('\x1b[2J\x1b[H');
|
|
133
|
+
printBanner();
|
|
134
|
+
console.log(`${blue('Select a skill file or folder')}`);
|
|
135
|
+
console.log(`${dim('Current folder:')} ${currentDir}`);
|
|
136
|
+
console.log(dim('Use arrows, Enter to open/select, q to quit.\n'));
|
|
137
|
+
if (message) {
|
|
138
|
+
console.log(`${red(message)}\n`);
|
|
139
|
+
}
|
|
140
|
+
const window = visibleWindow(entries, selected, 12);
|
|
141
|
+
for (const [index, entry] of window.items.entries()) {
|
|
142
|
+
const realIndex = window.offset + index;
|
|
143
|
+
const pointer = realIndex === selected ? blue('>') : ' ';
|
|
144
|
+
const label = realIndex === selected ? white(entry.label) : entry.runnable ? blue(entry.label) : entry.label;
|
|
145
|
+
const note = entry.runnable ? blue(entry.note) : dim(entry.note);
|
|
146
|
+
console.log(`${pointer} ${label.padEnd(34)} ${note}`);
|
|
147
|
+
}
|
|
148
|
+
console.log('');
|
|
149
|
+
}
|
|
150
|
+
function readKey() {
|
|
151
|
+
return new Promise((resolve) => {
|
|
152
|
+
process.stdin.once('keypress', (input, key) => resolve({ input, key }));
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
export async function selectSkillPath(startDir = process.cwd()) {
|
|
156
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
|
157
|
+
throw new Error(`Please give me a path to ${supportedSkillFilesText()}, or a folder containing one.`);
|
|
158
|
+
}
|
|
159
|
+
readline.emitKeypressEvents(process.stdin);
|
|
160
|
+
const wasRaw = process.stdin.isRaw === true;
|
|
161
|
+
process.stdin.setRawMode(true);
|
|
162
|
+
let currentDir = path.resolve(startDir);
|
|
163
|
+
let selected = 0;
|
|
164
|
+
let message;
|
|
165
|
+
try {
|
|
166
|
+
for (;;) {
|
|
167
|
+
const entries = await listPickerEntries(currentDir);
|
|
168
|
+
selected = Math.max(0, Math.min(selected, entries.length - 1));
|
|
169
|
+
renderPicker(currentDir, entries, selected, message);
|
|
170
|
+
message = undefined;
|
|
171
|
+
const { input, key } = await readKey();
|
|
172
|
+
if ((key.ctrl && key.name === 'c') || input === 'q') {
|
|
173
|
+
throw new Error('Selection cancelled.');
|
|
174
|
+
}
|
|
175
|
+
if (key.name === 'up') {
|
|
176
|
+
selected = selected <= 0 ? entries.length - 1 : selected - 1;
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (key.name === 'down') {
|
|
180
|
+
selected = selected >= entries.length - 1 ? 0 : selected + 1;
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
if (key.name !== 'return') {
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
const entry = entries[selected];
|
|
187
|
+
if (!entry) {
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
if (entry.kind === 'parent') {
|
|
191
|
+
currentDir = entry.fullPath;
|
|
192
|
+
selected = 0;
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
if (entry.kind === 'directory' && !entry.runnable) {
|
|
196
|
+
currentDir = entry.fullPath;
|
|
197
|
+
selected = 0;
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
if (entry.runnable) {
|
|
201
|
+
process.stdout.write('\x1b[2J\x1b[H');
|
|
202
|
+
return entry.fullPath;
|
|
203
|
+
}
|
|
204
|
+
message = `That is not a skill file. Select ${supportedSkillFilesText()}, or a folder containing one.`;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
process.stdin.setRawMode(wasRaw);
|
|
209
|
+
process.stdin.pause();
|
|
210
|
+
process.stdout.write('\x1b[?25h');
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
function progressLine(percent) {
|
|
214
|
+
const width = 32;
|
|
215
|
+
const filled = Math.round((width * percent) / 100);
|
|
216
|
+
const bar = `${'#'.repeat(filled)}${'-'.repeat(width - filled)}`;
|
|
217
|
+
return `${blue('Analyzing skill')} ${white(`[${bar}]`)} ${String(percent).padStart(3, ' ')}%`;
|
|
218
|
+
}
|
|
219
|
+
export function startProgress() {
|
|
220
|
+
if (!process.stdout.isTTY) {
|
|
221
|
+
return { finish: () => undefined, fail: () => undefined };
|
|
222
|
+
}
|
|
223
|
+
let percent = 0;
|
|
224
|
+
const render = () => {
|
|
225
|
+
process.stdout.write(`\r\x1b[2K${progressLine(percent)}`);
|
|
226
|
+
};
|
|
227
|
+
const timer = setInterval(() => {
|
|
228
|
+
percent = Math.min(94, percent + 2);
|
|
229
|
+
render();
|
|
230
|
+
}, 400);
|
|
231
|
+
process.stdout.write('\x1b[?25l');
|
|
232
|
+
render();
|
|
233
|
+
return {
|
|
234
|
+
finish: () => {
|
|
235
|
+
clearInterval(timer);
|
|
236
|
+
percent = 100;
|
|
237
|
+
render();
|
|
238
|
+
process.stdout.write('\n\x1b[?25h');
|
|
239
|
+
},
|
|
240
|
+
fail: () => {
|
|
241
|
+
clearInterval(timer);
|
|
242
|
+
process.stdout.write('\r\x1b[2K\x1b[?25h');
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
function asRecord(value) {
|
|
247
|
+
return typeof value === 'object' && value !== null ? value : {};
|
|
248
|
+
}
|
|
249
|
+
function formatPercent(value) {
|
|
250
|
+
return typeof value === 'number' ? `${(value * 100).toFixed(1)}%` : 'n/a';
|
|
251
|
+
}
|
|
252
|
+
function formatNumber(value) {
|
|
253
|
+
return typeof value === 'number' ? value.toFixed(1) : 'n/a';
|
|
254
|
+
}
|
|
255
|
+
function verdictColor(verdict) {
|
|
256
|
+
if (verdict === 'helps') {
|
|
257
|
+
return green(verdict.toUpperCase());
|
|
258
|
+
}
|
|
259
|
+
if (verdict === 'harms') {
|
|
260
|
+
return red(verdict.toUpperCase());
|
|
261
|
+
}
|
|
262
|
+
return yellow(verdict.toUpperCase());
|
|
263
|
+
}
|
|
264
|
+
function boxLine(label, value) {
|
|
265
|
+
return `${blue('|')} ${white(label.padEnd(15))} ${value}`;
|
|
266
|
+
}
|
|
267
|
+
export function formatResultCard(result, outputPath) {
|
|
268
|
+
const root = asRecord(result);
|
|
269
|
+
const skill = asRecord(root.skill);
|
|
270
|
+
const config = asRecord(root.config);
|
|
271
|
+
const score = asRecord(root.result);
|
|
272
|
+
const ci = Array.isArray(score.ci_pp) ? score.ci_pp : [];
|
|
273
|
+
const ciText = typeof ci[0] === 'number' && typeof ci[1] === 'number'
|
|
274
|
+
? `[${ci[0].toFixed(1)}, ${ci[1].toFixed(1)}] pp`
|
|
275
|
+
: 'n/a';
|
|
276
|
+
const verdict = String(score.verdict ?? 'unknown');
|
|
277
|
+
const lines = [
|
|
278
|
+
blue('+-----------------------------------------------+'),
|
|
279
|
+
`${blue('|')} ${white('SKILLCHECK RESULT')}`,
|
|
280
|
+
blue('+-----------------------------------------------+'),
|
|
281
|
+
boxLine('Skill', String(skill.name ?? 'unknown')),
|
|
282
|
+
boxLine('Verdict', verdictColor(verdict)),
|
|
283
|
+
boxLine('Effect', `${formatNumber(score.effect_pp)} pp`),
|
|
284
|
+
boxLine('Confidence', ciText),
|
|
285
|
+
boxLine('With skill', formatPercent(score.with_skill_pass)),
|
|
286
|
+
boxLine('Without skill', formatPercent(score.no_skill_pass)),
|
|
287
|
+
boxLine('Token cost', `${String(score.token_overhead ?? 'n/a')} tokens`),
|
|
288
|
+
boxLine('Run size', `${String(config.tasks ?? 'n/a')} tasks x ${String(config.trials ?? 'n/a')} trials`)
|
|
289
|
+
];
|
|
290
|
+
if (outputPath) {
|
|
291
|
+
lines.push(boxLine('Saved JSON', outputPath));
|
|
292
|
+
}
|
|
293
|
+
lines.push(blue('+-----------------------------------------------+'));
|
|
294
|
+
return lines.join('\n');
|
|
295
|
+
}
|
|
296
|
+
export function sanitizeCliError(error) {
|
|
297
|
+
const raw = error instanceof Error ? error.message : String(error);
|
|
298
|
+
if (/api[_ -]?key|credential|unauthorized|authentication|401/i.test(raw)) {
|
|
299
|
+
return 'Skillcheck Cloud is not connected for this workspace. Please try again later or contact the workspace owner.';
|
|
300
|
+
}
|
|
301
|
+
return raw
|
|
302
|
+
.replace(/NVIDIA NIM/gi, 'Skillcheck Cloud')
|
|
303
|
+
.replace(/NVIDIA_API_KEY/gi, 'Skillcheck Cloud')
|
|
304
|
+
.replace(/NVIDIA/gi, 'Skillcheck Cloud')
|
|
305
|
+
.replace(/API[_ -]?KEY/gi, 'credential');
|
|
306
|
+
}
|
|
307
|
+
export function formatFatalError(error) {
|
|
308
|
+
return `${red('Skillcheck stopped')}\n${sanitizeCliError(error)}`;
|
|
309
|
+
}
|
|
310
|
+
//# sourceMappingURL=ui.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ui.js","sourceRoot":"","sources":["../../packages/cli/src/ui.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,QAAQ,MAAM,eAAe,CAAC;AAErC,MAAM,qBAAqB,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;AACrF,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC;AAC5E,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,KAAa,EAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAC7G,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAChE,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;AACtE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC9D,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;AACzD,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC5D,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAC9D,MAAM,MAAM,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAgB/D,MAAM,UAAU,WAAW;IACzB,OAAO,CAAC,GAAG,CACT,QAAQ,CAAC;;;;qDAIwC,CAAC,CACnD,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,WAAW,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,YAAY,CAAC,yBAAyB,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,KAAK,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,kFAAkF,CAAC,CAAC;IAChG,OAAO,CAAC,GAAG,CAAC,wEAAwE,CAAC,CAAC;IACtF,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,uBAAuB;IACrC,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,OAAe;IAClD,KAAK,MAAM,QAAQ,IAAI,qBAAqB,EAAE,CAAC;QAC7C,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB;IACxD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACnE,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrD,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,sDAAsD,uBAAuB,EAAE,+BAA+B,CAAC,CAAC;IAClI,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,sDAAsD,uBAAuB,EAAE,+BAA+B,CAAC,CAAC;AAClI,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,UAAkB;IACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,OAAO;SACJ,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;SACzE,GAAG,CAAC,KAAK,EAAE,KAAK,EAAwB,EAAE;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YACvD,OAAO;gBACL,KAAK,EAAE,GAAG,KAAK,CAAC,IAAI,GAAG;gBACvB,QAAQ;gBACR,IAAI,EAAE,WAAW;gBACjB,QAAQ;gBACR,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ;aAC3C,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAClE,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,QAAQ;YACR,IAAI,EAAE,MAAM;YACZ,QAAQ;YACR,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa;SAC9C,CAAC;IACJ,CAAC,CAAC,CACL,CAAC;IAEF,OAAO;QACL;YACE,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK;YACf,IAAI,EAAE,MAAM;SACb;QACD,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACtB,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;gBACtB,OAAO,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC9B,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;YACD,OAAO,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxC,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB,EAAE,IAAY;IAClE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACxG,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;AAC/D,CAAC;AAED,SAAS,YAAY,CAAC,UAAkB,EAAE,OAAsB,EAAE,QAAgB,EAAE,OAAgB;IAClG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACtC,WAAW,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,+BAA+B,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,iBAAiB,CAAC,IAAI,UAAU,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC,CAAC;IAEnE,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IACpD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC;QACxC,MAAM,OAAO,GAAG,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACzD,MAAM,KAAK,GAAG,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7G,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjE,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,OAAO;IACd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,KAAa,EAAE,GAAa,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,EAAE;IAC5D,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,4BAA4B,uBAAuB,EAAE,+BAA+B,CAAC,CAAC;IACxG,CAAC;IAED,QAAQ,CAAC,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC;IAC5C,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAE/B,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAA2B,CAAC;IAEhC,IAAI,CAAC;QACH,SAAS,CAAC;YACR,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,UAAU,CAAC,CAAC;YACpD,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/D,YAAY,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,OAAO,GAAG,SAAS,CAAC;YAEpB,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,OAAO,EAAE,CAAC;YACvC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBACtB,QAAQ,GAAG,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAC7D,SAAS;YACX,CAAC;YACD,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxB,QAAQ,GAAG,QAAQ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAC7D,SAAS;YACX,CAAC;YACD,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC1B,SAAS;YACX,CAAC;YAED,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC5B,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,GAAG,CAAC,CAAC;gBACb,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClD,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,GAAG,CAAC,CAAC;gBACb,SAAS;YACX,CAAC;YACD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACtC,OAAO,KAAK,CAAC,QAAQ,CAAC;YACxB,CAAC;YACD,OAAO,GAAG,oCAAoC,uBAAuB,EAAE,+BAA+B,CAAC;QACzG,CAAC;IACH,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC;IACjE,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC;IAC5D,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5D,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QACpC,MAAM,EAAE,CAAC;IACX,CAAC,EAAE,GAAG,CAAC,CAAC;IAER,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAClC,MAAM,EAAE,CAAC;IAET,OAAO;QACL,MAAM,EAAE,GAAG,EAAE;YACX,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,GAAG,GAAG,CAAC;YACd,MAAM,EAAE,CAAC;YACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,EAAE,GAAG,EAAE;YACT,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC7C,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/F,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AAC5E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9D,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,OAAO,CAAC,KAAa,EAAE,KAAa;IAC3C,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAe,EAAE,UAAmB;IACnE,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,MAAM,GACV,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ;QACpD,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QACjD,CAAC,CAAC,KAAK,CAAC;IACZ,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,SAAS,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG;QACZ,IAAI,CAAC,mDAAmD,CAAC;QACzD,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,EAAE;QAC5C,IAAI,CAAC,mDAAmD,CAAC;QACzD,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC;QACjD,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,QAAQ,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC;QACxD,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC;QAC7B,OAAO,CAAC,YAAY,EAAE,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAC3D,OAAO,CAAC,eAAe,EAAE,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5D,OAAO,CAAC,YAAY,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,SAAS,CAAC;QACxE,OAAO,CAAC,UAAU,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC;KACzG,CAAC;IACF,IAAI,UAAU,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC,CAAC;IACtE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnE,IAAI,0DAA0D,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACzE,OAAO,8GAA8G,CAAC;IACxH,CAAC;IACD,OAAO,GAAG;SACP,OAAO,CAAC,cAAc,EAAE,kBAAkB,CAAC;SAC3C,OAAO,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;SAC/C,OAAO,CAAC,UAAU,EAAE,kBAAkB,CAAC;SACvC,OAAO,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,OAAO,GAAG,GAAG,CAAC,oBAAoB,CAAC,KAAK,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;AACpE,CAAC"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# Skillcheck Cloud setup
|
|
2
|
+
|
|
3
|
+
Use this when you want users to install the CLI and run checks without configuring model-provider secrets locally.
|
|
4
|
+
|
|
5
|
+
## Architecture
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
skillcheck CLI
|
|
9
|
+
-> Skillcheck Cloud API
|
|
10
|
+
-> model provider
|
|
11
|
+
|
|
12
|
+
Dashboard
|
|
13
|
+
-> user signs up
|
|
14
|
+
-> user creates a Skillcheck token
|
|
15
|
+
-> token is stored hashed in your database
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The CLI only needs:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
export SKILLCHECK_API_URL=https://api.yourdomain.com/v1
|
|
22
|
+
export SKILLCHECK_TOKEN=sk_live_...
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
If you want public free trials, the proxy can allow anonymous requests with strict rate limits and no token.
|
|
26
|
+
|
|
27
|
+
## API contract
|
|
28
|
+
|
|
29
|
+
The CLI expects an OpenAI-compatible endpoint:
|
|
30
|
+
|
|
31
|
+
```http
|
|
32
|
+
POST /v1/chat/completions
|
|
33
|
+
Authorization: Bearer <skillcheck-token>
|
|
34
|
+
Content-Type: application/json
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Response should match OpenAI chat completions enough for the `openai` Node SDK:
|
|
38
|
+
|
|
39
|
+
```json
|
|
40
|
+
{
|
|
41
|
+
"id": "chatcmpl_...",
|
|
42
|
+
"object": "chat.completion",
|
|
43
|
+
"created": 1780000000,
|
|
44
|
+
"model": "your-model",
|
|
45
|
+
"choices": [
|
|
46
|
+
{
|
|
47
|
+
"index": 0,
|
|
48
|
+
"message": {
|
|
49
|
+
"role": "assistant",
|
|
50
|
+
"content": "..."
|
|
51
|
+
},
|
|
52
|
+
"finish_reason": "stop"
|
|
53
|
+
}
|
|
54
|
+
],
|
|
55
|
+
"usage": {
|
|
56
|
+
"prompt_tokens": 1,
|
|
57
|
+
"completion_tokens": 1,
|
|
58
|
+
"total_tokens": 2
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Minimal proxy
|
|
64
|
+
|
|
65
|
+
The repo includes a tiny Node proxy in `examples/nvidia-proxy/`. It is useful for testing the `SKILLCHECK_API_URL` flow before building the dashboard.
|
|
66
|
+
|
|
67
|
+
Run it on a server:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
export NVIDIA_API_KEY=...
|
|
71
|
+
export SKILLCHECK_PROXY_TOKEN=dev-token
|
|
72
|
+
node examples/nvidia-proxy/server.mjs
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Point the CLI at it:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
export SKILLCHECK_API_URL=https://your-proxy.example.com/v1
|
|
79
|
+
export SKILLCHECK_TOKEN=dev-token
|
|
80
|
+
skillcheck check path/to/SKILL.md
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Dashboard requirements
|
|
84
|
+
|
|
85
|
+
- `users`: id, email, password/session provider, created_at.
|
|
86
|
+
- `tokens`: id, user_id, token_hash, prefix, created_at, last_used_at, revoked_at.
|
|
87
|
+
- `usage_events`: user_id, token_id, request_id, model, prompt_tokens, completion_tokens, created_at.
|
|
88
|
+
- Rate limit by token and IP.
|
|
89
|
+
- Store model-provider secrets only on the server.
|
|
90
|
+
- Never expose upstream provider secrets to the browser or CLI.
|
|
91
|
+
|
|
92
|
+
## First production path
|
|
93
|
+
|
|
94
|
+
1. Deploy the proxy API at `https://api.yourdomain.com/v1`.
|
|
95
|
+
2. Add dashboard auth with GitHub or email login.
|
|
96
|
+
3. Add “Create token” in the dashboard and show the token once.
|
|
97
|
+
4. Hash tokens before storing them.
|
|
98
|
+
5. Verify `Authorization: Bearer <token>` in the proxy.
|
|
99
|
+
6. Add rate limits and usage logging.
|
|
100
|
+
7. Ship the CLI with docs telling users to set `SKILLCHECK_API_URL` and `SKILLCHECK_TOKEN`.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# skillcheck NVIDIA proxy
|
|
2
|
+
|
|
3
|
+
This is the safe way to let CLI users run `skillcheck` without seeing your NVIDIA key.
|
|
4
|
+
|
|
5
|
+
Run the proxy on a server:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
export NVIDIA_API_KEY=...
|
|
9
|
+
node examples/nvidia-proxy/server.mjs
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Point the CLI at the proxy:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
export SKILLCHECK_API_URL=https://your-proxy.example.com/v1
|
|
16
|
+
skillcheck check path/to/SKILL.md
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Do not publish `NVIDIA_API_KEY` inside the npm package. If the proxy is public, put it behind rate limiting, quotas, or authentication before sharing it broadly.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createServer } from 'node:http';
|
|
3
|
+
|
|
4
|
+
const port = Number(process.env.PORT || 8787);
|
|
5
|
+
const nvidiaApiKey = process.env.NVIDIA_API_KEY?.trim();
|
|
6
|
+
const nvidiaBaseUrl = process.env.NVIDIA_BASE_URL?.trim() || 'https://integrate.api.nvidia.com/v1';
|
|
7
|
+
const proxyToken = process.env.SKILLCHECK_PROXY_TOKEN?.trim();
|
|
8
|
+
const maxBodyBytes = Number(process.env.MAX_BODY_BYTES || 5_000_000);
|
|
9
|
+
|
|
10
|
+
if (!nvidiaApiKey) {
|
|
11
|
+
throw new Error('Missing NVIDIA_API_KEY. Set it on the server, never inside the npm CLI.');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function writeJson(res, status, body) {
|
|
15
|
+
res.writeHead(status, {
|
|
16
|
+
'access-control-allow-headers': 'authorization, content-type',
|
|
17
|
+
'access-control-allow-methods': 'POST, OPTIONS',
|
|
18
|
+
'access-control-allow-origin': '*',
|
|
19
|
+
'content-type': 'application/json'
|
|
20
|
+
});
|
|
21
|
+
res.end(`${JSON.stringify(body)}\n`);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function readBody(req) {
|
|
25
|
+
return new Promise((resolve, reject) => {
|
|
26
|
+
const chunks = [];
|
|
27
|
+
let size = 0;
|
|
28
|
+
req.on('data', (chunk) => {
|
|
29
|
+
size += chunk.length;
|
|
30
|
+
if (size > maxBodyBytes) {
|
|
31
|
+
reject(new Error('Request body is too large'));
|
|
32
|
+
req.destroy();
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
chunks.push(chunk);
|
|
36
|
+
});
|
|
37
|
+
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
|
38
|
+
req.on('error', reject);
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function isAuthorized(req) {
|
|
43
|
+
if (!proxyToken) {
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
return req.headers.authorization === `Bearer ${proxyToken}`;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const server = createServer(async (req, res) => {
|
|
50
|
+
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
|
51
|
+
|
|
52
|
+
if (req.method === 'OPTIONS') {
|
|
53
|
+
writeJson(res, 204, {});
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (req.method === 'GET' && url.pathname === '/health') {
|
|
58
|
+
writeJson(res, 200, { ok: true });
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (req.method !== 'POST' || url.pathname !== '/v1/chat/completions') {
|
|
63
|
+
writeJson(res, 404, { error: 'Not found' });
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (!isAuthorized(req)) {
|
|
68
|
+
writeJson(res, 401, { error: 'Unauthorized' });
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
try {
|
|
73
|
+
const body = await readBody(req);
|
|
74
|
+
const upstream = await fetch(`${nvidiaBaseUrl}/chat/completions`, {
|
|
75
|
+
method: 'POST',
|
|
76
|
+
headers: {
|
|
77
|
+
authorization: `Bearer ${nvidiaApiKey}`,
|
|
78
|
+
'content-type': 'application/json'
|
|
79
|
+
},
|
|
80
|
+
body
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
res.writeHead(upstream.status, {
|
|
84
|
+
'access-control-allow-origin': '*',
|
|
85
|
+
'content-type': upstream.headers.get('content-type') || 'application/json'
|
|
86
|
+
});
|
|
87
|
+
res.end(await upstream.text());
|
|
88
|
+
} catch (error) {
|
|
89
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
90
|
+
writeJson(res, 500, { error: message });
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
server.listen(port, () => {
|
|
95
|
+
console.log(`skillcheck NVIDIA proxy listening on http://localhost:${port}`);
|
|
96
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sx4im/skillcheck",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Measure whether agent skills improve task performance.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -28,11 +28,13 @@
|
|
|
28
28
|
},
|
|
29
29
|
"files": [
|
|
30
30
|
"dist",
|
|
31
|
+
"docs",
|
|
32
|
+
"examples",
|
|
31
33
|
"README.md",
|
|
32
34
|
"METHODOLOGY.md"
|
|
33
35
|
],
|
|
34
36
|
"scripts": {
|
|
35
|
-
"build": "rm -rf dist && tsc -p tsconfig.json",
|
|
37
|
+
"build": "rm -rf dist && tsc -p tsconfig.json && chmod +x dist/bin/skillcheck.js",
|
|
36
38
|
"m0": "tsx packages/cli/bin/skillcheck.ts m0",
|
|
37
39
|
"site:build": "next build packages/site",
|
|
38
40
|
"test": "vitest run",
|