@sx4im/skillcheck 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/dist/src/adapters/nvidia-nim.d.ts +3 -1
- package/dist/src/adapters/nvidia-nim.js +7 -8
- package/dist/src/adapters/nvidia-nim.js.map +1 -1
- package/dist/src/cache.d.ts +9 -1
- package/dist/src/cache.js +15 -1
- package/dist/src/cache.js.map +1 -1
- package/dist/src/cli.js +30 -13
- package/dist/src/cli.js.map +1 -1
- package/dist/src/config.d.ts +15 -0
- package/dist/src/config.js +61 -0
- package/dist/src/config.js.map +1 -1
- package/dist/src/env.js +1 -1
- package/dist/src/env.js.map +1 -1
- package/dist/src/eval.js +5 -1
- package/dist/src/eval.js.map +1 -1
- package/dist/src/grade.js +5 -2
- package/dist/src/grade.js.map +1 -1
- package/dist/src/run.js +11 -2
- package/dist/src/run.js.map +1 -1
- package/dist/src/ui.d.ts +10 -1
- package/dist/src/ui.js +41 -4
- package/dist/src/ui.js.map +1 -1
- package/dist/src/verify.js +6 -2
- package/dist/src/verify.js.map +1 -1
- package/docs/skillcheck-cloud-build-plan.md +644 -0
- package/examples/dashboard/README.md +15 -0
- package/examples/dashboard/index.html +393 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -43,13 +43,18 @@ The quick run shows a blue/white result card in the terminal. It does not write
|
|
|
43
43
|
|
|
44
44
|
## Cloud Setup
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
The hosted option lets users run checks without any model-provider key of their own. Deploy the **`dashboard/`** folder on Vercel: users sign in with Google or GitHub (via Clerk), get a Skillcheck API key with 10 free runs, and the CLI proxies through your server-side NVIDIA key (after the free runs they upgrade via Stripe). Full click-by-click walkthrough in **`dashboard/SETUP.md`**.
|
|
47
|
+
|
|
48
|
+
Once deployed, users just run `skillcheck` and paste the API key from your dashboard when prompted — the hosted URL is built into the CLI, so there is no URL to configure. The key is verified before it is saved, then the file picker opens.
|
|
49
|
+
|
|
50
|
+
Power users can skip the prompt:
|
|
47
51
|
|
|
48
52
|
```bash
|
|
49
|
-
export
|
|
53
|
+
export SKILLCHECK_TOKEN=chk_live_... # from your dashboard (https://dashboard-skillcheck.vercel.app/app.html)
|
|
54
|
+
skillcheck check ./SKILL.md
|
|
50
55
|
```
|
|
51
56
|
|
|
52
|
-
|
|
57
|
+
Other references: `dashboard.md` is a drop-in single-file connect page for users who run their own proxy; `docs/skillcheck-cloud.md` documents the OpenAI-compatible API contract and a minimal proxy; `docs/skillcheck-cloud-build-plan.md` is the deeper multi-service architecture.
|
|
53
58
|
|
|
54
59
|
## Local Development
|
|
55
60
|
|
|
@@ -24,7 +24,9 @@ export declare class NvidiaNimClient {
|
|
|
24
24
|
private readonly requestDelayMs;
|
|
25
25
|
private readonly maxAttempts;
|
|
26
26
|
private readonly maxRetryDelayMs;
|
|
27
|
-
constructor(config: NvidiaConfig
|
|
27
|
+
constructor(config: NvidiaConfig, options?: {
|
|
28
|
+
defaultHeaders?: Record<string, string>;
|
|
29
|
+
});
|
|
28
30
|
private runSerializedRequest;
|
|
29
31
|
complete(request: CompletionRequest): Promise<CompletionResponse>;
|
|
30
32
|
}
|
|
@@ -78,7 +78,7 @@ export class NvidiaNimClient {
|
|
|
78
78
|
requestDelayMs;
|
|
79
79
|
maxAttempts;
|
|
80
80
|
maxRetryDelayMs;
|
|
81
|
-
constructor(config) {
|
|
81
|
+
constructor(config, options = {}) {
|
|
82
82
|
this.requestDelayMs = config.requestDelayMs;
|
|
83
83
|
this.maxAttempts = config.maxAttempts;
|
|
84
84
|
this.maxRetryDelayMs = config.maxRetryDelayMs;
|
|
@@ -86,7 +86,8 @@ export class NvidiaNimClient {
|
|
|
86
86
|
apiKey: config.apiKey,
|
|
87
87
|
baseURL: config.baseUrl,
|
|
88
88
|
maxRetries: 0,
|
|
89
|
-
timeout: config.timeoutMs
|
|
89
|
+
timeout: config.timeoutMs,
|
|
90
|
+
defaultHeaders: options.defaultHeaders
|
|
90
91
|
});
|
|
91
92
|
}
|
|
92
93
|
async runSerializedRequest(operation) {
|
|
@@ -115,11 +116,10 @@ export class NvidiaNimClient {
|
|
|
115
116
|
let lastError;
|
|
116
117
|
for (let attempt = 0; attempt < this.maxAttempts; attempt += 1) {
|
|
117
118
|
try {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
: undefined;
|
|
119
|
+
// NOTE: do not send `extra_body`. It is a Python-SDK convenience field, not a
|
|
120
|
+
// real OpenAI/NVIDIA parameter — NVIDIA NIM rejects it with
|
|
121
|
+
// "400 Validation: Unsupported parameter(s): `extra_body`". chat_template_kwargs
|
|
122
|
+
// is passed as a real top-level field, which NIM accepts.
|
|
123
123
|
const requestBody = {
|
|
124
124
|
model: request.model,
|
|
125
125
|
messages: request.messages,
|
|
@@ -127,7 +127,6 @@ export class NvidiaNimClient {
|
|
|
127
127
|
max_tokens: request.maxTokens,
|
|
128
128
|
response_format: request.responseFormat ? { type: request.responseFormat } : undefined,
|
|
129
129
|
chat_template_kwargs: request.chatTemplateKwargs,
|
|
130
|
-
extra_body: extraBody,
|
|
131
130
|
stream: false
|
|
132
131
|
};
|
|
133
132
|
const response = (await this.runSerializedRequest(() => this.client.chat.completions.create(requestBody)));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nvidia-nim.js","sourceRoot":"","sources":["../../../packages/cli/src/adapters/nvidia-nim.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAuB5B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAC;gBAC/C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAgF,CAAC;IAChG,OAAO,CACL,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;QAClC,kBAAkB,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACrE,MAAM,MAAM,GAAI,KAA8B,CAAC,MAAM,CAAC;QACtD,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,MAAM,UAAU,GACd,OAAO,YAAY,OAAO;QACxB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC5B,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,aAAa,IAAI,OAAO;YAC3E,CAAC,CAAE,OAAuC,CAAC,aAAa,CAAC;YACzD,CAAC,CAAC,SAAS,CAAC;IAElB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,MAAc,EAAE,WAAmB;IACzF,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG,EAAE,CAAC;QACjD,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,GAAG,CAAC,IAAI,WAAW,UAAU,KAAK,aAAa,MAAM,KAAK,CAAC,CAAC;AACzG,CAAC;AAED,MAAM,OAAO,eAAe;IAClB,MAAM,CAAC,YAAY,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IACvD,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;IAEhB,MAAM,CAAS;IACf,cAAc,CAAS;IACvB,WAAW,CAAS;IACpB,eAAe,CAAS;IAEzC,YAAY,MAAoB;
|
|
1
|
+
{"version":3,"file":"nvidia-nim.js","sourceRoot":"","sources":["../../../packages/cli/src/adapters/nvidia-nim.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAuB5B,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExE,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO;aAClB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAI,IAA2B,CAAC,IAAI,CAAC;gBAC/C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,MAAM,GAAG,OAAgF,CAAC;IAChG,OAAO,CACL,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC;QAClC,kBAAkB,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,QAAQ,IAAI,KAAK,EAAE,CAAC;QACrE,MAAM,MAAM,GAAI,KAA8B,CAAC,MAAM,CAAC;QACtD,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAI,KAA+B,CAAC,OAAO,CAAC;IACzD,MAAM,UAAU,GACd,OAAO,YAAY,OAAO;QACxB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC5B,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,aAAa,IAAI,OAAO;YAC3E,CAAC,CAAE,OAAuC,CAAC,aAAa,CAAC;YACzD,CAAC,CAAC,SAAS,CAAC;IAElB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,MAAc,EAAE,WAAmB;IACzF,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG,EAAE,CAAC;QACjD,OAAO;IACT,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC;IACvE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,GAAG,CAAC,IAAI,WAAW,UAAU,KAAK,aAAa,MAAM,KAAK,CAAC,CAAC;AACzG,CAAC;AAED,MAAM,OAAO,eAAe;IAClB,MAAM,CAAC,YAAY,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IACvD,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;IAEhB,MAAM,CAAS;IACf,cAAc,CAAS;IACvB,WAAW,CAAS;IACpB,eAAe,CAAS;IAEzC,YAAY,MAAoB,EAAE,UAAuD,EAAE;QACzF,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAI,SAA2B;QAC/D,MAAM,eAAe,GAAG,eAAe,CAAC,YAAY,CAAC;QACrD,IAAI,qBAAqB,GAAe,GAAG,EAAE,CAAC,SAAS,CAAC;QACxD,eAAe,CAAC,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC3D,qBAAqB,GAAG,OAAO,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,CAAC;QACtB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,aAAa,CAAC;gBAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;gBAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,eAAe,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3C,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,qBAAqB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,OAA0B;QACvC,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,8EAA8E;gBAC9E,4DAA4D;gBAC5D,iFAAiF;gBACjF,0DAA0D;gBAC1D,MAAM,WAAW,GAAG;oBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,UAAU,EAAE,OAAO,CAAC,SAAS;oBAC7B,eAAe,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS;oBACtF,oBAAoB,EAAE,OAAO,CAAC,kBAAkB;oBAChD,MAAM,EAAE,KAAK;iBACd,CAAC;gBACF,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,CACjD,CAAmB,CAAC;gBAErB,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC7C,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAC5C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;oBACtE,MAAM,IAAI,KAAK,CAAC,yEAAyE,IAAI,EAAE,CAAC,CAAC;gBACnG,CAAC;gBAED,OAAO;oBACL,OAAO;oBACP,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,KAAK,EAAE;wBACL,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa;wBAC3C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,iBAAiB;wBACnD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY;qBAC1C;iBACF,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACzE,IAAI,CAAC,SAAS,IAAI,OAAO,KAAK,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;oBACnD,MAAM;gBACR,CAAC;gBAED,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;gBAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;gBAC/C,MAAM,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,WAAW,GAAG,CAAC,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC;gBACnG,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxD,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC9E,CAAC"}
|
package/dist/src/cache.d.ts
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
export declare class JsonCache {
|
|
2
2
|
private readonly rootDir;
|
|
3
|
-
|
|
3
|
+
private readonly enabled;
|
|
4
|
+
constructor(rootDir?: string, enabled?: boolean);
|
|
5
|
+
/**
|
|
6
|
+
* A cache that never reads or writes: every `getOrSet` calls its factory.
|
|
7
|
+
* Used by `verify`, which must independently re-query the model rather than
|
|
8
|
+
* replay the original run's cached outputs (otherwise verification is a
|
|
9
|
+
* tautology on the machine that produced the result).
|
|
10
|
+
*/
|
|
11
|
+
static disabled(): JsonCache;
|
|
4
12
|
getOrSet<T>(namespace: string, keyParts: unknown, factory: () => Promise<T>): Promise<T>;
|
|
5
13
|
}
|
package/dist/src/cache.js
CHANGED
|
@@ -3,10 +3,24 @@ import path from 'node:path';
|
|
|
3
3
|
import { hashJson } from './hash.js';
|
|
4
4
|
export class JsonCache {
|
|
5
5
|
rootDir;
|
|
6
|
-
|
|
6
|
+
enabled;
|
|
7
|
+
constructor(rootDir = '.cache/skillcheck', enabled = true) {
|
|
7
8
|
this.rootDir = rootDir;
|
|
9
|
+
this.enabled = enabled;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* A cache that never reads or writes: every `getOrSet` calls its factory.
|
|
13
|
+
* Used by `verify`, which must independently re-query the model rather than
|
|
14
|
+
* replay the original run's cached outputs (otherwise verification is a
|
|
15
|
+
* tautology on the machine that produced the result).
|
|
16
|
+
*/
|
|
17
|
+
static disabled() {
|
|
18
|
+
return new JsonCache(undefined, false);
|
|
8
19
|
}
|
|
9
20
|
async getOrSet(namespace, keyParts, factory) {
|
|
21
|
+
if (!this.enabled) {
|
|
22
|
+
return factory();
|
|
23
|
+
}
|
|
10
24
|
const key = hashJson(keyParts);
|
|
11
25
|
const dir = path.join(this.rootDir, namespace);
|
|
12
26
|
const file = path.join(dir, `${key}.json`);
|
package/dist/src/cache.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../packages/cli/src/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,MAAM,OAAO,SAAS;
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../packages/cli/src/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,MAAM,OAAO,SAAS;IAED;IACA;IAFnB,YACmB,UAAU,mBAAmB,EAC7B,UAAU,IAAI;QADd,YAAO,GAAP,OAAO,CAAsB;QAC7B,YAAO,GAAP,OAAO,CAAO;IAC9B,CAAC;IAEJ;;;;;OAKG;IACH,MAAM,CAAC,QAAQ;QACb,OAAO,IAAI,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAI,SAAiB,EAAE,QAAiB,EAAE,OAAyB;QAC/E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,OAAO,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;QAE3C,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAM,CAAC;QACvD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,OAAO,EAAE,CAAC;QAC9B,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACtD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
package/dist/src/cli.js
CHANGED
|
@@ -1,36 +1,53 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { NvidiaNimClient } from './adapters/nvidia-nim.js';
|
|
3
|
-
import {
|
|
3
|
+
import { cloudApiUrl, cloudWebUrl, getConfiguredToken, loadUserConfig, saveUserConfig, verifyCloudKey } from './config.js';
|
|
4
4
|
import { runCorpus } from './corpus.js';
|
|
5
5
|
import { evalSkill } from './eval.js';
|
|
6
6
|
import { runM0Gate } from './m0/run.js';
|
|
7
7
|
import { runRot } from './rot.js';
|
|
8
|
-
import { formatResultCard, printHelpUi, printBanner, printSetupIntro, promptText, selectSkillPath, startProgress, validateSkillInput } from './ui.js';
|
|
8
|
+
import { formatResultCard, printHelpUi, printBanner, printSetupIntro, printKeyChecking, printKeyVerified, printKeyRejected, printKeyUnreachable, printKeyPromptHint, promptText, selectSkillPath, startProgress, validateSkillInput } from './ui.js';
|
|
9
9
|
import { verifyResult } from './verify.js';
|
|
10
10
|
function printHelp() {
|
|
11
11
|
printHelpUi();
|
|
12
12
|
}
|
|
13
|
+
// First-run onboarding. The CLI demands a Skillcheck Cloud API key, points the
|
|
14
|
+
// user at the website to grab one, authenticates the pasted key against the
|
|
15
|
+
// hosted /key/verify endpoint, and only saves it once it is confirmed valid.
|
|
16
|
+
// The hosted URL is baked in (config.cloudApiUrl), so the user never types a URL.
|
|
13
17
|
async function ensureCloudConfigured(force = false) {
|
|
14
|
-
|
|
18
|
+
// Maintainer/dev mode: a direct NVIDIA key bypasses the hosted proxy entirely.
|
|
19
|
+
if (process.env.NVIDIA_API_KEY?.trim()) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
if (!force && getConfiguredToken()) {
|
|
15
23
|
return;
|
|
16
24
|
}
|
|
17
25
|
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
|
18
|
-
throw new Error(
|
|
26
|
+
throw new Error(`Skillcheck needs an API key. Run \`skillcheck setup\` in an interactive terminal, or set SKILLCHECK_TOKEN. Get a key at ${cloudWebUrl()}`);
|
|
19
27
|
}
|
|
28
|
+
const apiUrl = cloudApiUrl();
|
|
29
|
+
const webUrl = cloudWebUrl();
|
|
20
30
|
printBanner();
|
|
21
|
-
printSetupIntro();
|
|
31
|
+
printSetupIntro(webUrl);
|
|
22
32
|
for (;;) {
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
33
|
+
const keyInput = await promptText('Paste your Skillcheck API key: ');
|
|
34
|
+
if (!keyInput) {
|
|
35
|
+
printKeyPromptHint(webUrl);
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
printKeyChecking();
|
|
39
|
+
const check = await verifyCloudKey(apiUrl, keyInput);
|
|
40
|
+
if (check.valid) {
|
|
26
41
|
const current = loadUserConfig();
|
|
27
|
-
const
|
|
28
|
-
|
|
42
|
+
const savedPath = saveUserConfig({ ...current, apiUrl, token: keyInput });
|
|
43
|
+
printKeyVerified(check, savedPath);
|
|
29
44
|
return;
|
|
30
45
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
46
|
+
if (check.reachable) {
|
|
47
|
+
printKeyRejected(check.message ?? 'The key was not accepted.', webUrl);
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
printKeyUnreachable(check.message ?? 'Network error.');
|
|
34
51
|
}
|
|
35
52
|
}
|
|
36
53
|
}
|
package/dist/src/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../packages/cli/src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../packages/cli/src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EACL,WAAW,EACX,WAAW,EACX,kBAAkB,EAClB,cAAc,EACd,cAAc,EACd,cAAc,EACf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,SAAS,EAAyB,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAoB,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,MAAM,EAAmB,MAAM,UAAU,CAAC;AACnD,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,kBAAkB,EAClB,UAAU,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,EACnB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,SAAS;IAChB,WAAW,EAAE,CAAC;AAChB,CAAC;AAED,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAC7E,kFAAkF;AAClF,KAAK,UAAU,qBAAqB,CAAC,KAAK,GAAG,KAAK;IAChD,+EAA+E;IAC/E,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,EAAE,EAAE,CAAC;QACvC,OAAO;IACT,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,kBAAkB,EAAE,EAAE,CAAC;QACnC,OAAO;IACT,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CACb,2HAA2H,WAAW,EAAE,EAAE,CAC3I,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAE7B,WAAW,EAAE,CAAC;IACd,eAAe,CAAC,MAAM,CAAC,CAAC;IAExB,SAAS,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,iCAAiC,CAAC,CAAC;QACrE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,gBAAgB,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,cAAc,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC1E,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YACnC,OAAO;QACT,CAAC;QAED,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACpB,gBAAgB,CAAC,KAAK,CAAC,OAAO,IAAI,2BAA2B,EAAE,MAAM,CAAC,CAAC;QACzE,CAAC;aAAM,CAAC;YACN,mBAAmB,CAAC,KAAK,CAAC,OAAO,IAAI,gBAAgB,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,IAAc,EAAE,IAAY;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,IAAY,EAAE,QAAgB;IACtE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAc,EAAE,IAAY;IAC5D,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,IAAc,EAAE,IAAY;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,UAAU,GAAG,CAAC;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAC;IACpD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,SAAS;QACT,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7C,IAAI;QACJ,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC;QAC1C,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC;KAC5C,CAAC;AACJ,CAAC;AAQD,MAAM,UAAU,iBAAiB,CAAC,IAAc,EAAE,UAAU,GAAG,CAAC;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CACb,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC5C,OAAO;QACL,WAAW,EAAE;YACX,SAAS;YACT,MAAM;YACN,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAC3C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAC7C,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;YACpC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;YACpC,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC;YAC1C,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC;YAC3C,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC;SAC/B;QACD,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;QAC7B,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,IAAc;IACrC,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,SAAS;QACtD,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,yBAAyB;QACjE,KAAK,EAAE,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC;QAClC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAc;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,OAAO;QACL,MAAM;QACN,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,gBAAgB;QAC5D,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7C,WAAW,EAAE,gBAAgB,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;QACvD,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,wBAAwB,CAAC,IAAI,EAAE,SAAS,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAAqB,EAAE,UAAU,GAAG,IAAI;IAC9D,IAAI,UAAU,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChC,WAAW,EAAE,CAAC;IAChB,CAAC;IACD,MAAM,kBAAkB,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,qBAAqB,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;IAC5D,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,EAAE,IAAI,EAAE,CAAC;QACjB,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,KAAK,UAAU,mBAAmB;IAChC,MAAM,qBAAqB,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,YAAY,GAAG,MAAM,eAAe,EAAE,CAAC;IAC7C,MAAM,QAAQ,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC1F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAc;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAExB,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QAC7C,SAAS,EAAE,CAAC;QACZ,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,mBAAmB,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,OAAO,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,MAAM,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;QACxC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,UAAU;YACV,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;SAC9C,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,MAAM,QAAQ,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,OAAO;IACT,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;AACjD,CAAC"}
|
package/dist/src/config.d.ts
CHANGED
|
@@ -2,6 +2,21 @@ export interface SkillcheckUserConfig {
|
|
|
2
2
|
apiUrl?: string;
|
|
3
3
|
token?: string;
|
|
4
4
|
}
|
|
5
|
+
export declare const DEFAULT_CLOUD_API_URL = "https://dashboard-skillcheck.vercel.app/api";
|
|
6
|
+
export declare const DEFAULT_CLOUD_WEB_URL = "https://dashboard-skillcheck.vercel.app/app.html";
|
|
7
|
+
export declare function cloudApiUrl(): string;
|
|
8
|
+
export declare function cloudWebUrl(): string;
|
|
9
|
+
export interface CloudKeyVerification {
|
|
10
|
+
valid: boolean;
|
|
11
|
+
plan?: string;
|
|
12
|
+
email?: string;
|
|
13
|
+
runsUsed?: number;
|
|
14
|
+
runsLimit?: number | null;
|
|
15
|
+
/** Distinguishes a rejected key (reachable, said no) from an unreachable cloud. */
|
|
16
|
+
reachable: boolean;
|
|
17
|
+
message?: string;
|
|
18
|
+
}
|
|
19
|
+
export declare function verifyCloudKey(apiUrl: string, token: string): Promise<CloudKeyVerification>;
|
|
5
20
|
export declare function userConfigPath(): string;
|
|
6
21
|
export declare function normalizeApiUrl(value: string): string;
|
|
7
22
|
export declare function loadUserConfig(): SkillcheckUserConfig;
|
package/dist/src/config.js
CHANGED
|
@@ -1,6 +1,67 @@
|
|
|
1
1
|
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { homedir } from 'node:os';
|
|
3
3
|
import path from 'node:path';
|
|
4
|
+
// Skillcheck Cloud's hosted endpoints. The CLI ships with these baked in so a
|
|
5
|
+
// user only ever needs to paste their API key — never a URL. Both are
|
|
6
|
+
// overridable via env for self-hosted deployments.
|
|
7
|
+
export const DEFAULT_CLOUD_API_URL = 'https://dashboard-skillcheck.vercel.app/api';
|
|
8
|
+
export const DEFAULT_CLOUD_WEB_URL = 'https://dashboard-skillcheck.vercel.app/app.html';
|
|
9
|
+
export function cloudApiUrl() {
|
|
10
|
+
return getConfiguredApiUrl() ?? DEFAULT_CLOUD_API_URL;
|
|
11
|
+
}
|
|
12
|
+
export function cloudWebUrl() {
|
|
13
|
+
return process.env.SKILLCHECK_WEB_URL?.trim() || DEFAULT_CLOUD_WEB_URL;
|
|
14
|
+
}
|
|
15
|
+
// Authenticate a pasted key against the hosted /key/verify endpoint. This does
|
|
16
|
+
// NOT consume a run. `reachable:false` means we never got a verdict (network
|
|
17
|
+
// down), so callers can treat it differently from an outright rejection.
|
|
18
|
+
export async function verifyCloudKey(apiUrl, token) {
|
|
19
|
+
const base = apiUrl.trim().replace(/\/+$/, '');
|
|
20
|
+
let response;
|
|
21
|
+
try {
|
|
22
|
+
response = await fetch(`${base}/key/verify`, {
|
|
23
|
+
method: 'POST',
|
|
24
|
+
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
|
25
|
+
body: '{}'
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
return { valid: false, reachable: false, message: error instanceof Error ? error.message : String(error) };
|
|
30
|
+
}
|
|
31
|
+
let data = {};
|
|
32
|
+
try {
|
|
33
|
+
data = (await response.json());
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
data = {};
|
|
37
|
+
}
|
|
38
|
+
if (response.ok && data.valid === true) {
|
|
39
|
+
const limit = data.runsLimit;
|
|
40
|
+
return {
|
|
41
|
+
valid: true,
|
|
42
|
+
reachable: true,
|
|
43
|
+
plan: typeof data.plan === 'string' ? data.plan : undefined,
|
|
44
|
+
email: typeof data.email === 'string' ? data.email : undefined,
|
|
45
|
+
runsUsed: typeof data.runsUsed === 'number' ? data.runsUsed : undefined,
|
|
46
|
+
runsLimit: typeof limit === 'number' ? limit : null
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
// The key-check endpoint is missing — almost always an older deployment that
|
|
50
|
+
// predates /api/key/verify. Don't reject a possibly-valid key; flag it as
|
|
51
|
+
// "unavailable" so the user gets an actionable message instead of "rejected".
|
|
52
|
+
if (response.status === 404 || response.status === 405) {
|
|
53
|
+
return {
|
|
54
|
+
valid: false,
|
|
55
|
+
reachable: false,
|
|
56
|
+
message: 'Skillcheck Cloud is missing the key-check endpoint (redeploy the dashboard so /api/key/verify ships).'
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
const errorRecord = data.error ?? {};
|
|
60
|
+
const message = typeof errorRecord.message === 'string'
|
|
61
|
+
? errorRecord.message
|
|
62
|
+
: `The key was not accepted (HTTP ${response.status}).`;
|
|
63
|
+
return { valid: false, reachable: true, message };
|
|
64
|
+
}
|
|
4
65
|
function configDir() {
|
|
5
66
|
if (process.env.SKILLCHECK_CONFIG_DIR?.trim()) {
|
|
6
67
|
return process.env.SKILLCHECK_CONFIG_DIR.trim();
|
package/dist/src/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../packages/cli/src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAO7B,SAAS,SAAS;IAChB,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,IAAI,EAAE,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,EAAE,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,YAAY,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,aAAa,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACjD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;QAChD,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC;IACvB,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,EAAE,MAAM,CAAC,CAAY,CAAC;QAC7E,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YAClD,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,MAAM,GAAG,MAAiC,CAAC;QACjD,OAAO;YACL,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YACrE,KAAK,EAAE,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SACnE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAA4B;IACzD,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;IAClC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACjF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,cAAc,EAAE,CAAC,MAAM,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,OAAO,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,cAAc,EAAE,CAAC,KAAK,CAAC;AAClH,CAAC"}
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../packages/cli/src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAO7B,8EAA8E;AAC9E,sEAAsE;AACtE,mDAAmD;AACnD,MAAM,CAAC,MAAM,qBAAqB,GAAG,6CAA6C,CAAC;AACnF,MAAM,CAAC,MAAM,qBAAqB,GAAG,kDAAkD,CAAC;AAExF,MAAM,UAAU,WAAW;IACzB,OAAO,mBAAmB,EAAE,IAAI,qBAAqB,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,qBAAqB,CAAC;AACzE,CAAC;AAaD,+EAA+E;AAC/E,6EAA6E;AAC7E,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAc,EAAE,KAAa;IAChE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/C,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,aAAa,EAAE;YAC3C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YACjF,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAC7G,CAAC;IAED,IAAI,IAAI,GAA4B,EAAE,CAAC;IACvC,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,GAAG,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE,IAAI;YACX,SAAS,EAAE,IAAI;YACf,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC3D,KAAK,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YAC9D,QAAQ,EAAE,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YACvE,SAAS,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;SACpD,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,0EAA0E;IAC1E,8EAA8E;IAC9E,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACvD,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,uGAAuG;SACjH,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAI,IAAI,CAAC,KAA6C,IAAI,EAAE,CAAC;IAC9E,MAAM,OAAO,GACX,OAAO,WAAW,CAAC,OAAO,KAAK,QAAQ;QACrC,CAAC,CAAC,WAAW,CAAC,OAAO;QACrB,CAAC,CAAC,kCAAkC,QAAQ,CAAC,MAAM,IAAI,CAAC;IAC5D,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,SAAS;IAChB,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,IAAI,EAAE,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,EAAE,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,YAAY,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,aAAa,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACjD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;QAChD,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC;IACvB,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,EAAE,MAAM,CAAC,CAAY,CAAC;QAC7E,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YAClD,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,MAAM,GAAG,MAAiC,CAAC;QACjD,OAAO;YACL,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YACrE,KAAK,EAAE,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SACnE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAA4B;IACzD,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;IAClC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACjF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,cAAc,EAAE,CAAC,MAAM,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,OAAO,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,EAAE,IAAI,cAAc,EAAE,CAAC,KAAK,CAAC;AAClH,CAAC"}
|
package/dist/src/env.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import dotenv from 'dotenv';
|
|
2
2
|
import { getConfiguredApiUrl, getConfiguredToken } from './config.js';
|
|
3
3
|
dotenv.config();
|
|
4
|
-
const DEFAULT_MODEL = '
|
|
4
|
+
const DEFAULT_MODEL = 'minimaxai/minimax-m2.7';
|
|
5
5
|
function readEnv(name) {
|
|
6
6
|
const value = process.env[name]?.trim();
|
|
7
7
|
return value || undefined;
|
package/dist/src/env.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../packages/cli/src/env.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtE,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,aAAa,GAAG,
|
|
1
|
+
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../packages/cli/src/env.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtE,MAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,aAAa,GAAG,wBAAwB,CAAC;AAc/C,SAAS,OAAO,CAAC,IAAY;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;IACxC,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED,SAAS,aAAa,CAAC,QAA4B;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC/C,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,WAAW,GAAG,kBAAkB,EAAE,CAAC;IACzC,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,WAAW,IAAI,kBAAkB,CAAC;IAC3C,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,kGAAkG,CAAC,CAAC;AACtH,CAAC;AAED,SAAS,YAAY,CAAC,IAAuC;IAC3D,OAAO,CACL,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC;QAC/B,OAAO,CAAC,cAAc,IAAI,QAAQ,CAAC;QACnC,OAAO,CAAC,kBAAkB,CAAC;QAC3B,aAAa,CACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,CAAC;IAC1E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;IACnF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,CAAC;IACvF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,QAAQ,GAAG,mBAAmB,EAAE,CAAC;IAEvC,OAAO;QACL,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC;QAC/B,OAAO,EAAE,QAAQ,IAAI,OAAO,CAAC,iBAAiB,CAAC,IAAI,qCAAqC;QACxF,SAAS;QACT,cAAc;QACd,WAAW;QACX,eAAe;QACf,cAAc,EAAE,YAAY,CAAC,WAAW,CAAC;QACzC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC;QACnC,WAAW,EAAE,YAAY,CAAC,QAAQ,CAAC;KACpC,CAAC;AACJ,CAAC"}
|
package/dist/src/eval.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
1
2
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
3
|
import path from 'node:path';
|
|
3
4
|
import { NvidiaNimClient } from './adapters/nvidia-nim.js';
|
|
@@ -87,7 +88,10 @@ export async function evalSkill(options) {
|
|
|
87
88
|
const skill = await normalizeSkill(options.inputPath);
|
|
88
89
|
const baseConfig = loadNvidiaConfig();
|
|
89
90
|
const config = applyModelOverrides(baseConfig, options);
|
|
90
|
-
|
|
91
|
+
// One run id per evalSkill call. The hosted proxy meters by distinct run id, so
|
|
92
|
+
// every model call in this run shares it and the whole check counts as one run.
|
|
93
|
+
const runId = randomUUID();
|
|
94
|
+
const client = new NvidiaNimClient(config, { defaultHeaders: { 'x-skillcheck-run': runId } });
|
|
91
95
|
const cache = new JsonCache();
|
|
92
96
|
const tasks = options.taskSuite
|
|
93
97
|
? parseTaskSuite(await readFile(options.taskSuite, 'utf8')).slice(0, options.tasks)
|
package/dist/src/eval.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../packages/cli/src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAqB,MAAM,UAAU,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,YAAY,CAAC;AAiB7E,SAAS,mBAAmB,CAAC,MAAoB,EAAE,OAAoB;IACrE,OAAO;QACL,GAAG,MAAM;QACT,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,cAAc,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAsB;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC;YAC9B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAyB,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAsB,EAAE,MAAsB;IACpE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QACrE,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,cAAc,EAAE,IAAI,CAAC,aAAa;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,eAAe,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;YAChF,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;SAC7E,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACjG,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,IAAI,QAAQ,CAAC;QAC5E,IAAI,aAAa,KAAK,QAAQ,IAAI,aAAa,KAAK,eAAe,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,aAAa;YACb,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;SACxC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAoB;IAClD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../packages/cli/src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAqB,MAAM,UAAU,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,YAAY,CAAC;AAiB7E,SAAS,mBAAmB,CAAC,MAAoB,EAAE,OAAoB;IACrE,OAAO;QACL,GAAG,MAAM;QACT,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,cAAc,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAsB;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC;YAC9B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAyB,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAsB,EAAE,MAAsB;IACpE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QACrE,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,cAAc,EAAE,IAAI,CAAC,aAAa;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,eAAe,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;YAChF,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;SAC7E,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACjG,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,IAAI,QAAQ,CAAC;QAC5E,IAAI,aAAa,KAAK,QAAQ,IAAI,aAAa,KAAK,eAAe,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,aAAa;YACb,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;SACxC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAoB;IAClD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,gFAAgF;IAChF,gFAAgF;IAChF,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IAC9F,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAC;IAE9B,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS;QAC7B,CAAC,CAAC,cAAc,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;QACnF,CAAC,CAAC,MAAM,aAAa,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC/F,MAAM,aAAa,GAAG,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,MAAM,mBAAmB,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;IAC1D,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,iBAAiB,aAAa,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACrH,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,aAAa,EAAE,CAAC;QACxC,MAAM,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,uBAAuB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACjD,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5G,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACxG,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,gBAAgB,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChH,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEtD,MAAM,MAAM,GAAG;QACb,KAAK,EAAE;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,UAAU;YAC/C,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB;QACD,MAAM,EAAE;YACN,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,cAAc,EAAE,MAAM,CAAC,WAAW;YAClC,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,cAAc,EAAE,MAAM,CAAC,WAAW;YAClC,eAAe,EAAE,MAAM,CAAC,cAAc;YACtC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,GAAG;YAChB,IAAI,EAAE,kBAAkB;SACzB;QACD,MAAM,EAAE;YACN,SAAS,EAAE,KAAK,CAAC,QAAQ;YACzB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,eAAe,EAAE,KAAK,CAAC,aAAa;YACpC,aAAa,EAAE,KAAK,CAAC,WAAW;YAChC,cAAc,EAAE,aAAa;YAC7B,mBAAmB,EAAE,gBAAgB;SACtC;QACD,KAAK,EAAE,UAAU;QACjB,eAAe,EAAE;YACf,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC;SAC7D;QACD,OAAO,EAAE;YACP;gBACE,cAAc,EAAE,MAAM,CAAC,WAAW;gBAClC,QAAQ,EAAE,OAAO;gBACjB,SAAS,EAAE,KAAK,CAAC,QAAQ;gBACzB,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB;SACF;QACD,QAAQ,EAAE,OAAO;KAClB,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/src/grade.js
CHANGED
|
@@ -65,13 +65,16 @@ export async function gradeOutputs(tasks, outputs, config, client, cache) {
|
|
|
65
65
|
model: config.graderModel,
|
|
66
66
|
criterion: task.criterion,
|
|
67
67
|
output: output.output,
|
|
68
|
-
promptVersion:
|
|
68
|
+
promptVersion: 7,
|
|
69
69
|
responseFormat: 'json_object',
|
|
70
70
|
attempt
|
|
71
71
|
}, () => client.complete({
|
|
72
72
|
model: config.graderModel,
|
|
73
73
|
temperature: 0,
|
|
74
|
-
|
|
74
|
+
// Reasoning models (e.g. minimax-m2.7) always emit ~500-600 reasoning
|
|
75
|
+
// tokens before the answer; a tight budget leaves `content` empty. Give
|
|
76
|
+
// the JSON verdict room to land after the reasoning.
|
|
77
|
+
maxTokens: 1536,
|
|
75
78
|
responseFormat: 'json_object',
|
|
76
79
|
chatTemplateKwargs: { thinking: false },
|
|
77
80
|
messages: [
|
package/dist/src/grade.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IAEnD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;gBAChC,GAAG,MAAM;gBACT,GAAG,KAAK;aACT,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,KAA+B,CAAC;QACpC,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,QAAQ,EACR;gBACE,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,CAAC;gBAChB,cAAc,EAAE,aAAa;gBAC7B,OAAO;aACR,EACD,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,
|
|
1
|
+
{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IAEnD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;gBAChC,GAAG,MAAM;gBACT,GAAG,KAAK;aACT,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,CAAC,KAAK,CAAC,sBAAsB,MAAM,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,KAA+B,CAAC;QACpC,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,QAAQ,EACR;gBACE,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,CAAC;gBAChB,cAAc,EAAE,aAAa;gBAC7B,OAAO;aACR,EACD,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,WAAW,EAAE,CAAC;gBACd,sEAAsE;gBACtE,wEAAwE;gBACxE,qDAAqD;gBACrD,SAAS,EAAE,IAAI;gBACf,cAAc,EAAE,aAAa;gBAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACvC,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EACL,qMAAqM;qBACxM;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,uBAAuB,IAAI,CAAC,SAAS,yBAAyB,MAAM,CAAC,MAAM,0MAA0M;qBAC/R;iBACF;aACF,CAAC,CACL,CAAC;YAEF,IAAI,CAAC;gBACH,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACrC,MAAM;YACR,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,KAAK,CAAC,wDAAwD,OAAO,IAAI,CAAC,CAAC;gBACrF,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAChH,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;YAChC,GAAG,MAAM;YACT,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,GAAG;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/src/run.js
CHANGED
|
@@ -13,11 +13,20 @@ function messagesForArm(skill, task, withSkill) {
|
|
|
13
13
|
}
|
|
14
14
|
async function runOne(skill, task, trial, arm, config, client, cache) {
|
|
15
15
|
const messages = messagesForArm(skill, task, arm === 'with_skill');
|
|
16
|
-
|
|
16
|
+
// `trial` MUST be part of the cache key. Without it, every trial for the same
|
|
17
|
+
// (skill, task, arm) produces identical `messages` and therefore the same key,
|
|
18
|
+
// so trials 2..K silently read trial 1's cached output. That collapses K
|
|
19
|
+
// independent stochastic samples (temperature 0.7) into one replicated K times
|
|
20
|
+
// and makes the paired-bootstrap CI ~sqrt(K) too narrow (pseudo-replication).
|
|
21
|
+
// Reasoning models (e.g. minimax-m2.7) spend ~500-600 tokens thinking before the
|
|
22
|
+
// answer, so the budget must cover reasoning + a full response or the answer is
|
|
23
|
+
// truncated. maxTokens is part of the cache key, so this also invalidates any
|
|
24
|
+
// answers captured under the old tighter budget.
|
|
25
|
+
const response = await cache.getOrSet('runner', { model: config.runnerModel, temperature: 0.7, maxTokens: 2048, promptVersion: 1, trial, messages }, () => client.complete({
|
|
17
26
|
model: config.runnerModel,
|
|
18
27
|
messages,
|
|
19
28
|
temperature: 0.7,
|
|
20
|
-
maxTokens:
|
|
29
|
+
maxTokens: 2048
|
|
21
30
|
}));
|
|
22
31
|
const transcriptHash = `sha256:${hashJson({ taskId: task.id, trial, arm, messages, output: response.content })}`;
|
|
23
32
|
return {
|
package/dist/src/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,8EAA8E;IAC9E,+EAA+E;IAC/E,yEAAyE;IACzE,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,gFAAgF;IAChF,8EAA8E;IAC9E,iDAAiD;IACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CACxJ,MAAM,CAAC,QAAQ,CAAC;QACd,KAAK,EAAE,MAAM,CAAC,WAAW;QACzB,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,IAAI;KAChB,CAAC,CACH,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAEjH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,IAAI,CAAC;QACvD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;QAC7C,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAsB,EACtB,KAAsB,EACtB,MAAc,EACd,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChD,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,aAAa,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,WAAW,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/src/ui.d.ts
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
export declare function printBanner(): void;
|
|
2
2
|
export declare function printHelpUi(): void;
|
|
3
3
|
export declare function promptText(question: string): Promise<string>;
|
|
4
|
-
export declare function printSetupIntro(): void;
|
|
4
|
+
export declare function printSetupIntro(webUrl: string): void;
|
|
5
|
+
export declare function printKeyChecking(): void;
|
|
6
|
+
export declare function printKeyVerified(info: {
|
|
7
|
+
plan?: string;
|
|
8
|
+
runsUsed?: number;
|
|
9
|
+
runsLimit?: number | null;
|
|
10
|
+
}, savedPath: string): void;
|
|
11
|
+
export declare function printKeyRejected(message: string, webUrl: string): void;
|
|
12
|
+
export declare function printKeyUnreachable(message: string): void;
|
|
13
|
+
export declare function printKeyPromptHint(webUrl: string): void;
|
|
5
14
|
export declare function supportedSkillFilesText(): string;
|
|
6
15
|
export declare function validateSkillInput(inputPath: string): Promise<string>;
|
|
7
16
|
export declare function selectSkillPath(startDir?: string): Promise<string>;
|