@cliwatch/cli-bench 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +6 -7
  2. package/dist/index.js +1 -1
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -79,7 +79,9 @@ Each referenced file is a plain array of tasks:
79
79
  | `concurrency` | No | Max concurrent API calls (default: 3) |
80
80
  | `workdir` | No | Working directory (default: temp dir per task) |
81
81
  | `upload` | No | auto, always, never (default: auto) |
82
- | `backend_url` | No | Dashboard URL (default: https://api.cliwatch.dev) |
82
+ | `repeat` | No | Run all tasks N times (default: 1, range: 1-100) |
83
+ | `system_prompt` | No | Custom prompt appended to the default agent system message |
84
+ | `thresholds` | No | Pass rate thresholds (see [docs](https://docs.cliwatch.com/configuration/thresholds)) |
83
85
  | `tasks` | Yes | Array of tasks or `file://` references |
84
86
 
85
87
  ## Assertion types
@@ -109,7 +111,7 @@ steps:
109
111
  - run: npm install -g my-cli
110
112
  - run: npx @cliwatch/cli-bench
111
113
  env:
112
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
114
+ AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
113
115
  CLIWATCH_API_KEY: ${{ secrets.CLIWATCH_API_KEY }} # optional, uploads to dashboard
114
116
  ```
115
117
 
@@ -119,11 +121,8 @@ No Docker required. Commands run directly on the CI runner.
119
121
 
120
122
  | Variable | Description |
121
123
  |----------|-------------|
122
- | `ANTHROPIC_API_KEY` | Anthropic API key (for `anthropic/*` models) |
123
- | `OPENAI_API_KEY` | OpenAI API key (for `openai/*` models) |
124
- | `GOOGLE_GENERATIVE_AI_API_KEY` | Google AI API key (for `google/*` models) |
125
- | `CLIWATCH_API_KEY` | API key for uploading results to cliwatch.dev |
126
- | `CLIWATCH_BACKEND_URL` | Override backend URL (default: `https://api.cliwatch.dev`) |
124
+ | `AI_GATEWAY_API_KEY` | Vercel AI Gateway key — provides access to all models |
125
+ | `CLIWATCH_API_KEY` | API key from [app.cliwatch.com](https://app.cliwatch.com) for uploading results |
127
126
 
128
127
  ## Uploading results
129
128
 
package/dist/index.js CHANGED
@@ -32,7 +32,7 @@ async function main() {
32
32
  }
33
33
  return;
34
34
  }
35
- console.log('@cliwatch/cli-bench v0.5.0');
35
+ console.log('@cliwatch/cli-bench v0.5.1');
36
36
  // Try to find a config file
37
37
  const configPath = await resolveConfigFile(config.configFile);
38
38
  let reports;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cliwatch/cli-bench",
3
- "version": "0.5.0",
3
+ "version": "0.5.1",
4
4
  "type": "module",
5
5
  "bin": {
6
6
  "cli-bench": "./dist/index.js"