@evolvingmachines/evolve 0.0.65 → 0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-DK2MGUPB.js → chunk-DI6LPW2B.js} +2 -2
- package/dist/cli/index.cjs +29 -29
- package/dist/cli/index.d.cts +3 -3
- package/dist/cli/index.d.ts +3 -3
- package/dist/cli/index.js +18 -18
- package/dist/index.cjs +64 -60
- package/dist/index.d.cts +297 -18
- package/dist/index.d.ts +297 -18
- package/dist/index.js +48 -44
- package/dist/{types-B_qmzlMk.d.cts → types-Dj0iMl5V.d.cts} +70 -90
- package/dist/{types-B_qmzlMk.d.ts → types-Dj0iMl5V.d.ts} +70 -90
- package/harness-capabilities.json +285 -0
- package/package.json +9 -6
- package/skills/create-adapter/SKILL.md +71 -26
- package/skills/create-task/SKILL.md +124 -66
- package/skills/evolve/SKILL.md +37 -20
- package/skills/evolve-evals/SKILL.md +142 -69
- package/skills/evolve-evals/references/cli-reference/agent.md +76 -0
- package/skills/evolve-evals/references/cli-reference/analysis.md +78 -0
- package/skills/evolve-evals/references/cli-reference/analyze.md +97 -0
- package/skills/evolve-evals/references/cli-reference/auth.md +96 -0
- package/skills/evolve-evals/references/cli-reference/check.md +157 -0
- package/skills/evolve-evals/references/cli-reference/dataset.md +154 -0
- package/skills/evolve-evals/references/cli-reference/filesystem.md +168 -0
- package/skills/evolve-evals/references/cli-reference/index.md +149 -0
- package/skills/evolve-evals/references/cli-reference/job.md +198 -0
- package/skills/evolve-evals/references/cli-reference/run.md +188 -0
- package/skills/evolve-evals/references/cli-reference/secrets.md +77 -0
- package/skills/evolve-evals/references/cli-reference/session.md +56 -0
- package/skills/evolve-evals/references/cli-reference/skill.md +52 -0
- package/skills/evolve-evals/references/cli-reference/skills.md +85 -0
- package/skills/evolve-evals/references/cli-reference/trial.md +115 -0
- package/skills/evolve-evals/references/cli-reference/upload.md +72 -0
- package/skills/evolve-evals/references/core-concepts/agents.md +101 -0
- package/skills/evolve-evals/references/core-concepts/analyze.md +143 -0
- package/skills/evolve-evals/references/core-concepts/check.md +116 -0
- package/skills/evolve-evals/references/core-concepts/compatibility.md +39 -0
- package/skills/evolve-evals/references/core-concepts/compose.md +103 -0
- package/skills/evolve-evals/references/core-concepts/datasets.md +234 -0
- package/skills/evolve-evals/references/core-concepts/index.md +52 -0
- package/skills/evolve-evals/references/core-concepts/jobs.md +223 -0
- package/skills/evolve-evals/references/core-concepts/models.md +73 -0
- package/skills/evolve-evals/references/core-concepts/multi-step.md +93 -0
- package/skills/evolve-evals/references/core-concepts/rewardkit.md +341 -0
- package/skills/evolve-evals/references/core-concepts/sandboxes.md +73 -0
- package/skills/evolve-evals/references/core-concepts/secrets.md +86 -0
- package/skills/evolve-evals/references/core-concepts/sharing.md +81 -0
- package/skills/evolve-evals/references/core-concepts/skills.md +100 -0
- package/skills/evolve-evals/references/core-concepts/task-config.md +169 -0
- package/skills/evolve-evals/references/core-concepts/task-environment.md +127 -0
- package/skills/evolve-evals/references/core-concepts/task-reference.md +191 -0
- package/skills/evolve-evals/references/core-concepts/task-verifiers.md +209 -0
- package/skills/evolve-evals/references/core-concepts/tasks.md +90 -0
- package/skills/evolve-evals/references/core-concepts/teams.md +128 -0
- package/skills/evolve-evals/references/core-concepts/trial-outputs.md +225 -0
- package/skills/evolve-evals/references/core-concepts/trials.md +129 -0
- package/skills/evolve-evals/references/core-concepts/upload-sdk-session.md +240 -0
- package/skills/evolve-evals/references/core-concepts/upload.md +166 -0
- package/skills/evolve-evals/references/dashboard/checks.md +54 -0
- package/skills/evolve-evals/references/dashboard/datasets.md +65 -0
- package/skills/evolve-evals/references/dashboard/jobs.md +106 -0
- package/skills/evolve-evals/references/dashboard/traces.md +54 -0
- package/skills/evolve-evals/references/dashboard/trial-viewer.md +77 -0
- package/skills/evolve-evals/references/getting-started/first-task.md +228 -0
- package/skills/evolve-evals/references/getting-started/installation.md +79 -0
- package/skills/evolve-evals/references/getting-started/quick-start.md +100 -0
- package/skills/evolve-evals/references/index.md +58 -0
- package/skills/evolve-evals/references/sdk/http.md +110 -0
- package/skills/evolve-evals/references/sdk/python.md +108 -0
- package/skills/evolve-evals/references/sdk/typescript.md +104 -0
- package/skills/evolve-evals/references/sdk-reference/agents.md +75 -0
- package/skills/evolve-evals/references/sdk-reference/analyses.md +125 -0
- package/skills/evolve-evals/references/sdk-reference/auth.md +171 -0
- package/skills/evolve-evals/references/sdk-reference/checks.md +146 -0
- package/skills/evolve-evals/references/sdk-reference/datasets.md +170 -0
- package/skills/evolve-evals/references/sdk-reference/error-codes.md +231 -0
- package/skills/evolve-evals/references/sdk-reference/errors.md +98 -0
- package/skills/evolve-evals/references/sdk-reference/filesystem.md +166 -0
- package/skills/evolve-evals/references/sdk-reference/index.md +161 -0
- package/skills/evolve-evals/references/sdk-reference/jobs.md +250 -0
- package/skills/evolve-evals/references/sdk-reference/meta.md +54 -0
- package/skills/evolve-evals/references/sdk-reference/methods/agents.md +154 -0
- package/skills/evolve-evals/references/sdk-reference/methods/analyses.md +211 -0
- package/skills/evolve-evals/references/sdk-reference/methods/auth.md +257 -0
- package/skills/evolve-evals/references/sdk-reference/methods/checks.md +462 -0
- package/skills/evolve-evals/references/sdk-reference/methods/datasets.md +687 -0
- package/skills/evolve-evals/references/sdk-reference/methods/filesystem.md +527 -0
- package/skills/evolve-evals/references/sdk-reference/methods/jobs.md +1023 -0
- package/skills/evolve-evals/references/sdk-reference/methods/meta.md +213 -0
- package/skills/evolve-evals/references/sdk-reference/methods/secrets.md +157 -0
- package/skills/evolve-evals/references/sdk-reference/methods/skills.md +125 -0
- package/skills/evolve-evals/references/sdk-reference/methods/trials.md +371 -0
- package/skills/evolve-evals/references/sdk-reference/secrets.md +104 -0
- package/skills/evolve-evals/references/sdk-reference/skills.md +66 -0
- package/skills/evolve-evals/references/sdk-reference/trials.md +164 -0
- package/skills/evolve-evals/references/sdk-reference/types.md +1255 -0
- package/skills/publish/SKILL.md +103 -79
- package/skills/rewardkit/SKILL.md +78 -23
- package/spec/openapi.yaml +164 -160
- package/skills/evolve-evals/references/cli-reference/agent.mdx +0 -79
- package/skills/evolve-evals/references/cli-reference/analysis.mdx +0 -160
- package/skills/evolve-evals/references/cli-reference/analyze.mdx +0 -84
- package/skills/evolve-evals/references/cli-reference/auth.mdx +0 -133
- package/skills/evolve-evals/references/cli-reference/check.mdx +0 -263
- package/skills/evolve-evals/references/cli-reference/dataset.mdx +0 -151
- package/skills/evolve-evals/references/cli-reference/index.mdx +0 -108
- package/skills/evolve-evals/references/cli-reference/job.mdx +0 -278
- package/skills/evolve-evals/references/cli-reference/run.mdx +0 -204
- package/skills/evolve-evals/references/cli-reference/secrets.mdx +0 -95
- package/skills/evolve-evals/references/cli-reference/skill.mdx +0 -61
- package/skills/evolve-evals/references/cli-reference/skills.mdx +0 -81
- package/skills/evolve-evals/references/cli-reference/trial.mdx +0 -181
- package/skills/evolve-evals/references/cli-reference/upload.mdx +0 -36
- package/skills/evolve-evals/references/core-concepts/agents.mdx +0 -106
- package/skills/evolve-evals/references/core-concepts/analyze.mdx +0 -86
- package/skills/evolve-evals/references/core-concepts/check.mdx +0 -64
- package/skills/evolve-evals/references/core-concepts/datasets.mdx +0 -138
- package/skills/evolve-evals/references/core-concepts/index.mdx +0 -42
- package/skills/evolve-evals/references/core-concepts/jobs.mdx +0 -151
- package/skills/evolve-evals/references/core-concepts/models.mdx +0 -68
- package/skills/evolve-evals/references/core-concepts/sandboxes.mdx +0 -52
- package/skills/evolve-evals/references/core-concepts/secrets.mdx +0 -41
- package/skills/evolve-evals/references/core-concepts/sharing.mdx +0 -35
- package/skills/evolve-evals/references/core-concepts/skills.mdx +0 -42
- package/skills/evolve-evals/references/core-concepts/tasks.mdx +0 -216
- package/skills/evolve-evals/references/core-concepts/teams.mdx +0 -36
- package/skills/evolve-evals/references/core-concepts/trial-outputs.mdx +0 -117
- package/skills/evolve-evals/references/core-concepts/trials.mdx +0 -103
- package/skills/evolve-evals/references/core-concepts/upload.mdx +0 -93
- package/skills/evolve-evals/references/dashboard/checks.mdx +0 -21
- package/skills/evolve-evals/references/dashboard/datasets.mdx +0 -24
- package/skills/evolve-evals/references/dashboard/jobs.mdx +0 -65
- package/skills/evolve-evals/references/dashboard/traces.mdx +0 -17
- package/skills/evolve-evals/references/dashboard/trial-viewer.mdx +0 -30
- package/skills/evolve-evals/references/getting-started/installation.mdx +0 -56
- package/skills/evolve-evals/references/getting-started/quick-start.mdx +0 -94
- package/skills/evolve-evals/references/index.mdx +0 -48
- package/skills/evolve-evals/references/sdk/python.mdx +0 -161
- package/skills/evolve-evals/references/sdk/typescript.mdx +0 -163
- package/skills/evolve-evals/references/sdk-reference/agents.mdx +0 -108
- package/skills/evolve-evals/references/sdk-reference/analyses.mdx +0 -222
- package/skills/evolve-evals/references/sdk-reference/auth.mdx +0 -116
- package/skills/evolve-evals/references/sdk-reference/checks.mdx +0 -300
- package/skills/evolve-evals/references/sdk-reference/datasets.mdx +0 -346
- package/skills/evolve-evals/references/sdk-reference/errors.mdx +0 -210
- package/skills/evolve-evals/references/sdk-reference/index.mdx +0 -60
- package/skills/evolve-evals/references/sdk-reference/jobs.mdx +0 -822
- package/skills/evolve-evals/references/sdk-reference/meta.mdx +0 -52
- package/skills/evolve-evals/references/sdk-reference/skills.mdx +0 -69
- package/skills/evolve-evals/references/sdk-reference/trials.mdx +0 -302
- package/skills/evolve-evals/references/sdk-reference/types.mdx +0 -274
- package/skills/evolve-evals/references/snippets/global-options.mdx +0 -13
- package/skills/evolve-evals/references/snippets/list-flags.mdx +0 -23
package/spec/openapi.yaml
CHANGED
|
@@ -101,9 +101,9 @@ info:
|
|
|
101
101
|
directory and runs it as a new local Harbor job
|
|
102
102
|
(their src/harbor/analyze/analyzer.py `run_analyze`). Here analysis is
|
|
103
103
|
a NATIVE capability of the platform — the worker runs the same
|
|
104
|
-
analyzer (claude-code over the trial's
|
|
105
|
-
task, rubric-driven, `analysis.json`
|
|
106
|
-
criterion) server-side, and the result is stored and served on the
|
|
104
|
+
analyzer (the chosen agent, claude-code by default, over the trial's
|
|
105
|
+
Harbor-shape tree plus its task, rubric-driven, `analysis.json`
|
|
106
|
+
validated criterion-for-criterion) server-side, and the result is stored and served on the
|
|
107
107
|
trial body (`Trial.analysis`) instead of written into a local
|
|
108
108
|
directory. Two named deviations from Harbor's shape: (1) server-side
|
|
109
109
|
execution — no wrapper job appears anywhere on this surface; the
|
|
@@ -111,8 +111,8 @@ info:
|
|
|
111
111
|
`JobCreate.analyze` runs the analyzer per trial as each trial settles,
|
|
112
112
|
which Harbor has no equivalent of (their analyze is always a manual
|
|
113
113
|
follow-up command; `POST /api/jobs/{jobId}/analyze` is that manual
|
|
114
|
-
verb here). Vocabulary is Harbor's: `model_name`, rubric criteria
|
|
115
|
-
`{name, description, guidance}` (their cli/analyze.py `--model` /
|
|
114
|
+
verb here). Vocabulary is Harbor's: `agent`, `model_name`, rubric criteria
|
|
115
|
+
`{name, description, guidance}` (their cli/analyze.py `--agent` / `--model` /
|
|
116
116
|
`--rubric`, cli/quality_checker/models.py RubricCriterion), result
|
|
117
117
|
`{summary, checks{outcome, explanation}, estimated_cost_usd}` (their
|
|
118
118
|
analyze/models.py AnalyzeResult). Selection and width are Harbor's
|
|
@@ -132,7 +132,7 @@ info:
|
|
|
132
132
|
own record: the caller uploads the task directory (or a directory of
|
|
133
133
|
task directories) as one archive (`POST /api/checks`, multipart —
|
|
134
134
|
`archive` + `config`), the worker runs the SAME checker per task
|
|
135
|
-
(claude-code over the task's byte-sacred tree, the platform's default
|
|
135
|
+
(the chosen agent, claude-code by default, over the task's byte-sacred tree, the platform's default
|
|
136
136
|
check prompt and rubric rendered through Harbor's own tokens,
|
|
137
137
|
`check-result.json` validated criterion for criterion) server-side,
|
|
138
138
|
and the report is served as the `Check`
|
|
@@ -145,7 +145,7 @@ info:
|
|
|
145
145
|
`claude-sonnet-4-6` (one roster, one default for both rubric agents);
|
|
146
146
|
and the platform's `reasoning_effort` and `sandbox_provider` beside
|
|
147
147
|
Harbor's knobs, exactly as on `AnalyzeConfigInput`. Vocabulary is
|
|
148
|
-
Harbor's: `model_name`, `rubric`, `prompt` (their cli/analyze.py:88-
|
|
148
|
+
Harbor's: `agent`, `model_name`, `rubric`, `prompt` (their cli/analyze.py:88-
|
|
149
149
|
103), `n_concurrent` (:117-119), `include_task_names` /
|
|
150
150
|
`exclude_task_names` / `n_tasks` (:121-131; fnmatch globs and the cap
|
|
151
151
|
of checker.py:132-138), result `{task_name, checks{outcome,
|
|
@@ -1459,9 +1459,11 @@ paths:
|
|
|
1459
1459
|
│ │ at Harbor's own tee name for the
|
|
1460
1460
|
│ │ harness (claude-code.txt, codex.txt,
|
|
1461
1461
|
│ │ gemini-cli.txt, qwen-code.txt,
|
|
1462
|
-
│ │ kimi-code.txt, opencode.txt
|
|
1463
|
-
│ │ droid.txt,
|
|
1464
|
-
│ │
|
|
1462
|
+
│ │ kimi-code.txt, opencode.txt, pi.txt,
|
|
1463
|
+
│ │ antigravity-stream.jsonl; droid.txt,
|
|
1464
|
+
│ │ prime-agent.txt, dsh.txt, zcode.txt, and
|
|
1465
|
+
│ │ stdout.log for a harness Harbor has
|
|
1466
|
+
│ │ no name for)
|
|
1465
1467
|
│ ├── stderr.log the harness process's stderr stream,
|
|
1466
1468
|
│ │ when stored
|
|
1467
1469
|
│ ├── trace-parsed.jsonl the parsed event trace, one
|
|
@@ -1801,9 +1803,9 @@ paths:
|
|
|
1801
1803
|
summary: Analyze a job's trial traces (rubric-driven)
|
|
1802
1804
|
description: |
|
|
1803
1805
|
Runs Harbor-style trace analysis over a terminal job's trials,
|
|
1804
|
-
server-side: for each trial the analyzer agent (claude-code
|
|
1805
|
-
own sandbox, under the network Harbor's analyze
|
|
1806
|
-
it — open by default) reads the trial's Harbor-shape tree plus its
|
|
1806
|
+
server-side: for each trial the analyzer agent (`agent`, claude-code
|
|
1807
|
+
by default, in its own sandbox, under the network Harbor's analyze
|
|
1808
|
+
wrapper task gives it — open by default) reads the trial's Harbor-shape tree plus its
|
|
1807
1809
|
original task and rules every rubric criterion, storing
|
|
1808
1810
|
`{summary, checks, estimated_cost_usd}` on the trial
|
|
1809
1811
|
(`Trial.analysis`) and the aggregate on the job
|
|
@@ -1823,7 +1825,7 @@ paths:
|
|
|
1823
1825
|
`max_concurrent_analyses`).
|
|
1824
1826
|
|
|
1825
1827
|
An analyzer run that completes without a valid `analysis.json` is
|
|
1826
|
-
automatically re-run at most once (fresh sandbox, same model and
|
|
1828
|
+
automatically re-run at most once (fresh sandbox, same agent, model and
|
|
1827
1829
|
rubric); the analysis object's `attempts` records how far the
|
|
1828
1830
|
ladder went. A run cut by its budget is not that class: it settles
|
|
1829
1831
|
`failed` with phase `timeout` at once and is never re-run — a
|
|
@@ -4994,8 +4996,8 @@ paths:
|
|
|
4994
4996
|
description: |
|
|
4995
4997
|
The resolved policy an empty `AnalyzeConfigInput` falls back to — the
|
|
4996
4998
|
five knobs every analysis that names none is stamped with:
|
|
4997
|
-
`model_name`, `rubric` (the platform's analyze rubric,
|
|
4998
|
-
|
|
4999
|
+
`model_name`, `rubric` (the platform's default analyze rubric),
|
|
5000
|
+
`prompt` (the built-in analyze
|
|
4999
5001
|
body, as the TEMPLATE a caller's own `prompt` replaces — with its
|
|
5000
5002
|
`{trial_path}`, `{task_section}` and `{criteria_guidance}` slots
|
|
5001
5003
|
unrendered; the output section Harbor appends is not part of it),
|
|
@@ -5692,8 +5694,8 @@ paths:
|
|
|
5692
5694
|
|
|
5693
5695
|
analyze-<trial>__<7>/
|
|
5694
5696
|
├── config.json TrialConfig, partial: the wrapper
|
|
5695
|
-
│ task's name, the agent (
|
|
5696
|
-
│ the model), and under `x_evolve`
|
|
5697
|
+
│ task's name, the agent (its Harbor
|
|
5698
|
+
│ name, the model), and under `x_evolve`
|
|
5697
5699
|
│ the frozen policy — rubric, prompt,
|
|
5698
5700
|
│ reasoning_effort — plus the analyzed
|
|
5699
5701
|
│ trial/job/task and the provider
|
|
@@ -5721,13 +5723,15 @@ paths:
|
|
|
5721
5723
|
│ validator (an infrastructure phase)
|
|
5722
5724
|
├── agent/
|
|
5723
5725
|
│ ├── claude-code.txt the analyzer's stdout stream at
|
|
5724
|
-
│ │ Harbor's own tee name for
|
|
5726
|
+
│ │ Harbor's own tee name for its agent
|
|
5727
|
+
│ │ (claude-code.txt, codex.txt, …)
|
|
5725
5728
|
│ ├── stderr.log its stderr, when stored
|
|
5726
5729
|
│ ├── trace-parsed.jsonl the run's parsed event trace
|
|
5727
5730
|
│ ├── .claude/ … the captured home at its real names,
|
|
5728
5731
|
│ ├── agent-home.json the capture record beside it
|
|
5729
|
-
│ └── sessions/ … Harbor's own
|
|
5730
|
-
│
|
|
5732
|
+
│ └── sessions/ … Harbor's own copies of the home, as
|
|
5733
|
+
│ on a trial (claude: the .claude/
|
|
5734
|
+
│ subtree, claude_code.py:482)
|
|
5731
5735
|
├── verifier/ ONLY when the validator ruled —
|
|
5732
5736
|
│ │ completed, or failed `invalid_result`
|
|
5733
5737
|
│ ├── test-stdout.txt the validator's printed reasons
|
|
@@ -5804,8 +5808,8 @@ paths:
|
|
|
5804
5808
|
gzipped tarball (`archive`), or a published dataset is named
|
|
5805
5809
|
instead (`dataset`, the hosted form; `CheckRequest.dataset` states
|
|
5806
5810
|
the deviation and its reason), and for each task directory it holds
|
|
5807
|
-
the checker agent (claude-code
|
|
5808
|
-
sandbox, under the network Harbor's check wrapper task gives it —
|
|
5811
|
+
the checker agent (`agent`, claude-code by default — Harbor's own
|
|
5812
|
+
check agent — in its own sandbox, under the network Harbor's check wrapper task gives it —
|
|
5809
5813
|
open by default) reads EVERY file of the task and rules every rubric
|
|
5810
5814
|
criterion `pass`, `fail`, `not_applicable` or `unknown` with a
|
|
5811
5815
|
rationale and the evidence behind it, writing `check-result.json`
|
|
@@ -5835,8 +5839,8 @@ paths:
|
|
|
5835
5839
|
refusal answers. The `config`
|
|
5836
5840
|
part is the JSON `CheckConfigInput`; absent or
|
|
5837
5841
|
`{}` means the defaults (openrouter/deepseek/deepseek-v4.1-flash at its
|
|
5838
|
-
per-model effort, the platform's default check rubric
|
|
5839
|
-
|
|
5842
|
+
per-model effort, the platform's default check rubric, served by
|
|
5843
|
+
`GET /api/checks/defaults`).
|
|
5840
5844
|
|
|
5841
5845
|
The task bytes are never modified and never land anywhere but the
|
|
5842
5846
|
checker's sandbox and the platform's rows: Harbor's own rule ("Do
|
|
@@ -5973,8 +5977,8 @@ paths:
|
|
|
5973
5977
|
description: |
|
|
5974
5978
|
The resolved policy an empty `CheckConfigInput` falls back to — the
|
|
5975
5979
|
five knobs the create door stamps on every check that names none:
|
|
5976
|
-
`model_name`, `rubric` (the platform's check rubric,
|
|
5977
|
-
|
|
5980
|
+
`model_name`, `rubric` (the platform's default check rubric),
|
|
5981
|
+
`prompt` (the built-in check
|
|
5978
5982
|
body, as the TEMPLATE a caller's own `prompt` replaces — with its
|
|
5979
5983
|
`{task_path}`, `{file_tree}` and `{criteria_guidance}` slots
|
|
5980
5984
|
unrendered; the output section Harbor appends is not part of it),
|
|
@@ -12318,13 +12322,10 @@ components:
|
|
|
12318
12322
|
pattern: '^[a-z][a-z0-9_]*$'
|
|
12319
12323
|
description: >
|
|
12320
12324
|
Criterion identifier, snake_case (it keys the result's `checks`
|
|
12321
|
-
object). The platform's default
|
|
12322
|
-
|
|
12323
|
-
|
|
12324
|
-
|
|
12325
|
-
check rubric eleven (CheckConfigInput lists them); the derived
|
|
12326
|
-
labels read those names, and a rubric with other names carries
|
|
12327
|
-
no label.
|
|
12325
|
+
object). The platform's default rubrics are served by
|
|
12326
|
+
`GET /api/analyses/defaults` and `GET /api/checks/defaults`; the
|
|
12327
|
+
platform reads no criterion name — a run's result is its
|
|
12328
|
+
per-criterion table under whatever names its rubric carries.
|
|
12328
12329
|
description:
|
|
12329
12330
|
type: string
|
|
12330
12331
|
minLength: 1
|
|
@@ -12362,7 +12363,7 @@ components:
|
|
|
12362
12363
|
type: object
|
|
12363
12364
|
description: |
|
|
12364
12365
|
Trace-analysis configuration — Harbor's `harbor analyze` vocabulary
|
|
12365
|
-
(their cli/analyze.py: `--model`, `--rubric`, `--prompt`). PRESENCE of this
|
|
12366
|
+
(their cli/analyze.py: `--agent`, `--model`, `--rubric`, `--prompt`). PRESENCE of this
|
|
12366
12367
|
object is the switch: on `JobCreate.analyze` it arms the embedded
|
|
12367
12368
|
trigger (each trial is analyzed server-side right after it settles;
|
|
12368
12369
|
CANCELLED trials are skipped — withdrawn work is not billed an
|
|
@@ -12370,17 +12371,17 @@ components:
|
|
|
12370
12371
|
it configures that manual wave. `{}` is legal and means "all
|
|
12371
12372
|
defaults": openrouter/deepseek/deepseek-v4.1-flash at its per-model effort
|
|
12372
12373
|
(high) over
|
|
12373
|
-
the platform's default analyze rubric (
|
|
12374
|
-
`
|
|
12375
|
-
still runs when passed as `rubric
|
|
12376
|
-
|
|
12377
|
-
The analyzer
|
|
12378
|
-
|
|
12379
|
-
analyze wrapper task gives it (open by default;
|
|
12380
|
-
unchanged) — on the provider `sandbox_provider`
|
|
12381
|
-
platform's analysis default when it names none; its
|
|
12382
|
-
per analysis and metered as its own line, never
|
|
12383
|
-
trial's own bill.
|
|
12374
|
+
the platform's default analyze rubric (served by
|
|
12375
|
+
`GET /api/analyses/defaults`; Harbor's own analyze-rubric.toml
|
|
12376
|
+
still runs when passed as `rubric`).
|
|
12377
|
+
|
|
12378
|
+
The analyzer runs the agent `agent` names (claude, Harbor's own
|
|
12379
|
+
default claude-code, when it names none) in its own sandbox, under
|
|
12380
|
+
the network Harbor's analyze wrapper task gives it (open by default;
|
|
12381
|
+
the model door is unchanged) — on the provider `sandbox_provider`
|
|
12382
|
+
names, or the platform's analysis default when it names none; its
|
|
12383
|
+
spend is capped per analysis and metered as its own line, never
|
|
12384
|
+
blended into the trial's own bill.
|
|
12384
12385
|
|
|
12385
12386
|
Which trials, and how wide, are Harbor's own analyze options with
|
|
12386
12387
|
their exact names: `n_concurrent` (`-n/--n-concurrent`), `passing`
|
|
@@ -12396,9 +12397,7 @@ components:
|
|
|
12396
12397
|
takes it; on `JobCreate.analyze` the trials do not exist yet.
|
|
12397
12398
|
|
|
12398
12399
|
Harbor's other analyze options, and why they are not here:
|
|
12399
|
-
`-
|
|
12400
|
-
instruction templates, the roster check and the sandbox policy are
|
|
12401
|
-
written against it); `--job-name` and `-o/--jobs-dir` — a hosted
|
|
12400
|
+
`--job-name` and `-o/--jobs-dir` — a hosted
|
|
12402
12401
|
analysis attaches to its source job and is served on that job's
|
|
12403
12402
|
trials, so there is no wrapper job to name and no directory to
|
|
12404
12403
|
write; `-k/--n-attempts` (k analyses per trial) — deferred: each
|
|
@@ -12413,6 +12412,21 @@ components:
|
|
|
12413
12412
|
(task quality checks, their cli/analyze.py:126-137) and are not
|
|
12414
12413
|
adopted here.
|
|
12415
12414
|
properties:
|
|
12415
|
+
agent:
|
|
12416
|
+
type: string
|
|
12417
|
+
default: claude
|
|
12418
|
+
description: >
|
|
12419
|
+
The agent the analyzer runs on — Harbor's `-a/--agent` (their
|
|
12420
|
+
cli/analyze.py:258, default claude-code), named in the
|
|
12421
|
+
`agents[].name` vocabulary (claude, not claude-code) so one set
|
|
12422
|
+
of names covers arms and reviewers: any built-in agent
|
|
12423
|
+
`analyze.agents` lists on `GET /api/meta`. `model_name` must be
|
|
12424
|
+
on this agent's roster and `reasoning_effort` must be one it
|
|
12425
|
+
accepts; with an agent whose roster lacks the default model,
|
|
12426
|
+
`model_name` is required. A registered (bring-your-own) agent
|
|
12427
|
+
is not accepted. Anything else is refused `invalid_input`
|
|
12428
|
+
naming `analyze.agent`. Stored as given and frozen into each
|
|
12429
|
+
analysis (`TrialAnalysis.agent`).
|
|
12416
12430
|
model_name:
|
|
12417
12431
|
type: string
|
|
12418
12432
|
default: openrouter/deepseek/deepseek-v4.1-flash
|
|
@@ -12435,9 +12449,10 @@ components:
|
|
|
12435
12449
|
default is the roster alias), the wire id is resolved only when
|
|
12436
12450
|
the analyzer runs, and every stored analysis serves the
|
|
12437
12451
|
spelling it was created under. Must be on
|
|
12438
|
-
the
|
|
12439
|
-
|
|
12440
|
-
|
|
12452
|
+
the agent's roster (`GET /api/meta`, `analyze.agents[].models`);
|
|
12453
|
+
anything else is refused at accept (`invalid_input`, roster in
|
|
12454
|
+
the message) — including the default model on an agent whose
|
|
12455
|
+
roster lacks it, which then says `model_name` is required.
|
|
12441
12456
|
rubric:
|
|
12442
12457
|
$ref: '#/components/schemas/Rubric'
|
|
12443
12458
|
prompt:
|
|
@@ -12475,20 +12490,22 @@ components:
|
|
|
12475
12490
|
type: string
|
|
12476
12491
|
description: >
|
|
12477
12492
|
Reasoning effort the analyzer runs at — the platform's
|
|
12478
|
-
`agents[].reasoning_effort` vocabulary applied to the analyzer
|
|
12479
|
-
|
|
12480
|
-
`analyze.reasoning_efforts` on
|
|
12481
|
-
|
|
12482
|
-
|
|
12483
|
-
|
|
12493
|
+
`agents[].reasoning_effort` vocabulary applied to the analyzer's
|
|
12494
|
+
agent, held to that agent's own rule exactly as an arm on it:
|
|
12495
|
+
the accepted values are `analyze.agents[].reasoning_efforts` on
|
|
12496
|
+
`GET /api/meta` (claude's `effort_support` is `level`, so the
|
|
12497
|
+
whole list; an agent that takes none, gemini, accepts none), and
|
|
12498
|
+
a refused value answers `invalid_input` with the same sentence
|
|
12499
|
+
an arm's does (`analyze.reasoning_effort must be one of: ...`).
|
|
12484
12500
|
Omitted, the PER-MODEL default applies —
|
|
12485
|
-
`analyze.models[].default_reasoning_effort`: `high` on
|
|
12501
|
+
`analyze.agents[].models[].default_reasoning_effort`: `high` on
|
|
12486
12502
|
openrouter/deepseek/deepseek-v4.1-flash (the default model;
|
|
12487
12503
|
DeepSeek's own documented default, the owner's ruling
|
|
12488
12504
|
2026-09-10), `max` on glm-5.3-flash (the platform's ruling
|
|
12489
|
-
2026-09-08, the effort its published scores use),
|
|
12490
|
-
|
|
12491
|
-
|
|
12505
|
+
2026-09-08, the effort its published scores use), on any agent
|
|
12506
|
+
that can honor them; the agent's own default
|
|
12507
|
+
(`agents[].default_effort`) on every other roster model; none
|
|
12508
|
+
on an agent that takes none. Whatever resolves is passed to the analyzer
|
|
12492
12509
|
EXPLICITLY on every call — never left to a provider's default —
|
|
12493
12510
|
and recorded as `TrialAnalysis.reasoning_effort`. A hosted
|
|
12494
12511
|
extension, recorded as such: Harbor's `harbor analyze` has no
|
|
@@ -12605,11 +12622,14 @@ components:
|
|
|
12605
12622
|
always states the policy it executes (same law as RetryConfig).
|
|
12606
12623
|
Echoed on the job body when the job was created with `analyze`;
|
|
12607
12624
|
each analysis additionally carries the exact policy IT ran under
|
|
12608
|
-
(`Trial.analysis.model_name` / `.rubric` / `.prompt` /
|
|
12625
|
+
(`Trial.analysis.agent` / `.model_name` / `.rubric` / `.prompt` /
|
|
12609
12626
|
`.reasoning_effort`), which a later manual re-analysis may have
|
|
12610
12627
|
changed.
|
|
12611
|
-
required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider, n_concurrent, passing, failing, n_trials, trial_ids]
|
|
12628
|
+
required: [agent, model_name, rubric, prompt, reasoning_effort, sandbox_provider, n_concurrent, passing, failing, n_trials, trial_ids]
|
|
12612
12629
|
properties:
|
|
12630
|
+
agent:
|
|
12631
|
+
type: string
|
|
12632
|
+
description: The agent this policy's analyses run on — as named, or the default (claude).
|
|
12613
12633
|
model_name:
|
|
12614
12634
|
type: string
|
|
12615
12635
|
rubric:
|
|
@@ -12621,14 +12641,15 @@ components:
|
|
|
12621
12641
|
null = the built-in prompt (the platform's default analyze body).
|
|
12622
12642
|
reasoning_effort:
|
|
12623
12643
|
x-wave: 3
|
|
12624
|
-
type: string
|
|
12644
|
+
type: [string, 'null']
|
|
12625
12645
|
description: >
|
|
12626
12646
|
The effort this policy's analyses run at. Named at create it is
|
|
12627
12647
|
served as stored; when the create named none, this echoes the
|
|
12628
|
-
per-model default of the day for `model_name`
|
|
12629
|
-
next enqueue under this policy stamps (the same nuance
|
|
12630
|
-
`sandbox_provider` below, for the same reason: the default is
|
|
12631
|
-
platform table, resolved where an analysis is enqueued).
|
|
12648
|
+
per-model default of the day for `model_name` on `agent` — the
|
|
12649
|
+
value the next enqueue under this policy stamps (the same nuance
|
|
12650
|
+
as `sandbox_provider` below, for the same reason: the default is
|
|
12651
|
+
a platform table, resolved where an analysis is enqueued). Null
|
|
12652
|
+
when the agent takes no effort.
|
|
12632
12653
|
sandbox_provider:
|
|
12633
12654
|
allOf:
|
|
12634
12655
|
- $ref: '#/components/schemas/SandboxProvider'
|
|
@@ -13967,6 +13988,7 @@ components:
|
|
|
13967
13988
|
- name
|
|
13968
13989
|
- status
|
|
13969
13990
|
- source
|
|
13991
|
+
- agent
|
|
13970
13992
|
- model_name
|
|
13971
13993
|
- reasoning_effort
|
|
13972
13994
|
- sandbox_provider
|
|
@@ -13995,11 +14017,14 @@ components:
|
|
|
13995
14017
|
source:
|
|
13996
14018
|
$ref: '#/components/schemas/CheckSource'
|
|
13997
14019
|
description: What was checked; the row's DATASETS cell (the dataset form's `name@version`; an archive shows as an archive with its sha256).
|
|
14020
|
+
agent:
|
|
14021
|
+
type: string
|
|
14022
|
+
description: The checker's agent, as Check.agent.
|
|
13998
14023
|
model_name:
|
|
13999
14024
|
type: string
|
|
14000
14025
|
description: The checker's model; the row's AGENTS cell, shown like an arm with its effort under it.
|
|
14001
14026
|
reasoning_effort:
|
|
14002
|
-
type: string
|
|
14027
|
+
type: [string, 'null']
|
|
14003
14028
|
description: The effort every task's checker ran at, as Check.reasoning_effort.
|
|
14004
14029
|
sandbox_provider:
|
|
14005
14030
|
allOf:
|
|
@@ -14786,8 +14811,7 @@ components:
|
|
|
14786
14811
|
AnalyzeResult (their analyze/models.py: `summary`, `checks` keyed
|
|
14787
14812
|
by criterion, `estimated_cost_usd`; `trial_name` is the enclosing
|
|
14788
14813
|
trial) with each check carrying the platform's evidence list and
|
|
14789
|
-
fourth outcome (AnalysisCheck),
|
|
14790
|
-
is provenance — which model, rubric and prompt THIS analysis ran
|
|
14814
|
+
fourth outcome (AnalysisCheck); the rest is provenance — which agent, model, rubric and prompt THIS analysis ran
|
|
14791
14815
|
under, its lifecycle status, and its typed failure when it failed.
|
|
14792
14816
|
|
|
14793
14817
|
`estimated_cost_usd` is the analyzer agent's OWN metered spend (its
|
|
@@ -14801,13 +14825,13 @@ components:
|
|
|
14801
14825
|
- job_id
|
|
14802
14826
|
- task_name
|
|
14803
14827
|
- status
|
|
14828
|
+
- agent
|
|
14804
14829
|
- model_name
|
|
14805
14830
|
- reasoning_effort
|
|
14806
14831
|
- rubric
|
|
14807
14832
|
- prompt
|
|
14808
14833
|
- summary
|
|
14809
14834
|
- checks
|
|
14810
|
-
- label
|
|
14811
14835
|
- estimated_cost_usd
|
|
14812
14836
|
- failure
|
|
14813
14837
|
- created_at
|
|
@@ -14846,6 +14870,12 @@ components:
|
|
|
14846
14870
|
`not_applicable` are their exact words) and the ladder follows
|
|
14847
14871
|
its object's dialect; the UI capitalizes on render, and nothing
|
|
14848
14872
|
ever compares this ladder's words against another ladder's.
|
|
14873
|
+
agent:
|
|
14874
|
+
type: string
|
|
14875
|
+
description: >
|
|
14876
|
+
The agent THIS analysis ran on (`AnalyzeConfigInput.agent`, or
|
|
14877
|
+
the default claude); every analysis from before the choice
|
|
14878
|
+
existed ran on claude and says so.
|
|
14849
14879
|
model_name:
|
|
14850
14880
|
type: string
|
|
14851
14881
|
reasoning_effort:
|
|
@@ -14855,8 +14885,8 @@ components:
|
|
|
14855
14885
|
The reasoning effort THIS analysis ran at — passed to the
|
|
14856
14886
|
analyzer explicitly, so it is what the model was asked for
|
|
14857
14887
|
(`AnalyzeConfigInput.reasoning_effort`, or the per-model
|
|
14858
|
-
default). Null
|
|
14859
|
-
stamped; never a guessed value.
|
|
14888
|
+
default). Null when its agent takes no effort, or on analyses
|
|
14889
|
+
recorded before the effort was stamped; never a guessed value.
|
|
14860
14890
|
rubric:
|
|
14861
14891
|
$ref: '#/components/schemas/Rubric'
|
|
14862
14892
|
prompt:
|
|
@@ -14878,24 +14908,6 @@ components:
|
|
|
14878
14908
|
criterion names (the frozen-criteria law). Null until completed.
|
|
14879
14909
|
additionalProperties:
|
|
14880
14910
|
$ref: '#/components/schemas/AnalysisCheck'
|
|
14881
|
-
label:
|
|
14882
|
-
type: [string, 'null']
|
|
14883
|
-
enum: [flagged, env_fault, unclear, clean, null]
|
|
14884
|
-
description: >
|
|
14885
|
-
The derived label — computed by the platform from the outcomes
|
|
14886
|
-
when the result is stored, never asked from the model, so it is
|
|
14887
|
-
the same rule for every analysis and can never contradict the
|
|
14888
|
-
checks. The rule reads the default analyze rubric's criteria,
|
|
14889
|
-
precedence in this order: `flagged` on a fail of
|
|
14890
|
-
score_is_earned, score_is_correct, task_was_fair or
|
|
14891
|
-
report_is_truthful; else `env_fault` on a fail of
|
|
14892
|
-
environment_worked; else `unclear` on an unknown of any of those
|
|
14893
|
-
five, or a not_applicable of score_is_earned (the agent never
|
|
14894
|
-
acted) or score_is_correct (no reward exists); else `clean`.
|
|
14895
|
-
ended_by_its_own_decision and worked_as_for_a_real_user never
|
|
14896
|
-
change it. Null until completed, and null on a completed
|
|
14897
|
-
analysis whose rubric is not the default one — a custom rubric
|
|
14898
|
-
carries its per-criterion outcomes and no label.
|
|
14899
14911
|
estimated_cost_usd:
|
|
14900
14912
|
type: [number, 'null']
|
|
14901
14913
|
usage:
|
|
@@ -15029,21 +15041,16 @@ components:
|
|
|
15029
15041
|
type: object
|
|
15030
15042
|
description: |
|
|
15031
15043
|
Task-check configuration — Harbor's `harbor check` vocabulary
|
|
15032
|
-
(their cli/analyze.py:84-148 check_command). The
|
|
15044
|
+
(their cli/analyze.py:84-148 check_command). The four rubric-agent
|
|
15033
15045
|
knobs are the analyze door's, with the SAME rules and refusals
|
|
15034
15046
|
(`AnalyzeConfigInput` states them; the param prefix here is
|
|
15035
|
-
`check.`): `
|
|
15047
|
+
`check.`): `agent` (`-a/--agent`, :100; default claude, Harbor's
|
|
15048
|
+
claude-code), `model_name` (`-m/--model`; Harbor's check default is
|
|
15036
15049
|
`claude-sonnet-4-6`, :101-103 — this platform's is the analyzer's
|
|
15037
15050
|
`openrouter/deepseek/deepseek-v4.1-flash`, one roster and one default for both rubric
|
|
15038
15051
|
agents, a recorded deviation), `rubric` (`-r/--rubric`; the default
|
|
15039
|
-
is the platform's check rubric,
|
|
15040
|
-
|
|
15041
|
-
verifier_is_correct, no_answer_leakage, grader_is_out_of_reach,
|
|
15042
|
-
reference_solution_is_valid, verifier_rejects_non_solutions,
|
|
15043
|
-
environment_builds_and_runs, verification_is_stable,
|
|
15044
|
-
limits_allow_the_task, task_is_solvable — the derived `label` and
|
|
15045
|
-
`executed` read them, and a rubric with other names carries
|
|
15046
|
-
neither; Harbor's own default-rubric.toml still runs when passed),
|
|
15052
|
+
is the platform's check rubric, served by `GET /api/checks/defaults`;
|
|
15053
|
+
Harbor's own default-rubric.toml still runs when passed),
|
|
15047
15054
|
`prompt` (`-p/--prompt`, the file's TEXT: it replaces the built-in
|
|
15048
15055
|
check prompt, in Harbor's check.txt slot, and is rendered with
|
|
15049
15056
|
`{task_path}`, `{file_tree}`, `{criteria_guidance}` — checker.py:
|
|
@@ -15064,10 +15071,9 @@ components:
|
|
|
15064
15071
|
matched against the directory NAME.
|
|
15065
15072
|
|
|
15066
15073
|
Harbor's other check options, and why they are not here: the same
|
|
15067
|
-
list as the analyze door's — `-
|
|
15068
|
-
|
|
15069
|
-
|
|
15070
|
-
to write), `-k/--n-attempts` (the one hosted auto re-run replaces
|
|
15074
|
+
list as the analyze door's — `-o/--jobs-dir` (a hosted check is its
|
|
15075
|
+
own record, no directory to write; `--job-name` is `name`),
|
|
15076
|
+
`-k/--n-attempts` (the one hosted auto re-run replaces
|
|
15071
15077
|
it; see `TaskCheck.attempts`), `-c/--config`, `--ak`, `--ae`,
|
|
15072
15078
|
`--ek` (local-runner knobs with no hosted meaning), `-q` (a client
|
|
15073
15079
|
rendering option, not a wire field).
|
|
@@ -15080,6 +15086,10 @@ components:
|
|
|
15080
15086
|
The caller's name for the check — Harbor's `--job-name` on
|
|
15081
15087
|
`harbor check` (cli/analyze.py:138). Omitted, the accept
|
|
15082
15088
|
timestamp in Harbor's own shape, `YYYY-MM-DD__HH-MM-SS` (UTC).
|
|
15089
|
+
agent:
|
|
15090
|
+
type: string
|
|
15091
|
+
default: claude
|
|
15092
|
+
description: The agent the checker runs on — the analyze door's `agent`, same rule.
|
|
15083
15093
|
model_name:
|
|
15084
15094
|
type: string
|
|
15085
15095
|
default: openrouter/deepseek/deepseek-v4.1-flash
|
|
@@ -15152,8 +15162,7 @@ components:
|
|
|
15152
15162
|
One task's quality check — Harbor's QualityCheckResult shape
|
|
15153
15163
|
(their cli/quality_checker/models.py:31-35: `task_name`, `checks`
|
|
15154
15164
|
keyed by criterion, `cost_usd`), its checks extended by the result
|
|
15155
|
-
schema
|
|
15156
|
-
hosted record: its own
|
|
15165
|
+
schema, plus the hosted record: its own
|
|
15157
15166
|
id, the check it belongs to, its lifecycle (the analysis ladder's
|
|
15158
15167
|
four lowercase words — Harbor's dialect), the bounded attempt count,
|
|
15159
15168
|
and a typed `failure` in place of Harbor's `error` string (the
|
|
@@ -15163,11 +15172,10 @@ components:
|
|
|
15163
15172
|
(Harbor's check-task-template/tests/validate.py extended by the
|
|
15164
15173
|
result schema): one key per rubric criterion, each an AnalysisCheck
|
|
15165
15174
|
`{outcome, explanation, evidence}` — no summary (a check has none;
|
|
15166
|
-
analyze does). `
|
|
15167
|
-
result is stored. `cost_usd` is the checker agent's OWN metered
|
|
15175
|
+
analyze does). `cost_usd` is the checker agent's OWN metered
|
|
15168
15176
|
spend on its capped key; null when nothing was measured, never a
|
|
15169
15177
|
fabricated 0.
|
|
15170
|
-
required: [id, check_id, task_name, status, checks,
|
|
15178
|
+
required: [id, check_id, task_name, status, checks, cost_usd, attempts, failure, created_at, finished_at]
|
|
15171
15179
|
properties:
|
|
15172
15180
|
id:
|
|
15173
15181
|
type: string
|
|
@@ -15186,30 +15194,6 @@ components:
|
|
|
15186
15194
|
description: One entry per rubric criterion, keys exactly the frozen criteria. Null until completed.
|
|
15187
15195
|
additionalProperties:
|
|
15188
15196
|
$ref: '#/components/schemas/AnalysisCheck'
|
|
15189
|
-
label:
|
|
15190
|
-
type: [string, 'null']
|
|
15191
|
-
enum: [has_a_problem, unclear, no_problem_found, null]
|
|
15192
|
-
description: >
|
|
15193
|
-
The derived label — computed by the platform from the outcomes
|
|
15194
|
-
when the result is stored, never asked from the model. The rule
|
|
15195
|
-
reads the default check rubric's criteria, precedence in this
|
|
15196
|
-
order: `has_a_problem` on a fail of any criterion; else
|
|
15197
|
-
`unclear` on an unknown of any of the six file-based criteria
|
|
15198
|
-
(instruction_is_sufficient, tests_match_the_instruction,
|
|
15199
|
-
verifier_is_correct, no_answer_leakage, grader_is_out_of_reach,
|
|
15200
|
-
task_is_solvable); else `no_problem_found`. Null until
|
|
15201
|
-
completed, and null on a completed check whose rubric is not
|
|
15202
|
-
the default one — a custom rubric carries its per-criterion
|
|
15203
|
-
outcomes and no label.
|
|
15204
|
-
executed:
|
|
15205
|
-
type: [boolean, 'null']
|
|
15206
|
-
description: >
|
|
15207
|
-
Whether the box ran the task's environment: true when none of
|
|
15208
|
-
the five run-based criteria (reference_solution_is_valid,
|
|
15209
|
-
verifier_rejects_non_solutions, environment_builds_and_runs,
|
|
15210
|
-
verification_is_stable, limits_allow_the_task) is unknown, so a
|
|
15211
|
-
reading-only `no_problem_found` is never mistaken for a run.
|
|
15212
|
-
Null exactly when `label` is null.
|
|
15213
15197
|
cost_usd:
|
|
15214
15198
|
type: [number, 'null']
|
|
15215
15199
|
attempts:
|
|
@@ -15268,6 +15252,7 @@ components:
|
|
|
15268
15252
|
- name
|
|
15269
15253
|
- status
|
|
15270
15254
|
- source
|
|
15255
|
+
- agent
|
|
15271
15256
|
- model_name
|
|
15272
15257
|
- reasoning_effort
|
|
15273
15258
|
- rubric
|
|
@@ -15294,11 +15279,14 @@ components:
|
|
|
15294
15279
|
enum: [queued, running, completed]
|
|
15295
15280
|
source:
|
|
15296
15281
|
$ref: '#/components/schemas/CheckSource'
|
|
15282
|
+
agent:
|
|
15283
|
+
type: string
|
|
15284
|
+
description: The agent every task's checker ran on — named at create, or the default (claude); every check from before the choice existed ran on claude.
|
|
15297
15285
|
model_name:
|
|
15298
15286
|
type: string
|
|
15299
15287
|
reasoning_effort:
|
|
15300
|
-
type: string
|
|
15301
|
-
description: The effort every task's checker ran at — named at create, or the model's default, resolved at accept and stamped.
|
|
15288
|
+
type: [string, 'null']
|
|
15289
|
+
description: The effort every task's checker ran at — named at create, or the model's default on the agent, resolved at accept and stamped; null when the agent takes none.
|
|
15302
15290
|
rubric:
|
|
15303
15291
|
$ref: '#/components/schemas/Rubric'
|
|
15304
15292
|
prompt:
|
|
@@ -15352,8 +15340,10 @@ components:
|
|
|
15352
15340
|
echoes for a job created with `analyze: {}`, except `prompt`, which
|
|
15353
15341
|
`AnalyzeConfig` serves as null (the built-in body) and this serves as
|
|
15354
15342
|
that body's template text.
|
|
15355
|
-
required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider]
|
|
15343
|
+
required: [agent, model_name, rubric, prompt, reasoning_effort, sandbox_provider]
|
|
15356
15344
|
properties:
|
|
15345
|
+
agent:
|
|
15346
|
+
type: string
|
|
15357
15347
|
model_name:
|
|
15358
15348
|
type: string
|
|
15359
15349
|
rubric:
|
|
@@ -15365,8 +15355,8 @@ components:
|
|
|
15365
15355
|
`AnalyzeConfigInput.prompt` to run the default body explicitly,
|
|
15366
15356
|
or edit it from here.
|
|
15367
15357
|
reasoning_effort:
|
|
15368
|
-
type: string
|
|
15369
|
-
description: The effort the default model runs at when the config names none.
|
|
15358
|
+
type: [string, 'null']
|
|
15359
|
+
description: The effort the default model runs at on the default agent when the config names none; null if that agent takes none.
|
|
15370
15360
|
sandbox_provider:
|
|
15371
15361
|
allOf:
|
|
15372
15362
|
- $ref: '#/components/schemas/SandboxProvider'
|
|
@@ -15379,8 +15369,10 @@ components:
|
|
|
15379
15369
|
a check created with no config, except `prompt`, which `Check`
|
|
15380
15370
|
serves as null (the built-in body) and this serves as that body's
|
|
15381
15371
|
template text.
|
|
15382
|
-
required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider]
|
|
15372
|
+
required: [agent, model_name, rubric, prompt, reasoning_effort, sandbox_provider]
|
|
15383
15373
|
properties:
|
|
15374
|
+
agent:
|
|
15375
|
+
type: string
|
|
15384
15376
|
model_name:
|
|
15385
15377
|
type: string
|
|
15386
15378
|
rubric:
|
|
@@ -15392,8 +15384,8 @@ components:
|
|
|
15392
15384
|
`CheckConfigInput.prompt` to run the default body explicitly,
|
|
15393
15385
|
or edit it from here.
|
|
15394
15386
|
reasoning_effort:
|
|
15395
|
-
type: string
|
|
15396
|
-
description: The effort the default model runs at when the config names none.
|
|
15387
|
+
type: [string, 'null']
|
|
15388
|
+
description: The effort the default model runs at on the default agent when the config names none; null if that agent takes none.
|
|
15397
15389
|
sandbox_provider:
|
|
15398
15390
|
allOf:
|
|
15399
15391
|
- $ref: '#/components/schemas/SandboxProvider'
|
|
@@ -18394,34 +18386,46 @@ components:
|
|
|
18394
18386
|
x-wave: 3
|
|
18395
18387
|
type: object
|
|
18396
18388
|
description: >
|
|
18397
|
-
The
|
|
18398
|
-
`AnalyzeConfigInput.
|
|
18399
|
-
`
|
|
18400
|
-
|
|
18389
|
+
The rubric agents' roster and defaults (the analyzer's and the
|
|
18390
|
+
checker's alike): the agents `AnalyzeConfigInput.agent` and
|
|
18391
|
+
`CheckConfigInput.agent` accept and the one an omitted agent
|
|
18392
|
+
takes, the model an omitted `model_name` takes, and per agent
|
|
18393
|
+
the efforts its `reasoning_effort` accepts and every roster
|
|
18401
18394
|
model with the effort an omitted `reasoning_effort` takes for
|
|
18402
18395
|
it. Published because the effort changes what an analysis costs
|
|
18403
18396
|
and how it reads, and a client comparing two waves must know
|
|
18404
18397
|
what "omitted" meant in each.
|
|
18405
|
-
required: [
|
|
18398
|
+
required: [default_agent, default_model, agents]
|
|
18406
18399
|
properties:
|
|
18400
|
+
default_agent:
|
|
18401
|
+
type: string
|
|
18407
18402
|
default_model:
|
|
18408
18403
|
type: string
|
|
18409
|
-
|
|
18410
|
-
type: array
|
|
18411
|
-
items:
|
|
18412
|
-
type: string
|
|
18413
|
-
models:
|
|
18404
|
+
agents:
|
|
18414
18405
|
type: array
|
|
18415
18406
|
items:
|
|
18416
18407
|
type: object
|
|
18417
|
-
required: [
|
|
18408
|
+
required: [name, reasoning_efforts, models]
|
|
18418
18409
|
properties:
|
|
18419
|
-
|
|
18420
|
-
type: string
|
|
18421
|
-
model_id:
|
|
18422
|
-
type: string
|
|
18423
|
-
default_reasoning_effort:
|
|
18410
|
+
name:
|
|
18424
18411
|
type: string
|
|
18412
|
+
reasoning_efforts:
|
|
18413
|
+
type: array
|
|
18414
|
+
description: Empty for an agent that takes no effort.
|
|
18415
|
+
items:
|
|
18416
|
+
type: string
|
|
18417
|
+
models:
|
|
18418
|
+
type: array
|
|
18419
|
+
items:
|
|
18420
|
+
type: object
|
|
18421
|
+
required: [alias, model_id, default_reasoning_effort]
|
|
18422
|
+
properties:
|
|
18423
|
+
alias:
|
|
18424
|
+
type: string
|
|
18425
|
+
model_id:
|
|
18426
|
+
type: string
|
|
18427
|
+
default_reasoning_effort:
|
|
18428
|
+
type: [string, 'null']
|
|
18425
18429
|
agent_registration:
|
|
18426
18430
|
type: object
|
|
18427
18431
|
description: Rules a bring-your-own agent registration must satisfy.
|