@evolvingmachines/evolve 0.0.66 → 0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RFLMRPGK.js → chunk-DI6LPW2B.js} +5 -5
- package/dist/cli/index.cjs +38 -38
- package/dist/cli/index.d.cts +1 -1
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +30 -30
- package/dist/index.cjs +64 -60
- package/dist/index.d.cts +295 -16
- package/dist/index.d.ts +295 -16
- package/dist/index.js +48 -44
- package/dist/{types-84M4J_Vm.d.cts → types-Dj0iMl5V.d.cts} +59 -46
- package/dist/{types-84M4J_Vm.d.ts → types-Dj0iMl5V.d.ts} +59 -46
- package/harness-capabilities.json +285 -0
- package/package.json +9 -6
- package/skills/evolve-evals/references/cli-reference/analyze.md +4 -2
- package/skills/evolve-evals/references/cli-reference/check.md +3 -2
- package/skills/evolve-evals/references/cli-reference/run.md +2 -1
- package/skills/evolve-evals/references/core-concepts/agents.md +5 -0
- package/skills/evolve-evals/references/core-concepts/analyze.md +1 -1
- package/skills/evolve-evals/references/core-concepts/check.md +2 -2
- package/skills/evolve-evals/references/core-concepts/models.md +7 -7
- package/skills/evolve-evals/references/sdk-reference/analyses.md +4 -3
- package/skills/evolve-evals/references/sdk-reference/checks.md +3 -2
- package/skills/evolve-evals/references/sdk-reference/meta.md +1 -1
- package/skills/evolve-evals/references/sdk-reference/methods/checks.md +2 -0
- package/skills/evolve-evals/references/sdk-reference/methods/jobs.md +3 -1
- package/skills/evolve-evals/references/sdk-reference/methods/meta.md +6 -2
- package/skills/evolve-evals/references/sdk-reference/types.md +11 -5
- package/spec/openapi.yaml +145 -86
|
@@ -208,10 +208,11 @@ interface RetryConfig {
|
|
|
208
208
|
}
|
|
209
209
|
|
|
210
210
|
interface AnalyzeConfig {
|
|
211
|
+
agent: string;
|
|
211
212
|
model_name: string;
|
|
212
213
|
rubric: Rubric;
|
|
213
214
|
prompt: string | null;
|
|
214
|
-
reasoning_effort: string;
|
|
215
|
+
reasoning_effort: string | null;
|
|
215
216
|
sandbox_provider: EvalSandboxProvider;
|
|
216
217
|
n_concurrent: number | null;
|
|
217
218
|
passing: boolean;
|
|
@@ -316,8 +317,9 @@ interface CheckRow {
|
|
|
316
317
|
name: string;
|
|
317
318
|
status: CheckStatus;
|
|
318
319
|
source: CheckSource;
|
|
320
|
+
agent: string;
|
|
319
321
|
model_name: string;
|
|
320
|
-
reasoning_effort: string;
|
|
322
|
+
reasoning_effort: string | null;
|
|
321
323
|
sandbox_provider: EvalSandboxProvider;
|
|
322
324
|
org: string;
|
|
323
325
|
visibility: JobVisibility;
|
|
@@ -757,6 +759,7 @@ interface TrialAnalysis {
|
|
|
757
759
|
job_id: string;
|
|
758
760
|
task_name: string;
|
|
759
761
|
status: AnalysisStatus;
|
|
762
|
+
agent: string;
|
|
760
763
|
model_name: string;
|
|
761
764
|
reasoning_effort: string | null;
|
|
762
765
|
rubric: Rubric;
|
|
@@ -788,10 +791,11 @@ interface AnalysisFailure {
|
|
|
788
791
|
}
|
|
789
792
|
|
|
790
793
|
interface AnalyzeDefaults {
|
|
794
|
+
agent: string;
|
|
791
795
|
model_name: string;
|
|
792
796
|
rubric: Rubric;
|
|
793
797
|
prompt: string;
|
|
794
|
-
reasoning_effort: string;
|
|
798
|
+
reasoning_effort: string | null;
|
|
795
799
|
sandbox_provider: EvalSandboxProvider;
|
|
796
800
|
}
|
|
797
801
|
```
|
|
@@ -806,8 +810,9 @@ interface Check {
|
|
|
806
810
|
name: string;
|
|
807
811
|
status: CheckStatus;
|
|
808
812
|
source: CheckSource;
|
|
813
|
+
agent: string;
|
|
809
814
|
model_name: string;
|
|
810
|
-
reasoning_effort: string;
|
|
815
|
+
reasoning_effort: string | null;
|
|
811
816
|
rubric: Rubric;
|
|
812
817
|
prompt: string | null;
|
|
813
818
|
sandbox_provider: EvalSandboxProvider;
|
|
@@ -843,10 +848,11 @@ interface TaskCheck {
|
|
|
843
848
|
}
|
|
844
849
|
|
|
845
850
|
interface CheckDefaults {
|
|
851
|
+
agent: string;
|
|
846
852
|
model_name: string;
|
|
847
853
|
rubric: Rubric;
|
|
848
854
|
prompt: string;
|
|
849
|
-
reasoning_effort: string;
|
|
855
|
+
reasoning_effort: string | null;
|
|
850
856
|
sandbox_provider: EvalSandboxProvider;
|
|
851
857
|
}
|
|
852
858
|
```
|
package/spec/openapi.yaml
CHANGED
|
@@ -101,9 +101,9 @@ info:
|
|
|
101
101
|
directory and runs it as a new local Harbor job
|
|
102
102
|
(their src/harbor/analyze/analyzer.py `run_analyze`). Here analysis is
|
|
103
103
|
a NATIVE capability of the platform — the worker runs the same
|
|
104
|
-
analyzer (claude-code over the trial's
|
|
105
|
-
task, rubric-driven, `analysis.json`
|
|
106
|
-
criterion) server-side, and the result is stored and served on the
|
|
104
|
+
analyzer (the chosen agent, claude-code by default, over the trial's
|
|
105
|
+
Harbor-shape tree plus its task, rubric-driven, `analysis.json`
|
|
106
|
+
validated criterion-for-criterion) server-side, and the result is stored and served on the
|
|
107
107
|
trial body (`Trial.analysis`) instead of written into a local
|
|
108
108
|
directory. Two named deviations from Harbor's shape: (1) server-side
|
|
109
109
|
execution — no wrapper job appears anywhere on this surface; the
|
|
@@ -111,8 +111,8 @@ info:
|
|
|
111
111
|
`JobCreate.analyze` runs the analyzer per trial as each trial settles,
|
|
112
112
|
which Harbor has no equivalent of (their analyze is always a manual
|
|
113
113
|
follow-up command; `POST /api/jobs/{jobId}/analyze` is that manual
|
|
114
|
-
verb here). Vocabulary is Harbor's: `model_name`, rubric criteria
|
|
115
|
-
`{name, description, guidance}` (their cli/analyze.py `--model` /
|
|
114
|
+
verb here). Vocabulary is Harbor's: `agent`, `model_name`, rubric criteria
|
|
115
|
+
`{name, description, guidance}` (their cli/analyze.py `--agent` / `--model` /
|
|
116
116
|
`--rubric`, cli/quality_checker/models.py RubricCriterion), result
|
|
117
117
|
`{summary, checks{outcome, explanation}, estimated_cost_usd}` (their
|
|
118
118
|
analyze/models.py AnalyzeResult). Selection and width are Harbor's
|
|
@@ -132,7 +132,7 @@ info:
|
|
|
132
132
|
own record: the caller uploads the task directory (or a directory of
|
|
133
133
|
task directories) as one archive (`POST /api/checks`, multipart —
|
|
134
134
|
`archive` + `config`), the worker runs the SAME checker per task
|
|
135
|
-
(claude-code over the task's byte-sacred tree, the platform's default
|
|
135
|
+
(the chosen agent, claude-code by default, over the task's byte-sacred tree, the platform's default
|
|
136
136
|
check prompt and rubric rendered through Harbor's own tokens,
|
|
137
137
|
`check-result.json` validated criterion for criterion) server-side,
|
|
138
138
|
and the report is served as the `Check`
|
|
@@ -145,7 +145,7 @@ info:
|
|
|
145
145
|
`claude-sonnet-4-6` (one roster, one default for both rubric agents);
|
|
146
146
|
and the platform's `reasoning_effort` and `sandbox_provider` beside
|
|
147
147
|
Harbor's knobs, exactly as on `AnalyzeConfigInput`. Vocabulary is
|
|
148
|
-
Harbor's: `model_name`, `rubric`, `prompt` (their cli/analyze.py:88-
|
|
148
|
+
Harbor's: `agent`, `model_name`, `rubric`, `prompt` (their cli/analyze.py:88-
|
|
149
149
|
103), `n_concurrent` (:117-119), `include_task_names` /
|
|
150
150
|
`exclude_task_names` / `n_tasks` (:121-131; fnmatch globs and the cap
|
|
151
151
|
of checker.py:132-138), result `{task_name, checks{outcome,
|
|
@@ -1459,9 +1459,11 @@ paths:
|
|
|
1459
1459
|
│ │ at Harbor's own tee name for the
|
|
1460
1460
|
│ │ harness (claude-code.txt, codex.txt,
|
|
1461
1461
|
│ │ gemini-cli.txt, qwen-code.txt,
|
|
1462
|
-
│ │ kimi-code.txt, opencode.txt
|
|
1463
|
-
│ │ droid.txt,
|
|
1464
|
-
│ │
|
|
1462
|
+
│ │ kimi-code.txt, opencode.txt, pi.txt,
|
|
1463
|
+
│ │ antigravity-stream.jsonl; droid.txt,
|
|
1464
|
+
│ │ prime-agent.txt, dsh.txt, zcode.txt, and
|
|
1465
|
+
│ │ stdout.log for a harness Harbor has
|
|
1466
|
+
│ │ no name for)
|
|
1465
1467
|
│ ├── stderr.log the harness process's stderr stream,
|
|
1466
1468
|
│ │ when stored
|
|
1467
1469
|
│ ├── trace-parsed.jsonl the parsed event trace, one
|
|
@@ -1801,9 +1803,9 @@ paths:
|
|
|
1801
1803
|
summary: Analyze a job's trial traces (rubric-driven)
|
|
1802
1804
|
description: |
|
|
1803
1805
|
Runs Harbor-style trace analysis over a terminal job's trials,
|
|
1804
|
-
server-side: for each trial the analyzer agent (claude-code
|
|
1805
|
-
own sandbox, under the network Harbor's analyze
|
|
1806
|
-
it — open by default) reads the trial's Harbor-shape tree plus its
|
|
1806
|
+
server-side: for each trial the analyzer agent (`agent`, claude-code
|
|
1807
|
+
by default, in its own sandbox, under the network Harbor's analyze
|
|
1808
|
+
wrapper task gives it — open by default) reads the trial's Harbor-shape tree plus its
|
|
1807
1809
|
original task and rules every rubric criterion, storing
|
|
1808
1810
|
`{summary, checks, estimated_cost_usd}` on the trial
|
|
1809
1811
|
(`Trial.analysis`) and the aggregate on the job
|
|
@@ -1823,7 +1825,7 @@ paths:
|
|
|
1823
1825
|
`max_concurrent_analyses`).
|
|
1824
1826
|
|
|
1825
1827
|
An analyzer run that completes without a valid `analysis.json` is
|
|
1826
|
-
automatically re-run at most once (fresh sandbox, same model and
|
|
1828
|
+
automatically re-run at most once (fresh sandbox, same agent, model and
|
|
1827
1829
|
rubric); the analysis object's `attempts` records how far the
|
|
1828
1830
|
ladder went. A run cut by its budget is not that class: it settles
|
|
1829
1831
|
`failed` with phase `timeout` at once and is never re-run — a
|
|
@@ -5692,8 +5694,8 @@ paths:
|
|
|
5692
5694
|
|
|
5693
5695
|
analyze-<trial>__<7>/
|
|
5694
5696
|
├── config.json TrialConfig, partial: the wrapper
|
|
5695
|
-
│ task's name, the agent (
|
|
5696
|
-
│ the model), and under `x_evolve`
|
|
5697
|
+
│ task's name, the agent (its Harbor
|
|
5698
|
+
│ name, the model), and under `x_evolve`
|
|
5697
5699
|
│ the frozen policy — rubric, prompt,
|
|
5698
5700
|
│ reasoning_effort — plus the analyzed
|
|
5699
5701
|
│ trial/job/task and the provider
|
|
@@ -5721,13 +5723,15 @@ paths:
|
|
|
5721
5723
|
│ validator (an infrastructure phase)
|
|
5722
5724
|
├── agent/
|
|
5723
5725
|
│ ├── claude-code.txt the analyzer's stdout stream at
|
|
5724
|
-
│ │ Harbor's own tee name for
|
|
5726
|
+
│ │ Harbor's own tee name for its agent
|
|
5727
|
+
│ │ (claude-code.txt, codex.txt, …)
|
|
5725
5728
|
│ ├── stderr.log its stderr, when stored
|
|
5726
5729
|
│ ├── trace-parsed.jsonl the run's parsed event trace
|
|
5727
5730
|
│ ├── .claude/ … the captured home at its real names,
|
|
5728
5731
|
│ ├── agent-home.json the capture record beside it
|
|
5729
|
-
│ └── sessions/ … Harbor's own
|
|
5730
|
-
│
|
|
5732
|
+
│ └── sessions/ … Harbor's own copies of the home, as
|
|
5733
|
+
│ on a trial (claude: the .claude/
|
|
5734
|
+
│ subtree, claude_code.py:482)
|
|
5731
5735
|
├── verifier/ ONLY when the validator ruled —
|
|
5732
5736
|
│ │ completed, or failed `invalid_result`
|
|
5733
5737
|
│ ├── test-stdout.txt the validator's printed reasons
|
|
@@ -5804,8 +5808,8 @@ paths:
|
|
|
5804
5808
|
gzipped tarball (`archive`), or a published dataset is named
|
|
5805
5809
|
instead (`dataset`, the hosted form; `CheckRequest.dataset` states
|
|
5806
5810
|
the deviation and its reason), and for each task directory it holds
|
|
5807
|
-
the checker agent (claude-code
|
|
5808
|
-
sandbox, under the network Harbor's check wrapper task gives it —
|
|
5811
|
+
the checker agent (`agent`, claude-code by default — Harbor's own
|
|
5812
|
+
check agent — in its own sandbox, under the network Harbor's check wrapper task gives it —
|
|
5809
5813
|
open by default) reads EVERY file of the task and rules every rubric
|
|
5810
5814
|
criterion `pass`, `fail`, `not_applicable` or `unknown` with a
|
|
5811
5815
|
rationale and the evidence behind it, writing `check-result.json`
|
|
@@ -12359,7 +12363,7 @@ components:
|
|
|
12359
12363
|
type: object
|
|
12360
12364
|
description: |
|
|
12361
12365
|
Trace-analysis configuration — Harbor's `harbor analyze` vocabulary
|
|
12362
|
-
(their cli/analyze.py: `--model`, `--rubric`, `--prompt`). PRESENCE of this
|
|
12366
|
+
(their cli/analyze.py: `--agent`, `--model`, `--rubric`, `--prompt`). PRESENCE of this
|
|
12363
12367
|
object is the switch: on `JobCreate.analyze` it arms the embedded
|
|
12364
12368
|
trigger (each trial is analyzed server-side right after it settles;
|
|
12365
12369
|
CANCELLED trials are skipped — withdrawn work is not billed an
|
|
@@ -12371,13 +12375,13 @@ components:
|
|
|
12371
12375
|
`GET /api/analyses/defaults`; Harbor's own analyze-rubric.toml
|
|
12372
12376
|
still runs when passed as `rubric`).
|
|
12373
12377
|
|
|
12374
|
-
The analyzer
|
|
12375
|
-
|
|
12376
|
-
analyze wrapper task gives it (open by default;
|
|
12377
|
-
unchanged) — on the provider `sandbox_provider`
|
|
12378
|
-
platform's analysis default when it names none; its
|
|
12379
|
-
per analysis and metered as its own line, never
|
|
12380
|
-
trial's own bill.
|
|
12378
|
+
The analyzer runs the agent `agent` names (claude, Harbor's own
|
|
12379
|
+
default claude-code, when it names none) in its own sandbox, under
|
|
12380
|
+
the network Harbor's analyze wrapper task gives it (open by default;
|
|
12381
|
+
the model door is unchanged) — on the provider `sandbox_provider`
|
|
12382
|
+
names, or the platform's analysis default when it names none; its
|
|
12383
|
+
spend is capped per analysis and metered as its own line, never
|
|
12384
|
+
blended into the trial's own bill.
|
|
12381
12385
|
|
|
12382
12386
|
Which trials, and how wide, are Harbor's own analyze options with
|
|
12383
12387
|
their exact names: `n_concurrent` (`-n/--n-concurrent`), `passing`
|
|
@@ -12393,9 +12397,7 @@ components:
|
|
|
12393
12397
|
takes it; on `JobCreate.analyze` the trials do not exist yet.
|
|
12394
12398
|
|
|
12395
12399
|
Harbor's other analyze options, and why they are not here:
|
|
12396
|
-
`-
|
|
12397
|
-
instruction templates, the roster check and the sandbox policy are
|
|
12398
|
-
written against it); `--job-name` and `-o/--jobs-dir` — a hosted
|
|
12400
|
+
`--job-name` and `-o/--jobs-dir` — a hosted
|
|
12399
12401
|
analysis attaches to its source job and is served on that job's
|
|
12400
12402
|
trials, so there is no wrapper job to name and no directory to
|
|
12401
12403
|
write; `-k/--n-attempts` (k analyses per trial) — deferred: each
|
|
@@ -12410,6 +12412,21 @@ components:
|
|
|
12410
12412
|
(task quality checks, their cli/analyze.py:126-137) and are not
|
|
12411
12413
|
adopted here.
|
|
12412
12414
|
properties:
|
|
12415
|
+
agent:
|
|
12416
|
+
type: string
|
|
12417
|
+
default: claude
|
|
12418
|
+
description: >
|
|
12419
|
+
The agent the analyzer runs on — Harbor's `-a/--agent` (their
|
|
12420
|
+
cli/analyze.py:258, default claude-code), named in the
|
|
12421
|
+
`agents[].name` vocabulary (claude, not claude-code) so one set
|
|
12422
|
+
of names covers arms and reviewers: any built-in agent
|
|
12423
|
+
`analyze.agents` lists on `GET /api/meta`. `model_name` must be
|
|
12424
|
+
on this agent's roster and `reasoning_effort` must be one it
|
|
12425
|
+
accepts; with an agent whose roster lacks the default model,
|
|
12426
|
+
`model_name` is required. A registered (bring-your-own) agent
|
|
12427
|
+
is not accepted. Anything else is refused `invalid_input`
|
|
12428
|
+
naming `analyze.agent`. Stored as given and frozen into each
|
|
12429
|
+
analysis (`TrialAnalysis.agent`).
|
|
12413
12430
|
model_name:
|
|
12414
12431
|
type: string
|
|
12415
12432
|
default: openrouter/deepseek/deepseek-v4.1-flash
|
|
@@ -12432,9 +12449,10 @@ components:
|
|
|
12432
12449
|
default is the roster alias), the wire id is resolved only when
|
|
12433
12450
|
the analyzer runs, and every stored analysis serves the
|
|
12434
12451
|
spelling it was created under. Must be on
|
|
12435
|
-
the
|
|
12436
|
-
|
|
12437
|
-
|
|
12452
|
+
the agent's roster (`GET /api/meta`, `analyze.agents[].models`);
|
|
12453
|
+
anything else is refused at accept (`invalid_input`, roster in
|
|
12454
|
+
the message) — including the default model on an agent whose
|
|
12455
|
+
roster lacks it, which then says `model_name` is required.
|
|
12438
12456
|
rubric:
|
|
12439
12457
|
$ref: '#/components/schemas/Rubric'
|
|
12440
12458
|
prompt:
|
|
@@ -12472,20 +12490,22 @@ components:
|
|
|
12472
12490
|
type: string
|
|
12473
12491
|
description: >
|
|
12474
12492
|
Reasoning effort the analyzer runs at — the platform's
|
|
12475
|
-
`agents[].reasoning_effort` vocabulary applied to the analyzer
|
|
12476
|
-
|
|
12477
|
-
`analyze.reasoning_efforts` on
|
|
12478
|
-
|
|
12479
|
-
|
|
12480
|
-
|
|
12493
|
+
`agents[].reasoning_effort` vocabulary applied to the analyzer's
|
|
12494
|
+
agent, held to that agent's own rule exactly as an arm on it:
|
|
12495
|
+
the accepted values are `analyze.agents[].reasoning_efforts` on
|
|
12496
|
+
`GET /api/meta` (claude's `effort_support` is `level`, so the
|
|
12497
|
+
whole list; an agent that takes none, gemini, accepts none), and
|
|
12498
|
+
a refused value answers `invalid_input` with the same sentence
|
|
12499
|
+
an arm's does (`analyze.reasoning_effort must be one of: ...`).
|
|
12481
12500
|
Omitted, the PER-MODEL default applies —
|
|
12482
|
-
`analyze.models[].default_reasoning_effort`: `high` on
|
|
12501
|
+
`analyze.agents[].models[].default_reasoning_effort`: `high` on
|
|
12483
12502
|
openrouter/deepseek/deepseek-v4.1-flash (the default model;
|
|
12484
12503
|
DeepSeek's own documented default, the owner's ruling
|
|
12485
12504
|
2026-09-10), `max` on glm-5.3-flash (the platform's ruling
|
|
12486
|
-
2026-09-08, the effort its published scores use),
|
|
12487
|
-
|
|
12488
|
-
|
|
12505
|
+
2026-09-08, the effort its published scores use), on any agent
|
|
12506
|
+
that can honor them; the agent's own default
|
|
12507
|
+
(`agents[].default_effort`) on every other roster model; none
|
|
12508
|
+
on an agent that takes none. Whatever resolves is passed to the analyzer
|
|
12489
12509
|
EXPLICITLY on every call — never left to a provider's default —
|
|
12490
12510
|
and recorded as `TrialAnalysis.reasoning_effort`. A hosted
|
|
12491
12511
|
extension, recorded as such: Harbor's `harbor analyze` has no
|
|
@@ -12602,11 +12622,14 @@ components:
|
|
|
12602
12622
|
always states the policy it executes (same law as RetryConfig).
|
|
12603
12623
|
Echoed on the job body when the job was created with `analyze`;
|
|
12604
12624
|
each analysis additionally carries the exact policy IT ran under
|
|
12605
|
-
(`Trial.analysis.model_name` / `.rubric` / `.prompt` /
|
|
12625
|
+
(`Trial.analysis.agent` / `.model_name` / `.rubric` / `.prompt` /
|
|
12606
12626
|
`.reasoning_effort`), which a later manual re-analysis may have
|
|
12607
12627
|
changed.
|
|
12608
|
-
required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider, n_concurrent, passing, failing, n_trials, trial_ids]
|
|
12628
|
+
required: [agent, model_name, rubric, prompt, reasoning_effort, sandbox_provider, n_concurrent, passing, failing, n_trials, trial_ids]
|
|
12609
12629
|
properties:
|
|
12630
|
+
agent:
|
|
12631
|
+
type: string
|
|
12632
|
+
description: The agent this policy's analyses run on — as named, or the default (claude).
|
|
12610
12633
|
model_name:
|
|
12611
12634
|
type: string
|
|
12612
12635
|
rubric:
|
|
@@ -12618,14 +12641,15 @@ components:
|
|
|
12618
12641
|
null = the built-in prompt (the platform's default analyze body).
|
|
12619
12642
|
reasoning_effort:
|
|
12620
12643
|
x-wave: 3
|
|
12621
|
-
type: string
|
|
12644
|
+
type: [string, 'null']
|
|
12622
12645
|
description: >
|
|
12623
12646
|
The effort this policy's analyses run at. Named at create it is
|
|
12624
12647
|
served as stored; when the create named none, this echoes the
|
|
12625
|
-
per-model default of the day for `model_name`
|
|
12626
|
-
next enqueue under this policy stamps (the same nuance
|
|
12627
|
-
`sandbox_provider` below, for the same reason: the default is
|
|
12628
|
-
platform table, resolved where an analysis is enqueued).
|
|
12648
|
+
per-model default of the day for `model_name` on `agent` — the
|
|
12649
|
+
value the next enqueue under this policy stamps (the same nuance
|
|
12650
|
+
as `sandbox_provider` below, for the same reason: the default is
|
|
12651
|
+
a platform table, resolved where an analysis is enqueued). Null
|
|
12652
|
+
when the agent takes no effort.
|
|
12629
12653
|
sandbox_provider:
|
|
12630
12654
|
allOf:
|
|
12631
12655
|
- $ref: '#/components/schemas/SandboxProvider'
|
|
@@ -13964,6 +13988,7 @@ components:
|
|
|
13964
13988
|
- name
|
|
13965
13989
|
- status
|
|
13966
13990
|
- source
|
|
13991
|
+
- agent
|
|
13967
13992
|
- model_name
|
|
13968
13993
|
- reasoning_effort
|
|
13969
13994
|
- sandbox_provider
|
|
@@ -13992,11 +14017,14 @@ components:
|
|
|
13992
14017
|
source:
|
|
13993
14018
|
$ref: '#/components/schemas/CheckSource'
|
|
13994
14019
|
description: What was checked; the row's DATASETS cell (the dataset form's `name@version`; an archive shows as an archive with its sha256).
|
|
14020
|
+
agent:
|
|
14021
|
+
type: string
|
|
14022
|
+
description: The checker's agent, as Check.agent.
|
|
13995
14023
|
model_name:
|
|
13996
14024
|
type: string
|
|
13997
14025
|
description: The checker's model; the row's AGENTS cell, shown like an arm with its effort under it.
|
|
13998
14026
|
reasoning_effort:
|
|
13999
|
-
type: string
|
|
14027
|
+
type: [string, 'null']
|
|
14000
14028
|
description: The effort every task's checker ran at, as Check.reasoning_effort.
|
|
14001
14029
|
sandbox_provider:
|
|
14002
14030
|
allOf:
|
|
@@ -14783,7 +14811,7 @@ components:
|
|
|
14783
14811
|
AnalyzeResult (their analyze/models.py: `summary`, `checks` keyed
|
|
14784
14812
|
by criterion, `estimated_cost_usd`; `trial_name` is the enclosing
|
|
14785
14813
|
trial) with each check carrying the platform's evidence list and
|
|
14786
|
-
fourth outcome (AnalysisCheck); the rest is provenance — which model, rubric and prompt THIS analysis ran
|
|
14814
|
+
fourth outcome (AnalysisCheck); the rest is provenance — which agent, model, rubric and prompt THIS analysis ran
|
|
14787
14815
|
under, its lifecycle status, and its typed failure when it failed.
|
|
14788
14816
|
|
|
14789
14817
|
`estimated_cost_usd` is the analyzer agent's OWN metered spend (its
|
|
@@ -14797,6 +14825,7 @@ components:
|
|
|
14797
14825
|
- job_id
|
|
14798
14826
|
- task_name
|
|
14799
14827
|
- status
|
|
14828
|
+
- agent
|
|
14800
14829
|
- model_name
|
|
14801
14830
|
- reasoning_effort
|
|
14802
14831
|
- rubric
|
|
@@ -14841,6 +14870,12 @@ components:
|
|
|
14841
14870
|
`not_applicable` are their exact words) and the ladder follows
|
|
14842
14871
|
its object's dialect; the UI capitalizes on render, and nothing
|
|
14843
14872
|
ever compares this ladder's words against another ladder's.
|
|
14873
|
+
agent:
|
|
14874
|
+
type: string
|
|
14875
|
+
description: >
|
|
14876
|
+
The agent THIS analysis ran on (`AnalyzeConfigInput.agent`, or
|
|
14877
|
+
the default claude); every analysis from before the choice
|
|
14878
|
+
existed ran on claude and says so.
|
|
14844
14879
|
model_name:
|
|
14845
14880
|
type: string
|
|
14846
14881
|
reasoning_effort:
|
|
@@ -14850,8 +14885,8 @@ components:
|
|
|
14850
14885
|
The reasoning effort THIS analysis ran at — passed to the
|
|
14851
14886
|
analyzer explicitly, so it is what the model was asked for
|
|
14852
14887
|
(`AnalyzeConfigInput.reasoning_effort`, or the per-model
|
|
14853
|
-
default). Null
|
|
14854
|
-
stamped; never a guessed value.
|
|
14888
|
+
default). Null when its agent takes no effort, or on analyses
|
|
14889
|
+
recorded before the effort was stamped; never a guessed value.
|
|
14855
14890
|
rubric:
|
|
14856
14891
|
$ref: '#/components/schemas/Rubric'
|
|
14857
14892
|
prompt:
|
|
@@ -15006,10 +15041,11 @@ components:
|
|
|
15006
15041
|
type: object
|
|
15007
15042
|
description: |
|
|
15008
15043
|
Task-check configuration — Harbor's `harbor check` vocabulary
|
|
15009
|
-
(their cli/analyze.py:84-148 check_command). The
|
|
15044
|
+
(their cli/analyze.py:84-148 check_command). The four rubric-agent
|
|
15010
15045
|
knobs are the analyze door's, with the SAME rules and refusals
|
|
15011
15046
|
(`AnalyzeConfigInput` states them; the param prefix here is
|
|
15012
|
-
`check.`): `
|
|
15047
|
+
`check.`): `agent` (`-a/--agent`, :100; default claude, Harbor's
|
|
15048
|
+
claude-code), `model_name` (`-m/--model`; Harbor's check default is
|
|
15013
15049
|
`claude-sonnet-4-6`, :101-103 — this platform's is the analyzer's
|
|
15014
15050
|
`openrouter/deepseek/deepseek-v4.1-flash`, one roster and one default for both rubric
|
|
15015
15051
|
agents, a recorded deviation), `rubric` (`-r/--rubric`; the default
|
|
@@ -15035,10 +15071,9 @@ components:
|
|
|
15035
15071
|
matched against the directory NAME.
|
|
15036
15072
|
|
|
15037
15073
|
Harbor's other check options, and why they are not here: the same
|
|
15038
|
-
list as the analyze door's — `-
|
|
15039
|
-
|
|
15040
|
-
|
|
15041
|
-
to write), `-k/--n-attempts` (the one hosted auto re-run replaces
|
|
15074
|
+
list as the analyze door's — `-o/--jobs-dir` (a hosted check is its
|
|
15075
|
+
own record, no directory to write; `--job-name` is `name`),
|
|
15076
|
+
`-k/--n-attempts` (the one hosted auto re-run replaces
|
|
15042
15077
|
it; see `TaskCheck.attempts`), `-c/--config`, `--ak`, `--ae`,
|
|
15043
15078
|
`--ek` (local-runner knobs with no hosted meaning), `-q` (a client
|
|
15044
15079
|
rendering option, not a wire field).
|
|
@@ -15051,6 +15086,10 @@ components:
|
|
|
15051
15086
|
The caller's name for the check — Harbor's `--job-name` on
|
|
15052
15087
|
`harbor check` (cli/analyze.py:138). Omitted, the accept
|
|
15053
15088
|
timestamp in Harbor's own shape, `YYYY-MM-DD__HH-MM-SS` (UTC).
|
|
15089
|
+
agent:
|
|
15090
|
+
type: string
|
|
15091
|
+
default: claude
|
|
15092
|
+
description: The agent the checker runs on — the analyze door's `agent`, same rule.
|
|
15054
15093
|
model_name:
|
|
15055
15094
|
type: string
|
|
15056
15095
|
default: openrouter/deepseek/deepseek-v4.1-flash
|
|
@@ -15213,6 +15252,7 @@ components:
|
|
|
15213
15252
|
- name
|
|
15214
15253
|
- status
|
|
15215
15254
|
- source
|
|
15255
|
+
- agent
|
|
15216
15256
|
- model_name
|
|
15217
15257
|
- reasoning_effort
|
|
15218
15258
|
- rubric
|
|
@@ -15239,11 +15279,14 @@ components:
|
|
|
15239
15279
|
enum: [queued, running, completed]
|
|
15240
15280
|
source:
|
|
15241
15281
|
$ref: '#/components/schemas/CheckSource'
|
|
15282
|
+
agent:
|
|
15283
|
+
type: string
|
|
15284
|
+
description: The agent every task's checker ran on — named at create, or the default (claude); every check from before the choice existed ran on claude.
|
|
15242
15285
|
model_name:
|
|
15243
15286
|
type: string
|
|
15244
15287
|
reasoning_effort:
|
|
15245
|
-
type: string
|
|
15246
|
-
description: The effort every task's checker ran at — named at create, or the model's default, resolved at accept and stamped.
|
|
15288
|
+
type: [string, 'null']
|
|
15289
|
+
description: The effort every task's checker ran at — named at create, or the model's default on the agent, resolved at accept and stamped; null when the agent takes none.
|
|
15247
15290
|
rubric:
|
|
15248
15291
|
$ref: '#/components/schemas/Rubric'
|
|
15249
15292
|
prompt:
|
|
@@ -15297,8 +15340,10 @@ components:
|
|
|
15297
15340
|
echoes for a job created with `analyze: {}`, except `prompt`, which
|
|
15298
15341
|
`AnalyzeConfig` serves as null (the built-in body) and this serves as
|
|
15299
15342
|
that body's template text.
|
|
15300
|
-
required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider]
|
|
15343
|
+
required: [agent, model_name, rubric, prompt, reasoning_effort, sandbox_provider]
|
|
15301
15344
|
properties:
|
|
15345
|
+
agent:
|
|
15346
|
+
type: string
|
|
15302
15347
|
model_name:
|
|
15303
15348
|
type: string
|
|
15304
15349
|
rubric:
|
|
@@ -15310,8 +15355,8 @@ components:
|
|
|
15310
15355
|
`AnalyzeConfigInput.prompt` to run the default body explicitly,
|
|
15311
15356
|
or edit it from here.
|
|
15312
15357
|
reasoning_effort:
|
|
15313
|
-
type: string
|
|
15314
|
-
description: The effort the default model runs at when the config names none.
|
|
15358
|
+
type: [string, 'null']
|
|
15359
|
+
description: The effort the default model runs at on the default agent when the config names none; null if that agent takes none.
|
|
15315
15360
|
sandbox_provider:
|
|
15316
15361
|
allOf:
|
|
15317
15362
|
- $ref: '#/components/schemas/SandboxProvider'
|
|
@@ -15324,8 +15369,10 @@ components:
|
|
|
15324
15369
|
a check created with no config, except `prompt`, which `Check`
|
|
15325
15370
|
serves as null (the built-in body) and this serves as that body's
|
|
15326
15371
|
template text.
|
|
15327
|
-
required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider]
|
|
15372
|
+
required: [agent, model_name, rubric, prompt, reasoning_effort, sandbox_provider]
|
|
15328
15373
|
properties:
|
|
15374
|
+
agent:
|
|
15375
|
+
type: string
|
|
15329
15376
|
model_name:
|
|
15330
15377
|
type: string
|
|
15331
15378
|
rubric:
|
|
@@ -15337,8 +15384,8 @@ components:
|
|
|
15337
15384
|
`CheckConfigInput.prompt` to run the default body explicitly,
|
|
15338
15385
|
or edit it from here.
|
|
15339
15386
|
reasoning_effort:
|
|
15340
|
-
type: string
|
|
15341
|
-
description: The effort the default model runs at when the config names none.
|
|
15387
|
+
type: [string, 'null']
|
|
15388
|
+
description: The effort the default model runs at on the default agent when the config names none; null if that agent takes none.
|
|
15342
15389
|
sandbox_provider:
|
|
15343
15390
|
allOf:
|
|
15344
15391
|
- $ref: '#/components/schemas/SandboxProvider'
|
|
@@ -18339,34 +18386,46 @@ components:
|
|
|
18339
18386
|
x-wave: 3
|
|
18340
18387
|
type: object
|
|
18341
18388
|
description: >
|
|
18342
|
-
The
|
|
18343
|
-
`AnalyzeConfigInput.
|
|
18344
|
-
`
|
|
18345
|
-
|
|
18389
|
+
The rubric agents' roster and defaults (the analyzer's and the
|
|
18390
|
+
checker's alike): the agents `AnalyzeConfigInput.agent` and
|
|
18391
|
+
`CheckConfigInput.agent` accept and the one an omitted agent
|
|
18392
|
+
takes, the model an omitted `model_name` takes, and per agent
|
|
18393
|
+
the efforts its `reasoning_effort` accepts and every roster
|
|
18346
18394
|
model with the effort an omitted `reasoning_effort` takes for
|
|
18347
18395
|
it. Published because the effort changes what an analysis costs
|
|
18348
18396
|
and how it reads, and a client comparing two waves must know
|
|
18349
18397
|
what "omitted" meant in each.
|
|
18350
|
-
required: [
|
|
18398
|
+
required: [default_agent, default_model, agents]
|
|
18351
18399
|
properties:
|
|
18400
|
+
default_agent:
|
|
18401
|
+
type: string
|
|
18352
18402
|
default_model:
|
|
18353
18403
|
type: string
|
|
18354
|
-
|
|
18355
|
-
type: array
|
|
18356
|
-
items:
|
|
18357
|
-
type: string
|
|
18358
|
-
models:
|
|
18404
|
+
agents:
|
|
18359
18405
|
type: array
|
|
18360
18406
|
items:
|
|
18361
18407
|
type: object
|
|
18362
|
-
required: [
|
|
18408
|
+
required: [name, reasoning_efforts, models]
|
|
18363
18409
|
properties:
|
|
18364
|
-
|
|
18365
|
-
type: string
|
|
18366
|
-
model_id:
|
|
18367
|
-
type: string
|
|
18368
|
-
default_reasoning_effort:
|
|
18410
|
+
name:
|
|
18369
18411
|
type: string
|
|
18412
|
+
reasoning_efforts:
|
|
18413
|
+
type: array
|
|
18414
|
+
description: Empty for an agent that takes no effort.
|
|
18415
|
+
items:
|
|
18416
|
+
type: string
|
|
18417
|
+
models:
|
|
18418
|
+
type: array
|
|
18419
|
+
items:
|
|
18420
|
+
type: object
|
|
18421
|
+
required: [alias, model_id, default_reasoning_effort]
|
|
18422
|
+
properties:
|
|
18423
|
+
alias:
|
|
18424
|
+
type: string
|
|
18425
|
+
model_id:
|
|
18426
|
+
type: string
|
|
18427
|
+
default_reasoning_effort:
|
|
18428
|
+
type: [string, 'null']
|
|
18370
18429
|
agent_registration:
|
|
18371
18430
|
type: object
|
|
18372
18431
|
description: Rules a bring-your-own agent registration must satisfy.
|