@evolvingmachines/evolve 0.0.66 → 0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RFLMRPGK.js → chunk-DI6LPW2B.js} +5 -5
- package/dist/cli/index.cjs +38 -38
- package/dist/cli/index.d.cts +1 -1
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +30 -30
- package/dist/index.cjs +64 -60
- package/dist/index.d.cts +295 -16
- package/dist/index.d.ts +295 -16
- package/dist/index.js +48 -44
- package/dist/{types-84M4J_Vm.d.cts → types-Dj0iMl5V.d.cts} +59 -46
- package/dist/{types-84M4J_Vm.d.ts → types-Dj0iMl5V.d.ts} +59 -46
- package/harness-capabilities.json +285 -0
- package/package.json +9 -6
- package/skills/evolve-evals/references/cli-reference/analyze.md +4 -2
- package/skills/evolve-evals/references/cli-reference/check.md +3 -2
- package/skills/evolve-evals/references/cli-reference/run.md +2 -1
- package/skills/evolve-evals/references/core-concepts/agents.md +5 -0
- package/skills/evolve-evals/references/core-concepts/analyze.md +1 -1
- package/skills/evolve-evals/references/core-concepts/check.md +2 -2
- package/skills/evolve-evals/references/core-concepts/models.md +7 -7
- package/skills/evolve-evals/references/sdk-reference/analyses.md +4 -3
- package/skills/evolve-evals/references/sdk-reference/checks.md +3 -2
- package/skills/evolve-evals/references/sdk-reference/meta.md +1 -1
- package/skills/evolve-evals/references/sdk-reference/methods/checks.md +2 -0
- package/skills/evolve-evals/references/sdk-reference/methods/jobs.md +3 -1
- package/skills/evolve-evals/references/sdk-reference/methods/meta.md +6 -2
- package/skills/evolve-evals/references/sdk-reference/types.md +11 -5
- package/spec/openapi.yaml +145 -86
|
@@ -450,8 +450,8 @@ interface Rubric {
|
|
|
450
450
|
* openrouter/deepseek/deepseek-v4.1-flash at its per-model effort (high)
|
|
451
451
|
* over the platform's default analyze rubric and its default prompt body.
|
|
452
452
|
*
|
|
453
|
-
* The analyzer
|
|
454
|
-
*
|
|
453
|
+
* The analyzer runs on the `agent` harness (default claude, Harbor's claude-code)
|
|
454
|
+
* in its own sealed sandbox — on the provider `sandbox_provider`
|
|
455
455
|
* names, or the platform's analysis default when it names none; its spend is
|
|
456
456
|
* capped per analysis and metered as its own line, never blended into the
|
|
457
457
|
* trial's own bill.
|
|
@@ -460,12 +460,14 @@ interface Rubric {
|
|
|
460
460
|
* exact names — `n_concurrent` (`-n/--n-concurrent`), `passing` / `failing`,
|
|
461
461
|
* `n_trials` (`-l/--n-trials`; their cli/analyze.py:278-290). All omitted
|
|
462
462
|
* is every analyzable trial, as wide as the organization's
|
|
463
|
-
* `max_concurrent_analyses` allows. Harbor's
|
|
463
|
+
* `max_concurrent_analyses` allows. Harbor's `--job-name`,
|
|
464
464
|
* `-o/--jobs-dir`, `-k/--n-attempts` and the local-runner kwargs are not
|
|
465
465
|
* on this surface; the contract (`AnalyzeConfigInput` in spec/openapi.yaml)
|
|
466
466
|
* records each with its reason.
|
|
467
467
|
*/
|
|
468
468
|
interface AnalyzeConfigInput {
|
|
469
|
+
/** Harbor's `-a/--agent`, spelled as the arms spell it (claude, not claude-code) so one set of names covers arms and reviewers; omitted: claude. The agents: `GET /api/meta` `analyze.agents`. */
|
|
470
|
+
agent?: string;
|
|
469
471
|
/**
|
|
470
472
|
* Model the analyzer agent runs — Harbor's `--model`. The default is
|
|
471
473
|
* openrouter/deepseek/deepseek-v4.1-flash on this platform's claude
|
|
@@ -483,7 +485,7 @@ interface AnalyzeConfigInput {
|
|
|
483
485
|
* spelling is accepted and stored AS GIVEN (the default is the roster
|
|
484
486
|
* alias), the wire id is resolved only when the analyzer runs, and every
|
|
485
487
|
* stored analysis serves the spelling it was created under. Must be on
|
|
486
|
-
* the
|
|
488
|
+
* the agent's roster (`GET /api/meta`, `analyze.agents[].models`); anything else
|
|
487
489
|
* is refused at accept (`invalid_input`, roster in the message).
|
|
488
490
|
*/
|
|
489
491
|
model_name?: string;
|
|
@@ -511,15 +513,14 @@ interface AnalyzeConfigInput {
|
|
|
511
513
|
prompt?: string;
|
|
512
514
|
/**
|
|
513
515
|
* Reasoning effort the analyzer runs at — the platform's `agents[].
|
|
514
|
-
* reasoning_effort` vocabulary applied to the analyzer
|
|
515
|
-
*
|
|
516
|
-
* `analyze.reasoning_efforts`, an unknown value is refused
|
|
516
|
+
* reasoning_effort` vocabulary applied to the analyzer's agent: the accepted
|
|
517
|
+
* values are `GET /api/meta`'s `analyze.agents[].reasoning_efforts`, an unknown value is refused
|
|
517
518
|
* `invalid_input` exactly as an arm's is. Omitted, the PER-MODEL default
|
|
518
|
-
* applies (`analyze.models[].default_reasoning_effort`: high on
|
|
519
|
+
* applies (`analyze.agents[].models[].default_reasoning_effort`: high on
|
|
519
520
|
* openrouter/deepseek/deepseek-v4.1-flash, the default model — DeepSeek's
|
|
520
521
|
* own documented default, the owner's ruling 2026-09-10; max on
|
|
521
522
|
* glm-5.3-flash — the platform's ruling 2026-09-08, the effort its
|
|
522
|
-
* published scores use; the
|
|
523
|
+
* published scores use; the agent's own default elsewhere, none where it takes none). The
|
|
523
524
|
* effort is always passed to the analyzer explicitly and
|
|
524
525
|
* recorded on the analysis (`TrialAnalysis.reasoning_effort`). A hosted
|
|
525
526
|
* extension: Harbor's analyze has no effort option; this is the run
|
|
@@ -596,10 +597,12 @@ interface AnalyzeConfigInput {
|
|
|
596
597
|
* the day, resolved at accept and stored, so the record always states the
|
|
597
598
|
* policy it executes (same law as RetryConfig). Echoed on the job body when
|
|
598
599
|
* the job was created with `analyze`; each analysis additionally carries the
|
|
599
|
-
* exact policy IT ran under (`Trial.analysis.
|
|
600
|
-
* `.prompt`), which a later manual re-analysis may have changed.
|
|
600
|
+
* exact policy IT ran under (`Trial.analysis.agent` / `.model_name` /
|
|
601
|
+
* `.rubric` / `.prompt`), which a later manual re-analysis may have changed.
|
|
601
602
|
*/
|
|
602
603
|
interface AnalyzeConfig {
|
|
604
|
+
/** The agent this policy's analyses run on — as named, or the default (claude). */
|
|
605
|
+
agent: string;
|
|
603
606
|
model_name: string;
|
|
604
607
|
rubric: Rubric;
|
|
605
608
|
/** The caller's prompt template as stored; null = the platform's default analyze body. */
|
|
@@ -607,10 +610,10 @@ interface AnalyzeConfig {
|
|
|
607
610
|
/**
|
|
608
611
|
* The effort this policy's analyses run at. Named at create it is served
|
|
609
612
|
* as stored; when the create named none, this echoes the per-model
|
|
610
|
-
* default of the day for `model_name` — the value the next enqueue under
|
|
611
|
-
* this policy stamps (the same nuance as `sandbox_provider` below).
|
|
613
|
+
* default of the day for `model_name` on `agent` — the value the next enqueue under
|
|
614
|
+
* this policy stamps (the same nuance as `sandbox_provider` below). Null when the agent takes no effort.
|
|
612
615
|
*/
|
|
613
|
-
reasoning_effort: string;
|
|
616
|
+
reasoning_effort: string | null;
|
|
614
617
|
/**
|
|
615
618
|
* The provider this policy's analyses run on. Named at create it is served
|
|
616
619
|
* as stored, forever. When the create named none, this echoes the
|
|
@@ -1523,11 +1526,13 @@ interface TrialAnalysis {
|
|
|
1523
1526
|
* `failed`, never left `running` forever.
|
|
1524
1527
|
*/
|
|
1525
1528
|
status: AnalysisStatus;
|
|
1529
|
+
/** The agent THIS analysis ran on; every analysis from before the choice existed ran on claude. */
|
|
1530
|
+
agent: string;
|
|
1526
1531
|
model_name: string;
|
|
1527
1532
|
/**
|
|
1528
1533
|
* The reasoning effort THIS analysis ran at — passed to the analyzer
|
|
1529
|
-
* explicitly, so it is what the model was asked for. Null
|
|
1530
|
-
* analyses recorded before the effort was stamped.
|
|
1534
|
+
* explicitly, so it is what the model was asked for. Null when its agent
|
|
1535
|
+
* takes no effort, or on analyses recorded before the effort was stamped.
|
|
1531
1536
|
*/
|
|
1532
1537
|
reasoning_effort: string | null;
|
|
1533
1538
|
rubric: Rubric;
|
|
@@ -2268,10 +2273,11 @@ interface CheckRow {
|
|
|
2268
2273
|
name: string;
|
|
2269
2274
|
status: CheckStatus;
|
|
2270
2275
|
source: CheckSource;
|
|
2276
|
+
agent: string;
|
|
2271
2277
|
/** The checker's model. */
|
|
2272
2278
|
model_name: string;
|
|
2273
|
-
/** The effort every task's checker ran at. */
|
|
2274
|
-
reasoning_effort: string;
|
|
2279
|
+
/** The effort every task's checker ran at; null when its agent takes none. */
|
|
2280
|
+
reasoning_effort: string | null;
|
|
2275
2281
|
sandbox_provider: EvalSandboxProvider;
|
|
2276
2282
|
/** The owning organization's slug (every check has one). */
|
|
2277
2283
|
org: string;
|
|
@@ -4583,10 +4589,10 @@ interface AnalysesClient {
|
|
|
4583
4589
|
* one directory named as Harbor names the wrapper trial
|
|
4584
4590
|
* (`analyze-<analyzed trial dir>__<7 chars>/`): config.json, lock.json,
|
|
4585
4591
|
* result.json, trial.log, exception.txt (an infrastructure failure only),
|
|
4586
|
-
* agent
|
|
4592
|
+
* agent/<Harbor's tee name for its agent> (claude-code.txt, codex.txt, …),
|
|
4587
4593
|
* agent/stderr.log, agent/trace-parsed.jsonl, the captured home at its
|
|
4588
|
-
* real names with agent/agent-home.json beside it and Harbor's copy at
|
|
4589
|
-
* agent/sessions
|
|
4594
|
+
* real names with agent/agent-home.json beside it and Harbor's copy at its
|
|
4595
|
+
* per-agent slot (agent/sessions/ for claude and codex), verifier/{test-stdout.txt,reward.txt,reward.json}
|
|
4590
4596
|
* when the validator ruled (reward 1 = a valid analysis.json, 0 = it was
|
|
4591
4597
|
* refused), and artifacts/manifest.json with artifacts/analysis.json (the
|
|
4592
4598
|
* validated {summary, checks}) on a completed run — absent artifacts are
|
|
@@ -4609,8 +4615,8 @@ interface AnalysesClient {
|
|
|
4609
4615
|
/**
|
|
4610
4616
|
* Task-check configuration — Harbor's `harbor check` vocabulary (their
|
|
4611
4617
|
* cli/analyze.py:84-148 check_command), the spec's `CheckConfigInput`. The
|
|
4612
|
-
* rubric-agent
|
|
4613
|
-
* (`AnalyzeConfigInput` states them; refusals name `check.*`): `model_name`
|
|
4618
|
+
* rubric-agent knobs are the analyze door's, under the same rules
|
|
4619
|
+
* (`AnalyzeConfigInput` states them; refusals name `check.*`): `agent` (`-a/--agent`), `model_name`
|
|
4614
4620
|
* (Harbor's check default is `claude-sonnet-4-6`; this platform's is the
|
|
4615
4621
|
* analyzer's `openrouter/deepseek/deepseek-v4.1-flash` — one roster, one
|
|
4616
4622
|
* default for both rubric
|
|
@@ -4628,14 +4634,15 @@ interface AnalysesClient {
|
|
|
4628
4634
|
* (`-x/--exclude-task-name`), `n_tasks` (`-l/--n-tasks`) — applied in
|
|
4629
4635
|
* checker.py's order (:132-138: include, exclude, then the cap) over the
|
|
4630
4636
|
* sorted task directory names, Python fnmatch globs against the NAME.
|
|
4631
|
-
* Harbor's `-
|
|
4632
|
-
*
|
|
4633
|
-
* each with its reason.
|
|
4637
|
+
* Harbor's `-o/--jobs-dir`, `-k/--n-attempts` and the local-runner kwargs
|
|
4638
|
+
* are not on this surface; the contract records each with its reason.
|
|
4634
4639
|
*/
|
|
4635
4640
|
interface CheckConfigInput {
|
|
4636
4641
|
/** A name for the check (Harbor's `--job-name`); omitted, the accept timestamp `YYYY-MM-DD__HH-MM-SS`. 1-120 characters. */
|
|
4637
4642
|
name?: string;
|
|
4638
|
-
/**
|
|
4643
|
+
/** The agent the checker runs on (Harbor's `-a/--agent`) — the analyze door's `agent`, same rule. Omitted: claude. */
|
|
4644
|
+
agent?: string;
|
|
4645
|
+
/** Model the checker agent runs (Harbor's `-m/--model`); must be on the agent's roster (`GET /api/meta`). */
|
|
4639
4646
|
model_name?: string;
|
|
4640
4647
|
/** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (`checks().defaults()`). */
|
|
4641
4648
|
rubric?: Rubric;
|
|
@@ -4740,9 +4747,11 @@ interface Check {
|
|
|
4740
4747
|
name: string;
|
|
4741
4748
|
status: CheckStatus;
|
|
4742
4749
|
source: CheckSource;
|
|
4750
|
+
/** The agent every task's checker ran on — named at create, or the default (claude). */
|
|
4751
|
+
agent: string;
|
|
4743
4752
|
model_name: string;
|
|
4744
|
-
/** The effort every task's checker ran at — named at create, or the model's default, resolved at accept. */
|
|
4745
|
-
reasoning_effort: string;
|
|
4753
|
+
/** The effort every task's checker ran at — named at create, or the model's default on the agent, resolved at accept; null when the agent takes none. */
|
|
4754
|
+
reasoning_effort: string | null;
|
|
4746
4755
|
rubric: Rubric;
|
|
4747
4756
|
/** The prompt template as stored; null = the platform's default check body. */
|
|
4748
4757
|
prompt: string | null;
|
|
@@ -4776,12 +4785,13 @@ interface Check {
|
|
|
4776
4785
|
* this serves as the template text.
|
|
4777
4786
|
*/
|
|
4778
4787
|
interface AnalyzeDefaults {
|
|
4788
|
+
agent: string;
|
|
4779
4789
|
model_name: string;
|
|
4780
4790
|
rubric: Rubric;
|
|
4781
4791
|
/** The built-in analyze prompt template, unrendered — pass it as `prompt` to run the default body explicitly, or edit it from here. */
|
|
4782
4792
|
prompt: string;
|
|
4783
|
-
/** The effort the default model runs at when the config names none. */
|
|
4784
|
-
reasoning_effort: string;
|
|
4793
|
+
/** The effort the default model runs at on the default agent when the config names none; null if that agent takes none. */
|
|
4794
|
+
reasoning_effort: string | null;
|
|
4785
4795
|
sandbox_provider: EvalSandboxProvider;
|
|
4786
4796
|
}
|
|
4787
4797
|
/**
|
|
@@ -4791,12 +4801,13 @@ interface AnalyzeDefaults {
|
|
|
4791
4801
|
* template text.
|
|
4792
4802
|
*/
|
|
4793
4803
|
interface CheckDefaults {
|
|
4804
|
+
agent: string;
|
|
4794
4805
|
model_name: string;
|
|
4795
4806
|
rubric: Rubric;
|
|
4796
4807
|
/** The built-in check prompt template, unrendered — pass it as `prompt` to run the default body explicitly, or edit it from here. */
|
|
4797
4808
|
prompt: string;
|
|
4798
|
-
/** The effort the default model runs at when the config names none. */
|
|
4799
|
-
reasoning_effort: string;
|
|
4809
|
+
/** The effort the default model runs at on the default agent when the config names none; null if that agent takes none. */
|
|
4810
|
+
reasoning_effort: string | null;
|
|
4800
4811
|
sandbox_provider: EvalSandboxProvider;
|
|
4801
4812
|
}
|
|
4802
4813
|
/** Options for checks().watch() */
|
|
@@ -4901,9 +4912,9 @@ interface ChecksClient {
|
|
|
4901
4912
|
* the TASK CHECK id (`Check.results[].id`). Each folder is Harbor's
|
|
4902
4913
|
* TrialPaths for the checker's run: config.json, lock.json, result.json,
|
|
4903
4914
|
* trial.log, exception.txt (an infrastructure failure only),
|
|
4904
|
-
* agent
|
|
4905
|
-
* captured home at its real names with agent/agent-home.json and Harbor's
|
|
4906
|
-
*
|
|
4915
|
+
* agent/<its agent's tee name>, agent/stderr.log, agent/trace-parsed.jsonl, the
|
|
4916
|
+
* captured home at its real names with agent/agent-home.json and Harbor's copy at its
|
|
4917
|
+
* per-agent slot (agent/sessions/ for claude and codex), verifier/{test-stdout.txt,reward.txt,
|
|
4907
4918
|
* reward.json} when the validator ruled (reward 1 = a valid
|
|
4908
4919
|
* check-result.json, 0 = it was refused), artifacts/manifest.json and
|
|
4909
4920
|
* artifacts/check-result.json (the validated flat checks) on a completed
|
|
@@ -5267,19 +5278,21 @@ interface CapabilityDocument {
|
|
|
5267
5278
|
dataset_version: StatusVocabulary;
|
|
5268
5279
|
};
|
|
5269
5280
|
/**
|
|
5270
|
-
* The
|
|
5271
|
-
*
|
|
5272
|
-
* accepts (the claude harness's — the analyzer IS that harness), and
|
|
5273
|
-
* every roster model with the effort an omitted `reasoning_effort` takes
|
|
5274
|
-
* for it. Absent on servers predating the field.
|
|
5281
|
+
* The rubric agents' roster and defaults, the analyzer's and the checker's alike, so a client
|
|
5282
|
+
* knows what "omitted" meant. Absent on servers predating the field.
|
|
5275
5283
|
*/
|
|
5276
5284
|
analyze?: {
|
|
5285
|
+
default_agent: string;
|
|
5277
5286
|
default_model: string;
|
|
5278
|
-
|
|
5279
|
-
|
|
5280
|
-
|
|
5281
|
-
|
|
5282
|
-
|
|
5287
|
+
agents: {
|
|
5288
|
+
name: string;
|
|
5289
|
+
/** Empty for an agent that takes no effort. */
|
|
5290
|
+
reasoning_efforts: string[];
|
|
5291
|
+
models: {
|
|
5292
|
+
alias: string;
|
|
5293
|
+
model_id: string;
|
|
5294
|
+
default_reasoning_effort: string | null;
|
|
5295
|
+
}[];
|
|
5283
5296
|
}[];
|
|
5284
5297
|
};
|
|
5285
5298
|
limits: {
|
|
@@ -450,8 +450,8 @@ interface Rubric {
|
|
|
450
450
|
* openrouter/deepseek/deepseek-v4.1-flash at its per-model effort (high)
|
|
451
451
|
* over the platform's default analyze rubric and its default prompt body.
|
|
452
452
|
*
|
|
453
|
-
* The analyzer
|
|
454
|
-
*
|
|
453
|
+
* The analyzer runs on the `agent` harness (default claude, Harbor's claude-code)
|
|
454
|
+
* in its own sealed sandbox — on the provider `sandbox_provider`
|
|
455
455
|
* names, or the platform's analysis default when it names none; its spend is
|
|
456
456
|
* capped per analysis and metered as its own line, never blended into the
|
|
457
457
|
* trial's own bill.
|
|
@@ -460,12 +460,14 @@ interface Rubric {
|
|
|
460
460
|
* exact names — `n_concurrent` (`-n/--n-concurrent`), `passing` / `failing`,
|
|
461
461
|
* `n_trials` (`-l/--n-trials`; their cli/analyze.py:278-290). All omitted
|
|
462
462
|
* is every analyzable trial, as wide as the organization's
|
|
463
|
-
* `max_concurrent_analyses` allows. Harbor's
|
|
463
|
+
* `max_concurrent_analyses` allows. Harbor's `--job-name`,
|
|
464
464
|
* `-o/--jobs-dir`, `-k/--n-attempts` and the local-runner kwargs are not
|
|
465
465
|
* on this surface; the contract (`AnalyzeConfigInput` in spec/openapi.yaml)
|
|
466
466
|
* records each with its reason.
|
|
467
467
|
*/
|
|
468
468
|
interface AnalyzeConfigInput {
|
|
469
|
+
/** Harbor's `-a/--agent`, spelled as the arms spell it (claude, not claude-code) so one set of names covers arms and reviewers; omitted: claude. The agents: `GET /api/meta` `analyze.agents`. */
|
|
470
|
+
agent?: string;
|
|
469
471
|
/**
|
|
470
472
|
* Model the analyzer agent runs — Harbor's `--model`. The default is
|
|
471
473
|
* openrouter/deepseek/deepseek-v4.1-flash on this platform's claude
|
|
@@ -483,7 +485,7 @@ interface AnalyzeConfigInput {
|
|
|
483
485
|
* spelling is accepted and stored AS GIVEN (the default is the roster
|
|
484
486
|
* alias), the wire id is resolved only when the analyzer runs, and every
|
|
485
487
|
* stored analysis serves the spelling it was created under. Must be on
|
|
486
|
-
* the
|
|
488
|
+
* the agent's roster (`GET /api/meta`, `analyze.agents[].models`); anything else
|
|
487
489
|
* is refused at accept (`invalid_input`, roster in the message).
|
|
488
490
|
*/
|
|
489
491
|
model_name?: string;
|
|
@@ -511,15 +513,14 @@ interface AnalyzeConfigInput {
|
|
|
511
513
|
prompt?: string;
|
|
512
514
|
/**
|
|
513
515
|
* Reasoning effort the analyzer runs at — the platform's `agents[].
|
|
514
|
-
* reasoning_effort` vocabulary applied to the analyzer
|
|
515
|
-
*
|
|
516
|
-
* `analyze.reasoning_efforts`, an unknown value is refused
|
|
516
|
+
* reasoning_effort` vocabulary applied to the analyzer's agent: the accepted
|
|
517
|
+
* values are `GET /api/meta`'s `analyze.agents[].reasoning_efforts`, an unknown value is refused
|
|
517
518
|
* `invalid_input` exactly as an arm's is. Omitted, the PER-MODEL default
|
|
518
|
-
* applies (`analyze.models[].default_reasoning_effort`: high on
|
|
519
|
+
* applies (`analyze.agents[].models[].default_reasoning_effort`: high on
|
|
519
520
|
* openrouter/deepseek/deepseek-v4.1-flash, the default model — DeepSeek's
|
|
520
521
|
* own documented default, the owner's ruling 2026-09-10; max on
|
|
521
522
|
* glm-5.3-flash — the platform's ruling 2026-09-08, the effort its
|
|
522
|
-
* published scores use; the
|
|
523
|
+
* published scores use; the agent's own default elsewhere, none where it takes none). The
|
|
523
524
|
* effort is always passed to the analyzer explicitly and
|
|
524
525
|
* recorded on the analysis (`TrialAnalysis.reasoning_effort`). A hosted
|
|
525
526
|
* extension: Harbor's analyze has no effort option; this is the run
|
|
@@ -596,10 +597,12 @@ interface AnalyzeConfigInput {
|
|
|
596
597
|
* the day, resolved at accept and stored, so the record always states the
|
|
597
598
|
* policy it executes (same law as RetryConfig). Echoed on the job body when
|
|
598
599
|
* the job was created with `analyze`; each analysis additionally carries the
|
|
599
|
-
* exact policy IT ran under (`Trial.analysis.
|
|
600
|
-
* `.prompt`), which a later manual re-analysis may have changed.
|
|
600
|
+
* exact policy IT ran under (`Trial.analysis.agent` / `.model_name` /
|
|
601
|
+
* `.rubric` / `.prompt`), which a later manual re-analysis may have changed.
|
|
601
602
|
*/
|
|
602
603
|
interface AnalyzeConfig {
|
|
604
|
+
/** The agent this policy's analyses run on — as named, or the default (claude). */
|
|
605
|
+
agent: string;
|
|
603
606
|
model_name: string;
|
|
604
607
|
rubric: Rubric;
|
|
605
608
|
/** The caller's prompt template as stored; null = the platform's default analyze body. */
|
|
@@ -607,10 +610,10 @@ interface AnalyzeConfig {
|
|
|
607
610
|
/**
|
|
608
611
|
* The effort this policy's analyses run at. Named at create it is served
|
|
609
612
|
* as stored; when the create named none, this echoes the per-model
|
|
610
|
-
* default of the day for `model_name` — the value the next enqueue under
|
|
611
|
-
* this policy stamps (the same nuance as `sandbox_provider` below).
|
|
613
|
+
* default of the day for `model_name` on `agent` — the value the next enqueue under
|
|
614
|
+
* this policy stamps (the same nuance as `sandbox_provider` below). Null when the agent takes no effort.
|
|
612
615
|
*/
|
|
613
|
-
reasoning_effort: string;
|
|
616
|
+
reasoning_effort: string | null;
|
|
614
617
|
/**
|
|
615
618
|
* The provider this policy's analyses run on. Named at create it is served
|
|
616
619
|
* as stored, forever. When the create named none, this echoes the
|
|
@@ -1523,11 +1526,13 @@ interface TrialAnalysis {
|
|
|
1523
1526
|
* `failed`, never left `running` forever.
|
|
1524
1527
|
*/
|
|
1525
1528
|
status: AnalysisStatus;
|
|
1529
|
+
/** The agent THIS analysis ran on; every analysis from before the choice existed ran on claude. */
|
|
1530
|
+
agent: string;
|
|
1526
1531
|
model_name: string;
|
|
1527
1532
|
/**
|
|
1528
1533
|
* The reasoning effort THIS analysis ran at — passed to the analyzer
|
|
1529
|
-
* explicitly, so it is what the model was asked for. Null
|
|
1530
|
-
* analyses recorded before the effort was stamped.
|
|
1534
|
+
* explicitly, so it is what the model was asked for. Null when its agent
|
|
1535
|
+
* takes no effort, or on analyses recorded before the effort was stamped.
|
|
1531
1536
|
*/
|
|
1532
1537
|
reasoning_effort: string | null;
|
|
1533
1538
|
rubric: Rubric;
|
|
@@ -2268,10 +2273,11 @@ interface CheckRow {
|
|
|
2268
2273
|
name: string;
|
|
2269
2274
|
status: CheckStatus;
|
|
2270
2275
|
source: CheckSource;
|
|
2276
|
+
agent: string;
|
|
2271
2277
|
/** The checker's model. */
|
|
2272
2278
|
model_name: string;
|
|
2273
|
-
/** The effort every task's checker ran at. */
|
|
2274
|
-
reasoning_effort: string;
|
|
2279
|
+
/** The effort every task's checker ran at; null when its agent takes none. */
|
|
2280
|
+
reasoning_effort: string | null;
|
|
2275
2281
|
sandbox_provider: EvalSandboxProvider;
|
|
2276
2282
|
/** The owning organization's slug (every check has one). */
|
|
2277
2283
|
org: string;
|
|
@@ -4583,10 +4589,10 @@ interface AnalysesClient {
|
|
|
4583
4589
|
* one directory named as Harbor names the wrapper trial
|
|
4584
4590
|
* (`analyze-<analyzed trial dir>__<7 chars>/`): config.json, lock.json,
|
|
4585
4591
|
* result.json, trial.log, exception.txt (an infrastructure failure only),
|
|
4586
|
-
* agent
|
|
4592
|
+
* agent/<Harbor's tee name for its agent> (claude-code.txt, codex.txt, …),
|
|
4587
4593
|
* agent/stderr.log, agent/trace-parsed.jsonl, the captured home at its
|
|
4588
|
-
* real names with agent/agent-home.json beside it and Harbor's copy at
|
|
4589
|
-
* agent/sessions
|
|
4594
|
+
* real names with agent/agent-home.json beside it and Harbor's copy at its
|
|
4595
|
+
* per-agent slot (agent/sessions/ for claude and codex), verifier/{test-stdout.txt,reward.txt,reward.json}
|
|
4590
4596
|
* when the validator ruled (reward 1 = a valid analysis.json, 0 = it was
|
|
4591
4597
|
* refused), and artifacts/manifest.json with artifacts/analysis.json (the
|
|
4592
4598
|
* validated {summary, checks}) on a completed run — absent artifacts are
|
|
@@ -4609,8 +4615,8 @@ interface AnalysesClient {
|
|
|
4609
4615
|
/**
|
|
4610
4616
|
* Task-check configuration — Harbor's `harbor check` vocabulary (their
|
|
4611
4617
|
* cli/analyze.py:84-148 check_command), the spec's `CheckConfigInput`. The
|
|
4612
|
-
* rubric-agent
|
|
4613
|
-
* (`AnalyzeConfigInput` states them; refusals name `check.*`): `model_name`
|
|
4618
|
+
* rubric-agent knobs are the analyze door's, under the same rules
|
|
4619
|
+
* (`AnalyzeConfigInput` states them; refusals name `check.*`): `agent` (`-a/--agent`), `model_name`
|
|
4614
4620
|
* (Harbor's check default is `claude-sonnet-4-6`; this platform's is the
|
|
4615
4621
|
* analyzer's `openrouter/deepseek/deepseek-v4.1-flash` — one roster, one
|
|
4616
4622
|
* default for both rubric
|
|
@@ -4628,14 +4634,15 @@ interface AnalysesClient {
|
|
|
4628
4634
|
* (`-x/--exclude-task-name`), `n_tasks` (`-l/--n-tasks`) — applied in
|
|
4629
4635
|
* checker.py's order (:132-138: include, exclude, then the cap) over the
|
|
4630
4636
|
* sorted task directory names, Python fnmatch globs against the NAME.
|
|
4631
|
-
* Harbor's `-
|
|
4632
|
-
*
|
|
4633
|
-
* each with its reason.
|
|
4637
|
+
* Harbor's `-o/--jobs-dir`, `-k/--n-attempts` and the local-runner kwargs
|
|
4638
|
+
* are not on this surface; the contract records each with its reason.
|
|
4634
4639
|
*/
|
|
4635
4640
|
interface CheckConfigInput {
|
|
4636
4641
|
/** A name for the check (Harbor's `--job-name`); omitted, the accept timestamp `YYYY-MM-DD__HH-MM-SS`. 1-120 characters. */
|
|
4637
4642
|
name?: string;
|
|
4638
|
-
/**
|
|
4643
|
+
/** The agent the checker runs on (Harbor's `-a/--agent`) — the analyze door's `agent`, same rule. Omitted: claude. */
|
|
4644
|
+
agent?: string;
|
|
4645
|
+
/** Model the checker agent runs (Harbor's `-m/--model`); must be on the agent's roster (`GET /api/meta`). */
|
|
4639
4646
|
model_name?: string;
|
|
4640
4647
|
/** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (`checks().defaults()`). */
|
|
4641
4648
|
rubric?: Rubric;
|
|
@@ -4740,9 +4747,11 @@ interface Check {
|
|
|
4740
4747
|
name: string;
|
|
4741
4748
|
status: CheckStatus;
|
|
4742
4749
|
source: CheckSource;
|
|
4750
|
+
/** The agent every task's checker ran on — named at create, or the default (claude). */
|
|
4751
|
+
agent: string;
|
|
4743
4752
|
model_name: string;
|
|
4744
|
-
/** The effort every task's checker ran at — named at create, or the model's default, resolved at accept. */
|
|
4745
|
-
reasoning_effort: string;
|
|
4753
|
+
/** The effort every task's checker ran at — named at create, or the model's default on the agent, resolved at accept; null when the agent takes none. */
|
|
4754
|
+
reasoning_effort: string | null;
|
|
4746
4755
|
rubric: Rubric;
|
|
4747
4756
|
/** The prompt template as stored; null = the platform's default check body. */
|
|
4748
4757
|
prompt: string | null;
|
|
@@ -4776,12 +4785,13 @@ interface Check {
|
|
|
4776
4785
|
* this serves as the template text.
|
|
4777
4786
|
*/
|
|
4778
4787
|
interface AnalyzeDefaults {
|
|
4788
|
+
agent: string;
|
|
4779
4789
|
model_name: string;
|
|
4780
4790
|
rubric: Rubric;
|
|
4781
4791
|
/** The built-in analyze prompt template, unrendered — pass it as `prompt` to run the default body explicitly, or edit it from here. */
|
|
4782
4792
|
prompt: string;
|
|
4783
|
-
/** The effort the default model runs at when the config names none. */
|
|
4784
|
-
reasoning_effort: string;
|
|
4793
|
+
/** The effort the default model runs at on the default agent when the config names none; null if that agent takes none. */
|
|
4794
|
+
reasoning_effort: string | null;
|
|
4785
4795
|
sandbox_provider: EvalSandboxProvider;
|
|
4786
4796
|
}
|
|
4787
4797
|
/**
|
|
@@ -4791,12 +4801,13 @@ interface AnalyzeDefaults {
|
|
|
4791
4801
|
* template text.
|
|
4792
4802
|
*/
|
|
4793
4803
|
interface CheckDefaults {
|
|
4804
|
+
agent: string;
|
|
4794
4805
|
model_name: string;
|
|
4795
4806
|
rubric: Rubric;
|
|
4796
4807
|
/** The built-in check prompt template, unrendered — pass it as `prompt` to run the default body explicitly, or edit it from here. */
|
|
4797
4808
|
prompt: string;
|
|
4798
|
-
/** The effort the default model runs at when the config names none. */
|
|
4799
|
-
reasoning_effort: string;
|
|
4809
|
+
/** The effort the default model runs at on the default agent when the config names none; null if that agent takes none. */
|
|
4810
|
+
reasoning_effort: string | null;
|
|
4800
4811
|
sandbox_provider: EvalSandboxProvider;
|
|
4801
4812
|
}
|
|
4802
4813
|
/** Options for checks().watch() */
|
|
@@ -4901,9 +4912,9 @@ interface ChecksClient {
|
|
|
4901
4912
|
* the TASK CHECK id (`Check.results[].id`). Each folder is Harbor's
|
|
4902
4913
|
* TrialPaths for the checker's run: config.json, lock.json, result.json,
|
|
4903
4914
|
* trial.log, exception.txt (an infrastructure failure only),
|
|
4904
|
-
* agent
|
|
4905
|
-
* captured home at its real names with agent/agent-home.json and Harbor's
|
|
4906
|
-
*
|
|
4915
|
+
* agent/<its agent's tee name>, agent/stderr.log, agent/trace-parsed.jsonl, the
|
|
4916
|
+
* captured home at its real names with agent/agent-home.json and Harbor's copy at its
|
|
4917
|
+
* per-agent slot (agent/sessions/ for claude and codex), verifier/{test-stdout.txt,reward.txt,
|
|
4907
4918
|
* reward.json} when the validator ruled (reward 1 = a valid
|
|
4908
4919
|
* check-result.json, 0 = it was refused), artifacts/manifest.json and
|
|
4909
4920
|
* artifacts/check-result.json (the validated flat checks) on a completed
|
|
@@ -5267,19 +5278,21 @@ interface CapabilityDocument {
|
|
|
5267
5278
|
dataset_version: StatusVocabulary;
|
|
5268
5279
|
};
|
|
5269
5280
|
/**
|
|
5270
|
-
* The
|
|
5271
|
-
*
|
|
5272
|
-
* accepts (the claude harness's — the analyzer IS that harness), and
|
|
5273
|
-
* every roster model with the effort an omitted `reasoning_effort` takes
|
|
5274
|
-
* for it. Absent on servers predating the field.
|
|
5281
|
+
* The rubric agents' roster and defaults, the analyzer's and the checker's alike, so a client
|
|
5282
|
+
* knows what "omitted" meant. Absent on servers predating the field.
|
|
5275
5283
|
*/
|
|
5276
5284
|
analyze?: {
|
|
5285
|
+
default_agent: string;
|
|
5277
5286
|
default_model: string;
|
|
5278
|
-
|
|
5279
|
-
|
|
5280
|
-
|
|
5281
|
-
|
|
5282
|
-
|
|
5287
|
+
agents: {
|
|
5288
|
+
name: string;
|
|
5289
|
+
/** Empty for an agent that takes no effort. */
|
|
5290
|
+
reasoning_efforts: string[];
|
|
5291
|
+
models: {
|
|
5292
|
+
alias: string;
|
|
5293
|
+
model_id: string;
|
|
5294
|
+
default_reasoning_effort: string | null;
|
|
5295
|
+
}[];
|
|
5283
5296
|
}[];
|
|
5284
5297
|
};
|
|
5285
5298
|
limits: {
|