@evolvingmachines/evolve 0.0.65 → 0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-DK2MGUPB.js → chunk-DI6LPW2B.js} +2 -2
- package/dist/cli/index.cjs +29 -29
- package/dist/cli/index.d.cts +3 -3
- package/dist/cli/index.d.ts +3 -3
- package/dist/cli/index.js +18 -18
- package/dist/index.cjs +64 -60
- package/dist/index.d.cts +297 -18
- package/dist/index.d.ts +297 -18
- package/dist/index.js +48 -44
- package/dist/{types-B_qmzlMk.d.cts → types-Dj0iMl5V.d.cts} +70 -90
- package/dist/{types-B_qmzlMk.d.ts → types-Dj0iMl5V.d.ts} +70 -90
- package/harness-capabilities.json +285 -0
- package/package.json +9 -6
- package/skills/create-adapter/SKILL.md +71 -26
- package/skills/create-task/SKILL.md +124 -66
- package/skills/evolve/SKILL.md +37 -20
- package/skills/evolve-evals/SKILL.md +142 -69
- package/skills/evolve-evals/references/cli-reference/agent.md +76 -0
- package/skills/evolve-evals/references/cli-reference/analysis.md +78 -0
- package/skills/evolve-evals/references/cli-reference/analyze.md +97 -0
- package/skills/evolve-evals/references/cli-reference/auth.md +96 -0
- package/skills/evolve-evals/references/cli-reference/check.md +157 -0
- package/skills/evolve-evals/references/cli-reference/dataset.md +154 -0
- package/skills/evolve-evals/references/cli-reference/filesystem.md +168 -0
- package/skills/evolve-evals/references/cli-reference/index.md +149 -0
- package/skills/evolve-evals/references/cli-reference/job.md +198 -0
- package/skills/evolve-evals/references/cli-reference/run.md +188 -0
- package/skills/evolve-evals/references/cli-reference/secrets.md +77 -0
- package/skills/evolve-evals/references/cli-reference/session.md +56 -0
- package/skills/evolve-evals/references/cli-reference/skill.md +52 -0
- package/skills/evolve-evals/references/cli-reference/skills.md +85 -0
- package/skills/evolve-evals/references/cli-reference/trial.md +115 -0
- package/skills/evolve-evals/references/cli-reference/upload.md +72 -0
- package/skills/evolve-evals/references/core-concepts/agents.md +101 -0
- package/skills/evolve-evals/references/core-concepts/analyze.md +143 -0
- package/skills/evolve-evals/references/core-concepts/check.md +116 -0
- package/skills/evolve-evals/references/core-concepts/compatibility.md +39 -0
- package/skills/evolve-evals/references/core-concepts/compose.md +103 -0
- package/skills/evolve-evals/references/core-concepts/datasets.md +234 -0
- package/skills/evolve-evals/references/core-concepts/index.md +52 -0
- package/skills/evolve-evals/references/core-concepts/jobs.md +223 -0
- package/skills/evolve-evals/references/core-concepts/models.md +73 -0
- package/skills/evolve-evals/references/core-concepts/multi-step.md +93 -0
- package/skills/evolve-evals/references/core-concepts/rewardkit.md +341 -0
- package/skills/evolve-evals/references/core-concepts/sandboxes.md +73 -0
- package/skills/evolve-evals/references/core-concepts/secrets.md +86 -0
- package/skills/evolve-evals/references/core-concepts/sharing.md +81 -0
- package/skills/evolve-evals/references/core-concepts/skills.md +100 -0
- package/skills/evolve-evals/references/core-concepts/task-config.md +169 -0
- package/skills/evolve-evals/references/core-concepts/task-environment.md +127 -0
- package/skills/evolve-evals/references/core-concepts/task-reference.md +191 -0
- package/skills/evolve-evals/references/core-concepts/task-verifiers.md +209 -0
- package/skills/evolve-evals/references/core-concepts/tasks.md +90 -0
- package/skills/evolve-evals/references/core-concepts/teams.md +128 -0
- package/skills/evolve-evals/references/core-concepts/trial-outputs.md +225 -0
- package/skills/evolve-evals/references/core-concepts/trials.md +129 -0
- package/skills/evolve-evals/references/core-concepts/upload-sdk-session.md +240 -0
- package/skills/evolve-evals/references/core-concepts/upload.md +166 -0
- package/skills/evolve-evals/references/dashboard/checks.md +54 -0
- package/skills/evolve-evals/references/dashboard/datasets.md +65 -0
- package/skills/evolve-evals/references/dashboard/jobs.md +106 -0
- package/skills/evolve-evals/references/dashboard/traces.md +54 -0
- package/skills/evolve-evals/references/dashboard/trial-viewer.md +77 -0
- package/skills/evolve-evals/references/getting-started/first-task.md +228 -0
- package/skills/evolve-evals/references/getting-started/installation.md +79 -0
- package/skills/evolve-evals/references/getting-started/quick-start.md +100 -0
- package/skills/evolve-evals/references/index.md +58 -0
- package/skills/evolve-evals/references/sdk/http.md +110 -0
- package/skills/evolve-evals/references/sdk/python.md +108 -0
- package/skills/evolve-evals/references/sdk/typescript.md +104 -0
- package/skills/evolve-evals/references/sdk-reference/agents.md +75 -0
- package/skills/evolve-evals/references/sdk-reference/analyses.md +125 -0
- package/skills/evolve-evals/references/sdk-reference/auth.md +171 -0
- package/skills/evolve-evals/references/sdk-reference/checks.md +146 -0
- package/skills/evolve-evals/references/sdk-reference/datasets.md +170 -0
- package/skills/evolve-evals/references/sdk-reference/error-codes.md +231 -0
- package/skills/evolve-evals/references/sdk-reference/errors.md +98 -0
- package/skills/evolve-evals/references/sdk-reference/filesystem.md +166 -0
- package/skills/evolve-evals/references/sdk-reference/index.md +161 -0
- package/skills/evolve-evals/references/sdk-reference/jobs.md +250 -0
- package/skills/evolve-evals/references/sdk-reference/meta.md +54 -0
- package/skills/evolve-evals/references/sdk-reference/methods/agents.md +154 -0
- package/skills/evolve-evals/references/sdk-reference/methods/analyses.md +211 -0
- package/skills/evolve-evals/references/sdk-reference/methods/auth.md +257 -0
- package/skills/evolve-evals/references/sdk-reference/methods/checks.md +462 -0
- package/skills/evolve-evals/references/sdk-reference/methods/datasets.md +687 -0
- package/skills/evolve-evals/references/sdk-reference/methods/filesystem.md +527 -0
- package/skills/evolve-evals/references/sdk-reference/methods/jobs.md +1023 -0
- package/skills/evolve-evals/references/sdk-reference/methods/meta.md +213 -0
- package/skills/evolve-evals/references/sdk-reference/methods/secrets.md +157 -0
- package/skills/evolve-evals/references/sdk-reference/methods/skills.md +125 -0
- package/skills/evolve-evals/references/sdk-reference/methods/trials.md +371 -0
- package/skills/evolve-evals/references/sdk-reference/secrets.md +104 -0
- package/skills/evolve-evals/references/sdk-reference/skills.md +66 -0
- package/skills/evolve-evals/references/sdk-reference/trials.md +164 -0
- package/skills/evolve-evals/references/sdk-reference/types.md +1255 -0
- package/skills/publish/SKILL.md +103 -79
- package/skills/rewardkit/SKILL.md +78 -23
- package/spec/openapi.yaml +164 -160
- package/skills/evolve-evals/references/cli-reference/agent.mdx +0 -79
- package/skills/evolve-evals/references/cli-reference/analysis.mdx +0 -160
- package/skills/evolve-evals/references/cli-reference/analyze.mdx +0 -84
- package/skills/evolve-evals/references/cli-reference/auth.mdx +0 -133
- package/skills/evolve-evals/references/cli-reference/check.mdx +0 -263
- package/skills/evolve-evals/references/cli-reference/dataset.mdx +0 -151
- package/skills/evolve-evals/references/cli-reference/index.mdx +0 -108
- package/skills/evolve-evals/references/cli-reference/job.mdx +0 -278
- package/skills/evolve-evals/references/cli-reference/run.mdx +0 -204
- package/skills/evolve-evals/references/cli-reference/secrets.mdx +0 -95
- package/skills/evolve-evals/references/cli-reference/skill.mdx +0 -61
- package/skills/evolve-evals/references/cli-reference/skills.mdx +0 -81
- package/skills/evolve-evals/references/cli-reference/trial.mdx +0 -181
- package/skills/evolve-evals/references/cli-reference/upload.mdx +0 -36
- package/skills/evolve-evals/references/core-concepts/agents.mdx +0 -106
- package/skills/evolve-evals/references/core-concepts/analyze.mdx +0 -86
- package/skills/evolve-evals/references/core-concepts/check.mdx +0 -64
- package/skills/evolve-evals/references/core-concepts/datasets.mdx +0 -138
- package/skills/evolve-evals/references/core-concepts/index.mdx +0 -42
- package/skills/evolve-evals/references/core-concepts/jobs.mdx +0 -151
- package/skills/evolve-evals/references/core-concepts/models.mdx +0 -68
- package/skills/evolve-evals/references/core-concepts/sandboxes.mdx +0 -52
- package/skills/evolve-evals/references/core-concepts/secrets.mdx +0 -41
- package/skills/evolve-evals/references/core-concepts/sharing.mdx +0 -35
- package/skills/evolve-evals/references/core-concepts/skills.mdx +0 -42
- package/skills/evolve-evals/references/core-concepts/tasks.mdx +0 -216
- package/skills/evolve-evals/references/core-concepts/teams.mdx +0 -36
- package/skills/evolve-evals/references/core-concepts/trial-outputs.mdx +0 -117
- package/skills/evolve-evals/references/core-concepts/trials.mdx +0 -103
- package/skills/evolve-evals/references/core-concepts/upload.mdx +0 -93
- package/skills/evolve-evals/references/dashboard/checks.mdx +0 -21
- package/skills/evolve-evals/references/dashboard/datasets.mdx +0 -24
- package/skills/evolve-evals/references/dashboard/jobs.mdx +0 -65
- package/skills/evolve-evals/references/dashboard/traces.mdx +0 -17
- package/skills/evolve-evals/references/dashboard/trial-viewer.mdx +0 -30
- package/skills/evolve-evals/references/getting-started/installation.mdx +0 -56
- package/skills/evolve-evals/references/getting-started/quick-start.mdx +0 -94
- package/skills/evolve-evals/references/index.mdx +0 -48
- package/skills/evolve-evals/references/sdk/python.mdx +0 -161
- package/skills/evolve-evals/references/sdk/typescript.mdx +0 -163
- package/skills/evolve-evals/references/sdk-reference/agents.mdx +0 -108
- package/skills/evolve-evals/references/sdk-reference/analyses.mdx +0 -222
- package/skills/evolve-evals/references/sdk-reference/auth.mdx +0 -116
- package/skills/evolve-evals/references/sdk-reference/checks.mdx +0 -300
- package/skills/evolve-evals/references/sdk-reference/datasets.mdx +0 -346
- package/skills/evolve-evals/references/sdk-reference/errors.mdx +0 -210
- package/skills/evolve-evals/references/sdk-reference/index.mdx +0 -60
- package/skills/evolve-evals/references/sdk-reference/jobs.mdx +0 -822
- package/skills/evolve-evals/references/sdk-reference/meta.mdx +0 -52
- package/skills/evolve-evals/references/sdk-reference/skills.mdx +0 -69
- package/skills/evolve-evals/references/sdk-reference/trials.mdx +0 -302
- package/skills/evolve-evals/references/sdk-reference/types.mdx +0 -274
- package/skills/evolve-evals/references/snippets/global-options.mdx +0 -13
- package/skills/evolve-evals/references/snippets/list-flags.mdx +0 -23
|
@@ -419,7 +419,7 @@ interface RetryConfig {
|
|
|
419
419
|
interface RubricCriterion {
|
|
420
420
|
/**
|
|
421
421
|
* Criterion identifier, snake_case (it keys the result's `checks` object).
|
|
422
|
-
* The platform's default rubrics
|
|
422
|
+
* The platform's default rubrics are served by `analyses().defaults()` and `checks().defaults()`.
|
|
423
423
|
*/
|
|
424
424
|
name: string;
|
|
425
425
|
/** What the criterion evaluates, one sentence. */
|
|
@@ -448,11 +448,10 @@ interface Rubric {
|
|
|
448
448
|
* as the body of `POST /api/jobs/{jobId}/analyze` it configures that manual
|
|
449
449
|
* wave. `{}` is legal and means "all defaults":
|
|
450
450
|
* openrouter/deepseek/deepseek-v4.1-flash at its per-model effort (high)
|
|
451
|
-
* over the platform's default analyze rubric
|
|
452
|
-
* first) and its default prompt body.
|
|
451
|
+
* over the platform's default analyze rubric and its default prompt body.
|
|
453
452
|
*
|
|
454
|
-
* The analyzer
|
|
455
|
-
*
|
|
453
|
+
* The analyzer runs on the `agent` harness (default claude, Harbor's claude-code)
|
|
454
|
+
* in its own sealed sandbox — on the provider `sandbox_provider`
|
|
456
455
|
* names, or the platform's analysis default when it names none; its spend is
|
|
457
456
|
* capped per analysis and metered as its own line, never blended into the
|
|
458
457
|
* trial's own bill.
|
|
@@ -461,12 +460,14 @@ interface Rubric {
|
|
|
461
460
|
* exact names — `n_concurrent` (`-n/--n-concurrent`), `passing` / `failing`,
|
|
462
461
|
* `n_trials` (`-l/--n-trials`; their cli/analyze.py:278-290). All omitted
|
|
463
462
|
* is every analyzable trial, as wide as the organization's
|
|
464
|
-
* `max_concurrent_analyses` allows. Harbor's
|
|
463
|
+
* `max_concurrent_analyses` allows. Harbor's `--job-name`,
|
|
465
464
|
* `-o/--jobs-dir`, `-k/--n-attempts` and the local-runner kwargs are not
|
|
466
465
|
* on this surface; the contract (`AnalyzeConfigInput` in spec/openapi.yaml)
|
|
467
466
|
* records each with its reason.
|
|
468
467
|
*/
|
|
469
468
|
interface AnalyzeConfigInput {
|
|
469
|
+
/** Harbor's `-a/--agent`, spelled as the arms spell it (claude, not claude-code) so one set of names covers arms and reviewers; omitted: claude. The agents: `GET /api/meta` `analyze.agents`. */
|
|
470
|
+
agent?: string;
|
|
470
471
|
/**
|
|
471
472
|
* Model the analyzer agent runs — Harbor's `--model`. The default is
|
|
472
473
|
* openrouter/deepseek/deepseek-v4.1-flash on this platform's claude
|
|
@@ -484,7 +485,7 @@ interface AnalyzeConfigInput {
|
|
|
484
485
|
* spelling is accepted and stored AS GIVEN (the default is the roster
|
|
485
486
|
* alias), the wire id is resolved only when the analyzer runs, and every
|
|
486
487
|
* stored analysis serves the spelling it was created under. Must be on
|
|
487
|
-
* the
|
|
488
|
+
* the agent's roster (`GET /api/meta`, `analyze.agents[].models`); anything else
|
|
488
489
|
* is refused at accept (`invalid_input`, roster in the message).
|
|
489
490
|
*/
|
|
490
491
|
model_name?: string;
|
|
@@ -512,15 +513,14 @@ interface AnalyzeConfigInput {
|
|
|
512
513
|
prompt?: string;
|
|
513
514
|
/**
|
|
514
515
|
* Reasoning effort the analyzer runs at — the platform's `agents[].
|
|
515
|
-
* reasoning_effort` vocabulary applied to the analyzer
|
|
516
|
-
*
|
|
517
|
-
* `analyze.reasoning_efforts`, an unknown value is refused
|
|
516
|
+
* reasoning_effort` vocabulary applied to the analyzer's agent: the accepted
|
|
517
|
+
* values are `GET /api/meta`'s `analyze.agents[].reasoning_efforts`, an unknown value is refused
|
|
518
518
|
* `invalid_input` exactly as an arm's is. Omitted, the PER-MODEL default
|
|
519
|
-
* applies (`analyze.models[].default_reasoning_effort`: high on
|
|
519
|
+
* applies (`analyze.agents[].models[].default_reasoning_effort`: high on
|
|
520
520
|
* openrouter/deepseek/deepseek-v4.1-flash, the default model — DeepSeek's
|
|
521
521
|
* own documented default, the owner's ruling 2026-09-10; max on
|
|
522
522
|
* glm-5.3-flash — the platform's ruling 2026-09-08, the effort its
|
|
523
|
-
* published scores use; the
|
|
523
|
+
* published scores use; the agent's own default elsewhere, none where it takes none). The
|
|
524
524
|
* effort is always passed to the analyzer explicitly and
|
|
525
525
|
* recorded on the analysis (`TrialAnalysis.reasoning_effort`). A hosted
|
|
526
526
|
* extension: Harbor's analyze has no effort option; this is the run
|
|
@@ -597,10 +597,12 @@ interface AnalyzeConfigInput {
|
|
|
597
597
|
* the day, resolved at accept and stored, so the record always states the
|
|
598
598
|
* policy it executes (same law as RetryConfig). Echoed on the job body when
|
|
599
599
|
* the job was created with `analyze`; each analysis additionally carries the
|
|
600
|
-
* exact policy IT ran under (`Trial.analysis.
|
|
601
|
-
* `.prompt`), which a later manual re-analysis may have changed.
|
|
600
|
+
* exact policy IT ran under (`Trial.analysis.agent` / `.model_name` /
|
|
601
|
+
* `.rubric` / `.prompt`), which a later manual re-analysis may have changed.
|
|
602
602
|
*/
|
|
603
603
|
interface AnalyzeConfig {
|
|
604
|
+
/** The agent this policy's analyses run on — as named, or the default (claude). */
|
|
605
|
+
agent: string;
|
|
604
606
|
model_name: string;
|
|
605
607
|
rubric: Rubric;
|
|
606
608
|
/** The caller's prompt template as stored; null = the platform's default analyze body. */
|
|
@@ -608,10 +610,10 @@ interface AnalyzeConfig {
|
|
|
608
610
|
/**
|
|
609
611
|
* The effort this policy's analyses run at. Named at create it is served
|
|
610
612
|
* as stored; when the create named none, this echoes the per-model
|
|
611
|
-
* default of the day for `model_name` — the value the next enqueue under
|
|
612
|
-
* this policy stamps (the same nuance as `sandbox_provider` below).
|
|
613
|
+
* default of the day for `model_name` on `agent` — the value the next enqueue under
|
|
614
|
+
* this policy stamps (the same nuance as `sandbox_provider` below). Null when the agent takes no effort.
|
|
613
615
|
*/
|
|
614
|
-
reasoning_effort: string;
|
|
616
|
+
reasoning_effort: string | null;
|
|
615
617
|
/**
|
|
616
618
|
* The provider this policy's analyses run on. Named at create it is served
|
|
617
619
|
* as stored, forever. When the create named none, this echoes the
|
|
@@ -1472,24 +1474,6 @@ interface AnalysisCheck {
|
|
|
1472
1474
|
explanation: string;
|
|
1473
1475
|
evidence: AnalysisEvidence[];
|
|
1474
1476
|
}
|
|
1475
|
-
/**
|
|
1476
|
-
* The derived label of an analysis — computed by the platform from the
|
|
1477
|
-
* outcomes when the result is stored, never asked from the model. Null until
|
|
1478
|
-
* completed, and null on a completed analysis whose rubric is not the
|
|
1479
|
-
* default one (a custom rubric carries its per-criterion outcomes and no
|
|
1480
|
-
* label). Precedence: `flagged` on a fail of score_is_earned,
|
|
1481
|
-
* score_is_correct, task_was_fair or report_is_truthful; else `env_fault` on
|
|
1482
|
-
* a fail of environment_worked; else `unclear` on an unknown of any of those
|
|
1483
|
-
* five, or a not_applicable of score_is_earned or score_is_correct; else
|
|
1484
|
-
* `clean`.
|
|
1485
|
-
*/
|
|
1486
|
-
type AnalysisLabel = "flagged" | "env_fault" | "unclear" | "clean";
|
|
1487
|
-
/**
|
|
1488
|
-
* The derived label of a task check — computed the same way. `has_a_problem`
|
|
1489
|
-
* on a fail of any criterion; else `unclear` on an unknown of any of the six
|
|
1490
|
-
* file-based criteria; else `no_problem_found`. Null under a custom rubric.
|
|
1491
|
-
*/
|
|
1492
|
-
type CheckLabel = "has_a_problem" | "unclear" | "no_problem_found";
|
|
1493
1477
|
/**
|
|
1494
1478
|
* Why an analysis FAILED — a stored typed failure, never a silent absence and
|
|
1495
1479
|
* never a fake pass. NOT under the key `error` for the same reason JobFailure
|
|
@@ -1531,7 +1515,7 @@ interface TrialAnalysis {
|
|
|
1531
1515
|
* Provenance: the analyzed trial, its job, and its task. Redundant on
|
|
1532
1516
|
* `Trial.analysis` (the trial is the enclosing object) and the whole point
|
|
1533
1517
|
* of a `analyses().list()` row, where nothing else says which run the
|
|
1534
|
-
*
|
|
1518
|
+
* result judged. Harbor's `trial_name` names the same thing by directory.
|
|
1535
1519
|
*/
|
|
1536
1520
|
trial_id: string;
|
|
1537
1521
|
job_id: string;
|
|
@@ -1542,11 +1526,13 @@ interface TrialAnalysis {
|
|
|
1542
1526
|
* `failed`, never left `running` forever.
|
|
1543
1527
|
*/
|
|
1544
1528
|
status: AnalysisStatus;
|
|
1529
|
+
/** The agent THIS analysis ran on; every analysis from before the choice existed ran on claude. */
|
|
1530
|
+
agent: string;
|
|
1545
1531
|
model_name: string;
|
|
1546
1532
|
/**
|
|
1547
1533
|
* The reasoning effort THIS analysis ran at — passed to the analyzer
|
|
1548
|
-
* explicitly, so it is what the model was asked for. Null
|
|
1549
|
-
* analyses recorded before the effort was stamped.
|
|
1534
|
+
* explicitly, so it is what the model was asked for. Null when its agent
|
|
1535
|
+
* takes no effort, or on analyses recorded before the effort was stamped.
|
|
1550
1536
|
*/
|
|
1551
1537
|
reasoning_effort: string | null;
|
|
1552
1538
|
rubric: Rubric;
|
|
@@ -1565,8 +1551,6 @@ interface TrialAnalysis {
|
|
|
1565
1551
|
* (the frozen-criteria law). Null until completed.
|
|
1566
1552
|
*/
|
|
1567
1553
|
checks: Record<string, AnalysisCheck> | null;
|
|
1568
|
-
/** The derived label (AnalysisLabel states the rule); null until completed, and null under a custom rubric. */
|
|
1569
|
-
label: AnalysisLabel | null;
|
|
1570
1554
|
estimated_cost_usd: number | null;
|
|
1571
1555
|
/**
|
|
1572
1556
|
* The analyzer's one-home usage reading — the SAME object, same keys, the
|
|
@@ -2289,10 +2273,11 @@ interface CheckRow {
|
|
|
2289
2273
|
name: string;
|
|
2290
2274
|
status: CheckStatus;
|
|
2291
2275
|
source: CheckSource;
|
|
2276
|
+
agent: string;
|
|
2292
2277
|
/** The checker's model. */
|
|
2293
2278
|
model_name: string;
|
|
2294
|
-
/** The effort every task's checker ran at. */
|
|
2295
|
-
reasoning_effort: string;
|
|
2279
|
+
/** The effort every task's checker ran at; null when its agent takes none. */
|
|
2280
|
+
reasoning_effort: string | null;
|
|
2296
2281
|
sandbox_provider: EvalSandboxProvider;
|
|
2297
2282
|
/** The owning organization's slug (every check has one). */
|
|
2298
2283
|
org: string;
|
|
@@ -4536,7 +4521,7 @@ interface AnalysisTranscript {
|
|
|
4536
4521
|
stored_at?: string[];
|
|
4537
4522
|
}
|
|
4538
4523
|
/**
|
|
4539
|
-
* Client for analysis runs — the analyzer's own transcript,
|
|
4524
|
+
* Client for analysis runs — the analyzer's own transcript, result document,
|
|
4540
4525
|
* and stored artifacts, all globally addressable by analysis id.
|
|
4541
4526
|
*
|
|
4542
4527
|
* DELIBERATELY OFF-CONTRACT: these three reads ride the dashboard's traces
|
|
@@ -4551,7 +4536,7 @@ interface AnalysisTranscript {
|
|
|
4551
4536
|
* GatewayUsageEvent prose, not as an operation. RECORDED TENSION: whether
|
|
4552
4537
|
* that feed and this one join the contract as operations (spec + both SDK
|
|
4553
4538
|
* shadows) is an open ruling, not something settled here. The contract-side
|
|
4554
|
-
*
|
|
4539
|
+
* result stays where it always was — `Trial.analysis` on the trial body;
|
|
4555
4540
|
* this client adds the reads the contract does not carry today.
|
|
4556
4541
|
*/
|
|
4557
4542
|
interface AnalysesClient {
|
|
@@ -4572,7 +4557,7 @@ interface AnalysesClient {
|
|
|
4572
4557
|
/** The defaults an analysis runs under when its config names nothing (GET /api/analyses/defaults): model, effort, provider, rubric and the unrendered prompt template. */
|
|
4573
4558
|
defaults(): Promise<AnalyzeDefaults>;
|
|
4574
4559
|
/**
|
|
4575
|
-
* The
|
|
4560
|
+
* The result document — the wire's TrialAnalysis, statuses and typed
|
|
4576
4561
|
* failure included, for EVERY analysis (not only completed ones). The same
|
|
4577
4562
|
* object the analyzed trial serves as `Trial.analysis` when this analysis
|
|
4578
4563
|
* is its latest; this door answers for earlier analyses too.
|
|
@@ -4604,10 +4589,10 @@ interface AnalysesClient {
|
|
|
4604
4589
|
* one directory named as Harbor names the wrapper trial
|
|
4605
4590
|
* (`analyze-<analyzed trial dir>__<7 chars>/`): config.json, lock.json,
|
|
4606
4591
|
* result.json, trial.log, exception.txt (an infrastructure failure only),
|
|
4607
|
-
* agent
|
|
4592
|
+
* agent/<Harbor's tee name for its agent> (claude-code.txt, codex.txt, …),
|
|
4608
4593
|
* agent/stderr.log, agent/trace-parsed.jsonl, the captured home at its
|
|
4609
|
-
* real names with agent/agent-home.json beside it and Harbor's copy at
|
|
4610
|
-
* agent/sessions
|
|
4594
|
+
* real names with agent/agent-home.json beside it and Harbor's copy at its
|
|
4595
|
+
* per-agent slot (agent/sessions/ for claude and codex), verifier/{test-stdout.txt,reward.txt,reward.json}
|
|
4611
4596
|
* when the validator ruled (reward 1 = a valid analysis.json, 0 = it was
|
|
4612
4597
|
* refused), and artifacts/manifest.json with artifacts/analysis.json (the
|
|
4613
4598
|
* validated {summary, checks}) on a completed run — absent artifacts are
|
|
@@ -4630,13 +4615,13 @@ interface AnalysesClient {
|
|
|
4630
4615
|
/**
|
|
4631
4616
|
* Task-check configuration — Harbor's `harbor check` vocabulary (their
|
|
4632
4617
|
* cli/analyze.py:84-148 check_command), the spec's `CheckConfigInput`. The
|
|
4633
|
-
* rubric-agent
|
|
4634
|
-
* (`AnalyzeConfigInput` states them; refusals name `check.*`): `model_name`
|
|
4618
|
+
* rubric-agent knobs are the analyze door's, under the same rules
|
|
4619
|
+
* (`AnalyzeConfigInput` states them; refusals name `check.*`): `agent` (`-a/--agent`), `model_name`
|
|
4635
4620
|
* (Harbor's check default is `claude-sonnet-4-6`; this platform's is the
|
|
4636
4621
|
* analyzer's `openrouter/deepseek/deepseek-v4.1-flash` — one roster, one
|
|
4637
4622
|
* default for both rubric
|
|
4638
4623
|
* agents, a recorded deviation), `rubric` (the default is the platform's
|
|
4639
|
-
* check rubric
|
|
4624
|
+
* check rubric), and
|
|
4640
4625
|
* `prompt` (the TEXT of Harbor's `-p/--prompt` file, replacing the platform's
|
|
4641
4626
|
* default check body and rendered with `{task_path}`, `{file_tree}`,
|
|
4642
4627
|
* `{criteria_guidance}`; the output contract is appended after it exactly
|
|
@@ -4649,16 +4634,17 @@ interface AnalysesClient {
|
|
|
4649
4634
|
* (`-x/--exclude-task-name`), `n_tasks` (`-l/--n-tasks`) — applied in
|
|
4650
4635
|
* checker.py's order (:132-138: include, exclude, then the cap) over the
|
|
4651
4636
|
* sorted task directory names, Python fnmatch globs against the NAME.
|
|
4652
|
-
* Harbor's `-
|
|
4653
|
-
*
|
|
4654
|
-
* each with its reason.
|
|
4637
|
+
* Harbor's `-o/--jobs-dir`, `-k/--n-attempts` and the local-runner kwargs
|
|
4638
|
+
* are not on this surface; the contract records each with its reason.
|
|
4655
4639
|
*/
|
|
4656
4640
|
interface CheckConfigInput {
|
|
4657
4641
|
/** A name for the check (Harbor's `--job-name`); omitted, the accept timestamp `YYYY-MM-DD__HH-MM-SS`. 1-120 characters. */
|
|
4658
4642
|
name?: string;
|
|
4659
|
-
/**
|
|
4643
|
+
/** The agent the checker runs on (Harbor's `-a/--agent`) — the analyze door's `agent`, same rule. Omitted: claude. */
|
|
4644
|
+
agent?: string;
|
|
4645
|
+
/** Model the checker agent runs (Harbor's `-m/--model`); must be on the agent's roster (`GET /api/meta`). */
|
|
4660
4646
|
model_name?: string;
|
|
4661
|
-
/** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (
|
|
4647
|
+
/** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (`checks().defaults()`). */
|
|
4662
4648
|
rubric?: Rubric;
|
|
4663
4649
|
/** The prompt template — the TEXT of Harbor's `-p/--prompt` file. */
|
|
4664
4650
|
prompt?: string;
|
|
@@ -4719,9 +4705,8 @@ interface CheckSource {
|
|
|
4719
4705
|
/**
|
|
4720
4706
|
* One task's quality check — Harbor's QualityCheckResult shape (their
|
|
4721
4707
|
* cli/quality_checker/models.py:31-35: `task_name`, `checks` keyed by
|
|
4722
|
-
* criterion, `cost_usd`), its checks extended by the result schema
|
|
4723
|
-
*
|
|
4724
|
-
* it belongs to, its lifecycle (the analysis ladder's four lowercase words),
|
|
4708
|
+
* criterion, `cost_usd`), its checks extended by the result schema, plus the
|
|
4709
|
+
* hosted provenance: its own id, the check it belongs to, its lifecycle (the analysis ladder's four lowercase words),
|
|
4725
4710
|
* the bounded attempt count, and a typed `failure` in place of Harbor's
|
|
4726
4711
|
* `error` string (the TrialAnalysis rule).
|
|
4727
4712
|
*
|
|
@@ -4740,17 +4725,6 @@ interface TaskCheck {
|
|
|
4740
4725
|
status: AnalysisStatus;
|
|
4741
4726
|
/** One entry per rubric criterion, keys exactly the frozen criteria. Null until completed. */
|
|
4742
4727
|
checks: Record<string, AnalysisCheck> | null;
|
|
4743
|
-
/** The derived label (CheckLabel states the rule); null until completed, and null under a custom rubric. */
|
|
4744
|
-
label: CheckLabel | null;
|
|
4745
|
-
/**
|
|
4746
|
-
* Whether the box ran the task's environment: true when none of the five
|
|
4747
|
-
* run-based criteria (reference_solution_is_valid,
|
|
4748
|
-
* verifier_rejects_non_solutions, environment_builds_and_runs,
|
|
4749
|
-
* verification_is_stable, limits_allow_the_task) is unknown, so a
|
|
4750
|
-
* reading-only `no_problem_found` is never mistaken for a run. Null
|
|
4751
|
-
* exactly when `label` is null.
|
|
4752
|
-
*/
|
|
4753
|
-
executed: boolean | null;
|
|
4754
4728
|
cost_usd: number | null;
|
|
4755
4729
|
/** 1, or 2 when the one automatic re-run fired (a run that produced no valid check-result.json, the missing file included, is re-run once — the analyze verb's hosted rule; a run cut by its budget is not that class: it settles `failed` with phase `timeout` at once and is never re-run). */
|
|
4756
4730
|
attempts: number;
|
|
@@ -4773,9 +4747,11 @@ interface Check {
|
|
|
4773
4747
|
name: string;
|
|
4774
4748
|
status: CheckStatus;
|
|
4775
4749
|
source: CheckSource;
|
|
4750
|
+
/** The agent every task's checker ran on — named at create, or the default (claude). */
|
|
4751
|
+
agent: string;
|
|
4776
4752
|
model_name: string;
|
|
4777
|
-
/** The effort every task's checker ran at — named at create, or the model's default, resolved at accept. */
|
|
4778
|
-
reasoning_effort: string;
|
|
4753
|
+
/** The effort every task's checker ran at — named at create, or the model's default on the agent, resolved at accept; null when the agent takes none. */
|
|
4754
|
+
reasoning_effort: string | null;
|
|
4779
4755
|
rubric: Rubric;
|
|
4780
4756
|
/** The prompt template as stored; null = the platform's default check body. */
|
|
4781
4757
|
prompt: string | null;
|
|
@@ -4809,12 +4785,13 @@ interface Check {
|
|
|
4809
4785
|
* this serves as the template text.
|
|
4810
4786
|
*/
|
|
4811
4787
|
interface AnalyzeDefaults {
|
|
4788
|
+
agent: string;
|
|
4812
4789
|
model_name: string;
|
|
4813
4790
|
rubric: Rubric;
|
|
4814
4791
|
/** The built-in analyze prompt template, unrendered — pass it as `prompt` to run the default body explicitly, or edit it from here. */
|
|
4815
4792
|
prompt: string;
|
|
4816
|
-
/** The effort the default model runs at when the config names none. */
|
|
4817
|
-
reasoning_effort: string;
|
|
4793
|
+
/** The effort the default model runs at on the default agent when the config names none; null if that agent takes none. */
|
|
4794
|
+
reasoning_effort: string | null;
|
|
4818
4795
|
sandbox_provider: EvalSandboxProvider;
|
|
4819
4796
|
}
|
|
4820
4797
|
/**
|
|
@@ -4824,12 +4801,13 @@ interface AnalyzeDefaults {
|
|
|
4824
4801
|
* template text.
|
|
4825
4802
|
*/
|
|
4826
4803
|
interface CheckDefaults {
|
|
4804
|
+
agent: string;
|
|
4827
4805
|
model_name: string;
|
|
4828
4806
|
rubric: Rubric;
|
|
4829
4807
|
/** The built-in check prompt template, unrendered — pass it as `prompt` to run the default body explicitly, or edit it from here. */
|
|
4830
4808
|
prompt: string;
|
|
4831
|
-
/** The effort the default model runs at when the config names none. */
|
|
4832
|
-
reasoning_effort: string;
|
|
4809
|
+
/** The effort the default model runs at on the default agent when the config names none; null if that agent takes none. */
|
|
4810
|
+
reasoning_effort: string | null;
|
|
4833
4811
|
sandbox_provider: EvalSandboxProvider;
|
|
4834
4812
|
}
|
|
4835
4813
|
/** Options for checks().watch() */
|
|
@@ -4934,9 +4912,9 @@ interface ChecksClient {
|
|
|
4934
4912
|
* the TASK CHECK id (`Check.results[].id`). Each folder is Harbor's
|
|
4935
4913
|
* TrialPaths for the checker's run: config.json, lock.json, result.json,
|
|
4936
4914
|
* trial.log, exception.txt (an infrastructure failure only),
|
|
4937
|
-
* agent
|
|
4938
|
-
* captured home at its real names with agent/agent-home.json and Harbor's
|
|
4939
|
-
*
|
|
4915
|
+
* agent/<its agent's tee name>, agent/stderr.log, agent/trace-parsed.jsonl, the
|
|
4916
|
+
* captured home at its real names with agent/agent-home.json and Harbor's copy at its
|
|
4917
|
+
* per-agent slot (agent/sessions/ for claude and codex), verifier/{test-stdout.txt,reward.txt,
|
|
4940
4918
|
* reward.json} when the validator ruled (reward 1 = a valid
|
|
4941
4919
|
* check-result.json, 0 = it was refused), artifacts/manifest.json and
|
|
4942
4920
|
* artifacts/check-result.json (the validated flat checks) on a completed
|
|
@@ -5300,19 +5278,21 @@ interface CapabilityDocument {
|
|
|
5300
5278
|
dataset_version: StatusVocabulary;
|
|
5301
5279
|
};
|
|
5302
5280
|
/**
|
|
5303
|
-
* The
|
|
5304
|
-
*
|
|
5305
|
-
* accepts (the claude harness's — the analyzer IS that harness), and
|
|
5306
|
-
* every roster model with the effort an omitted `reasoning_effort` takes
|
|
5307
|
-
* for it. Absent on servers predating the field.
|
|
5281
|
+
* The rubric agents' roster and defaults, the analyzer's and the checker's alike, so a client
|
|
5282
|
+
* knows what "omitted" meant. Absent on servers predating the field.
|
|
5308
5283
|
*/
|
|
5309
5284
|
analyze?: {
|
|
5285
|
+
default_agent: string;
|
|
5310
5286
|
default_model: string;
|
|
5311
|
-
|
|
5312
|
-
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
5287
|
+
agents: {
|
|
5288
|
+
name: string;
|
|
5289
|
+
/** Empty for an agent that takes no effort. */
|
|
5290
|
+
reasoning_efforts: string[];
|
|
5291
|
+
models: {
|
|
5292
|
+
alias: string;
|
|
5293
|
+
model_id: string;
|
|
5294
|
+
default_reasoning_effort: string | null;
|
|
5295
|
+
}[];
|
|
5316
5296
|
}[];
|
|
5317
5297
|
};
|
|
5318
5298
|
limits: {
|
|
@@ -5408,4 +5388,4 @@ interface CapabilityDocument {
|
|
|
5408
5388
|
error_codes: string[];
|
|
5409
5389
|
}
|
|
5410
5390
|
|
|
5411
|
-
export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type
|
|
5391
|
+
export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type FilesystemSource as a$, type AnalysisFailure as a0, type AnalysisList as a1, type AnalysisPage as a2, type AnalysisStatus as a3, type AnalysisTranscript as a4, type AnalysisTranscriptOptions as a5, type AnalyzeConfig as a6, type AnalyzeConfigInput as a7, type AnalyzeDefaults as a8, type ApiKey as a9, type DatasetVersionArchiveSource as aA, type DatasetVersionArchiveUrlSource as aB, type DatasetVersionGitSource as aC, type DatasetVersionHubSource as aD, type DatasetVersionSource as aE, type DatasetVersionState as aF, type DownloadDatasetOptions as aG, type DownloadJobOptions as aH, EVAL_SANDBOX_PROVIDERS as aI, type Agent as aJ, type ModelInfo as aK, type EvalSandboxProvider as aL, type StepResult as aM, type ExceptionInfo as aN, type FilesystemArchiveOptions as aO, type FilesystemBox as aP, type FilesystemCapture as aQ, type FilesystemChange as aR, type FilesystemChanges as aS, type FilesystemChangesOptions as aT, type FilesystemEntry as aU, type FilesystemListOptions as aV, type FilesystemListing as aW, type FilesystemReadOptions as aX, type FilesystemSearchHit as aY, type FilesystemSearchOptions as aZ, type FilesystemSearchResult as a_, type AttemptPhase as aa, type AuthStatus as ab, type Awaitable as ac, CHECK_STATUSES as ad, type CheckDefaults as ae, type CheckRow as af, type CheckRowList as ag, type CheckRowPage as ah, type CheckTaskTally as ai, type CompareCell as aj, type CompareCoverage as ak, type CompareJobAggregate as al, type CompareResponse as am, type CompareTaskRow as an, type Dataset as ao, type DatasetImportFailure as ap, type DatasetImportList as aq, type DatasetImportPage as ar, type DatasetImportStatus as as, type DatasetList as at, type DatasetPage as au, type DatasetPatch as av, type DatasetPreflight as aw, type DatasetRef as ax, type DatasetSelector as ay, type DatasetSource as az, type DatasetImport as b, type OrgQuota as b$, type FilesystemState as b0, type FilesystemStatus as b1, type FilesystemStreamEvent as b2, type FilesystemStreamOptions as b3, type FilesystemWatchResult as b4, GATEWAY_TRACE_SEQ_BASE as b5, type GatewayUsage as b6, type GetDatasetOptions as b7, type GrepJobOptions as b8, HOSTED_ERROR_CODES as b9, type JobPage as bA, type JobShareEmail as bB, type JobShareLink as bC, type JobStats as bD, type JobStatus as bE, type JobTaskLink as bF, type JobTaskRollup as bG, type JobTaskRollupList as bH, type JobTaskRollupPage as bI, type JobViewer as bJ, type JobWatch as bK, type JudgeResult as bL, type ListAgentsOptions as bM, type ListAnalysesOptions as bN, type ListDatasetsOptions as bO, type ListImportsOptions as bP, type ListJobImportsOptions as bQ, type ListJobTasksOptions as bR, type ListJobsOptions as bS, type ListSkillsOptions as bT, type ListTrialFilesOptions as bU, type ListTrialsOptions as bV, type ManagedProviderCapability as bW, type OrgInvite as bX, type OrgInviteCreated as bY, type OrgJoined as bZ, type OrgMember as b_, type ImportPhase as ba, type ImportPhaseProgress as bb, type ImportWarning as bc, type InfraFailureSignature as bd, JOB_LIST_KINDS as be, JOB_LIST_SCOPES as bf, type JobAnalysisStats as bg, type JobDeleteResult as bh, type JobFailure as bi, type JobGrepGroup as bj, type JobGrepPage as bk, type JobImport as bl, type JobImportFailure as bm, type JobImportList as bn, type JobImportPage as bo, type JobImportPhaseName as bp, type JobImportPhaseProgress as bq, type JobImportProgress as br, type JobImportSkippedTrial as bs, type JobImportSource as bt, type JobList as bu, type JobListItem as bv, type JobListItemList as bw, type JobListItemPage as bx, type JobListKind as by, type JobListScope as bz, type JobEvent as c, type UploadProvenance as c$, type OrgRole as c0, type OrgUsage as c1, type Organization as c2, type OrganizationDetail as c3, type Page as c4, type PageOptions as c5, type PassAtKGroup as c6, type PassAtKPoint as c7, type PreflightDatasetInput as c8, type PreflightDeferredCheck as c9, TASK_LINK_REASONS as cA, TRIAL_ARTIFACT_STREAMS as cB, TRIAL_STATUSES as cC, type Task as cD, type TaskCheckTranscript as cE, type TaskLinkReason as cF, type TaskLinkedBy as cG, type TaskNote as cH, type TaskPackageFiles as cI, type TaskPackageFilesystemStatus as cJ, type TaskProviderVerdict as cK, type TimingInfo as cL, type TraceEventPage as cM, type TraceOptions as cN, type TrialArtifactStream as cO, type TrialCounts as cP, type TrialFile as cQ, type TrialFilePage as cR, type TrialFileRange as cS, type TrialGpuCost as cT, type TrialList as cU, type TrialPage as cV, type TrialStatus as cW, type TrialStatusTally as cX, type TrialTaskLink as cY, type TrialUploadProvenance as cZ, type UploadJobOptions as c_, type PreflightManifestVerdict as ca, type PreflightTaskVerdict as cb, type ProviderCapability as cc, type PublishDatasetOptions as cd, type RegradeRequest as ce, type ResumeRequest as cf, type RetryRequest as cg, type RubricCriterion as ch, type RunFilesystem as ci, SANDBOX_LOG_STREAMS as cj, type SandboxLogEvent as ck, type SandboxLogLine as cl, type SandboxLogLines as cm, type SandboxLogOptions as cn, type SandboxLogStream as co, type SandboxProcs as cp, type SkillLock as cq, type SkillUpload as cr, type SkillUploadList as cs, type SkillUploadPage as ct, type SourceJob as cu, type SpendSource as cv, type StartJobOptions as cw, type StatusVocabulary as cx, type StopResponse as cy, TASK_LINKED_BY as cz, type DatasetFailedTask as d, type UploadSkillOptions as d0, type UpstreamStatus as d1, type VerifierEnvironmentMode as d2, type VerifierResult as d3, type WatchAnalysisOptions as d4, type WatchImportOptions as d5, type WatchJobImportOptions as d6, type WatchJobOptions as d7, gatewayUsageOf as d8, isHostedErrorCode as d9, passAtK as da, type DatasetImportProgress as e, type JobSecretInline as f, type JobSecretRef as g, type TraceEvent as h, type JobVisibility as i, type JobShareRequest as j, type JobShares as k, type Job as l, type TaskCheck as m, type DatasetsClient as n, type AgentsClient as o, type JobsClient as p, type TrialsClient as q, type AnalysesClient as r, type ChecksClient as s, type CapabilityDocument as t, type HostedClientConfig as u, type AuthClient as v, AGENT_EFFORT_SUPPORT_VALUES as w, ANALYSIS_ARTIFACT_STREAMS as x, ANALYSIS_STATUSES as y, type ActiveDataset as z };
|