@evolvingmachines/evolve 0.0.63 → 0.0.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-DK2MGUPB.js +427 -0
- package/dist/cli/index.cjs +39 -39
- package/dist/cli/index.d.cts +1 -1
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +30 -30
- package/dist/index.cjs +47 -47
- package/dist/index.d.cts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +22 -22
- package/dist/{types-LYvFBfCo.d.cts → types-B_qmzlMk.d.cts} +125 -5
- package/dist/{types-LYvFBfCo.d.ts → types-B_qmzlMk.d.ts} +125 -5
- package/package.json +4 -4
- package/skills/evolve-evals/SKILL.md +1 -1
- package/skills/evolve-evals/references/cli-reference/analyze.mdx +11 -1
- package/skills/evolve-evals/references/cli-reference/job.mdx +6 -1
- package/skills/evolve-evals/references/core-concepts/analyze.mdx +2 -2
- package/skills/evolve-evals/references/core-concepts/sharing.mdx +1 -1
- package/skills/evolve-evals/references/core-concepts/teams.mdx +3 -3
- package/skills/evolve-evals/references/core-concepts/upload.mdx +2 -0
- package/skills/evolve-evals/references/dashboard/checks.mdx +7 -1
- package/skills/evolve-evals/references/dashboard/jobs.mdx +4 -4
- package/skills/evolve-evals/references/dashboard/traces.mdx +2 -2
- package/skills/evolve-evals/references/dashboard/trial-viewer.mdx +1 -1
- package/skills/evolve-evals/references/sdk-reference/analyses.mdx +27 -0
- package/skills/evolve-evals/references/sdk-reference/jobs.mdx +16 -5
- package/skills/evolve-evals/references/sdk-reference/types.mdx +2 -1
- package/spec/openapi.yaml +313 -13
- package/dist/chunk-B4QFLLPV.js +0 -427
|
@@ -112,6 +112,13 @@ type EvalSandboxProvider = (typeof EVAL_SANDBOX_PROVIDERS)[number];
|
|
|
112
112
|
declare const JOB_LIST_SCOPES: readonly ["my", "shared", "org"];
|
|
113
113
|
/** One list scope — see JOB_LIST_SCOPES. */
|
|
114
114
|
type JobListScope = (typeof JOB_LIST_SCOPES)[number];
|
|
115
|
+
/**
|
|
116
|
+
* A check is a job, so the list takes a `kind` (the ruling and the Harbor lines live on the spec's
|
|
117
|
+
* JobListKind parameter). A runtime value so the CLI validates `--kind` against it.
|
|
118
|
+
*/
|
|
119
|
+
declare const JOB_LIST_KINDS: readonly ["job", "check", "all"];
|
|
120
|
+
/** One list kind — see JOB_LIST_KINDS. */
|
|
121
|
+
type JobListKind = (typeof JOB_LIST_KINDS)[number];
|
|
115
122
|
/**
|
|
116
123
|
* An analysis's own lifecycle ladder — lowercase, the object's Harbor
|
|
117
124
|
* dialect (spec TrialAnalysis.status). A runtime value so the CLI validates
|
|
@@ -568,6 +575,22 @@ interface AnalyzeConfigInput {
|
|
|
568
575
|
* anything else is refused `invalid_input` naming `analyze.n_trials`.
|
|
569
576
|
*/
|
|
570
577
|
n_trials?: number;
|
|
578
|
+
/**
|
|
579
|
+
* Analyze only these trials of the job — Harbor's `harbor analyze <trial
|
|
580
|
+
* directory>` for one trial (their cli/analyze.py:242-245), given as ids
|
|
581
|
+
* here; combinable with `passing`/`failing` (a listed trial on the other
|
|
582
|
+
* side of the filter is skipped) and applied before `n_trials` (Harbor's
|
|
583
|
+
* one-trial path skips its filter and cap, analyzer.py:180-181 before
|
|
584
|
+
* :189-191; hosted, both apply to the named trials too). An id that is not
|
|
585
|
+
* a trial of this job is refused `invalid_input` naming `analyze.trial_ids`
|
|
586
|
+
* with the unknown ids named in the message and in
|
|
587
|
+
* `details.unknown_trial_ids`; a list that leaves nothing analyzable is the
|
|
588
|
+
* 409 `no_analyzable_trials`; an empty list, a duplicate, an empty string
|
|
589
|
+
* or a non-string is refused `invalid_input`. Only `jobs().analyze()` takes
|
|
590
|
+
* it: on `start({ analyze })` (the embedded trigger) it is refused — the
|
|
591
|
+
* trials do not exist yet.
|
|
592
|
+
*/
|
|
593
|
+
trial_ids?: string[];
|
|
571
594
|
}
|
|
572
595
|
/**
|
|
573
596
|
* The RESOLVED trace-analysis policy — the caller's values or the defaults of
|
|
@@ -611,6 +634,11 @@ interface AnalyzeConfig {
|
|
|
611
634
|
failing: boolean;
|
|
612
635
|
/** The trial cap as stored (`AnalyzeConfigInput.n_trials`); null = no cap. */
|
|
613
636
|
n_trials: number | null;
|
|
637
|
+
/**
|
|
638
|
+
* Always null on the job body today: the stored policy is the create-time
|
|
639
|
+
* one, which refuses `trial_ids`; a manual wave's named trials are the rows it enqueues.
|
|
640
|
+
*/
|
|
641
|
+
trial_ids: string[] | null;
|
|
614
642
|
}
|
|
615
643
|
/** The job-creation body — POST /api/jobs. */
|
|
616
644
|
interface JobCreate {
|
|
@@ -1235,11 +1263,20 @@ interface JobBuildExclusion {
|
|
|
1235
1263
|
failed_task_names: string[];
|
|
1236
1264
|
note: string;
|
|
1237
1265
|
}
|
|
1266
|
+
/**
|
|
1267
|
+
* The caller's relation to a job on a read (Job.viewer): `creator` made it,
|
|
1268
|
+
* `member` belongs to its organization, `shared` reads it through an email
|
|
1269
|
+
* share, `link` through its unlisted link. Acting verbs (analyze, cancel,
|
|
1270
|
+
* resume, retry, regrade) are open to `creator` and `member` only.
|
|
1271
|
+
*/
|
|
1272
|
+
type JobViewer = "creator" | "member" | "shared" | "link";
|
|
1238
1273
|
/**
|
|
1239
1274
|
* THE job body — the same shape from create, get, list items, cancel, resume,
|
|
1240
1275
|
* and regrade responses; no field appears on some responses and not others.
|
|
1241
1276
|
*/
|
|
1242
1277
|
interface Job {
|
|
1278
|
+
/** The row's kind on the jobs list (JobListItem): always `job` on a Job body; a CheckRow says `check`. */
|
|
1279
|
+
kind: "job";
|
|
1243
1280
|
id: string;
|
|
1244
1281
|
/** User-facing label. */
|
|
1245
1282
|
job_name: string;
|
|
@@ -1328,6 +1365,14 @@ interface Job {
|
|
|
1328
1365
|
source_jobs: SourceJob[];
|
|
1329
1366
|
/** Derived: any source_jobs entry with action "regrade". */
|
|
1330
1367
|
is_regrade: boolean;
|
|
1368
|
+
/**
|
|
1369
|
+
* The caller's relation on a read (JobViewer). Null on the responses that
|
|
1370
|
+
* echo a job the caller just acted on (create, analyze, cancel, resume,
|
|
1371
|
+
* retry, regrade), where the caller is the creator or a member by
|
|
1372
|
+
* construction, and on list rows (batched; read one job to learn it);
|
|
1373
|
+
* null too from a server older than the field.
|
|
1374
|
+
*/
|
|
1375
|
+
viewer: JobViewer | null;
|
|
1331
1376
|
/**
|
|
1332
1377
|
* The upload provenance echo — null for every job this platform executed,
|
|
1333
1378
|
* non-null only on a job ingested by jobs().upload(). See UploadProvenance.
|
|
@@ -2225,6 +2270,54 @@ type CheckPage = Page<Check>;
|
|
|
2225
2270
|
/** The handle returned by checks().list() — one page on await, every page on for-await. */
|
|
2226
2271
|
interface CheckList extends Awaitable<CheckPage>, AsyncIterable<Check> {
|
|
2227
2272
|
}
|
|
2273
|
+
/**
|
|
2274
|
+
* Spec CheckTaskTally. A stopped task check is stored `failed` with the stop phase; the tally counts it
|
|
2275
|
+
* under `stopped`, never `failed`, so a stop never reads as a failure.
|
|
2276
|
+
*/
|
|
2277
|
+
interface CheckTaskTally {
|
|
2278
|
+
total: number;
|
|
2279
|
+
byStatus: Record<AnalysisStatus | "stopped", number>;
|
|
2280
|
+
}
|
|
2281
|
+
/**
|
|
2282
|
+
* Spec CheckRow, one row of `jobs().list({ kind: "check" | "all" })`: a check is a job, so it lists beside them.
|
|
2283
|
+
* Not a Job (no arms, attempts, caps, retry policy, trials or upload provenance), so those fields are absent, never faked.
|
|
2284
|
+
*/
|
|
2285
|
+
interface CheckRow {
|
|
2286
|
+
kind: "check";
|
|
2287
|
+
id: string;
|
|
2288
|
+
/** Check.name — Harbor's `--job-name`: the caller's, or the accept stamp. */
|
|
2289
|
+
name: string;
|
|
2290
|
+
status: CheckStatus;
|
|
2291
|
+
source: CheckSource;
|
|
2292
|
+
/** The checker's model. */
|
|
2293
|
+
model_name: string;
|
|
2294
|
+
/** The effort every task's checker ran at. */
|
|
2295
|
+
reasoning_effort: string;
|
|
2296
|
+
sandbox_provider: EvalSandboxProvider;
|
|
2297
|
+
/** The owning organization's slug (every check has one). */
|
|
2298
|
+
org: string;
|
|
2299
|
+
visibility: JobVisibility;
|
|
2300
|
+
/** How many task checks, and how they break down; done = completed + failed + stopped. */
|
|
2301
|
+
tasks: CheckTaskTally;
|
|
2302
|
+
/** The sum of the measured task costs; null when none was measured. */
|
|
2303
|
+
cost_usd: number | null;
|
|
2304
|
+
/** The accept instant; the list orders it with the jobs' `started_at`. */
|
|
2305
|
+
created_at: string;
|
|
2306
|
+
/** When the last task settled; null until every task has. */
|
|
2307
|
+
finished_at: string | null;
|
|
2308
|
+
}
|
|
2309
|
+
/** One row of the jobs list, told apart by `kind` (spec JobListItem). */
|
|
2310
|
+
type JobListItem = Job | CheckRow;
|
|
2311
|
+
/** Cursor page of check rows (newest first) — `jobs().list({ kind: "check" })`. */
|
|
2312
|
+
type CheckRowPage = Page<CheckRow>;
|
|
2313
|
+
/** The handle returned by `jobs().list({ kind: "check" })`. */
|
|
2314
|
+
interface CheckRowList extends Awaitable<CheckRowPage>, AsyncIterable<CheckRow> {
|
|
2315
|
+
}
|
|
2316
|
+
/** Cursor page of jobs and check rows (newest first) — `jobs().list({ kind: "all" })`. */
|
|
2317
|
+
type JobListItemPage = Page<JobListItem>;
|
|
2318
|
+
/** The handle returned by `jobs().list({ kind: "all" })`. */
|
|
2319
|
+
interface JobListItemList extends Awaitable<JobListItemPage>, AsyncIterable<JobListItem> {
|
|
2320
|
+
}
|
|
2228
2321
|
/**
|
|
2229
2322
|
* One task's rollup within a job: its trial tally, mean reward over SCORED
|
|
2230
2323
|
* trials, and measured cost. Sits between the job body and the trial list so
|
|
@@ -3140,6 +3233,8 @@ interface ListJobsOptions extends PageOptions {
|
|
|
3140
3233
|
* See JOB_LIST_SCOPES.
|
|
3141
3234
|
*/
|
|
3142
3235
|
scope?: JobListScope;
|
|
3236
|
+
/** `job` (the server's default), `check` or `all`; see JOB_LIST_KINDS. */
|
|
3237
|
+
kind?: JobListKind;
|
|
3143
3238
|
}
|
|
3144
3239
|
/** Options for analyses().list() (default page 50, max 200) */
|
|
3145
3240
|
interface ListAnalysesOptions extends PageOptions {
|
|
@@ -3755,11 +3850,19 @@ interface JobsClient {
|
|
|
3755
3850
|
/** Get one job */
|
|
3756
3851
|
get(id: string): Promise<Job>;
|
|
3757
3852
|
/**
|
|
3758
|
-
* List the caller's jobs, newest first (cursor-paged)
|
|
3759
|
-
*
|
|
3760
|
-
* cursor pages transparently.
|
|
3853
|
+
* List the caller's jobs, newest first (cursor-paged): await one page, or `for await` every page.
|
|
3854
|
+
* One overload per `kind`, so the rows are typed as what was asked for.
|
|
3761
3855
|
*/
|
|
3762
|
-
list(options?: ListJobsOptions
|
|
3856
|
+
list(options?: ListJobsOptions & {
|
|
3857
|
+
kind?: "job";
|
|
3858
|
+
}): JobList;
|
|
3859
|
+
list(options: ListJobsOptions & {
|
|
3860
|
+
kind: "check";
|
|
3861
|
+
}): CheckRowList;
|
|
3862
|
+
list(options: ListJobsOptions & {
|
|
3863
|
+
kind: "all";
|
|
3864
|
+
}): JobListItemList;
|
|
3865
|
+
list(options: ListJobsOptions): JobListItemList;
|
|
3763
3866
|
/**
|
|
3764
3867
|
* List a job's trials (cursor-paged; { status } filters, e.g. to
|
|
3765
3868
|
* the failed trials). Await the result for one page, or `for await` it to
|
|
@@ -4466,6 +4569,8 @@ interface AnalysesClient {
|
|
|
4466
4569
|
* feed's species-blind events door, which answers `trial_not_found`.
|
|
4467
4570
|
*/
|
|
4468
4571
|
list(options?: ListAnalysesOptions): AnalysisList;
|
|
4572
|
+
/** The defaults an analysis runs under when its config names nothing (GET /api/analyses/defaults): model, effort, provider, rubric and the unrendered prompt template. */
|
|
4573
|
+
defaults(): Promise<AnalyzeDefaults>;
|
|
4469
4574
|
/**
|
|
4470
4575
|
* The verdict document — the wire's TrialAnalysis, statuses and typed
|
|
4471
4576
|
* failure included, for EVERY analysis (not only completed ones). The same
|
|
@@ -4697,6 +4802,21 @@ interface Check {
|
|
|
4697
4802
|
/** When the last task settled; null until every task has. */
|
|
4698
4803
|
finished_at: string | null;
|
|
4699
4804
|
}
|
|
4805
|
+
/**
|
|
4806
|
+
* The policy an empty analyze config resolves to (GET /api/analyses/defaults):
|
|
4807
|
+
* each key the value `AnalyzeConfig` echoes for a job created with
|
|
4808
|
+
* `analyze: {}`, except `prompt`, which `AnalyzeConfig` serves as null and
|
|
4809
|
+
* this serves as the template text.
|
|
4810
|
+
*/
|
|
4811
|
+
interface AnalyzeDefaults {
|
|
4812
|
+
model_name: string;
|
|
4813
|
+
rubric: Rubric;
|
|
4814
|
+
/** The built-in analyze prompt template, unrendered — pass it as `prompt` to run the default body explicitly, or edit it from here. */
|
|
4815
|
+
prompt: string;
|
|
4816
|
+
/** The effort the default model runs at when the config names none. */
|
|
4817
|
+
reasoning_effort: string;
|
|
4818
|
+
sandbox_provider: EvalSandboxProvider;
|
|
4819
|
+
}
|
|
4700
4820
|
/**
|
|
4701
4821
|
* The policy an empty check config resolves to (GET /api/checks/defaults):
|
|
4702
4822
|
* each key the value `Check` echoes for a check created with no config,
|
|
@@ -5288,4 +5408,4 @@ interface CapabilityDocument {
|
|
|
5288
5408
|
error_codes: string[];
|
|
5289
5409
|
}
|
|
5290
5410
|
|
|
5291
|
-
export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type
|
|
5411
|
+
export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type FilesystemSearchOptions as a$, type AnalysisFailure as a0, type AnalysisLabel as a1, type AnalysisList as a2, type AnalysisPage as a3, type AnalysisStatus as a4, type AnalysisTranscript as a5, type AnalysisTranscriptOptions as a6, type AnalyzeConfig as a7, type AnalyzeConfigInput as a8, type AnalyzeDefaults as a9, type DatasetSelector as aA, type DatasetSource as aB, type DatasetVersionArchiveSource as aC, type DatasetVersionArchiveUrlSource as aD, type DatasetVersionGitSource as aE, type DatasetVersionHubSource as aF, type DatasetVersionSource as aG, type DatasetVersionState as aH, type DownloadDatasetOptions as aI, type DownloadJobOptions as aJ, EVAL_SANDBOX_PROVIDERS as aK, type Agent as aL, type ModelInfo as aM, type EvalSandboxProvider as aN, type StepResult as aO, type ExceptionInfo as aP, type FilesystemArchiveOptions as aQ, type FilesystemBox as aR, type FilesystemCapture as aS, type FilesystemChange as aT, type FilesystemChanges as aU, type FilesystemChangesOptions as aV, type FilesystemEntry as aW, type FilesystemListOptions as aX, type FilesystemListing as aY, type FilesystemReadOptions as aZ, type FilesystemSearchHit as a_, type ApiKey as aa, type AttemptPhase as ab, type AuthStatus as ac, type Awaitable as ad, CHECK_STATUSES as ae, type CheckDefaults as af, type CheckLabel as ag, type CheckRow as ah, type CheckRowList as ai, type CheckRowPage as aj, type CheckTaskTally as ak, type CompareCell as al, type CompareCoverage as am, type CompareJobAggregate as an, type CompareResponse as ao, type CompareTaskRow as ap, type Dataset as aq, type DatasetImportFailure as ar, type DatasetImportList as as, type DatasetImportPage as at, type DatasetImportStatus as au, type DatasetList as av, type DatasetPage as aw, type DatasetPatch as ax, type DatasetPreflight as ay, type DatasetRef as az, type DatasetImport as b, type OrgJoined as b$, type FilesystemSearchResult as b0, type FilesystemSource as b1, type FilesystemState as b2, type FilesystemStatus as b3, type FilesystemStreamEvent as b4, type FilesystemStreamOptions as b5, type FilesystemWatchResult as b6, GATEWAY_TRACE_SEQ_BASE as b7, type GatewayUsage as b8, type GetDatasetOptions as b9, type JobListKind as bA, type JobListScope as bB, type JobPage as bC, type JobShareEmail as bD, type JobShareLink as bE, type JobStats as bF, type JobStatus as bG, type JobTaskLink as bH, type JobTaskRollup as bI, type JobTaskRollupList as bJ, type JobTaskRollupPage as bK, type JobViewer as bL, type JobWatch as bM, type JudgeResult as bN, type ListAgentsOptions as bO, type ListAnalysesOptions as bP, type ListDatasetsOptions as bQ, type ListImportsOptions as bR, type ListJobImportsOptions as bS, type ListJobTasksOptions as bT, type ListJobsOptions as bU, type ListSkillsOptions as bV, type ListTrialFilesOptions as bW, type ListTrialsOptions as bX, type ManagedProviderCapability as bY, type OrgInvite as bZ, type OrgInviteCreated as b_, type GrepJobOptions as ba, HOSTED_ERROR_CODES as bb, type ImportPhase as bc, type ImportPhaseProgress as bd, type ImportWarning as be, type InfraFailureSignature as bf, JOB_LIST_KINDS as bg, JOB_LIST_SCOPES as bh, type JobAnalysisStats as bi, type JobDeleteResult as bj, type JobFailure as bk, type JobGrepGroup as bl, type JobGrepPage as bm, type JobImport as bn, type JobImportFailure as bo, type JobImportList as bp, type JobImportPage as bq, type JobImportPhaseName as br, type JobImportPhaseProgress as bs, type JobImportProgress as bt, type JobImportSkippedTrial as bu, type JobImportSource as bv, type JobList as bw, type JobListItem as bx, type JobListItemList as by, type JobListItemPage as bz, type JobEvent as c, type TrialUploadProvenance as c$, type OrgMember as c0, type OrgQuota as c1, type OrgRole as c2, type OrgUsage as c3, type Organization as c4, type OrganizationDetail as c5, type Page as c6, type PageOptions as c7, type PassAtKGroup as c8, type PassAtKPoint as c9, type StopResponse as cA, TASK_LINKED_BY as cB, TASK_LINK_REASONS as cC, TRIAL_ARTIFACT_STREAMS as cD, TRIAL_STATUSES as cE, type Task as cF, type TaskCheckTranscript as cG, type TaskLinkReason as cH, type TaskLinkedBy as cI, type TaskNote as cJ, type TaskPackageFiles as cK, type TaskPackageFilesystemStatus as cL, type TaskProviderVerdict as cM, type TimingInfo as cN, type TraceEventPage as cO, type TraceOptions as cP, type TrialArtifactStream as cQ, type TrialCounts as cR, type TrialFile as cS, type TrialFilePage as cT, type TrialFileRange as cU, type TrialGpuCost as cV, type TrialList as cW, type TrialPage as cX, type TrialStatus as cY, type TrialStatusTally as cZ, type TrialTaskLink as c_, type PreflightDatasetInput as ca, type PreflightDeferredCheck as cb, type PreflightManifestVerdict as cc, type PreflightTaskVerdict as cd, type ProviderCapability as ce, type PublishDatasetOptions as cf, type RegradeRequest as cg, type ResumeRequest as ch, type RetryRequest as ci, type RubricCriterion as cj, type RunFilesystem as ck, SANDBOX_LOG_STREAMS as cl, type SandboxLogEvent as cm, type SandboxLogLine as cn, type SandboxLogLines as co, type SandboxLogOptions as cp, type SandboxLogStream as cq, type SandboxProcs as cr, type SkillLock as cs, type SkillUpload as ct, type SkillUploadList as cu, type SkillUploadPage as cv, type SourceJob as cw, type SpendSource as cx, type StartJobOptions as cy, type StatusVocabulary as cz, type DatasetFailedTask as d, type UploadJobOptions as d0, type UploadProvenance as d1, type UploadSkillOptions as d2, type UpstreamStatus as d3, type VerifierEnvironmentMode as d4, type VerifierResult as d5, type WatchAnalysisOptions as d6, type WatchImportOptions as d7, type WatchJobImportOptions as d8, type WatchJobOptions as d9, gatewayUsageOf as da, isHostedErrorCode as db, passAtK as dc, type DatasetImportProgress as e, type JobSecretInline as f, type JobSecretRef as g, type TraceEvent as h, type JobVisibility as i, type JobShareRequest as j, type JobShares as k, type Job as l, type TaskCheck as m, type DatasetsClient as n, type AgentsClient as o, type JobsClient as p, type TrialsClient as q, type AnalysesClient as r, type ChecksClient as s, type CapabilityDocument as t, type HostedClientConfig as u, type AuthClient as v, AGENT_EFFORT_SUPPORT_VALUES as w, ANALYSIS_ARTIFACT_STREAMS as x, ANALYSIS_STATUSES as y, type ActiveDataset as z };
|
|
@@ -112,6 +112,13 @@ type EvalSandboxProvider = (typeof EVAL_SANDBOX_PROVIDERS)[number];
|
|
|
112
112
|
declare const JOB_LIST_SCOPES: readonly ["my", "shared", "org"];
|
|
113
113
|
/** One list scope — see JOB_LIST_SCOPES. */
|
|
114
114
|
type JobListScope = (typeof JOB_LIST_SCOPES)[number];
|
|
115
|
+
/**
|
|
116
|
+
* A check is a job, so the list takes a `kind` (the ruling and the Harbor lines live on the spec's
|
|
117
|
+
* JobListKind parameter). A runtime value so the CLI validates `--kind` against it.
|
|
118
|
+
*/
|
|
119
|
+
declare const JOB_LIST_KINDS: readonly ["job", "check", "all"];
|
|
120
|
+
/** One list kind — see JOB_LIST_KINDS. */
|
|
121
|
+
type JobListKind = (typeof JOB_LIST_KINDS)[number];
|
|
115
122
|
/**
|
|
116
123
|
* An analysis's own lifecycle ladder — lowercase, the object's Harbor
|
|
117
124
|
* dialect (spec TrialAnalysis.status). A runtime value so the CLI validates
|
|
@@ -568,6 +575,22 @@ interface AnalyzeConfigInput {
|
|
|
568
575
|
* anything else is refused `invalid_input` naming `analyze.n_trials`.
|
|
569
576
|
*/
|
|
570
577
|
n_trials?: number;
|
|
578
|
+
/**
|
|
579
|
+
* Analyze only these trials of the job — Harbor's `harbor analyze <trial
|
|
580
|
+
* directory>` for one trial (their cli/analyze.py:242-245), given as ids
|
|
581
|
+
* here; combinable with `passing`/`failing` (a listed trial on the other
|
|
582
|
+
* side of the filter is skipped) and applied before `n_trials` (Harbor's
|
|
583
|
+
* one-trial path skips its filter and cap, analyzer.py:180-181 before
|
|
584
|
+
* :189-191; hosted, both apply to the named trials too). An id that is not
|
|
585
|
+
* a trial of this job is refused `invalid_input` naming `analyze.trial_ids`
|
|
586
|
+
* with the unknown ids named in the message and in
|
|
587
|
+
* `details.unknown_trial_ids`; a list that leaves nothing analyzable is the
|
|
588
|
+
* 409 `no_analyzable_trials`; an empty list, a duplicate, an empty string
|
|
589
|
+
* or a non-string is refused `invalid_input`. Only `jobs().analyze()` takes
|
|
590
|
+
* it: on `start({ analyze })` (the embedded trigger) it is refused — the
|
|
591
|
+
* trials do not exist yet.
|
|
592
|
+
*/
|
|
593
|
+
trial_ids?: string[];
|
|
571
594
|
}
|
|
572
595
|
/**
|
|
573
596
|
* The RESOLVED trace-analysis policy — the caller's values or the defaults of
|
|
@@ -611,6 +634,11 @@ interface AnalyzeConfig {
|
|
|
611
634
|
failing: boolean;
|
|
612
635
|
/** The trial cap as stored (`AnalyzeConfigInput.n_trials`); null = no cap. */
|
|
613
636
|
n_trials: number | null;
|
|
637
|
+
/**
|
|
638
|
+
* Always null on the job body today: the stored policy is the create-time
|
|
639
|
+
* one, which refuses `trial_ids`; a manual wave's named trials are the rows it enqueues.
|
|
640
|
+
*/
|
|
641
|
+
trial_ids: string[] | null;
|
|
614
642
|
}
|
|
615
643
|
/** The job-creation body — POST /api/jobs. */
|
|
616
644
|
interface JobCreate {
|
|
@@ -1235,11 +1263,20 @@ interface JobBuildExclusion {
|
|
|
1235
1263
|
failed_task_names: string[];
|
|
1236
1264
|
note: string;
|
|
1237
1265
|
}
|
|
1266
|
+
/**
|
|
1267
|
+
* The caller's relation to a job on a read (Job.viewer): `creator` made it,
|
|
1268
|
+
* `member` belongs to its organization, `shared` reads it through an email
|
|
1269
|
+
* share, `link` through its unlisted link. Acting verbs (analyze, cancel,
|
|
1270
|
+
* resume, retry, regrade) are open to `creator` and `member` only.
|
|
1271
|
+
*/
|
|
1272
|
+
type JobViewer = "creator" | "member" | "shared" | "link";
|
|
1238
1273
|
/**
|
|
1239
1274
|
* THE job body — the same shape from create, get, list items, cancel, resume,
|
|
1240
1275
|
* and regrade responses; no field appears on some responses and not others.
|
|
1241
1276
|
*/
|
|
1242
1277
|
interface Job {
|
|
1278
|
+
/** The row's kind on the jobs list (JobListItem): always `job` on a Job body; a CheckRow says `check`. */
|
|
1279
|
+
kind: "job";
|
|
1243
1280
|
id: string;
|
|
1244
1281
|
/** User-facing label. */
|
|
1245
1282
|
job_name: string;
|
|
@@ -1328,6 +1365,14 @@ interface Job {
|
|
|
1328
1365
|
source_jobs: SourceJob[];
|
|
1329
1366
|
/** Derived: any source_jobs entry with action "regrade". */
|
|
1330
1367
|
is_regrade: boolean;
|
|
1368
|
+
/**
|
|
1369
|
+
* The caller's relation on a read (JobViewer). Null on the responses that
|
|
1370
|
+
* echo a job the caller just acted on (create, analyze, cancel, resume,
|
|
1371
|
+
* retry, regrade), where the caller is the creator or a member by
|
|
1372
|
+
* construction, and on list rows (batched; read one job to learn it);
|
|
1373
|
+
* null too from a server older than the field.
|
|
1374
|
+
*/
|
|
1375
|
+
viewer: JobViewer | null;
|
|
1331
1376
|
/**
|
|
1332
1377
|
* The upload provenance echo — null for every job this platform executed,
|
|
1333
1378
|
* non-null only on a job ingested by jobs().upload(). See UploadProvenance.
|
|
@@ -2225,6 +2270,54 @@ type CheckPage = Page<Check>;
|
|
|
2225
2270
|
/** The handle returned by checks().list() — one page on await, every page on for-await. */
|
|
2226
2271
|
interface CheckList extends Awaitable<CheckPage>, AsyncIterable<Check> {
|
|
2227
2272
|
}
|
|
2273
|
+
/**
|
|
2274
|
+
* Spec CheckTaskTally. A stopped task check is stored `failed` with the stop phase; the tally counts it
|
|
2275
|
+
* under `stopped`, never `failed`, so a stop never reads as a failure.
|
|
2276
|
+
*/
|
|
2277
|
+
interface CheckTaskTally {
|
|
2278
|
+
total: number;
|
|
2279
|
+
byStatus: Record<AnalysisStatus | "stopped", number>;
|
|
2280
|
+
}
|
|
2281
|
+
/**
|
|
2282
|
+
* Spec CheckRow, one row of `jobs().list({ kind: "check" | "all" })`: a check is a job, so it lists beside them.
|
|
2283
|
+
* Not a Job (no arms, attempts, caps, retry policy, trials or upload provenance), so those fields are absent, never faked.
|
|
2284
|
+
*/
|
|
2285
|
+
interface CheckRow {
|
|
2286
|
+
kind: "check";
|
|
2287
|
+
id: string;
|
|
2288
|
+
/** Check.name — Harbor's `--job-name`: the caller's, or the accept stamp. */
|
|
2289
|
+
name: string;
|
|
2290
|
+
status: CheckStatus;
|
|
2291
|
+
source: CheckSource;
|
|
2292
|
+
/** The checker's model. */
|
|
2293
|
+
model_name: string;
|
|
2294
|
+
/** The effort every task's checker ran at. */
|
|
2295
|
+
reasoning_effort: string;
|
|
2296
|
+
sandbox_provider: EvalSandboxProvider;
|
|
2297
|
+
/** The owning organization's slug (every check has one). */
|
|
2298
|
+
org: string;
|
|
2299
|
+
visibility: JobVisibility;
|
|
2300
|
+
/** How many task checks, and how they break down; done = completed + failed + stopped. */
|
|
2301
|
+
tasks: CheckTaskTally;
|
|
2302
|
+
/** The sum of the measured task costs; null when none was measured. */
|
|
2303
|
+
cost_usd: number | null;
|
|
2304
|
+
/** The accept instant; the list orders it with the jobs' `started_at`. */
|
|
2305
|
+
created_at: string;
|
|
2306
|
+
/** When the last task settled; null until every task has. */
|
|
2307
|
+
finished_at: string | null;
|
|
2308
|
+
}
|
|
2309
|
+
/** One row of the jobs list, told apart by `kind` (spec JobListItem). */
|
|
2310
|
+
type JobListItem = Job | CheckRow;
|
|
2311
|
+
/** Cursor page of check rows (newest first) — `jobs().list({ kind: "check" })`. */
|
|
2312
|
+
type CheckRowPage = Page<CheckRow>;
|
|
2313
|
+
/** The handle returned by `jobs().list({ kind: "check" })`. */
|
|
2314
|
+
interface CheckRowList extends Awaitable<CheckRowPage>, AsyncIterable<CheckRow> {
|
|
2315
|
+
}
|
|
2316
|
+
/** Cursor page of jobs and check rows (newest first) — `jobs().list({ kind: "all" })`. */
|
|
2317
|
+
type JobListItemPage = Page<JobListItem>;
|
|
2318
|
+
/** The handle returned by `jobs().list({ kind: "all" })`. */
|
|
2319
|
+
interface JobListItemList extends Awaitable<JobListItemPage>, AsyncIterable<JobListItem> {
|
|
2320
|
+
}
|
|
2228
2321
|
/**
|
|
2229
2322
|
* One task's rollup within a job: its trial tally, mean reward over SCORED
|
|
2230
2323
|
* trials, and measured cost. Sits between the job body and the trial list so
|
|
@@ -3140,6 +3233,8 @@ interface ListJobsOptions extends PageOptions {
|
|
|
3140
3233
|
* See JOB_LIST_SCOPES.
|
|
3141
3234
|
*/
|
|
3142
3235
|
scope?: JobListScope;
|
|
3236
|
+
/** `job` (the server's default), `check` or `all`; see JOB_LIST_KINDS. */
|
|
3237
|
+
kind?: JobListKind;
|
|
3143
3238
|
}
|
|
3144
3239
|
/** Options for analyses().list() (default page 50, max 200) */
|
|
3145
3240
|
interface ListAnalysesOptions extends PageOptions {
|
|
@@ -3755,11 +3850,19 @@ interface JobsClient {
|
|
|
3755
3850
|
/** Get one job */
|
|
3756
3851
|
get(id: string): Promise<Job>;
|
|
3757
3852
|
/**
|
|
3758
|
-
* List the caller's jobs, newest first (cursor-paged)
|
|
3759
|
-
*
|
|
3760
|
-
* cursor pages transparently.
|
|
3853
|
+
* List the caller's jobs, newest first (cursor-paged): await one page, or `for await` every page.
|
|
3854
|
+
* One overload per `kind`, so the rows are typed as what was asked for.
|
|
3761
3855
|
*/
|
|
3762
|
-
list(options?: ListJobsOptions
|
|
3856
|
+
list(options?: ListJobsOptions & {
|
|
3857
|
+
kind?: "job";
|
|
3858
|
+
}): JobList;
|
|
3859
|
+
list(options: ListJobsOptions & {
|
|
3860
|
+
kind: "check";
|
|
3861
|
+
}): CheckRowList;
|
|
3862
|
+
list(options: ListJobsOptions & {
|
|
3863
|
+
kind: "all";
|
|
3864
|
+
}): JobListItemList;
|
|
3865
|
+
list(options: ListJobsOptions): JobListItemList;
|
|
3763
3866
|
/**
|
|
3764
3867
|
* List a job's trials (cursor-paged; { status } filters, e.g. to
|
|
3765
3868
|
* the failed trials). Await the result for one page, or `for await` it to
|
|
@@ -4466,6 +4569,8 @@ interface AnalysesClient {
|
|
|
4466
4569
|
* feed's species-blind events door, which answers `trial_not_found`.
|
|
4467
4570
|
*/
|
|
4468
4571
|
list(options?: ListAnalysesOptions): AnalysisList;
|
|
4572
|
+
/** The defaults an analysis runs under when its config names nothing (GET /api/analyses/defaults): model, effort, provider, rubric and the unrendered prompt template. */
|
|
4573
|
+
defaults(): Promise<AnalyzeDefaults>;
|
|
4469
4574
|
/**
|
|
4470
4575
|
* The verdict document — the wire's TrialAnalysis, statuses and typed
|
|
4471
4576
|
* failure included, for EVERY analysis (not only completed ones). The same
|
|
@@ -4697,6 +4802,21 @@ interface Check {
|
|
|
4697
4802
|
/** When the last task settled; null until every task has. */
|
|
4698
4803
|
finished_at: string | null;
|
|
4699
4804
|
}
|
|
4805
|
+
/**
|
|
4806
|
+
* The policy an empty analyze config resolves to (GET /api/analyses/defaults):
|
|
4807
|
+
* each key the value `AnalyzeConfig` echoes for a job created with
|
|
4808
|
+
* `analyze: {}`, except `prompt`, which `AnalyzeConfig` serves as null and
|
|
4809
|
+
* this serves as the template text.
|
|
4810
|
+
*/
|
|
4811
|
+
interface AnalyzeDefaults {
|
|
4812
|
+
model_name: string;
|
|
4813
|
+
rubric: Rubric;
|
|
4814
|
+
/** The built-in analyze prompt template, unrendered — pass it as `prompt` to run the default body explicitly, or edit it from here. */
|
|
4815
|
+
prompt: string;
|
|
4816
|
+
/** The effort the default model runs at when the config names none. */
|
|
4817
|
+
reasoning_effort: string;
|
|
4818
|
+
sandbox_provider: EvalSandboxProvider;
|
|
4819
|
+
}
|
|
4700
4820
|
/**
|
|
4701
4821
|
* The policy an empty check config resolves to (GET /api/checks/defaults):
|
|
4702
4822
|
* each key the value `Check` echoes for a check created with no config,
|
|
@@ -5288,4 +5408,4 @@ interface CapabilityDocument {
|
|
|
5288
5408
|
error_codes: string[];
|
|
5289
5409
|
}
|
|
5290
5410
|
|
|
5291
|
-
export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type
|
|
5411
|
+
export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type FilesystemSearchOptions as a$, type AnalysisFailure as a0, type AnalysisLabel as a1, type AnalysisList as a2, type AnalysisPage as a3, type AnalysisStatus as a4, type AnalysisTranscript as a5, type AnalysisTranscriptOptions as a6, type AnalyzeConfig as a7, type AnalyzeConfigInput as a8, type AnalyzeDefaults as a9, type DatasetSelector as aA, type DatasetSource as aB, type DatasetVersionArchiveSource as aC, type DatasetVersionArchiveUrlSource as aD, type DatasetVersionGitSource as aE, type DatasetVersionHubSource as aF, type DatasetVersionSource as aG, type DatasetVersionState as aH, type DownloadDatasetOptions as aI, type DownloadJobOptions as aJ, EVAL_SANDBOX_PROVIDERS as aK, type Agent as aL, type ModelInfo as aM, type EvalSandboxProvider as aN, type StepResult as aO, type ExceptionInfo as aP, type FilesystemArchiveOptions as aQ, type FilesystemBox as aR, type FilesystemCapture as aS, type FilesystemChange as aT, type FilesystemChanges as aU, type FilesystemChangesOptions as aV, type FilesystemEntry as aW, type FilesystemListOptions as aX, type FilesystemListing as aY, type FilesystemReadOptions as aZ, type FilesystemSearchHit as a_, type ApiKey as aa, type AttemptPhase as ab, type AuthStatus as ac, type Awaitable as ad, CHECK_STATUSES as ae, type CheckDefaults as af, type CheckLabel as ag, type CheckRow as ah, type CheckRowList as ai, type CheckRowPage as aj, type CheckTaskTally as ak, type CompareCell as al, type CompareCoverage as am, type CompareJobAggregate as an, type CompareResponse as ao, type CompareTaskRow as ap, type Dataset as aq, type DatasetImportFailure as ar, type DatasetImportList as as, type DatasetImportPage as at, type DatasetImportStatus as au, type DatasetList as av, type DatasetPage as aw, type DatasetPatch as ax, type DatasetPreflight as ay, type DatasetRef as az, type DatasetImport as b, type OrgJoined as b$, type FilesystemSearchResult as b0, type FilesystemSource as b1, type FilesystemState as b2, type FilesystemStatus as b3, type FilesystemStreamEvent as b4, type FilesystemStreamOptions as b5, type FilesystemWatchResult as b6, GATEWAY_TRACE_SEQ_BASE as b7, type GatewayUsage as b8, type GetDatasetOptions as b9, type JobListKind as bA, type JobListScope as bB, type JobPage as bC, type JobShareEmail as bD, type JobShareLink as bE, type JobStats as bF, type JobStatus as bG, type JobTaskLink as bH, type JobTaskRollup as bI, type JobTaskRollupList as bJ, type JobTaskRollupPage as bK, type JobViewer as bL, type JobWatch as bM, type JudgeResult as bN, type ListAgentsOptions as bO, type ListAnalysesOptions as bP, type ListDatasetsOptions as bQ, type ListImportsOptions as bR, type ListJobImportsOptions as bS, type ListJobTasksOptions as bT, type ListJobsOptions as bU, type ListSkillsOptions as bV, type ListTrialFilesOptions as bW, type ListTrialsOptions as bX, type ManagedProviderCapability as bY, type OrgInvite as bZ, type OrgInviteCreated as b_, type GrepJobOptions as ba, HOSTED_ERROR_CODES as bb, type ImportPhase as bc, type ImportPhaseProgress as bd, type ImportWarning as be, type InfraFailureSignature as bf, JOB_LIST_KINDS as bg, JOB_LIST_SCOPES as bh, type JobAnalysisStats as bi, type JobDeleteResult as bj, type JobFailure as bk, type JobGrepGroup as bl, type JobGrepPage as bm, type JobImport as bn, type JobImportFailure as bo, type JobImportList as bp, type JobImportPage as bq, type JobImportPhaseName as br, type JobImportPhaseProgress as bs, type JobImportProgress as bt, type JobImportSkippedTrial as bu, type JobImportSource as bv, type JobList as bw, type JobListItem as bx, type JobListItemList as by, type JobListItemPage as bz, type JobEvent as c, type TrialUploadProvenance as c$, type OrgMember as c0, type OrgQuota as c1, type OrgRole as c2, type OrgUsage as c3, type Organization as c4, type OrganizationDetail as c5, type Page as c6, type PageOptions as c7, type PassAtKGroup as c8, type PassAtKPoint as c9, type StopResponse as cA, TASK_LINKED_BY as cB, TASK_LINK_REASONS as cC, TRIAL_ARTIFACT_STREAMS as cD, TRIAL_STATUSES as cE, type Task as cF, type TaskCheckTranscript as cG, type TaskLinkReason as cH, type TaskLinkedBy as cI, type TaskNote as cJ, type TaskPackageFiles as cK, type TaskPackageFilesystemStatus as cL, type TaskProviderVerdict as cM, type TimingInfo as cN, type TraceEventPage as cO, type TraceOptions as cP, type TrialArtifactStream as cQ, type TrialCounts as cR, type TrialFile as cS, type TrialFilePage as cT, type TrialFileRange as cU, type TrialGpuCost as cV, type TrialList as cW, type TrialPage as cX, type TrialStatus as cY, type TrialStatusTally as cZ, type TrialTaskLink as c_, type PreflightDatasetInput as ca, type PreflightDeferredCheck as cb, type PreflightManifestVerdict as cc, type PreflightTaskVerdict as cd, type ProviderCapability as ce, type PublishDatasetOptions as cf, type RegradeRequest as cg, type ResumeRequest as ch, type RetryRequest as ci, type RubricCriterion as cj, type RunFilesystem as ck, SANDBOX_LOG_STREAMS as cl, type SandboxLogEvent as cm, type SandboxLogLine as cn, type SandboxLogLines as co, type SandboxLogOptions as cp, type SandboxLogStream as cq, type SandboxProcs as cr, type SkillLock as cs, type SkillUpload as ct, type SkillUploadList as cu, type SkillUploadPage as cv, type SourceJob as cw, type SpendSource as cx, type StartJobOptions as cy, type StatusVocabulary as cz, type DatasetFailedTask as d, type UploadJobOptions as d0, type UploadProvenance as d1, type UploadSkillOptions as d2, type UpstreamStatus as d3, type VerifierEnvironmentMode as d4, type VerifierResult as d5, type WatchAnalysisOptions as d6, type WatchImportOptions as d7, type WatchJobImportOptions as d8, type WatchJobOptions as d9, gatewayUsageOf as da, isHostedErrorCode as db, passAtK as dc, type DatasetImportProgress as e, type JobSecretInline as f, type JobSecretRef as g, type TraceEvent as h, type JobVisibility as i, type JobShareRequest as j, type JobShares as k, type Job as l, type TaskCheck as m, type DatasetsClient as n, type AgentsClient as o, type JobsClient as p, type TrialsClient as q, type AnalysesClient as r, type ChecksClient as s, type CapabilityDocument as t, type HostedClientConfig as u, type AuthClient as v, AGENT_EFFORT_SUPPORT_VALUES as w, ANALYSIS_ARTIFACT_STREAMS as x, ANALYSIS_STATUSES as y, type ActiveDataset as z };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@evolvingmachines/evolve",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.65",
|
|
4
4
|
"keywords": [
|
|
5
5
|
"ai",
|
|
6
6
|
"agents",
|
|
@@ -129,9 +129,9 @@
|
|
|
129
129
|
},
|
|
130
130
|
"dependencies": {
|
|
131
131
|
"@agentclientprotocol/sdk": "^0.5.1",
|
|
132
|
-
"@evolvingmachines/daytona": "0.0.
|
|
133
|
-
"@evolvingmachines/e2b": "0.0.
|
|
134
|
-
"@evolvingmachines/modal": "0.0.
|
|
132
|
+
"@evolvingmachines/daytona": "0.0.65",
|
|
133
|
+
"@evolvingmachines/e2b": "0.0.65",
|
|
134
|
+
"@evolvingmachines/modal": "0.0.65",
|
|
135
135
|
"ajv": "^8.17.1",
|
|
136
136
|
"p-map": "^7.0.2",
|
|
137
137
|
"smol-toml": "^1.7.0",
|
|
@@ -57,7 +57,7 @@ The pages under `references/` are the documentation site's pages, byte for byte,
|
|
|
57
57
|
| [Datasets](references/dashboard/datasets.mdx) | Browse the catalog, and each dataset's tasks and versions, in the dashboard. |
|
|
58
58
|
| [Jobs](references/dashboard/jobs.mdx) | Start a job from the browser, follow it, and read its results. |
|
|
59
59
|
| [Trial viewer](references/dashboard/trial-viewer.mdx) | Read one trial's trace event by event, its tool calls, its spend, and download its artifacts. |
|
|
60
|
-
| [Checks](references/dashboard/checks.mdx) | Read a job's
|
|
60
|
+
| [Checks](references/dashboard/checks.mdx) | Read task quality checks in the dashboard, on a job's CHECK tab or on the check's own page. |
|
|
61
61
|
| [Traces](references/dashboard/traces.mdx) | Every recorded run in one list. |
|
|
62
62
|
|
|
63
63
|
### SDK
|
|
@@ -7,6 +7,7 @@ import GlobalOptions from '/snippets/global-options.mdx';
|
|
|
7
7
|
|
|
8
8
|
```bash
|
|
9
9
|
evolve analyze <job-id> [options]
|
|
10
|
+
evolve analyze --show-defaults
|
|
10
11
|
```
|
|
11
12
|
|
|
12
13
|
Analyze a terminal job's trial traces against a rubric, server-side. Without `--watch` the command returns at once with the job block and the hint `Follow it with: evolve job show <id>`; `--json` prints the accepted job body. Exit 0 on an accepted wave; with `--watch`, exit 1 when any analysis failed.
|
|
@@ -19,6 +20,7 @@ evolve analyze 3e1f9a2c-… \
|
|
|
19
20
|
-l 20 \
|
|
20
21
|
-n 2 \
|
|
21
22
|
--watch
|
|
23
|
+
evolve analyze 3e1f9a2c-… -t d1a10c4e-… -t d1a10f72-…
|
|
22
24
|
```
|
|
23
25
|
|
|
24
26
|
## Options
|
|
@@ -43,6 +45,10 @@ evolve analyze 3e1f9a2c-… \
|
|
|
43
45
|
Sandbox provider the analyzer runs on. Defaults to `daytona`.
|
|
44
46
|
</ParamField>
|
|
45
47
|
|
|
48
|
+
<ParamField path="--show-defaults">
|
|
49
|
+
Print the defaults an analysis runs under when you name none: model, effort, provider, the rubric's criteria and the built-in prompt template, then exit. With `--json`, one object. Takes no job id and no other analyze flag.
|
|
50
|
+
</ParamField>
|
|
51
|
+
|
|
46
52
|
<ParamField path="-n, --n-concurrent" type="n">
|
|
47
53
|
Max concurrent trial analyses, 1 to 150, beneath the organization's `max_concurrent_analyses` (fleet default 60). Defaults to the ceiling.
|
|
48
54
|
</ParamField>
|
|
@@ -55,8 +61,12 @@ evolve analyze 3e1f9a2c-… \
|
|
|
55
61
|
Only analyze failing trials (reward below 1.0, or an exception).
|
|
56
62
|
</ParamField>
|
|
57
63
|
|
|
64
|
+
<ParamField path="-t, --trial" type="trial-id">
|
|
65
|
+
Analyze only this trial; repeatable. Combines with `--passing` or `--failing` (a named trial on the other side is skipped) and comes before `-l`. An id that is not a trial of this job is refused, with the unknown ids named.
|
|
66
|
+
</ParamField>
|
|
67
|
+
|
|
58
68
|
<ParamField path="-l, --n-trials" type="n">
|
|
59
|
-
Max trials to analyze, after `--passing` or `--
|
|
69
|
+
Max trials to analyze, after `--passing`, `--failing` or `--trial`, in the job's trial order.
|
|
60
70
|
</ParamField>
|
|
61
71
|
|
|
62
72
|
<ParamField path="--watch">
|
|
@@ -26,7 +26,7 @@ Start a job; add `--watch` to follow it. Exactly the options of [`evolve run`](/
|
|
|
26
26
|
evolve job list [options]
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
List jobs, newest first: yours, `--scope shared` for what teammates created, `--scope org` for everything in your organizations.
|
|
29
|
+
List jobs, newest first: yours, `--scope shared` for what teammates created, `--scope org` for everything in your organizations. A task quality check is a job too (Harbor runs `harbor check` as one job): `--kind all` lists jobs and checks in one list, each row marked by its kind, and `--kind check` lists checks alone.
|
|
30
30
|
|
|
31
31
|
<ListFlags />
|
|
32
32
|
|
|
@@ -38,8 +38,13 @@ List jobs, newest first: yours, `--scope shared` for what teammates created, `--
|
|
|
38
38
|
Visibility scope: `my`, what you created (the default); `shared`, your organizations' rows that teammates created; or `org`, every row in your organizations, your own included.
|
|
39
39
|
</ParamField>
|
|
40
40
|
|
|
41
|
+
<ParamField path="--kind" type="job | check | all">
|
|
42
|
+
Which rows to list: `job` (the default), `check` for task quality checks, or `all` for both merged newest first. With `check` or `all` a `KIND` column leads each row; a check row reads its id, status, dataset or archive, task check count, cost and accept time, and under `--columns` its `name` and its checker's model and effort (`agents`).
|
|
43
|
+
</ParamField>
|
|
44
|
+
|
|
41
45
|
```bash
|
|
42
46
|
evolve job list --limit 20 -q
|
|
47
|
+
evolve job list --kind all
|
|
43
48
|
```
|
|
44
49
|
|
|
45
50
|
## job show
|
|
@@ -42,9 +42,9 @@ evolve analyze <job-id> \
|
|
|
42
42
|
--watch
|
|
43
43
|
```
|
|
44
44
|
|
|
45
|
-
`-r` is the rubric file, `-p` the prompt file. `--passing` and `--failing` restrict the batch of analyses to trials with reward 1.0, or below it; both together is refused. `-l` caps the number of trials, `-n` how many analyses run at once, beneath the organization's `max_concurrent_analyses` (fleet default 60). `-m` picks the analyzer's model, `--effort` its reasoning effort, `-e` its sandbox provider.
|
|
45
|
+
`-r` is the rubric file, `-p` the prompt file. `--passing` and `--failing` restrict the batch of analyses to trials with reward 1.0, or below it; both together is refused. `-t` names one trial to analyze, repeatable, applied with the filter and before `-l`. `-l` caps the number of trials, `-n` how many analyses run at once, beneath the organization's `max_concurrent_analyses` (fleet default 60). `-m` picks the analyzer's model, `--effort` its reasoning effort, `-e` its sandbox provider. `--show-defaults` prints the defaults a bare `evolve analyze` runs under and exits.
|
|
46
46
|
|
|
47
|
-
A result that misses or invents a criterion is a failed analysis, never a partial pass. A rubric with unknown keys, empty or duplicate criteria, or a length bound exceeded is refused with `invalid_rubric`; an off-roster model
|
|
47
|
+
A result that misses or invents a criterion is a failed analysis, never a partial pass. A rubric with unknown keys, empty or duplicate criteria, or a length bound exceeded is refused with `invalid_rubric`; an off-roster model, an unknown effort or provider, or a `-t` id that is not a trial of the job with `invalid_input`; a job with no analyzable trial, or a `--passing`, `--failing` or `-t` selection that leaves none, with `no_analyzable_trials`; a job still running with `job_not_terminal`.
|
|
48
48
|
|
|
49
49
|
On `evolve run` the same knobs are `--analyze-model`, `--analyze-rubric`, `--analyze-prompt`, `--analyze-effort` and `--analyze-provider`. Each of them implies `--analyze`.
|
|
50
50
|
|
|
@@ -28,7 +28,7 @@ In the browser a shared job opens on the run, its trials and each trial's trace,
|
|
|
28
28
|
|
|
29
29
|
## In the dashboard
|
|
30
30
|
|
|
31
|
-
SHARE sits on a job's page, on a task check's trace opened from
|
|
31
|
+
SHARE sits on a job's page, on a check's page, on a task check's trace opened from either, and on a session's trace under Traces. It opens the dialog: type an address and INVITE, or under General access turn on **Anyone with the link** and copy it. Turning it off stops the link at once. Only the run's creator sees SHARE. Runs shared with you by email are listed under SHARED WITH ME on your personal workspace's Traces.
|
|
32
32
|
|
|
33
33
|
<Card title="job reference" icon="terminal" href="/cli-reference/job">
|
|
34
34
|
`job share`, `job unshare`, `job shares` and their flags.
|
|
@@ -25,11 +25,11 @@ In the SDK the same choice is `org: "acme"` on `jobs().start()` and `datasets().
|
|
|
25
25
|
|
|
26
26
|
## In the dashboard
|
|
27
27
|
|
|
28
|
-
The box under your account in the sidebar is the
|
|
28
|
+
The box under your account in the sidebar is the workspace switcher. It always names one workspace: the one in the page's URL, otherwise the one you last opened in this tab, otherwise Personal. Its menu lists Personal and every team you belong to, each with your role, and **Create team**. Pick a workspace and Datasets, Jobs, Traces, Members and Settings show that workspace's work; on a page that is yours alone (Skills, Integrations, Secrets, API Keys, Usage, Billing) the pick is remembered for the next one.
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
TEAM in the sidebar has two pages. Members lists people with their role: an owner has MAKE OWNER, DEMOTE and REMOVE on every other row, and everyone has LEAVE on their own; below the list, an owner has NEW INVITE LINK, shown once, and REVOKE. A person who opens an invite link signs in, or signs up, and joins. Settings holds the team's name, and DELETE TEAM for an owner. In Personal, Members lists only you and Settings shows your workspace's name.
|
|
31
31
|
|
|
32
|
-
Traces
|
|
32
|
+
Traces is under the workspace too. In a team it is one list of every run in that team, with a CREATED BY column naming who ran each. In Personal it has two tabs: MINE (yours, in any team) and SHARED WITH ME (what teammates made, plus runs shared with you by email). Skills lists your own uploads.
|
|
33
33
|
|
|
34
34
|
<Card title="auth reference" icon="terminal" href="/cli-reference/auth">
|
|
35
35
|
Every `evolve auth org` command.
|
|
@@ -5,6 +5,8 @@ description: "Bring a job you ran elsewhere onto the platform: a Harbor job fold
|
|
|
5
5
|
|
|
6
6
|
A job you ran elsewhere uploads as a finished job. Its trials, traces and rewards become a job you read, analyze and share like any other.
|
|
7
7
|
|
|
8
|
+
A job from `harbor run` — local or through the Harbor hub — uploads as it is: point `upload` at the job folder, nothing to repack. Runs made with the Evolve SDK are packed into that same layout first (below).
|
|
9
|
+
|
|
8
10
|
```bash
|
|
9
11
|
evolve upload ./job-2026-08-27__12-00-00 -d terminal-bench-4@4.0
|
|
10
12
|
```
|