@evolvingmachines/evolve 0.0.65 → 0.0.66

package/dist/{types-B_qmzlMk.d.cts → types-84M4J_Vm.d.cts}

@@ -419,7 +419,7 @@ interface RetryConfig {
 interface RubricCriterion {
     /**
      * Criterion identifier, snake_case (it keys the result's `checks` object).
-     * The platform's default rubrics name seven criteria for analyze and eleven for check.
+     * The platform's default rubrics are served by `analyses().defaults()` and `checks().defaults()`.
      */
     name: string;
     /** What the criterion evaluates, one sentence. */
@@ -448,8 +448,7 @@ interface Rubric {
  * as the body of `POST /api/jobs/{jobId}/analyze` it configures that manual
  * wave. `{}` is legal and means "all defaults":
  * openrouter/deepseek/deepseek-v4.1-flash at its per-model effort (high)
- * over the platform's default analyze rubric (seven criteria, score_is_earned
- * first) and its default prompt body.
+ * over the platform's default analyze rubric and its default prompt body.
  *
  * The analyzer always runs the claude-code harness (Harbor's default analyze
  * agent) in its own sealed sandbox — on the provider `sandbox_provider`
@@ -1472,24 +1471,6 @@ interface AnalysisCheck {
     explanation: string;
     evidence: AnalysisEvidence[];
 }
-/**
- * The derived label of an analysis — computed by the platform from the
- * outcomes when the result is stored, never asked from the model. Null until
- * completed, and null on a completed analysis whose rubric is not the
- * default one (a custom rubric carries its per-criterion outcomes and no
- * label). Precedence: `flagged` on a fail of score_is_earned,
- * score_is_correct, task_was_fair or report_is_truthful; else `env_fault` on
- * a fail of environment_worked; else `unclear` on an unknown of any of those
- * five, or a not_applicable of score_is_earned or score_is_correct; else
- * `clean`.
- */
-type AnalysisLabel = "flagged" | "env_fault" | "unclear" | "clean";
-/**
- * The derived label of a task check — computed the same way. `has_a_problem`
- * on a fail of any criterion; else `unclear` on an unknown of any of the six
- * file-based criteria; else `no_problem_found`. Null under a custom rubric.
- */
-type CheckLabel = "has_a_problem" | "unclear" | "no_problem_found";
 /**
  * Why an analysis FAILED — a stored typed failure, never a silent absence and
  * never a fake pass. NOT under the key `error` for the same reason JobFailure
@@ -1531,7 +1512,7 @@ interface TrialAnalysis {
      * Provenance: the analyzed trial, its job, and its task. Redundant on
      * `Trial.analysis` (the trial is the enclosing object) and the whole point
      * of a `analyses().list()` row, where nothing else says which run the
-     * verdict judged. Harbor's `trial_name` names the same thing by directory.
+     * result judged. Harbor's `trial_name` names the same thing by directory.
      */
     trial_id: string;
     job_id: string;
@@ -1565,8 +1546,6 @@ interface TrialAnalysis {
      * (the frozen-criteria law). Null until completed.
      */
     checks: Record<string, AnalysisCheck> | null;
-    /** The derived label (AnalysisLabel states the rule); null until completed, and null under a custom rubric. */
-    label: AnalysisLabel | null;
     estimated_cost_usd: number | null;
     /**
      * The analyzer's one-home usage reading — the SAME object, same keys, the
@@ -4536,7 +4515,7 @@ interface AnalysisTranscript {
     stored_at?: string[];
 }
 /**
- * Client for analysis runs — the analyzer's own transcript, verdict document,
+ * Client for analysis runs — the analyzer's own transcript, result document,
  * and stored artifacts, all globally addressable by analysis id.
  *
  * DELIBERATELY OFF-CONTRACT: these three reads ride the dashboard's traces
@@ -4551,7 +4530,7 @@ interface AnalysisTranscript {
  * GatewayUsageEvent prose, not as an operation. RECORDED TENSION: whether
  * that feed and this one join the contract as operations (spec + both SDK
  * shadows) is an open ruling, not something settled here. The contract-side
- * verdict stays where it always was — `Trial.analysis` on the trial body;
+ * result stays where it always was — `Trial.analysis` on the trial body;
  * this client adds the reads the contract does not carry today.
  */
 interface AnalysesClient {
@@ -4572,7 +4551,7 @@ interface AnalysesClient {
     /** The defaults an analysis runs under when its config names nothing (GET /api/analyses/defaults): model, effort, provider, rubric and the unrendered prompt template. */
     defaults(): Promise<AnalyzeDefaults>;
     /**
-     * The verdict document — the wire's TrialAnalysis, statuses and typed
+     * The result document — the wire's TrialAnalysis, statuses and typed
      * failure included, for EVERY analysis (not only completed ones). The same
      * object the analyzed trial serves as `Trial.analysis` when this analysis
      * is its latest; this door answers for earlier analyses too.
@@ -4636,7 +4615,7 @@ interface AnalysesClient {
  * analyzer's `openrouter/deepseek/deepseek-v4.1-flash` — one roster, one
  * default for both rubric
  * agents, a recorded deviation), `rubric` (the default is the platform's
- * check rubric, eleven criteria), and
+ * check rubric), and
  * `prompt` (the TEXT of Harbor's `-p/--prompt` file, replacing the platform's
  * default check body and rendered with `{task_path}`, `{file_tree}`,
  * `{criteria_guidance}`; the output contract is appended after it exactly
@@ -4658,7 +4637,7 @@ interface CheckConfigInput {
     name?: string;
     /** Model the checker agent runs (Harbor's `-m/--model`); must be on the claude roster (`GET /api/meta`). */
     model_name?: string;
-    /** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (eleven criteria). */
+    /** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (`checks().defaults()`). */
     rubric?: Rubric;
     /** The prompt template — the TEXT of Harbor's `-p/--prompt` file. */
     prompt?: string;
@@ -4719,9 +4698,8 @@ interface CheckSource {
 /**
  * One task's quality check — Harbor's QualityCheckResult shape (their
  * cli/quality_checker/models.py:31-35: `task_name`, `checks` keyed by
- * criterion, `cost_usd`), its checks extended by the result schema and the
- * derived `label` and `executed` beside them, plus the hosted provenance: its own id, the check
- * it belongs to, its lifecycle (the analysis ladder's four lowercase words),
+ * criterion, `cost_usd`), its checks extended by the result schema, plus the
+ * hosted provenance: its own id, the check it belongs to, its lifecycle (the analysis ladder's four lowercase words),
  * the bounded attempt count, and a typed `failure` in place of Harbor's
  * `error` string (the TrialAnalysis rule).
  *
@@ -4740,17 +4718,6 @@ interface TaskCheck {
     status: AnalysisStatus;
     /** One entry per rubric criterion, keys exactly the frozen criteria. Null until completed. */
     checks: Record<string, AnalysisCheck> | null;
-    /** The derived label (CheckLabel states the rule); null until completed, and null under a custom rubric. */
-    label: CheckLabel | null;
-    /**
-     * Whether the box ran the task's environment: true when none of the five
-     * run-based criteria (reference_solution_is_valid,
-     * verifier_rejects_non_solutions, environment_builds_and_runs,
-     * verification_is_stable, limits_allow_the_task) is unknown, so a
-     * reading-only `no_problem_found` is never mistaken for a run. Null
-     * exactly when `label` is null.
-     */
-    executed: boolean | null;
     cost_usd: number | null;
     /** 1, or 2 when the one automatic re-run fired (a run that produced no valid check-result.json, the missing file included, is re-run once — the analyze verb's hosted rule; a run cut by its budget is not that class: it settles `failed` with phase `timeout` at once and is never re-run). */
     attempts: number;
@@ -5408,4 +5375,4 @@ interface CapabilityDocument {
     error_codes: string[];
 }
 
-export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type FilesystemSearchOptions as a$, type AnalysisFailure as a0, type AnalysisLabel as a1, type AnalysisList as a2, type AnalysisPage as a3, type AnalysisStatus as a4, type AnalysisTranscript as a5, type AnalysisTranscriptOptions as a6, type AnalyzeConfig as a7, type AnalyzeConfigInput as a8, type AnalyzeDefaults as a9, type DatasetSelector as aA, type DatasetSource as aB, type DatasetVersionArchiveSource as aC, type DatasetVersionArchiveUrlSource as aD, type DatasetVersionGitSource as aE, type DatasetVersionHubSource as aF, type DatasetVersionSource as aG, type DatasetVersionState as aH, type DownloadDatasetOptions as aI, type DownloadJobOptions as aJ, EVAL_SANDBOX_PROVIDERS as aK, type Agent as aL, type ModelInfo as aM, type EvalSandboxProvider as aN, type StepResult as aO, type ExceptionInfo as aP, type FilesystemArchiveOptions as aQ, type FilesystemBox as aR, type FilesystemCapture as aS, type FilesystemChange as aT, type FilesystemChanges as aU, type FilesystemChangesOptions as aV, type FilesystemEntry as aW, type FilesystemListOptions as aX, type FilesystemListing as aY, type FilesystemReadOptions as aZ, type FilesystemSearchHit as a_, type ApiKey as aa, type AttemptPhase as ab, type AuthStatus as ac, type Awaitable as ad, CHECK_STATUSES as ae, type CheckDefaults as af, type CheckLabel as ag, type CheckRow as ah, type CheckRowList as ai, type CheckRowPage as aj, type CheckTaskTally as ak, type CompareCell as al, type CompareCoverage as am, type CompareJobAggregate as an, type CompareResponse as ao, type CompareTaskRow as ap, type Dataset as aq, type DatasetImportFailure as ar, type DatasetImportList as as, type DatasetImportPage as at, type DatasetImportStatus as au, type DatasetList as av, type DatasetPage as aw, type DatasetPatch as ax, type DatasetPreflight as ay, type DatasetRef as az, type DatasetImport as b, type OrgJoined as b$, type FilesystemSearchResult as b0, type FilesystemSource as b1, type FilesystemState as b2, type FilesystemStatus as b3, type FilesystemStreamEvent as b4, type FilesystemStreamOptions as b5, type FilesystemWatchResult as b6, GATEWAY_TRACE_SEQ_BASE as b7, type GatewayUsage as b8, type GetDatasetOptions as b9, type JobListKind as bA, type JobListScope as bB, type JobPage as bC, type JobShareEmail as bD, type JobShareLink as bE, type JobStats as bF, type JobStatus as bG, type JobTaskLink as bH, type JobTaskRollup as bI, type JobTaskRollupList as bJ, type JobTaskRollupPage as bK, type JobViewer as bL, type JobWatch as bM, type JudgeResult as bN, type ListAgentsOptions as bO, type ListAnalysesOptions as bP, type ListDatasetsOptions as bQ, type ListImportsOptions as bR, type ListJobImportsOptions as bS, type ListJobTasksOptions as bT, type ListJobsOptions as bU, type ListSkillsOptions as bV, type ListTrialFilesOptions as bW, type ListTrialsOptions as bX, type ManagedProviderCapability as bY, type OrgInvite as bZ, type OrgInviteCreated as b_, type GrepJobOptions as ba, HOSTED_ERROR_CODES as bb, type ImportPhase as bc, type ImportPhaseProgress as bd, type ImportWarning as be, type InfraFailureSignature as bf, JOB_LIST_KINDS as bg, JOB_LIST_SCOPES as bh, type JobAnalysisStats as bi, type JobDeleteResult as bj, type JobFailure as bk, type JobGrepGroup as bl, type JobGrepPage as bm, type JobImport as bn, type JobImportFailure as bo, type JobImportList as bp, type JobImportPage as bq, type JobImportPhaseName as br, type JobImportPhaseProgress as bs, type JobImportProgress as bt, type JobImportSkippedTrial as bu, type JobImportSource as bv, type JobList as bw, type JobListItem as bx, type JobListItemList as by, type JobListItemPage as bz, type JobEvent as c, type TrialUploadProvenance as c$, type OrgMember as c0, type OrgQuota as c1, type OrgRole as c2, type OrgUsage as c3, type Organization as c4, type OrganizationDetail as c5, type Page as c6, type PageOptions as c7, type PassAtKGroup as c8, type PassAtKPoint as c9, type StopResponse as cA, TASK_LINKED_BY as cB, TASK_LINK_REASONS as cC, TRIAL_ARTIFACT_STREAMS as cD, TRIAL_STATUSES as cE, type Task as cF, type TaskCheckTranscript as cG, type TaskLinkReason as cH, type TaskLinkedBy as cI, type TaskNote as cJ, type TaskPackageFiles as cK, type TaskPackageFilesystemStatus as cL, type TaskProviderVerdict as cM, type TimingInfo as cN, type TraceEventPage as cO, type TraceOptions as cP, type TrialArtifactStream as cQ, type TrialCounts as cR, type TrialFile as cS, type TrialFilePage as cT, type TrialFileRange as cU, type TrialGpuCost as cV, type TrialList as cW, type TrialPage as cX, type TrialStatus as cY, type TrialStatusTally as cZ, type TrialTaskLink as c_, type PreflightDatasetInput as ca, type PreflightDeferredCheck as cb, type PreflightManifestVerdict as cc, type PreflightTaskVerdict as cd, type ProviderCapability as ce, type PublishDatasetOptions as cf, type RegradeRequest as cg, type ResumeRequest as ch, type RetryRequest as ci, type RubricCriterion as cj, type RunFilesystem as ck, SANDBOX_LOG_STREAMS as cl, type SandboxLogEvent as cm, type SandboxLogLine as cn, type SandboxLogLines as co, type SandboxLogOptions as cp, type SandboxLogStream as cq, type SandboxProcs as cr, type SkillLock as cs, type SkillUpload as ct, type SkillUploadList as cu, type SkillUploadPage as cv, type SourceJob as cw, type SpendSource as cx, type StartJobOptions as cy, type StatusVocabulary as cz, type DatasetFailedTask as d, type UploadJobOptions as d0, type UploadProvenance as d1, type UploadSkillOptions as d2, type UpstreamStatus as d3, type VerifierEnvironmentMode as d4, type VerifierResult as d5, type WatchAnalysisOptions as d6, type WatchImportOptions as d7, type WatchJobImportOptions as d8, type WatchJobOptions as d9, gatewayUsageOf as da, isHostedErrorCode as db, passAtK as dc, type DatasetImportProgress as e, type JobSecretInline as f, type JobSecretRef as g, type TraceEvent as h, type JobVisibility as i, type JobShareRequest as j, type JobShares as k, type Job as l, type TaskCheck as m, type DatasetsClient as n, type AgentsClient as o, type JobsClient as p, type TrialsClient as q, type AnalysesClient as r, type ChecksClient as s, type CapabilityDocument as t, type HostedClientConfig as u, type AuthClient as v, AGENT_EFFORT_SUPPORT_VALUES as w, ANALYSIS_ARTIFACT_STREAMS as x, ANALYSIS_STATUSES as y, type ActiveDataset as z };
+export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type FilesystemSource as a$, type AnalysisFailure as a0, type AnalysisList as a1, type AnalysisPage as a2, type AnalysisStatus as a3, type AnalysisTranscript as a4, type AnalysisTranscriptOptions as a5, type AnalyzeConfig as a6, type AnalyzeConfigInput as a7, type AnalyzeDefaults as a8, type ApiKey as a9, type DatasetVersionArchiveSource as aA, type DatasetVersionArchiveUrlSource as aB, type DatasetVersionGitSource as aC, type DatasetVersionHubSource as aD, type DatasetVersionSource as aE, type DatasetVersionState as aF, type DownloadDatasetOptions as aG, type DownloadJobOptions as aH, EVAL_SANDBOX_PROVIDERS as aI, type Agent as aJ, type ModelInfo as aK, type EvalSandboxProvider as aL, type StepResult as aM, type ExceptionInfo as aN, type FilesystemArchiveOptions as aO, type FilesystemBox as aP, type FilesystemCapture as aQ, type FilesystemChange as aR, type FilesystemChanges as aS, type FilesystemChangesOptions as aT, type FilesystemEntry as aU, type FilesystemListOptions as aV, type FilesystemListing as aW, type FilesystemReadOptions as aX, type FilesystemSearchHit as aY, type FilesystemSearchOptions as aZ, type FilesystemSearchResult as a_, type AttemptPhase as aa, type AuthStatus as ab, type Awaitable as ac, CHECK_STATUSES as ad, type CheckDefaults as ae, type CheckRow as af, type CheckRowList as ag, type CheckRowPage as ah, type CheckTaskTally as ai, type CompareCell as aj, type CompareCoverage as ak, type CompareJobAggregate as al, type CompareResponse as am, type CompareTaskRow as an, type Dataset as ao, type DatasetImportFailure as ap, type DatasetImportList as aq, type DatasetImportPage as ar, type DatasetImportStatus as as, type DatasetList as at, type DatasetPage as au, type DatasetPatch as av, type DatasetPreflight as aw, type DatasetRef as ax, type DatasetSelector as ay, type DatasetSource as az, type DatasetImport as b, type OrgQuota as b$, type FilesystemState as b0, type FilesystemStatus as b1, type FilesystemStreamEvent as b2, type FilesystemStreamOptions as b3, type FilesystemWatchResult as b4, GATEWAY_TRACE_SEQ_BASE as b5, type GatewayUsage as b6, type GetDatasetOptions as b7, type GrepJobOptions as b8, HOSTED_ERROR_CODES as b9, type JobPage as bA, type JobShareEmail as bB, type JobShareLink as bC, type JobStats as bD, type JobStatus as bE, type JobTaskLink as bF, type JobTaskRollup as bG, type JobTaskRollupList as bH, type JobTaskRollupPage as bI, type JobViewer as bJ, type JobWatch as bK, type JudgeResult as bL, type ListAgentsOptions as bM, type ListAnalysesOptions as bN, type ListDatasetsOptions as bO, type ListImportsOptions as bP, type ListJobImportsOptions as bQ, type ListJobTasksOptions as bR, type ListJobsOptions as bS, type ListSkillsOptions as bT, type ListTrialFilesOptions as bU, type ListTrialsOptions as bV, type ManagedProviderCapability as bW, type OrgInvite as bX, type OrgInviteCreated as bY, type OrgJoined as bZ, type OrgMember as b_, type ImportPhase as ba, type ImportPhaseProgress as bb, type ImportWarning as bc, type InfraFailureSignature as bd, JOB_LIST_KINDS as be, JOB_LIST_SCOPES as bf, type JobAnalysisStats as bg, type JobDeleteResult as bh, type JobFailure as bi, type JobGrepGroup as bj, type JobGrepPage as bk, type JobImport as bl, type JobImportFailure as bm, type JobImportList as bn, type JobImportPage as bo, type JobImportPhaseName as bp, type JobImportPhaseProgress as bq, type JobImportProgress as br, type JobImportSkippedTrial as bs, type JobImportSource as bt, type JobList as bu, type JobListItem as bv, type JobListItemList as bw, type JobListItemPage as bx, type JobListKind as by, type JobListScope as bz, type JobEvent as c, type UploadProvenance as c$, type OrgRole as c0, type OrgUsage as c1, type Organization as c2, type OrganizationDetail as c3, type Page as c4, type PageOptions as c5, type PassAtKGroup as c6, type PassAtKPoint as c7, type PreflightDatasetInput as c8, type PreflightDeferredCheck as c9, TASK_LINK_REASONS as cA, TRIAL_ARTIFACT_STREAMS as cB, TRIAL_STATUSES as cC, type Task as cD, type TaskCheckTranscript as cE, type TaskLinkReason as cF, type TaskLinkedBy as cG, type TaskNote as cH, type TaskPackageFiles as cI, type TaskPackageFilesystemStatus as cJ, type TaskProviderVerdict as cK, type TimingInfo as cL, type TraceEventPage as cM, type TraceOptions as cN, type TrialArtifactStream as cO, type TrialCounts as cP, type TrialFile as cQ, type TrialFilePage as cR, type TrialFileRange as cS, type TrialGpuCost as cT, type TrialList as cU, type TrialPage as cV, type TrialStatus as cW, type TrialStatusTally as cX, type TrialTaskLink as cY, type TrialUploadProvenance as cZ, type UploadJobOptions as c_, type PreflightManifestVerdict as ca, type PreflightTaskVerdict as cb, type ProviderCapability as cc, type PublishDatasetOptions as cd, type RegradeRequest as ce, type ResumeRequest as cf, type RetryRequest as cg, type RubricCriterion as ch, type RunFilesystem as ci, SANDBOX_LOG_STREAMS as cj, type SandboxLogEvent as ck, type SandboxLogLine as cl, type SandboxLogLines as cm, type SandboxLogOptions as cn, type SandboxLogStream as co, type SandboxProcs as cp, type SkillLock as cq, type SkillUpload as cr, type SkillUploadList as cs, type SkillUploadPage as ct, type SourceJob as cu, type SpendSource as cv, type StartJobOptions as cw, type StatusVocabulary as cx, type StopResponse as cy, TASK_LINKED_BY as cz, type DatasetFailedTask as d, type UploadSkillOptions as d0, type UpstreamStatus as d1, type VerifierEnvironmentMode as d2, type VerifierResult as d3, type WatchAnalysisOptions as d4, type WatchImportOptions as d5, type WatchJobImportOptions as d6, type WatchJobOptions as d7, gatewayUsageOf as d8, isHostedErrorCode as d9, passAtK as da, type DatasetImportProgress as e, type JobSecretInline as f, type JobSecretRef as g, type TraceEvent as h, type JobVisibility as i, type JobShareRequest as j, type JobShares as k, type Job as l, type TaskCheck as m, type DatasetsClient as n, type AgentsClient as o, type JobsClient as p, type TrialsClient as q, type AnalysesClient as r, type ChecksClient as s, type CapabilityDocument as t, type HostedClientConfig as u, type AuthClient as v, AGENT_EFFORT_SUPPORT_VALUES as w, ANALYSIS_ARTIFACT_STREAMS as x, ANALYSIS_STATUSES as y, type ActiveDataset as z };

package/dist/{types-B_qmzlMk.d.ts → types-84M4J_Vm.d.ts}

@@ -419,7 +419,7 @@ interface RetryConfig {
 interface RubricCriterion {
     /**
      * Criterion identifier, snake_case (it keys the result's `checks` object).
-     * The platform's default rubrics name seven criteria for analyze and eleven for check.
+     * The platform's default rubrics are served by `analyses().defaults()` and `checks().defaults()`.
      */
     name: string;
     /** What the criterion evaluates, one sentence. */
@@ -448,8 +448,7 @@ interface Rubric {
  * as the body of `POST /api/jobs/{jobId}/analyze` it configures that manual
  * wave. `{}` is legal and means "all defaults":
  * openrouter/deepseek/deepseek-v4.1-flash at its per-model effort (high)
- * over the platform's default analyze rubric (seven criteria, score_is_earned
- * first) and its default prompt body.
+ * over the platform's default analyze rubric and its default prompt body.
  *
  * The analyzer always runs the claude-code harness (Harbor's default analyze
  * agent) in its own sealed sandbox — on the provider `sandbox_provider`
@@ -1472,24 +1471,6 @@ interface AnalysisCheck {
     explanation: string;
     evidence: AnalysisEvidence[];
 }
-/**
- * The derived label of an analysis — computed by the platform from the
- * outcomes when the result is stored, never asked from the model. Null until
- * completed, and null on a completed analysis whose rubric is not the
- * default one (a custom rubric carries its per-criterion outcomes and no
- * label). Precedence: `flagged` on a fail of score_is_earned,
- * score_is_correct, task_was_fair or report_is_truthful; else `env_fault` on
- * a fail of environment_worked; else `unclear` on an unknown of any of those
- * five, or a not_applicable of score_is_earned or score_is_correct; else
- * `clean`.
- */
-type AnalysisLabel = "flagged" | "env_fault" | "unclear" | "clean";
-/**
- * The derived label of a task check — computed the same way. `has_a_problem`
- * on a fail of any criterion; else `unclear` on an unknown of any of the six
- * file-based criteria; else `no_problem_found`. Null under a custom rubric.
- */
-type CheckLabel = "has_a_problem" | "unclear" | "no_problem_found";
 /**
  * Why an analysis FAILED — a stored typed failure, never a silent absence and
  * never a fake pass. NOT under the key `error` for the same reason JobFailure
@@ -1531,7 +1512,7 @@ interface TrialAnalysis {
      * Provenance: the analyzed trial, its job, and its task. Redundant on
      * `Trial.analysis` (the trial is the enclosing object) and the whole point
      * of a `analyses().list()` row, where nothing else says which run the
-     * verdict judged. Harbor's `trial_name` names the same thing by directory.
+     * result judged. Harbor's `trial_name` names the same thing by directory.
      */
     trial_id: string;
     job_id: string;
@@ -1565,8 +1546,6 @@ interface TrialAnalysis {
      * (the frozen-criteria law). Null until completed.
      */
     checks: Record<string, AnalysisCheck> | null;
-    /** The derived label (AnalysisLabel states the rule); null until completed, and null under a custom rubric. */
-    label: AnalysisLabel | null;
     estimated_cost_usd: number | null;
     /**
      * The analyzer's one-home usage reading — the SAME object, same keys, the
@@ -4536,7 +4515,7 @@ interface AnalysisTranscript {
     stored_at?: string[];
 }
 /**
- * Client for analysis runs — the analyzer's own transcript, verdict document,
+ * Client for analysis runs — the analyzer's own transcript, result document,
  * and stored artifacts, all globally addressable by analysis id.
  *
  * DELIBERATELY OFF-CONTRACT: these three reads ride the dashboard's traces
@@ -4551,7 +4530,7 @@ interface AnalysisTranscript {
  * GatewayUsageEvent prose, not as an operation. RECORDED TENSION: whether
  * that feed and this one join the contract as operations (spec + both SDK
  * shadows) is an open ruling, not something settled here. The contract-side
- * verdict stays where it always was — `Trial.analysis` on the trial body;
+ * result stays where it always was — `Trial.analysis` on the trial body;
  * this client adds the reads the contract does not carry today.
  */
 interface AnalysesClient {
@@ -4572,7 +4551,7 @@ interface AnalysesClient {
     /** The defaults an analysis runs under when its config names nothing (GET /api/analyses/defaults): model, effort, provider, rubric and the unrendered prompt template. */
     defaults(): Promise<AnalyzeDefaults>;
     /**
-     * The verdict document — the wire's TrialAnalysis, statuses and typed
+     * The result document — the wire's TrialAnalysis, statuses and typed
      * failure included, for EVERY analysis (not only completed ones). The same
      * object the analyzed trial serves as `Trial.analysis` when this analysis
      * is its latest; this door answers for earlier analyses too.
@@ -4636,7 +4615,7 @@ interface AnalysesClient {
  * analyzer's `openrouter/deepseek/deepseek-v4.1-flash` — one roster, one
  * default for both rubric
  * agents, a recorded deviation), `rubric` (the default is the platform's
- * check rubric, eleven criteria), and
+ * check rubric), and
  * `prompt` (the TEXT of Harbor's `-p/--prompt` file, replacing the platform's
  * default check body and rendered with `{task_path}`, `{file_tree}`,
  * `{criteria_guidance}`; the output contract is appended after it exactly
@@ -4658,7 +4637,7 @@ interface CheckConfigInput {
     name?: string;
     /** Model the checker agent runs (Harbor's `-m/--model`); must be on the claude roster (`GET /api/meta`). */
     model_name?: string;
-    /** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (eleven criteria). */
+    /** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (`checks().defaults()`). */
     rubric?: Rubric;
     /** The prompt template — the TEXT of Harbor's `-p/--prompt` file. */
     prompt?: string;
@@ -4719,9 +4698,8 @@ interface CheckSource {
 /**
  * One task's quality check — Harbor's QualityCheckResult shape (their
  * cli/quality_checker/models.py:31-35: `task_name`, `checks` keyed by
- * criterion, `cost_usd`), its checks extended by the result schema and the
- * derived `label` and `executed` beside them, plus the hosted provenance: its own id, the check
- * it belongs to, its lifecycle (the analysis ladder's four lowercase words),
+ * criterion, `cost_usd`), its checks extended by the result schema, plus the
+ * hosted provenance: its own id, the check it belongs to, its lifecycle (the analysis ladder's four lowercase words),
  * the bounded attempt count, and a typed `failure` in place of Harbor's
  * `error` string (the TrialAnalysis rule).
  *
@@ -4740,17 +4718,6 @@ interface TaskCheck {
     status: AnalysisStatus;
     /** One entry per rubric criterion, keys exactly the frozen criteria. Null until completed. */
     checks: Record<string, AnalysisCheck> | null;
-    /** The derived label (CheckLabel states the rule); null until completed, and null under a custom rubric. */
-    label: CheckLabel | null;
-    /**
-     * Whether the box ran the task's environment: true when none of the five
-     * run-based criteria (reference_solution_is_valid,
-     * verifier_rejects_non_solutions, environment_builds_and_runs,
-     * verification_is_stable, limits_allow_the_task) is unknown, so a
-     * reading-only `no_problem_found` is never mistaken for a run. Null
-     * exactly when `label` is null.
-     */
-    executed: boolean | null;
     cost_usd: number | null;
     /** 1, or 2 when the one automatic re-run fired (a run that produced no valid check-result.json, the missing file included, is re-run once — the analyze verb's hosted rule; a run cut by its budget is not that class: it settles `failed` with phase `timeout` at once and is never re-run). */
     attempts: number;
@@ -5408,4 +5375,4 @@ interface CapabilityDocument {
     error_codes: string[];
 }
 
-export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type FilesystemSearchOptions as a$, type AnalysisFailure as a0, type AnalysisLabel as a1, type AnalysisList as a2, type AnalysisPage as a3, type AnalysisStatus as a4, type AnalysisTranscript as a5, type AnalysisTranscriptOptions as a6, type AnalyzeConfig as a7, type AnalyzeConfigInput as a8, type AnalyzeDefaults as a9, type DatasetSelector as aA, type DatasetSource as aB, type DatasetVersionArchiveSource as aC, type DatasetVersionArchiveUrlSource as aD, type DatasetVersionGitSource as aE, type DatasetVersionHubSource as aF, type DatasetVersionSource as aG, type DatasetVersionState as aH, type DownloadDatasetOptions as aI, type DownloadJobOptions as aJ, EVAL_SANDBOX_PROVIDERS as aK, type Agent as aL, type ModelInfo as aM, type EvalSandboxProvider as aN, type StepResult as aO, type ExceptionInfo as aP, type FilesystemArchiveOptions as aQ, type FilesystemBox as aR, type FilesystemCapture as aS, type FilesystemChange as aT, type FilesystemChanges as aU, type FilesystemChangesOptions as aV, type FilesystemEntry as aW, type FilesystemListOptions as aX, type FilesystemListing as aY, type FilesystemReadOptions as aZ, type FilesystemSearchHit as a_, type ApiKey as aa, type AttemptPhase as ab, type AuthStatus as ac, type Awaitable as ad, CHECK_STATUSES as ae, type CheckDefaults as af, type CheckLabel as ag, type CheckRow as ah, type CheckRowList as ai, type CheckRowPage as aj, type CheckTaskTally as ak, type CompareCell as al, type CompareCoverage as am, type CompareJobAggregate as an, type CompareResponse as ao, type CompareTaskRow as ap, type Dataset as aq, type DatasetImportFailure as ar, type DatasetImportList as as, type DatasetImportPage as at, type DatasetImportStatus as au, type DatasetList as av, type DatasetPage as aw, type DatasetPatch as ax, type DatasetPreflight as ay, type DatasetRef as az, type DatasetImport as b, type OrgJoined as b$, type FilesystemSearchResult as b0, type FilesystemSource as b1, type FilesystemState as b2, type FilesystemStatus as b3, type FilesystemStreamEvent as b4, type FilesystemStreamOptions as b5, type FilesystemWatchResult as b6, GATEWAY_TRACE_SEQ_BASE as b7, type GatewayUsage as b8, type GetDatasetOptions as b9, type JobListKind as bA, type JobListScope as bB, type JobPage as bC, type JobShareEmail as bD, type JobShareLink as bE, type JobStats as bF, type JobStatus as bG, type JobTaskLink as bH, type JobTaskRollup as bI, type JobTaskRollupList as bJ, type JobTaskRollupPage as bK, type JobViewer as bL, type JobWatch as bM, type JudgeResult as bN, type ListAgentsOptions as bO, type ListAnalysesOptions as bP, type ListDatasetsOptions as bQ, type ListImportsOptions as bR, type ListJobImportsOptions as bS, type ListJobTasksOptions as bT, type ListJobsOptions as bU, type ListSkillsOptions as bV, type ListTrialFilesOptions as bW, type ListTrialsOptions as bX, type ManagedProviderCapability as bY, type OrgInvite as bZ, type OrgInviteCreated as b_, type GrepJobOptions as ba, HOSTED_ERROR_CODES as bb, type ImportPhase as bc, type ImportPhaseProgress as bd, type ImportWarning as be, type InfraFailureSignature as bf, JOB_LIST_KINDS as bg, JOB_LIST_SCOPES as bh, type JobAnalysisStats as bi, type JobDeleteResult as bj, type JobFailure as bk, type JobGrepGroup as bl, type JobGrepPage as bm, type JobImport as bn, type JobImportFailure as bo, type JobImportList as bp, type JobImportPage as bq, type JobImportPhaseName as br, type JobImportPhaseProgress as bs, type JobImportProgress as bt, type JobImportSkippedTrial as bu, type JobImportSource as bv, type JobList as bw, type JobListItem as bx, type JobListItemList as by, type JobListItemPage as bz, type JobEvent as c, type TrialUploadProvenance as c$, type OrgMember as c0, type OrgQuota as c1, type OrgRole as c2, type OrgUsage as c3, type Organization as c4, type OrganizationDetail as c5, type Page as c6, type PageOptions as c7, type PassAtKGroup as c8, type PassAtKPoint as c9, type StopResponse as cA, TASK_LINKED_BY as cB, TASK_LINK_REASONS as cC, TRIAL_ARTIFACT_STREAMS as cD, TRIAL_STATUSES as cE, type Task as cF, type TaskCheckTranscript as cG, type TaskLinkReason as cH, type TaskLinkedBy as cI, type TaskNote as cJ, type TaskPackageFiles as cK, type TaskPackageFilesystemStatus as cL, type TaskProviderVerdict as cM, type TimingInfo as cN, type TraceEventPage as cO, type TraceOptions as cP, type TrialArtifactStream as cQ, type TrialCounts as cR, type TrialFile as cS, type TrialFilePage as cT, type TrialFileRange as cU, type TrialGpuCost as cV, type TrialList as cW, type TrialPage as cX, type TrialStatus as cY, type TrialStatusTally as cZ, type TrialTaskLink as c_, type PreflightDatasetInput as ca, type PreflightDeferredCheck as cb, type PreflightManifestVerdict as cc, type PreflightTaskVerdict as cd, type ProviderCapability as ce, type PublishDatasetOptions as cf, type RegradeRequest as cg, type ResumeRequest as ch, type RetryRequest as ci, type RubricCriterion as cj, type RunFilesystem as ck, SANDBOX_LOG_STREAMS as cl, type SandboxLogEvent as cm, type SandboxLogLine as cn, type SandboxLogLines as co, type SandboxLogOptions as cp, type SandboxLogStream as cq, type SandboxProcs as cr, type SkillLock as cs, type SkillUpload as ct, type SkillUploadList as cu, type SkillUploadPage as cv, type SourceJob as cw, type SpendSource as cx, type StartJobOptions as cy, type StatusVocabulary as cz, type DatasetFailedTask as d, type UploadJobOptions as d0, type UploadProvenance as d1, type UploadSkillOptions as d2, type UpstreamStatus as d3, type VerifierEnvironmentMode as d4, type VerifierResult as d5, type WatchAnalysisOptions as d6, type WatchImportOptions as d7, type WatchJobImportOptions as d8, type WatchJobOptions as d9, gatewayUsageOf as da, isHostedErrorCode as db, passAtK as dc, type DatasetImportProgress as e, type JobSecretInline as f, type JobSecretRef as g, type TraceEvent as h, type JobVisibility as i, type JobShareRequest as j, type JobShares as k, type Job as l, type TaskCheck as m, type DatasetsClient as n, type AgentsClient as o, type JobsClient as p, type TrialsClient as q, type AnalysesClient as r, type ChecksClient as s, type CapabilityDocument as t, type HostedClientConfig as u, type AuthClient as v, AGENT_EFFORT_SUPPORT_VALUES as w, ANALYSIS_ARTIFACT_STREAMS as x, ANALYSIS_STATUSES as y, type ActiveDataset as z };
+export { type AnalysisEvidence as $, type AgentInput as A, type AgentArm as B, type Check as C, type DatasetVersion as D, type AgentArmInput as E, type AgentCapability as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentDatasetStats as I, type JobCreate as J, type AgentEffortSupport as K, type AgentInfo as L, type AgentList as M, type AgentModelOption as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentPage as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentResult as V, type AgentSource as W, type AgentSourceInput as X, type AgentUpsertInput as Y, type AnalysisArtifactStream as Z, type AnalysisCheck as _, type TrialAnalysis as a, type FilesystemSource as a$, type AnalysisFailure as a0, type AnalysisList as a1, type AnalysisPage as a2, type AnalysisStatus as a3, type AnalysisTranscript as a4, type AnalysisTranscriptOptions as a5, type AnalyzeConfig as a6, type AnalyzeConfigInput as a7, type AnalyzeDefaults as a8, type ApiKey as a9, type DatasetVersionArchiveSource as aA, type DatasetVersionArchiveUrlSource as aB, type DatasetVersionGitSource as aC, type DatasetVersionHubSource as aD, type DatasetVersionSource as aE, type DatasetVersionState as aF, type DownloadDatasetOptions as aG, type DownloadJobOptions as aH, EVAL_SANDBOX_PROVIDERS as aI, type Agent as aJ, type ModelInfo as aK, type EvalSandboxProvider as aL, type StepResult as aM, type ExceptionInfo as aN, type FilesystemArchiveOptions as aO, type FilesystemBox as aP, type FilesystemCapture as aQ, type FilesystemChange as aR, type FilesystemChanges as aS, type FilesystemChangesOptions as aT, type FilesystemEntry as aU, type FilesystemListOptions as aV, type FilesystemListing as aW, type FilesystemReadOptions as aX, type FilesystemSearchHit as aY, type FilesystemSearchOptions as aZ, type FilesystemSearchResult as a_, type AttemptPhase as aa, type AuthStatus as ab, type Awaitable as ac, CHECK_STATUSES as ad, type CheckDefaults as ae, type CheckRow as af, type CheckRowList as ag, type CheckRowPage as ah, type CheckTaskTally as ai, type CompareCell as aj, type CompareCoverage as ak, type CompareJobAggregate as al, type CompareResponse as am, type CompareTaskRow as an, type Dataset as ao, type DatasetImportFailure as ap, type DatasetImportList as aq, type DatasetImportPage as ar, type DatasetImportStatus as as, type DatasetList as at, type DatasetPage as au, type DatasetPatch as av, type DatasetPreflight as aw, type DatasetRef as ax, type DatasetSelector as ay, type DatasetSource as az, type DatasetImport as b, type OrgQuota as b$, type FilesystemState as b0, type FilesystemStatus as b1, type FilesystemStreamEvent as b2, type FilesystemStreamOptions as b3, type FilesystemWatchResult as b4, GATEWAY_TRACE_SEQ_BASE as b5, type GatewayUsage as b6, type GetDatasetOptions as b7, type GrepJobOptions as b8, HOSTED_ERROR_CODES as b9, type JobPage as bA, type JobShareEmail as bB, type JobShareLink as bC, type JobStats as bD, type JobStatus as bE, type JobTaskLink as bF, type JobTaskRollup as bG, type JobTaskRollupList as bH, type JobTaskRollupPage as bI, type JobViewer as bJ, type JobWatch as bK, type JudgeResult as bL, type ListAgentsOptions as bM, type ListAnalysesOptions as bN, type ListDatasetsOptions as bO, type ListImportsOptions as bP, type ListJobImportsOptions as bQ, type ListJobTasksOptions as bR, type ListJobsOptions as bS, type ListSkillsOptions as bT, type ListTrialFilesOptions as bU, type ListTrialsOptions as bV, type ManagedProviderCapability as bW, type OrgInvite as bX, type OrgInviteCreated as bY, type OrgJoined as bZ, type OrgMember as b_, type ImportPhase as ba, type ImportPhaseProgress as bb, type ImportWarning as bc, type InfraFailureSignature as bd, JOB_LIST_KINDS as be, JOB_LIST_SCOPES as bf, type JobAnalysisStats as bg, type JobDeleteResult as bh, type JobFailure as bi, type JobGrepGroup as bj, type JobGrepPage as bk, type JobImport as bl, type JobImportFailure as bm, type JobImportList as bn, type JobImportPage as bo, type JobImportPhaseName as bp, type JobImportPhaseProgress as bq, type JobImportProgress as br, type JobImportSkippedTrial as bs, type JobImportSource as bt, type JobList as bu, type JobListItem as bv, type JobListItemList as bw, type JobListItemPage as bx, type JobListKind as by, type JobListScope as bz, type JobEvent as c, type UploadProvenance as c$, type OrgRole as c0, type OrgUsage as c1, type Organization as c2, type OrganizationDetail as c3, type Page as c4, type PageOptions as c5, type PassAtKGroup as c6, type PassAtKPoint as c7, type PreflightDatasetInput as c8, type PreflightDeferredCheck as c9, TASK_LINK_REASONS as cA, TRIAL_ARTIFACT_STREAMS as cB, TRIAL_STATUSES as cC, type Task as cD, type TaskCheckTranscript as cE, type TaskLinkReason as cF, type TaskLinkedBy as cG, type TaskNote as cH, type TaskPackageFiles as cI, type TaskPackageFilesystemStatus as cJ, type TaskProviderVerdict as cK, type TimingInfo as cL, type TraceEventPage as cM, type TraceOptions as cN, type TrialArtifactStream as cO, type TrialCounts as cP, type TrialFile as cQ, type TrialFilePage as cR, type TrialFileRange as cS, type TrialGpuCost as cT, type TrialList as cU, type TrialPage as cV, type TrialStatus as cW, type TrialStatusTally as cX, type TrialTaskLink as cY, type TrialUploadProvenance as cZ, type UploadJobOptions as c_, type PreflightManifestVerdict as ca, type PreflightTaskVerdict as cb, type ProviderCapability as cc, type PublishDatasetOptions as cd, type RegradeRequest as ce, type ResumeRequest as cf, type RetryRequest as cg, type RubricCriterion as ch, type RunFilesystem as ci, SANDBOX_LOG_STREAMS as cj, type SandboxLogEvent as ck, type SandboxLogLine as cl, type SandboxLogLines as cm, type SandboxLogOptions as cn, type SandboxLogStream as co, type SandboxProcs as cp, type SkillLock as cq, type SkillUpload as cr, type SkillUploadList as cs, type SkillUploadPage as ct, type SourceJob as cu, type SpendSource as cv, type StartJobOptions as cw, type StatusVocabulary as cx, type StopResponse as cy, TASK_LINKED_BY as cz, type DatasetFailedTask as d, type UploadSkillOptions as d0, type UpstreamStatus as d1, type VerifierEnvironmentMode as d2, type VerifierResult as d3, type WatchAnalysisOptions as d4, type WatchImportOptions as d5, type WatchJobImportOptions as d6, type WatchJobOptions as d7, gatewayUsageOf as d8, isHostedErrorCode as d9, passAtK as da, type DatasetImportProgress as e, type JobSecretInline as f, type JobSecretRef as g, type TraceEvent as h, type JobVisibility as i, type JobShareRequest as j, type JobShares as k, type Job as l, type TaskCheck as m, type DatasetsClient as n, type AgentsClient as o, type JobsClient as p, type TrialsClient as q, type AnalysesClient as r, type ChecksClient as s, type CapabilityDocument as t, type HostedClientConfig as u, type AuthClient as v, AGENT_EFFORT_SUPPORT_VALUES as w, ANALYSIS_ARTIFACT_STREAMS as x, ANALYSIS_STATUSES as y, type ActiveDataset as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evolvingmachines/evolve",
-  "version": "0.0.65",
+  "version": "0.0.66",
   "keywords": [
     "ai",
     "agents",
@@ -129,9 +129,9 @@
   },
   "dependencies": {
     "@agentclientprotocol/sdk": "^0.5.1",
-    "@evolvingmachines/daytona": "0.0.65",
-    "@evolvingmachines/e2b": "0.0.65",
-    "@evolvingmachines/modal": "0.0.65",
+    "@evolvingmachines/daytona": "0.0.66",
+    "@evolvingmachines/e2b": "0.0.66",
+    "@evolvingmachines/modal": "0.0.66",
     "ajv": "^8.17.1",
     "p-map": "^7.0.2",
     "smol-toml": "^1.7.0",

package/skills/create-adapter/SKILL.md

@@ -10,18 +10,27 @@ metadata:
 An adapter is a small program that reads an existing benchmark and writes one task
 directory per task, in the Harbor task format. Its output is a folder of tasks, ready for
 `evolve dataset publish`. This skill guides the conversion; `evolve skills get create-task`
-has the task format in full, and `evolve skills get publish` every publish option.
+guides task authoring, and `evolve skills get publish` guides publishing. Read
+`evolve skills get evals cli-reference/dataset` for every dataset publish option.
 
-## Authoritative reference
+## References
 
-The conversion rules below are Harbor's, from its adapter guide, and Evolve runs the
-task format unchanged. For the full guide, read
-https://github.com/laude-institute/harbor/blob/main/docs/content/docs/datasets/adapters.mdx
-(its steps on parity experiments, the registry and pull requests are Harbor's own
-process and do not apply here). The task format itself is at
+The conversion structure follows Harbor's adapter format. Evolve's supported task
+options and publishing workflow are documented in its bundled references:
+
+| Topic | Read |
+| --- | --- |
+| Task layout and configuration | `evolve skills get evals core-concepts/tasks` and `evolve skills get evals core-concepts/task-config` |
+| Environments and Compose | `evolve skills get evals core-concepts/task-environment` and `evolve skills get evals core-concepts/compose` |
+| Verifiers and rewards | `evolve skills get evals core-concepts/task-verifiers` and `evolve skills get evals core-concepts/rewardkit` |
+| Managed support limits | `evolve skills get evals core-concepts/compatibility` |
+
+The upstream task format is at
 https://docs.harborframework.com/core-concepts/tasks/overview.
 
-Do not invent structure, field names, or workflow beyond what the guide specifies.
+Use Evolve's references for commands and supported behavior. Upstream adapter
+examples below can help with conversion code; Harbor's registry and pull-request
+workflow do not apply to publishing on Evolve.
 
 ## Prerequisites
 
@@ -59,7 +68,7 @@ before proceeding.
 | Adapter name | Lowercase, hyphen-separated. Must match the benchmark's common identifier (e.g., `swe-bench`, `aider-polyglot`). Becomes the dataset name on Evolve and, with dashes turned to underscores, the Python package name. |
 | Human-readable name | Appears in the README. |
 | Upstream repo URL | Needed for step 1 (benchmark analysis) and for the README. |
-| Reference solutions available? | If the benchmark ships reference solutions, use them. If not, they must be written, with LLM help, before the tasks can be checked. |
+| Reference solutions available? | Use them when available. Otherwise record the gap and write solutions when possible; Evolve can check tasks without them. |
 | Subset | Adapting a subset of tasks is acceptable (e.g., only a verified split). Document every exclusion in the README. |
 
 ### 3. Write the converter
@@ -90,8 +99,10 @@ The `evolve` CLI has no adapter scaffold; create this layout by hand:
 `--overwrite`, and `--task-ids`. Run it as
 `uv run python -m <adapter_name>.main --output-dir <path>`.
 
-Each generated task directory must contain at minimum `task.toml`, `instruction.md`,
-`environment/Dockerfile`, `solution/solve.sh`, and `tests/test.sh`:
+For a single-step benchmark using Dockerfile-built images, generate this layout.
+`solution/solve.sh` is optional in the format, but useful for testing the conversion.
+Tasks using a prebuilt image can omit `environment/Dockerfile`; follow the
+environment reference above.
 
 ```
 <output-dir>/
@@ -149,11 +160,13 @@ OPENAI_API_KEY = "${OPENAI_API_KEY}"
 **`tests/test.sh`:** must write a numeric reward (integer or float, 0 to 1) to
 `/logs/verifier/reward.txt`, or named numbers to `/logs/verifier/reward.json`.
 `/logs/verifier/` exists at run time. Use the same metrics as the original benchmark.
+For multiple JSON metrics, include a `reward` key in `[0, 1]` for the primary score.
+Install pytest in the image that runs verification; do not fetch it during the test.
 
 ```bash
 #!/bin/bash
-pytest /tests/test_*.py
-if [ $? -eq 0 ]; then
+set -uo pipefail
+if python3 -m pytest /tests/test_*.py; then
   echo 1 > /logs/verifier/reward.txt
 else
   echo 0 > /logs/verifier/reward.txt
@@ -179,6 +192,9 @@ RUN apt-get update && apt-get install -y \
     git \
     && rm -rf /var/lib/apt/lists/*
 
+# The shared verifier uses the task image
+RUN pip install --no-cache-dir 'pytest==8.4.1'
+
 # Install benchmark-specific dependencies
 # RUN pip install --no-cache-dir <packages>
 
@@ -211,15 +227,32 @@ run each task.
 
 ### 5. Verify the conversion
 
-Check the generated tasks. The check reads each task and, when it can, runs its
-environment, its `solution/solve.sh` and its verifier, then rules on every criterion of a
-rubric; every task should come back `no_problem_found` with `executed` true.
+Validate the metadata, then request a hosted review of each task:
 
 ```bash
+evolve dataset check "<output-dir>"
 evolve check "<output-dir>" --watch
-evolve check show <check-id>
 ```
 
+Set `CHECK_ID` to the parent check ID returned by `evolve check`, then read it:
+
+```bash
+evolve check show "$CHECK_ID"
+```
+
+The first command sends `task.toml` files and any `dataset.toml` to the API for
+validation, without uploading the full task packages or executing them. The
+second runs a paid checker agent against a rubric. It can run the environment,
+reference solution, and verifier when its sandbox supports them; it does not
+guarantee that execution.
+
+Read every task's status, findings, and evidence. The result is the
+per-criterion JSON; Evolve derives no verdict from it. The default rubric's
+execution criteria say whether the checker ran the task, and `unknown` on them
+is not proof that the solution or verifier ran.
+Use `evolve check trace <task-check-id>` to inspect the checker's work. Test any
+unresolved execution in an environment that can run the task.
+
 A failing check usually means one of three things, in this order:
 
 1. **Adaptation error:** the instruction, environment or test does not match upstream.
@@ -229,14 +262,18 @@ A failing check usually means one of three things, in this order:
 3. **Environment error:** a Dockerfile that does not build or a test that cannot run makes
    the task impossible for every agent, so catch it here.
 
-Where Harbor is installed, `harbor run -p "<output-dir>" -a oracle` runs every reference
-solution locally (optional); the reward should be `1.0` on every task.
+The Evolve CLI has no standalone oracle command. Do not treat `evolve check` as
+one. When a reference solution is available, verify that it earns full credit and
+that invalid submissions fail; keep the execution evidence with the conversion.
 
-**Benchmarks without reference solutions:** write them, with LLM help, before publishing.
-A cheap agent and model can take a first pass over all the tasks; complete the rest with
-a stronger model plus human review.
+**Benchmarks without reference solutions:** document that limitation. The default
+checker marks the solution criterion `not_applicable` when none is shipped.
+Write and validate solutions where possible; a model's proposed solution still
+needs review.
 
-**Step complete when:** every task passes `evolve check`.
+**Step complete when:** every generated task has been reviewed, confirmed defects
+are fixed, and any unresolved execution or missing-solution limits are documented.
+Read `evolve skills get evals core-concepts/check` for the result rules.
 
 ### 6. Document and publish
 
@@ -245,7 +282,8 @@ and every exclusion; benchmark bugs found and how they were handled; prompt
 modifications, environment adjustments and other deviations from the original, with the
 reason; known limitations; the exact commands to regenerate the tasks and to run them.
 
-Then publish the output folder as a dataset (`evolve skills get publish` has every option):
+Then publish the output folder as a dataset (`evolve skills get publish` explains the
+workflow; `evolve skills get evals cli-reference/dataset` lists the options):
 
 ```bash
 evolve dataset check "<output-dir>"
@@ -256,11 +294,18 @@ evolve run -d "<adapter-name>@1.0" -a codex -m gpt-5.5 --watch
 To see how faithful the conversion is, run a job with the same agent and model the
 benchmark's own leaderboard reports, and compare the scores.
 
+For TypeScript or Python automation, read
+`evolve skills get evals sdk-reference/methods/datasets`,
+`evolve skills get evals sdk-reference/methods/checks`, and
+`evolve skills get evals sdk-reference/methods/jobs`.
+
 ## Reference adapters by shape
 
 Harbor's repository holds one adapter per benchmark, public at
 https://github.com/laude-institute/harbor/tree/main/adapters. When implementation
-questions come up, read the one that matches the benchmark's shape:
+questions come up, read the one that matches the benchmark's shape. These are
+optional conversion examples; verify their task settings against Evolve's
+supported scope before using them:
 
 | Shape | Example adapter |
 |-------|----------------|
@@ -283,5 +328,5 @@ questions come up, read the one that matches the benchmark's shape:
 |---------|--------------|--------|
 | `evolve: command not found` | The CLI is not installed | `npm install -g @evolvingmachines/evolve`. |
 | `evolve dataset check` refuses a task by name | Its `task.toml` breaks a rule (a field, a value, a name) | Fix the converter, regenerate, check again. The refusal names the field. |
-| A task check comes back `has_a_problem` | One criterion failed | `evolve check show <check-id>` prints the criterion, its explanation and its evidence. |
+| A task check has a `fail` outcome | One criterion failed | `evolve check show <check-id>` prints the criterion, its explanation and its evidence. |
 | Every task fails the check the same way | An error in the task template | Fix `task-template/` in the converter, not the generated tasks. |