@evolvingmachines/evolve 0.0.65 → 0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d

package/harness-capabilities.json

@@ -18,6 +18,40 @@
   ],
   "defaultReasoningEffort": "medium",
   "harnesses": {
+    "antigravity": {
+      "defaultModel": "gemini-3.8-flash",
+      "models": [
+        {
+          "alias": "gemini-3.8-flash",
+          "modelId": "vertex_ai/gemini-3.8-flash",
+          "description": "Latest Flash on Vertex AI: coding + agentic planning"
+        },
+        {
+          "alias": "gemini-3.5-flash-lite",
+          "modelId": "vertex_ai/gemini-3.5-flash-lite",
+          "description": "Latest Flash-Lite on Vertex AI: most cost-effective"
+        },
+        {
+          "alias": "gemini-3.1-pro-preview",
+          "modelId": "vertex_ai/gemini-3.1-pro-preview",
+          "description": "Latest Pro on Vertex AI: complex agentic + coding"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": false,
+      "presets": []
+    },
     "claude": {
       "defaultModel": "opus",
       "models": [
@@ -236,6 +270,40 @@
       "supportsConfig": false,
       "presets": []
     },
+    "dsh": {
+      "defaultModel": "openrouter/deepseek/deepseek-v4.1-flash",
+      "models": [
+        {
+          "alias": "openrouter/deepseek/deepseek-v4.1-flash",
+          "modelId": "openrouter/deepseek/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via OpenRouter"
+        },
+        {
+          "alias": "fireworks/deepseek-v4.1-flash",
+          "modelId": "fireworks/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via Fireworks"
+        },
+        {
+          "alias": "openrouter/deepseek/deepseek-v4-pro-0813",
+          "modelId": "openrouter/deepseek/deepseek-v4-pro-0813",
+          "description": "DeepSeek V4 Pro via OpenRouter"
+        },
+        {
+          "alias": "fireworks/deepseek-v4-pro-0813",
+          "modelId": "fireworks/deepseek-v4-pro-0813",
+          "description": "DeepSeek V4 Pro via Fireworks"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "low",
+        "medium",
+        "high"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": false,
+      "presets": []
+    },
     "gemini": {
       "defaultModel": "gemini-3.5-flash",
       "models": [
@@ -396,6 +464,184 @@
       "supportsConfig": false,
       "presets": []
     },
+    "pi": {
+      "defaultModel": "openrouter/anthropic/claude-opus-5",
+      "models": [
+        {
+          "alias": "openrouter/anthropic/claude-fable-5.1",
+          "modelId": "openrouter/anthropic/claude-fable-5.1",
+          "description": "Anthropic Fable 5.1 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-opus-5",
+          "modelId": "openrouter/anthropic/claude-opus-5",
+          "description": "Anthropic Opus 5 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-sonnet-5",
+          "modelId": "openrouter/anthropic/claude-sonnet-5",
+          "description": "Anthropic Sonnet 5 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-haiku-4.5",
+          "modelId": "openrouter/anthropic/claude-haiku-4.5",
+          "description": "Anthropic Haiku via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-6-astra",
+          "modelId": "openrouter/openai/gpt-6-astra",
+          "description": "OpenAI GPT-6 Astra via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-sol",
+          "modelId": "openrouter/openai/gpt-5.6-sol",
+          "description": "OpenAI GPT-5.6 Sol via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-terra",
+          "modelId": "openrouter/openai/gpt-5.6-terra",
+          "description": "OpenAI GPT-5.6 Terra via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-luna",
+          "modelId": "openrouter/openai/gpt-5.6-luna",
+          "description": "OpenAI GPT-5.6 Luna via OpenRouter"
+        },
+        {
+          "alias": "openrouter/google/gemini-3.6-flash",
+          "modelId": "openrouter/google/gemini-3.6-flash",
+          "description": "Gemini 3.6 Flash via OpenRouter"
+        },
+        {
+          "alias": "openrouter/qwen/qwen3.7-max",
+          "modelId": "openrouter/qwen/qwen3.7-max",
+          "description": "Qwen 3.7 Max via OpenRouter"
+        },
+        {
+          "alias": "openrouter/moonshotai/kimi-k3",
+          "modelId": "openrouter/moonshotai/kimi-k3",
+          "description": "Kimi K3 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/z-ai/glm-5.3",
+          "modelId": "openrouter/z-ai/glm-5.3",
+          "description": "Zhipu GLM-5.3 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/z-ai/glm-5.3-flash",
+          "modelId": "openrouter/z-ai/glm-5.3-flash",
+          "description": "Zhipu GLM-5.3 Flash (OpenRouter id; the Evolve gateway serves it from Fireworks)"
+        },
+        {
+          "alias": "openrouter/deepseek/deepseek-v4.1-flash",
+          "modelId": "openrouter/deepseek/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via OpenRouter"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": false,
+      "presets": []
+    },
+    "prime-agent": {
+      "defaultModel": "openrouter/anthropic/claude-opus-5",
+      "models": [
+        {
+          "alias": "openrouter/anthropic/claude-fable-5.1",
+          "modelId": "openrouter/anthropic/claude-fable-5.1",
+          "description": "Anthropic Fable 5.1 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-opus-5",
+          "modelId": "openrouter/anthropic/claude-opus-5",
+          "description": "Anthropic Opus 5 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-sonnet-5",
+          "modelId": "openrouter/anthropic/claude-sonnet-5",
+          "description": "Anthropic Sonnet 5 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-haiku-4.5",
+          "modelId": "openrouter/anthropic/claude-haiku-4.5",
+          "description": "Anthropic Haiku via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-6-astra",
+          "modelId": "openrouter/openai/gpt-6-astra",
+          "description": "OpenAI GPT-6 Astra via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-sol",
+          "modelId": "openrouter/openai/gpt-5.6-sol",
+          "description": "OpenAI GPT-5.6 Sol via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-terra",
+          "modelId": "openrouter/openai/gpt-5.6-terra",
+          "description": "OpenAI GPT-5.6 Terra via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-luna",
+          "modelId": "openrouter/openai/gpt-5.6-luna",
+          "description": "OpenAI GPT-5.6 Luna via OpenRouter"
+        },
+        {
+          "alias": "openrouter/google/gemini-3.6-flash",
+          "modelId": "openrouter/google/gemini-3.6-flash",
+          "description": "Gemini 3.6 Flash via OpenRouter"
+        },
+        {
+          "alias": "openrouter/qwen/qwen3.7-max",
+          "modelId": "openrouter/qwen/qwen3.7-max",
+          "description": "Qwen 3.7 Max via OpenRouter"
+        },
+        {
+          "alias": "openrouter/moonshotai/kimi-k3",
+          "modelId": "openrouter/moonshotai/kimi-k3",
+          "description": "Kimi K3 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/z-ai/glm-5.3",
+          "modelId": "openrouter/z-ai/glm-5.3",
+          "description": "Zhipu GLM-5.3 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/z-ai/glm-5.3-flash",
+          "modelId": "openrouter/z-ai/glm-5.3-flash",
+          "description": "Zhipu GLM-5.3 Flash (OpenRouter id; the Evolve gateway serves it from Fireworks)"
+        },
+        {
+          "alias": "openrouter/deepseek/deepseek-v4.1-flash",
+          "modelId": "openrouter/deepseek/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via OpenRouter"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": false,
+      "presets": []
+    },
     "qwen": {
       "defaultModel": "qwen3.7-max",
       "models": [
@@ -425,6 +671,45 @@
       "defaultEffort": "thinking",
       "supportsConfig": false,
       "presets": []
+    },
+    "zcode": {
+      "defaultModel": "openrouter/z-ai/glm-5.3",
+      "models": [
+        {
+          "alias": "openrouter/z-ai/glm-5.3",
+          "modelId": "openrouter/z-ai/glm-5.3",
+          "description": "Zhipu GLM-5.3 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/z-ai/glm-5.3-flash",
+          "modelId": "openrouter/z-ai/glm-5.3-flash",
+          "description": "Zhipu GLM-5.3 Flash via OpenRouter"
+        },
+        {
+          "alias": "fireworks/glm-5.3",
+          "modelId": "fireworks/glm-5.3",
+          "description": "Zhipu GLM-5.3 via Fireworks"
+        },
+        {
+          "alias": "fireworks/glm-5.3-flash",
+          "modelId": "fireworks/glm-5.3-flash",
+          "description": "Zhipu GLM-5.3 Flash via Fireworks"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": false,
+      "presets": []
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@evolvingmachines/evolve",
-  "version": "0.0.65",
+  "version": "0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d",
   "keywords": [
     "ai",
     "agents",
@@ -49,14 +49,15 @@
     "build": "tsx scripts/generate-harness-capabilities.ts --check && tsup --minify && node scripts/copy-spec.mjs && node scripts/copy-skills.mjs",
     "generate:capabilities": "tsx scripts/generate-harness-capabilities.ts",
     "dev": "tsup src/index.ts --watch",
-    "type-check": "tsc --noEmit && tsc -p tsconfig.types-test.json",
+    "type-check": "tsc --noEmit && tsc -p tsconfig.types-test.json && node scripts/check-test-syntax.mjs",
     "test": "npm run test:unit && npm run test:integration",
     "pretest:unit": "npm run build",
-    "test:unit": "tsx tests/unit/semaphore.test.ts && tsx tests/unit/swarm-concurrency.test.ts && tsx tests/unit/swarm-verify.test.ts && tsx tests/unit/swarm-retry-verify.test.ts && tsx tests/unit/prompt-construction.test.ts && tsx tests/unit/observability-metadata.test.ts && tsx tests/unit/observability-identity.test.ts && tsx tests/unit/skills-integration.test.ts && tsx tests/unit/skills-resolver.test.ts && tsx tests/unit/skills-tag-peel.test.ts && tsx tests/unit/auth-config.test.ts && tsx tests/unit/config-validation.test.ts && tsx tests/unit/agent-presets.test.ts && tsx tests/unit/run-resume-option.test.ts && tsx tests/unit/native-config-delivery.test.ts && tsx tests/unit/browser-config.test.ts && tsx tests/unit/integrations-config.test.ts && tsx tests/unit/managed-secrets.test.ts && tsx tests/unit/managed-modal.test.ts && tsx tests/unit/plugins-config.test.ts && tsx tests/unit/session-runtime.test.ts && tsx tests/unit/cost-api.test.ts && tsx tests/unit/storage-config.test.ts && tsx tests/unit/checkpoint-tar.test.ts && tsx tests/unit/checkpoint-tar-listing.test.ts && tsx tests/unit/checkpoint-errors.test.ts && tsx tests/unit/checkpoint-flows.test.ts && tsx tests/unit/checkpoint-dx.test.ts && tsx tests/unit/checkpoint-edge-cases.test.ts && tsx tests/unit/codex-parser-errors.test.ts && tsx tests/unit/parser-harness-errors.test.ts && tsx tests/unit/codex-toml.test.ts && tsx tests/unit/kimi-parser.test.ts && tsx tests/unit/kimi-mcp.test.ts && tsx tests/unit/opencode-parser.test.ts && tsx tests/unit/droid-parser.test.ts && tsx tests/unit/droid-mcp.test.ts && tsx tests/unit/parser-tool-name.test.ts && tsx tests/unit/parser-atif-fields.test.ts && tsx tests/unit/provider-parity.test.ts && tsx tests/unit/storage-client.test.ts && tsx tests/unit/sessions-client.test.ts && tsx tests/unit/hosted-client.test.ts && tsx tests/unit/hosted-trial-tree.test.ts && tsx tests/unit/hosted-ergonomics.test.ts && tsx tests/unit/cli.test.ts && tsx tests/unit/cli-bin.test.ts && tsx tests/unit/cli-skills.test.ts && tsx tests/unit/cli-help.test.ts && tsx tests/unit/hosted-types.test.ts && tsx tests/unit/hosted-error-codes.test.ts && tsx tests/unit/hosted-spec-gate.test.ts && tsx tests/unit/harness-capabilities.test.ts && tsx tests/unit/sandbox-artifacts.test.ts && tsx tests/unit/upload-file-from-path.test.ts && tsx tests/unit/file-utils.test.ts && tsx tests/unit/hosted-upload-stream.test.ts && tsx tests/unit/hosted-upload-memory.test.ts && tsx tests/unit/hosted-resumable.test.ts && tsx tests/unit/cli-ndjson-pipe.test.ts && tsx tests/unit/sandbox-errors.test.ts && tsx --expose-gc tests/unit/hosted-tar.test.ts",
+    "test:unit": "node scripts/check-test-syntax.mjs && tsx tests/unit/semaphore.test.ts && tsx tests/unit/swarm-concurrency.test.ts && tsx tests/unit/swarm-verify.test.ts && tsx tests/unit/swarm-retry-verify.test.ts && tsx tests/unit/prompt-construction.test.ts && tsx tests/unit/observability-metadata.test.ts && tsx tests/unit/observability-identity.test.ts && tsx tests/unit/skills-integration.test.ts && tsx tests/unit/skills-resolver.test.ts && tsx tests/unit/skills-tag-peel.test.ts && tsx tests/unit/auth-config.test.ts && tsx tests/unit/config-validation.test.ts && tsx tests/unit/agent-presets.test.ts && tsx tests/unit/run-resume-option.test.ts && tsx tests/unit/native-config-delivery.test.ts && tsx tests/unit/browser-config.test.ts && tsx tests/unit/integrations-config.test.ts && tsx tests/unit/managed-secrets.test.ts && tsx tests/unit/managed-modal.test.ts && tsx tests/unit/plugins-config.test.ts && tsx tests/unit/session-runtime.test.ts && tsx tests/unit/cost-api.test.ts && tsx tests/unit/storage-config.test.ts && tsx tests/unit/checkpoint-tar.test.ts && tsx tests/unit/checkpoint-tar-listing.test.ts && tsx tests/unit/checkpoint-errors.test.ts && tsx tests/unit/checkpoint-flows.test.ts && tsx tests/unit/checkpoint-dx.test.ts && tsx tests/unit/checkpoint-edge-cases.test.ts && tsx tests/unit/codex-parser-errors.test.ts && tsx tests/unit/parser-harness-errors.test.ts && tsx tests/unit/codex-toml.test.ts && tsx tests/unit/kimi-parser.test.ts && tsx tests/unit/kimi-mcp.test.ts && tsx tests/unit/opencode-parser.test.ts && tsx tests/unit/droid-parser.test.ts && tsx tests/unit/droid-mcp.test.ts && tsx tests/unit/pi-parser.test.ts && tsx tests/unit/prime-agent-parser.test.ts && tsx tests/unit/pi-family-mcp.test.ts && tsx tests/unit/dsh-parser.test.ts && tsx tests/unit/dsh-mcp.test.ts && tsx tests/unit/zcode-parser.test.ts && tsx tests/unit/zcode-mcp.test.ts && tsx tests/unit/antigravity-parser.test.ts && tsx tests/unit/antigravity-mcp.test.ts && tsx tests/unit/parser-tool-name.test.ts && tsx tests/unit/parser-atif-fields.test.ts && tsx tests/unit/provider-parity.test.ts && tsx tests/unit/storage-client.test.ts && tsx tests/unit/sessions-client.test.ts && tsx tests/unit/hosted-client.test.ts && tsx tests/unit/hosted-trial-tree.test.ts && tsx tests/unit/hosted-ergonomics.test.ts && tsx tests/unit/cli.test.ts && tsx tests/unit/cli-bin.test.ts && tsx tests/unit/cli-skills.test.ts && tsx tests/unit/cli-help.test.ts && tsx tests/unit/hosted-types.test.ts && tsx tests/unit/hosted-error-codes.test.ts && tsx tests/unit/hosted-spec-gate.test.ts && tsx tests/unit/harness-capabilities.test.ts && tsx tests/unit/harness-uniformity.test.ts && tsx tests/unit/sandbox-artifacts.test.ts && tsx tests/unit/upload-file-from-path.test.ts && tsx tests/unit/file-utils.test.ts && tsx tests/unit/hosted-upload-stream.test.ts && tsx tests/unit/hosted-upload-memory.test.ts && tsx tests/unit/hosted-resumable.test.ts && tsx tests/unit/cli-ndjson-pipe.test.ts && tsx tests/unit/sandbox-errors.test.ts && tsx --expose-gc tests/unit/hosted-tar.test.ts",
     "test:unit:upload-from-path": "tsx tests/unit/upload-file-from-path.test.ts",
     "test:unit:upload-stream": "tsx tests/unit/hosted-upload-stream.test.ts",
     "test:unit:upload-memory": "tsx tests/unit/hosted-upload-memory.test.ts",
     "test:unit:parity": "tsx tests/unit/provider-parity.test.ts",
+    "test:unit:uniformity": "tsx tests/unit/harness-uniformity.test.ts",
     "test:unit:semaphore": "tsx tests/unit/semaphore.test.ts",
     "test:unit:swarm": "tsx tests/unit/swarm-concurrency.test.ts",
     "test:unit:verify": "tsx tests/unit/swarm-verify.test.ts",
@@ -89,6 +90,8 @@
     "test:unit:tool-name": "tsx tests/unit/parser-tool-name.test.ts",
     "test:unit:kimi-parser": "tsx tests/unit/kimi-parser.test.ts",
     "test:unit:kimi-mcp": "tsx tests/unit/kimi-mcp.test.ts",
+    "test:unit:dsh-parser": "tsx tests/unit/dsh-parser.test.ts",
+    "test:unit:dsh-mcp": "tsx tests/unit/dsh-mcp.test.ts",
     "test:unit:codex-toml": "tsx tests/unit/codex-toml.test.ts",
     "pretest:integration": "npm run build",
     "test:integration": "npm run test:01",
@@ -129,9 +132,9 @@
   },
   "dependencies": {
     "@agentclientprotocol/sdk": "^0.5.1",
-    "@evolvingmachines/daytona": "0.0.65",
-    "@evolvingmachines/e2b": "0.0.65",
-    "@evolvingmachines/modal": "0.0.65",
+    "@evolvingmachines/daytona": "0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d",
+    "@evolvingmachines/e2b": "0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d",
+    "@evolvingmachines/modal": "0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d",
     "ajv": "^8.17.1",
     "p-map": "^7.0.2",
     "smol-toml": "^1.7.0",

package/skills/create-adapter/SKILL.md

@@ -10,18 +10,27 @@ metadata:
 An adapter is a small program that reads an existing benchmark and writes one task
 directory per task, in the Harbor task format. Its output is a folder of tasks, ready for
 `evolve dataset publish`. This skill guides the conversion; `evolve skills get create-task`
-has the task format in full, and `evolve skills get publish` every publish option.
+guides task authoring, and `evolve skills get publish` guides publishing. Read
+`evolve skills get evals cli-reference/dataset` for every dataset publish option.
 
-## Authoritative reference
+## References
 
-The conversion rules below are Harbor's, from its adapter guide, and Evolve runs the
-task format unchanged. For the full guide, read
-https://github.com/laude-institute/harbor/blob/main/docs/content/docs/datasets/adapters.mdx
-(its steps on parity experiments, the registry and pull requests are Harbor's own
-process and do not apply here). The task format itself is at
+The conversion structure follows Harbor's adapter format. Evolve's supported task
+options and publishing workflow are documented in its bundled references:
+
+| Topic | Read |
+| --- | --- |
+| Task layout and configuration | `evolve skills get evals core-concepts/tasks` and `evolve skills get evals core-concepts/task-config` |
+| Environments and Compose | `evolve skills get evals core-concepts/task-environment` and `evolve skills get evals core-concepts/compose` |
+| Verifiers and rewards | `evolve skills get evals core-concepts/task-verifiers` and `evolve skills get evals core-concepts/rewardkit` |
+| Managed support limits | `evolve skills get evals core-concepts/compatibility` |
+
+The upstream task format is at
 https://docs.harborframework.com/core-concepts/tasks/overview.
 
-Do not invent structure, field names, or workflow beyond what the guide specifies.
+Use Evolve's references for commands and supported behavior. Upstream adapter
+examples below can help with conversion code; Harbor's registry and pull-request
+workflow do not apply to publishing on Evolve.
 
 ## Prerequisites
 
@@ -59,7 +68,7 @@ before proceeding.
 | Adapter name | Lowercase, hyphen-separated. Must match the benchmark's common identifier (e.g., `swe-bench`, `aider-polyglot`). Becomes the dataset name on Evolve and, with dashes turned to underscores, the Python package name. |
 | Human-readable name | Appears in the README. |
 | Upstream repo URL | Needed for step 1 (benchmark analysis) and for the README. |
-| Reference solutions available? | If the benchmark ships reference solutions, use them. If not, they must be written, with LLM help, before the tasks can be checked. |
+| Reference solutions available? | Use them when available. Otherwise record the gap and write solutions when possible; Evolve can check tasks without them. |
 | Subset | Adapting a subset of tasks is acceptable (e.g., only a verified split). Document every exclusion in the README. |
 
 ### 3. Write the converter
@@ -90,8 +99,10 @@ The `evolve` CLI has no adapter scaffold; create this layout by hand:
 `--overwrite`, and `--task-ids`. Run it as
 `uv run python -m <adapter_name>.main --output-dir <path>`.
 
-Each generated task directory must contain at minimum `task.toml`, `instruction.md`,
-`environment/Dockerfile`, `solution/solve.sh`, and `tests/test.sh`:
+For a single-step benchmark using Dockerfile-built images, generate this layout.
+`solution/solve.sh` is optional in the format, but useful for testing the conversion.
+Tasks using a prebuilt image can omit `environment/Dockerfile`; follow the
+environment reference above.
 
 ```
 <output-dir>/
@@ -149,11 +160,13 @@ OPENAI_API_KEY = "${OPENAI_API_KEY}"
 **`tests/test.sh`:** must write a numeric reward (integer or float, 0 to 1) to
 `/logs/verifier/reward.txt`, or named numbers to `/logs/verifier/reward.json`.
 `/logs/verifier/` exists at run time. Use the same metrics as the original benchmark.
+For multiple JSON metrics, include a `reward` key in `[0, 1]` for the primary score.
+Install pytest in the image that runs verification; do not fetch it during the test.
 
 ```bash
 #!/bin/bash
-pytest /tests/test_*.py
-if [ $? -eq 0 ]; then
+set -uo pipefail
+if python3 -m pytest /tests/test_*.py; then
   echo 1 > /logs/verifier/reward.txt
 else
   echo 0 > /logs/verifier/reward.txt
@@ -179,6 +192,9 @@ RUN apt-get update && apt-get install -y \
     git \
     && rm -rf /var/lib/apt/lists/*
 
+# The shared verifier uses the task image
+RUN pip install --no-cache-dir 'pytest==8.4.1'
+
 # Install benchmark-specific dependencies
 # RUN pip install --no-cache-dir <packages>
 
@@ -211,15 +227,32 @@ run each task.
 
 ### 5. Verify the conversion
 
-Check the generated tasks. The check reads each task and, when it can, runs its
-environment, its `solution/solve.sh` and its verifier, then rules on every criterion of a
-rubric; every task should come back `no_problem_found` with `executed` true.
+Validate the metadata, then request a hosted review of each task:
 
 ```bash
+evolve dataset check "<output-dir>"
 evolve check "<output-dir>" --watch
-evolve check show <check-id>
 ```
 
+Set `CHECK_ID` to the parent check ID returned by `evolve check`, then read it:
+
+```bash
+evolve check show "$CHECK_ID"
+```
+
+The first command sends `task.toml` files and any `dataset.toml` to the API for
+validation, without uploading the full task packages or executing them. The
+second runs a paid checker agent against a rubric. It can run the environment,
+reference solution, and verifier when its sandbox supports them; it does not
+guarantee that execution.
+
+Read every task's status, findings, and evidence. The result is the
+per-criterion JSON; Evolve derives no verdict from it. The default rubric's
+execution criteria say whether the checker ran the task, and `unknown` on them
+is not proof that the solution or verifier ran.
+Use `evolve check trace <task-check-id>` to inspect the checker's work. Test any
+unresolved execution in an environment that can run the task.
+
 A failing check usually means one of three things, in this order:
 
 1. **Adaptation error:** the instruction, environment or test does not match upstream.
@@ -229,14 +262,18 @@ A failing check usually means one of three things, in this order:
 3. **Environment error:** a Dockerfile that does not build or a test that cannot run makes
    the task impossible for every agent, so catch it here.
 
-Where Harbor is installed, `harbor run -p "<output-dir>" -a oracle` runs every reference
-solution locally (optional); the reward should be `1.0` on every task.
+The Evolve CLI has no standalone oracle command. Do not treat `evolve check` as
+one. When a reference solution is available, verify that it earns full credit and
+that invalid submissions fail; keep the execution evidence with the conversion.
 
-**Benchmarks without reference solutions:** write them, with LLM help, before publishing.
-A cheap agent and model can take a first pass over all the tasks; complete the rest with
-a stronger model plus human review.
+**Benchmarks without reference solutions:** document that limitation. The default
+checker marks the solution criterion `not_applicable` when none is shipped.
+Write and validate solutions where possible; a model's proposed solution still
+needs review.
 
-**Step complete when:** every task passes `evolve check`.
+**Step complete when:** every generated task has been reviewed, confirmed defects
+are fixed, and any unresolved execution or missing-solution limits are documented.
+Read `evolve skills get evals core-concepts/check` for the result rules.
 
 ### 6. Document and publish
 
@@ -245,7 +282,8 @@ and every exclusion; benchmark bugs found and how they were handled; prompt
 modifications, environment adjustments and other deviations from the original, with the
 reason; known limitations; the exact commands to regenerate the tasks and to run them.
 
-Then publish the output folder as a dataset (`evolve skills get publish` has every option):
+Then publish the output folder as a dataset (`evolve skills get publish` explains the
+workflow; `evolve skills get evals cli-reference/dataset` lists the options):
 
 ```bash
 evolve dataset check "<output-dir>"
@@ -256,11 +294,18 @@ evolve run -d "<adapter-name>@1.0" -a codex -m gpt-5.5 --watch
 To see how faithful the conversion is, run a job with the same agent and model the
 benchmark's own leaderboard reports, and compare the scores.
 
+For TypeScript or Python automation, read
+`evolve skills get evals sdk-reference/methods/datasets`,
+`evolve skills get evals sdk-reference/methods/checks`, and
+`evolve skills get evals sdk-reference/methods/jobs`.
+
 ## Reference adapters by shape
 
 Harbor's repository holds one adapter per benchmark, public at
 https://github.com/laude-institute/harbor/tree/main/adapters. When implementation
-questions come up, read the one that matches the benchmark's shape:
+questions come up, read the one that matches the benchmark's shape. These are
+optional conversion examples; verify their task settings against Evolve's
+supported scope before using them:
 
 | Shape | Example adapter |
 |-------|----------------|
@@ -283,5 +328,5 @@ questions come up, read the one that matches the benchmark's shape:
 |---------|--------------|--------|
 | `evolve: command not found` | The CLI is not installed | `npm install -g @evolvingmachines/evolve`. |
 | `evolve dataset check` refuses a task by name | Its `task.toml` breaks a rule (a field, a value, a name) | Fix the converter, regenerate, check again. The refusal names the field. |
-| A task check comes back `has_a_problem` | One criterion failed | `evolve check show <check-id>` prints the criterion, its explanation and its evidence. |
+| A task check has a `fail` outcome | One criterion failed | `evolve check show <check-id>` prints the criterion, its explanation and its evidence. |
 | Every task fails the check the same way | An error in the task template | Fix `task-template/` in the converter, not the generated tasks. |