@evolvingmachines/evolve 0.0.55-project-sable.20260916.db0bc0bb

package/package.json

@@ -0,0 +1,163 @@
+{
+  "name": "@evolvingmachines/evolve",
+  "version": "0.0.55-project-sable.20260916.db0bc0bb",
+  "keywords": [
+    "ai",
+    "agents",
+    "orchestration",
+    "sandbox",
+    "e2b",
+    "daytona",
+    "modal",
+    "gpu",
+    "claude code",
+    "codex",
+    "gemini cli",
+    "qwen code",
+    "automation",
+    "evolve"
+  ],
+  "homepage": "https://github.com/evolving-machines-lab/evolve",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/evolving-machines-lab/evolve.git"
+  },
+  "type": "module",
+  "bin": {
+    "evolve": "dist/cli/index.js"
+  },
+  "main": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "LICENSE",
+    "spec",
+    "docs-evals",
+    "docs-agents",
+    "skills",
+    "hosted-error-codes.json",
+    "harness-capabilities.json"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    },
+    "./harness-capabilities.json": "./harness-capabilities.json"
+  },
+  "scripts": {
+    "build": "tsx scripts/generate-harness-capabilities.ts --check && tsup --minify && node scripts/copy-spec.mjs && node scripts/copy-skills.mjs",
+    "generate:capabilities": "tsx scripts/generate-harness-capabilities.ts",
+    "dev": "tsup src/index.ts --watch",
+    "type-check": "tsc --noEmit && tsc -p tsconfig.types-test.json",
+    "test": "npm run test:unit && npm run test:integration",
+    "pretest:unit": "npm run build",
+    "test:unit": "tsx tests/unit/semaphore.test.ts && tsx tests/unit/swarm-concurrency.test.ts && tsx tests/unit/swarm-verify.test.ts && tsx tests/unit/swarm-retry-verify.test.ts && tsx tests/unit/prompt-construction.test.ts && tsx tests/unit/observability-metadata.test.ts && tsx tests/unit/observability-identity.test.ts && tsx tests/unit/skills-integration.test.ts && tsx tests/unit/skills-resolver.test.ts && tsx tests/unit/skills-tag-peel.test.ts && tsx tests/unit/auth-config.test.ts && tsx tests/unit/config-validation.test.ts && tsx tests/unit/agent-presets.test.ts && tsx tests/unit/run-resume-option.test.ts && tsx tests/unit/native-config-delivery.test.ts && tsx tests/unit/browser-config.test.ts && tsx tests/unit/integrations-config.test.ts && tsx tests/unit/managed-secrets.test.ts && tsx tests/unit/managed-modal.test.ts && tsx tests/unit/plugins-config.test.ts && tsx tests/unit/session-runtime.test.ts && tsx tests/unit/cost-api.test.ts && tsx tests/unit/storage-config.test.ts && tsx tests/unit/checkpoint-tar.test.ts && tsx tests/unit/checkpoint-tar-listing.test.ts && tsx tests/unit/checkpoint-errors.test.ts && tsx tests/unit/checkpoint-flows.test.ts && tsx tests/unit/checkpoint-dx.test.ts && tsx tests/unit/checkpoint-edge-cases.test.ts && tsx tests/unit/codex-parser-errors.test.ts && tsx tests/unit/parser-harness-errors.test.ts && tsx tests/unit/codex-toml.test.ts && tsx tests/unit/kimi-parser.test.ts && tsx tests/unit/kimi-mcp.test.ts && tsx tests/unit/opencode-parser.test.ts && tsx tests/unit/droid-parser.test.ts && tsx tests/unit/droid-mcp.test.ts && tsx tests/unit/parser-tool-name.test.ts && tsx tests/unit/parser-atif-fields.test.ts && tsx tests/unit/provider-parity.test.ts && tsx tests/unit/storage-client.test.ts && tsx tests/unit/sessions-client.test.ts && tsx tests/unit/hosted-client.test.ts && tsx tests/unit/hosted-trial-tree.test.ts && tsx tests/unit/hosted-ergonomics.test.ts && tsx tests/unit/cli.test.ts && tsx tests/unit/cli-bin.test.ts && tsx tests/unit/cli-skills.test.ts && tsx tests/unit/cli-help.test.ts && tsx tests/unit/hosted-types.test.ts && tsx tests/unit/hosted-error-codes.test.ts && tsx tests/unit/hosted-spec-gate.test.ts && tsx tests/unit/harness-capabilities.test.ts && tsx tests/unit/sandbox-artifacts.test.ts && tsx tests/unit/upload-file-from-path.test.ts && tsx tests/unit/file-utils.test.ts && tsx --expose-gc tests/unit/hosted-tar.test.ts && tsx tests/unit/hosted-upload-stream.test.ts && tsx tests/unit/hosted-upload-memory.test.ts && tsx tests/unit/hosted-resumable.test.ts && tsx tests/unit/cli-ndjson-pipe.test.ts",
+    "test:unit:upload-from-path": "tsx tests/unit/upload-file-from-path.test.ts",
+    "test:unit:upload-stream": "tsx tests/unit/hosted-upload-stream.test.ts",
+    "test:unit:upload-memory": "tsx tests/unit/hosted-upload-memory.test.ts",
+    "test:unit:parity": "tsx tests/unit/provider-parity.test.ts",
+    "test:unit:semaphore": "tsx tests/unit/semaphore.test.ts",
+    "test:unit:swarm": "tsx tests/unit/swarm-concurrency.test.ts",
+    "test:unit:verify": "tsx tests/unit/swarm-verify.test.ts",
+    "test:unit:retry": "tsx tests/unit/swarm-retry-verify.test.ts",
+    "test:unit:prompts": "tsx tests/unit/prompt-construction.test.ts",
+    "test:unit:observability": "tsx tests/unit/observability-metadata.test.ts",
+    "test:unit:observability-identity": "tsx tests/unit/observability-identity.test.ts",
+    "test:unit:skills": "tsx tests/unit/skills-integration.test.ts",
+    "test:unit:skills-resolver": "tsx tests/unit/skills-resolver.test.ts",
+    "test:unit:auth": "tsx tests/unit/auth-config.test.ts",
+    "test:unit:config-validation": "tsx tests/unit/config-validation.test.ts",
+    "test:unit:agent-presets": "tsx tests/unit/agent-presets.test.ts",
+    "test:unit:native-config-delivery": "tsx tests/unit/native-config-delivery.test.ts",
+    "test:unit:browser": "tsx tests/unit/browser-config.test.ts",
+    "test:unit:integrations": "tsx tests/unit/integrations-config.test.ts",
+    "test:unit:managed-secrets": "tsx tests/unit/managed-secrets.test.ts",
+    "test:unit:plugins": "tsx tests/unit/plugins-config.test.ts",
+    "test:unit:storage-config": "tsx tests/unit/storage-config.test.ts",
+    "test:unit:error-codes": "tsx tests/unit/hosted-error-codes.test.ts",
+    "test:unit:capabilities": "tsx tests/unit/harness-capabilities.test.ts",
+    "test:unit:checkpoint-tar": "tsx tests/unit/checkpoint-tar.test.ts",
+    "test:unit:checkpoint-tar-listing": "tsx tests/unit/checkpoint-tar-listing.test.ts",
+    "test:unit:checkpoint-errors": "tsx tests/unit/checkpoint-errors.test.ts",
+    "test:unit:checkpoint-flows": "tsx tests/unit/checkpoint-flows.test.ts",
+    "test:unit:checkpoint-dx": "tsx tests/unit/checkpoint-dx.test.ts",
+    "test:unit:checkpoint-edge-cases": "tsx tests/unit/checkpoint-edge-cases.test.ts",
+    "test:unit:storage-client": "tsx tests/unit/storage-client.test.ts",
+    "test:unit:sessions-client": "tsx tests/unit/sessions-client.test.ts",
+    "test:unit:hosted-client": "tsx tests/unit/hosted-client.test.ts",
+    "test:unit:tool-name": "tsx tests/unit/parser-tool-name.test.ts",
+    "test:unit:kimi-parser": "tsx tests/unit/kimi-parser.test.ts",
+    "test:unit:kimi-mcp": "tsx tests/unit/kimi-mcp.test.ts",
+    "test:unit:codex-toml": "tsx tests/unit/codex-toml.test.ts",
+    "pretest:integration": "npm run build",
+    "test:integration": "npm run test:01",
+    "test:integration:all": "npm run test:01 && npm run test:02 && npm run test:03 && npm run test:04 && npm run test:05 && npm run test:06 && npm run test:07 && npm run test:08 && npm run test:09 && npm run test:10 && npm run test:11 && npm run test:12 && npm run test:13 && npm run test:14 && npm run test:15 && npm run test:24",
+    "test:01": "tsx tests/integration/01-all-agents-parallel.ts",
+    "test:02": "tsx tests/integration/02-execute-command-streaming.ts",
+    "test:03": "tsx tests/integration/03-file-operations.ts",
+    "test:04": "tsx tests/integration/04-session-lifecycle.ts",
+    "test:05": "tsx tests/integration/05-workspace-config.ts",
+    "test:06": "tsx tests/integration/06-observability.ts",
+    "test:07": "tsx tests/integration/07-background-timeouts.ts",
+    "test:08": "tsx tests/integration/08-network.ts",
+    "test:09": "tsx tests/integration/09-advanced-features.ts",
+    "test:10": "tsx tests/integration/10-swarm-abstractions.ts",
+    "test:11": "tsx tests/integration/11-schema-validation.ts",
+    "test:12": "tsx tests/integration/12-swarm-verify.ts",
+    "test:13": "tsx tests/integration/13-pipeline-abstractions.ts",
+    "test:14": "tsx tests/integration/14-byok-direct-mode.ts",
+    "test:15": "tsx tests/integration/15-oauth-mode.ts",
+    "test:20": "tsx tests/integration/20-storage-checkpoints.ts",
+    "test:21": "tsx tests/integration/21-storage-restore-fidelity.ts",
+    "test:22": "tsx tests/integration/22-storage-dx.ts",
+    "test:23": "tsx tests/integration/23-storage-edge-cases.ts",
+    "test:24": "tsx tests/integration/24-kimi-gateway-hello.ts",
+    "test:26": "tsx tests/integration/26-storage-full-surface.ts",
+    "test:claude": "tsx tests/integration/01-all-agents-parallel.ts claude",
+    "test:codex": "tsx tests/integration/01-all-agents-parallel.ts codex",
+    "test:gemini": "tsx tests/integration/01-all-agents-parallel.ts gemini",
+    "test:qwen": "tsx tests/integration/01-all-agents-parallel.ts qwen",
+    "test:unit:cli": "tsx tests/unit/cli.test.ts",
+    "test:unit:cli-bin": "tsx tests/unit/cli-bin.test.ts",
+    "test:unit:cli-skills": "tsx tests/unit/cli-skills.test.ts",
+    "test:unit:cli-help": "tsx tests/unit/cli-help.test.ts",
+    "test:unit:cli-ndjson-pipe": "tsx tests/unit/cli-ndjson-pipe.test.ts",
+    "test:unit:sandbox-artifacts": "tsx tests/unit/sandbox-artifacts.test.ts",
+    "test:unit:hosted-tar": "tsx --expose-gc tests/unit/hosted-tar.test.ts",
+    "prepack": "node scripts/copy-spec.mjs && node scripts/copy-skills.mjs"
+  },
+  "dependencies": {
+    "@agentclientprotocol/sdk": "^0.5.1",
+    "@evolvingmachines/daytona": "0.0.55-project-sable.20260916.db0bc0bb",
+    "@evolvingmachines/e2b": "0.0.55-project-sable.20260916.db0bc0bb",
+    "@evolvingmachines/modal": "0.0.55-project-sable.20260916.db0bc0bb",
+    "ajv": "^8.17.1",
+    "p-map": "^7.0.2",
+    "smol-toml": "^1.7.0",
+    "tar-stream": "^3.1.7",
+    "yaml": "^2.9.0",
+    "zod": "^3.24.0",
+    "zod-to-json-schema": "^3.25.0"
+  },
+  "peerDependencies": {
+    "@aws-sdk/client-s3": "^3.0.0",
+    "@aws-sdk/s3-request-presigner": "^3.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@aws-sdk/client-s3": {
+      "optional": true
+    },
+    "@aws-sdk/s3-request-presigner": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/node": "^22.15.18",
+    "@types/tar-stream": "^3.1.4",
+    "tsup": "^8.4.0",
+    "typescript": "^5.8.3"
+  }
+}

package/skills/create-adapter/SKILL.md

@@ -0,0 +1,287 @@
+---
+name: create-adapter
+description: Convert an existing benchmark into a folder of Harbor-format tasks ready for `evolve dataset publish`. Use when the user wants to port, adapt, or import a benchmark (a paper's task set, a repository of problems, a leaderboard's dataset) onto Evolve. Guides the conversion and its verification with evolve check.
+metadata:
+  internal: true
+---
+
+# Create Adapter
+
+An adapter is a small program that reads an existing benchmark and writes one task
+directory per task, in the Harbor task format. Its output is a folder of tasks, ready for
+`evolve dataset publish`. This skill guides the conversion; `evolve skills get create-task`
+has the task format in full, and `evolve skills get publish` every publish option.
+
+## Authoritative reference
+
+The conversion rules below are Harbor's, from its adapter guide, and Evolve runs the
+task format unchanged. For the full guide, read
+https://github.com/laude-institute/harbor/blob/main/docs/content/docs/datasets/adapters.mdx
+(its steps on parity experiments, the registry and pull requests are Harbor's own
+process and do not apply here). The task format itself is at
+https://docs.harborframework.com/core-concepts/tasks/overview.
+
+Do not invent structure, field names, or workflow beyond what the guide specifies.
+
+## Prerequisites
+
+- The `evolve` CLI: `npm install -g @evolvingmachines/evolve` (`evolve --version` succeeds).
+- `EVOLVE_API_KEY` exported, from the dashboard's API keys page
+  (https://dashboard.evolvingmachines.ai/api-keys); `evolve auth status` prints who you are.
+- Docker, to build and enter a task's environment locally (optional).
+- The upstream benchmark's repository, cloned.
+
+## Workflow
+
+### 1. Understand the original benchmark
+
+Identify these four components for every task in the benchmark:
+
+| Component | What to find |
+|-----------|-------------|
+| **Instructions** | How tasks are described; what information agents receive |
+| **Environments** | Docker setup, system dependencies, file structures |
+| **Tests** | Evaluation method: deterministic unit tests, LLM-as-a-Judge, etc. |
+| **Solutions** | Oracle/reference solutions; if none exist, whether LLM generation is feasible |
+
+Study the benchmark's repository, documentation, and code structure.
+
+**Step complete when:** you can describe, for each task, the instruction text,
+environment setup, test/verification method, and reference solution.
+
+### 2. Gather benchmark context from the user
+
+Collect the following before writing code. If the user has not provided an item, ask
+before proceeding.
+
+| Field | Why it matters |
+|-------|---------------|
+| Adapter name | Lowercase, hyphen-separated. Must match the benchmark's common identifier (e.g., `swe-bench`, `aider-polyglot`). Becomes the dataset name on Evolve and, with dashes turned to underscores, the Python package name. |
+| Human-readable name | Appears in the README. |
+| Upstream repo URL | Needed for step 1 (benchmark analysis) and for the README. |
+| Reference solutions available? | If the benchmark ships reference solutions, use them. If not, they must be written, with LLM help, before the tasks can be checked. |
+| Subset | Adapting a subset of tasks is acceptable (e.g., only a verified split). Document every exclusion in the README. |
+
+### 3. Write the converter
+
+The `evolve` CLI has no adapter scaffold; create this layout by hand:
+
+```
+<adapter-name>/
+├── README.md                  # final documentation (step 6)
+├── pyproject.toml             # Python package config
+└── src/
+    └── <adapter_name>/        # adapter-name with dashes → underscores
+        ├── __init__.py
+        ├── adapter.py         # main logic: parse benchmark, generate task dirs
+        ├── main.py            # CLI entry point
+        └── task-template/     # template files copied into each task
+            ├── task.toml
+            ├── instruction.md
+            ├── environment/
+            │   └── Dockerfile
+            ├── solution/
+            │   └── solve.sh
+            └── tests/
+                └── test.sh
+```
+
+`main.py` must support `--output-dir` (where generated tasks are written), `--limit`,
+`--overwrite`, and `--task-ids`. Run it as
+`uv run python -m <adapter_name>.main --output-dir <path>`.
+
+Each generated task directory must contain at minimum `task.toml`, `instruction.md`,
+`environment/Dockerfile`, `solution/solve.sh`, and `tests/test.sh`:
+
+```
+<output-dir>/
+└── <task-id>/
+    ├── task.toml              # task configuration and metadata
+    ├── instruction.md         # task instructions for the agent
+    ├── environment/
+    │   └── Dockerfile         # container environment definition
+    ├── solution/
+    │   └── solve.sh           # reference solution script
+    └── tests/
+        ├── test.sh            # test execution script
+        └── test_*.py          # (optional) pytest test files
+```
+
+**`task.toml`:** every task must include it. Adjust timeouts to match your benchmark's
+complexity.
+
+```toml
+schema_version = "1.4"
+
+[task]
+name = "<adapter-name>/<task-id>"
+version = "1.0.0"
+
+[metadata]
+author_name = "Original benchmark authors' names"
+author_email = "benchmark-authors@email.com"
+difficulty = "medium"
+category = "programming"
+tags = ["debugging", "python"]
+
+[agent]
+timeout_sec = 1800.0
+
+[verifier]
+timeout_sec = 120.0
+
+[environment]
+build_timeout_sec = 600.0
+cpus = 1
+memory_mb = 2048
+storage_mb = 10240
+```
+
+For LLM-as-a-Judge verifiers, request the judge credential in `[verifier.env]`; on Evolve
+you never put a real key in the task, the credential is supplied at run time
+(`evolve skills get rewardkit` has the details):
+
+```toml
+[verifier.env]
+OPENAI_API_KEY = "${OPENAI_API_KEY}"
+```
+
+**`tests/test.sh`:** must write a numeric reward (integer or float, 0 to 1) to
+`/logs/verifier/reward.txt`, or named numbers to `/logs/verifier/reward.json`.
+`/logs/verifier/` exists at run time. Use the same metrics as the original benchmark.
+
+```bash
+#!/bin/bash
+pytest /tests/test_*.py
+if [ $? -eq 0 ]; then
+  echo 1 > /logs/verifier/reward.txt
+else
+  echo 0 > /logs/verifier/reward.txt
+fi
+```
+
+**`instruction.md`:** write agent-actionable instructions, not raw benchmark
+descriptions. Include the goal, constraints, expected output location, and any files the
+agent should modify. Do not include test answers or reference solutions. Prompt
+modifications (e.g., "write files in place without asking") are acceptable if you apply
+them to both the original benchmark and the adapter, and document them.
+
+**`environment/Dockerfile`:** set up the container the agent will work in. Install system
+and Python dependencies, copy any benchmark-specific data files, and set the working
+directory. The agent and the verifier both run inside this container unless the task
+declares a separate verifier environment.
+
+```dockerfile
+FROM python:3.13-slim
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install benchmark-specific dependencies
+# RUN pip install --no-cache-dir <packages>
+
+# Copy task-specific files
+# COPY . /workspace/
+```
+
+**GPU tasks:** set `gpus` (and optionally `gpu_types`) under `[environment]` in
+`task.toml`. After publishing, `evolve dataset show` prints which sandbox providers can
+run each task.
+
+**Step complete when:** `main.py` produces a valid task directory for each task.
+
+### 4. Keep task names right
+
+- **Every task directory name is the task's name on Evolve.** Letters, digits, `.`, `_`
+  and `-`, at most 128 characters, starting with a letter or digit; use lowercase (Harbor's
+  convention). Put the same identifier in `[task] name` as `<adapter-name>/<task-id>`. A
+  `[metadata] task_id`, when present, must equal the directory name.
+- **Task names must be unique within the dataset and stable across adapter runs.** An
+  unstable name makes the same task look like a different one on republish. If upstream
+  lacks stable identifiers, mint a deterministic scheme in adapter code (e.g.,
+  `{dataset}-1`, `{dataset}-2`, ...) derived from a reproducible sort of upstream tasks.
+- **Sanitize upstream identifiers** before using them as names: lowercase, replace
+  spaces/slashes/special characters with hyphens, avoid leading/trailing separators.
+- **Treat `main.py` as the source of truth for task names.** Do not hand-edit generated
+  task directories; fix the converter and regenerate.
+- Use `schema_version = "1.4"` at the top of `task.toml`; `[task].version` is the task's
+  own version and is distinct from it.
+
+### 5. Verify the conversion
+
+Check the generated tasks. The check reads each task and, when it can, runs its
+environment, its `solution/solve.sh` and its verifier, then rules on every criterion of a
+rubric; every task should come back `no_problem_found` with `executed` true.
+
+```bash
+evolve check "<output-dir>" --watch
+evolve check show <check-id>
+```
+
+A failing check usually means one of three things, in this order:
+
+1. **Adaptation error:** the instruction, environment or test does not match upstream.
+2. **A broken reference solution:** run the solution on the original benchmark side too, to
+   tell a wrong solution from a wrong adaptation. If the fix is simple, propose it
+   upstream and document it in the README; exclude tasks that cannot be reliably fixed.
+3. **Environment error:** a Dockerfile that does not build or a test that cannot run makes
+   the task impossible for every agent, so catch it here.
+
+Where Harbor is installed, `harbor run -p "<output-dir>" -a oracle` runs every reference
+solution locally (optional); the reward should be `1.0` on every task.
+
+**Benchmarks without reference solutions:** write them, with LLM help, before publishing.
+A cheap agent and model can take a first pass over all the tasks; complete the rest with
+a stronger model plus human review.
+
+**Step complete when:** every task passes `evolve check`.
+
+### 6. Document and publish
+
+Write the README with: what the benchmark measures and a link to it; the subset adapted
+and every exclusion; benchmark bugs found and how they were handled; prompt
+modifications, environment adjustments and other deviations from the original, with the
+reason; known limitations; the exact commands to regenerate the tasks and to run them.
+
+Then publish the output folder as a dataset (`evolve skills get publish` has every option):
+
+```bash
+evolve dataset check "<output-dir>"
+evolve dataset publish --dir "<output-dir>" --name "<adapter-name>" --version 1.0 --watch
+evolve run -d "<adapter-name>@1.0" -a codex -m gpt-5.5 --watch
+```
+
+To see how faithful the conversion is, run a job with the same agent and model the
+benchmark's own leaderboard reports, and compare the scores.
+
+## Reference adapters by shape
+
+Harbor's repository holds one adapter per benchmark, public at
+https://github.com/laude-institute/harbor/tree/main/adapters. When implementation
+questions come up, read the one that matches the benchmark's shape:
+
+| Shape | Example adapter |
+|-------|----------------|
+| Repository-level coding tasks with unit-test verifiers | `adapters/swebench/` |
+| Many small tasks from one dataset file | `adapters/evoeval/` |
+| Data-analysis tasks with a custom dataset | `adapters/bixbench/` |
+| LLM-as-a-Judge verification | `adapters/financeagent/` |
+| GPU tasks | `adapters/featurebench/` |
+
+## What this skill does NOT do
+
+- Implement `adapter.py`, `main.py`, or the task-template files. Those are the
+  contributor's work, guided by the rules above.
+- Run `evolve check` or publish on its own. Both act on the user's account and need the
+  user's explicit intent; a check spends credits (it runs a checker model).
+
+## Failure modes
+
+| Symptom | Likely cause | Action |
+|---------|--------------|--------|
+| `evolve: command not found` | The CLI is not installed | `npm install -g @evolvingmachines/evolve`. |
+| `evolve dataset check` refuses a task by name | Its `task.toml` breaks a rule (a field, a value, a name) | Fix the converter, regenerate, check again. The refusal names the field. |
+| A task check comes back `has_a_problem` | One criterion failed | `evolve check show <check-id>` prints the criterion, its explanation and its evidence. |
+| Every task fails the check the same way | An error in the task template | Fix `task-template/` in the converter, not the generated tasks. |