@tonyclaw/eval-harness 0.2.18

package/config/eval.example.json

@@ -0,0 +1,70 @@
+{
+  "agent": {
+    "type": "opencode",
+    "variant": "high"
+  },
+  "source_repo": {
+    "url": "https://example.com/your-org/task-repository.git",
+    "ssh_url": "",
+    "ref": "main",
+    "fallback_urls": []
+  },
+  "submission_repo": {
+    "url": "https://example.com/your-org/submitted-work.git",
+    "ssh_url": "",
+    "ref": "main",
+    "work_dir": "work",
+    "fallback_urls": []
+  },
+  "run": {
+    "runs_dir": "runs",
+    "project_dir_name": "project",
+    "clean_runtime_artifacts": true
+  },
+  "execution": {
+    "timeout_minutes": 90,
+    "idle_timeout_minutes": 10,
+    "subagent_idle_grace_windows": 1,
+    "session_activity_check_seconds": 15,
+    "artifact_activity_check_seconds": 5,
+    "provider_api_failure_grace": 5,
+    "isolate_runtime_env": true,
+    "isolate_maven_repo": true,
+    "seed_runtime_profile": true,
+    "stream_output": true,
+    "console_log_mode": "important",
+    "heartbeat_seconds": 60,
+    "final_result_recovery_helpers": [],
+    "agent_inspector": {
+      "enabled": false,
+      "port": 25947,
+      "base_url": "http://127.0.0.1:25947",
+      "proxy_url": "http://127.0.0.1:25947/proxy",
+      "health_url": "http://127.0.0.1:25947/api/health",
+      "providers_url": "http://127.0.0.1:25947/api/providers",
+      "mcp_url": "http://127.0.0.1:25947/api/mcp",
+      "mcp_enabled": false,
+      "mcp_required_tools": [],
+      "required_model": "",
+      "start_command": ["agent-inspector", "--background", "--no-open", "--port", "25947"],
+      "restart_command": ["agent-inspector", "--background", "--no-open", "--port", "25947", "--force-restart"],
+      "doctor_command": ["agent-inspector", "doctor", "--port", "25947"],
+      "startup_timeout_seconds": 15
+    }
+  },
+  "container": {
+    "enabled": false,
+    "engine": "docker",
+    "image": "eval-harness-worker:latest",
+    "workspace_mount": "/workspace",
+    "extra_args": []
+  },
+  "scoring": {
+    "thresholds": {
+      "accept_score": 85,
+      "investigate_score": 60
+    },
+    "protected_paths": [],
+    "validation_commands": []
+  }
+}

package/docs/agent-runner-compatibility.md

@@ -0,0 +1,47 @@
+# Agent Runner Compatibility
+
+This page records the current validated runner adapter baseline for the
+eval harness. Treat these versions as known-good compatibility evidence, not as
+a strict lower or upper bound.
+
+Validated on: `2026-07-06`
+
+| Runner | `agent.type` | Validated CLI version | Version command | Version output |
+| --- | --- | --- | --- | --- |
+| OpenCode | `opencode` | `1.17.11` | `opencode --version` | `1.17.11` |
+| MiMo Code | `mimocode`, `mimo` | `0.1.4` | `mimo --version` | `0.1.4` |
+| Codex CLI | `codex`, `codex-cli` | `0.142.5` | `codex --version` | `codex-cli 0.142.5` |
+| Claude Code | `claude`, `claude-code` | `2.1.195` | `claude --version` | `2.1.195 (Claude Code)` |
+
+The same baseline is exposed in machine-readable form:
+
+```powershell
+python -m eval runner-presets --format json
+```
+
+Check the installed runner on the current machine against that baseline:
+
+```powershell
+python -m eval runner-version-check --config config/eval.local.json --markdown-output runs/preflight/runner-version-check.md
+```
+
+Jenkins runs the same check during environment preparation and archives
+`runs/preflight/runner-version-check.json` plus `.md`. Set
+`REQUIRE_RUNNER_COMPATIBILITY=true` when the job should fail fast on a version
+drift instead of only recording a warning.
+
+## Compatibility Notes
+
+- OpenCode `1.17.11`: agent frontmatter `tools` must be a record in this baseline. The harness normalizes inline `tools: [...]` arrays when installing `.opencode/agents` assets.
+- MiMo Code `0.1.4`: uses `--agent` for runner variant selection and shares the agent `tools` record normalization path for `.mimocode/agents` assets.
+- Codex CLI `0.142.5`: ignores `REASONING_VARIANT`; Agent Inspector routes are passed as `--model` after route-name normalization. On Windows service deployments, the harness honors configured executable paths and `OEVAL_CODEX_EXECUTABLE`, then auto-discovers the current user's `%LOCALAPPDATA%\OpenAI\Codex\bin\*\codex.exe` before falling back to `PATH`.
+- Claude Code `2.1.195`: ignores the generic `REASONING_VARIANT`. Claude Code's own `--agent` flag selects a named Claude agent, not a reasoning level, so configure it only in a Claude-specific command when needed. Agent Inspector routing uses `ANTHROPIC_BASE_URL` and `ANTHROPIC_API_KEY`; the harness also pins `ANTHROPIC_MODEL` and `CLAUDE_CODE_SUBAGENT_MODEL` to the resolved worker model so Claude subagents stay on the same Inspector route.
+
+## Upgrade Check
+
+When upgrading any runner:
+
+- Run `python -m eval runner-presets --format json` and compare the expected model flag, variant flag, asset directories, and validated baseline.
+- Run `python -m eval runner-version-check --config <config>` and inspect `status`, `actual_version`, and `validated_version`.
+- Run a short Jenkins smoke or single-worker real dispatch with `DRY_RUN=false`.
+- Check `<agent-log-dir>/opencode.log`, `sessions.json`, Inspector evidence, and `evaluation.json` for changed CLI behavior before updating this compatibility baseline.

package/docs/runners.md

@@ -0,0 +1,82 @@
+# Runner Adapters
+
+The eval harness is runner-based. OpenCode, MiMo Code, Codex CLI, and Claude Code are supported runner adapters, but none of them is the platform itself.
+
+## Runner Model
+
+The harness owns the evaluation lifecycle:
+
+- materialize task and work inputs
+- prepare isolated worker directories
+- install work-provided Skills and Agents into the selected runner layout
+- execute the selected runner CLI
+- collect logs, artifacts, Inspector evidence, scores, and reports
+
+Each runner owns its own CLI details. The current validated CLI versions are
+tracked in [Agent Runner Compatibility](agent-runner-compatibility.md).
+
+| Runner | `agent.type` | Default binary | Model flag | Variant flag | Default tool check | Default asset targets |
+| --- | --- | --- | --- | --- | --- | --- |
+| OpenCode | `opencode` | `opencode` | `--model` | `--variant` | `opencode,node` | `.opencode/skills`, `.opencode/agents` |
+| MiMo Code | `mimocode`, `mimo` | `mimo` | `--model` | `--agent` | `mimo,node` | `.mimocode/skills`, `.mimocode/agents` |
+| Codex CLI | `codex`, `codex-cli` | discovered Codex CLI or configured executable | `--model` | none | discovered/configured executable | `.codex/skills`, `.codex/agents` |
+| Claude Code | `claude`, `claude-code` | `claude` or configured executable | `--model` | none | configured executable | `.claude/skills`, `.claude/agents` |
+
+The same data can be printed from the CLI:
+
+```powershell
+python -m eval runner-presets
+python -m eval runner-presets --format json
+python -m eval runner-version-check --config config/eval.local.json
+```
+
+## Jenkins Parameters
+
+Jenkins resolves the selected runner from `agent.type` after writing the effective config. Runner-specific behavior is then derived automatically:
+
+- `WORKER_TOOLS` can be empty; Jenkins uses the runner default tool list.
+- `REASONING_VARIANT` is applied only when the runner exposes a variant flag.
+- `RUNNER_MATRIX` is a generic per-worker runner adapter list.
+- `MODEL_MATRIX` is a generic per-worker model route list.
+- `AGENT_ASSET_INSTALLS` is a generic extra asset mapping list.
+
+This prevents runner-specific flags such as OpenCode `--variant` from leaking into Codex CLI and Claude Code runs, while allowing MiMo Code to use `--agent`.
+
+When `RUNNER_MATRIX` is set, Jenkins writes a runner-specific effective config for each worker before `prepare`. The same fetched task/work inputs are reused, but worker 1 can be Codex, worker 2 can be OpenCode, worker 3 can be MiMo Code, and worker 4 can be Claude Code in the same Jenkins build. `MODEL_MATRIX` still lines up by worker index, so `RUNNER_MATRIX=codex,opencode,mimocode,claude` plus `MODEL_MATRIX=agent-inspector/MiniMax-M3,agent-inspector/deepseek-v4-pro,agent-inspector/glm-5.2,agent-inspector/MiniMax-M3` creates four isolated runner/model branches.
+
+Claude Code also has a `--agent` flag, but it selects a named Claude agent rather than a reasoning level. Configure that explicitly in a Claude-specific command when needed instead of using the generic `REASONING_VARIANT` Jenkins parameter.
+
+For Windows Jenkins service deployments, Codex CLI first honors `agent.executable`, `codex.executable`, `execution.codex_executable`, or `execution.codex.executable`, then environment overrides such as `OEVAL_CODEX_EXECUTABLE`, and then auto-discovers the current user's desktop install under `%LOCALAPPDATA%\OpenAI\Codex\bin\*\codex.exe` before falling back to `PATH`. Claude Code can also be configured with the real executable path rather than relying on `PATH`. Use `agent.executable`, `<runner>.executable`, or `execution.<runner>_executable` in a local, uncommitted config. For mixed `RUNNER_MATRIX` jobs, prefer runner-specific executable keys such as `codex.executable` and `claude.executable`; a generic `agent.executable` is only safe for a single-runner job and is intentionally not copied when Jenkins creates a different runner's worker config.
+
+When Claude Code is routed through Agent Inspector, the runner injects Anthropic-compatible environment variables into the isolated worker environment:
+
+- `ANTHROPIC_BASE_URL=<agent-inspector proxy_url>`
+- `ANTHROPIC_API_KEY=<provider apiKey or agent-inspector>`
+- `ANTHROPIC_MODEL=<resolved model>`
+- `CLAUDE_CODE_SUBAGENT_MODEL=<resolved model>`
+
+The runner also writes the same model pin into isolated `.claude/settings.json` and Claude alias model environment variables. This prevents Claude Code subagents from falling back to a default Haiku/Sonnet route when the worker is intentionally using an Inspector model such as `glm-5.2`.
+
+The common `agent-inspector/<model>` model route is passed to Claude Code and Codex CLI as `--model <model>`, for example `agent-inspector/MiniMax-M3` becomes `--model MiniMax-M3`. Codex CLI also normalizes Agent Inspector display names such as `agent-inspector/MiniMax M3` to `--model MiniMax-M3`. For `codex exec`, the harness inserts `--model` before the final prompt argument even when Jenkins uses an `{agent_executable}` placeholder, so the model flag stays in the CLI option area.
+
+For OpenCode and MiMo Code runs, the selected worker model is also written to the worker's isolated runtime config as `model`. MiMo Code additionally pins both the harness-managed `harness-build` agent and the built-in `build` agent to the same route. This keeps subagent/internal sessions aligned with the same Jenkins/Inspector model route without adding `model:` to work-provided agent markdown files or otherwise polluting the reusable work package.
+
+When a run requests `agent-inspector/*`, the harness records an `inspector_route` check in `manifest.json` and `evaluation/evaluation.json`. It compares requested Inspector routing against runtime provider/model evidence from runner logs and Inspector `/api/logs` backfill. A log route such as `mimo/mimo-auto` during an `agent-inspector/glm-5.2` run is reported as `inspector route mismatch`, so aggregate reports show the actual route that escaped Inspector.
+
+All runners write their transcript to `<agent-log-dir>/opencode.log` for compatibility with existing exports. Session summaries are written to `<agent-log-dir>/sessions.json` and `.md`. When Agent Inspector is enabled, the harness also attempts a best-effort `/api/logs` backfill so Inspector evidence remains linked even when a worker times out before exposing a normal session id. Backfill prefers the worker's known `sessionId` or `clientPid` before falling back to model-based queries, stores the full compact log set in `<agent-log-dir>/inspector-logs.json`, and keeps only a preview table in `sessions.md`.
+
+Default runner commands include a shared harness execution contract. Workers receive `AGENT_EVAL_TIMEOUT_MINUTES`, `AGENT_EVAL_IDLE_TIMEOUT_MINUTES`, `AGENT_EVAL_FINALIZE_BEFORE_MINUTES`, and `AGENT_EVAL_REQUIRED_FINAL_ARTIFACTS`, and the prompt asks them to reserve finalization time, write required final artifacts even for blocked or partial runs, avoid UTF-8 BOM output for Windows source edits, and write full helper/finalization output to `work/logs/trace` before previewing saved logs. Keep these requirements when overriding `execution.<runner>_command`.
+
+## Adding Another Runner
+
+Add a new `AgentRunner` implementation under `src/eval/runners/`, register it in `_RUNNER_REGISTRY`, and define:
+
+- `name`, `binary`, and `config_key`
+- `default_command`
+- model and variant injection behavior
+- default asset install targets
+- runtime profile locations
+- log/session parsing
+- `default_worker_tools`
+
+Keep runner-specific config under `agent`, or under a runner-specific section such as `codex`, `mimocode`, `claude`, or a future runner key. Legacy OpenCode keys remain compatibility inputs only.

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@tonyclaw/eval-harness",
+  "version": "0.2.18",
+  "description": "Evaluation harness for configurable runner work packages",
+  "bin": {
+    "eval-harness": "bin/eval-harness.js"
+  },
+  "files": [
+    "bin/",
+    "src/eval/**/*.py",
+    "config/eval.example.json",
+    "schemas/",
+    "prompts/",
+    "docs/runners.md",
+    "docs/agent-runner-compatibility.md",
+    "README.md",
+    "README.zh-CN.md",
+    "pyproject.toml"
+  ],
+  "scripts": {
+    "test": "python -m pytest",
+    "smoke": "node bin/eval-harness.js --help",
+    "pack:dry-run": "npm pack --dry-run"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@gitcode.com/TonyClaw/opencode-eval-harness.git"
+  },
+  "keywords": [
+    "eval",
+    "harness",
+    "agent",
+    "codex",
+    "opencode",
+    "mimocode",
+    "claude-code",
+    "jenkins"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "license": "UNLICENSED"
+}

package/prompts/judge.md

@@ -0,0 +1,27 @@
+# Judge Prompt
+
+You are evaluating an OpenCode run for a configurable design-implementation consistency task.
+
+Use deterministic evidence first. Do not reward chat summaries when required artifacts are missing.
+
+Assess:
+
+- Whether `work/result/output.md` exists and has a final status.
+- Whether `work/reports/FINAL_RESULT.json` reports `is_final: true`.
+- Whether configured protected paths were changed.
+- Whether validation commands passed, failed, or were not run.
+- Whether the implementation diff appears aligned with the task specification instead of public tests only.
+- Whether logs under `work/logs/trace/` are sufficient to audit the run.
+- Which skill, subagent, or helper rule should be improved next.
+
+Return concise JSON plus a short Markdown explanation:
+
+```json
+{
+  "score": 0,
+  "hard_failures": [],
+  "strengths": [],
+  "risks": [],
+  "recommended_skill_changes": []
+}
+```

package/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "eval-harness"
+version = "0.2.18"
+description = "Evaluation harness for configurable runner work packages"
+requires-python = ">=3.10"
+dependencies = []
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+]
+
+[project.optional-dependencies]
+dev = [
+  "coverage[toml]>=7",
+  "pytest>=8",
+  "ruff>=0.6",
+]
+
+[project.scripts]
+eval-harness = "eval.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+
+[tool.coverage.run]
+branch = true
+source = ["eval"]
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F"]

package/schemas/batch-comparison.schema.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Eval Harness Batch Comparison",
+  "type": "object",
+  "required": [
+    "schema",
+    "created_at",
+    "baseline_batch_id",
+    "candidate_batch_id",
+    "regression_tolerance",
+    "baseline_best_score",
+    "candidate_best_score",
+    "best_score_delta",
+    "regression",
+    "regression_reasons"
+  ],
+  "properties": {
+    "schema": {
+      "const": "eval-harness.batch-comparison.v1"
+    },
+    "created_at": {
+      "type": "string"
+    },
+    "baseline_batch_id": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "candidate_batch_id": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "regression_tolerance": {
+      "type": "integer"
+    },
+    "baseline_best_score": {
+      "type": [
+        "integer",
+        "null"
+      ]
+    },
+    "candidate_best_score": {
+      "type": [
+        "integer",
+        "null"
+      ]
+    },
+    "best_score_delta": {
+      "type": [
+        "integer",
+        "null"
+      ]
+    },
+    "regression": {
+      "type": "boolean"
+    },
+    "regression_reasons": {
+      "type": "array"
+    }
+  }
+}

package/schemas/batch-summary.schema.json

@@ -0,0 +1,85 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Eval Harness Batch Summary",
+  "type": "object",
+  "required": [
+    "schema",
+    "batch_id",
+    "created_at",
+    "count",
+    "parallel",
+    "dry_run",
+    "run_ids",
+    "results",
+    "best_by_model",
+    "decision_counts",
+    "recommendation_area_counts",
+    "model_report",
+    "experiment"
+  ],
+  "properties": {
+    "schema": {
+      "const": "eval-harness.batch.v1"
+    },
+    "score_dimensions": {
+      "type": "array"
+    },
+    "batch_id": {
+      "type": "string"
+    },
+    "created_at": {
+      "type": "string"
+    },
+    "count": {
+      "type": "integer"
+    },
+    "parallel": {
+      "type": "integer"
+    },
+    "dry_run": {
+      "type": "boolean"
+    },
+    "run_ids": {
+      "type": "array"
+    },
+    "results": {
+      "type": "array"
+    },
+    "best_run_id": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "best_score": {
+      "type": [
+        "integer",
+        "null"
+      ]
+    },
+    "best_subscores": {
+      "type": "object"
+    },
+    "best_subscore_summary": {
+      "type": "string"
+    },
+    "best_by_model": {
+      "type": "object"
+    },
+    "decision_counts": {
+      "type": "object"
+    },
+    "recommendation_area_counts": {
+      "type": "object"
+    },
+    "model_report": {
+      "type": "object"
+    },
+    "experiment": {
+      "type": "object"
+    },
+    "comparison": {
+      "type": "object"
+    }
+  }
+}

package/schemas/evaluation.schema.json

@@ -0,0 +1,81 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Eval Harness Evaluation",
+  "type": "object",
+  "required": [
+    "schema",
+    "run_id",
+    "score",
+    "subscores",
+    "final_status",
+    "findings",
+    "decision",
+    "recommendations",
+    "next_action"
+  ],
+  "properties": {
+    "schema": {
+      "const": "eval-harness.evaluation.v1"
+    },
+    "run_id": {
+      "type": "string"
+    },
+    "score": {
+      "type": "integer"
+    },
+    "subscores": {
+      "type": "object"
+    },
+    "score_dimensions": {
+      "type": "array"
+    },
+    "semantic_memory": {
+      "type": "object"
+    },
+    "semantic_memory_score": {
+      "type": [
+        "integer",
+        "null"
+      ]
+    },
+    "semantic_memory_summary": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "project_diff_mode": {
+      "type": "string"
+    },
+    "project_diff_baseline": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "final_status": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "findings": {
+      "type": "array"
+    },
+    "decision": {
+      "type": "string"
+    },
+    "recommendations": {
+      "type": "array"
+    },
+    "failure_taxonomy": {
+      "type": "array"
+    },
+    "skill_improvement_input": {
+      "type": "object"
+    },
+    "next_action": {
+      "type": "object"
+    }
+  }
+}

package/schemas/experiment.schema.json

@@ -0,0 +1,63 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Eval Harness Experiment",
+  "type": "object",
+  "required": [
+    "schema",
+    "batch_id",
+    "created_at",
+    "count",
+    "parallel",
+    "dry_run",
+    "models",
+    "run_ids",
+    "run_matrix"
+  ],
+  "properties": {
+    "schema": {
+      "const": "eval-harness.experiment.v1"
+    },
+    "score_dimensions": {
+      "type": "array"
+    },
+    "batch_id": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "created_at": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "count": {
+      "type": [
+        "integer",
+        "null"
+      ]
+    },
+    "parallel": {
+      "type": [
+        "integer",
+        "null"
+      ]
+    },
+    "dry_run": {
+      "type": [
+        "boolean",
+        "null"
+      ]
+    },
+    "models": {
+      "type": "array"
+    },
+    "run_ids": {
+      "type": "array"
+    },
+    "run_matrix": {
+      "type": "array"
+    }
+  }
+}

package/schemas/history-index.schema.json

@@ -0,0 +1,37 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Eval Harness History Index",
+  "type": "object",
+  "required": [
+    "schema",
+    "updated_at",
+    "batch_count",
+    "records"
+  ],
+  "properties": {
+    "schema": {
+      "const": "eval-harness.history-index.v1"
+    },
+    "updated_at": {
+      "type": "string"
+    },
+    "batch_count": {
+      "type": "integer"
+    },
+    "latest": {
+      "type": [
+        "object",
+        "null"
+      ]
+    },
+    "best_overall": {
+      "type": [
+        "object",
+        "null"
+      ]
+    },
+    "records": {
+      "type": "array"
+    }
+  }
+}