@evolvingmachines/evolve 0.0.55-project-sable.20260916.db0bc0bb

package/docs-evals/snippets/list-flags.mdx

@@ -0,0 +1,23 @@
+<ParamField path="-l, --limit" type="n">
+  Page size.
+</ParamField>
+
+<ParamField path="--cursor" type="c">
+  Resume from a page cursor.
+</ParamField>
+
+<ParamField path="--columns" type="keys | all | help">
+  Choose and order columns, comma-separated. `help` lists the keys.
+</ParamField>
+
+<ParamField path="-q, --quiet">
+  Print only ids, one per line, for piping.
+</ParamField>
+
+<ParamField path="--no-trunc">
+  Full cell content instead of one-line truncation.
+</ParamField>
+
+<ParamField path="--no-headers">
+  Omit the header row in piped (TSV) output.
+</ParamField>

package/harness-capabilities.json

@@ -0,0 +1,430 @@
+{
+  "$comment": "Per-harness models and reasoning efforts, checked in so two repos that share no test runner can be held to one table. The evolve SDK owns it: AGENT_REGISTRY in packages/sdk-ts/src/registry.ts is the only place models, efforts, or defaults may change, and scripts/generate-harness-capabilities.ts regenerates this file from it (npm run generate:capabilities; the build fails when it is stale). The hosted-evals dashboard consumes this file for its capability document and its parity test diffs the served document against it, the TypeScript SDK test regenerates these exact bytes from the registry, and the Python SDK test asserts its own Literals cover them. Changing a model or an effort therefore means: registry.ts, regenerate this file, and nothing else.",
+  "reasoningEfforts": [
+    "off",
+    "minimal",
+    "low",
+    "medium",
+    "high",
+    "xhigh",
+    "max",
+    "thinking"
+  ],
+  "binaryEffortValues": [
+    "off",
+    "minimal",
+    "medium",
+    "thinking"
+  ],
+  "defaultReasoningEffort": "medium",
+  "harnesses": {
+    "claude": {
+      "defaultModel": "opus",
+      "models": [
+        {
+          "alias": "fable",
+          "modelId": "claude-fable-5-1",
+          "description": "Highest capability, long-horizon agentic work"
+        },
+        {
+          "alias": "opus",
+          "modelId": "claude-opus-5",
+          "description": "Complex reasoning, R&D, architecting"
+        },
+        {
+          "alias": "sonnet",
+          "modelId": "claude-sonnet-5",
+          "description": "Daily coding, features, tests"
+        },
+        {
+          "alias": "haiku",
+          "modelId": "claude-haiku-4-5-20251001",
+          "description": "Quick tasks, syntax correction"
+        },
+        {
+          "alias": "opus[1m]",
+          "modelId": "opus[1m]",
+          "description": "Complex reasoning with 1M context window"
+        },
+        {
+          "alias": "sonnet[1m]",
+          "modelId": "sonnet[1m]",
+          "description": "Daily coding with 1M context window"
+        },
+        {
+          "alias": "glm-5.3",
+          "modelId": "glm-5.3",
+          "description": "Zhipu GLM-5.3 via the Evolve gateway"
+        },
+        {
+          "alias": "glm-5.3-flash",
+          "modelId": "glm-5.3-flash",
+          "description": "Zhipu GLM-5.3 Flash via the Evolve gateway"
+        },
+        {
+          "alias": "openrouter/deepseek/deepseek-v4.1-flash",
+          "modelId": "openrouter/deepseek/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via OpenRouter"
+        },
+        {
+          "alias": "fireworks/deepseek-v4.1-flash",
+          "modelId": "fireworks/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via Fireworks"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": true,
+      "presets": [
+        "no-internet",
+        "pinned-context"
+      ]
+    },
+    "codex": {
+      "defaultModel": "gpt-5.6-sol",
+      "models": [
+        {
+          "alias": "gpt-6-astra",
+          "modelId": "gpt-6-astra",
+          "description": "Newest frontier flagship"
+        },
+        {
+          "alias": "gpt-5.6-sol",
+          "modelId": "gpt-5.6-sol",
+          "description": "GPT-5.6 flagship (previous generation)"
+        },
+        {
+          "alias": "gpt-5.6-terra",
+          "modelId": "gpt-5.6-terra",
+          "description": "Balances intelligence and cost"
+        },
+        {
+          "alias": "gpt-5.6-luna",
+          "modelId": "gpt-5.6-luna",
+          "description": "High-volume, cost-sensitive tier"
+        },
+        {
+          "alias": "gpt-5.5",
+          "modelId": "gpt-5.5",
+          "description": "GPT-5.5 frontier model (two generations back)"
+        },
+        {
+          "alias": "gpt-5.3-codex",
+          "modelId": "gpt-5.3-codex",
+          "description": "Industry-leading code-optimized"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": true,
+      "presets": [
+        "no-internet",
+        "pinned-context"
+      ]
+    },
+    "droid": {
+      "defaultModel": "claude-opus-5",
+      "models": [
+        {
+          "alias": "claude-fable-5.1",
+          "modelId": "claude-fable-5-1",
+          "description": "Factory-managed Claude Fable 5.1"
+        },
+        {
+          "alias": "claude-opus-5",
+          "modelId": "claude-opus-5",
+          "description": "Factory-managed Claude Opus 5"
+        },
+        {
+          "alias": "claude-sonnet-5",
+          "modelId": "claude-sonnet-5",
+          "description": "Factory-managed Claude Sonnet 5"
+        },
+        {
+          "alias": "claude-haiku-4-5",
+          "modelId": "claude-haiku-4-5-20251001",
+          "description": "Factory-managed Claude Haiku 4.5"
+        },
+        {
+          "alias": "gpt-6-astra",
+          "modelId": "gpt-6-astra",
+          "description": "Factory-managed GPT-6 Astra"
+        },
+        {
+          "alias": "gpt-5.6-sol",
+          "modelId": "gpt-5.6-sol",
+          "description": "Factory-managed GPT-5.6 Sol"
+        },
+        {
+          "alias": "gpt-5.6-terra",
+          "modelId": "gpt-5.6-terra",
+          "description": "Factory-managed GPT-5.6 Terra"
+        },
+        {
+          "alias": "gpt-5.6-luna",
+          "modelId": "gpt-5.6-luna",
+          "description": "Factory-managed GPT-5.6 Luna"
+        },
+        {
+          "alias": "gemini-3.6-flash",
+          "modelId": "gemini-3.6-flash",
+          "description": "Factory-managed Gemini 3.6 Flash"
+        },
+        {
+          "alias": "qwen3.7-max",
+          "modelId": "qwen3.7-max",
+          "description": "Qwen 3.7 Max via the Evolve gateway"
+        },
+        {
+          "alias": "kimi-k3",
+          "modelId": "kimi-k3",
+          "description": "Factory-managed Droid Core Kimi K3"
+        },
+        {
+          "alias": "glm-5.3",
+          "modelId": "glm-5.3",
+          "description": "Zhipu GLM-5.3 via the Evolve gateway"
+        },
+        {
+          "alias": "glm-5.3-flash",
+          "modelId": "glm-5.3-flash",
+          "description": "Zhipu GLM-5.3 Flash via the Evolve gateway"
+        },
+        {
+          "alias": "openrouter/deepseek/deepseek-v4.1-flash",
+          "modelId": "openrouter/deepseek/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via OpenRouter"
+        },
+        {
+          "alias": "fireworks/deepseek-v4.1-flash",
+          "modelId": "fireworks/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via Fireworks"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": false,
+      "presets": []
+    },
+    "gemini": {
+      "defaultModel": "gemini-3.5-flash",
+      "models": [
+        {
+          "alias": "gemini-3.5-flash",
+          "modelId": "gemini-3.5-flash",
+          "description": "Latest flash the stable gemini CLI serves: coding + agentic planning"
+        },
+        {
+          "alias": "gemini-3.5-flash-lite",
+          "modelId": "gemini-3.5-flash-lite",
+          "description": "Most cost-effective 3.5-class model"
+        },
+        {
+          "alias": "gemini-3.1-pro-preview",
+          "modelId": "gemini-3.1-pro-preview",
+          "description": "Latest pro, complex agentic + coding"
+        }
+      ],
+      "effortSupport": "none",
+      "efforts": [],
+      "defaultEffort": null,
+      "supportsConfig": false,
+      "presets": []
+    },
+    "kimi": {
+      "defaultModel": "kimi-k3",
+      "models": [
+        {
+          "alias": "kimi-k3",
+          "modelId": "moonshot/kimi-k3",
+          "description": "Latest flagship: 1M context, always-on thinking",
+          "maxContextSize": 1048576
+        },
+        {
+          "alias": "kimi-k2.7-code",
+          "modelId": "moonshot/kimi-k2.7-code",
+          "description": "Latest coding-specialized standard model"
+        },
+        {
+          "alias": "kimi-k3-raptor",
+          "modelId": "kimi-k3-raptor",
+          "description": "Evolve-managed Kimi K3 Raptor route for latency-sensitive agent runs",
+          "maxContextSize": 1048576
+        },
+        {
+          "alias": "kimi-k2p7-code-raptor",
+          "modelId": "kimi-k2p7-code-raptor",
+          "description": "Evolve-managed Kimi K2.7 Code Raptor route for latency-sensitive agent runs"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "max",
+      "supportsConfig": false,
+      "presets": []
+    },
+    "opencode": {
+      "defaultModel": "openrouter/anthropic/claude-opus-5",
+      "models": [
+        {
+          "alias": "openrouter/anthropic/claude-fable-5.1",
+          "modelId": "openrouter/anthropic/claude-fable-5.1",
+          "description": "Anthropic Fable 5.1 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-opus-5",
+          "modelId": "openrouter/anthropic/claude-opus-5",
+          "description": "Anthropic Opus 5 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-sonnet-5",
+          "modelId": "openrouter/anthropic/claude-sonnet-5",
+          "description": "Anthropic Sonnet 5 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/anthropic/claude-haiku-4.5",
+          "modelId": "openrouter/anthropic/claude-haiku-4.5",
+          "description": "Anthropic Haiku via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-6-astra",
+          "modelId": "openrouter/openai/gpt-6-astra",
+          "description": "OpenAI GPT-6 Astra via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-sol",
+          "modelId": "openrouter/openai/gpt-5.6-sol",
+          "description": "OpenAI GPT-5.6 Sol via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-terra",
+          "modelId": "openrouter/openai/gpt-5.6-terra",
+          "description": "OpenAI GPT-5.6 Terra via OpenRouter"
+        },
+        {
+          "alias": "openrouter/openai/gpt-5.6-luna",
+          "modelId": "openrouter/openai/gpt-5.6-luna",
+          "description": "OpenAI GPT-5.6 Luna via OpenRouter"
+        },
+        {
+          "alias": "openrouter/google/gemini-3.6-flash",
+          "modelId": "openrouter/google/gemini-3.6-flash",
+          "description": "Gemini 3.6 Flash via OpenRouter"
+        },
+        {
+          "alias": "openrouter/qwen/qwen3.7-max",
+          "modelId": "openrouter/qwen/qwen3.7-max",
+          "description": "Qwen 3.7 Max via OpenRouter"
+        },
+        {
+          "alias": "openrouter/moonshotai/kimi-k3",
+          "modelId": "openrouter/moonshotai/kimi-k3",
+          "description": "Kimi K3 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/z-ai/glm-5.3",
+          "modelId": "openrouter/z-ai/glm-5.3",
+          "description": "Zhipu GLM-5.3 via OpenRouter"
+        },
+        {
+          "alias": "openrouter/z-ai/glm-5.3-flash",
+          "modelId": "openrouter/z-ai/glm-5.3-flash",
+          "description": "Zhipu GLM-5.3 Flash (OpenRouter id; the Evolve gateway serves it from Fireworks)"
+        },
+        {
+          "alias": "openrouter/deepseek/deepseek-v4.1-flash",
+          "modelId": "openrouter/deepseek/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via OpenRouter"
+        },
+        {
+          "alias": "fireworks/deepseek-v4.1-flash",
+          "modelId": "fireworks/deepseek-v4.1-flash",
+          "description": "DeepSeek V4.1 Flash via Fireworks"
+        }
+      ],
+      "effortSupport": "level",
+      "efforts": [
+        "off",
+        "minimal",
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max",
+        "thinking"
+      ],
+      "defaultEffort": "high",
+      "supportsConfig": false,
+      "presets": []
+    },
+    "qwen": {
+      "defaultModel": "qwen3.7-max",
+      "models": [
+        {
+          "alias": "qwen3.7-max",
+          "modelId": "qwen3.7-max",
+          "description": "Strongest reasoning and coding option"
+        },
+        {
+          "alias": "qwen3.7-plus",
+          "modelId": "qwen3.7-plus",
+          "description": "Latest balanced Qwen Cloud recommendation"
+        },
+        {
+          "alias": "qwen3.6-flash",
+          "modelId": "qwen3.6-flash",
+          "description": "Fast and cost-effective option"
+        }
+      ],
+      "effortSupport": "binary",
+      "efforts": [
+        "off",
+        "minimal",
+        "medium",
+        "thinking"
+      ],
+      "defaultEffort": "thinking",
+      "supportsConfig": false,
+      "presets": []
+    }
+  }
+}

package/hosted-error-codes.json

@@ -0,0 +1,120 @@
+{
+  "$comment": "The hosted evals API's error-code vocabulary, checked in so two repos that share no test runner can be held to one list. The contract owns it: the ErrorCode enum in spec/openapi.yaml is the only place a code may be added. This file is that enum's shadow, and the tests keep everyone honest — the server regenerates these exact bytes and diffs them, the TypeScript SDK asserts HOSTED_ERROR_CODES equals codes[], and the Python SDK asserts both its tuple and its Literal do. Adding a code therefore means: spec enum, this file, both SDK lists.",
+  "codes": [
+    "missing_authorization",
+    "invalid_api_key",
+    "read_only_key",
+    "credential_service_unavailable",
+    "rate_limited",
+    "insufficient_credits",
+    "quota_exceeded",
+    "invalid_json",
+    "invalid_input",
+    "invalid_limit",
+    "invalid_status",
+    "invalid_visibility",
+    "invalid_cursor",
+    "invalid_after",
+    "invalid_format",
+    "invalid_ids",
+    "invalid_multipart",
+    "idempotency_key_reused",
+    "dataset_not_found",
+    "dataset_version_not_found",
+    "dataset_name_taken",
+    "dataset_in_use",
+    "dataset_not_owned",
+    "dataset_import_in_progress",
+    "upstream_not_watchable",
+    "no_active_version",
+    "version_not_ready",
+    "version_not_activatable",
+    "unknown_task_names",
+    "no_tasks",
+    "task_not_found",
+    "task_failed_to_build",
+    "upload_session_not_found",
+    "upload_offset_mismatch",
+    "upload_chunk_digest_mismatch",
+    "upload_incomplete",
+    "upload_archive_digest_mismatch",
+    "upload_session_failed",
+    "too_many_concurrent_upload_chunks",
+    "agent_not_found",
+    "agent_name_taken",
+    "agent_name_reserved",
+    "agent_invalid_name",
+    "agent_source_required",
+    "agent_source_conflict",
+    "agent_invalid_env",
+    "agent_too_large",
+    "agent_limit_reached",
+    "skill_not_found",
+    "skill_name_not_found",
+    "skill_ref_invalid",
+    "skill_unresolvable",
+    "skill_invalid",
+    "skill_in_use",
+    "skill_too_large",
+    "skill_limit_reached",
+    "too_many_concurrent_skill_uploads",
+    "secret_not_found",
+    "secret_ambiguous",
+    "secret_brokered_unsupported",
+    "secret_exists",
+    "secret_not_attached",
+    "agent_version_not_found",
+    "agent_version_unresolvable",
+    "agent_kwarg_unsupported",
+    "agent_config_unsupported",
+    "agent_config_key_refused",
+    "agent_preset_unsupported",
+    "provider_unsupported",
+    "job_not_found",
+    "job_not_terminal",
+    "no_failed_trials",
+    "trial_not_found",
+    "trial_not_settled",
+    "concurrent_update",
+    "regrade_source_ineligible",
+    "no_regradable_trials",
+    "invalid_rubric",
+    "analysis_already_running",
+    "analysis_not_found",
+    "analysis_not_terminal",
+    "check_not_found",
+    "check_not_terminal",
+    "no_checkable_tasks",
+    "too_many_concurrent_check_uploads",
+    "no_analyzable_trials",
+    "not_a_job_dir",
+    "invalid_trial",
+    "trial_too_large",
+    "upload_too_large",
+    "job_uploaded",
+    "job_already_uploaded",
+    "too_many_concurrent_job_uploads",
+    "job_import_not_found",
+    "import_not_found",
+    "import_too_large",
+    "too_many_concurrent_imports",
+    "invalid_archive",
+    "unpinned_git_ref",
+    "hub_package_not_found",
+    "hub_unreachable",
+    "package_not_retained",
+    "package_corrupt",
+    "package_missing",
+    "too_many_concurrent_package_downloads",
+    "org_not_found",
+    "org_slug_taken",
+    "org_forbidden",
+    "org_personal_immutable",
+    "org_last_owner",
+    "org_in_use",
+    "org_member_not_found",
+    "invite_not_found",
+    "invite_invalid",
+    "internal_error"
+  ]
+}