@lmmslab/harness 0.0.0-stage → 0.81.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1356 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +232 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
"""Multi-turn text completion for the high level.
|
|
2
|
+
|
|
3
|
+
Chat Completions covers OpenRouter and self-hosted OpenAI-compatible servers
|
|
4
|
+
(vLLM, SGLang) through `openai_chat` with a `base_url`. Messages use the
|
|
5
|
+
harness content parts (`input_text`, `input_image`).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import time
|
|
14
|
+
from urllib.error import HTTPError
|
|
15
|
+
from urllib.parse import urlsplit
|
|
16
|
+
|
|
17
|
+
from harness.core.http import open_url, post_json, read_sse, redact
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# The usage of model calls, one call's (ChatReply.usage, with its finish_reason) or a
|
|
21
|
+
# total of several (an episode's summary, a vision model's, which add a count of calls):
|
|
22
|
+
# tokens in and out, and the cost in US dollars when the provider reports it.
|
|
23
|
+
USAGE = {"input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def add_usage(total: dict, usage: dict) -> None:
|
|
27
|
+
"""Add `usage` (one call's or a total) to `total`, key by key of USAGE."""
|
|
28
|
+
for key, zero in USAGE.items():
|
|
29
|
+
total[key] = total.get(key, zero) + type(zero)(usage.get(key) or zero)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class ChatReply:
|
|
34
|
+
"""One completion: its text, usage (USAGE and finish_reason; cost_usd None when the
|
|
35
|
+
provider does not report it), the provider's response id, and the seconds the request
|
|
36
|
+
took with its retries."""
|
|
37
|
+
|
|
38
|
+
text: str
|
|
39
|
+
usage: dict = field(default_factory=dict)
|
|
40
|
+
response_id: str | None = None
|
|
41
|
+
seconds: float = 0.0
|
|
42
|
+
# Reasoning text or summary, when the provider returns one (e.g. DeepSeek, GPT-6
|
|
43
|
+
# Luna on OpenRouter); for the trace only, never sent back to the model.
|
|
44
|
+
reasoning: str = ""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# The keys of a model role (configs/models/<name>.toml: [main], [vlm], [pointer],
|
|
48
|
+
# [verifier]). Documentation; ChatModel reads them.
|
|
49
|
+
MODEL_KEYS = {
|
|
50
|
+
"service": "a service of the deploy instead of a hosted model: its client config, with this role's keys on top",
|
|
51
|
+
"backend": "openrouter (default) or openai_chat (any OpenAI-compatible server)",
|
|
52
|
+
"model": "the model id",
|
|
53
|
+
"base_url": "openai_chat: the server's URL, e.g. http://127.0.0.1:8800/v1",
|
|
54
|
+
"base_url_env": "openai_chat: environment variable holding the server URL; overrides base_url when set",
|
|
55
|
+
"api_key_env": "the environment variable that holds the key (default OPENROUTER_API_KEY for openrouter)",
|
|
56
|
+
"reasoning_effort": "the model provider’s reasoning option ID (effort_fields); omit to use its default",
|
|
57
|
+
"max_output_tokens": "tokens per reply, reasoning included on OpenRouter (default 8192)",
|
|
58
|
+
"max_tokens_key": "the request field of that limit (default max_tokens; OpenAI's newer models take "
|
|
59
|
+
"max_completion_tokens)",
|
|
60
|
+
"temperature": "sampling temperature (default: the provider's)",
|
|
61
|
+
"stream": "stream the reply: a dead connection shows within the read timeout while a model thinks",
|
|
62
|
+
"timeout_seconds": "read timeout of one request (default 300)",
|
|
63
|
+
"retries": "retries of a failed request (default 2)",
|
|
64
|
+
"retry_seconds": "keep retrying transient failures until this long after the request started",
|
|
65
|
+
"extra_body": "extra request fields for OpenAI-compatible servers (e.g. max_image_side)",
|
|
66
|
+
"images": "false for a model that reads no images: none is sent to it (default: it reads them)",
|
|
67
|
+
"on_demand": "written by the resolution for an on-demand service: the broker starts it first",
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# Thinking tokens for providers that take a budget instead of a level.
|
|
72
|
+
BUDGETS = {"low": 2000, "medium": 8000, "high": 24000}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def valid_effort(value):
|
|
76
|
+
return isinstance(value, str) and (not value or re.fullmatch(r"[A-Za-z0-9_-]{1,64}", value) is not None)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def effort_fields(backend: str, base_url, model: str, effort, max_tokens: int) -> dict:
|
|
80
|
+
"""The request fields that set `effort` (none, low, medium, high) for this provider,
|
|
81
|
+
each in the field its API reads; {} when unset. OpenAI, Gemini, Ollama and other
|
|
82
|
+
OpenAI-compatible servers take OpenAI's reasoning_effort."""
|
|
83
|
+
if not effort:
|
|
84
|
+
return {}
|
|
85
|
+
if backend == "openrouter":
|
|
86
|
+
return {"reasoning": {"enabled": False} if effort == "none" else {"effort": effort}}
|
|
87
|
+
host = str(base_url or "")
|
|
88
|
+
budget = max(1024, min(BUDGETS.get(effort, 0), max_tokens - 1024))
|
|
89
|
+
if "anthropic.com" in host:
|
|
90
|
+
# Its OpenAI-compatible endpoint ignores reasoning_effort; thinking takes a budget.
|
|
91
|
+
return {"thinking": {"type": "disabled"} if effort == "none" else {"type": "enabled", "budget_tokens": budget}}
|
|
92
|
+
if "dashscope" in host:
|
|
93
|
+
return {"enable_thinking": False} if effort == "none" else {"enable_thinking": True, "thinking_budget": budget}
|
|
94
|
+
if "bigmodel.cn" in host or "z.ai" in host:
|
|
95
|
+
return {"thinking": {"type": "disabled" if effort == "none" else "enabled"}}
|
|
96
|
+
if "deepseek.com" in host:
|
|
97
|
+
# Levels low, high and max; thinking is a switch of its own.
|
|
98
|
+
return {"thinking": {"type": "disabled"}} if effort == "none" else {
|
|
99
|
+
"thinking": {"type": "enabled"}, "reasoning_effort": "low" if effort == "low" else "high"}
|
|
100
|
+
if "moonshot" in host:
|
|
101
|
+
if "k3" in model:
|
|
102
|
+
return {} if effort == "none" else {"reasoning_effort": "low" if effort == "low" else "high"}
|
|
103
|
+
return {"thinking": {"type": "disabled" if effort == "none" else "enabled"}}
|
|
104
|
+
if "x.ai" in host:
|
|
105
|
+
# Grok takes low or high, on the models that accept it.
|
|
106
|
+
return {} if effort == "none" else {"reasoning_effort": "low" if effort == "low" else "high"}
|
|
107
|
+
return {"reasoning_effort": effort}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
CHAT_COMPLETIONS = "/chat/completions"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def chat_url(provider, base_url):
|
|
114
|
+
"""The Chat Completions URL of a model config (backend, base_url)."""
|
|
115
|
+
if provider == "openrouter" and not base_url:
|
|
116
|
+
return "https://openrouter.ai/api/v1/chat/completions"
|
|
117
|
+
if not base_url:
|
|
118
|
+
raise ValueError("openai_chat requires base_url")
|
|
119
|
+
root = base_url.rstrip("/")
|
|
120
|
+
if root.endswith(CHAT_COMPLETIONS):
|
|
121
|
+
return root
|
|
122
|
+
return root + CHAT_COMPLETIONS
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def chat_part(part):
|
|
126
|
+
if part["type"] == "input_text":
|
|
127
|
+
return {"type": "text", "text": part["text"]}
|
|
128
|
+
if part["type"] == "input_image":
|
|
129
|
+
return {"type": "image_url", "image_url": {"url": part["image_url"]}}
|
|
130
|
+
raise ValueError(f"Unsupported content part: {part['type']}")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class ChatModel:
|
|
134
|
+
"""A Chat Completions client for one model role (MODEL_KEYS): OpenRouter or an
|
|
135
|
+
OpenAI-compatible server. The openrouter backend needs its key in the environment;
|
|
136
|
+
`transport` replaces the HTTP call (tests)."""
|
|
137
|
+
|
|
138
|
+
def __init__(self, config: dict, transport=None):
|
|
139
|
+
self.provider = config.get("backend", "openrouter")
|
|
140
|
+
if self.provider not in {"openrouter", "openai_chat"}:
|
|
141
|
+
raise ValueError("The program harness supports the openrouter and openai_chat backends")
|
|
142
|
+
self.model = config["model"]
|
|
143
|
+
# A URL in base_url_env is resolved with the agent (config/agent.py _role).
|
|
144
|
+
self.base_url = config.get("base_url")
|
|
145
|
+
if self.provider == "openai_chat" and not self.base_url:
|
|
146
|
+
named = config.get("base_url_env")
|
|
147
|
+
raise ValueError(f"openai_chat requires base_url.{f' Set {named}.' if named else ''}")
|
|
148
|
+
# A self-hosted model placed on demand is started by the deploy's broker first.
|
|
149
|
+
self.on_demand = config.get("on_demand")
|
|
150
|
+
self.url = chat_url(self.provider, self.base_url)
|
|
151
|
+
self.timeout = float(config.get("timeout_seconds", 300.0))
|
|
152
|
+
self.max_output_tokens = int(config.get("max_output_tokens", 8192))
|
|
153
|
+
self.max_tokens_key = str(config.get("max_tokens_key") or "max_tokens")
|
|
154
|
+
self.reasoning_effort = config.get("reasoning_effort")
|
|
155
|
+
self.temperature = config.get("temperature")
|
|
156
|
+
self.retries = int(config.get("retries", 2))
|
|
157
|
+
# Keep retrying transient failures until this many seconds have passed since the
|
|
158
|
+
# request started (at least `retries` times). The simulator does not advance while
|
|
159
|
+
# the model is being called, so waiting out an outage costs wall time only.
|
|
160
|
+
self.retry_seconds = config.get("retry_seconds")
|
|
161
|
+
self.retry_seconds = None if self.retry_seconds is None else float(self.retry_seconds)
|
|
162
|
+
# Extra request fields for OpenAI-compatible servers (e.g. max_image_side for serve_vlm.py).
|
|
163
|
+
self.extra_body = dict(config.get("extra_body") or {})
|
|
164
|
+
# Streaming keeps bytes flowing while a reasoning model thinks; some networks reset
|
|
165
|
+
# a connection that stays silent for minutes waiting for a non-streamed reply.
|
|
166
|
+
self.stream = bool(config.get("stream", False))
|
|
167
|
+
self.transport = transport or (sse_transport if self.stream else urllib_transport)
|
|
168
|
+
key_env = config.get("api_key_env") or ("OPENROUTER_API_KEY" if self.provider == "openrouter" else None)
|
|
169
|
+
self._api_key = os.environ.get(key_env) if key_env else None
|
|
170
|
+
if self.provider == "openrouter" and not self._api_key:
|
|
171
|
+
raise ValueError(f"Set {key_env} before selecting the openrouter backend")
|
|
172
|
+
self._api_key = self._api_key or "EMPTY"
|
|
173
|
+
|
|
174
|
+
def payload(self, system: str, messages: list[dict]) -> dict:
|
|
175
|
+
"""The request body: `system` and the harness messages as chat messages, the output
|
|
176
|
+
limit and extra_body fields, and on OpenRouter usage accounting and reasoning."""
|
|
177
|
+
# Some models (Molmo2) take no system message; an empty one is left out.
|
|
178
|
+
converted = [{"role": "system", "content": system}] if system else []
|
|
179
|
+
for message in messages:
|
|
180
|
+
content = message["content"]
|
|
181
|
+
if isinstance(content, str):
|
|
182
|
+
converted.append({"role": message["role"], "content": content})
|
|
183
|
+
else:
|
|
184
|
+
converted.append({"role": message["role"],
|
|
185
|
+
"content": [chat_part(part) for part in content]})
|
|
186
|
+
payload = {"model": self.model, "messages": converted, self.max_tokens_key: self.max_output_tokens,
|
|
187
|
+
**self.extra_body}
|
|
188
|
+
if self.stream:
|
|
189
|
+
payload["stream"] = True
|
|
190
|
+
if self.temperature is not None:
|
|
191
|
+
payload["temperature"] = self.temperature
|
|
192
|
+
if self.provider == "openrouter":
|
|
193
|
+
payload["usage"] = {"include": True}
|
|
194
|
+
payload.update(effort_fields(self.provider, self.base_url, self.model, self.reasoning_effort,
|
|
195
|
+
self.max_output_tokens))
|
|
196
|
+
return payload
|
|
197
|
+
|
|
198
|
+
@property
|
|
199
|
+
def streams_text(self) -> bool:
|
|
200
|
+
"""Whether complete() can hand over the reply's text as it streams (on_text)."""
|
|
201
|
+
return self.stream and self.transport is sse_transport
|
|
202
|
+
|
|
203
|
+
def complete(self, system: str, messages: list[dict], on_text=None, *, cancellation=None) -> ChatReply:
|
|
204
|
+
"""One completion of `messages` under `system`, starting an on-demand service through
|
|
205
|
+
the broker first. Failed requests are retried (`retries` times, or until
|
|
206
|
+
`retry_seconds` have passed), except client errors other than HTTP 429; the last
|
|
207
|
+
failure is raised as a RuntimeError. When it streams (streams_text), `on_text(kind,
|
|
208
|
+
text)` gets each piece of the reply ("text") and of its reasoning ("reasoning") as
|
|
209
|
+
it comes. A cancellation closes built-in transports and stops retries."""
|
|
210
|
+
if cancellation is not None:
|
|
211
|
+
cancellation.check()
|
|
212
|
+
if self.on_demand:
|
|
213
|
+
from harness.services.broker import ensure
|
|
214
|
+
|
|
215
|
+
ensure({"on_demand": self.on_demand})
|
|
216
|
+
payload = self.payload(system, messages)
|
|
217
|
+
headers = {"authorization": f"Bearer {self._api_key}"}
|
|
218
|
+
# Explicit OpenRouter backends include relays; compatible clients can also
|
|
219
|
+
# target OpenRouter directly. Match the hostname, never URL substrings.
|
|
220
|
+
if self.provider == "openrouter" or urlsplit(self.url).hostname == "openrouter.ai":
|
|
221
|
+
headers.update({"http-referer": "https://harness.lmms-lab.com/",
|
|
222
|
+
"x-openrouter-title": "the-harness"})
|
|
223
|
+
started = time.perf_counter()
|
|
224
|
+
last_error = None
|
|
225
|
+
# A completion has no side effects, so transient provider failures may be retried.
|
|
226
|
+
attempt = 0
|
|
227
|
+
while True:
|
|
228
|
+
if attempt:
|
|
229
|
+
waited = time.perf_counter() - started
|
|
230
|
+
if attempt > self.retries and (self.retry_seconds is None or waited >= self.retry_seconds):
|
|
231
|
+
raise last_error
|
|
232
|
+
delay = min(2 ** (attempt - 1) * 5, 30 if self.retry_seconds is None else 60)
|
|
233
|
+
if cancellation is None:
|
|
234
|
+
time.sleep(delay)
|
|
235
|
+
else:
|
|
236
|
+
cancellation.event.wait(delay)
|
|
237
|
+
if cancellation is not None:
|
|
238
|
+
cancellation.check()
|
|
239
|
+
attempt += 1
|
|
240
|
+
try:
|
|
241
|
+
body = self.transport(self.url, headers, payload, self.timeout,
|
|
242
|
+
**({"cancellation": cancellation} if cancellation is not None
|
|
243
|
+
and self.transport in (sse_transport, urllib_transport) else {}),
|
|
244
|
+
**({"on_text": on_text} if on_text is not None and self.streams_text else {}))
|
|
245
|
+
except Exception as error:
|
|
246
|
+
if cancellation is not None:
|
|
247
|
+
cancellation.check()
|
|
248
|
+
last_error = RuntimeError(redact(str(error), self._api_key))
|
|
249
|
+
if "Provider HTTP 4" in str(error) and "HTTP 429" not in str(error):
|
|
250
|
+
raise last_error from None # a client error will not pass on retry
|
|
251
|
+
continue
|
|
252
|
+
if isinstance(body, dict) and body.get("error"):
|
|
253
|
+
detail = body["error"]
|
|
254
|
+
text = detail.get("message", str(detail)) if isinstance(detail, dict) else str(detail)
|
|
255
|
+
last_error = RuntimeError(f"{self.provider} error: {redact(text, self._api_key)}")
|
|
256
|
+
continue
|
|
257
|
+
choices = body.get("choices") or []
|
|
258
|
+
if len(choices) != 1:
|
|
259
|
+
raise ValueError(f"Expected one completion choice, got {len(choices)}")
|
|
260
|
+
message = choices[0].get("message") or {}
|
|
261
|
+
usage = body.get("usage") or {}
|
|
262
|
+
return ChatReply(
|
|
263
|
+
text=(message.get("content") or "") + _tool_blocks(message.get("tool_calls")),
|
|
264
|
+
reasoning=message.get("reasoning") or "",
|
|
265
|
+
usage={"input_tokens": usage.get("prompt_tokens", 0),
|
|
266
|
+
"output_tokens": usage.get("completion_tokens", 0),
|
|
267
|
+
"cost_usd": usage.get("cost"),
|
|
268
|
+
"finish_reason": choices[0].get("finish_reason")},
|
|
269
|
+
response_id=body.get("id"),
|
|
270
|
+
seconds=time.perf_counter() - started,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
# DeepSeek models sometimes answer with a tool call of the provider's own where the text
|
|
275
|
+
# should hold the block (finish_reason "tool_calls", no tools offered): on agent 1.51's
|
|
276
|
+
# Minecraft practice (accept8, seed 0) two calls ran nothing so, one reply prose only and
|
|
277
|
+
# one empty. A call named for a block (python, or a name with python, code or exec in it)
|
|
278
|
+
# is read as that block, its program the one text argument.
|
|
279
|
+
_TOOL_KINDS = ("python", "verify", "finish", "action", "wait")
|
|
280
|
+
_TOOL_ARGUMENTS = ("code", "program", "source", "script", "python", "input", "content", "text", "claim", "result")
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _tool_blocks(calls) -> str:
|
|
284
|
+
"""Native tool calls of a reply as the blocks they name, after its text."""
|
|
285
|
+
blocks = []
|
|
286
|
+
for call in calls or []:
|
|
287
|
+
function = (call or {}).get("function") or {}
|
|
288
|
+
name = str(function.get("name") or "").strip()
|
|
289
|
+
body = function.get("arguments")
|
|
290
|
+
if isinstance(body, str):
|
|
291
|
+
try:
|
|
292
|
+
body = json.loads(body)
|
|
293
|
+
except ValueError:
|
|
294
|
+
pass
|
|
295
|
+
if isinstance(body, dict):
|
|
296
|
+
texts = [body[key] for key in _TOOL_ARGUMENTS if isinstance(body.get(key), str)]
|
|
297
|
+
others = [value for value in body.values() if isinstance(value, str)]
|
|
298
|
+
body = texts[0] if texts else others[0] if len(others) == 1 else json.dumps(body)
|
|
299
|
+
kind = (name if name in _TOOL_KINDS else
|
|
300
|
+
"python" if re.search(r"python|code|exec", name, re.IGNORECASE) else name or "python")
|
|
301
|
+
program = str(body if body is not None else "").strip("\n")
|
|
302
|
+
blocks.append(f"\n<{kind}>\n{program}\n</{kind}>")
|
|
303
|
+
return "".join(blocks)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def urllib_transport(url, headers, payload, timeout, *, cancellation=None):
|
|
307
|
+
"""POST a chat completion and return the provider's response."""
|
|
308
|
+
with _provider_errors():
|
|
309
|
+
return post_json(url, payload, headers, timeout, **({"cancellation": cancellation} if cancellation else {}))
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def sse_transport(url, headers, payload, timeout, on_text=None, *, cancellation=None):
|
|
313
|
+
"""POST a streamed chat completion and assemble the chunks into the non-streamed
|
|
314
|
+
response shape. `timeout` bounds each read, so keep-alive comments reset it.
|
|
315
|
+
`on_text(kind, text)` gets each piece as it comes (ChatModel.complete)."""
|
|
316
|
+
options = {"cancellation": cancellation} if cancellation is not None else {}
|
|
317
|
+
with _provider_errors(), open_url(url, payload, headers, timeout, **options) as response:
|
|
318
|
+
return _read_sse(response, on_text)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
@contextmanager
|
|
322
|
+
def _provider_errors():
|
|
323
|
+
"""A failed request as a RuntimeError that says what the provider answered."""
|
|
324
|
+
try:
|
|
325
|
+
yield
|
|
326
|
+
except HTTPError as error:
|
|
327
|
+
detail = error.read().decode("utf-8", "replace")[:500]
|
|
328
|
+
raise RuntimeError(f"Provider HTTP {error.code}: {detail}") from error
|
|
329
|
+
except OSError as error:
|
|
330
|
+
raise RuntimeError(f"Provider request failed: {getattr(error, 'reason', error)}") from error
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _read_sse(lines, on_text=None):
|
|
334
|
+
response_id, usage = None, None
|
|
335
|
+
content, reasoning = [], []
|
|
336
|
+
finish = None
|
|
337
|
+
# Native tool calls, by index: their name and the pieces of their arguments.
|
|
338
|
+
tools = {}
|
|
339
|
+
for event, data in read_sse(lines):
|
|
340
|
+
if event is None:
|
|
341
|
+
# ": keep-alive" comments, which a provider sends while the model has not
|
|
342
|
+
# begun: on_text hears of them as an empty piece, so a caller can cut off a
|
|
343
|
+
# reply that has not started (runtime/conversation.py).
|
|
344
|
+
if on_text is not None:
|
|
345
|
+
on_text("", "")
|
|
346
|
+
continue
|
|
347
|
+
data = data.strip()
|
|
348
|
+
if data == "[DONE]":
|
|
349
|
+
break
|
|
350
|
+
chunk = json.loads(data)
|
|
351
|
+
if chunk.get("error"):
|
|
352
|
+
return {"error": chunk["error"]}
|
|
353
|
+
response_id = response_id or chunk.get("id")
|
|
354
|
+
usage = chunk.get("usage") or usage
|
|
355
|
+
for choice in chunk.get("choices") or []:
|
|
356
|
+
delta = choice.get("delta") or {}
|
|
357
|
+
content.append(delta.get("content") or "")
|
|
358
|
+
reasoning.append(delta.get("reasoning") or "")
|
|
359
|
+
for piece in delta.get("tool_calls") or []:
|
|
360
|
+
tool = tools.setdefault(piece.get("index", 0), {"name": "", "arguments": ""})
|
|
361
|
+
function = piece.get("function") or {}
|
|
362
|
+
tool["name"] += function.get("name") or ""
|
|
363
|
+
arguments = function.get("arguments")
|
|
364
|
+
tool["arguments"] += arguments if isinstance(arguments, str) else json.dumps(arguments or "")
|
|
365
|
+
if on_text is not None:
|
|
366
|
+
for kind, piece in (("reasoning", reasoning[-1]), ("text", content[-1])):
|
|
367
|
+
if piece:
|
|
368
|
+
on_text(kind, piece)
|
|
369
|
+
finish = choice.get("finish_reason") or finish
|
|
370
|
+
if finish is None:
|
|
371
|
+
# The connection closed before the completion finished (the provider dropped the
|
|
372
|
+
# stream): a transport failure to retry, not an empty reply.
|
|
373
|
+
raise OSError(f"stream ended before the reply finished ({len(''.join(content))} characters received)")
|
|
374
|
+
return {"id": response_id, "usage": usage or {},
|
|
375
|
+
"choices": [{"message": {"content": "".join(content),
|
|
376
|
+
**({"reasoning": "".join(reasoning)} if any(reasoning) else {}),
|
|
377
|
+
**({"tool_calls": [{"function": tools[index]} for index in sorted(tools)]}
|
|
378
|
+
if tools else {})},
|
|
379
|
+
"finish_reason": finish}]}
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def scripted_models(script) -> dict:
|
|
383
|
+
"""The models of a scripted episode: `script` is a list of replies, or
|
|
384
|
+
{"replies": [...], "verifier": [...]} as scripts/replay_script.py writes it; without
|
|
385
|
+
recorded verifier answers the verifier says YES."""
|
|
386
|
+
if isinstance(script, list):
|
|
387
|
+
script = {"replies": script}
|
|
388
|
+
return {"chat": ScriptedChat(script["replies"]),
|
|
389
|
+
"verifier": ScriptedChat(script.get("verifier") or ["YES (scripted verifier)"] * 100)}
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
_AHEAD = re.compile(r"Program \d+ (?:is running now\.|has run )")
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def made_ahead(messages: list[dict]) -> bool:
|
|
396
|
+
"""Whether the final content part marks a request made while a program runs."""
|
|
397
|
+
content = messages[-1].get("content") if messages else None
|
|
398
|
+
last = content if isinstance(content, str) else (content[-1].get("text", "") if content else "")
|
|
399
|
+
return bool(_AHEAD.match(last or ""))
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
class ScriptedChat:
|
|
403
|
+
"""Replays fixed replies in order. For pipeline checks without model calls. A call
|
|
404
|
+
made while a program runs is answered <wait> without taking a reply (`waits` counts
|
|
405
|
+
them), so the script is the replies after each program, whichever programs the model
|
|
406
|
+
thinks past; with `ahead=True` such calls take the next reply too."""
|
|
407
|
+
|
|
408
|
+
def __init__(self, replies: list[str], ahead: bool = False):
|
|
409
|
+
self.replies = list(replies)
|
|
410
|
+
self.calls = 0
|
|
411
|
+
self.ahead = ahead
|
|
412
|
+
self.waits = 0
|
|
413
|
+
|
|
414
|
+
def complete(self, system: str, messages: list[dict]) -> ChatReply:
|
|
415
|
+
"""The next scripted reply (a `<finish>` block once they run out), with zero usage."""
|
|
416
|
+
if not self.ahead and made_ahead(messages):
|
|
417
|
+
self.waits += 1
|
|
418
|
+
return ChatReply(text="<wait>the program's result</wait>", usage=dict(USAGE))
|
|
419
|
+
if self.calls >= len(self.replies):
|
|
420
|
+
text = "<finish>scripted replies exhausted</finish>"
|
|
421
|
+
else:
|
|
422
|
+
text = self.replies[self.calls]
|
|
423
|
+
self.calls += 1
|
|
424
|
+
return ChatReply(text=text, usage=dict(USAGE))
|