@lmmslab/harness 0.0.0-stage → 0.81.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +124 -2
  3. package/bin/harness.js +71 -0
  4. package/package.json +34 -4
  5. package/python/harness/README.md +72 -0
  6. package/python/harness/__init__.py +8 -0
  7. package/python/harness/__main__.py +3 -0
  8. package/python/harness/bench.py +124 -0
  9. package/python/harness/builtin/benches/README.md +5 -0
  10. package/python/harness/builtin/benches/none/bench.toml +6 -0
  11. package/python/harness/builtin/configs/README.md +26 -0
  12. package/python/harness/builtin/configs/assistant.toml +11 -0
  13. package/python/harness/builtin/configs/default.toml +33 -0
  14. package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
  15. package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
  16. package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
  17. package/python/harness/builtin/configs/models/grok-low.toml +8 -0
  18. package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
  19. package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
  20. package/python/harness/builtin/environment/README.md +53 -0
  21. package/python/harness/builtin/environment/constraints.txt +31 -0
  22. package/python/harness/builtin/model-settings.js +143 -0
  23. package/python/harness/builtin/prompts/conversation/core.md +5 -0
  24. package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
  25. package/python/harness/builtin/prompts/core/core.md +88 -0
  26. package/python/harness/builtin/prompts/core/pack.toml +9 -0
  27. package/python/harness/builtin/prompts/reflect/core.md +45 -0
  28. package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
  29. package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
  30. package/python/harness/builtin/prompts/shared/agents.md +14 -0
  31. package/python/harness/builtin/prompts/shared/checklist.md +21 -0
  32. package/python/harness/builtin/prompts/shared/entries.md +86 -0
  33. package/python/harness/builtin/prompts/shared/environments.md +21 -0
  34. package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
  35. package/python/harness/builtin/prompts/shared/learning.md +27 -0
  36. package/python/harness/builtin/prompts/shared/mcp.md +9 -0
  37. package/python/harness/builtin/prompts/shared/world.md +1 -0
  38. package/python/harness/builtin/skills/README.md +10 -0
  39. package/python/harness/builtin/skills/agents/__init__.py +88 -0
  40. package/python/harness/builtin/skills/agents/skill.toml +11 -0
  41. package/python/harness/builtin/skills/checklist/__init__.py +32 -0
  42. package/python/harness/builtin/skills/checklist/skill.toml +9 -0
  43. package/python/harness/builtin/skills/conversation/__init__.py +23 -0
  44. package/python/harness/builtin/skills/conversation/skill.toml +8 -0
  45. package/python/harness/builtin/skills/entries/__init__.py +62 -0
  46. package/python/harness/builtin/skills/entries/skill.toml +8 -0
  47. package/python/harness/builtin/skills/environments/__init__.py +69 -0
  48. package/python/harness/builtin/skills/environments/skill.toml +9 -0
  49. package/python/harness/builtin/skills/experience/__init__.py +27 -0
  50. package/python/harness/builtin/skills/experience/skill.toml +9 -0
  51. package/python/harness/builtin/skills/instructions/__init__.py +29 -0
  52. package/python/harness/builtin/skills/instructions/skill.toml +9 -0
  53. package/python/harness/builtin/skills/learning/__init__.py +189 -0
  54. package/python/harness/builtin/skills/learning/skill.toml +7 -0
  55. package/python/harness/builtin/skills/mcp/__init__.py +54 -0
  56. package/python/harness/builtin/skills/mcp/skill.toml +10 -0
  57. package/python/harness/builtin/skills/memory/__init__.py +130 -0
  58. package/python/harness/builtin/skills/memory/skill.toml +13 -0
  59. package/python/harness/builtin/skills/verify/__init__.py +31 -0
  60. package/python/harness/builtin/skills/verify/skill.toml +7 -0
  61. package/python/harness/builtin/third_party/sources.toml +40 -0
  62. package/python/harness/cli.py +1182 -0
  63. package/python/harness/config/__init__.py +224 -0
  64. package/python/harness/config/agent.py +777 -0
  65. package/python/harness/core/__init__.py +52 -0
  66. package/python/harness/core/artifacts.py +46 -0
  67. package/python/harness/core/codec.py +160 -0
  68. package/python/harness/core/contracts.py +75 -0
  69. package/python/harness/core/episodes.py +150 -0
  70. package/python/harness/core/format.py +51 -0
  71. package/python/harness/core/http.py +124 -0
  72. package/python/harness/core/lanes.py +324 -0
  73. package/python/harness/core/linux_sandbox.py +55 -0
  74. package/python/harness/core/media.py +96 -0
  75. package/python/harness/core/memory.py +147 -0
  76. package/python/harness/core/ports.py +63 -0
  77. package/python/harness/core/procs.py +227 -0
  78. package/python/harness/core/remote.py +102 -0
  79. package/python/harness/core/spans.py +166 -0
  80. package/python/harness/core/trace.py +139 -0
  81. package/python/harness/core/wait.py +12 -0
  82. package/python/harness/core/worker.py +388 -0
  83. package/python/harness/core/worker_pipe.py +119 -0
  84. package/python/harness/embodiments.py +217 -0
  85. package/python/harness/extensions.py +112 -0
  86. package/python/harness/host/__init__.py +397 -0
  87. package/python/harness/host/client.py +201 -0
  88. package/python/harness/host/http.py +720 -0
  89. package/python/harness/host/listing.py +68 -0
  90. package/python/harness/host/server.py +189 -0
  91. package/python/harness/host/sessions.py +406 -0
  92. package/python/harness/host/tasks.py +1356 -0
  93. package/python/harness/host/workers.py +257 -0
  94. package/python/harness/host/workspaces.py +256 -0
  95. package/python/harness/model_list.py +117 -0
  96. package/python/harness/packages/__init__.py +73 -0
  97. package/python/harness/packages/addons.py +121 -0
  98. package/python/harness/packages/archive.py +122 -0
  99. package/python/harness/packages/describe.py +46 -0
  100. package/python/harness/packages/environments.py +448 -0
  101. package/python/harness/packages/fetch.py +57 -0
  102. package/python/harness/packages/manager.py +587 -0
  103. package/python/harness/packages/manifest.py +232 -0
  104. package/python/harness/packages/migrate.py +68 -0
  105. package/python/harness/packages/provision.py +38 -0
  106. package/python/harness/packages/registry.py +275 -0
  107. package/python/harness/packages/resolve.py +89 -0
  108. package/python/harness/packages/sources.py +141 -0
  109. package/python/harness/packages/state.py +213 -0
  110. package/python/harness/packages/versions.py +112 -0
  111. package/python/harness/paths.py +218 -0
  112. package/python/harness/providers.py +303 -0
  113. package/python/harness/runtime/__init__.py +8 -0
  114. package/python/harness/runtime/decision/__init__.py +1 -0
  115. package/python/harness/runtime/decision/loop.py +198 -0
  116. package/python/harness/runtime/decision/messages.py +271 -0
  117. package/python/harness/runtime/decision/model.py +424 -0
  118. package/python/harness/runtime/decision/prompts.py +248 -0
  119. package/python/harness/runtime/decision/protocol.py +169 -0
  120. package/python/harness/runtime/decision/thoughts.py +199 -0
  121. package/python/harness/runtime/events.py +213 -0
  122. package/python/harness/runtime/execution/__init__.py +1 -0
  123. package/python/harness/runtime/execution/attached.py +96 -0
  124. package/python/harness/runtime/execution/control.py +211 -0
  125. package/python/harness/runtime/execution/entries.py +665 -0
  126. package/python/harness/runtime/execution/kernel.py +811 -0
  127. package/python/harness/runtime/execution/noenv.py +84 -0
  128. package/python/harness/runtime/execution/results.py +40 -0
  129. package/python/harness/runtime/execution/session.py +871 -0
  130. package/python/harness/runtime/execution/skills.py +436 -0
  131. package/python/harness/runtime/learning/__init__.py +1 -0
  132. package/python/harness/runtime/learning/active.py +314 -0
  133. package/python/harness/runtime/learning/experience.py +48 -0
  134. package/python/harness/runtime/learning/fixtures.py +167 -0
  135. package/python/harness/runtime/learning/learner.py +651 -0
  136. package/python/harness/runtime/learning/library.py +504 -0
  137. package/python/harness/runtime/learning/reflect.py +151 -0
  138. package/python/harness/runtime/learning/usage.py +30 -0
  139. package/python/harness/runtime/learning/views.py +89 -0
  140. package/python/harness/runtime/reactive/__init__.py +1 -0
  141. package/python/harness/runtime/reactive/rules.py +549 -0
  142. package/python/harness/runtime/safety/__init__.py +1 -0
  143. package/python/harness/runtime/safety/approval.py +131 -0
  144. package/python/harness/runtime/safety/input.py +62 -0
  145. package/python/harness/runtime/safety/isolation.py +419 -0
  146. package/python/harness/runtime/safety/shield.py +129 -0
  147. package/python/harness/runtime/status.py +23 -0
  148. package/python/harness/runtime/task/__init__.py +7 -0
  149. package/python/harness/runtime/task/context.py +431 -0
  150. package/python/harness/runtime/task/delegation.py +468 -0
  151. package/python/harness/runtime/task/environments.py +268 -0
  152. package/python/harness/runtime/task/run.py +2878 -0
  153. package/python/harness/runtime/tick.py +909 -0
  154. package/python/harness/sdk/__init__.py +281 -0
  155. package/python/harness/services/__init__.py +6 -0
  156. package/python/harness/services/broker.py +35 -0
  157. package/python/harness/services/connectors.py +88 -0
  158. package/python/harness/services/mcp.py +298 -0
  159. package/python/harness/services/mcp_stdio.py +131 -0
  160. package/python/harness/services/placement.py +82 -0
  161. package/python/harness/services/registry.py +156 -0
  162. package/python/harness/settings.py +536 -0
  163. package/python/harness/terminal.py +79 -0
  164. package/python/harness/transcript.py +297 -0
  165. package/python/harness/tui/__init__.py +68 -0
  166. package/python/harness/tui/chat.py +1181 -0
  167. package/python/harness/tui/pty.py +81 -0
  168. package/python/harness/tui/renderer.py +45 -0
  169. package/python/harness/tui/timeline_rows.py +58 -0
  170. package/python/harness/tui/views.py +434 -0
@@ -0,0 +1,424 @@
1
+ """Multi-turn text completion for the high level.
2
+
3
+ Chat Completions covers OpenRouter and self-hosted OpenAI-compatible servers
4
+ (vLLM, SGLang) through `openai_chat` with a `base_url`. Messages use the
5
+ harness content parts (`input_text`, `input_image`).
6
+ """
7
+
8
+ from contextlib import contextmanager
9
+ from dataclasses import dataclass, field
10
+ import json
11
+ import os
12
+ import re
13
+ import time
14
+ from urllib.error import HTTPError
15
+ from urllib.parse import urlsplit
16
+
17
+ from harness.core.http import open_url, post_json, read_sse, redact
18
+
19
+
20
+ # The usage of model calls, one call's (ChatReply.usage, with its finish_reason) or a
21
+ # total of several (an episode's summary, a vision model's, which add a count of calls):
22
+ # tokens in and out, and the cost in US dollars when the provider reports it.
23
+ USAGE = {"input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0}
24
+
25
+
26
+ def add_usage(total: dict, usage: dict) -> None:
27
+ """Add `usage` (one call's or a total) to `total`, key by key of USAGE."""
28
+ for key, zero in USAGE.items():
29
+ total[key] = total.get(key, zero) + type(zero)(usage.get(key) or zero)
30
+
31
+
32
+ @dataclass
33
+ class ChatReply:
34
+ """One completion: its text, usage (USAGE and finish_reason; cost_usd None when the
35
+ provider does not report it), the provider's response id, and the seconds the request
36
+ took with its retries."""
37
+
38
+ text: str
39
+ usage: dict = field(default_factory=dict)
40
+ response_id: str | None = None
41
+ seconds: float = 0.0
42
+ # Reasoning text or summary, when the provider returns one (e.g. DeepSeek, GPT-6
43
+ # Luna on OpenRouter); for the trace only, never sent back to the model.
44
+ reasoning: str = ""
45
+
46
+
47
+ # The keys of a model role (configs/models/<name>.toml: [main], [vlm], [pointer],
48
+ # [verifier]). Documentation; ChatModel reads them.
49
+ MODEL_KEYS = {
50
+ "service": "a service of the deploy instead of a hosted model: its client config, with this role's keys on top",
51
+ "backend": "openrouter (default) or openai_chat (any OpenAI-compatible server)",
52
+ "model": "the model id",
53
+ "base_url": "openai_chat: the server's URL, e.g. http://127.0.0.1:8800/v1",
54
+ "base_url_env": "openai_chat: environment variable holding the server URL; overrides base_url when set",
55
+ "api_key_env": "the environment variable that holds the key (default OPENROUTER_API_KEY for openrouter)",
56
+ "reasoning_effort": "the model provider’s reasoning option ID (effort_fields); omit to use its default",
57
+ "max_output_tokens": "tokens per reply, reasoning included on OpenRouter (default 8192)",
58
+ "max_tokens_key": "the request field of that limit (default max_tokens; OpenAI's newer models take "
59
+ "max_completion_tokens)",
60
+ "temperature": "sampling temperature (default: the provider's)",
61
+ "stream": "stream the reply: a dead connection shows within the read timeout while a model thinks",
62
+ "timeout_seconds": "read timeout of one request (default 300)",
63
+ "retries": "retries of a failed request (default 2)",
64
+ "retry_seconds": "keep retrying transient failures until this long after the request started",
65
+ "extra_body": "extra request fields for OpenAI-compatible servers (e.g. max_image_side)",
66
+ "images": "false for a model that reads no images: none is sent to it (default: it reads them)",
67
+ "on_demand": "written by the resolution for an on-demand service: the broker starts it first",
68
+ }
69
+
70
+
71
+ # Thinking tokens for providers that take a budget instead of a level.
72
+ BUDGETS = {"low": 2000, "medium": 8000, "high": 24000}
73
+
74
+
75
+ def valid_effort(value):
76
+ return isinstance(value, str) and (not value or re.fullmatch(r"[A-Za-z0-9_-]{1,64}", value) is not None)
77
+
78
+
79
+ def effort_fields(backend: str, base_url, model: str, effort, max_tokens: int) -> dict:
80
+ """The request fields that set `effort` (none, low, medium, high) for this provider,
81
+ each in the field its API reads; {} when unset. OpenAI, Gemini, Ollama and other
82
+ OpenAI-compatible servers take OpenAI's reasoning_effort."""
83
+ if not effort:
84
+ return {}
85
+ if backend == "openrouter":
86
+ return {"reasoning": {"enabled": False} if effort == "none" else {"effort": effort}}
87
+ host = str(base_url or "")
88
+ budget = max(1024, min(BUDGETS.get(effort, 0), max_tokens - 1024))
89
+ if "anthropic.com" in host:
90
+ # Its OpenAI-compatible endpoint ignores reasoning_effort; thinking takes a budget.
91
+ return {"thinking": {"type": "disabled"} if effort == "none" else {"type": "enabled", "budget_tokens": budget}}
92
+ if "dashscope" in host:
93
+ return {"enable_thinking": False} if effort == "none" else {"enable_thinking": True, "thinking_budget": budget}
94
+ if "bigmodel.cn" in host or "z.ai" in host:
95
+ return {"thinking": {"type": "disabled" if effort == "none" else "enabled"}}
96
+ if "deepseek.com" in host:
97
+ # Levels low, high and max; thinking is a switch of its own.
98
+ return {"thinking": {"type": "disabled"}} if effort == "none" else {
99
+ "thinking": {"type": "enabled"}, "reasoning_effort": "low" if effort == "low" else "high"}
100
+ if "moonshot" in host:
101
+ if "k3" in model:
102
+ return {} if effort == "none" else {"reasoning_effort": "low" if effort == "low" else "high"}
103
+ return {"thinking": {"type": "disabled" if effort == "none" else "enabled"}}
104
+ if "x.ai" in host:
105
+ # Grok takes low or high, on the models that accept it.
106
+ return {} if effort == "none" else {"reasoning_effort": "low" if effort == "low" else "high"}
107
+ return {"reasoning_effort": effort}
108
+
109
+
110
+ CHAT_COMPLETIONS = "/chat/completions"
111
+
112
+
113
+ def chat_url(provider, base_url):
114
+ """The Chat Completions URL of a model config (backend, base_url)."""
115
+ if provider == "openrouter" and not base_url:
116
+ return "https://openrouter.ai/api/v1/chat/completions"
117
+ if not base_url:
118
+ raise ValueError("openai_chat requires base_url")
119
+ root = base_url.rstrip("/")
120
+ if root.endswith(CHAT_COMPLETIONS):
121
+ return root
122
+ return root + CHAT_COMPLETIONS
123
+
124
+
125
+ def chat_part(part):
126
+ if part["type"] == "input_text":
127
+ return {"type": "text", "text": part["text"]}
128
+ if part["type"] == "input_image":
129
+ return {"type": "image_url", "image_url": {"url": part["image_url"]}}
130
+ raise ValueError(f"Unsupported content part: {part['type']}")
131
+
132
+
133
+ class ChatModel:
134
+ """A Chat Completions client for one model role (MODEL_KEYS): OpenRouter or an
135
+ OpenAI-compatible server. The openrouter backend needs its key in the environment;
136
+ `transport` replaces the HTTP call (tests)."""
137
+
138
+ def __init__(self, config: dict, transport=None):
139
+ self.provider = config.get("backend", "openrouter")
140
+ if self.provider not in {"openrouter", "openai_chat"}:
141
+ raise ValueError("The program harness supports the openrouter and openai_chat backends")
142
+ self.model = config["model"]
143
+ # A URL in base_url_env is resolved with the agent (config/agent.py _role).
144
+ self.base_url = config.get("base_url")
145
+ if self.provider == "openai_chat" and not self.base_url:
146
+ named = config.get("base_url_env")
147
+ raise ValueError(f"openai_chat requires base_url.{f' Set {named}.' if named else ''}")
148
+ # A self-hosted model placed on demand is started by the deploy's broker first.
149
+ self.on_demand = config.get("on_demand")
150
+ self.url = chat_url(self.provider, self.base_url)
151
+ self.timeout = float(config.get("timeout_seconds", 300.0))
152
+ self.max_output_tokens = int(config.get("max_output_tokens", 8192))
153
+ self.max_tokens_key = str(config.get("max_tokens_key") or "max_tokens")
154
+ self.reasoning_effort = config.get("reasoning_effort")
155
+ self.temperature = config.get("temperature")
156
+ self.retries = int(config.get("retries", 2))
157
+ # Keep retrying transient failures until this many seconds have passed since the
158
+ # request started (at least `retries` times). The simulator does not advance while
159
+ # the model is being called, so waiting out an outage costs wall time only.
160
+ self.retry_seconds = config.get("retry_seconds")
161
+ self.retry_seconds = None if self.retry_seconds is None else float(self.retry_seconds)
162
+ # Extra request fields for OpenAI-compatible servers (e.g. max_image_side for serve_vlm.py).
163
+ self.extra_body = dict(config.get("extra_body") or {})
164
+ # Streaming keeps bytes flowing while a reasoning model thinks; some networks reset
165
+ # a connection that stays silent for minutes waiting for a non-streamed reply.
166
+ self.stream = bool(config.get("stream", False))
167
+ self.transport = transport or (sse_transport if self.stream else urllib_transport)
168
+ key_env = config.get("api_key_env") or ("OPENROUTER_API_KEY" if self.provider == "openrouter" else None)
169
+ self._api_key = os.environ.get(key_env) if key_env else None
170
+ if self.provider == "openrouter" and not self._api_key:
171
+ raise ValueError(f"Set {key_env} before selecting the openrouter backend")
172
+ self._api_key = self._api_key or "EMPTY"
173
+
174
+ def payload(self, system: str, messages: list[dict]) -> dict:
175
+ """The request body: `system` and the harness messages as chat messages, the output
176
+ limit and extra_body fields, and on OpenRouter usage accounting and reasoning."""
177
+ # Some models (Molmo2) take no system message; an empty one is left out.
178
+ converted = [{"role": "system", "content": system}] if system else []
179
+ for message in messages:
180
+ content = message["content"]
181
+ if isinstance(content, str):
182
+ converted.append({"role": message["role"], "content": content})
183
+ else:
184
+ converted.append({"role": message["role"],
185
+ "content": [chat_part(part) for part in content]})
186
+ payload = {"model": self.model, "messages": converted, self.max_tokens_key: self.max_output_tokens,
187
+ **self.extra_body}
188
+ if self.stream:
189
+ payload["stream"] = True
190
+ if self.temperature is not None:
191
+ payload["temperature"] = self.temperature
192
+ if self.provider == "openrouter":
193
+ payload["usage"] = {"include": True}
194
+ payload.update(effort_fields(self.provider, self.base_url, self.model, self.reasoning_effort,
195
+ self.max_output_tokens))
196
+ return payload
197
+
198
+ @property
199
+ def streams_text(self) -> bool:
200
+ """Whether complete() can hand over the reply's text as it streams (on_text)."""
201
+ return self.stream and self.transport is sse_transport
202
+
203
+ def complete(self, system: str, messages: list[dict], on_text=None, *, cancellation=None) -> ChatReply:
204
+ """One completion of `messages` under `system`, starting an on-demand service through
205
+ the broker first. Failed requests are retried (`retries` times, or until
206
+ `retry_seconds` have passed), except client errors other than HTTP 429; the last
207
+ failure is raised as a RuntimeError. When it streams (streams_text), `on_text(kind,
208
+ text)` gets each piece of the reply ("text") and of its reasoning ("reasoning") as
209
+ it comes. A cancellation closes built-in transports and stops retries."""
210
+ if cancellation is not None:
211
+ cancellation.check()
212
+ if self.on_demand:
213
+ from harness.services.broker import ensure
214
+
215
+ ensure({"on_demand": self.on_demand})
216
+ payload = self.payload(system, messages)
217
+ headers = {"authorization": f"Bearer {self._api_key}"}
218
+ # Explicit OpenRouter backends include relays; compatible clients can also
219
+ # target OpenRouter directly. Match the hostname, never URL substrings.
220
+ if self.provider == "openrouter" or urlsplit(self.url).hostname == "openrouter.ai":
221
+ headers.update({"http-referer": "https://harness.lmms-lab.com/",
222
+ "x-openrouter-title": "the-harness"})
223
+ started = time.perf_counter()
224
+ last_error = None
225
+ # A completion has no side effects, so transient provider failures may be retried.
226
+ attempt = 0
227
+ while True:
228
+ if attempt:
229
+ waited = time.perf_counter() - started
230
+ if attempt > self.retries and (self.retry_seconds is None or waited >= self.retry_seconds):
231
+ raise last_error
232
+ delay = min(2 ** (attempt - 1) * 5, 30 if self.retry_seconds is None else 60)
233
+ if cancellation is None:
234
+ time.sleep(delay)
235
+ else:
236
+ cancellation.event.wait(delay)
237
+ if cancellation is not None:
238
+ cancellation.check()
239
+ attempt += 1
240
+ try:
241
+ body = self.transport(self.url, headers, payload, self.timeout,
242
+ **({"cancellation": cancellation} if cancellation is not None
243
+ and self.transport in (sse_transport, urllib_transport) else {}),
244
+ **({"on_text": on_text} if on_text is not None and self.streams_text else {}))
245
+ except Exception as error:
246
+ if cancellation is not None:
247
+ cancellation.check()
248
+ last_error = RuntimeError(redact(str(error), self._api_key))
249
+ if "Provider HTTP 4" in str(error) and "HTTP 429" not in str(error):
250
+ raise last_error from None # a client error will not pass on retry
251
+ continue
252
+ if isinstance(body, dict) and body.get("error"):
253
+ detail = body["error"]
254
+ text = detail.get("message", str(detail)) if isinstance(detail, dict) else str(detail)
255
+ last_error = RuntimeError(f"{self.provider} error: {redact(text, self._api_key)}")
256
+ continue
257
+ choices = body.get("choices") or []
258
+ if len(choices) != 1:
259
+ raise ValueError(f"Expected one completion choice, got {len(choices)}")
260
+ message = choices[0].get("message") or {}
261
+ usage = body.get("usage") or {}
262
+ return ChatReply(
263
+ text=(message.get("content") or "") + _tool_blocks(message.get("tool_calls")),
264
+ reasoning=message.get("reasoning") or "",
265
+ usage={"input_tokens": usage.get("prompt_tokens", 0),
266
+ "output_tokens": usage.get("completion_tokens", 0),
267
+ "cost_usd": usage.get("cost"),
268
+ "finish_reason": choices[0].get("finish_reason")},
269
+ response_id=body.get("id"),
270
+ seconds=time.perf_counter() - started,
271
+ )
272
+
273
+
274
+ # DeepSeek models sometimes answer with a tool call of the provider's own where the text
275
+ # should hold the block (finish_reason "tool_calls", no tools offered): on agent 1.51's
276
+ # Minecraft practice (accept8, seed 0) two calls ran nothing so, one reply prose only and
277
+ # one empty. A call named for a block (python, or a name with python, code or exec in it)
278
+ # is read as that block, its program the one text argument.
279
+ _TOOL_KINDS = ("python", "verify", "finish", "action", "wait")
280
+ _TOOL_ARGUMENTS = ("code", "program", "source", "script", "python", "input", "content", "text", "claim", "result")
281
+
282
+
283
+ def _tool_blocks(calls) -> str:
284
+ """Native tool calls of a reply as the blocks they name, after its text."""
285
+ blocks = []
286
+ for call in calls or []:
287
+ function = (call or {}).get("function") or {}
288
+ name = str(function.get("name") or "").strip()
289
+ body = function.get("arguments")
290
+ if isinstance(body, str):
291
+ try:
292
+ body = json.loads(body)
293
+ except ValueError:
294
+ pass
295
+ if isinstance(body, dict):
296
+ texts = [body[key] for key in _TOOL_ARGUMENTS if isinstance(body.get(key), str)]
297
+ others = [value for value in body.values() if isinstance(value, str)]
298
+ body = texts[0] if texts else others[0] if len(others) == 1 else json.dumps(body)
299
+ kind = (name if name in _TOOL_KINDS else
300
+ "python" if re.search(r"python|code|exec", name, re.IGNORECASE) else name or "python")
301
+ program = str(body if body is not None else "").strip("\n")
302
+ blocks.append(f"\n<{kind}>\n{program}\n</{kind}>")
303
+ return "".join(blocks)
304
+
305
+
306
+ def urllib_transport(url, headers, payload, timeout, *, cancellation=None):
307
+ """POST a chat completion and return the provider's response."""
308
+ with _provider_errors():
309
+ return post_json(url, payload, headers, timeout, **({"cancellation": cancellation} if cancellation else {}))
310
+
311
+
312
+ def sse_transport(url, headers, payload, timeout, on_text=None, *, cancellation=None):
313
+ """POST a streamed chat completion and assemble the chunks into the non-streamed
314
+ response shape. `timeout` bounds each read, so keep-alive comments reset it.
315
+ `on_text(kind, text)` gets each piece as it comes (ChatModel.complete)."""
316
+ options = {"cancellation": cancellation} if cancellation is not None else {}
317
+ with _provider_errors(), open_url(url, payload, headers, timeout, **options) as response:
318
+ return _read_sse(response, on_text)
319
+
320
+
321
+ @contextmanager
322
+ def _provider_errors():
323
+ """A failed request as a RuntimeError that says what the provider answered."""
324
+ try:
325
+ yield
326
+ except HTTPError as error:
327
+ detail = error.read().decode("utf-8", "replace")[:500]
328
+ raise RuntimeError(f"Provider HTTP {error.code}: {detail}") from error
329
+ except OSError as error:
330
+ raise RuntimeError(f"Provider request failed: {getattr(error, 'reason', error)}") from error
331
+
332
+
333
+ def _read_sse(lines, on_text=None):
334
+ response_id, usage = None, None
335
+ content, reasoning = [], []
336
+ finish = None
337
+ # Native tool calls, by index: their name and the pieces of their arguments.
338
+ tools = {}
339
+ for event, data in read_sse(lines):
340
+ if event is None:
341
+ # ": keep-alive" comments, which a provider sends while the model has not
342
+ # begun: on_text hears of them as an empty piece, so a caller can cut off a
343
+ # reply that has not started (runtime/conversation.py).
344
+ if on_text is not None:
345
+ on_text("", "")
346
+ continue
347
+ data = data.strip()
348
+ if data == "[DONE]":
349
+ break
350
+ chunk = json.loads(data)
351
+ if chunk.get("error"):
352
+ return {"error": chunk["error"]}
353
+ response_id = response_id or chunk.get("id")
354
+ usage = chunk.get("usage") or usage
355
+ for choice in chunk.get("choices") or []:
356
+ delta = choice.get("delta") or {}
357
+ content.append(delta.get("content") or "")
358
+ reasoning.append(delta.get("reasoning") or "")
359
+ for piece in delta.get("tool_calls") or []:
360
+ tool = tools.setdefault(piece.get("index", 0), {"name": "", "arguments": ""})
361
+ function = piece.get("function") or {}
362
+ tool["name"] += function.get("name") or ""
363
+ arguments = function.get("arguments")
364
+ tool["arguments"] += arguments if isinstance(arguments, str) else json.dumps(arguments or "")
365
+ if on_text is not None:
366
+ for kind, piece in (("reasoning", reasoning[-1]), ("text", content[-1])):
367
+ if piece:
368
+ on_text(kind, piece)
369
+ finish = choice.get("finish_reason") or finish
370
+ if finish is None:
371
+ # The connection closed before the completion finished (the provider dropped the
372
+ # stream): a transport failure to retry, not an empty reply.
373
+ raise OSError(f"stream ended before the reply finished ({len(''.join(content))} characters received)")
374
+ return {"id": response_id, "usage": usage or {},
375
+ "choices": [{"message": {"content": "".join(content),
376
+ **({"reasoning": "".join(reasoning)} if any(reasoning) else {}),
377
+ **({"tool_calls": [{"function": tools[index]} for index in sorted(tools)]}
378
+ if tools else {})},
379
+ "finish_reason": finish}]}
380
+
381
+
382
+ def scripted_models(script) -> dict:
383
+ """The models of a scripted episode: `script` is a list of replies, or
384
+ {"replies": [...], "verifier": [...]} as scripts/replay_script.py writes it; without
385
+ recorded verifier answers the verifier says YES."""
386
+ if isinstance(script, list):
387
+ script = {"replies": script}
388
+ return {"chat": ScriptedChat(script["replies"]),
389
+ "verifier": ScriptedChat(script.get("verifier") or ["YES (scripted verifier)"] * 100)}
390
+
391
+
392
+ _AHEAD = re.compile(r"Program \d+ (?:is running now\.|has run )")
393
+
394
+
395
+ def made_ahead(messages: list[dict]) -> bool:
396
+ """Whether the final content part marks a request made while a program runs."""
397
+ content = messages[-1].get("content") if messages else None
398
+ last = content if isinstance(content, str) else (content[-1].get("text", "") if content else "")
399
+ return bool(_AHEAD.match(last or ""))
400
+
401
+
402
+ class ScriptedChat:
403
+ """Replays fixed replies in order. For pipeline checks without model calls. A call
404
+ made while a program runs is answered <wait> without taking a reply (`waits` counts
405
+ them), so the script is the replies after each program, whichever programs the model
406
+ thinks past; with `ahead=True` such calls take the next reply too."""
407
+
408
+ def __init__(self, replies: list[str], ahead: bool = False):
409
+ self.replies = list(replies)
410
+ self.calls = 0
411
+ self.ahead = ahead
412
+ self.waits = 0
413
+
414
+ def complete(self, system: str, messages: list[dict]) -> ChatReply:
415
+ """The next scripted reply (a `<finish>` block once they run out), with zero usage."""
416
+ if not self.ahead and made_ahead(messages):
417
+ self.waits += 1
418
+ return ChatReply(text="<wait>the program's result</wait>", usage=dict(USAGE))
419
+ if self.calls >= len(self.replies):
420
+ text = "<finish>scripted replies exhausted</finish>"
421
+ else:
422
+ text = self.replies[self.calls]
423
+ self.calls += 1
424
+ return ChatReply(text=text, usage=dict(USAGE))