@lmmslab/harness 0.81.1 → 0.83.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -35
- package/package.json +2 -2
- package/python/harness/README.md +3 -3
- package/python/harness/__init__.py +1 -1
- package/python/harness/builtin/configs/assistant.toml +0 -1
- package/python/harness/builtin/configs/default.toml +1 -1
- package/python/harness/builtin/prompts/conversation/core.md +2 -2
- package/python/harness/builtin/prompts/core/core.md +197 -45
- package/python/harness/builtin/prompts/core/pack.toml +2 -2
- package/python/harness/builtin/prompts/reflect/core.md +4 -13
- package/python/harness/builtin/prompts/reflect/pack.toml +3 -2
- package/python/harness/builtin/prompts/shared/learning-goal.md +5 -29
- package/python/harness/builtin/prompts/shared/learning.md +8 -10
- package/python/harness/builtin/prompts/shared/web.md +11 -0
- package/python/harness/builtin/skills/agents/__init__.py +24 -22
- package/python/harness/builtin/skills/entries/__init__.py +43 -28
- package/python/harness/builtin/skills/entries/skill.toml +2 -3
- package/python/harness/builtin/skills/environments/__init__.py +2 -9
- package/python/harness/builtin/skills/mcp/__init__.py +2 -3
- package/python/harness/builtin/skills/memory/__init__.py +2 -5
- package/python/harness/builtin/skills/web/__init__.py +92 -0
- package/python/harness/builtin/skills/web/skill.toml +9 -0
- package/python/harness/cli.py +85 -17
- package/python/harness/config/agent.py +36 -9
- package/python/harness/core/episodes.py +5 -4
- package/python/harness/core/lanes.py +3 -0
- package/python/harness/core/memory.py +5 -1
- package/python/harness/core/remote.py +4 -3
- package/python/harness/core/worker.py +58 -3
- package/python/harness/extensions.py +23 -2
- package/python/harness/host/__init__.py +43 -1
- package/python/harness/host/client.py +36 -6
- package/python/harness/host/http.py +19 -0
- package/python/harness/host/server.py +17 -3
- package/python/harness/host/sessions.py +3 -2
- package/python/harness/host/tasks.py +107 -26
- package/python/harness/host/workspaces.py +69 -7
- package/python/harness/model_list.py +1 -1
- package/python/harness/packages/manager.py +5 -1
- package/python/harness/providers.py +14 -5
- package/python/harness/runtime/decision/loop.py +105 -117
- package/python/harness/runtime/decision/messages.py +158 -55
- package/python/harness/runtime/decision/model.py +85 -22
- package/python/harness/runtime/decision/notices.py +66 -0
- package/python/harness/runtime/decision/prompts.py +15 -11
- package/python/harness/runtime/decision/protocol.py +12 -6
- package/python/harness/runtime/decision/thoughts.py +10 -16
- package/python/harness/runtime/events.py +28 -13
- package/python/harness/runtime/execution/attached.py +1 -1
- package/python/harness/runtime/execution/control.py +17 -35
- package/python/harness/runtime/execution/entries.py +204 -124
- package/python/harness/runtime/execution/kernel.py +471 -281
- package/python/harness/runtime/execution/managed.py +231 -0
- package/python/harness/runtime/execution/noenv.py +2 -2
- package/python/harness/runtime/execution/results.py +2 -0
- package/python/harness/runtime/execution/session.py +284 -115
- package/python/harness/runtime/learning/active.py +41 -50
- package/python/harness/runtime/learning/fixtures.py +2 -2
- package/python/harness/runtime/learning/learner.py +75 -2
- package/python/harness/runtime/learning/library.py +77 -10
- package/python/harness/runtime/learning/views.py +12 -1
- package/python/harness/runtime/reactive/__init__.py +1 -1
- package/python/harness/runtime/reactive/rules.py +29 -394
- package/python/harness/runtime/reactive/tree.py +611 -0
- package/python/harness/runtime/safety/approval.py +8 -3
- package/python/harness/runtime/safety/permissions.py +242 -0
- package/python/harness/runtime/status.py +7 -2
- package/python/harness/runtime/task/delegation.py +97 -58
- package/python/harness/runtime/task/environments.py +3 -4
- package/python/harness/runtime/task/inbound.py +64 -0
- package/python/harness/runtime/task/run.py +1231 -790
- package/python/harness/runtime/tick.py +257 -91
- package/python/harness/sdk/__init__.py +7 -4
- package/python/harness/settings.py +30 -5
- package/python/harness/terminal.py +10 -6
- package/python/harness/transcript.py +18 -6
- package/python/harness/tui/answer.py +51 -0
- package/python/harness/tui/chat.py +75 -26
- package/python/harness/tui/views.py +1 -0
- package/python/harness/builtin/prompts/shared/entries.md +0 -86
|
@@ -1,13 +1,30 @@
|
|
|
1
|
-
"""One
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
"""One conversation with one model: turns in it, model decisions and Automatic Control.
|
|
2
|
+
|
|
3
|
+
A conversation starts with one model. Every program the model writes runs as a unit of its
|
|
4
|
+
own in the kernel (runtime/execution/kernel.py): the model's call comes back at once, with
|
|
5
|
+
the program's result when it was over within a short grace, else with its handle, and the
|
|
6
|
+
result reaches the model when the unit ends. What reaches the model by itself (a message,
|
|
7
|
+
an event, a unit's end, a note of the host) is a notice in one queue with a priority
|
|
8
|
+
(runtime/decision/notices.py): `now` sets a reply made before it aside, `next` is told with
|
|
9
|
+
the next call, `later` after the model's next wait. One place tells the model with each
|
|
10
|
+
call (_deliver): what its last reply came to, the notices, and how things stand now.
|
|
11
|
+
|
|
12
|
+
A turn is the model's work from what started it (the instruction, a message, a unit's end,
|
|
13
|
+
an event) to its answer: a reply that is text, with no program and no wait (<finish> and a
|
|
14
|
+
program's finish() are that same answer). A limit (model calls and wall time, both of the
|
|
15
|
+
turn; cost, of the conversation), an error, an interrupt and the environment's end end a
|
|
16
|
+
turn too. Every turn ends with a result record (decisions.TURN_ENDS); then the conversation
|
|
17
|
+
is IDLE and keeps its messages, the kernel's namespace, rules, facts and the environment.
|
|
18
|
+
The units still running after an answer go on, and what reaches the model starts the next
|
|
19
|
+
turn. Only the user or the host ends the conversation (close()), or a kernel that died; a
|
|
20
|
+
caller without a user (interactive False: a benchmark's run) ends it after the first turn.
|
|
21
|
+
|
|
22
|
+
A conversation with a user writes what continues it in another process as it goes
|
|
23
|
+
(conversation.json beside its trace: its messages, what the user said, its facts): a
|
|
24
|
+
TaskRun given that (`restore`: saved()) goes on in the same directory and trace with a new
|
|
25
|
+
kernel, and tells the model what died with the old process.
|
|
8
26
|
"""
|
|
9
27
|
|
|
10
|
-
import ast
|
|
11
28
|
import base64
|
|
12
29
|
import hashlib
|
|
13
30
|
import json
|
|
@@ -19,34 +36,53 @@ from types import SimpleNamespace
|
|
|
19
36
|
import uuid
|
|
20
37
|
|
|
21
38
|
from harness.runtime.decision import loop as decisions
|
|
22
|
-
from harness.runtime.decision.messages import _inbox_parts
|
|
39
|
+
from harness.runtime.decision.messages import _inbox_parts, content_parts
|
|
40
|
+
from harness.runtime.decision.notices import LATER, NEXT, NOW, Notice, Notices
|
|
23
41
|
from harness.runtime.execution.results import _taken_over, _cell_usage, _wrap_up
|
|
24
42
|
from harness.runtime.safety.approval import CONFIRMATION_POLICIES
|
|
25
|
-
from harness.runtime.status import Closed
|
|
43
|
+
from harness.runtime.status import Closed, Interrupted
|
|
26
44
|
|
|
27
45
|
from harness.bench import bench_names, bench_spec
|
|
28
46
|
from harness.core.remote import RemoteEnvironment
|
|
29
47
|
from harness.core.media import write_episode_videos
|
|
30
|
-
from harness.core import sha256
|
|
48
|
+
from harness.core import atomic_write, sha256
|
|
31
49
|
from harness.core.http import Preempted
|
|
32
50
|
from harness.core.trace import Trace, context_summary, provenance, write_json
|
|
33
51
|
from harness.embodiments import KINDS as EMBODIMENT_KINDS, embodiment_kind
|
|
34
52
|
from harness.runtime.decision.thoughts import Thought, ThoughtCancelled, ThoughtQueue
|
|
35
53
|
from harness.runtime.execution.kernel import DEFAULT_BOOTSTRAP, Kernel, KernelDied, kernel_python
|
|
36
|
-
from harness.runtime.learning.library import SECTIONS, eligible, executable, library_for
|
|
54
|
+
from harness.runtime.learning.library import SECTIONS, admitted_by, eligible, executable, library_for
|
|
37
55
|
from harness.runtime.task import context as memory_view
|
|
38
56
|
from harness.runtime.task.context import WorkingMemory
|
|
39
57
|
from harness.runtime.learning.usage import fold_uses
|
|
40
58
|
from harness.runtime.decision.messages import (EMBODIMENT_TEXT, attachment_parts, cell_feedback, image_parts,
|
|
41
|
-
prune_images, prune_views, state_text,
|
|
59
|
+
prune_images, prune_views, state_text, steps_text, text_only,
|
|
60
|
+
unit_feedback, unit_text, view_parts)
|
|
42
61
|
from harness.runtime.events import EVENTS
|
|
43
62
|
from harness.runtime.status import ERROR, FINISHED, IDLE, PAUSED, RUNNING, STATUSES, WAITING_FOR_CONFIRMATION
|
|
44
63
|
from harness.runtime.learning.active import sandbox_of
|
|
45
|
-
from harness.runtime.
|
|
64
|
+
from harness.runtime.reactive.tree import told as tree_told
|
|
65
|
+
from harness.runtime.decision.messages import clear_results, latest, recent_start
|
|
66
|
+
from harness.runtime.decision.protocol import (DIRECT_KINDS, KINDS, ProtocolError, announces, looks_like_action,
|
|
67
|
+
parse_reply, wait_seconds)
|
|
46
68
|
|
|
47
69
|
import harness.runtime.learning.views as library_view
|
|
48
70
|
|
|
49
71
|
|
|
72
|
+
# What continues a conversation in another process, in its directory.
|
|
73
|
+
CONVERSATION_FILE = "conversation.json"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def saved(directory) -> dict | None:
|
|
77
|
+
"""What the conversation in `directory` kept to be continued (TaskRun's `restore`):
|
|
78
|
+
{"messages", "said", "facts", "units", "turns", "summary"}; None when it kept nothing."""
|
|
79
|
+
try:
|
|
80
|
+
kept = json.loads((Path(directory) / CONVERSATION_FILE).read_text(encoding="utf-8"))
|
|
81
|
+
except (OSError, ValueError):
|
|
82
|
+
return None
|
|
83
|
+
return kept if isinstance(kept, dict) and isinstance(kept.get("messages"), list) and kept["messages"] else None
|
|
84
|
+
|
|
85
|
+
|
|
50
86
|
class TaskRun:
|
|
51
87
|
"""One episode of `config.task` with `seed`, its trace and files under `directory`.
|
|
52
88
|
`chat`, `verifier` and `environment` replace the configured models and worker.
|
|
@@ -54,11 +90,13 @@ class TaskRun:
|
|
|
54
90
|
CONFIRMATION_POLICIES; `instruction`: the user's own instruction in place of the
|
|
55
91
|
task's (the evaluator's outcome still scores the task). `agents` (runtime/task/delegation.py)
|
|
56
92
|
starts the sub-agents its programs ask for; `parent`: the agent that started this one.
|
|
93
|
+
`restore`: what an earlier process kept of this conversation (saved(directory)): it goes
|
|
94
|
+
on from there, in the same directory and trace.
|
|
57
95
|
Managed workspaces are acquired by the host before the conversation starts."""
|
|
58
96
|
|
|
59
97
|
def __init__(self, config, seed: int, directory, *, chat=None, verifier=None, environment=None,
|
|
60
98
|
interactive: bool = False, confirmation: str = "never", instruction: str | None = None,
|
|
61
|
-
agents=None, parent=None, memory=None):
|
|
99
|
+
agents=None, parent=None, memory=None, restore: dict | None = None):
|
|
62
100
|
from copy import copy
|
|
63
101
|
|
|
64
102
|
from harness.config.agent import resolve_for
|
|
@@ -77,10 +115,25 @@ class TaskRun:
|
|
|
77
115
|
self.working_memory = memory if memory is not None else WorkingMemory()
|
|
78
116
|
self._active_library = {}
|
|
79
117
|
self._library_prompt = ""
|
|
118
|
+
from harness.runtime.safety.permissions import Programs
|
|
119
|
+
|
|
120
|
+
self.programs = Programs(self)
|
|
121
|
+
# The tokens of context the model's last call read and wrote (what compaction goes
|
|
122
|
+
# by), the summaries in a row the model did not give, and the learning report as
|
|
123
|
+
# the model was last told it.
|
|
124
|
+
self._context_tokens, self._compact_failures, self._learning_told = 0, 0, None
|
|
125
|
+
# When the model last replied (what clearing old results goes by).
|
|
126
|
+
self._replied_at = None
|
|
80
127
|
self._captured_library = None
|
|
81
128
|
self._activation_queue = {}
|
|
82
129
|
self._activation_interrupt = 0
|
|
83
130
|
self._fixture_attempts = set()
|
|
131
|
+
# What the host allows for admitting learned code (learner.ADMISSION): set by the host
|
|
132
|
+
# that starts the conversation from the user's setting learning.admission; None is
|
|
133
|
+
# $HARNESS_LEARNING_ADMISSION, else "fixture". Never part of the run's configuration.
|
|
134
|
+
self.library_admission = None
|
|
135
|
+
self._admission_level = None
|
|
136
|
+
self._trial_interrupted = set()
|
|
84
137
|
self._fixture_status = "unchecked"
|
|
85
138
|
self._fixture_descriptions = []
|
|
86
139
|
self._fixture_feedback = {}
|
|
@@ -93,9 +146,17 @@ class TaskRun:
|
|
|
93
146
|
config = copy(config)
|
|
94
147
|
if table and table.get("learn") == "on":
|
|
95
148
|
config.system_prompt = config.base_prompt
|
|
96
|
-
|
|
97
|
-
if
|
|
98
|
-
raise ValueError(f"runtime.
|
|
149
|
+
grace = config.runtime.get("turn_grace_seconds", decisions.TURN_GRACE_SECONDS)
|
|
150
|
+
if isinstance(grace, bool) or not isinstance(grace, (int, float)) or grace < 0:
|
|
151
|
+
raise ValueError(f"runtime.turn_grace_seconds must be a number of seconds >= 0, got {grace!r}")
|
|
152
|
+
told = config.runtime.get("tell_budget", "never")
|
|
153
|
+
if told not in decisions.TELL_BUDGET:
|
|
154
|
+
raise ValueError(f"runtime.tell_budget must be one of {decisions.TELL_BUDGET}, got {told!r}")
|
|
155
|
+
# Whether the model is told what the run has left (its calls, its steps, its time).
|
|
156
|
+
self._budget_told = told == "always"
|
|
157
|
+
longest = config.runtime.get("longest_wait_seconds", decisions.LONGEST_WAIT_SECONDS)
|
|
158
|
+
if isinstance(longest, bool) or not isinstance(longest, (int, float)) or longest <= 0:
|
|
159
|
+
raise ValueError(f"runtime.longest_wait_seconds must be a number of seconds > 0, got {longest!r}")
|
|
99
160
|
thinking = self.thinking_level = str(config.runtime.get("thinking") or "model")
|
|
100
161
|
if thinking not in decisions.THINKING:
|
|
101
162
|
raise ValueError(f"runtime.thinking must be one of {tuple(decisions.THINKING)}, got {thinking!r}")
|
|
@@ -107,12 +168,11 @@ class TaskRun:
|
|
|
107
168
|
config.base_prompt += decisions.THINK_BRIEFLY
|
|
108
169
|
if interactive:
|
|
109
170
|
config = copy(config)
|
|
110
|
-
message_prompt = decisions.MESSAGE_PROMPT
|
|
111
171
|
if parent is None:
|
|
112
|
-
role_prompt = decisions.INTERACTIVE_PROMPT
|
|
172
|
+
role_prompt = decisions.INTERACTIVE_PROMPT
|
|
113
173
|
else:
|
|
114
174
|
deepest = agents is None or self.depth >= agents.max_depth
|
|
115
|
-
role_prompt = SUB_AGENT_PROMPT +
|
|
175
|
+
role_prompt = SUB_AGENT_PROMPT + (
|
|
116
176
|
NO_DEEPER if deepest and "agents" in (config.runtime.get("skills") or []) else "")
|
|
117
177
|
config.system_prompt += role_prompt
|
|
118
178
|
config.base_prompt += role_prompt
|
|
@@ -124,40 +184,62 @@ class TaskRun:
|
|
|
124
184
|
self.confirmation = confirmation
|
|
125
185
|
self.instruction = instruction
|
|
126
186
|
self.chat, self.verifier, self.env = chat, verifier, environment
|
|
127
|
-
self.trace = Trace(directory, EVENTS)
|
|
187
|
+
self.trace = Trace(directory, EVENTS, append=restore is not None)
|
|
188
|
+
self._restore = restore
|
|
189
|
+
if restore is not None:
|
|
190
|
+
# Its last summary said it had ended: it is open again, so nothing takes it for ended.
|
|
191
|
+
write_json(directory / "summary.json", {**(restore.get("summary") or {}), "open": True})
|
|
192
|
+
# The facts of its programs as the kernel reports their changes: kept to be restored.
|
|
193
|
+
self._facts = dict((restore or {}).get("facts") or {})
|
|
128
194
|
self.status = IDLE
|
|
129
195
|
self.messages = []
|
|
130
196
|
self.kernel = None
|
|
131
197
|
self.episode = None
|
|
132
198
|
self.summary = None
|
|
133
199
|
self._condition = threading.Condition()
|
|
134
|
-
|
|
200
|
+
# What reaches the model by itself: one queue (runtime/decision/notices.py). With it:
|
|
201
|
+
# what the model's last output came to (told first with its next call), the
|
|
202
|
+
# priority down to which that call is told the queue, the kinds of notices told
|
|
203
|
+
# that no reply has answered yet with the events among them, how many notices
|
|
204
|
+
# set a call aside, and the state as the kernel last reported it.
|
|
205
|
+
self._notices = Notices()
|
|
206
|
+
self._outcome = []
|
|
207
|
+
self._upto = LATER
|
|
208
|
+
self._unanswered = set()
|
|
209
|
+
self._told_events = []
|
|
210
|
+
self._urgent = 0
|
|
211
|
+
self._state_now = None
|
|
135
212
|
self._pause = False
|
|
136
213
|
self._closing = False
|
|
214
|
+
# The user asked to end the turn that runs (cancel()).
|
|
215
|
+
self._interrupt = False
|
|
137
216
|
self._decision = None
|
|
217
|
+
# Of the turn that runs: when it began, the time it waited for the user, the
|
|
218
|
+
# model's calls before it, and how many turns began.
|
|
219
|
+
self._turn_began = None
|
|
138
220
|
self._waited = 0.0
|
|
139
|
-
self.
|
|
221
|
+
self._calls_before = 0
|
|
222
|
+
self._turns = 0
|
|
140
223
|
self._program_span = None
|
|
141
|
-
#
|
|
142
|
-
#
|
|
143
|
-
|
|
144
|
-
#
|
|
145
|
-
self.
|
|
146
|
-
|
|
147
|
-
self.
|
|
224
|
+
# The model's units that run ({unit id: {"name", "kind", "cell", "started"}}), its
|
|
225
|
+
# programs whose result has not come ({the kernel's cell: {"number", "code", "action",
|
|
226
|
+
# "span", "unit"}}), and how long a program may hold the model's turn before the
|
|
227
|
+
# turn comes back with its handle.
|
|
228
|
+
self._units = {}
|
|
229
|
+
self._programs = {}
|
|
230
|
+
self._grace = float(grace)
|
|
231
|
+
# The longest a <wait> of the model lasts.
|
|
232
|
+
self._longest_wait = float(longest)
|
|
233
|
+
# A unit ended the turn: "episode_ended", or ("finished", the summary its finish() gave).
|
|
234
|
+
self._run_end = None
|
|
235
|
+
# The model's call count when the learned policy first ran (runtime.post_policy_calls).
|
|
236
|
+
self._post_from = None
|
|
148
237
|
self._request = None
|
|
149
|
-
self._revision =
|
|
150
|
-
self.
|
|
238
|
+
self._revision = 0
|
|
239
|
+
self._inbound = None # the listener its worker tells through (runtime/task/inbound.py)
|
|
151
240
|
self._seen_events = set()
|
|
152
|
-
self.
|
|
153
|
-
self._ahead_due = None
|
|
154
|
-
self._program_completing = False
|
|
155
|
-
self._interruptions = 0
|
|
156
|
-
self._thoughts = ThoughtQueue(self._condition, advance=self._decision_tick,
|
|
157
|
-
on_exit=self._came_back, trace=self.trace)
|
|
241
|
+
self._thoughts = ThoughtQueue(self._condition, on_exit=self._came_back, trace=self.trace)
|
|
158
242
|
self._cancelled_origins = set()
|
|
159
|
-
# Set when the episode deadline expires while a program runs.
|
|
160
|
-
self._wall_up = False
|
|
161
243
|
self._logged = 0
|
|
162
244
|
self._replaces = 0
|
|
163
245
|
# What the user said, in order: the instruction, then every message.
|
|
@@ -187,8 +269,6 @@ class TaskRun:
|
|
|
187
269
|
# watch), kept for `_view_keep` replies or while pinned (the model says).
|
|
188
270
|
self._standing = "cameras" in config.runtime
|
|
189
271
|
self._view_keep, self._view_pins = 1, set()
|
|
190
|
-
# The views a watch(every=seconds) sent while a program runs, for the model's next look.
|
|
191
|
-
self._watched = None
|
|
192
272
|
# A main model that reads no images gets none (its role's `images` is false).
|
|
193
273
|
self._images = config.main.get("images") is not False
|
|
194
274
|
# {session, agent} in a sessions registry (host/sessions.py).
|
|
@@ -212,7 +292,6 @@ class TaskRun:
|
|
|
212
292
|
self._charged = (0, 0.0)
|
|
213
293
|
self._iterations = 0
|
|
214
294
|
self.result = None
|
|
215
|
-
self.question = None
|
|
216
295
|
self.waiting_since = None
|
|
217
296
|
self.trace.context = self._who
|
|
218
297
|
|
|
@@ -229,8 +308,9 @@ class TaskRun:
|
|
|
229
308
|
|
|
230
309
|
def send_message(self, text: str, sender: str = "user", files=()) -> None:
|
|
231
310
|
"""A message from the user (or, `sender` "parent", from the agent that started this
|
|
232
|
-
one): recorded now,
|
|
233
|
-
(
|
|
311
|
+
one): recorded now, told with the model's next call (a notice of priority `next`:
|
|
312
|
+
the call in flight goes on; cancel() is what ends a turn). `files`: paths the user
|
|
313
|
+
attached (read now: messages.attachment_parts)."""
|
|
234
314
|
text, files = str(text).strip(), [str(path) for path in files or ()]
|
|
235
315
|
if not text and not files:
|
|
236
316
|
return
|
|
@@ -238,27 +318,27 @@ class TaskRun:
|
|
|
238
318
|
with self._condition:
|
|
239
319
|
if self.status in (FINISHED, ERROR):
|
|
240
320
|
raise RuntimeError("The episode has ended")
|
|
241
|
-
self._inbox.append((sender, text, parts))
|
|
242
321
|
said = " ".join([text] + [f"@{path}" for path in files]).strip()
|
|
243
322
|
self._said.append(said if sender == "user" else f"(from the agent that started you) {said}")
|
|
244
|
-
self.question = None
|
|
245
323
|
self.trace.emit("user_message", text=text, **({"files": files} if files else {}),
|
|
246
324
|
**({"sender": sender} if sender != "user" else {}))
|
|
247
325
|
self._revision += 1
|
|
248
326
|
if self._managed_dispatcher is not None:
|
|
249
|
-
|
|
327
|
+
# (The message stops no program; an approval asked before it is void.)
|
|
250
328
|
self._cancel_managed_worker_approvals("a new user message arrived")
|
|
251
329
|
if self._physical is not None and self._physical["id"] in self._worker_approvals:
|
|
252
330
|
request = self._physical
|
|
253
331
|
self._worker_approvals.pop(request["id"])
|
|
254
332
|
self._physical = None
|
|
255
333
|
self._answer_physical(request, False, "once", "superseded by a user message")
|
|
256
|
-
|
|
257
|
-
self.
|
|
334
|
+
# (The roles' calls that judged without it are stale.)
|
|
335
|
+
self._thoughts.cancel_where(
|
|
336
|
+
lambda item: item.role in ("verifier", "observer", "vlm", "pointer", "reflection"), "user message")
|
|
337
|
+
self._put("message", NEXT, (sender, text, parts))
|
|
258
338
|
|
|
259
339
|
def pause(self) -> None:
|
|
260
|
-
"""Pause:
|
|
261
|
-
called again until resume() or a message."""
|
|
340
|
+
"""Pause: every unit of the model stops at its next execution step or wait, and
|
|
341
|
+
nothing of it acts; the model is not called again until resume() or a message."""
|
|
262
342
|
with self._condition:
|
|
263
343
|
if self.status in (FINISHED, ERROR):
|
|
264
344
|
return
|
|
@@ -268,8 +348,7 @@ class TaskRun:
|
|
|
268
348
|
if self._managed_dispatcher is not None:
|
|
269
349
|
self._managed_dispatcher.stop()
|
|
270
350
|
self._cancel_managed_worker_approvals("paused")
|
|
271
|
-
kernel = self.kernel
|
|
272
|
-
self.invalidate_lookahead()
|
|
351
|
+
kernel = self.kernel
|
|
273
352
|
self._condition.notify_all()
|
|
274
353
|
if self.agents is not None:
|
|
275
354
|
self.agents.wake()
|
|
@@ -281,13 +360,16 @@ class TaskRun:
|
|
|
281
360
|
"""Go on after a pause."""
|
|
282
361
|
with self._condition:
|
|
283
362
|
self._pause = False
|
|
363
|
+
kernel = self.kernel
|
|
284
364
|
self._condition.notify_all()
|
|
365
|
+
if kernel is not None:
|
|
366
|
+
kernel.resume()
|
|
285
367
|
|
|
286
368
|
def set_model(self, model: str | None = None, reasoning_effort: str | None = None) -> dict:
|
|
287
369
|
"""Change the main model (another id of its provider) or its reasoning effort (EFFORTS,
|
|
288
370
|
"" for the provider's default; runtime.thinking still bounds it) from its next call on:
|
|
289
371
|
a model_changed record. Returns the model and effort now."""
|
|
290
|
-
from harness.runtime.decision.model import ChatModel
|
|
372
|
+
from harness.runtime.decision.model import BUILT_IN, ChatModel
|
|
291
373
|
|
|
292
374
|
main = dict(self.config.main)
|
|
293
375
|
if model:
|
|
@@ -309,11 +391,15 @@ class TaskRun:
|
|
|
309
391
|
if chat is not None:
|
|
310
392
|
if not isinstance(chat, ChatModel):
|
|
311
393
|
raise ValueError("this conversation's model is not one a model id names")
|
|
312
|
-
# Its connection as it is (a provider's transport), with the new role
|
|
313
|
-
|
|
394
|
+
# Its connection as it is (a provider's transport), with the new role; a
|
|
395
|
+
# package's backend (an ACP agent) is made for its role.
|
|
396
|
+
packaged = chat.packaged or main.get("backend", "openrouter") not in BUILT_IN
|
|
397
|
+
chat = ChatModel(main, transport=None if packaged else chat.transport)
|
|
314
398
|
self.config.main = main
|
|
315
399
|
if chat is not None:
|
|
316
|
-
self.chat = chat
|
|
400
|
+
replaced, self.chat = self.chat, chat
|
|
401
|
+
if replaced.packaged:
|
|
402
|
+
replaced.close()
|
|
317
403
|
changed = {"model": main["model"], "reasoning_effort": main.get("reasoning_effort")}
|
|
318
404
|
self.trace.emit("model_changed", **changed)
|
|
319
405
|
return changed
|
|
@@ -372,8 +458,22 @@ class TaskRun:
|
|
|
372
458
|
return
|
|
373
459
|
self._answer_physical(request, accept, remember or "once", reason)
|
|
374
460
|
|
|
461
|
+
# What a frontend may ask of it besides messages (the host's /api/conversations/cancel).
|
|
462
|
+
capabilities = {"cancel_turn": True}
|
|
463
|
+
|
|
464
|
+
def cancel(self) -> None:
|
|
465
|
+
"""End the turn that runs: the call in flight is set aside, the model's units are
|
|
466
|
+
cancelled, and the conversation waits for the next message (nothing between turns)."""
|
|
467
|
+
with self._condition:
|
|
468
|
+
if self.status in (FINISHED, ERROR, IDLE) or self._interrupt:
|
|
469
|
+
return
|
|
470
|
+
self._interrupt = True
|
|
471
|
+
self._revision += 1
|
|
472
|
+
self._preempt("interrupted")
|
|
473
|
+
self._condition.notify_all()
|
|
474
|
+
|
|
375
475
|
def close(self) -> None:
|
|
376
|
-
"""End the
|
|
476
|
+
"""End the conversation: at once while it waits, else at the next step boundary."""
|
|
377
477
|
with self._condition:
|
|
378
478
|
self._closing = True
|
|
379
479
|
if self._request is not None:
|
|
@@ -381,13 +481,14 @@ class TaskRun:
|
|
|
381
481
|
if self._managed_dispatcher is not None:
|
|
382
482
|
self._managed_dispatcher.stop()
|
|
383
483
|
self._cancel_managed_worker_approvals("closed")
|
|
384
|
-
kernel = self.kernel
|
|
484
|
+
kernel = self.kernel
|
|
385
485
|
self._condition.notify_all()
|
|
386
486
|
if self.agents is not None:
|
|
387
487
|
self.agents.wake()
|
|
388
488
|
self._drop_physical("closed")
|
|
389
489
|
if kernel is not None:
|
|
390
|
-
|
|
490
|
+
# The run ends: so do its units.
|
|
491
|
+
kernel.cancel()
|
|
391
492
|
|
|
392
493
|
# Physical input the policy asks the user for (runtime/safety/input.py).
|
|
393
494
|
|
|
@@ -559,15 +660,15 @@ class TaskRun:
|
|
|
559
660
|
return (getattr(self, "profile", None) or {}).get("steps_shown", True) is not False
|
|
560
661
|
|
|
561
662
|
def _kernel_span(self, record: dict) -> None:
|
|
562
|
-
"""A
|
|
563
|
-
program
|
|
663
|
+
"""A record from the kernel: a span (a skill's execution, a service call) under the
|
|
664
|
+
program whose unit made it, a note, a unit's start or end."""
|
|
564
665
|
record = dict(record)
|
|
565
666
|
kind = record.pop("op")
|
|
566
667
|
if kind == "worker_request":
|
|
567
668
|
self._queue_host_request(record, worker=True)
|
|
568
669
|
return
|
|
569
|
-
if kind == "
|
|
570
|
-
self.
|
|
670
|
+
if kind == "unit_end":
|
|
671
|
+
self._unit_arrived(record)
|
|
571
672
|
return
|
|
572
673
|
if kind == "note":
|
|
573
674
|
note = record.pop("kind", None)
|
|
@@ -583,14 +684,15 @@ class TaskRun:
|
|
|
583
684
|
elif note == "environment":
|
|
584
685
|
threading.Thread(target=self._environment_request, args=(record,), name="environment",
|
|
585
686
|
daemon=True).start()
|
|
687
|
+
elif note == "web":
|
|
688
|
+
threading.Thread(target=self._web_request, args=(record,), name="web", daemon=True).start()
|
|
689
|
+
elif note == "program":
|
|
690
|
+
# A step of a program on the machine (runtime/safety/permissions.py): answered
|
|
691
|
+
# through the kernel, on a thread of its own (the user may be asked).
|
|
692
|
+
threading.Thread(target=lambda: self.kernel is not None and self.kernel.answer(
|
|
693
|
+
record.get("id"), self.programs.handle(record)), name="program-step", daemon=True).start()
|
|
586
694
|
elif note == "views":
|
|
587
695
|
self._views_changed(record)
|
|
588
|
-
elif note == "view":
|
|
589
|
-
with self._condition:
|
|
590
|
-
if self._cell_running and not self._standing and isinstance(record.get("images"), dict):
|
|
591
|
-
self._watched = [{"camera": name, "image_url": url}
|
|
592
|
-
for name, url in record["images"].items()]
|
|
593
|
-
self._thoughts.wake()
|
|
594
696
|
elif note == "mcp_request":
|
|
595
697
|
self._queue_host_request(record)
|
|
596
698
|
elif note == "mcp_cancel":
|
|
@@ -609,31 +711,68 @@ class TaskRun:
|
|
|
609
711
|
elif note == "library_use":
|
|
610
712
|
record.pop("time", None)
|
|
611
713
|
self.trace.emit("library_use", **record)
|
|
612
|
-
elif note == "cell_finishing":
|
|
613
|
-
with self._condition:
|
|
614
|
-
if self._cell_running:
|
|
615
|
-
self._program_completing = True
|
|
616
|
-
self._condition.notify_all()
|
|
617
714
|
elif note == "escalation":
|
|
618
715
|
self._escalation(record)
|
|
619
716
|
elif note == "memory":
|
|
620
717
|
self._memory_request(record)
|
|
718
|
+
elif note == "reflexes":
|
|
719
|
+
# How the tree of reflexes stands (runtime/reactive/tree.py tell): told with every turn.
|
|
720
|
+
with self._condition:
|
|
721
|
+
self._reflex_nodes = dict(record.get("nodes") or {})
|
|
722
|
+
elif note == "state":
|
|
723
|
+
# The state as the kernel last had it (runtime/execution/session.py): told with every call.
|
|
724
|
+
with self._condition:
|
|
725
|
+
self._state_now = record.get("state")
|
|
726
|
+
elif note == "preempted":
|
|
727
|
+
self._taken(record)
|
|
728
|
+
elif note == "held_back":
|
|
729
|
+
self._held_back(record)
|
|
730
|
+
elif note == "priority":
|
|
731
|
+
# A program said with what priority it holds the environment (at_priority).
|
|
732
|
+
with self._condition:
|
|
733
|
+
if record.get("unit") in self._units:
|
|
734
|
+
self._units[record["unit"]]["priority"] = record.get("priority")
|
|
735
|
+
elif note == "fact":
|
|
736
|
+
record.pop("time", None)
|
|
737
|
+
data = record.pop("data", None)
|
|
738
|
+
with self._condition:
|
|
739
|
+
if record.get("dropped"):
|
|
740
|
+
self._facts.pop(record.get("name"), None)
|
|
741
|
+
elif data is not None:
|
|
742
|
+
self._facts[record["name"]] = json.loads(data)
|
|
743
|
+
self.trace.emit("fact", **record)
|
|
621
744
|
elif note in EVENTS:
|
|
745
|
+
if note == "program_output":
|
|
746
|
+
with self._condition:
|
|
747
|
+
unit = self._units.get(record.get("unit"))
|
|
748
|
+
if unit is not None and "printed" in unit:
|
|
749
|
+
unit["printed"] = time.time()
|
|
622
750
|
record.pop("time", None)
|
|
623
751
|
self.trace.emit(note, **record)
|
|
624
752
|
if note == "guard" and self.summary is not None:
|
|
625
753
|
self.summary["guard_triggers"] += 1
|
|
626
|
-
if note == "memory":
|
|
627
|
-
self.invalidate_lookahead()
|
|
628
754
|
return
|
|
629
755
|
if kind in ("entry_start", "entry_end"):
|
|
630
756
|
with self._condition:
|
|
631
757
|
if kind == "entry_start":
|
|
632
758
|
self._mcp_entries.add(record["id"])
|
|
759
|
+
if record.get("kind") in ("program", "agent"):
|
|
760
|
+
self._units[record["id"]] = {"name": record.get("name"), "kind": record["kind"],
|
|
761
|
+
"cell": record.get("cell"), "started": time.time()}
|
|
762
|
+
if record["kind"] == "program" and self._managed_dispatcher is None:
|
|
763
|
+
# What it prints reaches the host while it runs (program_output).
|
|
764
|
+
self._units[record["id"]]["printed"] = None
|
|
765
|
+
program = self._programs.get(record.get("cell"))
|
|
766
|
+
if program is not None:
|
|
767
|
+
program["unit"] = record["id"]
|
|
633
768
|
else:
|
|
634
769
|
self._mcp_entries.discard(record["id"])
|
|
770
|
+
self._units.pop(record["id"], None)
|
|
771
|
+
self._condition.notify_all()
|
|
635
772
|
if kind == "span_start" and record.get("parent") is None:
|
|
636
|
-
|
|
773
|
+
# Under the program whose unit made it, else under the host's own program span.
|
|
774
|
+
cell = (self._units.get(record.get("entry")) or {}).get("cell")
|
|
775
|
+
record["parent"] = (self._programs.get(cell) or {}).get("span") or self._program_span
|
|
637
776
|
self.trace.emit(kind, **record)
|
|
638
777
|
|
|
639
778
|
def _queue_host_request(self, request, *, worker=False):
|
|
@@ -643,7 +782,7 @@ class TaskRun:
|
|
|
643
782
|
return
|
|
644
783
|
with self._condition:
|
|
645
784
|
kernel = self.kernel
|
|
646
|
-
if kernel is None or (not worker and not
|
|
785
|
+
if kernel is None or (not worker and owner not in self._mcp_entries) or self._closing or self._pause:
|
|
647
786
|
if worker and kernel is not None:
|
|
648
787
|
kernel.answer(identifier, {"worker": True, "error": "The calling program has stopped"})
|
|
649
788
|
return
|
|
@@ -658,12 +797,10 @@ class TaskRun:
|
|
|
658
797
|
if not slot.acquire(blocking=False):
|
|
659
798
|
kernel.answer(identifier, {"worker": worker, "error": "Another request is still running"})
|
|
660
799
|
return
|
|
661
|
-
revision, cell = self._revision, self.summary["cells"]
|
|
662
800
|
|
|
663
801
|
def still_current():
|
|
664
|
-
return (self.kernel is kernel and (worker or
|
|
665
|
-
|
|
666
|
-
and (owner or self._revision == revision) and not self._pause and not self._closing
|
|
802
|
+
return (self.kernel is kernel and (worker or owner in self._mcp_entries)
|
|
803
|
+
and not self._pause and not self._closing
|
|
667
804
|
and (worker or identifier not in self._mcp_cancelled))
|
|
668
805
|
|
|
669
806
|
threading.Thread(target=self._host_request, args=(kernel, request, still_current, slot),
|
|
@@ -812,11 +949,11 @@ class TaskRun:
|
|
|
812
949
|
from harness.runtime.safety import approval
|
|
813
950
|
|
|
814
951
|
with self._condition:
|
|
815
|
-
kernel, revision
|
|
952
|
+
kernel, revision = self.kernel, self._revision
|
|
816
953
|
|
|
817
954
|
def still_current():
|
|
818
|
-
return (self.kernel is kernel and self.
|
|
819
|
-
and
|
|
955
|
+
return (self.kernel is kernel and self._revision == revision and not self._pause
|
|
956
|
+
and not self._closing)
|
|
820
957
|
|
|
821
958
|
request = approval.request(kind, target, [kind], why=str(why)[:500], **detail)
|
|
822
959
|
return self._await_host_decision(request, still_current)
|
|
@@ -833,8 +970,6 @@ class TaskRun:
|
|
|
833
970
|
self._view_pins.clear()
|
|
834
971
|
else:
|
|
835
972
|
self._view_pins.discard(str(record["pin"]))
|
|
836
|
-
elif action == "unwatch":
|
|
837
|
-
self._watched = None
|
|
838
973
|
self.trace.emit("views", **{key: value for key, value in record.items() if key not in ("time", "id")})
|
|
839
974
|
|
|
840
975
|
def _described(self, looks) -> list[dict]:
|
|
@@ -864,6 +999,42 @@ class TaskRun:
|
|
|
864
999
|
parts.append({"type": "input_text", "text": f"{name}: not described ({type(error).__name__}: {error})"})
|
|
865
1000
|
return parts
|
|
866
1001
|
|
|
1002
|
+
def _web_request(self, request):
|
|
1003
|
+
"""A program searches the web (skills/web): with the search this conversation is
|
|
1004
|
+
configured with (runtime.search: a model on OpenRouter, whose web plugin searches;
|
|
1005
|
+
the pages it cites are the results), answered through the kernel."""
|
|
1006
|
+
from harness.runtime.decision.model import ChatModel
|
|
1007
|
+
|
|
1008
|
+
config, query = self.config.runtime.get("search"), str(request.get("query") or "")
|
|
1009
|
+
count = max(1, min(10, int(request.get("results") or 5)))
|
|
1010
|
+
try:
|
|
1011
|
+
if not isinstance(config, dict) or not config.get("model"):
|
|
1012
|
+
raise ValueError("this conversation has no search (runtime.search names none); fetch(url) reads a "
|
|
1013
|
+
"page whose address you know")
|
|
1014
|
+
plugin = {"id": "web", "max_results": count, **({"engine": config["engine"]} if config.get("engine") else {})}
|
|
1015
|
+
chat = ChatModel({"backend": "openrouter", "max_output_tokens": 512,
|
|
1016
|
+
**{key: value for key, value in config.items() if key != "engine"},
|
|
1017
|
+
"extra_body": {"plugins": [plugin]}})
|
|
1018
|
+
reply = self._ask_role("search", chat, SEARCHER, [{"role": "user", "content": [
|
|
1019
|
+
{"type": "input_text", "text": query}]}])
|
|
1020
|
+
found, seen = [], set()
|
|
1021
|
+
for item in reply.annotations:
|
|
1022
|
+
cited = item.get("url_citation") if isinstance(item, dict) and item.get("type") == "url_citation" else None
|
|
1023
|
+
if cited and cited.get("url") and cited["url"] not in seen:
|
|
1024
|
+
seen.add(cited["url"])
|
|
1025
|
+
found.append({"title": str(cited.get("title") or ""), "url": str(cited["url"]),
|
|
1026
|
+
"text": str(cited.get("content") or "")[:2000]})
|
|
1027
|
+
if not found and reply.text.strip():
|
|
1028
|
+
# (A search that cites nothing: what it said is the one result.)
|
|
1029
|
+
found = [{"title": "", "url": "", "text": reply.text.strip()[:4000]}]
|
|
1030
|
+
answer = {"value": found[:count]}
|
|
1031
|
+
self.trace.emit("web_search", query=query, results=len(found[:count]))
|
|
1032
|
+
except Exception as error:
|
|
1033
|
+
answer = {"error": str(error) if isinstance(error, ValueError) else f"{type(error).__name__}: {error}"}
|
|
1034
|
+
self.trace.emit("web_search", query=query, error=answer["error"][:300])
|
|
1035
|
+
if self.kernel is not None:
|
|
1036
|
+
self.kernel.answer(request.get("id"), answer)
|
|
1037
|
+
|
|
867
1038
|
def _environment_request(self, request):
|
|
868
1039
|
"""A program asks for another environment, to give one back, or for sub-agents
|
|
869
1040
|
(skills/environments): answered through the kernel."""
|
|
@@ -880,7 +1051,6 @@ class TaskRun:
|
|
|
880
1051
|
with self._condition:
|
|
881
1052
|
self.config = copy(self.config)
|
|
882
1053
|
self.config.system_prompt += text
|
|
883
|
-
self.invalidate_lookahead()
|
|
884
1054
|
|
|
885
1055
|
def _prompt_removed(self, text):
|
|
886
1056
|
from copy import copy
|
|
@@ -888,7 +1058,6 @@ class TaskRun:
|
|
|
888
1058
|
with self._condition:
|
|
889
1059
|
self.config = copy(self.config)
|
|
890
1060
|
self.config.system_prompt = self.config.system_prompt.replace(text, "", 1)
|
|
891
|
-
self.invalidate_lookahead()
|
|
892
1061
|
|
|
893
1062
|
def _skill_added(self, name) -> bool:
|
|
894
1063
|
"""A skill switched on while the conversation runs: part of its skills and prompt now."""
|
|
@@ -924,42 +1093,101 @@ class TaskRun:
|
|
|
924
1093
|
if result["status"] != "ok":
|
|
925
1094
|
self.trace.emit("cleanup_error", message=f"skills {due}: {result.get('error')}")
|
|
926
1095
|
|
|
927
|
-
def
|
|
1096
|
+
def tell(self, content, source: str | None = None) -> None:
|
|
1097
|
+
"""The environment tells the model something that happened by itself: a message, in
|
|
1098
|
+
the form conversations give content (a string, or a list of text and image parts:
|
|
1099
|
+
decision/messages.py content_parts). It reaches the model at once (a notice of
|
|
1100
|
+
priority `now`): a wait ends, a call in flight is set aside and asked again; the model reads it as
|
|
1101
|
+
"Message from the environment". Called by whoever knows of it: the run's worker
|
|
1102
|
+
through its listener (runtime/task/inbound.py), or the code that owns this run."""
|
|
1103
|
+
parts = content_parts(content)
|
|
1104
|
+
if not parts:
|
|
1105
|
+
return
|
|
1106
|
+
sender = "environment" + (f" ({source})" if source and source != "environment" else "")
|
|
1107
|
+
text = "\n".join(part["text"] for part in parts if part["type"] == "input_text")
|
|
1108
|
+
with self._condition:
|
|
1109
|
+
if self.status in (FINISHED, ERROR):
|
|
1110
|
+
raise RuntimeError("The episode has ended")
|
|
1111
|
+
self.trace.emit("user_message", text=text, sender=sender,
|
|
1112
|
+
**({"images": sum(part["type"] == "input_image" for part in parts)}
|
|
1113
|
+
if any(part["type"] == "input_image" for part in parts) else {}))
|
|
1114
|
+
self._put("message", NOW, (sender, text, parts), reason="environment message")
|
|
1115
|
+
|
|
1116
|
+
def _put(self, kind, priority, data, *, wakes=True, reason=None) -> None:
|
|
1117
|
+
"""Put a notice into the queue (runtime/decision/notices.py; the caller holds the
|
|
1118
|
+
condition). One of priority `now` sets the model's call in flight aside."""
|
|
1119
|
+
self._notices.put(Notice(kind, priority, data, wakes))
|
|
1120
|
+
if priority == NOW:
|
|
1121
|
+
self._revision += 1
|
|
1122
|
+
self._urgent += 1
|
|
1123
|
+
self._preempt(reason or kind)
|
|
1124
|
+
self._condition.notify_all()
|
|
1125
|
+
|
|
1126
|
+
def _note(self, text, priority=NEXT) -> None:
|
|
1127
|
+
"""A note of the host for the model."""
|
|
1128
|
+
with self._condition:
|
|
1129
|
+
self._put("note", priority, str(text))
|
|
1130
|
+
|
|
1131
|
+
def _message_waits(self) -> bool:
|
|
1132
|
+
"""Whether a message waits to be told (the caller holds the condition)."""
|
|
1133
|
+
return bool(self._notices.pending(kinds=("message",)))
|
|
1134
|
+
|
|
1135
|
+
def _escalation(self, record, priority=NOW):
|
|
1136
|
+
"""An event for the model: a notice, at once unless its source says otherwise."""
|
|
928
1137
|
with self._condition:
|
|
929
1138
|
identifier = record["id"]
|
|
930
1139
|
if identifier in self._seen_events:
|
|
931
1140
|
return
|
|
932
1141
|
self._seen_events.add(identifier)
|
|
933
|
-
self._pending_events[identifier] = record
|
|
934
1142
|
self.trace.emit("escalation", **{key: value for key, value in record.items() if key != "event"},
|
|
935
1143
|
reported=record["event"])
|
|
936
|
-
self.
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
1144
|
+
self._put("event", priority, record, reason="unhandled escalation")
|
|
1145
|
+
|
|
1146
|
+
def _held_back(self, record: dict) -> None:
|
|
1147
|
+
"""A reflex's condition came to hold while a program of higher priority had the
|
|
1148
|
+
environment (at_priority; the kernel's `held_back` note, once per reflex and unit):
|
|
1149
|
+
the reflex did not act and is judged again when that unit has ended. An event to the
|
|
1150
|
+
model of priority `later`: the program said itself that it stands above the reflex,
|
|
1151
|
+
so nothing the model thinks is out of date; it may still let the reflex act by
|
|
1152
|
+
cancelling the unit."""
|
|
1153
|
+
record.pop("time", None)
|
|
1154
|
+
self.trace.emit("held_back", **record)
|
|
1155
|
+
self._escalation({"id": uuid.uuid4().hex, "entry": record.get("by"), "event": {
|
|
1156
|
+
"kind": "HELD_BACK", "source": record.get("rule"),
|
|
1157
|
+
"detail": {key: record[key] for key in ("rule", "priority", "by", "by_name") if record.get(key) is not None}}},
|
|
1158
|
+
LATER)
|
|
1159
|
+
|
|
1160
|
+
def _taken(self, record: dict) -> None:
|
|
1161
|
+
"""A unit of higher priority (a rule's program) took the environment from an
|
|
1162
|
+
execution that ran (the tick's `preempted` note): an event to the model, told at
|
|
1163
|
+
once as every event that reaches it is: a wait ends, a call in flight is set
|
|
1164
|
+
aside and asked again. It says which unit and rule took the environment, from
|
|
1165
|
+
which unit and execution, whether that execution is started again, and the state."""
|
|
1166
|
+
record.pop("time", None)
|
|
1167
|
+
state = record.pop("state", None) or {}
|
|
1168
|
+
self.trace.emit("preempted", **record)
|
|
1169
|
+
if record.get("of_reflex"):
|
|
1170
|
+
# The call of a reflex's program: its node begins again by itself when its turn comes.
|
|
1171
|
+
return
|
|
1172
|
+
told = {key: record[key] for key in ("rule", "by", "by_name", "unit", "name", "execution", "restart")
|
|
1173
|
+
if record.get(key) is not None}
|
|
1174
|
+
self._escalation({"id": uuid.uuid4().hex, "entry": record.get("unit"), "event": {
|
|
1175
|
+
"kind": "PREEMPTED", "source": record.get("rule") or record.get("by_name") or record.get("by"),
|
|
1176
|
+
"detail": {**told, **({"state": state_text(state, self.config.embodiment, self._steps_shown())}
|
|
1177
|
+
if state else {})}}})
|
|
944
1178
|
|
|
945
1179
|
def _preempt(self, reason):
|
|
946
|
-
|
|
1180
|
+
"""Something reached the model while its call is in flight: the call is set aside
|
|
1181
|
+
(its reply would not know), and so are the role calls that reason makes stale."""
|
|
947
1182
|
thought = self._request
|
|
948
1183
|
if thought is not None and thought.outcome is None:
|
|
949
1184
|
self._settle(thought, "discarded", reason)
|
|
950
|
-
if reason == "
|
|
951
|
-
self._thoughts.cancel_where(lambda item: item.role in ("verifier", "observer", "vlm", "pointer", "reflection"),
|
|
952
|
-
reason)
|
|
953
|
-
elif reason == "unhandled escalation":
|
|
1185
|
+
if reason == "unhandled escalation":
|
|
954
1186
|
self._thoughts.cancel_where(lambda item: item.role == "observer", reason)
|
|
955
1187
|
|
|
956
|
-
def
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
self._ahead_generation += 1
|
|
960
|
-
if self._request is not None and self._request.trigger == "ahead":
|
|
961
|
-
self._settle(self._request, "discarded", "inputs changed")
|
|
962
|
-
self._condition.notify_all()
|
|
1188
|
+
def _busy(self) -> bool:
|
|
1189
|
+
"""Whether a program of the model runs (code it spawned to go on beside does not count)."""
|
|
1190
|
+
return bool(self._programs)
|
|
963
1191
|
|
|
964
1192
|
def _learner(self):
|
|
965
1193
|
"""The run's library ([runtime.library]) and what its settings allow, opened once per
|
|
@@ -981,6 +1209,28 @@ class TaskRun:
|
|
|
981
1209
|
chosen = settings(table)
|
|
982
1210
|
return chosen if chosen["learn"] == "on" else None
|
|
983
1211
|
|
|
1212
|
+
def _admission(self) -> str:
|
|
1213
|
+
"""How learned code is admitted in this conversation (learner.admission): what the
|
|
1214
|
+
host allows (the conversation that started this one decides for its sub-agents),
|
|
1215
|
+
for this worker's profile. Read once, when the profile is known."""
|
|
1216
|
+
if self._admission_level is None:
|
|
1217
|
+
import os
|
|
1218
|
+
|
|
1219
|
+
from harness.runtime.learning.learner import ADMISSION_VARIABLE, admission
|
|
1220
|
+
|
|
1221
|
+
root = self
|
|
1222
|
+
while root.parent is not None:
|
|
1223
|
+
root = root.parent
|
|
1224
|
+
allowed = os.environ.get(ADMISSION_VARIABLE) or root.library_admission
|
|
1225
|
+
profile = getattr(self, "profile", None)
|
|
1226
|
+
if profile is None:
|
|
1227
|
+
return admission(None, allowed) # before the worker answered: asked again later
|
|
1228
|
+
self._admission_level = admission(profile, allowed)
|
|
1229
|
+
if self.config.runtime.get("library"):
|
|
1230
|
+
self.trace.emit("library", action="admission", level=self._admission_level,
|
|
1231
|
+
allowed=allowed or "fixture", physical=self.profile.get("physical"))
|
|
1232
|
+
return self._admission_level
|
|
1233
|
+
|
|
984
1234
|
def _learning_rows(self, snapshot=None):
|
|
985
1235
|
rows = self._learner().entries(snapshot=snapshot)
|
|
986
1236
|
with self._condition:
|
|
@@ -997,7 +1247,8 @@ class TaskRun:
|
|
|
997
1247
|
continue
|
|
998
1248
|
info = self._active_row(item, row)
|
|
999
1249
|
rows.append({**info, "section": key.split("/", 1)[0], "file": key, "loaded": True,
|
|
1000
|
-
"verified": bool(row.get("proof")), "
|
|
1250
|
+
"verified": bool(row.get("proof")), "admitted_by": admitted_by(row),
|
|
1251
|
+
"uses": info["seen"], "success_rate": info["rate"]})
|
|
1001
1252
|
return rows
|
|
1002
1253
|
|
|
1003
1254
|
def _learning_progress(self):
|
|
@@ -1005,52 +1256,67 @@ class TaskRun:
|
|
|
1005
1256
|
|
|
1006
1257
|
return {**progress(self._learning_rows(), self.seconds_left(), self.config.max_wall_seconds or 0.0,
|
|
1007
1258
|
self._situations()), "fixture_status": self._fixture_status,
|
|
1008
|
-
"fixture_feedback": dict(self._fixture_feedback)}
|
|
1259
|
+
"fixture_feedback": dict(self._fixture_feedback), "admission": self._admission()}
|
|
1009
1260
|
|
|
1010
1261
|
def _learning_report(self) -> str:
|
|
1011
1262
|
from harness.runtime.learning import active
|
|
1012
1263
|
|
|
1013
1264
|
if not active.is_active(self.config.runtime):
|
|
1014
1265
|
return ""
|
|
1015
|
-
|
|
1266
|
+
level = self._admission()
|
|
1267
|
+
report = active.report(self._learning_rows(), self.seconds_left() if self._budget_told else None,
|
|
1268
|
+
self._situations(), level)
|
|
1016
1269
|
if self._fixture_descriptions:
|
|
1017
1270
|
report += "\n" + "\n".join(self._fixture_descriptions)
|
|
1018
|
-
|
|
1271
|
+
if level == "off":
|
|
1272
|
+
report += "\nThe host admits no code here. Keep what you learn as notes and memory entries."
|
|
1273
|
+
elif level == "own_test":
|
|
1274
|
+
report += ("\nThe host runs a pending proposal's own test between programs: end the program after "
|
|
1275
|
+
"proposing. A proposal whose test passes is active in your next program.")
|
|
1276
|
+
elif not self._fixture_descriptions and self._fixture_status == "unavailable":
|
|
1019
1277
|
report += "\nNo trusted fixture is configured here. Keep observations and leave executable proposals pending."
|
|
1020
1278
|
if self._fixture_feedback:
|
|
1021
1279
|
report += "\n" + "\n".join(f"{key}: {why}" for key, why in self._fixture_feedback.items())
|
|
1022
1280
|
return "\n" + report
|
|
1023
1281
|
|
|
1024
1282
|
def _trial_at_boundary(self, *, replay=False):
|
|
1025
|
-
"""Run at most one trial between programs, or recheck pinned entries at startup.
|
|
1283
|
+
"""Run at most one trial between programs, or recheck pinned entries at startup. What
|
|
1284
|
+
verifies a pending proposal is the admission level in force (_admission): a fixture
|
|
1285
|
+
the host has for it; where the level is own_test and it has none, its own test
|
|
1286
|
+
(_own_test_trial); nothing where the level is off."""
|
|
1026
1287
|
from harness.runtime.learning import fixtures
|
|
1027
|
-
from harness.runtime.learning.learner import CANDIDATES
|
|
1288
|
+
from harness.runtime.learning.learner import CANDIDATES, OWN_TEST
|
|
1028
1289
|
|
|
1029
1290
|
learning = self._learning()
|
|
1030
1291
|
if not learning or self.parent is not None or (replay and not learning["replay"]):
|
|
1031
1292
|
return
|
|
1293
|
+
level = self._admission()
|
|
1294
|
+
if level == "off":
|
|
1295
|
+
self._fixture_status = "off"
|
|
1296
|
+
return
|
|
1032
1297
|
with self._condition:
|
|
1033
|
-
if self._closing or self._pause or self.
|
|
1298
|
+
if self._closing or self._pause or self._busy() or (self._message_waits() and not replay):
|
|
1034
1299
|
return
|
|
1035
1300
|
generation = self._activation_interrupt, self._revision
|
|
1036
1301
|
|
|
1037
1302
|
def cancelled():
|
|
1038
1303
|
with self._condition:
|
|
1039
|
-
return (self._closing or self._pause or self.
|
|
1304
|
+
return (self._closing or self._pause or self._busy()
|
|
1040
1305
|
or generation != (self._activation_interrupt, self._revision))
|
|
1041
1306
|
|
|
1307
|
+
own_test = level == OWN_TEST and not replay
|
|
1042
1308
|
try:
|
|
1043
1309
|
inventory = fixtures.configured(self.profile)
|
|
1044
1310
|
except (OSError, TypeError, ValueError) as error:
|
|
1045
|
-
|
|
1046
|
-
self.
|
|
1047
|
-
|
|
1311
|
+
inventory = ()
|
|
1312
|
+
if self._fixture_status != "unavailable":
|
|
1313
|
+
self.trace.emit("library", action="fixture_unavailable", reason=str(error))
|
|
1048
1314
|
self._fixture_descriptions = fixtures.describe(inventory)
|
|
1049
1315
|
self._fixture_status = "configured" if self._fixture_descriptions else "unavailable"
|
|
1050
|
-
if not self._fixture_descriptions:
|
|
1316
|
+
if not self._fixture_descriptions and not own_test:
|
|
1051
1317
|
return
|
|
1052
1318
|
learner = self._learner()
|
|
1053
|
-
trials = {fixture.get("skill"): fixture for fixture in inventory}
|
|
1319
|
+
trials = {fixture.get("skill"): fixture for fixture in inventory} if self._fixture_descriptions else {}
|
|
1054
1320
|
for index_key, row in learner.index().items():
|
|
1055
1321
|
pending = index_key.startswith(f"{CANDIDATES}/")
|
|
1056
1322
|
if pending and (replay or row.get("evidence") != "proposed"):
|
|
@@ -1062,10 +1328,20 @@ class TaskRun:
|
|
|
1062
1328
|
continue
|
|
1063
1329
|
fixture = trials.get(Path(key).stem)
|
|
1064
1330
|
expected = row.get("revision")
|
|
1065
|
-
if
|
|
1066
|
-
|
|
1331
|
+
if not fixture and own_test and pending and (row.get("demoted") or {}).get("revision") == expected:
|
|
1332
|
+
uses = row["demoted"]
|
|
1333
|
+
self._fixture_feedback[key] = (f"Sent back to pending after failing in use ({uses.get('ok', 0)} of "
|
|
1334
|
+
f"{uses.get('ok', 0) + uses.get('failed', 0)} uses succeeded): rewrite it "
|
|
1335
|
+
"and propose it again.")
|
|
1336
|
+
continue
|
|
1337
|
+
if ((not fixture and not (own_test and pending)) or not expected or expected in self._fixture_attempts
|
|
1338
|
+
or cancelled() or self.seconds_left() <= 0):
|
|
1067
1339
|
continue
|
|
1068
1340
|
self._fixture_attempts.add(expected)
|
|
1341
|
+
if not fixture:
|
|
1342
|
+
# A fixture, where the host has one for the entry, wins: this is the rest.
|
|
1343
|
+
self._own_test_trial(learner, key, expected, cancelled)
|
|
1344
|
+
return
|
|
1069
1345
|
budget = fixtures.Budget(self.seconds_left(), cancelled)
|
|
1070
1346
|
try:
|
|
1071
1347
|
entry = learner.candidate_from(key, pending=pending)
|
|
@@ -1120,6 +1396,116 @@ class TaskRun:
|
|
|
1120
1396
|
if not replay:
|
|
1121
1397
|
return
|
|
1122
1398
|
|
|
1399
|
+
def _own_test_trial(self, learner, key, expected, cancelled):
|
|
1400
|
+
"""Verify one pending proposal by its own test, where the admission level is own_test:
|
|
1401
|
+
the kernel stages the exact revision, the host runs its test on the staged revision
|
|
1402
|
+
as a cell of its own (Kernel.run_staged: bounded, counted as testing, the entry not
|
|
1403
|
+
yet in the programs' namespace) and reads the cell's result. The kernel never grades:
|
|
1404
|
+
a pass makes the host write the proof (method "own_test", this worker's profile, the
|
|
1405
|
+
receipt) and admit the revision as provisional, then commit it; anything else leaves
|
|
1406
|
+
nothing loaded, and a test that ran and did not pass marks the proposal failed."""
|
|
1407
|
+
from harness.runtime.learning import fixtures
|
|
1408
|
+
from harness.runtime.learning.learner import OWN_TEST, OWN_TEST_SECONDS, STAGED_TEST
|
|
1409
|
+
|
|
1410
|
+
def pending(reason, tell=True):
|
|
1411
|
+
if tell:
|
|
1412
|
+
self._fixture_feedback[key] = f"Still pending: {reason[:300]}."
|
|
1413
|
+
self.trace.emit("library", action="trial_pending", key=key, revision=expected, reason=reason)
|
|
1414
|
+
|
|
1415
|
+
try:
|
|
1416
|
+
entry = learner.candidate_from(key, pending=True)
|
|
1417
|
+
except (OSError, ValueError) as error:
|
|
1418
|
+
return pending(str(error))
|
|
1419
|
+
if entry is None or not entry.test or entry.revision().revision != expected:
|
|
1420
|
+
return
|
|
1421
|
+
if entry.test.startswith(STAGED_TEST) and not sandbox_of(self.config.runtime):
|
|
1422
|
+
return pending("its test stages a scene in the practice sandbox, and this run has none")
|
|
1423
|
+
item = entry.revision()
|
|
1424
|
+
budget = fixtures.Budget(self.seconds_left(), cancelled, limit=OWN_TEST_SECONDS + 60.0)
|
|
1425
|
+
observed = {}
|
|
1426
|
+
|
|
1427
|
+
def test(kernel):
|
|
1428
|
+
"""Why the staged revision's own test did not pass; None when it passed."""
|
|
1429
|
+
limit = max(0.01, min(OWN_TEST_SECONDS, self.seconds_left()))
|
|
1430
|
+
self.trace.emit("library", action="trial_started", key=key, revision=expected, method=OWN_TEST)
|
|
1431
|
+
began = time.perf_counter()
|
|
1432
|
+
try:
|
|
1433
|
+
with self._span("program", f"test of {key}") as end:
|
|
1434
|
+
# The test runs as a unit with `limit` as its time: the kernel ends it then.
|
|
1435
|
+
result = kernel.run_staged(entry.test, timeout=limit)
|
|
1436
|
+
end.update(status=result["status"])
|
|
1437
|
+
finally:
|
|
1438
|
+
self.summary["kernel_seconds"] += time.perf_counter() - began
|
|
1439
|
+
_cell_usage(self.summary, result)
|
|
1440
|
+
self.charge_up()
|
|
1441
|
+
why = None
|
|
1442
|
+
if result["status"] != "ok" and "deadline exceeded" in str(result.get("error") or "").lower():
|
|
1443
|
+
why = f"its test did not end within {limit:.0f} s"
|
|
1444
|
+
elif result["status"] != "ok":
|
|
1445
|
+
why = str(result.get("error") or f"its test ended the program with {result['status']}")
|
|
1446
|
+
elif (result.get("state") or {}).get("ended"):
|
|
1447
|
+
why = "its test ended the episode"
|
|
1448
|
+
receipt = {"ok": why is None, "method": OWN_TEST, "run": str(self.directory), "episode_id": self.episode,
|
|
1449
|
+
"revision": expected, "seconds": round(time.perf_counter() - began, 3),
|
|
1450
|
+
**({"error": why[-600:]} if why else {})}
|
|
1451
|
+
observed["receipt"] = receipt
|
|
1452
|
+
self.trace.emit("library", action="trial_observed", key=key, **receipt,
|
|
1453
|
+
stdout=str(result.get("stdout") or "")[-2000:])
|
|
1454
|
+
return why
|
|
1455
|
+
|
|
1456
|
+
def publish():
|
|
1457
|
+
budget.remaining()
|
|
1458
|
+
receipt = observed["receipt"]
|
|
1459
|
+
proof = {"revision": item.revision, "receipt": receipt,
|
|
1460
|
+
"conditions": {"method": OWN_TEST, "profile_sha256": sha256(self.profile)}}
|
|
1461
|
+
value = learner.admit(entry, "provisional", proof=proof, test_run=receipt, expected=expected,
|
|
1462
|
+
check=budget.remaining)
|
|
1463
|
+
self._library_admitted(learner, value, entry.text, entry.kind, entry.source, entry.origin)
|
|
1464
|
+
self._fixture_feedback.pop(key, None)
|
|
1465
|
+
observed["admitted"] = value
|
|
1466
|
+
return value["row"]
|
|
1467
|
+
|
|
1468
|
+
proof = {"revision": item.revision, "conditions": {"method": OWN_TEST, "profile_sha256": sha256(self.profile)}}
|
|
1469
|
+
row = {**(learner.index().get(f"candidates/{key}") or {}), "revision": item.revision, "proof": proof,
|
|
1470
|
+
"evidence": "provisional"}
|
|
1471
|
+
loads = learner.wanted(entry, "provisional") and self._library_eligible(
|
|
1472
|
+
item, row, learner.layers, learner.settings["load"])
|
|
1473
|
+
try:
|
|
1474
|
+
outcome = self._activate_library([(item, row)], publish=publish, budget=budget, test=test, commit=loads)
|
|
1475
|
+
except (OSError, ValueError, RuntimeError, TimeoutError) as error:
|
|
1476
|
+
if isinstance(error, KernelDied):
|
|
1477
|
+
raise
|
|
1478
|
+
outcome = {"status": "error", "error": str(error)}
|
|
1479
|
+
receipt = observed.get("receipt")
|
|
1480
|
+
active = (self.config.runtime.get("library") or {}).get("trigger") == "active"
|
|
1481
|
+
if "admitted" in observed:
|
|
1482
|
+
if not active:
|
|
1483
|
+
self._note(f"The host ran the test of your proposal {key} and it passed"
|
|
1484
|
+
+ (": it is active from your next program." if outcome.get("status") == "activated"
|
|
1485
|
+
else "; it waits for validation before it is loaded."))
|
|
1486
|
+
return
|
|
1487
|
+
if receipt is not None and not receipt["ok"]:
|
|
1488
|
+
try:
|
|
1489
|
+
with self._admission_lock, self._condition:
|
|
1490
|
+
learner.record_trial(key, expected, item, pending=True, evidence="failed", receipt=receipt,
|
|
1491
|
+
check=lambda: None)
|
|
1492
|
+
except (OSError, ValueError) as error:
|
|
1493
|
+
self.trace.emit("library", action="trial_pending", key=key, revision=expected, reason=str(error))
|
|
1494
|
+
self._fixture_feedback[key] = f"Its own test failed when the host ran it: {receipt['error'][-300:]}"
|
|
1495
|
+
if not active:
|
|
1496
|
+
self._note(f"The host ran the test of your proposal {key} and it failed: {receipt['error'][-300:]}\n"
|
|
1497
|
+
"It stays out of your library; rewrite it and propose it again if you still need it.")
|
|
1498
|
+
return
|
|
1499
|
+
# No verdict: the revision could not be staged, or the boundary was taken (a pause, a
|
|
1500
|
+
# message, an event). One more try at a later boundary, then it waits.
|
|
1501
|
+
if outcome.get("status") == "cancelled" or receipt is not None:
|
|
1502
|
+
if expected not in self._trial_interrupted:
|
|
1503
|
+
self._trial_interrupted.add(expected)
|
|
1504
|
+
self._fixture_attempts.discard(expected)
|
|
1505
|
+
pending("its test was interrupted", tell=False)
|
|
1506
|
+
else:
|
|
1507
|
+
pending(f"it could not be loaded for its test: {outcome.get('error') or 'unknown'}")
|
|
1508
|
+
|
|
1123
1509
|
def _startup_library(self, learning):
|
|
1124
1510
|
library = self._learner().library
|
|
1125
1511
|
with self._admission_lock:
|
|
@@ -1150,22 +1536,28 @@ class TaskRun:
|
|
|
1150
1536
|
self._activation_queue[item.key] = (item, row)
|
|
1151
1537
|
|
|
1152
1538
|
def _library_eligible(self, item, row, layers, load):
|
|
1539
|
+
if executable(item.key) and row and row.get("origin") != "hand" and self._admission() == "off":
|
|
1540
|
+
return False # the host admits no learned code: it is not loaded either
|
|
1153
1541
|
conditions = (row.get("proof") or {}).get("conditions") or {}
|
|
1154
1542
|
if isinstance(conditions, dict) and conditions.get("profile_sha256") not in (None, sha256(self.profile)):
|
|
1155
1543
|
return False
|
|
1156
1544
|
return eligible(item, row, layers, load)
|
|
1157
1545
|
|
|
1158
|
-
def _activate_library(self, selected, *, startup=False, publish=None, budget=None):
|
|
1546
|
+
def _activate_library(self, selected, *, startup=False, publish=None, budget=None, test=None, commit=True):
|
|
1547
|
+
"""Stage `selected` in the kernel, publish (the admission, when given) and commit.
|
|
1548
|
+
`test(kernel)` runs after staging and says why the staged revision must not be
|
|
1549
|
+
published (None: go on); without `commit` the staged revision is admitted and not
|
|
1550
|
+
loaded. {"status": "activated", "admitted", "cancelled" or "error"[, "error"]}."""
|
|
1159
1551
|
if not selected:
|
|
1160
|
-
return
|
|
1552
|
+
return {"status": "cancelled"}
|
|
1161
1553
|
def can_switch():
|
|
1162
|
-
return startup or (not (self.
|
|
1554
|
+
return startup or (not (self._busy() or self._message_waits())
|
|
1163
1555
|
and (publish is not None or all(
|
|
1164
1556
|
self._activation_queue.get(item.key, (None,))[0] == item for item, _ in selected)))
|
|
1165
1557
|
|
|
1166
1558
|
with self._condition:
|
|
1167
1559
|
if self._pause or self._closing or not can_switch():
|
|
1168
|
-
return
|
|
1560
|
+
return {"status": "cancelled"}
|
|
1169
1561
|
kernel = self.kernel
|
|
1170
1562
|
interrupt = self._activation_interrupt
|
|
1171
1563
|
for item, _ in selected:
|
|
@@ -1174,21 +1566,30 @@ class TaskRun:
|
|
|
1174
1566
|
with self._span("program", "library"):
|
|
1175
1567
|
prepared = kernel.load_library("prepare", entries, **({"timeout": budget.remaining()} if budget else {}))
|
|
1176
1568
|
rejected = None
|
|
1569
|
+
if prepared["status"] == "ok" and test is not None:
|
|
1570
|
+
try:
|
|
1571
|
+
rejected = test(kernel)
|
|
1572
|
+
except KernelDied:
|
|
1573
|
+
raise
|
|
1574
|
+
except (OSError, ValueError, RuntimeError) as error:
|
|
1575
|
+
rejected = str(error)
|
|
1177
1576
|
with self._admission_lock, self._condition:
|
|
1178
1577
|
cancelled = (self._closing or self._pause or self.kernel is not kernel
|
|
1179
1578
|
or self._activation_interrupt != interrupt or not can_switch())
|
|
1180
|
-
if not cancelled and prepared["status"] == "ok" and publish is not None:
|
|
1579
|
+
if not cancelled and prepared["status"] == "ok" and rejected is None and publish is not None:
|
|
1181
1580
|
try:
|
|
1182
1581
|
selected = [(selected[0][0], publish())]
|
|
1183
1582
|
except (OSError, ValueError, RuntimeError) as error:
|
|
1184
1583
|
rejected = str(error)
|
|
1185
|
-
if prepared["status"] != "ok" or cancelled or rejected:
|
|
1584
|
+
if prepared["status"] != "ok" or cancelled or rejected or not commit:
|
|
1186
1585
|
if prepared["status"] == "ok" and self.kernel is kernel:
|
|
1187
1586
|
kernel.load_library("discard")
|
|
1587
|
+
if prepared["status"] == "ok" and not cancelled and not rejected:
|
|
1588
|
+
return {"status": "admitted"}
|
|
1188
1589
|
self.trace.emit("library", action="load_failed", status="cancelled" if cancelled else "error",
|
|
1189
1590
|
error=rejected or prepared.get("error"), revisions={item.key: item.revision for item, _ in selected})
|
|
1190
1591
|
if cancelled or rejected:
|
|
1191
|
-
return
|
|
1592
|
+
return {"status": "cancelled" if cancelled else "error", "error": rejected}
|
|
1192
1593
|
if startup and any(not hand for _, hand in entries):
|
|
1193
1594
|
self._activate_library([(item, row) for item, row in selected
|
|
1194
1595
|
if row.get("origin", "hand") == "hand"], startup=True)
|
|
@@ -1199,7 +1600,7 @@ class TaskRun:
|
|
|
1199
1600
|
for item, _ in selected:
|
|
1200
1601
|
if self._activation_queue.get(item.key, (None,))[0] == item:
|
|
1201
1602
|
del self._activation_queue[item.key]
|
|
1202
|
-
return
|
|
1603
|
+
return {"status": "error", "error": prepared.get("error")}
|
|
1203
1604
|
# The boundary check reserved the switch. The kernel reader can drain notes while
|
|
1204
1605
|
# the commit is acknowledged and the decision loop still owns program scheduling.
|
|
1205
1606
|
result = kernel.load_library("commit")
|
|
@@ -1225,7 +1626,7 @@ class TaskRun:
|
|
|
1225
1626
|
self.config.system_prompt = (self.config.system_prompt.replace(prior, updated, 1)
|
|
1226
1627
|
if prior else self.config.system_prompt + updated)
|
|
1227
1628
|
self._library_prompt = updated
|
|
1228
|
-
|
|
1629
|
+
return {"status": "activated"}
|
|
1229
1630
|
|
|
1230
1631
|
@staticmethod
|
|
1231
1632
|
def _active_row(item, row):
|
|
@@ -1237,14 +1638,14 @@ class TaskRun:
|
|
|
1237
1638
|
return {**row_data, "key": item.key, "revision": item.revision, "sha256": item.source_sha256,
|
|
1238
1639
|
"source": row.get("origin", "hand"), "summary": row.get("summary", ""),
|
|
1239
1640
|
"text": item.source.decode("utf-8")[:400], "active": True,
|
|
1240
|
-
**({"verification": row["proof"]} if row.get("proof") else {})}
|
|
1641
|
+
**({"verification": row["proof"], "admitted_by": admitted_by(row)} if row.get("proof") else {})}
|
|
1241
1642
|
|
|
1242
1643
|
def _startup_holdings(self):
|
|
1243
1644
|
from harness.runtime.learning import active
|
|
1244
1645
|
|
|
1245
1646
|
situations = self._situations()
|
|
1246
1647
|
rows = [{"section": item.key.split("/", 1)[0], "name": Path(item.key).stem,
|
|
1247
|
-
"summary": row.get("summary"), "passed": bool(row.get("proof"))}
|
|
1648
|
+
"summary": row.get("summary"), "passed": bool(row.get("proof")), "admitted_by": admitted_by(row)}
|
|
1248
1649
|
for item, row in self._active_library.values()]
|
|
1249
1650
|
holdings = active.holdings(rows, situations)
|
|
1250
1651
|
return f"\n{holdings}\n" if holdings else ""
|
|
@@ -1304,7 +1705,6 @@ class TaskRun:
|
|
|
1304
1705
|
with self._condition:
|
|
1305
1706
|
self.config = copy(self.config)
|
|
1306
1707
|
self.config.system_prompt += f"\nLearned in this run: {str(request.get('text', '')).strip()}\n"
|
|
1307
|
-
self.invalidate_lookahead()
|
|
1308
1708
|
else:
|
|
1309
1709
|
self.trace.emit("library", action="rejected", kind=request.get("entry"), name=request.get("name"),
|
|
1310
1710
|
reason=answer["error"])
|
|
@@ -1373,11 +1773,6 @@ class TaskRun:
|
|
|
1373
1773
|
"""A program asks for sub-agents (skills/agents): done on a thread of its own (a wait
|
|
1374
1774
|
takes long; the kernel's notes must go on), answered through the kernel."""
|
|
1375
1775
|
|
|
1376
|
-
if request.get("action") == "wait" and request.get("own", True):
|
|
1377
|
-
with self._condition:
|
|
1378
|
-
request["interrupt_at"] = self._interruptions
|
|
1379
|
-
self.invalidate_lookahead()
|
|
1380
|
-
|
|
1381
1776
|
def answer():
|
|
1382
1777
|
if self.agents is None:
|
|
1383
1778
|
value = {"error": "this conversation cannot start sub-agents"}
|
|
@@ -1422,33 +1817,36 @@ class TaskRun:
|
|
|
1422
1817
|
|
|
1423
1818
|
def _wait(self, status: str, ready, **detail):
|
|
1424
1819
|
"""Wait in `status` until `ready()` holds (under the lock); Closed when the user
|
|
1425
|
-
closes the conversation meanwhile."""
|
|
1820
|
+
closes the conversation meanwhile, Interrupted when they end the turn."""
|
|
1426
1821
|
self._set_status(status, **detail)
|
|
1427
1822
|
began = time.perf_counter()
|
|
1428
1823
|
with self._condition:
|
|
1429
|
-
while not self._closing and not ready():
|
|
1824
|
+
while not self._closing and not self._interrupt and not ready():
|
|
1430
1825
|
self._condition.wait()
|
|
1431
|
-
closing = self._closing
|
|
1826
|
+
closing, interrupted = self._closing, self._interrupt
|
|
1432
1827
|
self._waited += time.perf_counter() - began
|
|
1433
1828
|
if closing:
|
|
1434
1829
|
raise Closed()
|
|
1830
|
+
if interrupted:
|
|
1831
|
+
raise Interrupted()
|
|
1435
1832
|
self._set_status(RUNNING)
|
|
1436
1833
|
|
|
1437
|
-
def _take_inbox(self) -> list[str]:
|
|
1438
|
-
with self._condition:
|
|
1439
|
-
pending, self._inbox = self._inbox, []
|
|
1440
|
-
return pending
|
|
1441
|
-
|
|
1442
1834
|
def _checkpoint(self) -> None:
|
|
1443
1835
|
"""Between steps: stop when closing, wait while paused."""
|
|
1444
1836
|
with self._condition:
|
|
1445
|
-
closing, paused = self._closing, self._pause
|
|
1837
|
+
closing, interrupted, paused = self._closing, self._interrupt, self._pause
|
|
1446
1838
|
if closing:
|
|
1447
1839
|
raise Closed()
|
|
1840
|
+
if interrupted:
|
|
1841
|
+
raise Interrupted()
|
|
1448
1842
|
if paused:
|
|
1449
|
-
self._wait(PAUSED, lambda: not self._pause or
|
|
1843
|
+
self._wait(PAUSED, lambda: not self._pause or self._message_waits())
|
|
1450
1844
|
with self._condition:
|
|
1451
1845
|
self._pause = False
|
|
1846
|
+
kernel = self.kernel
|
|
1847
|
+
if kernel is not None:
|
|
1848
|
+
# (A message ends the pause as Continue does: the units may act again.)
|
|
1849
|
+
kernel.resume()
|
|
1452
1850
|
self._handed_back()
|
|
1453
1851
|
|
|
1454
1852
|
def _handed_back(self) -> None:
|
|
@@ -1463,72 +1861,126 @@ class TaskRun:
|
|
|
1463
1861
|
except (ValueError, RuntimeError, OSError):
|
|
1464
1862
|
pass
|
|
1465
1863
|
|
|
1466
|
-
def
|
|
1467
|
-
"""
|
|
1864
|
+
def _woken(self) -> bool:
|
|
1865
|
+
"""Whether something reached the model that starts a turn: a notice that wakes (the
|
|
1866
|
+
end of a unit that was cancelled is told with the next turn, it starts none)."""
|
|
1867
|
+
return bool(self._notices.pending(waking=True)) or self._run_end is not None
|
|
1868
|
+
|
|
1869
|
+
def _idle(self, reason: str) -> None:
|
|
1870
|
+
"""Between turns: the conversation waits, IDLE with the reason its last turn ended
|
|
1871
|
+
for, with everything it has (messages, the kernel and its units, rules, facts, the
|
|
1872
|
+
environment) until something reaches the model. Closed when the user or the host
|
|
1873
|
+
ends the conversation meanwhile."""
|
|
1468
1874
|
self.waiting_since = time.time()
|
|
1875
|
+
# No turn runs: the next one has its whole time.
|
|
1876
|
+
self._turn_began = None
|
|
1469
1877
|
try:
|
|
1470
|
-
self.
|
|
1878
|
+
self._set_status(IDLE, reason=reason)
|
|
1879
|
+
with self._condition:
|
|
1880
|
+
while not self._closing and not self._woken():
|
|
1881
|
+
self._condition.wait()
|
|
1882
|
+
# (An interrupt asked for when the turn was over ends nothing.)
|
|
1883
|
+
closing, self._interrupt = self._closing, False
|
|
1884
|
+
if closing:
|
|
1885
|
+
raise Closed()
|
|
1886
|
+
self._set_status(RUNNING)
|
|
1471
1887
|
finally:
|
|
1472
1888
|
self.waiting_since = None
|
|
1473
|
-
self.reason = "user"
|
|
1474
1889
|
|
|
1475
|
-
def
|
|
1476
|
-
"""
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1890
|
+
def _tell(self, content) -> None:
|
|
1891
|
+
"""Add a user message to the history."""
|
|
1892
|
+
self.messages.append({"role": "user", "content": content})
|
|
1893
|
+
|
|
1894
|
+
def _came_to(self, content) -> None:
|
|
1895
|
+
"""What the model's last output came to (its program's handle, the end of its wait,
|
|
1896
|
+
why nothing ran): the first thing its next call is told (_deliver)."""
|
|
1897
|
+
self._outcome += content_parts(content)
|
|
1898
|
+
|
|
1899
|
+
def _say(self, parts) -> None:
|
|
1900
|
+
"""Tell the model `parts` with its next call: in the user message that ends the
|
|
1901
|
+
history, or as a new one after a reply."""
|
|
1902
|
+
if self.messages and self.messages[-1]["role"] == "user":
|
|
1483
1903
|
self._extend_request(parts)
|
|
1484
1904
|
else:
|
|
1485
1905
|
self._tell(parts)
|
|
1486
1906
|
|
|
1487
|
-
def
|
|
1488
|
-
"""
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1907
|
+
def _tree_told(self) -> str:
|
|
1908
|
+
"""What this turn is told of the tree of reflexes: what changed in it since the
|
|
1909
|
+
model was last told (runtime/reactive/tree.py told)."""
|
|
1910
|
+
nodes = getattr(self, "_reflex_nodes", {})
|
|
1911
|
+
text = tree_told(nodes, getattr(self, "_reflex_told", {}))
|
|
1912
|
+
self._reflex_told = dict(nodes)
|
|
1913
|
+
return text
|
|
1914
|
+
|
|
1915
|
+
def _compact_if_due(self, loop) -> None:
|
|
1916
|
+
"""Compaction: when the context the model's last call read has reached
|
|
1917
|
+
runtime.compact_at_tokens, the model writes a summary of the conversation
|
|
1918
|
+
(decisions.COMPACT), which then stands in the first message in place of every
|
|
1919
|
+
message before the recent ones (messages.recent_start). Interactive: the summary is
|
|
1920
|
+
followed by everything the user said (they hold the task). The system prompt, the
|
|
1921
|
+
kernel, the library, the tree of reflexes and the running units are as they were;
|
|
1922
|
+
what a turn is told of the tree and of what was learned is told anew."""
|
|
1923
|
+
import re
|
|
1924
|
+
from copy import deepcopy
|
|
1494
1925
|
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1926
|
+
limit, messages = self.config.runtime.get("compact_at_tokens"), self.messages
|
|
1927
|
+
if (not limit or self._context_tokens < int(limit) or len(messages) <= 3
|
|
1928
|
+
or loop.malformed_from is not None):
|
|
1929
|
+
return
|
|
1930
|
+
tail = messages[recent_start(messages, decisions.KEEP_MIN_TOKENS, decisions.KEEP_MIN_MESSAGES,
|
|
1931
|
+
decisions.KEEP_MAX_TOKENS):]
|
|
1932
|
+
request = deepcopy(messages) if self._images else text_only(deepcopy(messages))
|
|
1933
|
+
ask = {"type": "input_text", "text": decisions.COMPACT}
|
|
1934
|
+
if request[-1]["role"] == "user":
|
|
1935
|
+
content = request[-1]["content"]
|
|
1936
|
+
request[-1]["content"] = [*([{"type": "input_text", "text": content}] if isinstance(content, str)
|
|
1937
|
+
else content), ask]
|
|
1938
|
+
else:
|
|
1939
|
+
request.append({"role": "user", "content": [ask]})
|
|
1940
|
+
before, tokens = len(messages), self._context_tokens
|
|
1941
|
+
reply = self._ask_role("compaction", self.chat, self.config.system_prompt, request)
|
|
1942
|
+
found = re.search(r"<summary>(.*?)(?:</summary>|$)", reply.text, re.S)
|
|
1943
|
+
text = (found.group(1) if found else "").strip()
|
|
1944
|
+
if not text:
|
|
1945
|
+
self._compact_failures += 1
|
|
1946
|
+
self.trace.emit("compaction", action="failed", tokens=tokens, attempts=self._compact_failures)
|
|
1947
|
+
if self._compact_failures >= 3:
|
|
1948
|
+
raise RuntimeError("The conversation could not be compacted: the model gave no summary")
|
|
1949
|
+
return
|
|
1950
|
+
if self.interactive and self._said:
|
|
1951
|
+
text += "\n\nThe user's messages so far, oldest first:\n" + "\n".join(
|
|
1952
|
+
f"{number}. {said}" for number, said in enumerate(self._said, 1))
|
|
1953
|
+
with self._condition:
|
|
1954
|
+
first = dict(messages[0])
|
|
1955
|
+
content = first["content"]
|
|
1956
|
+
first["content"] = [
|
|
1957
|
+
*([{"type": "input_text", "text": content}] if isinstance(content, str)
|
|
1958
|
+
else [part for part in content if not part.get("summary")]),
|
|
1959
|
+
{"type": "input_text", "text": decisions.COMPACTED.format(summary=text), "summary": True}]
|
|
1960
|
+
messages[:] = [first, *tail]
|
|
1961
|
+
# The next request's record replaces every message the trace has (rebuild_messages).
|
|
1962
|
+
self._replaces, self._logged = self._replaces + self._logged, 0
|
|
1963
|
+
self._context_tokens, self._compact_failures = 0, 0
|
|
1964
|
+
self._reflex_told, self._learning_told = {}, None
|
|
1965
|
+
self.trace.emit("compaction", action="done", tokens=tokens, messages_before=before, messages=len(messages),
|
|
1966
|
+
summary=text)
|
|
1967
|
+
self._save()
|
|
1506
1968
|
|
|
1507
1969
|
def seconds_left(self) -> float:
|
|
1508
|
-
"""What the
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
ran = time.perf_counter() -
|
|
1970
|
+
"""What the turn has left of its wall time (max_wall_seconds): the time it ran, less
|
|
1971
|
+
the time it waited for the user, counts; the whole budget before it starts."""
|
|
1972
|
+
began = self._turn_began
|
|
1973
|
+
ran = time.perf_counter() - began - self._waited if began is not None else 0.0
|
|
1512
1974
|
return float(self.config.max_wall_seconds) - ran
|
|
1513
1975
|
|
|
1514
1976
|
def _cell_timeout(self, *, policy=False) -> float:
|
|
1515
|
-
"""The program's declared deadline, bounded by the
|
|
1977
|
+
"""The program's declared deadline, bounded by the turn."""
|
|
1516
1978
|
seconds = self.seconds_left()
|
|
1517
1979
|
configured = self.config.runtime.get("cell_timeout_seconds")
|
|
1518
1980
|
if configured is not None and not policy:
|
|
1519
1981
|
seconds = min(seconds, float(configured))
|
|
1520
1982
|
return max(0.01, seconds)
|
|
1521
1983
|
|
|
1522
|
-
def _program_deadline(self, number, episode_deadline):
|
|
1523
|
-
with self._condition:
|
|
1524
|
-
if self._cell_running and not self._program_completing and self.summary["cells"] == number:
|
|
1525
|
-
if episode_deadline:
|
|
1526
|
-
self._wall_up = True
|
|
1527
|
-
self.kernel.pause()
|
|
1528
|
-
else:
|
|
1529
|
-
self._program_timed_out = True
|
|
1530
|
-
self._stop_program_for_decision()
|
|
1531
|
-
|
|
1532
1984
|
def _stuck(self, program: str, result: dict) -> None:
|
|
1533
1985
|
"""Interactive: note the same program, or the same unhandled event, three times
|
|
1534
1986
|
in a row (after OpenHands' stuck detector). Only noted; nothing changes."""
|
|
@@ -1542,9 +1994,10 @@ class TaskRun:
|
|
|
1542
1994
|
self._recent = []
|
|
1543
1995
|
|
|
1544
1996
|
def run(self) -> dict:
|
|
1545
|
-
"""Run the
|
|
1546
|
-
model calls, tokens, cost, times), also written with the
|
|
1547
|
-
the directory. An error ends the
|
|
1997
|
+
"""Run the conversation to its end and return its summary (the status of its last
|
|
1998
|
+
turn, evaluator outcome, model calls, tokens, cost, times), also written with the
|
|
1999
|
+
trace and manifest under the directory at every turn's end. An error ends the turn
|
|
2000
|
+
(before the first one: the conversation) with status "error" instead of raising."""
|
|
1548
2001
|
from harness.runtime.decision.model import USAGE, ChatModel
|
|
1549
2002
|
from harness.config.agent import image_roles, model_services
|
|
1550
2003
|
|
|
@@ -1591,6 +2044,12 @@ class TaskRun:
|
|
|
1591
2044
|
# The embodiment profile is part of the frozen configuration (App. A).
|
|
1592
2045
|
self.profile = {key: value for key, value in env.call("profile").items()
|
|
1593
2046
|
if key not in ("task", "step_limit", "fk_check_error_m")}
|
|
2047
|
+
if self.profile.get("tells"):
|
|
2048
|
+
# The worker tells what happens by itself: it is given where (inbound.py).
|
|
2049
|
+
from harness.runtime.task.inbound import Inbound
|
|
2050
|
+
|
|
2051
|
+
self._inbound = Inbound(self)
|
|
2052
|
+
env.call("tell_to", episode_id=self.episode, url=self._inbound.url, token=self._inbound.token)
|
|
1594
2053
|
# The skills, prompt and session were chosen for one embodiment kind; the worker
|
|
1595
2054
|
# says which it drives.
|
|
1596
2055
|
if embodiment_kind(self.profile) != embodiment:
|
|
@@ -1645,12 +2104,13 @@ class TaskRun:
|
|
|
1645
2104
|
if self._managed_workspace is not None:
|
|
1646
2105
|
self._managed_dispatcher = self._managed_workspace.dispatcher(self, self.workspace_kind)
|
|
1647
2106
|
operations = ", ".join(self._managed_dispatcher.options()["operations"])
|
|
1648
|
-
scope = f"\nThis isolated workspace exposes only these
|
|
2107
|
+
scope = f"\nThis isolated workspace exposes only these operations: {operations}.\n"
|
|
1649
2108
|
config.system_prompt += scope
|
|
1650
2109
|
config.base_prompt += scope
|
|
1651
2110
|
self.kernel = Kernel(DEFAULT_BOOTSTRAP, {}, managed=self._managed_dispatcher,
|
|
1652
2111
|
managed_launcher=self._managed_workspace.program_command,
|
|
1653
|
-
log_path=directory / "kernel.log", start_timeout=30
|
|
2112
|
+
log_path=directory / "kernel.log", start_timeout=30,
|
|
2113
|
+
on_span=self._kernel_span)
|
|
1654
2114
|
else:
|
|
1655
2115
|
# (The verifier is asked through the conversation, a thought: control.think.)
|
|
1656
2116
|
services = model_services(runtime, roles)
|
|
@@ -1673,6 +2133,8 @@ class TaskRun:
|
|
|
1673
2133
|
"skills": skills,
|
|
1674
2134
|
**kernel_services, **image_clients,
|
|
1675
2135
|
"memory": self._recall_library,
|
|
2136
|
+
# The facts an earlier process of this conversation kept.
|
|
2137
|
+
**({"facts": self._facts} if self._facts else {}),
|
|
1676
2138
|
# App. A: programs see only the Python installation and the harness source,
|
|
1677
2139
|
# and reach only the services named here (runtime/safety/isolation.py).
|
|
1678
2140
|
"isolation": runtime.get("isolation", True),
|
|
@@ -1689,6 +2151,8 @@ class TaskRun:
|
|
|
1689
2151
|
**({"sandbox": sandbox_of(runtime)} if sandbox_of(runtime) else {}),
|
|
1690
2152
|
# The worker's private files (a game's instance): never a program's.
|
|
1691
2153
|
**({"private_paths": self.profile["private_paths"]} if self.profile.get("private_paths") else {}),
|
|
2154
|
+
# What the environment gives to act with, where programs can read it (core/worker.py).
|
|
2155
|
+
**({"references": list(self.profile["references"])} if self.profile.get("references") else {}),
|
|
1692
2156
|
}, python=kernel_python(runtime), cwd=str(self.program_directory) if self.program_directory else None,
|
|
1693
2157
|
log_path=directory / "kernel.log",
|
|
1694
2158
|
on_span=self._kernel_span)
|
|
@@ -1737,17 +2201,35 @@ class TaskRun:
|
|
|
1737
2201
|
if self.cameras:
|
|
1738
2202
|
opening += (f"Views of this environment: {', '.join(self.cameras)}. None is shown until a "
|
|
1739
2203
|
"program looks: look().\n")
|
|
2204
|
+
if self.profile.get("references"):
|
|
2205
|
+
opening += ("What this environment gives you to act with, its documentation and source, is in these "
|
|
2206
|
+
"folders, which a program can read: " + ", ".join(map(str, self.profile["references"])) + "\n")
|
|
1740
2207
|
self.messages = [{"role": "user", "content": [
|
|
1741
2208
|
{"type": "input_text",
|
|
1742
2209
|
"text": opening + f"{told['state_label']}: "
|
|
1743
2210
|
f"{state_text(observation.state, embodiment, self._steps_shown())}"},
|
|
1744
2211
|
*first]}]
|
|
1745
|
-
|
|
1746
|
-
|
|
2212
|
+
if self._restore is not None:
|
|
2213
|
+
self._continue(self._restore, f"{told['state_label']}: "
|
|
2214
|
+
f"{state_text(observation.state, embodiment, self._steps_shown())}")
|
|
2215
|
+
self._idle("resumed")
|
|
2216
|
+
while True:
|
|
2217
|
+
reason = self._turn(skills, told)
|
|
2218
|
+
if not self.interactive or self.kernel.process.poll() is not None:
|
|
2219
|
+
# Nobody is there for another turn, or the kernel died with what it held.
|
|
2220
|
+
break
|
|
2221
|
+
self._turn_result(reason, open=True)
|
|
2222
|
+
self._idle(reason)
|
|
2223
|
+
if reason == "answer":
|
|
2224
|
+
# The benchmark's convention at the end of an episode that was answered.
|
|
2225
|
+
self.kernel.cancel()
|
|
2226
|
+
_wrap_up(self.kernel, trace)
|
|
1747
2227
|
except Closed:
|
|
1748
|
-
#
|
|
1749
|
-
#
|
|
1750
|
-
|
|
2228
|
+
# Closed by the user or the host: the benchmark's convention ends it. Between
|
|
2229
|
+
# turns the summary keeps how the last turn ended; a turn that was cut short
|
|
2230
|
+
# (or a conversation that had none) is "closed".
|
|
2231
|
+
if self._turn_began is not None or not self._turns:
|
|
2232
|
+
summary["status"] = "closed"
|
|
1751
2233
|
if self.kernel is not None:
|
|
1752
2234
|
_wrap_up(self.kernel, trace)
|
|
1753
2235
|
except KeyboardInterrupt:
|
|
@@ -1762,40 +2244,147 @@ class TaskRun:
|
|
|
1762
2244
|
self._close_episode(token)
|
|
1763
2245
|
return summary
|
|
1764
2246
|
|
|
1765
|
-
def
|
|
2247
|
+
def _continue(self, kept: dict, state: str) -> None:
|
|
2248
|
+
"""Go on with what an earlier process kept of this conversation (saved()): its
|
|
2249
|
+
messages, what the user said, its turns and the model's spending so far (its facts
|
|
2250
|
+
are in the new kernel already). What died with that process (the namespace, the
|
|
2251
|
+
rules, the units that ran) is a note the model reads with its next prompt, with
|
|
2252
|
+
the state as it is now; it starts no turn."""
|
|
2253
|
+
summary = self.summary
|
|
2254
|
+
self.messages = list(kept["messages"])
|
|
2255
|
+
self._said = [str(said) for said in kept.get("said") or self._said]
|
|
2256
|
+
self._turns = int(kept.get("turns") or 0)
|
|
2257
|
+
self._replied_at = kept.get("replied")
|
|
2258
|
+
spent = kept.get("summary") or {}
|
|
2259
|
+
for key, value in spent.items():
|
|
2260
|
+
if key in summary and type(value) in (int, float):
|
|
2261
|
+
summary[key] = value
|
|
2262
|
+
summary["decisions"] = dict(spent.get("decisions") or {})
|
|
2263
|
+
if isinstance(spent.get("status"), str):
|
|
2264
|
+
# (How its last turn ended, until its next one ends.)
|
|
2265
|
+
summary["status"] = spent["status"]
|
|
2266
|
+
self._logged = len(self.messages)
|
|
2267
|
+
units = [str(unit) for unit in kept.get("units") or ()]
|
|
2268
|
+
self._say([{"type": "input_text", "text": decisions.CONTINUED.format(
|
|
2269
|
+
units=": " + "; ".join(units) if units else "") + "\n" + state}])
|
|
2270
|
+
self.trace.emit("resumed", messages=len(kept["messages"]), facts=sorted(self._facts), units=units)
|
|
2271
|
+
|
|
2272
|
+
def _save(self) -> None:
|
|
2273
|
+
"""Write what continues this conversation in another process (CONVERSATION_FILE), at
|
|
2274
|
+
every call of the model and every turn's end: with a user or without one (a run
|
|
2275
|
+
nobody talks to is continued with a message as well). A sub-agent lives and ends
|
|
2276
|
+
with the process of the agent that started it."""
|
|
2277
|
+
if self.parent is not None:
|
|
2278
|
+
return
|
|
2279
|
+
with self._condition:
|
|
2280
|
+
kept = json.dumps({
|
|
2281
|
+
"schema_version": 1, "messages": self.messages, "said": self._said, "facts": self._facts,
|
|
2282
|
+
"units": [unit_text(identifier, unit, printed=False) for identifier, unit in self._units.items()],
|
|
2283
|
+
"turns": self._turns, "replied": self._replied_at, "summary": self.summary}, ensure_ascii=False)
|
|
2284
|
+
atomic_write(self.directory / CONVERSATION_FILE, kept)
|
|
2285
|
+
|
|
2286
|
+
def _turn(self, skills, told) -> str:
|
|
2287
|
+
"""One turn, from what started it to how it ended (a key of decisions.TURN_ENDS,
|
|
2288
|
+
whose status the summary takes). Its limits are its own: the model's calls and the
|
|
2289
|
+
wall time begin anew. An error ends the turn; so does cancel(). Only an answer
|
|
2290
|
+
leaves the model's units running: after any other end they are cancelled, so that
|
|
2291
|
+
nothing starts a turn until the user says how to go on."""
|
|
2292
|
+
summary, trace = self.summary, self.trace
|
|
2293
|
+
summary.pop("error", None)
|
|
2294
|
+
with self._condition:
|
|
2295
|
+
self._turns += 1
|
|
2296
|
+
self._turn_began, self._waited = time.perf_counter(), 0.0
|
|
2297
|
+
self._calls_before = summary["model_calls"] + self.agents_usage["model_calls"]
|
|
2298
|
+
self._iterations = 0
|
|
2299
|
+
try:
|
|
2300
|
+
reason = self._loop(skills, told)
|
|
2301
|
+
except Interrupted:
|
|
2302
|
+
trace.emit("interrupted")
|
|
2303
|
+
reason = "interrupted"
|
|
2304
|
+
except Closed:
|
|
2305
|
+
raise
|
|
2306
|
+
except Exception as error:
|
|
2307
|
+
summary["error"] = f"{type(error).__name__}: {error}"
|
|
2308
|
+
trace.emit("error", message=summary["error"])
|
|
2309
|
+
reason = "error"
|
|
2310
|
+
with self._condition:
|
|
2311
|
+
self._interrupt = False
|
|
2312
|
+
if reason in decisions.TURN_STOPS:
|
|
2313
|
+
# What reached the model in this turn and met no call of it is in its
|
|
2314
|
+
# history: it starts no turn by itself, the user's next message does.
|
|
2315
|
+
parts = [part for notice in self._notices.take(kinds=("message", "event", "note"))
|
|
2316
|
+
for part in self._notice_parts(notice)]
|
|
2317
|
+
if parts:
|
|
2318
|
+
self._say(parts)
|
|
2319
|
+
summary["status"] = decisions.TURN_ENDS[reason]
|
|
2320
|
+
if reason != "answer":
|
|
2321
|
+
self.kernel.cancel()
|
|
2322
|
+
return reason
|
|
2323
|
+
|
|
2324
|
+
def _turn_result(self, reason: str, open: bool = False) -> None:
|
|
2325
|
+
"""The end of a turn: learning reflects on it, the summary is written (`open`: the
|
|
2326
|
+
conversation goes on, which the file says, so that nothing takes it for ended),
|
|
2327
|
+
and the turn's result record says why it ended, in which session, the model's
|
|
2328
|
+
calls in it (its sub-agents' counted), the conversation's cost so far and the
|
|
2329
|
+
turn's seconds."""
|
|
2330
|
+
summary = self.summary
|
|
2331
|
+
began = self._turn_began
|
|
2332
|
+
if self.kernel is not None and began is not None:
|
|
2333
|
+
self._reflect_learning()
|
|
2334
|
+
self._write_summary(open)
|
|
2335
|
+
self.trace.emit(
|
|
2336
|
+
"turn_result", reason=reason, turn=self._turns, session=(self.identity or {}).get("session"),
|
|
2337
|
+
model_calls=summary["model_calls"] + self.agents_usage["model_calls"] - self._calls_before,
|
|
2338
|
+
cost_usd=round(summary["cost_usd"] + self.agents_usage["cost_usd"], 6),
|
|
2339
|
+
seconds=round(time.perf_counter() - began - self._waited, 3) if began is not None else 0.0,
|
|
2340
|
+
**({"result": self.result} if reason == "answer" and self.result is not None else {}),
|
|
2341
|
+
**({"error": summary["error"]} if summary.get("error") else {}))
|
|
2342
|
+
|
|
2343
|
+
def _write_summary(self, open: bool = False) -> None:
|
|
2344
|
+
"""Write the summary as it stands, and what continues the conversation; its
|
|
2345
|
+
spending is charged to the agent that started it."""
|
|
2346
|
+
summary = self.summary
|
|
2347
|
+
summary["wall_seconds"] = time.perf_counter() - self.started
|
|
2348
|
+
if self.agents_usage["started"] or self.agents_usage["model_calls"]:
|
|
2349
|
+
summary["agents"] = {**self.agents_usage, "cost_usd": round(self.agents_usage["cost_usd"], 6)}
|
|
2350
|
+
write_json(self.directory / "summary.json", {**summary, "open": True} if open else summary)
|
|
2351
|
+
self._save()
|
|
2352
|
+
self.charge_up()
|
|
2353
|
+
|
|
2354
|
+
def _loop(self, skills, told) -> str:
|
|
1766
2355
|
try:
|
|
1767
|
-
self._steps(skills, told)
|
|
2356
|
+
return self._steps(skills, told)
|
|
1768
2357
|
finally:
|
|
1769
2358
|
with self._condition:
|
|
1770
2359
|
if self._request is not None:
|
|
1771
|
-
self._settle(self._request, "unused", "the
|
|
2360
|
+
self._settle(self._request, "unused", "the turn ended first")
|
|
1772
2361
|
self._request = None
|
|
1773
2362
|
|
|
1774
|
-
def _steps(self, skills, told) ->
|
|
1775
|
-
"""The loop
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
2363
|
+
def _steps(self, skills, told) -> str:
|
|
2364
|
+
"""The loop of one turn: the model is told what its units ended with, asked on the
|
|
2365
|
+
trigger of what happened last (self.reason), and its reply's output is done
|
|
2366
|
+
(OUTPUTS), until an output, a unit, a limit or the environment ends the turn: how
|
|
2367
|
+
it ended (decisions.TURN_ENDS). No program holds the loop."""
|
|
2368
|
+
config, summary, runtime = self.config, self.summary, self.config.runtime
|
|
1779
2369
|
direct = runtime.get("mode", "program") == "direct"
|
|
1780
2370
|
post_calls = runtime.get("post_policy_calls")
|
|
1781
2371
|
# What the outputs share in this loop. Direct control: the agent's action vocabulary
|
|
1782
|
-
# turns each <action> into a program (runtime.actions, "module:function").
|
|
1783
|
-
# turns sent to the model, lowered after the model server runs out of memory.
|
|
2372
|
+
# turns each <action> into a program (runtime.actions, "module:function").
|
|
1784
2373
|
# malformed_from: the first reply without a valid block since the last valid one (such
|
|
1785
2374
|
# exchanges leave the history once a valid reply comes). nudged: replies in a row sent
|
|
1786
2375
|
# back for only announcing an action (ANNOUNCED); empties: empty replies in a row asked
|
|
1787
2376
|
# again (EMPTY_RETRIES). post_calls: the model's calls after the learned policy's first
|
|
1788
2377
|
# run are bounded (finishing with its own tools cost half the spend for no score), from
|
|
1789
|
-
#
|
|
2378
|
+
# self._post_from on; then the policy runs on.
|
|
1790
2379
|
loop = SimpleNamespace(skills=skills, told=told, direct=direct, kinds=DIRECT_KINDS if direct else KINDS,
|
|
1791
2380
|
actions=_object(runtime["actions"]) if direct else None,
|
|
1792
|
-
|
|
1793
|
-
post_calls=None if post_calls is None else int(post_calls)
|
|
1794
|
-
|
|
1795
|
-
#
|
|
1796
|
-
|
|
2381
|
+
malformed_from=None, nudged=0, empties=0,
|
|
2382
|
+
post_calls=None if post_calls is None else int(post_calls))
|
|
2383
|
+
# What the first call of the turn is made for: the instruction, else what started
|
|
2384
|
+
# the turn (a message or an event it was told comes first: _think).
|
|
2385
|
+
self.reason = "start" if self._turns == 1 else "unit_end"
|
|
1797
2386
|
with self._condition:
|
|
1798
|
-
self.
|
|
2387
|
+
self._upto = LATER
|
|
1799
2388
|
while True:
|
|
1800
2389
|
self._checkpoint()
|
|
1801
2390
|
self._load_due_skills()
|
|
@@ -1804,94 +2393,77 @@ class TaskRun:
|
|
|
1804
2393
|
self._activate_library(selected)
|
|
1805
2394
|
self._trial_at_boundary()
|
|
1806
2395
|
self._checkpoint()
|
|
1807
|
-
|
|
2396
|
+
if ended := self._ended_by_unit(loop):
|
|
2397
|
+
return ended
|
|
2398
|
+
self._deliver(loop)
|
|
2399
|
+
if ended := self._ended_by_unit(loop):
|
|
2400
|
+
return ended
|
|
2401
|
+
self._compact_if_due(loop)
|
|
1808
2402
|
with self._condition:
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
break_for_policy = False
|
|
1824
|
-
if self.seconds_left() <= 0:
|
|
1825
|
-
summary["status"] = "wall_time_exhausted"
|
|
1826
|
-
break
|
|
1827
|
-
if not self._ahead_allowed():
|
|
1828
|
-
summary["status"] = "cost_exhausted"
|
|
1829
|
-
break
|
|
1830
|
-
if not break_for_policy:
|
|
1831
|
-
current = self._think("between", loop.turns, learning_report=learning_report)
|
|
1832
|
-
else:
|
|
1833
|
-
break_for_policy = False
|
|
1834
|
-
if not break_for_policy:
|
|
1835
|
-
self._condition.wait_for(lambda: current.back(0) or self._closing)
|
|
2403
|
+
if self._iterations >= config.max_model_calls:
|
|
2404
|
+
return "budget"
|
|
2405
|
+
policy = (self._post_from is not None and loop.post_calls is not None
|
|
2406
|
+
and summary["model_calls"] - self._post_from >= loop.post_calls and not self._message_waits())
|
|
2407
|
+
if self.seconds_left() <= 0:
|
|
2408
|
+
return "wall_time"
|
|
2409
|
+
if not self._within_budget():
|
|
2410
|
+
return "cost"
|
|
2411
|
+
if not policy:
|
|
2412
|
+
if self._notices.pending(self._upto):
|
|
2413
|
+
continue # (It came meanwhile: told first.)
|
|
2414
|
+
asked = self._urgent
|
|
2415
|
+
current = self._think()
|
|
2416
|
+
self._condition.wait_for(lambda: current.back(0) or self._closing or self._interrupt)
|
|
1836
2417
|
if self._closing:
|
|
1837
2418
|
raise Closed()
|
|
2419
|
+
if self._interrupt:
|
|
2420
|
+
raise Interrupted()
|
|
2421
|
+
self._request = None
|
|
1838
2422
|
if current.cancelled:
|
|
2423
|
+
# Set aside (a message, an event, a unit's end came first), or out of time.
|
|
1839
2424
|
if isinstance(current.error, ThoughtCancelled) and "deadline" in str(current.error):
|
|
1840
2425
|
raise current.error
|
|
1841
2426
|
continue
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
else:
|
|
1849
|
-
used = {}
|
|
2427
|
+
reply = current.wait()
|
|
2428
|
+
self._replied_at = time.time()
|
|
2429
|
+
self._context_tokens = (
|
|
2430
|
+
int(reply.usage.get("input_tokens") or 0) + int(reply.usage.get("output_tokens") or 0)
|
|
2431
|
+
or (sum(row["chars"] for row in context_summary(current.messages, current.system))
|
|
2432
|
+
+ len(reply.text)) // 4)
|
|
1850
2433
|
self._settle(current, "used")
|
|
1851
|
-
self.
|
|
1852
|
-
|
|
1853
|
-
loop.
|
|
1854
|
-
loop.ahead_generation = current.ahead_generation if current.trigger == "ahead" else None
|
|
2434
|
+
self._unanswered.clear()
|
|
2435
|
+
self._told_events = []
|
|
2436
|
+
loop.revision, loop.urgent = current.revision, asked
|
|
1855
2437
|
loop.call, loop.thought = current.call, current.id
|
|
1856
|
-
if
|
|
1857
|
-
self._policy_to_end()
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
return False
|
|
1875
|
-
if parsed is not None and parsed.kind != "wait" and not decisions._STOP.search(thought.reply.text):
|
|
1876
|
-
if parsed.kind not in ("python", "action") or (parsed.kind == "python" and decisions._beside(parsed.body)):
|
|
1877
|
-
return False
|
|
1878
|
-
self._settle(thought, "waited", "the decision needs the completed program result")
|
|
1879
|
-
self._pending_events.clear()
|
|
1880
|
-
if self._carry:
|
|
1881
|
-
self._extend_request(self._carry)
|
|
1882
|
-
self._carry = []
|
|
1883
|
-
return True
|
|
2438
|
+
if policy:
|
|
2439
|
+
return self._policy_to_end()
|
|
2440
|
+
output, parsed = self._output(reply, loop)
|
|
2441
|
+
if ended := getattr(self, decisions.OUTPUTS[output])(reply, parsed, loop):
|
|
2442
|
+
return ended
|
|
2443
|
+
|
|
2444
|
+
def _ended_by_unit(self, loop) -> str | None:
|
|
2445
|
+
"""How a unit ended the turn, if one did: the episode ended in it, or it called
|
|
2446
|
+
finish() (which a message or an unresolved event for the model overrides)."""
|
|
2447
|
+
with self._condition:
|
|
2448
|
+
end, self._run_end = self._run_end, None
|
|
2449
|
+
if end is None:
|
|
2450
|
+
return None
|
|
2451
|
+
if end == "episode_ended":
|
|
2452
|
+
return "episode_ended"
|
|
2453
|
+
if self._unanswered or self._notices.pending(kinds=("message", "event")):
|
|
2454
|
+
return None
|
|
2455
|
+
return self._out_answer(None, SimpleNamespace(body=end[1], source="program"), loop)
|
|
1884
2456
|
|
|
1885
|
-
def _output(self, reply,
|
|
1886
|
-
"""What a reply
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
if (not
|
|
2457
|
+
def _output(self, reply, loop) -> tuple[str, object]:
|
|
2458
|
+
"""What a reply does (a key of OUTPUTS) and its parsed block (or the error that kept
|
|
2459
|
+
it from parsing); the reply joins the trace and the history."""
|
|
2460
|
+
trace, messages = self.trace, self.messages
|
|
2461
|
+
if (not reply.text.strip() and reply.usage.get("finish_reason") != "length"
|
|
1890
2462
|
and loop.empties < decisions.EMPTY_RETRIES):
|
|
1891
2463
|
return "empty", None
|
|
1892
2464
|
loop.empties = 0
|
|
1893
|
-
# An empty reply is a format error after EMPTY_RETRIES, never
|
|
1894
|
-
plain = (
|
|
2465
|
+
# An empty reply is a format error after EMPTY_RETRIES, never an answer.
|
|
2466
|
+
plain = (reply.usage.get("finish_reason") != "length"
|
|
1895
2467
|
and bool(reply.text.strip()) and not looks_like_action(reply.text))
|
|
1896
2468
|
# A plain reply that only announces the next step ("I'll open the page ...") is
|
|
1897
2469
|
# sent back as a format error.
|
|
@@ -1905,12 +2477,7 @@ class TaskRun:
|
|
|
1905
2477
|
parsed.program = (loop.actions(parsed.body, allow_policy="vla" in loop.skills,
|
|
1906
2478
|
allow_skills=bool(self.config.runtime.get("fixed_skills", False)))
|
|
1907
2479
|
if parsed.kind == "action" else parsed.body)
|
|
1908
|
-
|
|
1909
|
-
assumed = decisions._ASSUME.search(thought_text)
|
|
1910
|
-
presentation = {"kind": parsed.kind,
|
|
1911
|
-
"thought": (decisions._ASSUME.sub("", thought_text) if assumed else thought_text).strip(),
|
|
1912
|
-
"body": parsed.body,
|
|
1913
|
-
**({"assume": " ".join(assumed[1].split())} if assumed else {})}
|
|
2480
|
+
presentation = {"kind": parsed.kind, "thought": (parsed.thought or "").strip(), "body": parsed.body}
|
|
1914
2481
|
output = {"python": "program", "action": "program"}.get(parsed.kind, parsed.kind)
|
|
1915
2482
|
except (ProtocolError, ValueError) as failed:
|
|
1916
2483
|
parsed, output = failed, "message" if plain else "invalid"
|
|
@@ -1918,11 +2485,10 @@ class TaskRun:
|
|
|
1918
2485
|
parsed = ProtocolError("The reply said what it would do, without a block")
|
|
1919
2486
|
presentation = ({"kind": "message", "body": reply.text.strip(), "announced": True} if announced else
|
|
1920
2487
|
{"kind": "message" if plain else "invalid", "body": reply.text.strip() if plain else ""})
|
|
1921
|
-
self._record_reply(reply, loop.call + 1, thought=loop.thought, presentation=presentation
|
|
2488
|
+
self._record_reply(reply, loop.call + 1, thought=loop.thought, presentation=presentation)
|
|
1922
2489
|
messages.append({"role": "assistant", "content": reply.text})
|
|
1923
2490
|
if output == "invalid":
|
|
1924
|
-
if loop.malformed_from is None
|
|
1925
|
-
# (A used reply's exchange stays: its message holds its program's result.)
|
|
2491
|
+
if loop.malformed_from is None:
|
|
1926
2492
|
loop.malformed_from = len(messages) - 1
|
|
1927
2493
|
elif loop.malformed_from is not None:
|
|
1928
2494
|
dropped = len(messages) - 1 - loop.malformed_from
|
|
@@ -1932,7 +2498,7 @@ class TaskRun:
|
|
|
1932
2498
|
loop.malformed_from = None
|
|
1933
2499
|
return output, parsed
|
|
1934
2500
|
|
|
1935
|
-
# The outputs of a reply
|
|
2501
|
+
# The outputs of a reply (OUTPUTS): what one returns is how the turn ended (None: it goes on).
|
|
1936
2502
|
|
|
1937
2503
|
def _out_empty(self, reply, parsed, loop):
|
|
1938
2504
|
"""Reasoning only: asked again at once, with no format error to show."""
|
|
@@ -1943,19 +2509,9 @@ class TaskRun:
|
|
|
1943
2509
|
self.reason = "empty"
|
|
1944
2510
|
|
|
1945
2511
|
def _current_output(self, loop):
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
def _out_message(self, reply, parsed, loop):
|
|
1950
|
-
"""A plain reply: the model talks to the user (or a sub-agent asks its parent)."""
|
|
1951
|
-
with self._condition:
|
|
1952
|
-
if not self._current_output(loop):
|
|
1953
|
-
return None
|
|
1954
|
-
self.question = reply.text.strip()
|
|
1955
|
-
# Keep the notification for task/inbox consumers, but identify the reply
|
|
1956
|
-
# already recorded by _output so transcript consumers do not show it twice.
|
|
1957
|
-
self.trace.emit("agent_message", text=reply.text.strip(), reply_call=loop.call + 1)
|
|
1958
|
-
self._await_input()
|
|
2512
|
+
"""Whether the reply being done knew of everything that set a call aside: no notice
|
|
2513
|
+
of priority `now` came since its call was made."""
|
|
2514
|
+
return loop.urgent == self._urgent
|
|
1959
2515
|
|
|
1960
2516
|
def _out_invalid(self, reply, parsed, loop):
|
|
1961
2517
|
"""A reply nothing came of: told why, and asked again."""
|
|
@@ -1968,40 +2524,70 @@ class TaskRun:
|
|
|
1968
2524
|
who = "the agent that started you" if self.parent is not None else "the user"
|
|
1969
2525
|
text = decisions.ANNOUNCED.format(block="<action>" if loop.direct else "<python>", who=who)
|
|
1970
2526
|
else:
|
|
1971
|
-
blocks = "<action> or <
|
|
2527
|
+
blocks = "<action> or <wait>" if loop.direct else "<python>, <verify> or <wait>"
|
|
1972
2528
|
text = (f"Format error: {parsed}. Nothing ran. Reply with the {blocks} block itself, starting with its "
|
|
1973
|
-
"opening tag.")
|
|
1974
|
-
self.
|
|
2529
|
+
"opening tag, or with your answer in plain text.")
|
|
2530
|
+
self._came_to(text)
|
|
1975
2531
|
self.trace.emit("protocol_error", message=str(parsed), truncated=reply.usage.get("finish_reason") == "length",
|
|
1976
2532
|
**({"announced": True} if text.startswith("Nothing ran") else {}))
|
|
1977
2533
|
self.reason = "format"
|
|
1978
2534
|
|
|
1979
|
-
def
|
|
1980
|
-
"""The
|
|
2535
|
+
def _out_answer(self, reply, parsed, loop):
|
|
2536
|
+
"""The model's answer: a reply that is text, the reply's <finish>, or a program's
|
|
2537
|
+
finish() (source "program"), all the same. The turn ends with it; the conversation
|
|
2538
|
+
and the units that run go on."""
|
|
1981
2539
|
source = getattr(parsed, "source", None)
|
|
1982
2540
|
if source is None:
|
|
1983
|
-
if loop.assumption is not None and not self._check_assumption(*loop.assumption):
|
|
1984
|
-
self._tell("The lookahead condition changed before the finish could be applied.")
|
|
1985
|
-
self.reason = "program_end"
|
|
1986
|
-
return None
|
|
1987
2541
|
with self._condition:
|
|
1988
2542
|
if not self._current_output(loop):
|
|
1989
2543
|
return None
|
|
2544
|
+
# (A reply without a block parsed to no block: its text is the answer.)
|
|
2545
|
+
text = reply.text if isinstance(parsed, Exception) else parsed.body
|
|
1990
2546
|
from harness.runtime.learning.active import is_active
|
|
1991
2547
|
|
|
1992
|
-
if is_active(self.config.runtime) and self.
|
|
2548
|
+
if is_active(self.config.runtime) and self._within_budget():
|
|
1993
2549
|
before = getattr(self, "_library_version", None)
|
|
1994
2550
|
self._trial_at_boundary()
|
|
1995
2551
|
if getattr(self, "_library_version", None) != before:
|
|
1996
|
-
self.
|
|
1997
|
-
|
|
2552
|
+
self._came_to("The host activated your proposal. Use the new revision in a brief practice program, "
|
|
2553
|
+
"check what it does, then answer with the result.")
|
|
1998
2554
|
self.reason = "program_end"
|
|
1999
2555
|
return None
|
|
2000
|
-
self.result =
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2556
|
+
self.result = text.strip()
|
|
2557
|
+
if isinstance(parsed, Exception):
|
|
2558
|
+
# (The reply is already recorded by _output: reply_call names it, so that
|
|
2559
|
+
# transcript consumers do not show it twice.)
|
|
2560
|
+
self.trace.emit("agent_message", text=self.result, reply_call=loop.call + 1)
|
|
2561
|
+
else:
|
|
2562
|
+
self.trace.emit("finish", text=text, **({"source": source} if source else {}))
|
|
2563
|
+
return "answer"
|
|
2564
|
+
|
|
2565
|
+
def _out_wait(self, reply, parsed, loop):
|
|
2566
|
+
"""The model waits by choice: no call is made until a notice reaches it (the end of
|
|
2567
|
+
a unit it started, a message, an event, a note), or its time is over (the block's
|
|
2568
|
+
seconds, at most the run's longest wait, which a wait without a time gets). It is
|
|
2569
|
+
told which."""
|
|
2570
|
+
asked = wait_seconds(parsed.body)
|
|
2571
|
+
limit = max(0.0, min(asked if asked is not None else self._longest_wait, self._longest_wait,
|
|
2572
|
+
self.seconds_left()))
|
|
2573
|
+
began = time.monotonic()
|
|
2574
|
+
|
|
2575
|
+
def reached():
|
|
2576
|
+
waking = self._notices.pending(waking=True)
|
|
2577
|
+
return "unit" if self._run_end is not None else waking[0].kind if waking else None
|
|
2578
|
+
|
|
2579
|
+
with self._condition:
|
|
2580
|
+
self.trace.emit("wait", seconds=round(limit, 3), **({"asked": asked} if asked is not None else {}))
|
|
2581
|
+
self._condition.wait_for(lambda: reached() or self._closing or self._pause or self._interrupt,
|
|
2582
|
+
timeout=limit)
|
|
2583
|
+
why = reached() or ("time" if time.monotonic() - began >= limit else "pause")
|
|
2584
|
+
waited = time.monotonic() - began
|
|
2585
|
+
self.trace.emit("wait_end", why=why, seconds=round(waited, 3))
|
|
2586
|
+
self._came_to(decisions.WAIT_ENDED.format(seconds=waited,
|
|
2587
|
+
why=decisions.WAIT_ENDS.get(why, "the user paused")))
|
|
2588
|
+
# (After a wait the queue is told whole.)
|
|
2589
|
+
self._upto = LATER
|
|
2590
|
+
self.reason = "wait_end"
|
|
2005
2591
|
|
|
2006
2592
|
def _out_verify(self, reply, parsed, loop):
|
|
2007
2593
|
"""The verifier's answer on the reply's claim, with the views it judged."""
|
|
@@ -2012,12 +2598,13 @@ class TaskRun:
|
|
|
2012
2598
|
answer, view = self._verify(parsed.body)
|
|
2013
2599
|
except ThoughtCancelled:
|
|
2014
2600
|
with self._condition:
|
|
2015
|
-
if self.
|
|
2601
|
+
if self._notices.pending() or self._closing:
|
|
2016
2602
|
return None
|
|
2017
2603
|
raise
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2604
|
+
with self._condition:
|
|
2605
|
+
self._state_now = view["state"]
|
|
2606
|
+
self._came_to([{"type": "input_text", "text": f"Verifier: {answer}"},
|
|
2607
|
+
*(image_parts(view["images"], self.cameras) if self._standing else [])])
|
|
2021
2608
|
self.reason = "verify"
|
|
2022
2609
|
|
|
2023
2610
|
def _verify(self, claim, cameras=None, request=None):
|
|
@@ -2056,7 +2643,7 @@ class TaskRun:
|
|
|
2056
2643
|
"""What the environment reports as text for a claim (its session's __evidence__: a
|
|
2057
2644
|
window's elements, a page's text), read between programs; empty when it has none."""
|
|
2058
2645
|
kernel = self.kernel
|
|
2059
|
-
if kernel is None
|
|
2646
|
+
if kernel is None:
|
|
2060
2647
|
return ""
|
|
2061
2648
|
try:
|
|
2062
2649
|
result = kernel.run("__evidence__()", timeout=30)
|
|
@@ -2073,13 +2660,8 @@ class TaskRun:
|
|
|
2073
2660
|
if self.config.max_cost_usd is not None and (
|
|
2074
2661
|
self.summary["cost_usd"] + self.agents_usage["cost_usd"] >= self.config.max_cost_usd):
|
|
2075
2662
|
raise RuntimeError("The session model cost budget is exhausted")
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
if current is not None and current.trigger == "ahead":
|
|
2079
|
-
self._settle(current, "discarded", "program requested a model result")
|
|
2080
|
-
thought = Thought(role, decisions.TRIGGERS["request"][role], chat, system, messages,
|
|
2081
|
-
timeout=max(0, self.seconds_left()), program=self.summary["cells"] if self._cell_running else None,
|
|
2082
|
-
revision=self._revision, origin=origin,
|
|
2663
|
+
thought = Thought(role, decisions.REQUESTS[role], chat, system, messages,
|
|
2664
|
+
timeout=max(0, self.seconds_left()), revision=self._revision, origin=origin,
|
|
2083
2665
|
owner={**self._who().get("who", {}), "entry": request.get("entry"),
|
|
2084
2666
|
"frame": request.get("frame")})
|
|
2085
2667
|
self._thoughts.submit(thought)
|
|
@@ -2128,8 +2710,9 @@ class TaskRun:
|
|
|
2128
2710
|
max_output_tokens=int(observer.get("max_output_tokens", 1200)))
|
|
2129
2711
|
return ChatModel({"max_output_tokens": 1024, "temperature": 0.0, "extra_body": extra, **config})
|
|
2130
2712
|
|
|
2131
|
-
def _policy_to_end(self) ->
|
|
2132
|
-
"""The model's calls after the learned policy's run are used up: it runs on to the
|
|
2713
|
+
def _policy_to_end(self) -> str:
|
|
2714
|
+
"""The model's calls after the learned policy's run are used up: it runs on to the
|
|
2715
|
+
end, which ends the turn (how: decisions.TURN_ENDS)."""
|
|
2133
2716
|
trace, summary, kernel = self.trace, self.summary, self.kernel
|
|
2134
2717
|
with self._span("program", "policy to the end"):
|
|
2135
2718
|
result = kernel.run("__policy_to_end__()", timeout=self._cell_timeout(policy=True))
|
|
@@ -2138,24 +2721,19 @@ class TaskRun:
|
|
|
2138
2721
|
if result.get("strategy"):
|
|
2139
2722
|
summary["strategy"] = result["strategy"]
|
|
2140
2723
|
if result["status"] == "episode_ended":
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
summary["status"] = "finished"
|
|
2145
|
-
_wrap_up(kernel, trace)
|
|
2724
|
+
return "episode_ended"
|
|
2725
|
+
trace.emit("finish", text="the model's calls after the policy's run were used up", source="harness")
|
|
2726
|
+
return "answer"
|
|
2146
2727
|
|
|
2147
2728
|
def _out_program(self, reply, parsed, loop):
|
|
2148
|
-
"""
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2729
|
+
"""Start the reply's program as a unit of the model. The turn comes back at once:
|
|
2730
|
+
with the program's result when it was over within the grace, else with its handle;
|
|
2731
|
+
the result then reaches the model when the unit ends (_unit_arrived)."""
|
|
2732
|
+
trace, summary, kernel = self.trace, self.summary, self.kernel
|
|
2733
|
+
program = parsed.program
|
|
2152
2734
|
if not self._approved(program):
|
|
2153
2735
|
return None
|
|
2154
2736
|
self._load_due_skills()
|
|
2155
|
-
if loop.assumption is not None and not self._check_assumption(*loop.assumption):
|
|
2156
|
-
self._tell("The lookahead condition changed before the program could start.")
|
|
2157
|
-
self.reason = "program_end"
|
|
2158
|
-
return None
|
|
2159
2737
|
with self._condition:
|
|
2160
2738
|
waiting = self._pause or self._closing
|
|
2161
2739
|
if waiting:
|
|
@@ -2164,109 +2742,167 @@ class TaskRun:
|
|
|
2164
2742
|
trace.emit("program_held")
|
|
2165
2743
|
self._checkpoint()
|
|
2166
2744
|
with self._condition:
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2745
|
+
# (After a pause a message of the user replaces the program.)
|
|
2746
|
+
replaced = waiting and self._message_waits()
|
|
2747
|
+
if replaced or not self._current_output(loop):
|
|
2748
|
+
trace.emit("program_dropped", reason="user_message" if replaced else "inputs_changed")
|
|
2749
|
+
self._came_to("The user paused before this program ran, so nothing ran." if waiting
|
|
2750
|
+
else "Something reached you before this program ran, so nothing ran.")
|
|
2171
2751
|
self.reason = "user"
|
|
2172
2752
|
return None
|
|
2173
2753
|
summary["cells"] += 1
|
|
2174
|
-
number,
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
self.
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
before = time.perf_counter()
|
|
2184
|
-
remaining = self._cell_timeout()
|
|
2185
|
-
limit = config.runtime.get("cell_timeout_seconds")
|
|
2186
|
-
episode_deadline = limit is None or float(limit) >= self.seconds_left()
|
|
2187
|
-
deadline = threading.Timer(remaining, self._program_deadline, (number, episode_deadline))
|
|
2188
|
-
deadline.daemon = True
|
|
2189
|
-
deadline.start()
|
|
2754
|
+
number, span = summary["cells"], uuid.uuid4().hex[:12]
|
|
2755
|
+
started = {"number": number, "code": program, "action": parsed.body if loop.direct else None,
|
|
2756
|
+
"span": span, "unit": None}
|
|
2757
|
+
trace.emit("span_start", id=span, lane="program", name=f"program {number}", parent=None)
|
|
2758
|
+
|
|
2759
|
+
def announce(cell):
|
|
2760
|
+
with self._condition:
|
|
2761
|
+
self._programs[cell] = started
|
|
2762
|
+
|
|
2190
2763
|
try:
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
end.update(status=result["status"])
|
|
2764
|
+
result = kernel.start(program, self._grace, timeout=self._cell_timeout(), name=f"program {number}",
|
|
2765
|
+
announce=announce)
|
|
2194
2766
|
except KernelDied as error:
|
|
2195
|
-
|
|
2767
|
+
trace.emit("span_end", id=span, error=f"KernelDied: {error}"[:300])
|
|
2196
2768
|
trace.emit("kernel_died", message=str(error))
|
|
2197
2769
|
if self.seconds_left() > 0:
|
|
2198
2770
|
raise
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
self.
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2771
|
+
return "wall_time"
|
|
2772
|
+
if result["status"] != "running":
|
|
2773
|
+
self._unit_arrived(result)
|
|
2774
|
+
return None
|
|
2775
|
+
with self._condition:
|
|
2776
|
+
if started in self._programs.values():
|
|
2777
|
+
# (Else it ended meanwhile: its result is what the model is told.)
|
|
2778
|
+
self._came_to(decisions.RUNNING.format(number=number, unit=result["unit"]))
|
|
2779
|
+
self.reason = "program_running"
|
|
2780
|
+
return None
|
|
2781
|
+
|
|
2782
|
+
def _unit_arrived(self, result: dict) -> None:
|
|
2783
|
+
"""A unit of the model ended: its result came as the reply to its program, or as the
|
|
2784
|
+
kernel's unit_end message (any thread). It is recorded now and is a notice for the
|
|
2785
|
+
model: told with its next call, or at once (a call in flight is set aside) when the
|
|
2786
|
+
end is an event (an error, an unhandled event, the episode's end, finish()). The
|
|
2787
|
+
end of a unit that was cancelled wakes nobody."""
|
|
2788
|
+
trace, summary = self.trace, self.summary
|
|
2789
|
+
result = {key: value for key, value in result.items() if key not in ("op", "time")}
|
|
2790
|
+
number = result.get("cell")
|
|
2791
|
+
with self._condition:
|
|
2792
|
+
program = self._programs.pop(number, None) if number is not None else None
|
|
2793
|
+
recorded = {key: value for key, value in result.items() if key not in ("attachments", "report", "looks")}
|
|
2794
|
+
if program is not None:
|
|
2795
|
+
if result.get("report"):
|
|
2796
|
+
recorded["report"] = {"text": result["report"].get("text", ""),
|
|
2797
|
+
"images": sorted(result["report"].get("images") or {})}
|
|
2798
|
+
if result.get("looks"):
|
|
2799
|
+
recorded["looks"] = result["looks"]
|
|
2800
|
+
trace.emit("cell", number=program["number"], code=program["code"], action=program["action"],
|
|
2801
|
+
result=recorded, attachments=len(result.get("attachments", [])))
|
|
2802
|
+
trace.emit("span_end", id=program["span"], status=result["status"])
|
|
2803
|
+
summary["kernel_seconds"] += float(result.get("seconds") or 0.0)
|
|
2804
|
+
result["program"] = program
|
|
2805
|
+
elif number is None:
|
|
2806
|
+
trace.emit("unit_end", **recorded, attachments=len(result.get("attachments", [])))
|
|
2807
|
+
if result.get("quiet"):
|
|
2808
|
+
# The tree of reflexes started it and answers its end itself
|
|
2809
|
+
# (runtime/reactive/tree.py): recorded, and nothing for the model.
|
|
2810
|
+
_cell_usage(summary, result)
|
|
2811
|
+
return
|
|
2812
|
+
else:
|
|
2813
|
+
# A program the host ran itself (wrap-up, a library load): nothing for the model.
|
|
2814
|
+
return
|
|
2815
|
+
_cell_usage(summary, result)
|
|
2816
|
+
if result.get("strategy"):
|
|
2817
|
+
summary["strategy"] = result["strategy"]
|
|
2818
|
+
if self._post_from is None and result["strategy"].get("policy_ran"):
|
|
2819
|
+
self._post_from = summary["model_calls"]
|
|
2820
|
+
status = result["status"]
|
|
2821
|
+
if status == "episode_ended":
|
|
2822
|
+
self._run_end = "episode_ended"
|
|
2823
|
+
elif status == "finished" and self._run_end is None:
|
|
2824
|
+
self._run_end = ("finished", str(result.get("finish") or ""))
|
|
2825
|
+
if result.get("state"):
|
|
2826
|
+
self._state_now = result["state"]
|
|
2827
|
+
urgent = status not in ("ok", "cancelled") and not result.get("handled")
|
|
2828
|
+
self._put("unit", NOW if urgent else NEXT, result, wakes=status != "cancelled",
|
|
2829
|
+
reason="a unit ended with an event")
|
|
2223
2830
|
self.charge_up()
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
return "
|
|
2236
|
-
|
|
2831
|
+
|
|
2832
|
+
def _notice_parts(self, notice) -> list:
|
|
2833
|
+
"""A message, an event or a note of the queue as the model reads it."""
|
|
2834
|
+
if notice.kind == "message":
|
|
2835
|
+
return _inbox_parts([notice.data])
|
|
2836
|
+
if notice.kind == "event":
|
|
2837
|
+
record = notice.data
|
|
2838
|
+
event = record.get("event") or {}
|
|
2839
|
+
more = {key: value for key, value in record.items() if key not in ("id", "event", "time")}
|
|
2840
|
+
if event.get("detail") not in (None, {}, ""):
|
|
2841
|
+
more["detail"] = event["detail"]
|
|
2842
|
+
return [{"type": "input_text", "text": decisions.EVENT.format(
|
|
2843
|
+
kind=event.get("kind"), source=event.get("source")) + (": " + json.dumps(more) if more else "")}]
|
|
2844
|
+
return [{"type": "input_text", "text": str(notice.data)}]
|
|
2845
|
+
|
|
2846
|
+
def _deliver(self, loop) -> None:
|
|
2847
|
+
"""Tell the model what there is to tell, with its next call: the one place that
|
|
2848
|
+
does. First what its last output came to (_came_to), then the notices of the queue
|
|
2849
|
+
down to the priority that is due (runtime/decision/notices.py), each as its kind
|
|
2850
|
+
reads (a program's feedback with what it showed and looked at, another unit's end,
|
|
2851
|
+
a message, an event, a note), then how things stand now (_ambient). Nothing when
|
|
2852
|
+
there is neither."""
|
|
2237
2853
|
with self._condition:
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2854
|
+
upto, self._upto = self._upto, NEXT
|
|
2855
|
+
notices = self._notices.take(upto)
|
|
2856
|
+
outcome, self._outcome = self._outcome, []
|
|
2857
|
+
if not notices and not outcome:
|
|
2858
|
+
return
|
|
2859
|
+
trace, cameras = self.trace, self.cameras
|
|
2860
|
+
order = ("unit_end", "program_end", "escalation", "error")
|
|
2861
|
+
content, looks, outlines, reason, taken, state, programs = list(outcome), [], {}, None, None, None, 0
|
|
2862
|
+
for notice in notices:
|
|
2863
|
+
if notice.kind != "unit":
|
|
2864
|
+
content += self._notice_parts(notice)
|
|
2865
|
+
continue
|
|
2866
|
+
result = notice.data
|
|
2867
|
+
program, status = result.get("program"), result["status"]
|
|
2868
|
+
if program is not None:
|
|
2869
|
+
programs += 1
|
|
2870
|
+
if self.interactive:
|
|
2871
|
+
self._stuck(program["code"], result)
|
|
2872
|
+
taken = taken or _taken_over(result)
|
|
2873
|
+
why = {"escalated": "escalation", "error": "error"}.get(status, "program_end")
|
|
2874
|
+
text = cell_feedback(result, program["number"])
|
|
2875
|
+
if why == "program_end":
|
|
2876
|
+
text += decisions.PROGRAM_ENDED
|
|
2877
|
+
state = result.get("state") or state
|
|
2246
2878
|
else:
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2879
|
+
why = "escalation" if status in ("error", "escalated") and not result.get("handled") else "unit_end"
|
|
2880
|
+
text = unit_feedback(result)
|
|
2881
|
+
reason = why if reason is None else max(reason, why, key=order.index)
|
|
2882
|
+
content.append({"type": "input_text", "text": text, "result": True})
|
|
2883
|
+
for item in result.get("attachments", [])[:6]:
|
|
2884
|
+
content.append({"type": "input_text", "text": f"show(): {item['caption']}"})
|
|
2885
|
+
content.append({"type": "input_image", "image_url": item["image_url"], "detail": "auto"})
|
|
2886
|
+
# A skill's report (the tracked scene) may outline objects in the same views.
|
|
2887
|
+
outlined = (result.get("report") or {}).get("images") or {}
|
|
2888
|
+
outlines.update(outlined)
|
|
2889
|
+
seen = [{**item, "image_url": outlined[item["camera"]]}
|
|
2890
|
+
if item["camera"] in outlined and not item.get("region") else item
|
|
2891
|
+
for item in result.get("looks") or ()]
|
|
2892
|
+
if seen and not self._standing:
|
|
2893
|
+
trace.emit("views", action="looked", program=(program or {}).get("number"), unit=result.get("unit"),
|
|
2894
|
+
views=[{key: item[key] for key in ("camera", "region", "pin") if key in item}
|
|
2895
|
+
for item in seen])
|
|
2896
|
+
looks += seen
|
|
2897
|
+
if reason is not None:
|
|
2898
|
+
self.reason = reason
|
|
2899
|
+
with self._condition:
|
|
2900
|
+
for notice in notices:
|
|
2901
|
+
if notice.kind in ("message", "event"):
|
|
2902
|
+
self._unanswered.add(notice.kind)
|
|
2903
|
+
if notice.kind == "event":
|
|
2904
|
+
self._told_events.append(notice.data.get("id"))
|
|
2905
|
+
if not taken and reason in ("error", "escalation"):
|
|
2270
2906
|
self._reflect_learning()
|
|
2271
2907
|
if taken is not None:
|
|
2272
2908
|
# The user brought a window of theirs to the front: the computer is theirs
|
|
@@ -2275,164 +2911,115 @@ class TaskRun:
|
|
|
2275
2911
|
self.pause()
|
|
2276
2912
|
self._checkpoint()
|
|
2277
2913
|
if self._standing:
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2914
|
+
if programs:
|
|
2915
|
+
with self._span("env", "observe"):
|
|
2916
|
+
view = self.env.call("observe", episode_id=self.episode, cameras=cameras)
|
|
2917
|
+
content.extend(image_parts({**view["images"], **outlines}, cameras))
|
|
2918
|
+
state = view["state"]
|
|
2919
|
+
with self._condition:
|
|
2920
|
+
self._state_now = state
|
|
2283
2921
|
else:
|
|
2284
|
-
# Only what the
|
|
2285
|
-
view = {"state": result.get("state") or {}}
|
|
2286
|
-
looks = list(result.get("looks") or ())
|
|
2287
|
-
outlined = (result.get("report") or {}).get("images") or {}
|
|
2288
|
-
looks = [{**item, "image_url": outlined[item["camera"]]}
|
|
2289
|
-
if item["camera"] in outlined and not item.get("region") else item for item in looks]
|
|
2922
|
+
# Only what the units looked at (look, a standing watch of the model's own).
|
|
2290
2923
|
with self._condition:
|
|
2291
2924
|
self._view_pins |= {item["pin"] for item in looks if item.get("pin")}
|
|
2292
|
-
if looks:
|
|
2293
|
-
trace.emit("views", action="looked", program=number,
|
|
2294
|
-
views=[{key: item[key] for key in ("camera", "region", "pin") if key in item}
|
|
2295
|
-
for item in looks])
|
|
2296
2925
|
content.extend(view_parts(looks) if self._images else self._described(looks))
|
|
2297
2926
|
with self._condition:
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
self.
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
if
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
reply = thought.reply
|
|
2332
|
-
if reply is None or reply.usage.get("finish_reason") == "length":
|
|
2333
|
-
return "discarded", "lookahead returned no complete reply"
|
|
2334
|
-
parsed, _ = decisions._running_reply(reply.text, kinds)
|
|
2335
|
-
if parsed is None or parsed.kind == "wait":
|
|
2336
|
-
return "waited", "lookahead deferred to the program result"
|
|
2337
|
-
condition = decisions._ASSUME.search(reply.text)
|
|
2338
|
-
if condition is None or decisions._STOP.search(reply.text):
|
|
2339
|
-
return "discarded", "lookahead requires a checkable completion condition"
|
|
2340
|
-
thought.assumption = (condition[1].strip(), result)
|
|
2341
|
-
if not self._check_assumption(*thought.assumption):
|
|
2342
|
-
return "discarded", "lookahead condition was false or could not be checked"
|
|
2343
|
-
return "used", ""
|
|
2344
|
-
|
|
2345
|
-
def _check_assumption(self, condition, result):
|
|
2346
|
-
"""Evaluate a bounded expression over JSON data, never the program namespace."""
|
|
2347
|
-
try:
|
|
2348
|
-
tree = ast.parse(condition, mode="eval")
|
|
2349
|
-
nodes = list(ast.walk(tree))
|
|
2350
|
-
if len(nodes) > 64 or not all(isinstance(node, decisions._ASSUME_NODES) for node in nodes):
|
|
2351
|
-
return False
|
|
2352
|
-
names = {node.id for node in nodes if isinstance(node, ast.Name)}
|
|
2353
|
-
if not names <= {"result", "output", "state"} or not names:
|
|
2354
|
-
return False
|
|
2355
|
-
snapshot = json.loads(json.dumps({key: value for key, value in result.items()
|
|
2356
|
-
if key not in ("attachments", "report")}))
|
|
2357
|
-
state = (self.env.call("state", episode_id=self.episode) if "state" in names else None)
|
|
2358
|
-
values = {"result": snapshot, "output": snapshot.get("stdout") or "", "state": state}
|
|
2359
|
-
return bool(eval(compile(tree, "<assume>", "eval"), {"__builtins__": {}}, values))
|
|
2360
|
-
except Exception:
|
|
2361
|
-
return False
|
|
2362
|
-
|
|
2363
|
-
def _think(self, phase, turns, *, learning_report=""):
|
|
2364
|
-
"""Select a decision from the table and queue its immutable input."""
|
|
2365
|
-
triggers = decisions.TRIGGERS[phase]
|
|
2366
|
-
now = time.monotonic()
|
|
2367
|
-
trigger = next((name for name, check in triggers.items() if check and getattr(self, check)(now)), None)
|
|
2368
|
-
running = phase == "running"
|
|
2369
|
-
if trigger is None:
|
|
2370
|
-
if running:
|
|
2371
|
-
return None
|
|
2372
|
-
trigger = self.reason
|
|
2373
|
-
if trigger not in triggers:
|
|
2374
|
-
raise ValueError(f"No model trigger for {trigger!r}")
|
|
2375
|
-
|
|
2376
|
-
number = self.summary["cells"] if running else None
|
|
2377
|
-
earlier = self._request
|
|
2378
|
-
if running and earlier is not None and (earlier.said or earlier.side is not None):
|
|
2379
|
-
self.messages.append({"role": "assistant", "content": earlier.reply.text})
|
|
2380
|
-
if earlier.side is not None:
|
|
2381
|
-
self._tell(self._beside_report(earlier))
|
|
2382
|
-
self._include_escalations()
|
|
2383
|
-
self._add_user_messages()
|
|
2927
|
+
# How things stand is told once, last: where the message this call ends with
|
|
2928
|
+
# already says it (a call that was set aside), that part gives way to this one,
|
|
2929
|
+
# which tells again what that one told as changed.
|
|
2930
|
+
last = self.messages[-1] if self.messages else None
|
|
2931
|
+
if last is not None and last["role"] == "user" and not isinstance(last["content"], str) and any(
|
|
2932
|
+
part.get("ambient") for part in last["content"]):
|
|
2933
|
+
last["content"] = [part for part in last["content"] if not part.get("ambient")]
|
|
2934
|
+
self._reflex_told, self._learning_told = getattr(self, "_ambient_before", ({}, None))
|
|
2935
|
+
self._ambient_before = dict(getattr(self, "_reflex_told", {})), self._learning_told
|
|
2936
|
+
content.append(self._ambient(loop))
|
|
2937
|
+
with self._condition:
|
|
2938
|
+
self._say(content)
|
|
2939
|
+
if (state or {}).get("ended"):
|
|
2940
|
+
self._run_end = "episode_ended"
|
|
2941
|
+
|
|
2942
|
+
def _ambient(self, loop) -> dict:
|
|
2943
|
+
"""How things stand now, told last with every call that tells the model anything,
|
|
2944
|
+
whatever it is told: the state, the units that run, what changed in the tree of
|
|
2945
|
+
reflexes, what was learned. What the run has left (its steps, its calls, its time)
|
|
2946
|
+
is told only where the run asks for it (runtime.tell_budget): the limits are the
|
|
2947
|
+
host's to keep, and a countdown with every call weighs on every judgement."""
|
|
2948
|
+
config, embodiment = self.config, self.config.embodiment
|
|
2949
|
+
with self._condition:
|
|
2950
|
+
calls_left = config.max_model_calls - self._iterations
|
|
2951
|
+
running = [unit_text(identifier, unit) for identifier, unit in self._units.items()]
|
|
2952
|
+
tree = self._tree_told()
|
|
2953
|
+
state = self._state_now
|
|
2954
|
+
lines = []
|
|
2955
|
+
if state:
|
|
2956
|
+
lines.append(f"{EMBODIMENT_TEXT[embodiment]['state_label']}: "
|
|
2957
|
+
+ state_text(state, embodiment, self._steps_shown()))
|
|
2958
|
+
if self._budget_told and self._steps_shown() and state.get("step") is not None:
|
|
2959
|
+
lines.append(steps_text(state, self.profile.get("control_hz")))
|
|
2384
2960
|
if running:
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2961
|
+
lines.append("Units running: " + latest(running))
|
|
2962
|
+
if tree:
|
|
2963
|
+
lines.append(tree)
|
|
2964
|
+
# What was learned so far, where this run learns: whole when it changed, else its first line.
|
|
2965
|
+
head, _, body = self._learning_report().strip().partition("\n")
|
|
2966
|
+
if head:
|
|
2967
|
+
lines.append(head + ("\n" + body if body != self._learning_told else " as you were last told"))
|
|
2968
|
+
self._learning_told = body
|
|
2969
|
+
if self._budget_told:
|
|
2970
|
+
if self._post_from is not None and loop.post_calls:
|
|
2971
|
+
calls_left = min(calls_left, loop.post_calls - (self.summary["model_calls"] - self._post_from))
|
|
2972
|
+
if calls_left > 0:
|
|
2973
|
+
lines.append("After these calls the learned policy runs on by itself to the end of the episode.")
|
|
2974
|
+
lines.append(f"Model calls left: {calls_left}")
|
|
2975
|
+
return {"type": "input_text", "text": "\n".join(lines), "ambient": True}
|
|
2976
|
+
|
|
2977
|
+
def _think(self):
|
|
2978
|
+
"""Queue the model's next call with its immutable input, on the trigger of what it
|
|
2979
|
+
was told and has not answered (a message, then an event), else of what happened
|
|
2980
|
+
last (self.reason)."""
|
|
2981
|
+
trigger = ("user" if "message" in self._unanswered else "escalation" if "event" in self._unanswered
|
|
2982
|
+
else self.reason)
|
|
2983
|
+
if trigger not in decisions.REASONS:
|
|
2984
|
+
raise ValueError(f"No model trigger for {trigger!r}")
|
|
2403
2985
|
config, summary, messages = self.config, self.summary, self.messages
|
|
2404
2986
|
if self._standing:
|
|
2405
2987
|
prune_images(messages, int(config.runtime.get("keep_images", 2)))
|
|
2406
2988
|
else:
|
|
2407
2989
|
prune_views(messages, self._view_keep, self._view_pins)
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2990
|
+
idle = time.time() - self._replied_at if self._replied_at else 0.0
|
|
2991
|
+
after = float(config.runtime.get("clear_results_after_minutes") or 0) * 60
|
|
2992
|
+
if after and idle >= after:
|
|
2993
|
+
# The provider's cache of the conversation is cold: this call reads all of it anew.
|
|
2994
|
+
cleared = clear_results(messages, decisions.KEPT_RESULTS, decisions.CLEARED)
|
|
2995
|
+
if cleared:
|
|
2996
|
+
self._replaces, self._logged = self._replaces + self._logged, 0
|
|
2997
|
+
self.trace.emit("results_cleared", results=cleared, idle_seconds=round(idle))
|
|
2998
|
+
request = messages if self._images else text_only(messages)
|
|
2411
2999
|
call = summary["model_calls"]
|
|
2412
3000
|
from copy import deepcopy
|
|
2413
3001
|
history = deepcopy(messages)
|
|
2414
|
-
system = config.system_prompt
|
|
3002
|
+
system = config.system_prompt
|
|
2415
3003
|
thought = Thought("main", trigger, self.chat, system, request,
|
|
2416
|
-
timeout=max(0, self.seconds_left()),
|
|
2417
|
-
revision=self._revision, call=call,
|
|
2418
|
-
invoke=lambda item: self._complete(item, history))
|
|
2419
|
-
thought.ahead_generation = self._ahead_generation
|
|
3004
|
+
timeout=max(0, self.seconds_left()), owner=self._who().get("who"),
|
|
3005
|
+
revision=self._revision, call=call, invoke=self._complete)
|
|
2420
3006
|
self.trace.emit("model_request", call=call, thought=thought.id, **decisions._model_of(thought.chat), reason=trigger,
|
|
2421
|
-
|
|
3007
|
+
total_messages=len(messages), sent_messages=len(request),
|
|
2422
3008
|
new_messages=history[self._logged:], context=context_summary(request, system),
|
|
2423
3009
|
library_version=getattr(self, "_library_version", None), revision=thought.revision,
|
|
2424
|
-
events=list(self.
|
|
2425
|
-
**({"message": True} if trigger == "message" else {}), **self._replaced())
|
|
3010
|
+
events=list(self._told_events), **self._replaced())
|
|
2426
3011
|
self._logged = len(messages)
|
|
2427
3012
|
summary["model_calls"] += 1
|
|
2428
3013
|
summary["decisions"][trigger] = summary["decisions"].get(trigger, 0) + 1
|
|
2429
3014
|
self._iterations += 1
|
|
3015
|
+
self._save()
|
|
2430
3016
|
self._request = thought
|
|
2431
3017
|
self._thoughts.submit(thought)
|
|
2432
3018
|
return thought
|
|
2433
3019
|
|
|
2434
|
-
def
|
|
2435
|
-
|
|
3020
|
+
def _within_budget(self):
|
|
3021
|
+
"""Whether the model may be called again: calls and cost left."""
|
|
3022
|
+
return (self._iterations < self.config.max_model_calls
|
|
2436
3023
|
and (self.config.max_cost_usd is None
|
|
2437
3024
|
or self.summary["cost_usd"] + self.agents_usage["cost_usd"] < self.config.max_cost_usd))
|
|
2438
3025
|
|
|
@@ -2446,58 +3033,11 @@ class TaskRun:
|
|
|
2446
3033
|
self._logged -= 1
|
|
2447
3034
|
self._replaces = 1
|
|
2448
3035
|
|
|
2449
|
-
def _include_escalations(self):
|
|
2450
|
-
parts = []
|
|
2451
|
-
for identifier, report in self._pending_events.items():
|
|
2452
|
-
if identifier not in self._reported_events:
|
|
2453
|
-
self._reported_events.add(identifier)
|
|
2454
|
-
parts.append({"type": "input_text", "text": f"Unresolved event: {json.dumps(report)}"})
|
|
2455
|
-
if parts:
|
|
2456
|
-
if self.messages[-1]["role"] == "user":
|
|
2457
|
-
self._extend_request(parts)
|
|
2458
|
-
else:
|
|
2459
|
-
self._tell(parts)
|
|
2460
|
-
|
|
2461
|
-
def _pending_message(self, now):
|
|
2462
|
-
return bool(self._inbox)
|
|
2463
|
-
|
|
2464
|
-
def _unresolved(self, now):
|
|
2465
|
-
return bool(self._pending_events)
|
|
2466
|
-
|
|
2467
|
-
def _pending_escalation(self, now):
|
|
2468
|
-
return bool(self._pending_events) and not self._pending_events.keys() <= self._reported_events
|
|
2469
|
-
|
|
2470
|
-
def _pending_watch(self, now):
|
|
2471
|
-
return (self._watched is not None and self._request is None and self._ahead_allowed(reserve=1)
|
|
2472
|
-
and not self._thoughts.pending)
|
|
2473
|
-
|
|
2474
|
-
def _pending_ahead(self, now):
|
|
2475
|
-
return (self._ahead_due is not None and self._ahead_due <= now and self._request is None
|
|
2476
|
-
and self._ahead_allowed(reserve=1) and not self._thoughts.pending)
|
|
2477
|
-
|
|
2478
|
-
def _decision_tick(self):
|
|
2479
|
-
"""Coalesce what is visible at this tick before acquiring a transport."""
|
|
2480
|
-
if not self._cell_running or self._program_completing or self._closing or self._pause:
|
|
2481
|
-
return None
|
|
2482
|
-
if not self._ahead_allowed():
|
|
2483
|
-
if self._inbox and not self._budget_stop:
|
|
2484
|
-
self._budget_stop = True
|
|
2485
|
-
self._stop_program_for_decision()
|
|
2486
|
-
return None
|
|
2487
|
-
if self._thoughts.active is not None:
|
|
2488
|
-
return None
|
|
2489
|
-
now = time.monotonic()
|
|
2490
|
-
if self._think("running", self._turns) is not None:
|
|
2491
|
-
return None
|
|
2492
|
-
if self._ahead_due is not None and self._ahead_due > now:
|
|
2493
|
-
return self._ahead_due
|
|
2494
|
-
return None
|
|
2495
|
-
|
|
2496
3036
|
def _came_back(self, thought):
|
|
2497
3037
|
from harness.runtime.decision.model import add_usage
|
|
2498
3038
|
|
|
2499
3039
|
summary = self.summary
|
|
2500
|
-
key = decisions.
|
|
3040
|
+
key = decisions.COUNTS.get(thought.role)
|
|
2501
3041
|
if key is not None:
|
|
2502
3042
|
summary[key] = summary.get(key, 0) + 1
|
|
2503
3043
|
summary["agent_seconds"] += thought.seconds
|
|
@@ -2507,108 +3047,8 @@ class TaskRun:
|
|
|
2507
3047
|
if key == "vision_calls":
|
|
2508
3048
|
summary["vision_cost_usd"] += float(usage.get("cost_usd") or 0.0)
|
|
2509
3049
|
self.charge_up()
|
|
2510
|
-
if key is not None:
|
|
2511
|
-
return
|
|
2512
|
-
if thought.outcome is not None:
|
|
3050
|
+
if key is None and thought.outcome is not None:
|
|
2513
3051
|
self._record_settlement(thought)
|
|
2514
|
-
elif (self._cell_running and not self._program_completing and self._request is thought
|
|
2515
|
-
and thought.revision == self._revision
|
|
2516
|
-
and not self._inbox and not self._pause):
|
|
2517
|
-
self._dispatch_running_reply(thought)
|
|
2518
|
-
|
|
2519
|
-
def _dispatch_running_reply(self, thought):
|
|
2520
|
-
if thought.cancelled or thought.trigger == "ahead" or thought.reply is None:
|
|
2521
|
-
return
|
|
2522
|
-
text = thought.reply.text
|
|
2523
|
-
if thought.reply.usage.get("finish_reason") == "length":
|
|
2524
|
-
return
|
|
2525
|
-
parsed, prose = decisions._running_reply(text, self._kinds)
|
|
2526
|
-
kind = getattr(parsed, "kind", None)
|
|
2527
|
-
stop = bool(decisions._STOP.search(text))
|
|
2528
|
-
if prose and thought.trigger == "message":
|
|
2529
|
-
output = "message"
|
|
2530
|
-
elif kind == "python" and not stop and self.confirmation != "always" and decisions._beside(parsed.body):
|
|
2531
|
-
output = "parallel"
|
|
2532
|
-
elif kind not in (None, "wait") and (stop or thought.trigger == "message"):
|
|
2533
|
-
output = "replace"
|
|
2534
|
-
else:
|
|
2535
|
-
output = "wait"
|
|
2536
|
-
handler = decisions.OUTPUTS["running"][output]
|
|
2537
|
-
if handler is not None:
|
|
2538
|
-
if output == "parallel":
|
|
2539
|
-
self._run_beside(thought, parsed.body)
|
|
2540
|
-
else:
|
|
2541
|
-
getattr(self, handler)(thought)
|
|
2542
|
-
|
|
2543
|
-
def _interrupt(self, thought: Thought) -> None:
|
|
2544
|
-
"""Stop the running program at its next step for `thought`'s reply, which runs next
|
|
2545
|
-
(the loop uses it when the program ends on the stop); a failed thought made for a
|
|
2546
|
-
message stops it too: the model reads the message once it has."""
|
|
2547
|
-
if thought.trigger in ("escalation", "watch"):
|
|
2548
|
-
found = decisions._STOP.search(thought.reply.text)
|
|
2549
|
-
why = " ".join(found.group(1).split()) if found else ""
|
|
2550
|
-
why = f"a look: {why}"[:300] if why else "a look"
|
|
2551
|
-
else:
|
|
2552
|
-
why = "the call failed" if thought.error is not None else "the reply to a message"
|
|
2553
|
-
with self._condition:
|
|
2554
|
-
if self.kernel is None or not self._cell_running or self._request is not thought:
|
|
2555
|
-
return
|
|
2556
|
-
thought.stopped = thought.error is None
|
|
2557
|
-
self.trace.emit("program_stopped", program=thought.program, call=thought.call + 1, why=why)
|
|
2558
|
-
self._stop_program_for_decision()
|
|
2559
|
-
|
|
2560
|
-
def _stop_program_for_decision(self):
|
|
2561
|
-
self._interruptions += 1
|
|
2562
|
-
if self.agents is not None:
|
|
2563
|
-
self.agents.wake()
|
|
2564
|
-
self.kernel.message()
|
|
2565
|
-
|
|
2566
|
-
def _say_beside(self, thought: Thought) -> bool:
|
|
2567
|
-
"""A reply in prose to a message that came while a program runs: it reaches the user
|
|
2568
|
-
at once (an agent_message), the program going on, and joins the history when the
|
|
2569
|
-
program ends; False when the program has ended."""
|
|
2570
|
-
with self._condition:
|
|
2571
|
-
if not (self._cell_running and self._request is thought):
|
|
2572
|
-
return False
|
|
2573
|
-
thought.said = True
|
|
2574
|
-
self._settle(thought, "said", "told the user while the program ran")
|
|
2575
|
-
self._pending_events.clear()
|
|
2576
|
-
self.question = thought.reply.text.strip()
|
|
2577
|
-
self.trace.emit("agent_message", text=self.question, program=thought.program)
|
|
2578
|
-
return True
|
|
2579
|
-
|
|
2580
|
-
def _beside_report(self, ahead: Thought) -> str:
|
|
2581
|
-
"""What a block run beside the program did, for the model."""
|
|
2582
|
-
side = ahead.side["result"] or {"status": "still running"}
|
|
2583
|
-
return (f"The program you ran beside program {ahead.program}: {side.get('status')}"
|
|
2584
|
-
+ (f"\n{side['error']}" if side.get("error") else "")
|
|
2585
|
-
+ (f"\nEvent: {side['escalation']['event']['kind']}" if side.get("escalation") else ""))
|
|
2586
|
-
|
|
2587
|
-
def _run_beside(self, ahead: Thought, program: str) -> bool:
|
|
2588
|
-
"""Start a block that can run beside the program."""
|
|
2589
|
-
identifier = uuid.uuid4().hex[:12]
|
|
2590
|
-
with self._condition:
|
|
2591
|
-
kernel = self.kernel if self._cell_running and self._request is ahead else None
|
|
2592
|
-
if kernel is None or not kernel.side(program, identifier):
|
|
2593
|
-
return False
|
|
2594
|
-
ahead.side = {"id": identifier, "code": program, "result": None}
|
|
2595
|
-
self._sides[identifier] = ahead
|
|
2596
|
-
self.trace.emit("side_program", id=identifier, program=ahead.program, call=ahead.call + 1,
|
|
2597
|
-
code=program)
|
|
2598
|
-
self._settle(ahead, "beside", "ran beside the program")
|
|
2599
|
-
self._pending_events.clear()
|
|
2600
|
-
return True
|
|
2601
|
-
|
|
2602
|
-
def _side_ended(self, record: dict) -> None:
|
|
2603
|
-
"""How a program run beside the running one ended (the kernel's side_result)."""
|
|
2604
|
-
with self._condition:
|
|
2605
|
-
ahead = self._sides.pop(record.get("id"), None)
|
|
2606
|
-
result = {key: record.get(key) for key in ("status", "error", "escalation") if record.get(key) is not None}
|
|
2607
|
-
self.trace.emit("side_program_end", id=record.get("id"), **result)
|
|
2608
|
-
if ahead is not None:
|
|
2609
|
-
ahead.side["result"] = result
|
|
2610
|
-
if result.get("status") in ("error", "escalated"):
|
|
2611
|
-
self._escalation({"id": record["id"], "entry": record["id"], "event": result})
|
|
2612
3052
|
|
|
2613
3053
|
def _replaced(self) -> dict:
|
|
2614
3054
|
"""What the next request record says of the history messages it replaces."""
|
|
@@ -2616,10 +3056,12 @@ class TaskRun:
|
|
|
2616
3056
|
return {"replaces": replaces} if replaces else {}
|
|
2617
3057
|
|
|
2618
3058
|
def _settle(self, thought, outcome, why=""):
|
|
3059
|
+
"""What became of a call of the model: used, or set aside (discarded: something
|
|
3060
|
+
reached the model first; unused: the run ended first)."""
|
|
2619
3061
|
if thought.outcome is not None:
|
|
2620
3062
|
return
|
|
2621
3063
|
thought.outcome = outcome, why
|
|
2622
|
-
if outcome
|
|
3064
|
+
if outcome != "used":
|
|
2623
3065
|
if thought.exited:
|
|
2624
3066
|
thought.cancel(why)
|
|
2625
3067
|
self.trace.emit("thought_cancelled", thought=thought.id, role=thought.role,
|
|
@@ -2631,53 +3073,31 @@ class TaskRun:
|
|
|
2631
3073
|
|
|
2632
3074
|
def _record_settlement(self, thought):
|
|
2633
3075
|
outcome, why = thought.outcome
|
|
2634
|
-
|
|
2635
|
-
if
|
|
2636
|
-
outcomes = summary.setdefault("ahead_outcomes", {})
|
|
2637
|
-
outcomes[outcome] = outcomes.get(outcome, 0) + 1
|
|
2638
|
-
self.trace.emit("ahead", thought=thought.id, call=thought.call + 1, program=thought.program,
|
|
2639
|
-
trigger=thought.trigger, outcome=outcome, why=why,
|
|
2640
|
-
seconds=round(thought.seconds, 3), cancelled=thought.cancelled)
|
|
2641
|
-
if thought.program is None and outcome != "used":
|
|
3076
|
+
reply = thought.reply
|
|
3077
|
+
if outcome != "used":
|
|
2642
3078
|
self.trace.emit("model_preempted", thought=thought.id, call=thought.call + 1, reason=why,
|
|
2643
3079
|
back=reply is not None, **({"text": reply.text, "usage": reply.usage} if reply else {}))
|
|
2644
|
-
elif reply is not None and outcome != "used":
|
|
2645
|
-
self._record_reply(reply, thought.call + 1, thought=thought.id, ahead=outcome)
|
|
2646
3080
|
|
|
2647
3081
|
def _record_reply(self, reply, call, **detail):
|
|
2648
3082
|
self.trace.emit("model_reply", call=call, text=reply.text, usage=reply.usage, response_id=reply.response_id,
|
|
2649
3083
|
seconds=reply.seconds, **detail, **({"reasoning": reply.reasoning} if reply.reasoning else {}))
|
|
2650
3084
|
|
|
2651
|
-
def _complete(self, thought
|
|
2652
|
-
"""The main transport
|
|
3085
|
+
def _complete(self, thought):
|
|
3086
|
+
"""The main transport."""
|
|
2653
3087
|
from harness.runtime.decision.model import ChatModel
|
|
2654
3088
|
|
|
2655
|
-
|
|
3089
|
+
chat = thought.chat
|
|
2656
3090
|
options = {"on_text": self._streamed(thought.call + 1, stop=lambda: thought.cancelled)} if getattr(
|
|
2657
3091
|
chat, "streams_text", False) else {}
|
|
2658
3092
|
if isinstance(chat, ChatModel):
|
|
2659
3093
|
options["cancellation"] = thought.cancellation
|
|
2660
|
-
attempt = 0
|
|
2661
3094
|
name = decisions._model_of(chat).get("model") or str(self.config.main.get("model", "model"))
|
|
2662
3095
|
with self._span("model", name, call=thought.call + 1,
|
|
2663
|
-
thought=thought.id, role="main", reason=thought.trigger
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
reply = chat.complete(thought.system, request, **options)
|
|
2669
|
-
end.update(output_tokens=reply.usage.get("output_tokens"), attempts=attempt)
|
|
2670
|
-
thought.turns = turns
|
|
2671
|
-
return reply
|
|
2672
|
-
except RuntimeError as error:
|
|
2673
|
-
if "out of GPU memory" not in str(error) or len(request) <= 2:
|
|
2674
|
-
raise
|
|
2675
|
-
turns = max(1, (turns or (len(history) - 1) // 2) - 1)
|
|
2676
|
-
shorter = window(history, turns)
|
|
2677
|
-
if len(shorter) >= len(request):
|
|
2678
|
-
raise
|
|
2679
|
-
self.trace.emit("context_shortened", history_turns=turns, sent_messages=len(shorter))
|
|
2680
|
-
request = shorter
|
|
3096
|
+
thought=thought.id, role="main", reason=thought.trigger) as end:
|
|
3097
|
+
thought.cancellation.check()
|
|
3098
|
+
reply = chat.complete(thought.system, thought.messages, **options)
|
|
3099
|
+
end.update(output_tokens=reply.usage.get("output_tokens"))
|
|
3100
|
+
return reply
|
|
2681
3101
|
|
|
2682
3102
|
def _streamed(self, call, every=decisions.STREAMED_SECONDS, stop=None):
|
|
2683
3103
|
"""What gets the pieces of a streamed reply of call `call`: model_text records of
|
|
@@ -2708,7 +3128,7 @@ class TaskRun:
|
|
|
2708
3128
|
identifier = self._program_approval_id = uuid.uuid4().hex
|
|
2709
3129
|
self.trace.emit("confirmation_request", id=identifier, code=program)
|
|
2710
3130
|
try:
|
|
2711
|
-
self._wait(WAITING_FOR_CONFIRMATION, lambda: self._decision is not None or
|
|
3131
|
+
self._wait(WAITING_FOR_CONFIRMATION, lambda: self._decision is not None or self._message_waits(),
|
|
2712
3132
|
reason="program", request={"id": identifier, "code": program})
|
|
2713
3133
|
finally:
|
|
2714
3134
|
with self._condition:
|
|
@@ -2721,7 +3141,7 @@ class TaskRun:
|
|
|
2721
3141
|
self.trace.emit("confirmation", id=identifier, accepted=accepted, reason=reason)
|
|
2722
3142
|
if accepted:
|
|
2723
3143
|
return True
|
|
2724
|
-
self.
|
|
3144
|
+
self._came_to("The user rejected this program, so nothing ran." + (f" Their reason: {reason}" if reason else ""))
|
|
2725
3145
|
self.reason = "rejected"
|
|
2726
3146
|
return False
|
|
2727
3147
|
|
|
@@ -2741,7 +3161,7 @@ class TaskRun:
|
|
|
2741
3161
|
or cells <= self._reflected_cells or self.seconds_left() <= 0):
|
|
2742
3162
|
return
|
|
2743
3163
|
with self._condition:
|
|
2744
|
-
if self._pause or self._closing or self.
|
|
3164
|
+
if self._pause or self._closing or self._message_waits() or self._busy():
|
|
2745
3165
|
return
|
|
2746
3166
|
revision = self._revision
|
|
2747
3167
|
since = self._reflected_cells
|
|
@@ -2765,7 +3185,6 @@ class TaskRun:
|
|
|
2765
3185
|
|
|
2766
3186
|
self.config = copy(self.config)
|
|
2767
3187
|
self.config.system_prompt += f"\nObservation from this run: {found.text.strip()}\n"
|
|
2768
|
-
self.invalidate_lookahead()
|
|
2769
3188
|
except ThoughtCancelled:
|
|
2770
3189
|
pass
|
|
2771
3190
|
except Exception as error:
|
|
@@ -2780,8 +3199,9 @@ class TaskRun:
|
|
|
2780
3199
|
and "code -15" in summary["error"])
|
|
2781
3200
|
|
|
2782
3201
|
def _close_episode(self, token) -> None:
|
|
2783
|
-
"""
|
|
2784
|
-
|
|
3202
|
+
"""The conversation ends: end its sub-agents, close the kernel, read the evaluator's
|
|
3203
|
+
outcome with the token (a worker evaluates once: at the end), and give the result
|
|
3204
|
+
of the turn that ran (none ran when it ends between turns)."""
|
|
2785
3205
|
trace, summary, directory, env = self.trace, self.summary, self.directory, self.env
|
|
2786
3206
|
with self._condition:
|
|
2787
3207
|
self._cancel_managed_worker_approvals("the episode ended")
|
|
@@ -2797,6 +3217,18 @@ class TaskRun:
|
|
|
2797
3217
|
trace.emit("cleanup_error", message=f"kernel: {type(error).__name__}: {error}")
|
|
2798
3218
|
if self.environments is not None:
|
|
2799
3219
|
self.environments.close()
|
|
3220
|
+
for model in (self.chat, self.verifier):
|
|
3221
|
+
# A package's backend holds a process of its own (an ACP agent).
|
|
3222
|
+
if getattr(model, "packaged", False):
|
|
3223
|
+
try:
|
|
3224
|
+
model.close()
|
|
3225
|
+
except Exception as error:
|
|
3226
|
+
trace.emit("cleanup_error", message=f"model: {type(error).__name__}: {error}")
|
|
3227
|
+
if self._inbound is not None:
|
|
3228
|
+
try:
|
|
3229
|
+
self._inbound.close()
|
|
3230
|
+
except Exception as error:
|
|
3231
|
+
trace.emit("cleanup_error", message=f"listener: {type(error).__name__}: {error}")
|
|
2800
3232
|
if hasattr(self, "_mcp_access"):
|
|
2801
3233
|
try:
|
|
2802
3234
|
self._mcp_access.close()
|
|
@@ -2836,9 +3268,15 @@ class TaskRun:
|
|
|
2836
3268
|
except Exception as error:
|
|
2837
3269
|
trace.emit("cleanup_error", message=f"outcome: {type(error).__name__}: {error}")
|
|
2838
3270
|
learning = self._learning()
|
|
2839
|
-
if self.kernel is not None:
|
|
2840
|
-
self._reflect_learning()
|
|
2841
3271
|
table = self.config.runtime.get("library") or {}
|
|
3272
|
+
if self._turn_began is not None or not self._turns:
|
|
3273
|
+
# The turn that ran (or a conversation that had none) ends with the conversation.
|
|
3274
|
+
ending = summary["status"]
|
|
3275
|
+
self._turn_result(next((reason for reason, status in decisions.TURN_ENDS.items() if status == ending),
|
|
3276
|
+
ending))
|
|
3277
|
+
else:
|
|
3278
|
+
# Ended between turns: the last turn gave its result; the summary is closed.
|
|
3279
|
+
self._write_summary()
|
|
2842
3280
|
uses = table.get("uses", {"learn": "on", "off": "off"}.get(table.get("experience"), "learn"))
|
|
2843
3281
|
self._thoughts.close()
|
|
2844
3282
|
with self._condition:
|
|
@@ -2849,14 +3287,16 @@ class TaskRun:
|
|
|
2849
3287
|
# What this episode's closed loops of library skills did, as their uses in the
|
|
2850
3288
|
# library's index (runtime/learning/usage.py fold_uses).
|
|
2851
3289
|
try:
|
|
2852
|
-
|
|
3290
|
+
learner = self._learner()
|
|
3291
|
+
trace.emit("library", action="used", uses=fold_uses(learner, directory))
|
|
3292
|
+
for entry in learner.demoted:
|
|
3293
|
+
# Admitted by its own test, it kept failing in use (learner.record_uses).
|
|
3294
|
+
trace.emit("library", action="demoted", **entry)
|
|
3295
|
+
learner.demoted.clear()
|
|
2853
3296
|
except Exception as error:
|
|
2854
3297
|
trace.emit("cleanup_error", message=f"library uses: {type(error).__name__}: {error}")
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
summary["agents"] = {**self.agents_usage, "cost_usd": round(self.agents_usage["cost_usd"], 6)}
|
|
2858
|
-
write_json(directory / "summary.json", summary)
|
|
2859
|
-
self.charge_up()
|
|
3298
|
+
if "model_transport_pending" in summary:
|
|
3299
|
+
write_json(directory / "summary.json", summary)
|
|
2860
3300
|
self._set_status(ERROR if summary["status"] == "error" else FINISHED, reason=summary["status"])
|
|
2861
3301
|
|
|
2862
3302
|
|
|
@@ -2865,6 +3305,7 @@ VERIFIER_EVIDENCE = (" Besides any pictures, the message gives the state and wha
|
|
|
2865
3305
|
"(a window's elements, a page's text): judge on all of it, and answer NO when together it "
|
|
2866
3306
|
"does not show enough to confirm the claim.")
|
|
2867
3307
|
# What the vision role is asked of a view, for a main model that reads no images.
|
|
3308
|
+
SEARCHER = "Search the web for what the user names and list what you found, one line for each page."
|
|
2868
3309
|
VIEWER = ("You describe a view for an agent that cannot see it. Say what is shown: the windows or objects, the "
|
|
2869
3310
|
"text and numbers exactly as written, the controls and where each is (pixel coordinates from the top "
|
|
2870
3311
|
"left). Facts only, no advice.")
|