@lmmslab/harness 0.0.0-stage → 0.81.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1356 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +232 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Decision protocol, dispatch tables and speculative-reply rules.
|
|
2
|
+
|
|
3
|
+
TaskRun coordinates these decisions with task lifecycle and program execution.
|
|
4
|
+
"""
|
|
5
|
+
import ast
|
|
6
|
+
import re
|
|
7
|
+
from harness.runtime.decision.protocol import AHEAD_KINDS, ProtocolError, looks_like_action, parse_reply
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# Seconds between two model_text records of a streamed reply.
|
|
11
|
+
STREAMED_SECONDS = 0.3
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Added to the system prompt of an interactive conversation.
|
|
15
|
+
INTERACTIVE_PROMPT = (
|
|
16
|
+
"\n\nA user is with you in this episode. Their messages reach you as \"Message from the user: ...\". "
|
|
17
|
+
"To ask for information needed by this task, reply in plain text without a block; the task waits "
|
|
18
|
+
"for the answer. Use <finish> for the completed task and your final answer. That ends this run; "
|
|
19
|
+
"it does not open a new conversation turn."
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Added to an interactive conversation's prompt (a sub-agent's too): a message while a program runs.
|
|
24
|
+
MESSAGE_PROMPT = (
|
|
25
|
+
" A message that comes while a program runs reaches you at once: reply <wait> to let the program go on "
|
|
26
|
+
"(you are asked again with its result), a block that moves nothing to run it beside the program, or what "
|
|
27
|
+
"should run instead, which stops the program at its next step."
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# Sent back for a reply that only announces what the model will do, at most ANNOUNCED_TIMES
|
|
32
|
+
# in a row (replayed: it then wrote its program in 32 of 33 calls).
|
|
33
|
+
ANNOUNCED_TIMES = 2
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
ANNOUNCED = ("Nothing ran: your reply says what you will do, but it has no {block} block. Do it now: reply with "
|
|
37
|
+
"the {block} block itself, starting with its opening tag. If you meant only to tell {who} "
|
|
38
|
+
"something, reply with that in plain text again.")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# An empty reply (reasoning only, not cut off: the provider's text never came) is asked
|
|
42
|
+
# again at once, without a format error, at most this many times in a row.
|
|
43
|
+
EMPTY_RETRIES = 2
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# Old profiles can still disable speculation; the other values share one policy.
|
|
47
|
+
THINK_WHILE_ACTING = ("off", "on", "on_events", "periodic")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
AHEAD_SECONDS = 3.0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
_ASSUME = re.compile(r"<assume>(.*?)</assume>", re.DOTALL)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
_ASSUME_NODES = (
|
|
57
|
+
ast.Expression, ast.BoolOp, ast.And, ast.Or, ast.UnaryOp, ast.Not, ast.USub,
|
|
58
|
+
ast.Compare, ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE, ast.Is,
|
|
59
|
+
ast.IsNot, ast.In, ast.NotIn, ast.Name, ast.Load, ast.Constant, ast.Subscript,
|
|
60
|
+
ast.Slice, ast.Tuple, ast.List,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
AHEAD_RUNNING = (
|
|
65
|
+
"Program {number} is running now. Prepare what comes after it. State a check on its result in "
|
|
66
|
+
"<assume>...</assume> before the next block. The check can read result (the final "
|
|
67
|
+
"execution receipt), output (stdout) and state (the latest observation). "
|
|
68
|
+
"Reply <wait> if you need to see the result first."
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
LOOK_RUNNING = (
|
|
73
|
+
"Program {number} is still running. Reply <wait> to let it continue, a Python block that "
|
|
74
|
+
"can run alongside it, or <stop>reason</stop> and the replacement block."
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# The message of a call made because the model's own watch(every=seconds) is due while
|
|
79
|
+
# program N runs: the views it asked for come with it.
|
|
80
|
+
WATCH_RUNNING = (
|
|
81
|
+
"Program {number} is still running; these are the views your watch() asked for. Reply <wait> to let it "
|
|
82
|
+
"continue, a Python block that can run alongside it, or <stop>reason</stop> and the replacement block."
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
_STOP = re.compile(r"<stop>(.*?)</stop>", re.DOTALL)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# The end of the message of a call made when a message for the model comes while program
|
|
90
|
+
# N runs (A6's demo 3: a budget typed during a 17 s page load sat unread until it ended).
|
|
91
|
+
MESSAGE_RUNNING = ("Program {number} is still running. Reply <wait> to let it finish: you are asked again with its "
|
|
92
|
+
"result. Or reply with what should run instead: the program stops at its next step, and your "
|
|
93
|
+
"block runs then.")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# What a reply may do beside the running program at once, instead of stopping it: tell or
|
|
97
|
+
# start agents and entries, update the plan, note places, plain Python.
|
|
98
|
+
_BESIDE_CALLS = {"send_agent", "cancel_agent", "delegate", "spawn", "plan", "step_done", "recall", "read_skill",
|
|
99
|
+
"remember", "forget", "things", "thing", "forget_thing",
|
|
100
|
+
"print", "len", "str", "int", "float", "bool", "list", "dict", "set", "tuple", "range",
|
|
101
|
+
"enumerate", "zip", "sorted", "min", "max", "sum", "any", "all", "isinstance", "round", "abs",
|
|
102
|
+
"repr", "format"}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
_BESIDE_METHODS = {"append", "extend", "get", "items", "keys", "values", "format", "join", "strip", "split",
|
|
106
|
+
"lower", "upper", "replace", "startswith", "endswith", "update", "pop", "copy", "count",
|
|
107
|
+
"index"}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _beside(program: str) -> bool:
|
|
111
|
+
"""Whether `program` needs no body: it calls only _BESIDE_CALLS, _BESIDE_METHODS and
|
|
112
|
+
functions it defines itself, so it may run beside a running program. What it hands to
|
|
113
|
+
spawn() runs as an entry of its own, whose channels the tick keeps apart
|
|
114
|
+
(CHANNEL_CONFLICT), so it may do anything."""
|
|
115
|
+
try:
|
|
116
|
+
tree = ast.parse(program)
|
|
117
|
+
except SyntaxError:
|
|
118
|
+
return False
|
|
119
|
+
own = {node.name for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))}
|
|
120
|
+
spawned = {id(inner) for node in ast.walk(tree)
|
|
121
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "spawn"
|
|
122
|
+
for argument in [*node.args, *(keyword.value for keyword in node.keywords)]
|
|
123
|
+
for inner in ast.walk(argument)}
|
|
124
|
+
for node in ast.walk(tree):
|
|
125
|
+
if isinstance(node, ast.Call) and id(node) not in spawned:
|
|
126
|
+
func = node.func
|
|
127
|
+
if not (isinstance(func, ast.Name) and (func.id in _BESIDE_CALLS or func.id in own)
|
|
128
|
+
or isinstance(func, ast.Attribute) and func.attr in _BESIDE_METHODS):
|
|
129
|
+
return False
|
|
130
|
+
return True
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# runtime.thinking: a level bounds the reasoning effort (a model set lower keeps its own)
|
|
134
|
+
# and the reply's tokens; "model" leaves them. (Replayed: nine in ten replies under 500.)
|
|
135
|
+
THINKING = {"model": (None, None), "none": ("none", 2000), "low": ("low", 2000), "medium": ("medium", 8000),
|
|
136
|
+
"high": ("high", None)}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
EFFORTS = ("none", "low", "medium", "high")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# Added to the system prompt at "none" and "low".
|
|
143
|
+
THINK_BRIEFLY = ("\n\nThink briefly: a sentence or two, then the block. Time goes on while you think, and a long "
|
|
144
|
+
"reply is cut off with nothing run.")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _model_of(chat) -> dict:
|
|
148
|
+
"""The model a call goes to and its reasoning effort, as a record says them (none for a
|
|
149
|
+
scripted model)."""
|
|
150
|
+
model = getattr(chat, "model", None)
|
|
151
|
+
if not isinstance(model, str):
|
|
152
|
+
return {}
|
|
153
|
+
effort = getattr(chat, "reasoning_effort", None)
|
|
154
|
+
return {"model": model, **({"reasoning_effort": effort} if effort else {})}
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _thinking(main: dict, level: str) -> dict:
|
|
158
|
+
"""The main model's role with its effort and reply length bounded by `level`."""
|
|
159
|
+
effort, tokens = THINKING[level]
|
|
160
|
+
main = dict(main)
|
|
161
|
+
if effort is not None:
|
|
162
|
+
own = main.get("reasoning_effort")
|
|
163
|
+
if own not in EFFORTS or EFFORTS.index(own) > EFFORTS.index(effort):
|
|
164
|
+
main["reasoning_effort"] = effort
|
|
165
|
+
if tokens is not None:
|
|
166
|
+
main["max_output_tokens"] = min(int(main.get("max_output_tokens") or tokens), tokens)
|
|
167
|
+
return main
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# Pending events take priority over the event that brought the loop to a decision.
|
|
171
|
+
# Explicit requests have a trigger per role and share the same transport as decisions.
|
|
172
|
+
TRIGGERS = {
|
|
173
|
+
"running": {"message": "_pending_message", "escalation": "_pending_escalation",
|
|
174
|
+
"ahead": "_pending_ahead", "watch": "_pending_watch"},
|
|
175
|
+
"between": {"user": "_pending_message", "escalation": "_unresolved",
|
|
176
|
+
"start": None, "program_end": None, "error": None, "rejected": None,
|
|
177
|
+
"finish_refused": None, "verify": None, "empty": None, "format": None},
|
|
178
|
+
"request": {"verifier": "request", "observer": "request", "vlm": "request",
|
|
179
|
+
"pointer": "request", "reflection": "reflect"},
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
OUTPUTS = {
|
|
184
|
+
"between": {"program": "_out_program", "verify": "_out_verify", "finish": "_out_finish",
|
|
185
|
+
"message": "_out_message", "invalid": "_out_invalid", "empty": "_out_empty"},
|
|
186
|
+
"running": {"wait": None, "message": "_say_beside", "parallel": "_run_beside",
|
|
187
|
+
"replace": "_interrupt"},
|
|
188
|
+
"request": {"verifier": "verifier_calls", "reflection": "reflection_calls",
|
|
189
|
+
"observer": "vision_calls", "vlm": "vision_calls", "pointer": "vision_calls"},
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _running_reply(text, kinds):
|
|
194
|
+
"""Parse one running-program reply and distinguish ordinary prose from malformed blocks."""
|
|
195
|
+
try:
|
|
196
|
+
return parse_reply(text, (*kinds, *AHEAD_KINDS)), False
|
|
197
|
+
except ProtocolError:
|
|
198
|
+
return None, bool(text.strip()) and not looks_like_action(text)
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""The model's messages: what the conversation (runtime/conversation.py) tells the model
|
|
2
|
+
of an embodiment and of each program it ran. The text of each embodiment kind
|
|
3
|
+
(EMBODIMENT_TEXT), the state and the camera images of a message, a program's feedback
|
|
4
|
+
(its status, output, events, entries, the skills' turn report, the state and steps), and
|
|
5
|
+
what a request keeps of the history (prune_images, window).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
import json
|
|
10
|
+
|
|
11
|
+
from harness.embodiments import KINDS
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# What the model is told about each embodiment kind (embodiments.KINDS): the
|
|
15
|
+
# verifier's instructions, the label and keys of the state in every message, and the
|
|
16
|
+
# image streams the model sees unless runtime.cameras names others.
|
|
17
|
+
# The kinds are the packages' (kinds.toml), read when a kind is first asked for.
|
|
18
|
+
class _EmbodimentText(Mapping):
|
|
19
|
+
def __getitem__(self, name):
|
|
20
|
+
kind = KINDS[name]
|
|
21
|
+
return {"verifier": kind.verifier, "state_label": kind.state_label, "state_keys": kind.state_keys,
|
|
22
|
+
"cameras": kind.cameras}
|
|
23
|
+
|
|
24
|
+
def __iter__(self):
|
|
25
|
+
return iter(KINDS)
|
|
26
|
+
|
|
27
|
+
def __len__(self):
|
|
28
|
+
return len(KINDS)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
EMBODIMENT_TEXT = _EmbodimentText()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def image_parts(images: dict, cameras) -> list[dict]:
|
|
35
|
+
"""The images of `cameras` among `images`, each after a line with its name."""
|
|
36
|
+
parts = []
|
|
37
|
+
for camera in cameras:
|
|
38
|
+
if camera in images:
|
|
39
|
+
parts.append({"type": "input_text", "text": f"Camera: {camera}"})
|
|
40
|
+
parts.append({"type": "input_image", "image_url": images[camera], "detail": "auto"})
|
|
41
|
+
return parts
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def attachment_parts(paths, limit: int = 100_000) -> list[dict]:
|
|
45
|
+
"""The files a user attached to a message as content parts, each after a line naming it:
|
|
46
|
+
an image as it is, a folder as its names, any other file as its text (at most `limit`
|
|
47
|
+
characters)."""
|
|
48
|
+
from pathlib import Path
|
|
49
|
+
|
|
50
|
+
from harness.core.codec import data_url
|
|
51
|
+
|
|
52
|
+
parts = []
|
|
53
|
+
for path in map(Path, paths):
|
|
54
|
+
mime = IMAGE_TYPES.get(path.suffix.lower())
|
|
55
|
+
if path.is_dir():
|
|
56
|
+
names = sorted(child.name + ("/" if child.is_dir() else "") for child in path.iterdir())
|
|
57
|
+
parts.append({"type": "input_text", "text": f"Attached folder {path}:\n" + "\n".join(names[:500])})
|
|
58
|
+
elif mime:
|
|
59
|
+
parts += [{"type": "input_text", "text": f"Attached image {path}:"},
|
|
60
|
+
{"type": "input_image", "image_url": data_url(path.read_bytes(), mime), "detail": "auto"}]
|
|
61
|
+
else:
|
|
62
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
63
|
+
more = f"\n… {len(text) - limit} more characters" if len(text) > limit else ""
|
|
64
|
+
parts.append({"type": "input_text", "text": f"Attached file {path}:\n{text[:limit]}{more}"})
|
|
65
|
+
return parts
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
IMAGE_TYPES = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif",
|
|
69
|
+
".webp": "image/webp"}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def state_text(state: dict, embodiment: str = "robot", steps: bool = True) -> str:
|
|
73
|
+
"""The state the model is shown; without `steps` (a worker that keeps its step budget
|
|
74
|
+
to itself: profile steps_shown false) no step count."""
|
|
75
|
+
keys = [key for key in EMBODIMENT_TEXT[embodiment]["state_keys"] if steps or key != "step"]
|
|
76
|
+
return json.dumps({key: state[key] for key in keys if key in state},
|
|
77
|
+
default=lambda v: round(v, 4) if isinstance(v, float) else v)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _round(value):
|
|
81
|
+
if isinstance(value, float):
|
|
82
|
+
return round(value, 4)
|
|
83
|
+
if isinstance(value, list):
|
|
84
|
+
return [_round(item) for item in value]
|
|
85
|
+
if isinstance(value, dict):
|
|
86
|
+
return {key: _round(item) for key, item in value.items()}
|
|
87
|
+
return value
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def cell_feedback(result: dict, number: int, hz: float | None = None, embodiment: str = "robot",
|
|
91
|
+
steps: bool = True) -> str:
|
|
92
|
+
"""What the model is told of program `number` it ran: its status, output and error, the
|
|
93
|
+
event that ended it, the executions and entries, the skills' turn report, the state and
|
|
94
|
+
the steps. It names the program, since the result of one program can reach the model
|
|
95
|
+
after the next program's code (a reply made while it ran)."""
|
|
96
|
+
lines = [f"Program {number} status: {result['status']} ({result.get('seconds', 0):.1f} s)"]
|
|
97
|
+
if result.get("stdout"):
|
|
98
|
+
lines.append("Output:\n" + result["stdout"].rstrip())
|
|
99
|
+
if result.get("error"):
|
|
100
|
+
lines.append("Error:\n" + result["error"].rstrip())
|
|
101
|
+
trace = result.get("escalation") or result.get("episode_ended")
|
|
102
|
+
if trace:
|
|
103
|
+
event = trace["event"]
|
|
104
|
+
lines.append("Unhandled event: " + json.dumps(_round({
|
|
105
|
+
"kind": event["kind"], "source": event["source"], "detail": event["detail"],
|
|
106
|
+
"skill": trace["skill"], "parameters": trace["parameters"],
|
|
107
|
+
"ticks": trace["end_tick"] - trace["start_tick"]})))
|
|
108
|
+
if result.get("events"):
|
|
109
|
+
executed = [f"{item['skill']}: {item['event']['kind']}" + (" (handled)" if item.get("handled") else "")
|
|
110
|
+
for item in result["events"]]
|
|
111
|
+
lines.append("Executions: " + "; ".join(executed[-20:]))
|
|
112
|
+
if result.get("event_counts"):
|
|
113
|
+
lines.append("Execution totals (details in trace): " + json.dumps(result["event_counts"]))
|
|
114
|
+
if result.get("entries"):
|
|
115
|
+
lines.append("Entries: " + "; ".join(entry_text(item) for item in result["entries"][-20:]))
|
|
116
|
+
if (result.get("report") or {}).get("text"):
|
|
117
|
+
lines.append(result["report"]["text"])
|
|
118
|
+
if result.get("state"):
|
|
119
|
+
lines.append(f"{EMBODIMENT_TEXT[embodiment]['state_label']}: "
|
|
120
|
+
+ state_text(result["state"], embodiment, steps))
|
|
121
|
+
if steps and result["state"].get("step") is not None:
|
|
122
|
+
lines.append(steps_text(result["state"], hz))
|
|
123
|
+
for name, part in (result.get("environments") or {}).items():
|
|
124
|
+
# An environment attached to the conversation (runtime/execution/attached.py).
|
|
125
|
+
if part.get("events"):
|
|
126
|
+
lines.append(f"{name} executions: " + "; ".join(
|
|
127
|
+
f"{item['skill']}: {item['event']['kind']}" + (" (handled)" if item.get("handled") else "")
|
|
128
|
+
for item in part["events"][-20:]))
|
|
129
|
+
if part.get("error"):
|
|
130
|
+
lines.append(f"{name}: {part['error']}")
|
|
131
|
+
if part.get("state"):
|
|
132
|
+
lines.append(f"{name} state: " + json.dumps(_round({
|
|
133
|
+
key: value for key, value in part["state"].items()
|
|
134
|
+
if isinstance(value, (str, int, float, bool)) or key in ("screen",)}))[:1200])
|
|
135
|
+
return "\n".join(lines)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def entry_text(item: dict) -> str:
|
|
139
|
+
"""An entry as the model hears of it after a program (runtime/execution/entries.py)."""
|
|
140
|
+
text = f"{item['entry']} {item['name']} ({item['kind']}"
|
|
141
|
+
if item.get("parent") not in (None, "e0"):
|
|
142
|
+
text += f", started by {item['parent']}"
|
|
143
|
+
text += f"): {item['status']} after {item['seconds']:g} s"
|
|
144
|
+
if item.get("error"):
|
|
145
|
+
text += f" ({item['error']})"
|
|
146
|
+
elif item.get("result") is not None:
|
|
147
|
+
text += f", returned {item['result']}"
|
|
148
|
+
return text
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def steps_text(state: dict, hz: float | None) -> str:
|
|
152
|
+
"""The env steps used, and left where the episode's limit is reported (Z2 showed both
|
|
153
|
+
every turn): every motion tick counts against the limit."""
|
|
154
|
+
step, limit = int(state["step"]), state.get("step_limit")
|
|
155
|
+
used = f"Env steps used: {step}" + (f" ({hz:g} per second)" if hz else "")
|
|
156
|
+
return used + (f"; left: {int(limit) - step}" if limit else "; the limit is not reported")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def prune_images(messages: list[dict], keep: int) -> None:
|
|
160
|
+
"""Keep images only in the last `keep` user messages."""
|
|
161
|
+
seen = 0
|
|
162
|
+
for message in reversed(messages):
|
|
163
|
+
if message["role"] != "user" or isinstance(message["content"], str):
|
|
164
|
+
continue
|
|
165
|
+
if any(part["type"] == "input_image" for part in message["content"]):
|
|
166
|
+
seen += 1
|
|
167
|
+
if seen > keep:
|
|
168
|
+
message["content"] = [part if part["type"] != "input_image"
|
|
169
|
+
else {"type": "input_text", "text": "[earlier image omitted]"}
|
|
170
|
+
for part in message["content"]]
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
OMITTED = {"type": "input_text", "text": "[earlier image omitted]"}
|
|
174
|
+
# What a model that reads no images gets in place of one.
|
|
175
|
+
NOT_SENT = "[image not sent: this model reads no images]"
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def view_parts(looks, limit: int = 8) -> list[dict]:
|
|
179
|
+
"""The views a program looked at (look, watch: runtime/execution/session.py) as content
|
|
180
|
+
parts: each after a line naming it, its image marked as a view (and with its pin), so
|
|
181
|
+
that prune_views keeps it only as long as the model said."""
|
|
182
|
+
parts = []
|
|
183
|
+
for item in list(looks)[-limit:]:
|
|
184
|
+
where = f" region {item['region']}" if item.get("region") else ""
|
|
185
|
+
pinned = f" (pinned as {item['pin']!r})" if item.get("pin") else ""
|
|
186
|
+
parts.append({"type": "input_text", "text": f"View: {item['camera']}{where}{pinned}"})
|
|
187
|
+
parts.append({"type": "input_image", "image_url": item["image_url"], "detail": "auto",
|
|
188
|
+
"view": item["camera"], **({"pin": item["pin"]} if item.get("pin") else {})})
|
|
189
|
+
return parts
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def prune_views(messages: list[dict], keep: int = 1, pins=(), others: int = 2) -> None:
|
|
193
|
+
"""What stays in the history when the model looks on demand. A view is in the model's
|
|
194
|
+
sight for the `keep` replies after the program that looked (1: its next reply only),
|
|
195
|
+
a pinned one while its name is in `pins`; any other image (a file the user attached,
|
|
196
|
+
a show()) stays in the last `others` messages that have one, as prune_images keeps them."""
|
|
197
|
+
turns = seen = 0
|
|
198
|
+
pins = set(pins)
|
|
199
|
+
for message in reversed(messages):
|
|
200
|
+
if message["role"] != "user" or isinstance(message["content"], str):
|
|
201
|
+
continue
|
|
202
|
+
turns += 1
|
|
203
|
+
other = any(part["type"] == "input_image" and "view" not in part for part in message["content"])
|
|
204
|
+
seen += other
|
|
205
|
+
if not any(part["type"] == "input_image" for part in message["content"]):
|
|
206
|
+
continue
|
|
207
|
+
message["content"] = [
|
|
208
|
+
OMITTED if part["type"] == "input_image" and (
|
|
209
|
+
(turns > keep and part.get("pin") not in pins) if "view" in part else seen > others)
|
|
210
|
+
else part for part in message["content"]]
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def text_only(messages: list[dict]) -> list[dict]:
|
|
214
|
+
"""`messages` for a model that reads no images: each image as a line saying it was not sent."""
|
|
215
|
+
out = []
|
|
216
|
+
for message in messages:
|
|
217
|
+
content = message["content"]
|
|
218
|
+
if not isinstance(content, str) and any(part["type"] == "input_image" for part in content):
|
|
219
|
+
message = {**message, "content": [
|
|
220
|
+
{"type": "input_text", "text": NOT_SENT} if part["type"] == "input_image" else part
|
|
221
|
+
for part in content]}
|
|
222
|
+
out.append(message)
|
|
223
|
+
return out
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def window(messages: list[dict], turns: int | None) -> list[dict]:
|
|
227
|
+
"""The first message (instruction and initial views) plus the last `turns`
|
|
228
|
+
assistant/user exchanges. Program state lives in the kernel, so older turns
|
|
229
|
+
can drop out of the prompt without losing variables."""
|
|
230
|
+
if not turns or len(messages) <= 1 + 2 * turns:
|
|
231
|
+
return messages
|
|
232
|
+
# messages alternate user, assistant, user, ...; keep the alternation.
|
|
233
|
+
recent = messages[-2 * turns:]
|
|
234
|
+
if recent[0]["role"] != "assistant":
|
|
235
|
+
recent = recent[1:]
|
|
236
|
+
omitted = len(messages) - 1 - len(recent)
|
|
237
|
+
first = dict(messages[0])
|
|
238
|
+
note = {"type": "input_text", "text": f"[{omitted} later messages are omitted here; kernel variables persist]"}
|
|
239
|
+
first["content"] = (list(first["content"]) + [note] if isinstance(first["content"], list)
|
|
240
|
+
else f"{first['content']}\n{note['text']}")
|
|
241
|
+
return [first] + recent
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _inbox_parts(pending) -> list[dict]:
|
|
245
|
+
"""Messages taken from the inbox as content parts: one text with each message, then the
|
|
246
|
+
files attached to them."""
|
|
247
|
+
text = "\n".join(f"Message from {'the user' if sender == 'user' else 'the agent that started you'}: "
|
|
248
|
+
+ (item or "(the files below)") for sender, item, _ in pending)
|
|
249
|
+
return [{"type": "input_text", "text": text}] + [part for *_, parts in pending for part in parts]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def rebuild_messages(records) -> list[dict]:
|
|
253
|
+
"""The main model's history as the trace has it, up to its last reply: what each
|
|
254
|
+
request added (from the previous reply on, in place of the last messages it
|
|
255
|
+
replaces), less the malformed exchanges dropped after it, and the last reply (but a
|
|
256
|
+
reply to a call made while a program ran that was not used). Image pruning and the
|
|
257
|
+
history window are applied per request (runtime/decision/messages.py prune_images, window),
|
|
258
|
+
so the rebuilt list, pruned the same way, is the conversation's own."""
|
|
259
|
+
history, reply = [], None
|
|
260
|
+
for record in records:
|
|
261
|
+
if record["event"] == "model_request":
|
|
262
|
+
if record.get("replaces"):
|
|
263
|
+
del history[len(history) - int(record["replaces"]):]
|
|
264
|
+
history.extend(record["new_messages"])
|
|
265
|
+
reply = None
|
|
266
|
+
elif record["event"] == "model_reply" and record.get("ahead", "used") == "used":
|
|
267
|
+
reply = {"role": "assistant", "content": record["text"]}
|
|
268
|
+
elif record["event"] == "malformed_dropped":
|
|
269
|
+
# The exchanges before the reply that came after them.
|
|
270
|
+
del history[len(history) - int(record["messages"]):]
|
|
271
|
+
return history + ([reply] if reply else [])
|