@lmmslab/harness 0.85.1 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/python/harness/__init__.py +2 -2
- package/python/harness/bench.py +2 -2
- package/python/harness/builtin/benches/README.md +1 -2
- package/python/harness/builtin/configs/default.toml +4 -19
- package/python/harness/builtin/configs/deploy/example-api.toml +1 -1
- package/python/harness/builtin/configs/models/deepseek-flash.toml +1 -1
- package/python/harness/builtin/configs/models/grok-low.toml +1 -1
- package/python/harness/builtin/configs/models/grok-medium.toml +3 -6
- package/python/harness/builtin/environment/README.md +8 -25
- package/python/harness/builtin/prompts/conversation/core.md +6 -5
- package/python/harness/builtin/prompts/core/core.md +147 -203
- package/python/harness/builtin/prompts/core/pack.toml +0 -1
- package/python/harness/builtin/prompts/reflect/core.md +8 -32
- package/python/harness/builtin/prompts/reflect/pack.toml +2 -4
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -3
- package/python/harness/builtin/prompts/shared/agents.md +20 -10
- package/python/harness/builtin/prompts/shared/checklist.md +13 -17
- package/python/harness/builtin/prompts/shared/environments.md +20 -15
- package/python/harness/builtin/prompts/shared/learning-goal.md +2 -3
- package/python/harness/builtin/prompts/shared/learning.md +73 -23
- package/python/harness/builtin/prompts/shared/mcp.md +11 -6
- package/python/harness/builtin/prompts/shared/reflex-method.md +243 -0
- package/python/harness/builtin/prompts/shared/units.md +138 -0
- package/python/harness/builtin/prompts/shared/web.md +5 -6
- package/python/harness/builtin/skills/agents/__init__.py +18 -3
- package/python/harness/builtin/skills/agents/skill.toml +3 -3
- package/python/harness/builtin/skills/checklist/__init__.py +34 -32
- package/python/harness/builtin/skills/checklist/skill.toml +9 -9
- package/python/harness/builtin/skills/environments/__init__.py +9 -5
- package/python/harness/builtin/skills/environments/skill.toml +1 -1
- package/python/harness/builtin/skills/experience/skill.toml +1 -1
- package/python/harness/builtin/skills/learning/__init__.py +19 -5
- package/python/harness/builtin/skills/mcp/skill.toml +1 -1
- package/python/harness/builtin/skills/memory/__init__.py +4 -1
- package/python/harness/builtin/skills/memory/skill.toml +0 -5
- package/python/harness/builtin/skills/verify/skill.toml +1 -0
- package/python/harness/builtin/skills/web/__init__.py +4 -1
- package/python/harness/builtin/third_party/sources.toml +2 -40
- package/python/harness/cli.py +16 -25
- package/python/harness/config/__init__.py +2 -2
- package/python/harness/config/agent.py +59 -47
- package/python/harness/core/__init__.py +1 -1
- package/python/harness/core/codec.py +4 -4
- package/python/harness/core/contracts.py +1 -1
- package/python/harness/core/media.py +1 -1
- package/python/harness/core/ports.py +2 -2
- package/python/harness/core/procs.py +1 -1
- package/python/harness/core/remote.py +1 -1
- package/python/harness/core/worker.py +19 -13
- package/python/harness/embodiments.py +53 -44
- package/python/harness/extensions.py +22 -0
- package/python/harness/host/__init__.py +1 -1
- package/python/harness/host/http.py +6 -12
- package/python/harness/host/sessions.py +1 -1
- package/python/harness/host/tasks.py +15 -11
- package/python/harness/host/workspaces.py +36 -8
- package/python/harness/packages/__init__.py +5 -5
- package/python/harness/packages/environments.py +30 -18
- package/python/harness/packages/manager.py +1 -1
- package/python/harness/packages/manifest.py +6 -6
- package/python/harness/packages/provision.py +1 -1
- package/python/harness/packages/sources.py +1 -1
- package/python/harness/paths.py +1 -1
- package/python/harness/runtime/decision/loop.py +95 -5
- package/python/harness/runtime/decision/messages.py +128 -2
- package/python/harness/runtime/decision/model.py +4 -5
- package/python/harness/runtime/decision/prompts.py +20 -32
- package/python/harness/runtime/events.py +7 -5
- package/python/harness/runtime/execution/attached.py +3 -3
- package/python/harness/runtime/execution/entries.py +189 -8
- package/python/harness/runtime/execution/kernel.py +26 -1
- package/python/harness/runtime/execution/results.py +2 -3
- package/python/harness/runtime/execution/session.py +417 -32
- package/python/harness/runtime/execution/skills.py +63 -16
- package/python/harness/runtime/learning/active.py +7 -7
- package/python/harness/runtime/learning/learner.py +3 -2
- package/python/harness/runtime/learning/library.py +9 -4
- package/python/harness/runtime/learning/reflect.py +10 -9
- package/python/harness/runtime/reactive/rules.py +4 -4
- package/python/harness/runtime/reactive/tree.py +506 -25
- package/python/harness/runtime/safety/input.py +1 -1
- package/python/harness/runtime/safety/isolation.py +3 -3
- package/python/harness/runtime/safety/shield.py +4 -50
- package/python/harness/runtime/task/delegation.py +5 -5
- package/python/harness/runtime/task/environments.py +10 -0
- package/python/harness/runtime/task/run.py +222 -27
- package/python/harness/runtime/tick.py +24 -1
- package/python/harness/sdk/__init__.py +2 -5
- package/python/harness/services/registry.py +23 -19
- package/python/harness/settings.py +9 -9
- package/python/harness/builtin/environment/constraints.txt +0 -31
- package/python/harness/builtin/prompts/shared/world.md +0 -1
|
@@ -43,6 +43,52 @@ TURN_GRACE_SECONDS = 2.0
|
|
|
43
43
|
# and the model is not told them; "always": told with every call (a benchmark whose
|
|
44
44
|
# episodes are a few dozen calls, where the model must plan its end).
|
|
45
45
|
TELL_BUDGET = ("never", "always")
|
|
46
|
+
# Whether the model's nodes of the tree of reflexes follow the method: the page on how a tree
|
|
47
|
+
# is made is read first, every node does something, and its condition can be judged
|
|
48
|
+
# (runtime.reflex_method; runtime/execution/session.py).
|
|
49
|
+
REFLEX_METHOD = ("required", "optional")
|
|
50
|
+
# A page of instructions given again after a summary, when the model had read it (as Claude
|
|
51
|
+
# Code shows the skills invoked before a compaction again).
|
|
52
|
+
# A page of instructions the model asked for (guide(name)), given whole with the result of
|
|
53
|
+
# the unit that asked: from here on it is read (as Claude Code's Read returns a file).
|
|
54
|
+
GUIDE_PAGE = "guide(\"{name}\"), the page you asked for:\n\n{page}"
|
|
55
|
+
GUIDE_AGAIN = ("You read guide(\"{name}\") before this summary. It is given again so that you still have "
|
|
56
|
+
"it:\n\n{page}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Whether a program has to say what it is for (runtime.purpose). "required": one that calls
|
|
60
|
+
# neither until(condition) nor until(None) is not started and the model is told NO_PURPOSE, as
|
|
61
|
+
# Claude Code refuses a call that leaves out what it is for.
|
|
62
|
+
PURPOSES = ("required", "optional")
|
|
63
|
+
NO_PURPOSE = ("Nothing ran: this program does not say what it is for. Call until(condition) in it, with a "
|
|
64
|
+
"function of the view that holds when the program's work is done; a program that only reads "
|
|
65
|
+
"says until(None). Send the program again with it.")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# What the model is told with a program's end when all it expected held and what it was for
|
|
69
|
+
# is not the case.
|
|
70
|
+
ALL_HELD = ("Every expectation held and the result is not there: the program relied on something it did not "
|
|
71
|
+
"check, or its until does not say the result.")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def calls(program: str, name: str) -> bool:
|
|
75
|
+
"""Whether the program calls name(...). One that does not parse says nothing either way:
|
|
76
|
+
it is started, and its error is the answer."""
|
|
77
|
+
import ast
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
tree = ast.parse(program)
|
|
81
|
+
except (SyntaxError, ValueError):
|
|
82
|
+
return True
|
|
83
|
+
return any(isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == name
|
|
84
|
+
for node in ast.walk(tree))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def says_purpose(program: str) -> bool:
|
|
88
|
+
"""Whether the program calls until(...)."""
|
|
89
|
+
return calls(program, "until")
|
|
90
|
+
|
|
91
|
+
|
|
46
92
|
|
|
47
93
|
|
|
48
94
|
# runtime.longest_wait_seconds: the longest a <wait> lasts; one that names no time lasts
|
|
@@ -58,7 +104,33 @@ WAIT_ENDS = {"unit": "a unit you started ended", "message": "a message reached y
|
|
|
58
104
|
|
|
59
105
|
|
|
60
106
|
# How the model reads an event that reached it; what the event's record holds follows.
|
|
107
|
+
# The plan (the checklist skill) is shown to the model again when it has made PLAN_STALE_CALLS
|
|
108
|
+
# calls without touching it, as Claude Code reminds of a task list that was not used recently.
|
|
109
|
+
PLAN_STALE_CALLS = 10
|
|
110
|
+
PLAN_STALE = ("Your plan has not been touched for {calls} calls. As it stands:\n{steps}\n"
|
|
111
|
+
"Check off what is done, and call plan() again if it no longer matches what you are doing. "
|
|
112
|
+
"This is only a reminder: ignore it if it does not apply.")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# The part of the state the environment calls the agent's situation (its profile: "situation")
|
|
116
|
+
# took a value it has not had in this run.
|
|
117
|
+
SITUATION_NEW = ("The state's {key} is now {now}, for the first time in this run (it was {before}). "
|
|
118
|
+
"The tree of reflexes has {nodes}, set before that. Before a program acts here, go through what "
|
|
119
|
+
"can happen to the agent here and give situations(...) the lines for it.{hint}")
|
|
120
|
+
|
|
121
|
+
|
|
61
122
|
EVENT = "Event {kind} from {source}"
|
|
123
|
+
# An event of a kind the model was told before in this run: its count, and how long ago the
|
|
124
|
+
# last EARLIER_TOLD of the earlier ones were told.
|
|
125
|
+
EVENT_AGAIN = " (the {count} of this kind in this run; the earlier ones {earlier} ago)"
|
|
126
|
+
EARLIER_TOLD = 5
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def ordinal(number: int) -> str:
|
|
130
|
+
"""1st, 2nd, 3rd, 4th, 11th, 22nd."""
|
|
131
|
+
if 10 <= number % 100 <= 20:
|
|
132
|
+
return f"{number}th"
|
|
133
|
+
return f"{number}" + {1: "st", 2: "nd", 3: "rd"}.get(number % 10, "th")
|
|
62
134
|
|
|
63
135
|
|
|
64
136
|
# What the model is told, with its next prompt, when its conversation goes on in a new
|
|
@@ -68,11 +140,26 @@ CONTINUED = ("Note from the harness: this conversation was continued in a new pr
|
|
|
68
140
|
"defined, the rules you set, and the units that still ran{units}. Set up again what you still need.")
|
|
69
141
|
|
|
70
142
|
|
|
71
|
-
# What the model is told when its turn comes back while its program still runs.
|
|
143
|
+
# What the model is told when its turn comes back while its program still runs: the fact.
|
|
144
|
+
# What a running program asks of the model is told with every call while one runs
|
|
145
|
+
# (WHILE_RUNNING), whatever the call is for.
|
|
72
146
|
RUNNING = "Program {number} is still running, as unit {unit}."
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
#
|
|
147
|
+
# Told with how things stand, at every call while a program of the model runs: when it has
|
|
148
|
+
# just been started, when a wait ended, when another program ended, when an event came. One
|
|
149
|
+
# text for all of them: they are the same moment, the model looking in on work that runs.
|
|
150
|
+
WHILE_RUNNING = ("A program of yours runs ({units}). It is yours to see through. Say how long it still has to "
|
|
151
|
+
"run, and compare where it is now with where it should be by now: change or end it when it is "
|
|
152
|
+
"behind. If you wait for its end, you learn that it failed only then, and all that time is "
|
|
153
|
+
"lost: look again well before it is due to end (a wait much shorter than what it has left), "
|
|
154
|
+
"have its report show the numbers that tell how far it is (until(report=)), and have a node "
|
|
155
|
+
"tell you when what it relies on is gone. Between the looks, prepare what comes after it with "
|
|
156
|
+
"what needs no acting on the environment: read the pages the next goal needs, write and keep "
|
|
157
|
+
"the code it needs, set the nodes for where the agent goes next, bring the plan up to date. "
|
|
158
|
+
"The program for the next goal can be sent now: it begins with after(\"{unit}\"), which "
|
|
159
|
+
"returns when that one has ended, then checks with expect, then acts.")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# Added to the feedback of a program that ended, when no other program of the model runs.
|
|
76
163
|
PROGRAM_ENDED = ("\nThe program ended. If the task is done, answer in plain text; otherwise write a program "
|
|
77
164
|
"for the rest of the task.")
|
|
78
165
|
|
|
@@ -160,7 +247,10 @@ COMPACTED = (
|
|
|
160
247
|
"This session is being continued from a previous conversation that ran out of context. The summary below "
|
|
161
248
|
"covers the earlier portion of the conversation. Recent messages are preserved verbatim. The kernel with "
|
|
162
249
|
"what your programs defined, the library, the tree of reflexes and the units that run are as they were."
|
|
163
|
-
"\n\nSummary:\n{summary}"
|
|
250
|
+
"\n\nSummary:\n{summary}"
|
|
251
|
+
"\n\nIf you need specific details from before this summary (what you were told, what a program printed, "
|
|
252
|
+
"what you replied), read the full record at: {record}\nIt has one JSON line for each message, in order: "
|
|
253
|
+
"time, role, text.")
|
|
164
254
|
|
|
165
255
|
|
|
166
256
|
# What a reply of the model does: the method of TaskRun for each output. A reply that is
|
|
@@ -69,7 +69,7 @@ IMAGE_TYPES = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
|
|
69
69
|
".webp": "image/webp"}
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
def state_text(state: dict, embodiment: str = "
|
|
72
|
+
def state_text(state: dict, embodiment: str = "none", steps: bool = True) -> str:
|
|
73
73
|
"""The state the model is shown; without `steps` (a worker that keeps its step budget
|
|
74
74
|
to itself: profile steps_shown false) no step count."""
|
|
75
75
|
keys = [key for key in EMBODIMENT_TEXT[embodiment]["state_keys"] if steps or key != "step"]
|
|
@@ -110,6 +110,117 @@ def ended_text(result: dict) -> str:
|
|
|
110
110
|
+ f" ({result.get('seconds', 0):.1f} s)")
|
|
111
111
|
|
|
112
112
|
|
|
113
|
+
# The last line of a message about a reflex: the model is told because it chose to be, when
|
|
114
|
+
# it wrote the program or set the node, and this is what it does with the message.
|
|
115
|
+
CHOSE_PROGRAM = ("You are told this because you chose to be when you wrote {names}: it matters to that work. Do "
|
|
116
|
+
"not wait on as before. Compare what {names} reports with what it is for, find out why the "
|
|
117
|
+
"reflex had to act during it, and decide: it goes on as it is, you change it, or you end it.")
|
|
118
|
+
CHOSE_NODE = ("You are told this because you chose to be when you set {name}. Decide what it changes in what you "
|
|
119
|
+
"are doing now.")
|
|
120
|
+
CHOSE_STOP = ("{name} ended because you chose so when you wrote it. Find out why the reflex had to act during it, "
|
|
121
|
+
"then decide what is done next.")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def report_parts(name: str, told: dict) -> list[dict]:
|
|
125
|
+
"""The pictures and files of what `name` (a node, a program) reports of itself, as the
|
|
126
|
+
parts of a message, each after a line saying whose it is (`told`: {"shows": [{"caption",
|
|
127
|
+
"image_url"}], "files": [{"name", "text"}]}; the text is said with the event)."""
|
|
128
|
+
parts = []
|
|
129
|
+
for item in told.get("shows") or ():
|
|
130
|
+
parts.append({"type": "input_text", "text": f"{name} shows" + (f" {item['caption']}" if item.get("caption") else "") + ":"})
|
|
131
|
+
parts.append({"type": "input_image", "image_url": item["image_url"], "detail": "auto"})
|
|
132
|
+
for item in told.get("files") or ():
|
|
133
|
+
parts.append({"type": "input_text", "text": f"{name} attaches {item['name']}:\n{item['text']}"})
|
|
134
|
+
return parts
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def without_pictures(detail):
|
|
138
|
+
"""An event's detail as it is recorded: the pictures and files of the reports in it are
|
|
139
|
+
counted, not kept."""
|
|
140
|
+
if not isinstance(detail, dict):
|
|
141
|
+
return detail
|
|
142
|
+
out = {key: value for key, value in detail.items() if key not in ("shows", "files", "interrupted")}
|
|
143
|
+
for key in ("shows", "files"):
|
|
144
|
+
if detail.get(key):
|
|
145
|
+
out[key] = len(detail[key])
|
|
146
|
+
if detail.get("interrupted"):
|
|
147
|
+
out["interrupted"] = [without_pictures(row) for row in detail["interrupted"]]
|
|
148
|
+
return out
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _times(count: int) -> str:
|
|
152
|
+
return "once" if count == 1 else f"{count} times"
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def reflex_text(kind: str, detail: dict) -> str:
|
|
156
|
+
"""What a node of the tree of reflexes did, as the model reads it: the node with its own
|
|
157
|
+
sentence (`about`), how its program ended, the units it took the environment from (how
|
|
158
|
+
often, for how long), the node's own report (`says`; without one, what its program
|
|
159
|
+
returned), and what each of those units says of itself."""
|
|
160
|
+
name, rows = detail.get("reflex"), detail.get("interrupted") or []
|
|
161
|
+
did = ("did its work" if kind == "REFLEX_ACTED"
|
|
162
|
+
else "ran, and the agent is still in its situation with no other node answering it")
|
|
163
|
+
lines = [f"{name}" + (f" ({detail['about']})" if detail.get("about") else "") + f" {did}: its program ended "
|
|
164
|
+
f"{detail.get('ended')} after {detail.get('seconds', 0):g} s."]
|
|
165
|
+
for row in rows:
|
|
166
|
+
lines.append(f"It took the environment from {row['name']}, {_times(row.get('times') or 1)} so far "
|
|
167
|
+
f"({row.get('paused_seconds', 0):g} s paused in all); {row['name']} goes on.")
|
|
168
|
+
if detail.get("says") is not None:
|
|
169
|
+
lines.append(f"{name} reports: {detail['says']}")
|
|
170
|
+
elif detail.get("returned") is not None:
|
|
171
|
+
lines.append(f"{name} returned: {detail['returned']}")
|
|
172
|
+
if detail.get("error"):
|
|
173
|
+
lines.append(f"{name} failed: {detail['error']}")
|
|
174
|
+
for row in rows:
|
|
175
|
+
done = "" if row.get("done") is None else "; done: yes" if row["done"] else "; done: no"
|
|
176
|
+
told = (f"reports: {row['says']}" if row.get("says") is not None
|
|
177
|
+
else "reports with what is shown below" if row.get("shows") or row.get("files") else "gives no report")
|
|
178
|
+
lines.append(f"{row['name']} {told} (running {row.get('seconds', 0):g} s{done})")
|
|
179
|
+
# Whose choice it was that the model is told, and what it is told to do with it.
|
|
180
|
+
asked = [row["name"] for row in rows if row.get("asked")]
|
|
181
|
+
if asked:
|
|
182
|
+
lines.append(CHOSE_PROGRAM.format(names=" and ".join(asked)))
|
|
183
|
+
else:
|
|
184
|
+
lines.append(CHOSE_NODE.format(name=name))
|
|
185
|
+
return "\n".join(lines)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def taken_text(detail: dict) -> str:
|
|
189
|
+
"""A unit that ended where a unit of higher priority took the environment from it, as
|
|
190
|
+
the model reads it: who took it, with the node's own sentence, from which unit and
|
|
191
|
+
execution, what that unit says of itself, whether it is started again, and the state."""
|
|
192
|
+
who = str(detail.get("rule") or detail.get("by_name") or detail.get("by"))
|
|
193
|
+
name = detail.get("name") or detail.get("unit")
|
|
194
|
+
lines = [f"{who}" + (f" ({detail['about']})" if detail.get("about") else "")
|
|
195
|
+
+ f" took the environment from {name}" + (f" during {detail['execution']}" if detail.get("execution") else "")
|
|
196
|
+
+ (", which is started again when its turn comes." if detail.get("restart")
|
|
197
|
+
else ", which ended there" + (" as it asked to (until(interrupted=\"stop\"))." if detail.get("asked") else "."))]
|
|
198
|
+
if detail.get("says") is not None or detail.get("done") is not None or detail.get("shows") or detail.get("files"):
|
|
199
|
+
done = "" if detail.get("done") is None else " (done: yes)" if detail["done"] else " (done: no)"
|
|
200
|
+
told = (f"reports: {detail['says']}" if detail.get("says") is not None
|
|
201
|
+
else "reports with what is shown below" if detail.get("shows") or detail.get("files") else "gives no report")
|
|
202
|
+
lines.append(f"{name} {told}{done}")
|
|
203
|
+
if detail.get("state"):
|
|
204
|
+
lines.append(f"State: {detail['state']}")
|
|
205
|
+
if detail.get("asked"):
|
|
206
|
+
lines.append(CHOSE_STOP.format(name=name))
|
|
207
|
+
return "\n".join(lines)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def expected_lines(result: dict) -> list:
|
|
211
|
+
"""What a unit expected of the environment (expect(text, check)), one line each with what
|
|
212
|
+
its check showed; and, when all of it held and what the unit was for is not the case,
|
|
213
|
+
that it relied on something it did not check."""
|
|
214
|
+
from harness.runtime.decision.loop import ALL_HELD
|
|
215
|
+
|
|
216
|
+
rows = result.get("expected") or []
|
|
217
|
+
lines = [f'Expected "{row["text"]}": ' + ("it held." if row["held"] else f"it does not hold: {row.get('found')}")
|
|
218
|
+
for row in rows]
|
|
219
|
+
if rows and result.get("done") is False and not result.get("error") and all(row["held"] for row in rows):
|
|
220
|
+
lines.append(ALL_HELD)
|
|
221
|
+
return lines
|
|
222
|
+
|
|
223
|
+
|
|
113
224
|
def cell_feedback(result: dict, number: int) -> str:
|
|
114
225
|
"""What the model is told of program `number` it ran: its status, output and error, the
|
|
115
226
|
event that ended it, the executions and the skills' turn report (the state as it is
|
|
@@ -117,6 +228,12 @@ def cell_feedback(result: dict, number: int) -> str:
|
|
|
117
228
|
program, since programs run side by side and the result of one can reach the model
|
|
118
229
|
after the code of the next."""
|
|
119
230
|
lines = [f"Program {number} " + ended_text(result)]
|
|
231
|
+
lines += expected_lines(result)
|
|
232
|
+
if result.get("says") is not None:
|
|
233
|
+
lines.append("It reports: " + str(result["says"]))
|
|
234
|
+
if result.get("interruptions_untold"):
|
|
235
|
+
lines.append(f"Reflexes took the environment from it {_times(result['interruptions_untold'])} more than you "
|
|
236
|
+
"were told one by one.")
|
|
120
237
|
if result.get("stdout"):
|
|
121
238
|
lines.append("Output:\n" + result["stdout"].rstrip())
|
|
122
239
|
if result.get("error"):
|
|
@@ -156,6 +273,7 @@ def unit_feedback(result: dict) -> str:
|
|
|
156
273
|
program spawned, a sub-agent): how it ended, what it printed and returned."""
|
|
157
274
|
lines = [f"Unit {result['unit']} {result.get('name') or ''} ({result.get('kind')}) " + ended_text(result)
|
|
158
275
|
+ (" (its failure was handled)" if result.get("handled") else "")]
|
|
276
|
+
lines += expected_lines(result)
|
|
159
277
|
if result.get("stdout"):
|
|
160
278
|
lines.append("Output:\n" + result["stdout"].rstrip())
|
|
161
279
|
if result.get("error"):
|
|
@@ -187,7 +305,15 @@ def unit_text(identifier: str, unit: dict, printed: bool = True) -> str:
|
|
|
187
305
|
last = (", has printed nothing" if unit["printed"] is None
|
|
188
306
|
else f", last printed {max(0.0, now - unit['printed']):.0f} s ago")
|
|
189
307
|
kept = f", priority {unit['priority']:g}" if unit.get("priority") else ""
|
|
190
|
-
|
|
308
|
+
# Which reflexes took the environment from it and gave it back, and how often.
|
|
309
|
+
paused = unit.get("paused_by") or {}
|
|
310
|
+
taken = ("; paused " + ", ".join(f"{count} time{'' if count == 1 else 's'} by {name}"
|
|
311
|
+
for name, count in sorted(paused.items(), key=lambda item: -item[1]))
|
|
312
|
+
if paused else "")
|
|
313
|
+
# What it says of itself (until(report=)), as of the state last reported.
|
|
314
|
+
says = f": {unit['says']}" if unit.get("says") is not None else ""
|
|
315
|
+
return (f"{identifier} {unit.get('name') or ''} ({unit.get('kind')}{kept}, "
|
|
316
|
+
f"{max(0.0, now - unit['started']):.0f} s{last}{taken}){says}")
|
|
191
317
|
|
|
192
318
|
|
|
193
319
|
def steps_text(state: dict, hz: float | None) -> str:
|
|
@@ -189,7 +189,7 @@ class ChatModel:
|
|
|
189
189
|
self.temperature = config.get("temperature")
|
|
190
190
|
self.retries = int(config.get("retries", 2))
|
|
191
191
|
# Keep retrying transient failures until this many seconds have passed since the
|
|
192
|
-
# request started (at least `retries` times). The
|
|
192
|
+
# request started (at least `retries` times). The environment does not advance while
|
|
193
193
|
# the model is being called, so waiting out an outage costs wall time only.
|
|
194
194
|
self.retry_seconds = config.get("retry_seconds")
|
|
195
195
|
self.retry_seconds = None if self.retry_seconds is None else float(self.retry_seconds)
|
|
@@ -207,7 +207,7 @@ class ChatModel:
|
|
|
207
207
|
def payload(self, system: str, messages: list[dict]) -> dict:
|
|
208
208
|
"""The request body: `system` and the harness messages as chat messages, the output
|
|
209
209
|
limit and extra_body fields, and on OpenRouter usage accounting and reasoning."""
|
|
210
|
-
# Some models
|
|
210
|
+
# Some models take no system message; an empty one is left out.
|
|
211
211
|
converted = [{"role": "system", "content": system}] if system else []
|
|
212
212
|
for message in messages:
|
|
213
213
|
content = message["content"]
|
|
@@ -324,9 +324,8 @@ class ChatModel:
|
|
|
324
324
|
|
|
325
325
|
|
|
326
326
|
# DeepSeek models sometimes answer with a tool call of the provider's own where the text
|
|
327
|
-
# should hold the block (finish_reason "tool_calls", no tools offered):
|
|
328
|
-
#
|
|
329
|
-
# one empty. A call named for a block (python, or a name with python, code or exec in it)
|
|
327
|
+
# should hold the block (finish_reason "tool_calls", no tools offered): in one run two calls ran nothing so, one
|
|
328
|
+
# reply prose only and one empty. A call named for a block (python, or a name with python, code or exec in it)
|
|
330
329
|
# is read as that block, its program the one text argument.
|
|
331
330
|
_TOOL_KINDS = ("python", "verify", "finish", "action", "wait")
|
|
332
331
|
_TOOL_ARGUMENTS = ("code", "program", "source", "script", "python", "input", "content", "text", "claim", "result")
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
"""System prompts assembled from a prompt pack for the skills a run enables.
|
|
2
2
|
|
|
3
|
-
A pack is a directory of Markdown sections (prompts/
|
|
3
|
+
A pack is a directory of Markdown sections (prompts/core, a package's prompts/<name>). Its
|
|
4
4
|
pack.toml lists the sections in order; a section with `skills` is included only when
|
|
5
5
|
one of them is enabled:
|
|
6
6
|
|
|
7
7
|
sections = [
|
|
8
8
|
{ file = "core.md" },
|
|
9
|
-
{ file = "
|
|
10
|
-
{ file = "
|
|
9
|
+
{ file = "search.md", skills = ["web"] },
|
|
10
|
+
{ file = "rules_search.md", skills = ["web"], optional = true },
|
|
11
11
|
]
|
|
12
12
|
|
|
13
13
|
A section { installed = true } is the skills a user installed (the app) that are on: each
|
|
@@ -16,21 +16,21 @@ Harness skill's summary, calls and prompt.md, and each Agent Skill's name and de
|
|
|
16
16
|
|
|
17
17
|
Inside a section, lines can be kept for some skills only:
|
|
18
18
|
|
|
19
|
-
<!-- if:
|
|
19
|
+
<!-- if: web, agents --> kept when all named skills are enabled
|
|
20
20
|
...
|
|
21
21
|
<!-- else --> kept otherwise (optional)
|
|
22
22
|
...
|
|
23
23
|
<!-- end -->
|
|
24
24
|
|
|
25
|
-
Markers may also name features of the connection (FEATURES
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
channels when the bench's embodiment declares channels (
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
25
|
+
Markers may also name features of the connection (FEATURES, and those a kind's package
|
|
26
|
+
names in its kinds.toml: native_features, which hold under the native contract, and
|
|
27
|
+
prompt_features, which always hold): step_limit when the episode's step limit is reported
|
|
28
|
+
(core/contract.py), channels when the bench's embodiment declares channels (inputs in
|
|
29
|
+
groups that units claim: runtime/tick.py), look when the environment has views and the
|
|
30
|
+
model looks at them on its own demand (no runtime.cameras: runtime/task/run.py), senses
|
|
31
|
+
when the kind says its environment computes values the model defines, and pauses when it
|
|
32
|
+
says its environment pauses a call for a unit of higher priority instead of ending it
|
|
33
|
+
(runtime/tick.py). think_while_acting never holds (the model's turn is always its own
|
|
34
34
|
now): a pack's text for it is left out. The marker lines themselves never reach the
|
|
35
35
|
model, so a prompt with every skill and feature enabled is the pack's text without them. The API documentation stays in the pack
|
|
36
36
|
because it describes one embodiment (its frames, reach and numbers); the skills'
|
|
@@ -44,25 +44,12 @@ import tomllib
|
|
|
44
44
|
from harness.paths import locate
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
# Sections of a pack without pack.toml (the RoboDojo layout before packs had one).
|
|
48
|
-
DEFAULT_SECTIONS = (
|
|
49
|
-
{"file": "core.md"},
|
|
50
|
-
{"file": "learned.md", "skills": ["vla", "wam"]},
|
|
51
|
-
{"file": "reactive.md", "skills": ["reactive"]},
|
|
52
|
-
{"file": "events.md"},
|
|
53
|
-
{"file": "handlers.md", "skills": ["handlers"]},
|
|
54
|
-
{"file": "rules.md"},
|
|
55
|
-
{"file": "rules_learned.md", "skills": ["vla"], "optional": True},
|
|
56
|
-
{"file": "rules_reactive.md", "skills": ["reactive"], "optional": True},
|
|
57
|
-
{"file": "rules_handlers.md", "skills": ["handlers"], "optional": True},
|
|
58
|
-
)
|
|
59
47
|
# Features of the connection that prompt text may depend on (not skills).
|
|
60
48
|
# `look`: the environment has views and the model looks on demand (look, watch): the
|
|
61
49
|
# configuration declares no standing watch (runtime.cameras).
|
|
62
50
|
# `machine`: programs are not isolated (runtime.isolation false): they run on the machine as
|
|
63
51
|
# its user, under the user's approvals (runtime/safety/permissions.py).
|
|
64
|
-
FEATURES = ("
|
|
65
|
-
"machine")
|
|
52
|
+
FEATURES = ("step_limit", "channels", "think_while_acting", "vision_service", "look", "senses", "pauses", "machine")
|
|
66
53
|
|
|
67
54
|
|
|
68
55
|
def features_for(runtime: dict, channels: bool = False, *, embodiment=None) -> set[str]:
|
|
@@ -92,7 +79,7 @@ def pack(directory: Path) -> dict:
|
|
|
92
79
|
"""The pack's settings: sections and the exported names it leaves undocumented."""
|
|
93
80
|
path = Path(directory) / "pack.toml"
|
|
94
81
|
if not path.is_file():
|
|
95
|
-
|
|
82
|
+
raise ValueError(f"{directory}: a prompt pack needs pack.toml with its `sections`")
|
|
96
83
|
with path.open("rb") as handle:
|
|
97
84
|
data = tomllib.load(handle)
|
|
98
85
|
unknown = set(data) - {"sections", "undocumented"}
|
|
@@ -104,7 +91,7 @@ def pack(directory: Path) -> dict:
|
|
|
104
91
|
|
|
105
92
|
def sections(directory: Path) -> list[dict]:
|
|
106
93
|
"""The pack's sections in order ({"file", and optionally "skills", "optional"}):
|
|
107
|
-
pack.toml's
|
|
94
|
+
pack.toml's. A section { pack = "../core" } stands
|
|
108
95
|
for that pack's sections (the core's page first, then the package's own), their files
|
|
109
96
|
named by absolute path."""
|
|
110
97
|
out = []
|
|
@@ -181,7 +168,7 @@ def installed_section(skills, installed=(), agent_skills=None, *, embodiment=Non
|
|
|
181
168
|
|
|
182
169
|
|
|
183
170
|
def assemble_prompt(directory: Path, skills, overrides: dict | None = None,
|
|
184
|
-
features=(
|
|
171
|
+
features=(), installed=(), agent_skills=None, *, embodiment=None,
|
|
185
172
|
own: bool = False, only=None) -> str:
|
|
186
173
|
"""The pack's sections for the enabled `skills` and connection `features`, joined.
|
|
187
174
|
`overrides` maps a section name (file stem) to another file, e.g. another
|
|
@@ -231,10 +218,11 @@ def mentions(text: str, name: str) -> bool:
|
|
|
231
218
|
|
|
232
219
|
|
|
233
220
|
# These calls are present in EpisodeCore even with no reactive skills enabled.
|
|
234
|
-
CORE_RULE_CALLS = {"set_reflex", "drop_reflex", "reflexes", "run_controller", "at_priority", "until"
|
|
221
|
+
CORE_RULE_CALLS = {"set_reflex", "drop_reflex", "reflexes", "run_controller", "at_priority", "until",
|
|
222
|
+
"expect", "after", "situations"}
|
|
235
223
|
|
|
236
224
|
|
|
237
|
-
def documentation_gaps(directory: Path, skills, exported: dict, features=(
|
|
225
|
+
def documentation_gaps(directory: Path, skills, exported: dict, features=()) -> dict:
|
|
238
226
|
"""What the assembled prompt gets wrong about the API: `missing` are functions of
|
|
239
227
|
enabled skills it never shows being called (except those the pack lists as
|
|
240
228
|
undocumented), `stray` are names of disabled skills it still mentions.
|
|
@@ -13,15 +13,16 @@ from typing import Any
|
|
|
13
13
|
|
|
14
14
|
# The kinds of an execution's event (Event.kind; a guard may raise one of its own name):
|
|
15
15
|
# nominal ends (NOMINAL), a time bound, the execution's scope, a guard, the shield, reactive
|
|
16
|
-
# code that broke its rules, a skill's error, the
|
|
16
|
+
# code that broke its rules, a skill's error, the body held back short of its target (contact
|
|
17
17
|
# or a load), the episode's end, the user's pause (it stops every unit of the model), a unit
|
|
18
18
|
# that failed, a channel another unit has, and a unit of higher priority that took the
|
|
19
19
|
# embodiment from an execution that is not to be started again.
|
|
20
20
|
BUILTIN = ("COMPLETED", "PREDICATE_SATISFIED", "TIMEOUT", "REGION_EXIT", "GUARD_TRIGGERED", "SHIELD_REJECTED",
|
|
21
21
|
"REACTIVE_FAULT", "SKILL_ERROR", "BLOCKED", "EPISODE_ENDED", "USER_PAUSED",
|
|
22
|
-
"ENTRY_FAILED", "CHANNEL_CONFLICT", "PREEMPTED", "REFLEX_FAILED")
|
|
22
|
+
"ENTRY_FAILED", "CHANNEL_CONFLICT", "PREEMPTED", "REFLEX_FAILED", "REFLEX_ACTED")
|
|
23
23
|
(COMPLETED, PREDICATE_SATISFIED, TIMEOUT, REGION_EXIT, GUARD_TRIGGERED, SHIELD_REJECTED, REACTIVE_FAULT, SKILL_ERROR,
|
|
24
|
-
BLOCKED, EPISODE_ENDED, USER_PAUSED, ENTRY_FAILED, CHANNEL_CONFLICT, PREEMPTED, REFLEX_FAILED
|
|
24
|
+
BLOCKED, EPISODE_ENDED, USER_PAUSED, ENTRY_FAILED, CHANNEL_CONFLICT, PREEMPTED, REFLEX_FAILED,
|
|
25
|
+
REFLEX_ACTED) = BUILTIN
|
|
25
26
|
# A program may continue after these without consulting a handler.
|
|
26
27
|
NOMINAL = frozenset({COMPLETED, PREDICATE_SATISFIED})
|
|
27
28
|
|
|
@@ -30,7 +31,7 @@ NOMINAL = frozenset({COMPLETED, PREDICATE_SATISFIED})
|
|
|
30
31
|
class Event:
|
|
31
32
|
"""Why an execution ended: its kind (a built-in kind above or a guard's own action),
|
|
32
33
|
the source that raised it (a skill, a guard, "shield", "environment"), details, and
|
|
33
|
-
the
|
|
34
|
+
the step it happened at."""
|
|
34
35
|
|
|
35
36
|
kind: str
|
|
36
37
|
source: str
|
|
@@ -72,7 +73,7 @@ class ExecutionTrace:
|
|
|
72
73
|
|
|
73
74
|
@property
|
|
74
75
|
def ticks(self) -> int:
|
|
75
|
-
"""
|
|
76
|
+
"""Steps the execution took."""
|
|
76
77
|
return self.end_tick - self.start_tick
|
|
77
78
|
|
|
78
79
|
def to_dict(self) -> dict:
|
|
@@ -163,6 +164,7 @@ EVENTS = {
|
|
|
163
164
|
"fact": {"source": "program", "shown": False},
|
|
164
165
|
"program_held": {"source": "harness", "shown": False},
|
|
165
166
|
"program_dropped": {"source": "harness", "shown": False},
|
|
167
|
+
"program_refused": {"source": "harness", "shown": False},
|
|
166
168
|
"user_took_over": {"source": "harness", "shown": False},
|
|
167
169
|
"stuck": {"source": "harness", "shown": False},
|
|
168
170
|
"policy_to_end": {"source": "program", "shown": False},
|
|
@@ -7,12 +7,12 @@ takes a workspace of that kind and answers with what its session needs. The kern
|
|
|
7
7
|
builds that kind's session beside its own, each with its tick (runtime/tick.py) and its
|
|
8
8
|
entries, and gives programs its calls under the kind's name:
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
KIND.NAME(...) the call NAME of the attached environment of kind KIND
|
|
11
11
|
|
|
12
12
|
The attached session never connects to its worker itself. Every call goes over the
|
|
13
13
|
kernel's connection to the host (RelayedEnvironment), which holds the worker's key, checks
|
|
14
14
|
the call as it checks the conversation's own and keeps the user's approvals: a step with
|
|
15
|
-
the real mouse and keyboard stays under the physical-input policy after the
|
|
15
|
+
the real mouse and keyboard stays under the physical-input policy after the environment is
|
|
16
16
|
attached. So the kernel's network guard is as it was when the kernel started.
|
|
17
17
|
"""
|
|
18
18
|
|
|
@@ -43,7 +43,7 @@ class RelayedEnvironment:
|
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
class Attached:
|
|
46
|
-
"""The calls of an attached environment as attributes:
|
|
46
|
+
"""The calls of an attached environment as attributes: KIND.NAME.
|
|
47
47
|
A call made after it was released, or after its episode ended, says so."""
|
|
48
48
|
|
|
49
49
|
def __init__(self, name: str, session):
|