@lmmslab/harness 0.0.0-stage → 0.81.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1356 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +232 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,871 @@
|
|
|
1
|
+
"""The episode core: the session skeleton every embodiment's kernel session is.
|
|
2
|
+
|
|
3
|
+
A kernel session connects to a worker, keeps the state it reports, the model clients of the
|
|
4
|
+
skills (VLM, pointer, verifier, segmenter) and the episode's events, and runs bounded
|
|
5
|
+
executions as program entries on its always-on tick (runtime/tick.py): at each step the
|
|
6
|
+
reactive layer's rules that stop are asked, the termination conditions are checked, and one
|
|
7
|
+
command goes to the worker, whose shield passes or rejects it. An execution ends with an
|
|
8
|
+
event; one that is not nominal goes to the rules that answer it (on()), up the tree of
|
|
9
|
+
entries (runtime/execution/entries.py), and one nobody answers escalates to the model.
|
|
10
|
+
|
|
11
|
+
An embodiment (a robot, a computer, a game) is a worker that serves the worker protocol
|
|
12
|
+
(core/worker.py WORKER_METHODS) behind a shield (runtime/safety/shield.py CommandShield), and a
|
|
13
|
+
session that subclasses EpisodeCore and says what a step's command is and how it reaches the
|
|
14
|
+
worker (_tick), the command that leaves everything as it is (_base_command), what reactive
|
|
15
|
+
code observes (_reactive_view), when a command leaves an execution's scope (_outside), what
|
|
16
|
+
a trace samples, and the guard actions it takes itself. With channels it also says how a
|
|
17
|
+
command takes them and how the tick merges the parts (_plan, _merge, _still, _admit,
|
|
18
|
+
_channels_of, _channel_of_reason). The kernel's session class is the embodiment's session
|
|
19
|
+
plus the mixin of every skill for that kind (session_class()); its namespace holds the
|
|
20
|
+
exports of the skills enabled for the run (runtime/execution/skills.py).
|
|
21
|
+
|
|
22
|
+
Bootstrap: harness.runtime.execution.kernel --bootstrap harness.runtime.execution.session:create_session
|
|
23
|
+
Options: {"endpoint", "episode_id", "embodiment", "skills", "reactive_budget_ms", "reaction_delay_ms", "observer",
|
|
24
|
+
"default_max_seconds", "vlm", "pointer", "verifier", "segmenter", "vision", "mcp",
|
|
25
|
+
"memory", "library", "agent_skills", "policies", "world_models"}, and an embodiment's
|
|
26
|
+
own (a robot's: "strategy_first", "policy_progress_seconds", harness_robot/session.py)
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import importlib
|
|
30
|
+
import math
|
|
31
|
+
import time
|
|
32
|
+
|
|
33
|
+
from harness.core import spans
|
|
34
|
+
from harness.core.remote import RemoteEnvironment
|
|
35
|
+
from harness.embodiments import KINDS
|
|
36
|
+
from harness.runtime.events import EpisodeEnded, Escalation, Event, ExecutionTrace
|
|
37
|
+
from harness.runtime.execution import control
|
|
38
|
+
from harness.runtime import events as ev
|
|
39
|
+
from harness.runtime.decision.model import USAGE
|
|
40
|
+
from harness.runtime.execution.entries import ClosedLoop, Tree
|
|
41
|
+
from harness.runtime.reactive.rules import HOLD, RAISES, STOP, ReactiveFault, ReactiveLayer, call_reactive, fault_event
|
|
42
|
+
from harness.runtime.tick import Tick, delay_ticks
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# The session of each embodiment kind, as "module:Class" (embodiments.KINDS).
|
|
46
|
+
SESSIONS = {name: kind.session for name, kind in KINDS.items()}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class EpisodeCore:
|
|
50
|
+
"""The session skeleton every embodiment and skill builds on: the worker connection and
|
|
51
|
+
profile, the state, the model clients, the reactive layer (guards, reflexes, answers to
|
|
52
|
+
events), the tree of entries and bounded executions on the tick. An embodiment's session
|
|
53
|
+
subclasses it (see the module docstring); its `namespace` holds the enabled skills'
|
|
54
|
+
exports."""
|
|
55
|
+
|
|
56
|
+
# The embodiment kind of this session (embodiments.KINDS).
|
|
57
|
+
embodiment = None
|
|
58
|
+
# Guard actions the session takes itself (when(predicate, action)); any other action
|
|
59
|
+
# is an event name.
|
|
60
|
+
guard_actions = (STOP, HOLD)
|
|
61
|
+
# Executions that change nothing: the run lets them act whatever it refuses (_refuses).
|
|
62
|
+
still_executions = ("hold",)
|
|
63
|
+
# The channels of the embodiment ({name: channel}, runtime/tick.py); none: it is one
|
|
64
|
+
# resource, and executions take turns on it.
|
|
65
|
+
channels = {}
|
|
66
|
+
hold_fields = ()
|
|
67
|
+
observation_event_kinds = frozenset()
|
|
68
|
+
# How show() encodes an array: "png" (exact pixels) or "jpeg" (camera pictures).
|
|
69
|
+
show_format = "png"
|
|
70
|
+
|
|
71
|
+
def __init__(self, options, *, environment=None):
|
|
72
|
+
self.env = environment if environment is not None else RemoteEnvironment(
|
|
73
|
+
options["endpoint"], timeout=float(options.get("timeout_seconds", 300)),
|
|
74
|
+
host_controls=options.get("host_controls", ()), relay=options.get("worker_relay"))
|
|
75
|
+
self.episode_id = options["episode_id"]
|
|
76
|
+
self.profile = self.env.call("profile")
|
|
77
|
+
self.hz = float(self.profile.get("control_hz", 1))
|
|
78
|
+
self.default_max_seconds = float(options.get("default_max_seconds", 30.0))
|
|
79
|
+
self.reactive = ReactiveLayer(float(options.get("reactive_budget_ms", 5.0)) / 1000.0, self.guard_actions)
|
|
80
|
+
# Between an observation and the action decided on it (runtime/tick.py).
|
|
81
|
+
self.reaction_delay_ms = float(options.get("reaction_delay_ms") or 0.0)
|
|
82
|
+
delay_ticks(self.reaction_delay_ms, self.hz)
|
|
83
|
+
# The world memory's observer as the run gives it (the world package's skill reads it).
|
|
84
|
+
self.observer = options.get("observer")
|
|
85
|
+
# MCP tool servers for the mcp skill: {name: {"url", "description"}}.
|
|
86
|
+
self.mcp_servers = dict(options.get("mcp") or {})
|
|
87
|
+
# Agent Skills the user installed in the app: {name: {"description", "path"}}.
|
|
88
|
+
self.agent_skills = dict(options.get("agent_skills") or {})
|
|
89
|
+
# Learned model services belong to the episode, not to a robot session.
|
|
90
|
+
self.policies = dict(options.get("policies") or {})
|
|
91
|
+
self._policy_clients = {}
|
|
92
|
+
self.world_models = dict(options.get("world_models") or {})
|
|
93
|
+
self._attachments = []
|
|
94
|
+
self._events = []
|
|
95
|
+
self._event_counts = {}
|
|
96
|
+
# Other environments taken into this conversation, by name (runtime/execution/attached.py),
|
|
97
|
+
# and the names the namespace gained or gave up since the last cell ended (the kernel
|
|
98
|
+
# keeps its API in step).
|
|
99
|
+
self._attached = {}
|
|
100
|
+
self._api_changes = {}
|
|
101
|
+
# What runs at the same time (runtime/execution/entries.py); a guard belongs to the entry that
|
|
102
|
+
# set it.
|
|
103
|
+
self._entry_tree = Tree(self)
|
|
104
|
+
self.reactive.owner = lambda: self._entries.current().id
|
|
105
|
+
# The guards a wall clock's idle tick saw firing (each is an event once: reactive.apply).
|
|
106
|
+
self._idle_fired = set()
|
|
107
|
+
# Latest views of a running controller that servos on images.
|
|
108
|
+
self._live = None
|
|
109
|
+
self.state = self.env.call("state", episode_id=self.episode_id)
|
|
110
|
+
self._embodiment_setup(options)
|
|
111
|
+
# Success verifier (App. E, Tab. 5): a separate model a program may ask.
|
|
112
|
+
self.verifier = image_role(options.get("verifier"), "verifier")
|
|
113
|
+
self.vision = image_role(options.get("vlm"), "vlm")
|
|
114
|
+
# Open-vocabulary pointing (Molmo2 in the paper); without it point_to uses the vlm.
|
|
115
|
+
self.pointer = image_role(options.get("pointer"), "pointer")
|
|
116
|
+
self.segmenter = None
|
|
117
|
+
if options.get("segmenter"):
|
|
118
|
+
self.segmenter = service_client(options["segmenter"])
|
|
119
|
+
# Generic perception for a game learned from its pictures (skills/vision).
|
|
120
|
+
self.vision_service = None
|
|
121
|
+
if options.get("vision"):
|
|
122
|
+
self.vision_service = service_client(options["vision"])
|
|
123
|
+
# The run's library and its learning settings ({"directory", "settings"}), for
|
|
124
|
+
# in-context learning (skills/learning).
|
|
125
|
+
self.library_given = dict(options.get("library") or {})
|
|
126
|
+
# The practice sandbox of an active run whose environment has one ({"what", "ops"}:
|
|
127
|
+
# the bench's [sandbox]); absent in every other run (skills/learning `sandbox`).
|
|
128
|
+
self.sandbox_given = dict(options.get("sandbox") or {})
|
|
129
|
+
# The skills this kernel exposes, as the configuration resolved them
|
|
130
|
+
# (config/agent.py resolve_for).
|
|
131
|
+
from harness.runtime.execution.skills import namespace, reporters
|
|
132
|
+
|
|
133
|
+
self.skills = list(options["skills"])
|
|
134
|
+
self.namespace = {
|
|
135
|
+
"__name__": "__program__", **namespace(self, self.skills),
|
|
136
|
+
"rule": self.rule, "when": self.reactive.when, "watch": self.reactive.watch,
|
|
137
|
+
"on": self.reactive.on, "clear_rules": self.reactive.clear,
|
|
138
|
+
"run_controller": self.run_controller,
|
|
139
|
+
# The host switches skills on between programs (sub-agents the user turned on).
|
|
140
|
+
"__load_skills__": self.load_skills,
|
|
141
|
+
# The host reads what the environment reports as text for the verifier.
|
|
142
|
+
"__evidence__": lambda: print(self._evidence()),
|
|
143
|
+
}
|
|
144
|
+
self._reporters = reporters(self.skills)
|
|
145
|
+
# The views of the environment and how the model gets them (the run's "views"):
|
|
146
|
+
# "demand", the model looks itself (look, watch); else the run sends them with
|
|
147
|
+
# every turn (a configuration that declares runtime.cameras) and nothing is added.
|
|
148
|
+
views = dict(options.get("views") or {})
|
|
149
|
+
self._views = {"cameras": list(views.get("cameras") or ()), "watch": None}
|
|
150
|
+
self._looks = []
|
|
151
|
+
if views.get("mode") == "demand":
|
|
152
|
+
guard = self.reactive.watch
|
|
153
|
+
|
|
154
|
+
def watch(*args, **kwargs):
|
|
155
|
+
"""watch(cameras=None, every="program", keep=None): the views with every
|
|
156
|
+
program's result from now on. watch(signal, low=..., high=...): a guard on
|
|
157
|
+
a number (a rule)."""
|
|
158
|
+
if len(args) > 1 or {"low", "high", "event", "name", "stops", "signal_name"} & set(kwargs) \
|
|
159
|
+
or (args and callable(args[0])):
|
|
160
|
+
return guard(*args, **kwargs)
|
|
161
|
+
return self.watch_views(*args, **kwargs)
|
|
162
|
+
|
|
163
|
+
self.namespace.update(look=self.look, watch=watch, unwatch=self.unwatch, unpin=self.unpin)
|
|
164
|
+
self._embodiment_ready(options)
|
|
165
|
+
self.reactive.wake_observer = self._ticker._wake
|
|
166
|
+
self._ticker.start()
|
|
167
|
+
|
|
168
|
+
def _evidence(self) -> str:
|
|
169
|
+
"""What this environment reports as text about where it stands, as evidence for a
|
|
170
|
+
claim beside its state and views (a window's elements, a page's text). An
|
|
171
|
+
embodiment's skills say what; none by default."""
|
|
172
|
+
return ""
|
|
173
|
+
|
|
174
|
+
def _embodiment_setup(self, options):
|
|
175
|
+
"""What the embodiment's session keeps besides the core's, set before the skills'
|
|
176
|
+
namespace is built (the profile and the first state are there)."""
|
|
177
|
+
|
|
178
|
+
def _embodiment_ready(self, options):
|
|
179
|
+
"""What the embodiment's session sets once the skills' namespace is built (it may
|
|
180
|
+
read `skills`), before the tick runs."""
|
|
181
|
+
|
|
182
|
+
def show(self, what, caption=""):
|
|
183
|
+
"""Attach an observation (each of its pictures) or an RGB array (a crop, say) to the
|
|
184
|
+
next message you receive."""
|
|
185
|
+
urls = getattr(what, "_urls", None)
|
|
186
|
+
if isinstance(urls, dict):
|
|
187
|
+
for camera, url in urls.items():
|
|
188
|
+
name = (caption or camera) if camera == "screen" else f"{caption} {camera}".strip()
|
|
189
|
+
self._attachments.append({"caption": name, "image_url": url})
|
|
190
|
+
return
|
|
191
|
+
from harness.core.codec import image_data_url
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
url = image_data_url(what, self.show_format)
|
|
195
|
+
except ValueError as error:
|
|
196
|
+
raise ValueError(f"show() takes an observation or an HxWx3 RGB array ({error})") from None
|
|
197
|
+
self._attachments.append({"caption": caption, "image_url": url})
|
|
198
|
+
|
|
199
|
+
# Views on the model's demand.
|
|
200
|
+
|
|
201
|
+
def _camera_names(self, cameras):
|
|
202
|
+
known = self._views["cameras"]
|
|
203
|
+
if not known:
|
|
204
|
+
raise RuntimeError("This environment has no view to look at: read its state and what its calls return")
|
|
205
|
+
names = list(known) if cameras is None else [cameras] if isinstance(cameras, str) else list(cameras)
|
|
206
|
+
unknown = [name for name in names if name not in known]
|
|
207
|
+
if unknown or not names:
|
|
208
|
+
raise ValueError(f"no view {unknown or names}; this environment has {', '.join(known)}")
|
|
209
|
+
return names
|
|
210
|
+
|
|
211
|
+
def look(self, camera=None, region=None, keep=None, pin=None):
|
|
212
|
+
"""Put a view of the environment in your next turn, with this program's result:
|
|
213
|
+
all its views, one by name, or the part region=(x, y, width, height) of it in the
|
|
214
|
+
view's pixels. Nothing is shown unless a program looks. A view stays for your next
|
|
215
|
+
reply only; keep=N keeps the views of the last N programs in sight, pin="name"
|
|
216
|
+
keeps this one until unpin("name"). Returns what it saw (.state, and an image
|
|
217
|
+
show() takes)."""
|
|
218
|
+
names = self._camera_names(camera)
|
|
219
|
+
data = self.env.call("observe", episode_id=self.episode_id, cameras=names)
|
|
220
|
+
images = data.get("images") or {}
|
|
221
|
+
if isinstance(data.get("state"), dict):
|
|
222
|
+
self.state = data["state"]
|
|
223
|
+
if region is not None:
|
|
224
|
+
region = [int(value) for value in region]
|
|
225
|
+
if len(region) != 4 or region[2] <= 0 or region[3] <= 0:
|
|
226
|
+
raise ValueError("region is (x, y, width, height) in the view's pixels")
|
|
227
|
+
seen = {}
|
|
228
|
+
for name in names:
|
|
229
|
+
if name not in images:
|
|
230
|
+
continue
|
|
231
|
+
url = images[name] if region is None else _cropped(images[name], region, self.show_format)
|
|
232
|
+
seen[name] = url
|
|
233
|
+
self._looks = [item for item in self._looks
|
|
234
|
+
if (item["camera"], item.get("region"), item.get("pin")) != (name, region, pin)]
|
|
235
|
+
self._looks.append({"camera": name, "image_url": url, **({"region": region} if region else {}),
|
|
236
|
+
**({"pin": str(pin)} if pin else {})})
|
|
237
|
+
if keep is not None:
|
|
238
|
+
self.keep_views(keep)
|
|
239
|
+
return Looked(self.state, seen)
|
|
240
|
+
|
|
241
|
+
def keep_views(self, keep):
|
|
242
|
+
if isinstance(keep, bool) or int(keep) < 1 or int(keep) > 20:
|
|
243
|
+
raise ValueError("keep is how many programs' views stay in sight: 1 to 20")
|
|
244
|
+
spans.note("views", action="keep", keep=int(keep))
|
|
245
|
+
|
|
246
|
+
def watch_views(self, cameras=None, every="program", keep=None):
|
|
247
|
+
"""From now on the views (all, or those named) come with every program's result,
|
|
248
|
+
until unwatch(); every=seconds: also that often while a long program runs. For
|
|
249
|
+
work that is seen, not read: a game, a moving scene."""
|
|
250
|
+
names = self._camera_names(cameras)
|
|
251
|
+
seconds = None
|
|
252
|
+
if every != "program":
|
|
253
|
+
if isinstance(every, bool) or not isinstance(every, (int, float)) or every < 2:
|
|
254
|
+
raise ValueError('every is "program" or a number of seconds, at least 2')
|
|
255
|
+
seconds = float(every)
|
|
256
|
+
if keep is not None:
|
|
257
|
+
self.keep_views(keep)
|
|
258
|
+
watch = self._views["watch"] = {"cameras": names, "every": seconds}
|
|
259
|
+
spans.note("views", action="watch", cameras=names, every=every)
|
|
260
|
+
if seconds is not None:
|
|
261
|
+
import threading
|
|
262
|
+
|
|
263
|
+
def run():
|
|
264
|
+
# While this watch stands: the views to the host, which gives them to the
|
|
265
|
+
# model while a program runs.
|
|
266
|
+
while self._views["watch"] is watch and not self.state.get("ended"):
|
|
267
|
+
time.sleep(seconds)
|
|
268
|
+
if self._views["watch"] is not watch:
|
|
269
|
+
return
|
|
270
|
+
try:
|
|
271
|
+
data = self.env.call("observe", episode_id=self.episode_id, cameras=names)
|
|
272
|
+
except Exception:
|
|
273
|
+
continue
|
|
274
|
+
spans.note("view", images={name: url for name, url in (data.get("images") or {}).items()
|
|
275
|
+
if name in names})
|
|
276
|
+
|
|
277
|
+
threading.Thread(target=run, name="watch", daemon=True).start()
|
|
278
|
+
return names
|
|
279
|
+
|
|
280
|
+
def unwatch(self):
|
|
281
|
+
"""Stop watch(): views come again only when a program looks."""
|
|
282
|
+
watching, self._views["watch"] = self._views["watch"] is not None, None
|
|
283
|
+
spans.note("views", action="unwatch")
|
|
284
|
+
return watching
|
|
285
|
+
|
|
286
|
+
def unpin(self, name=None):
|
|
287
|
+
"""Let go of the view pinned as `name` (all pinned views without one)."""
|
|
288
|
+
spans.note("views", action="unpin", **({"pin": str(name)} if name is not None else {}))
|
|
289
|
+
|
|
290
|
+
def drain(self):
|
|
291
|
+
"""What the last cell produced, for the kernel's reply: attachments and events
|
|
292
|
+
gathered since the previous drain (then cleared), the state, model usage, and the
|
|
293
|
+
enabled skills' turn report."""
|
|
294
|
+
watch = self._views["watch"]
|
|
295
|
+
if watch is not None and not self.state.get("ended"):
|
|
296
|
+
# A standing watch: the views the program did not look at itself.
|
|
297
|
+
looked = {item["camera"] for item in self._looks if not item.get("region")}
|
|
298
|
+
due = [name for name in watch["cameras"] if name not in looked]
|
|
299
|
+
if due:
|
|
300
|
+
try:
|
|
301
|
+
self.look(due)
|
|
302
|
+
except Exception as error:
|
|
303
|
+
spans.note("cleanup_error", message=f"watch: {type(error).__name__}: {error}")
|
|
304
|
+
report = self._turn_report()
|
|
305
|
+
data = {"attachments": self._attachments, "events": self._events,
|
|
306
|
+
"state": self.state, "usage": self._drain_usage()}
|
|
307
|
+
if self._event_counts:
|
|
308
|
+
data["event_counts"] = self._event_counts
|
|
309
|
+
self._event_counts = {}
|
|
310
|
+
if report is not None:
|
|
311
|
+
data["report"] = report
|
|
312
|
+
entries = self._entries.drain()
|
|
313
|
+
if entries:
|
|
314
|
+
data["entries"] = entries
|
|
315
|
+
self._attachments, self._events = [], []
|
|
316
|
+
if self._looks:
|
|
317
|
+
data["looks"], self._looks = self._looks, []
|
|
318
|
+
for name, other in list(self._attached.items()):
|
|
319
|
+
# An attached environment's part of the cell: its state and executions under
|
|
320
|
+
# its name, its pictures with the conversation's own.
|
|
321
|
+
try:
|
|
322
|
+
part = other.drain()
|
|
323
|
+
except Exception as error:
|
|
324
|
+
part = {"state": {}, "events": [], "attachments": [],
|
|
325
|
+
"error": f"{type(error).__name__}: {error}"}
|
|
326
|
+
data["attachments"] = data["attachments"] + [
|
|
327
|
+
{**item, "caption": f"{name}: {item.get('caption') or ''}".rstrip(": ")}
|
|
328
|
+
for item in part.pop("attachments", [])]
|
|
329
|
+
for key, usage in (part.pop("usage", None) or {}).items():
|
|
330
|
+
data["usage"].setdefault(key, usage)
|
|
331
|
+
if part.get("looks"):
|
|
332
|
+
data["looks"] = data.get("looks", []) + [{**item, "camera": f"{name}.{item['camera']}"}
|
|
333
|
+
for item in part.pop("looks")]
|
|
334
|
+
data.setdefault("environments", {})[name] = {
|
|
335
|
+
key: value for key, value in part.items() if key in ("state", "events", "event_counts", "error")}
|
|
336
|
+
return data
|
|
337
|
+
|
|
338
|
+
def attach(self, name, given):
|
|
339
|
+
"""Take another environment into this session under `name`: its session beside this
|
|
340
|
+
one (runtime/execution/attached.py), its calls in the namespace as name.call(...)."""
|
|
341
|
+
from harness.runtime.execution.attached import Attached, build
|
|
342
|
+
|
|
343
|
+
if name in self._attached:
|
|
344
|
+
return self.namespace[name]
|
|
345
|
+
if name in self.namespace:
|
|
346
|
+
raise ValueError(f"{name} is a name of this conversation's own; it cannot be an environment's")
|
|
347
|
+
session = build(name, given)
|
|
348
|
+
self._attached[name] = session
|
|
349
|
+
proxy = self.namespace[name] = self._api_changes[name] = Attached(name, session)
|
|
350
|
+
return proxy
|
|
351
|
+
|
|
352
|
+
def detach(self, name):
|
|
353
|
+
"""Give an attached environment back: its session closed, its name gone."""
|
|
354
|
+
session = self._attached.pop(name, None)
|
|
355
|
+
if session is None:
|
|
356
|
+
return False
|
|
357
|
+
proxy = self.namespace.pop(name, None)
|
|
358
|
+
if proxy is not None:
|
|
359
|
+
object.__setattr__(proxy, "_live", False)
|
|
360
|
+
self._api_changes[name] = None
|
|
361
|
+
try:
|
|
362
|
+
session.close()
|
|
363
|
+
except Exception as error:
|
|
364
|
+
spans.note("cleanup_error", message=f"{name}: {type(error).__name__}: {error}")
|
|
365
|
+
return True
|
|
366
|
+
|
|
367
|
+
def load_skills(self, names):
|
|
368
|
+
"""Switch skills on while the conversation runs (sub-agents the user agreed to): their
|
|
369
|
+
exports join the namespace."""
|
|
370
|
+
from harness.runtime.execution.skills import namespace, reporters
|
|
371
|
+
|
|
372
|
+
new = [name for name in names if name not in self.skills]
|
|
373
|
+
if not new:
|
|
374
|
+
return []
|
|
375
|
+
added = namespace(self, new)
|
|
376
|
+
self.skills = [*self.skills, *new]
|
|
377
|
+
self.namespace.update(added)
|
|
378
|
+
self._api_changes.update(added)
|
|
379
|
+
self._reporters = reporters(self.skills)
|
|
380
|
+
return sorted(name for name in added if not name.startswith("_"))
|
|
381
|
+
|
|
382
|
+
def _turn_report(self):
|
|
383
|
+
"""What the enabled skills add to the model's next turn after every program (each
|
|
384
|
+
skill's turn_report: text, and camera images to show in place of the plain ones),
|
|
385
|
+
joined; None when none adds anything or the episode has ended. A report that fails
|
|
386
|
+
becomes a note, never a failed program."""
|
|
387
|
+
if self.state.get("ended"):
|
|
388
|
+
return None
|
|
389
|
+
texts, images = [], {}
|
|
390
|
+
for report in getattr(self, "_reporters", ()):
|
|
391
|
+
try:
|
|
392
|
+
part = report(self)
|
|
393
|
+
except Exception as error:
|
|
394
|
+
texts.append(f"({report.__module__} could not report this turn: {type(error).__name__}: {error})")
|
|
395
|
+
continue
|
|
396
|
+
if part:
|
|
397
|
+
if part.get("text"):
|
|
398
|
+
texts.append(part["text"])
|
|
399
|
+
images.update(part.get("images") or {})
|
|
400
|
+
if not texts and not images:
|
|
401
|
+
return None
|
|
402
|
+
return {"text": "\n".join(texts), "images": images}
|
|
403
|
+
|
|
404
|
+
def close(self):
|
|
405
|
+
"""Cancel the entries, stop the tick and close the worker connection."""
|
|
406
|
+
for name in list(self.__dict__.get("_attached", {})):
|
|
407
|
+
self.detach(name)
|
|
408
|
+
self._entries.close()
|
|
409
|
+
ticker = self.__dict__.get("_tick_loop")
|
|
410
|
+
if ticker is not None:
|
|
411
|
+
ticker.stop()
|
|
412
|
+
try:
|
|
413
|
+
for name, close in self.__dict__.pop("_policy_closers", {}).items():
|
|
414
|
+
try:
|
|
415
|
+
close()
|
|
416
|
+
except Exception as error:
|
|
417
|
+
spans.note("policy_cleanup_error", policy=name, error=str(error))
|
|
418
|
+
finally:
|
|
419
|
+
self.env.close()
|
|
420
|
+
|
|
421
|
+
def _drain_usage(self):
|
|
422
|
+
"""Model usage of the perception and verifier calls since the last drain."""
|
|
423
|
+
usage = {}
|
|
424
|
+
for name, model in (("vision", self.vision), ("pointer", self.pointer), ("verifier", self.verifier)):
|
|
425
|
+
if model is not None and model.usage["calls"]:
|
|
426
|
+
usage[name] = dict(model.usage)
|
|
427
|
+
model.usage.update(calls=0, **USAGE)
|
|
428
|
+
return usage
|
|
429
|
+
|
|
430
|
+
def _vision(self):
|
|
431
|
+
if self.vision is None:
|
|
432
|
+
raise RuntimeError("No vision model is configured for this harness (the vlm role, with the "
|
|
433
|
+
"perception package)")
|
|
434
|
+
return self.vision
|
|
435
|
+
|
|
436
|
+
def _execute(self, skill, parameters, *args, **kwargs):
|
|
437
|
+
"""One bounded execution (_bounded), recorded as a span of the skill lane with the
|
|
438
|
+
event it ended with (core/spans.py)."""
|
|
439
|
+
with spans.span("skill", skill, parameters=parameters, tick=self.state.get("step")) as end:
|
|
440
|
+
try:
|
|
441
|
+
trace = self._bounded(skill, parameters, *args, **kwargs)
|
|
442
|
+
except (Escalation, EpisodeEnded) as stopped:
|
|
443
|
+
end.update(event_kind=stopped.trace.event.kind, end_tick=stopped.trace.end_tick, escalated=True,
|
|
444
|
+
**_reason(stopped.trace.event))
|
|
445
|
+
raise
|
|
446
|
+
end.update(event_kind=trace.event.kind, end_tick=trace.end_tick, **_reason(trace.event))
|
|
447
|
+
return trace
|
|
448
|
+
|
|
449
|
+
def _bounded(self, skill, parameters, command_fn, max_seconds, region, until, timeout_is_nominal=False,
|
|
450
|
+
cameras_at=None):
|
|
451
|
+
"""One bounded execution: command_fn(state, tick) gives each step's command (None
|
|
452
|
+
when the skill is done, or an Event that ends it), for at most `max_seconds`,
|
|
453
|
+
inside `region` (the embodiment's scope), until `until(view)` holds. It runs on the
|
|
454
|
+
tick as a program entry (_steps) and returns its trace once that ended."""
|
|
455
|
+
max_seconds = self.default_max_seconds if max_seconds is None else float(max_seconds)
|
|
456
|
+
steps = self._steps(skill, parameters, command_fn, max_seconds, region, until, timeout_is_nominal,
|
|
457
|
+
cameras_at)
|
|
458
|
+
result = Execution(session=self, skill=skill, parameters=parameters, steps=steps)
|
|
459
|
+
trace = result.detail.get("trace")
|
|
460
|
+
if trace is None:
|
|
461
|
+
# Not confirmed: the run does not let it act now.
|
|
462
|
+
steps.close()
|
|
463
|
+
step = self.state["step"]
|
|
464
|
+
trace = ExecutionTrace(skill, parameters, step, step, Event(
|
|
465
|
+
ev.SKILL_ERROR, skill, {"reason": result.reason, **result.detail}, step))
|
|
466
|
+
return self._finish(trace)
|
|
467
|
+
|
|
468
|
+
def _refuses(self, skill):
|
|
469
|
+
"""Why the run does not let the execution `skill` act now ({"reason", "hint"}), or
|
|
470
|
+
None (Execution.confirm). The skeleton refuses nothing; an embodiment may (a robot's
|
|
471
|
+
learned policy)."""
|
|
472
|
+
return None
|
|
473
|
+
|
|
474
|
+
def _steps(self, skill, parameters, command_fn, max_seconds, region, until, timeout_is_nominal, cameras_at):
|
|
475
|
+
"""The program entry of a bounded execution (runtime/tick.py): every step it
|
|
476
|
+
observes, runs the guards and the termination conditions, and yields the command
|
|
477
|
+
to send with the cameras whose images the reply should carry; it receives the
|
|
478
|
+
worker's reply to the step the tick sent, and returns the execution's trace."""
|
|
479
|
+
max_ticks = max(1, math.ceil(max_seconds * self.hz))
|
|
480
|
+
# The guards of the entry that runs it and of those above it watch it.
|
|
481
|
+
scope = {entry.id for entry in self._entries.current().lineage()}
|
|
482
|
+
start = self.state["step"]
|
|
483
|
+
samples = []
|
|
484
|
+
receipt = None
|
|
485
|
+
value = None
|
|
486
|
+
deadline = time.monotonic() + max_seconds
|
|
487
|
+
discrete = self.profile.get("clock") == "events"
|
|
488
|
+
tick = 0
|
|
489
|
+
event = None
|
|
490
|
+
|
|
491
|
+
def check(predicate=None):
|
|
492
|
+
state = self.state
|
|
493
|
+
if control.pause_requested():
|
|
494
|
+
return Event(ev.USER_PAUSED, "user", tick=state["step"])
|
|
495
|
+
if discrete and time.monotonic() >= deadline:
|
|
496
|
+
return Event(ev.TIMEOUT, skill, {"max_seconds": max_seconds}, state["step"])
|
|
497
|
+
view = self._reactive_view(state, tick)
|
|
498
|
+
if cameras_at is not None:
|
|
499
|
+
view["observation"] = self._live
|
|
500
|
+
try:
|
|
501
|
+
guard = self.reactive.evaluate(view, scope)
|
|
502
|
+
if guard is not None or predicate is None:
|
|
503
|
+
return guard
|
|
504
|
+
if call_reactive("until", predicate, view, budget_seconds=self.reactive.budget_seconds):
|
|
505
|
+
return Event(ev.PREDICATE_SATISFIED, "until", tick=state["step"])
|
|
506
|
+
except ReactiveFault as fault:
|
|
507
|
+
return fault
|
|
508
|
+
|
|
509
|
+
stop = None
|
|
510
|
+
self._motion_started(self.state)
|
|
511
|
+
while event is None:
|
|
512
|
+
state = self.state
|
|
513
|
+
if state.get("ended"):
|
|
514
|
+
# The worker ended the episode between steps (a wall clock's observation saw
|
|
515
|
+
# it: a practice segment's goal or time limit): nothing more is sent.
|
|
516
|
+
self._motion_stopped(state)
|
|
517
|
+
raise EpisodeEnded(ExecutionTrace(skill, parameters, start, state["step"],
|
|
518
|
+
Event(ev.EPISODE_ENDED, "environment", tick=state["step"]),
|
|
519
|
+
samples))
|
|
520
|
+
paused = control.pause_requested()
|
|
521
|
+
if paused or control.message_arrived() and self._entries.current() is self._entries.root:
|
|
522
|
+
self._motion_stopped(state)
|
|
523
|
+
event = Event(ev.USER_PAUSED if paused else ev.USER_MESSAGE, "user", tick=state["step"])
|
|
524
|
+
break
|
|
525
|
+
outcome, observed = stop or (check(until), state)
|
|
526
|
+
if outcome is not None:
|
|
527
|
+
if isinstance(outcome, Event):
|
|
528
|
+
event = outcome
|
|
529
|
+
break
|
|
530
|
+
self._motion_stopped(observed)
|
|
531
|
+
if isinstance(outcome, ReactiveFault):
|
|
532
|
+
command = self._base_command(observed)
|
|
533
|
+
else:
|
|
534
|
+
self._idle_fired.add(outcome.name)
|
|
535
|
+
self._let_go_under(outcome.owner)
|
|
536
|
+
command = self._guard_command(outcome.action, observed) if (
|
|
537
|
+
outcome.action in self.guard_actions and outcome.action != STOP) else None
|
|
538
|
+
held = (yield command, None) if command is not None else None
|
|
539
|
+
if self.state.get("ended"):
|
|
540
|
+
continue
|
|
541
|
+
event = self._rule_event(outcome, observed, held)
|
|
542
|
+
break
|
|
543
|
+
if region is not None and self._outside(region, state):
|
|
544
|
+
event = Event(ev.REGION_EXIT, skill, {"region": region}, state["step"])
|
|
545
|
+
break
|
|
546
|
+
if not discrete and tick >= max_ticks:
|
|
547
|
+
kind = ev.COMPLETED if timeout_is_nominal else ev.TIMEOUT
|
|
548
|
+
event = Event(kind, skill, {"max_seconds": max_seconds}, state["step"])
|
|
549
|
+
break
|
|
550
|
+
try:
|
|
551
|
+
target = command_fn(state, tick)
|
|
552
|
+
except ReactiveFault as fault:
|
|
553
|
+
target = self._rule_event(fault, state)
|
|
554
|
+
if target is None or isinstance(target, Event):
|
|
555
|
+
event = target if target is not None else Event(ev.COMPLETED, skill, receipt or {}, state["step"])
|
|
556
|
+
break
|
|
557
|
+
result = yield target, (cameras_at(tick) if cameras_at else None), check
|
|
558
|
+
if result is None and self.state.get("ended"):
|
|
559
|
+
continue
|
|
560
|
+
if isinstance(result, tuple):
|
|
561
|
+
stop = result
|
|
562
|
+
continue
|
|
563
|
+
if event := ev.input_refusal(result, result["state"]["step"]):
|
|
564
|
+
if "action_id" in result:
|
|
565
|
+
event.detail["action_id"] = result["action_id"]
|
|
566
|
+
break
|
|
567
|
+
if "action_id" in result:
|
|
568
|
+
receipt = {"action_id": result["action_id"]}
|
|
569
|
+
if delivery := result.get("delivery"):
|
|
570
|
+
receipt["delivery_status"] = delivery.get("status")
|
|
571
|
+
if "evidence" in result:
|
|
572
|
+
receipt["evidence"] = result["evidence"]
|
|
573
|
+
value = result.get("result")
|
|
574
|
+
self._motion_ticked(result["shield"]["command"])
|
|
575
|
+
if tick % 5 == 0:
|
|
576
|
+
samples.append(self._sample(self.state))
|
|
577
|
+
tick += 1
|
|
578
|
+
if self.state["ended"]:
|
|
579
|
+
trace = ExecutionTrace(skill, parameters, start, self.state["step"],
|
|
580
|
+
Event(ev.EPISODE_ENDED, "environment", tick=self.state["step"]),
|
|
581
|
+
samples)
|
|
582
|
+
raise EpisodeEnded(trace)
|
|
583
|
+
self._motion_stopped(self.state, arrived=event.kind == ev.COMPLETED)
|
|
584
|
+
return ExecutionTrace(skill, parameters, start, self.state["step"], event, samples, result=value)
|
|
585
|
+
|
|
586
|
+
@property
|
|
587
|
+
def _entries(self):
|
|
588
|
+
"""The session's tree of entries (runtime/execution/entries.py), made on first use."""
|
|
589
|
+
tree = self.__dict__.get("_entry_tree")
|
|
590
|
+
if tree is None:
|
|
591
|
+
tree = self.__dict__["_entry_tree"] = Tree(self)
|
|
592
|
+
return tree
|
|
593
|
+
|
|
594
|
+
@property
|
|
595
|
+
def _ticker(self):
|
|
596
|
+
"""The session's tick (runtime/tick.py), made on first use."""
|
|
597
|
+
ticker = self.__dict__.get("_tick_loop")
|
|
598
|
+
if ticker is None:
|
|
599
|
+
ticker = self.__dict__["_tick_loop"] = self._new_tick()
|
|
600
|
+
return ticker
|
|
601
|
+
|
|
602
|
+
def _new_tick(self):
|
|
603
|
+
return Tick(self)
|
|
604
|
+
|
|
605
|
+
def _finish(self, trace):
|
|
606
|
+
tree = self._entries
|
|
607
|
+
entry = tree.current()
|
|
608
|
+
handled_here = not trace.ok and trace.event.kind not in (ev.USER_PAUSED, ev.USER_MESSAGE)
|
|
609
|
+
record = tree.record_event(trace, entry if handled_here and entry is not tree.root else None)
|
|
610
|
+
if trace.ok:
|
|
611
|
+
return trace
|
|
612
|
+
if not handled_here or not tree.handle(entry, trace):
|
|
613
|
+
raise Escalation(trace)
|
|
614
|
+
record["handled"] = True
|
|
615
|
+
return trace
|
|
616
|
+
|
|
617
|
+
def _failed(self, skill, parameters, reason):
|
|
618
|
+
step = self.state["step"]
|
|
619
|
+
return ExecutionTrace(skill, parameters, step, step, Event(ev.SKILL_ERROR, skill, {"reason": reason}, step))
|
|
620
|
+
|
|
621
|
+
def rule(self, decide, name=None, priority=0):
|
|
622
|
+
"""Register an observation rule that can raise an event or stop an entry in scope."""
|
|
623
|
+
return self.reactive.rule(decide, RAISES, name, priority).name
|
|
624
|
+
|
|
625
|
+
def run_controller(self, fn, until=None, max_seconds=None):
|
|
626
|
+
"""Run `fn(view)` for each step, until it returns None or a stopping condition holds."""
|
|
627
|
+
def command(state, tick):
|
|
628
|
+
return call_reactive("controller", fn, self._reactive_view(state, tick),
|
|
629
|
+
budget_seconds=self.reactive.budget_seconds)
|
|
630
|
+
|
|
631
|
+
return self._execute("run_controller", {}, command, max_seconds, None, until)
|
|
632
|
+
|
|
633
|
+
def reflex(self, control, priority="survival", name=None):
|
|
634
|
+
"""Run `control(view)` on the tick every control period from now on, as a rule of
|
|
635
|
+
this entry that acts (runtime/reactive/rules.py): it returns None or the period's act ({"keys",
|
|
636
|
+
"buttons", "dx", "dy", "take"}) and wins the channels it needs over lower
|
|
637
|
+
priorities and over what the entries do. Its name."""
|
|
638
|
+
if not self.channels:
|
|
639
|
+
raise ValueError("reflexes run on an embodiment with channels (a game's)")
|
|
640
|
+
name = name or getattr(control, "__name__", None) or type(control).__name__
|
|
641
|
+
self.reactive.reflex(name, control, priority)
|
|
642
|
+
self._ticker.start()
|
|
643
|
+
return name
|
|
644
|
+
|
|
645
|
+
def _let_go_under(self, owner):
|
|
646
|
+
"""A guard that stops fired: with channels, the keys and buttons that the entry
|
|
647
|
+
`owner` and the entries below it hold down (key_down) go up at the next period, as
|
|
648
|
+
a stop of all they do (None: every entry's)."""
|
|
649
|
+
tick = self.__dict__.get("_tick_loop")
|
|
650
|
+
if tick is None or not tick.channels:
|
|
651
|
+
return
|
|
652
|
+
tree = self._entries
|
|
653
|
+
tick.let_go(tree.under(tree.get(owner) if owner is not None else tree.root))
|
|
654
|
+
|
|
655
|
+
def _rule_event(self, guard, state, held=None):
|
|
656
|
+
detail = {"hold_rejected": held["shield"]["reasons"]} if held is not None and not held["executed"] else {}
|
|
657
|
+
if isinstance(guard, ReactiveFault):
|
|
658
|
+
return fault_event(guard, state.get("step"), **detail)
|
|
659
|
+
detail = {"guard": guard.describe(), **detail}
|
|
660
|
+
# Counted per episode (summary guard_triggers) and tallied by practice.
|
|
661
|
+
spans.note("guard", name=guard.name, action=guard.action, stops=guard.does == "stop")
|
|
662
|
+
if guard.action in self.guard_actions:
|
|
663
|
+
return Event(ev.GUARD_TRIGGERED, guard.name, {**detail, "action": guard.action}, state["step"])
|
|
664
|
+
return Event(guard.action, guard.name, detail, state["step"])
|
|
665
|
+
|
|
666
|
+
def _guard_command(self, action, state):
|
|
667
|
+
"""The command a guard action other than stop sends: hold keeps everything as it
|
|
668
|
+
is."""
|
|
669
|
+
return self._base_command(state)
|
|
670
|
+
|
|
671
|
+
# What the embodiment's session defines.
|
|
672
|
+
|
|
673
|
+
def _base_command(self, state):
|
|
674
|
+
"""The command that leaves everything as it is."""
|
|
675
|
+
raise NotImplementedError
|
|
676
|
+
|
|
677
|
+
def _tick(self, command, cameras=None):
|
|
678
|
+
"""Send one step's command to the worker; the worker's reply {"executed",
|
|
679
|
+
"shield", "state", ...}, with self.state updated."""
|
|
680
|
+
raise NotImplementedError
|
|
681
|
+
|
|
682
|
+
def _motion_started(self, state):
|
|
683
|
+
"""An execution begins (what an embodiment holds afterwards may depend on it)."""
|
|
684
|
+
|
|
685
|
+
def _motion_ticked(self, command):
|
|
686
|
+
"""The shield passed this command of the running execution."""
|
|
687
|
+
|
|
688
|
+
def _motion_stopped(self, state, arrived=False):
|
|
689
|
+
"""The running execution was interrupted (a guard, a fault) or ended; `arrived`
|
|
690
|
+
when it completed."""
|
|
691
|
+
|
|
692
|
+
def _take_observation_events(self, state):
|
|
693
|
+
"""An embodiment's new observation events, offered once to existing handlers."""
|
|
694
|
+
return ()
|
|
695
|
+
|
|
696
|
+
def _reactive_view(self, state, tick):
|
|
697
|
+
"""What guards, controllers and until predicates observe at this step."""
|
|
698
|
+
raise NotImplementedError
|
|
699
|
+
|
|
700
|
+
# With channels (runtime/tick.py).
|
|
701
|
+
|
|
702
|
+
def _plan(self, command):
|
|
703
|
+
"""A command taken apart: (the channels it drives, its periods' parts, a change of
|
|
704
|
+
what its entry holds down or None); raises tick.Refused for one it cannot take."""
|
|
705
|
+
raise NotImplementedError
|
|
706
|
+
|
|
707
|
+
def _merge(self, parts, idle):
|
|
708
|
+
"""One period's command from the parts of the executions and holds, with the
|
|
709
|
+
channels of `idle` at their default."""
|
|
710
|
+
raise NotImplementedError
|
|
711
|
+
|
|
712
|
+
def _still(self, last):
|
|
713
|
+
"""The command that keeps what the last sent command (None: none yet) held."""
|
|
714
|
+
raise NotImplementedError
|
|
715
|
+
|
|
716
|
+
def _admit(self, command, commit=False):
|
|
717
|
+
"""The shield's verdict on a merged command, judged here before it is sent;
|
|
718
|
+
`commit` once the worker executed it."""
|
|
719
|
+
raise NotImplementedError
|
|
720
|
+
|
|
721
|
+
def _channels_of(self, part) -> set:
|
|
722
|
+
"""The channels a part uses."""
|
|
723
|
+
raise NotImplementedError
|
|
724
|
+
|
|
725
|
+
def _channel_of_reason(self, reason):
|
|
726
|
+
"""The channel a shield's reason is about, when it names an input and not the
|
|
727
|
+
channel."""
|
|
728
|
+
return None
|
|
729
|
+
|
|
730
|
+
def _observe(self):
|
|
731
|
+
"""Refresh the state at the start of a tick of a wall-clock embodiment (between
|
|
732
|
+
two ticks the world moved on); a steps clock's last reply is the state."""
|
|
733
|
+
self.state = self.env.call("state", episode_id=self.episode_id)
|
|
734
|
+
|
|
735
|
+
def _outside(self, region, state):
|
|
736
|
+
"""Whether the state has left an execution's scope."""
|
|
737
|
+
raise NotImplementedError
|
|
738
|
+
|
|
739
|
+
def _sample(self, state):
|
|
740
|
+
"""What a trace keeps of a step (every fifth)."""
|
|
741
|
+
return {"step": state["step"]}
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
class Looked:
|
|
745
|
+
"""What look() saw: .state, .cameras, and the pictures show() takes (a crop of one is
|
|
746
|
+
look(region=...))."""
|
|
747
|
+
|
|
748
|
+
def __init__(self, state, urls):
|
|
749
|
+
self.state, self._urls = state, dict(urls)
|
|
750
|
+
self.cameras = list(urls)
|
|
751
|
+
|
|
752
|
+
def __repr__(self):
|
|
753
|
+
return f"Looked({', '.join(self.cameras) or 'nothing'}): it is in your next turn"
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def _cropped(url, region, fmt):
|
|
757
|
+
"""The part `region` (x, y, width, height) of the image at data URL `url`, as a data URL."""
|
|
758
|
+
from harness.core.codec import decode_data_url, image_data_url
|
|
759
|
+
|
|
760
|
+
try:
|
|
761
|
+
import numpy as np
|
|
762
|
+
except ImportError:
|
|
763
|
+
raise RuntimeError("look(region=...) needs NumPy in this kernel; look without region, or read the "
|
|
764
|
+
"structure (elements(), read_page())") from None
|
|
765
|
+
raw = decode_data_url(url)
|
|
766
|
+
try:
|
|
767
|
+
import cv2
|
|
768
|
+
|
|
769
|
+
image = cv2.cvtColor(cv2.imdecode(np.frombuffer(raw, dtype=np.uint8), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)
|
|
770
|
+
except ImportError:
|
|
771
|
+
import io
|
|
772
|
+
|
|
773
|
+
from PIL import Image
|
|
774
|
+
|
|
775
|
+
image = np.asarray(Image.open(io.BytesIO(raw)).convert("RGB"))
|
|
776
|
+
x, y, width, height = region
|
|
777
|
+
part = image[max(y, 0):y + height, max(x, 0):x + width]
|
|
778
|
+
if part.size == 0:
|
|
779
|
+
raise ValueError(f"region {region} is outside the view ({image.shape[1]} x {image.shape[0]})")
|
|
780
|
+
return image_data_url(part, fmt)
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
class Execution(ClosedLoop):
|
|
784
|
+
"""A bounded execution as a closed loop: confirm the run lets it act (_refuses), do its
|
|
785
|
+
steps on the tick, check it ended nominally. One try (a motion that may have run is never
|
|
786
|
+
silently retried); every exception ends the program; its span is its record. The trace
|
|
787
|
+
is the Result's detail["trace"] (none when not confirmed: then the reason and hint)."""
|
|
788
|
+
|
|
789
|
+
tries = 1
|
|
790
|
+
max_seconds = math.inf
|
|
791
|
+
catches = ()
|
|
792
|
+
|
|
793
|
+
def confirm(self):
|
|
794
|
+
refused = self.session._refuses(self.skill)
|
|
795
|
+
if refused is None:
|
|
796
|
+
return True
|
|
797
|
+
self.detail["hint"] = refused.get("hint")
|
|
798
|
+
return refused["reason"]
|
|
799
|
+
|
|
800
|
+
def do(self):
|
|
801
|
+
self.detail["trace"] = self.session._ticker.run(self.steps)
|
|
802
|
+
|
|
803
|
+
def check(self):
|
|
804
|
+
trace = self.detail["trace"]
|
|
805
|
+
return True if trace.ok else f"{trace.event.kind} from {trace.event.source}"
|
|
806
|
+
|
|
807
|
+
def report(self, number, what, ok, seconds, reason):
|
|
808
|
+
pass
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
_SESSION_CLASSES = {}
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def _import(spec: str):
|
|
815
|
+
module, _, attribute = spec.partition(":")
|
|
816
|
+
return getattr(importlib.import_module(module), attribute)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _reason(event) -> dict:
|
|
820
|
+
"""The reason an execution's event gives, for its span (practice tallies failures by it)."""
|
|
821
|
+
reason = (event.detail or {}).get("reason") if isinstance(event.detail, dict) else None
|
|
822
|
+
return {"reason": str(reason)[:120]} if reason and event.kind not in ev.NOMINAL else {}
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def image_role(config: dict | None, role: str):
|
|
826
|
+
"""A model role that answers about images (vlm, pointer, verifier): an instance of the
|
|
827
|
+
class its config names ([class] "module:Class", a package's: config/agent.py
|
|
828
|
+
image_roles), or None without a config or a class."""
|
|
829
|
+
module, _, name = str((config or {}).get("class") or "").partition(":")
|
|
830
|
+
if not module or not name:
|
|
831
|
+
return None
|
|
832
|
+
return getattr(importlib.import_module(module), name)(
|
|
833
|
+
{key: value for key, value in config.items() if key != "class"}, role=role)
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def service_client(config: dict):
|
|
837
|
+
"""The client of a service the episode's programs use (a segmenter, a vision service): the
|
|
838
|
+
class its manifest names ([client] class, "module:Class"), given its url, timeout and
|
|
839
|
+
on-demand start."""
|
|
840
|
+
import importlib
|
|
841
|
+
|
|
842
|
+
module, _, name = str(config.get("class") or "").partition(":")
|
|
843
|
+
if not module or not name:
|
|
844
|
+
raise ValueError(f"the service at {config.get('url')} names no client class ([client] class)")
|
|
845
|
+
return getattr(importlib.import_module(module), name)(
|
|
846
|
+
config["url"], timeout=float(config.get("timeout_seconds", 120.0)), on_demand=config.get("on_demand"))
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def session_class(embodiment: str = "robot", knowledge: str | None = None):
|
|
850
|
+
"""The session of an embodiment kind with the mixin of every skill for that kind (the
|
|
851
|
+
skills enabled for a run only decide the namespace; a skill's methods may use
|
|
852
|
+
another's); with knowledge "learned" (a game the agent learns itself), without the
|
|
853
|
+
skills written for a game by hand, whose modules are then never imported."""
|
|
854
|
+
knowledge = knowledge or KINDS[embodiment].knowledge or "handwritten"
|
|
855
|
+
key = (embodiment, knowledge)
|
|
856
|
+
if key not in _SESSION_CLASSES:
|
|
857
|
+
from harness.runtime.execution.skills import mixins
|
|
858
|
+
|
|
859
|
+
if embodiment not in SESSIONS:
|
|
860
|
+
raise ValueError(f"No session for the embodiment kind {embodiment!r}; sessions exist for "
|
|
861
|
+
f"{sorted(SESSIONS)}")
|
|
862
|
+
base = _import(SESSIONS[embodiment])
|
|
863
|
+
_SESSION_CLASSES[key] = type("Session", (*mixins(embodiment=embodiment, knowledge=knowledge), base),
|
|
864
|
+
{"__module__": base.__module__, "embodiment": embodiment,
|
|
865
|
+
"knowledge": knowledge})
|
|
866
|
+
return _SESSION_CLASSES[key]
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
def create_session(options):
|
|
870
|
+
"""Kernel bootstrap: the session of options["embodiment"] (default robot)."""
|
|
871
|
+
return session_class(options.get("embodiment", "robot"), options.get("knowledge"))(options)
|