@lmmslab/harness 0.85.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/python/harness/__init__.py +2 -2
- package/python/harness/bench.py +2 -2
- package/python/harness/builtin/benches/README.md +1 -2
- package/python/harness/builtin/configs/default.toml +4 -19
- package/python/harness/builtin/configs/deploy/example-api.toml +1 -1
- package/python/harness/builtin/configs/models/deepseek-flash.toml +1 -1
- package/python/harness/builtin/configs/models/grok-low.toml +1 -1
- package/python/harness/builtin/configs/models/grok-medium.toml +3 -6
- package/python/harness/builtin/environment/README.md +8 -25
- package/python/harness/builtin/prompts/conversation/core.md +6 -5
- package/python/harness/builtin/prompts/core/core.md +147 -203
- package/python/harness/builtin/prompts/core/pack.toml +0 -1
- package/python/harness/builtin/prompts/reflect/core.md +8 -32
- package/python/harness/builtin/prompts/reflect/pack.toml +2 -4
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -3
- package/python/harness/builtin/prompts/shared/agents.md +20 -10
- package/python/harness/builtin/prompts/shared/checklist.md +13 -17
- package/python/harness/builtin/prompts/shared/environments.md +20 -15
- package/python/harness/builtin/prompts/shared/learning-goal.md +2 -3
- package/python/harness/builtin/prompts/shared/learning.md +73 -23
- package/python/harness/builtin/prompts/shared/mcp.md +11 -6
- package/python/harness/builtin/prompts/shared/reflex-method.md +243 -0
- package/python/harness/builtin/prompts/shared/units.md +138 -0
- package/python/harness/builtin/prompts/shared/web.md +5 -6
- package/python/harness/builtin/skills/agents/__init__.py +18 -3
- package/python/harness/builtin/skills/agents/skill.toml +3 -3
- package/python/harness/builtin/skills/checklist/__init__.py +34 -32
- package/python/harness/builtin/skills/checklist/skill.toml +9 -9
- package/python/harness/builtin/skills/environments/__init__.py +9 -5
- package/python/harness/builtin/skills/environments/skill.toml +1 -1
- package/python/harness/builtin/skills/experience/skill.toml +1 -1
- package/python/harness/builtin/skills/learning/__init__.py +19 -5
- package/python/harness/builtin/skills/mcp/skill.toml +1 -1
- package/python/harness/builtin/skills/memory/__init__.py +4 -1
- package/python/harness/builtin/skills/memory/skill.toml +0 -5
- package/python/harness/builtin/skills/verify/skill.toml +1 -0
- package/python/harness/builtin/skills/web/__init__.py +4 -1
- package/python/harness/builtin/third_party/sources.toml +2 -40
- package/python/harness/cli.py +16 -25
- package/python/harness/config/__init__.py +2 -2
- package/python/harness/config/agent.py +59 -47
- package/python/harness/core/__init__.py +1 -1
- package/python/harness/core/codec.py +4 -4
- package/python/harness/core/contracts.py +1 -1
- package/python/harness/core/media.py +1 -1
- package/python/harness/core/ports.py +2 -2
- package/python/harness/core/procs.py +1 -1
- package/python/harness/core/remote.py +1 -1
- package/python/harness/core/worker.py +19 -13
- package/python/harness/embodiments.py +53 -44
- package/python/harness/extensions.py +22 -0
- package/python/harness/host/__init__.py +1 -1
- package/python/harness/host/http.py +6 -12
- package/python/harness/host/sessions.py +1 -1
- package/python/harness/host/tasks.py +15 -11
- package/python/harness/host/workspaces.py +36 -8
- package/python/harness/packages/__init__.py +5 -5
- package/python/harness/packages/environments.py +30 -18
- package/python/harness/packages/manager.py +1 -1
- package/python/harness/packages/manifest.py +6 -6
- package/python/harness/packages/provision.py +1 -1
- package/python/harness/packages/sources.py +1 -1
- package/python/harness/paths.py +1 -1
- package/python/harness/runtime/decision/loop.py +95 -5
- package/python/harness/runtime/decision/messages.py +128 -2
- package/python/harness/runtime/decision/model.py +4 -5
- package/python/harness/runtime/decision/prompts.py +20 -32
- package/python/harness/runtime/events.py +7 -5
- package/python/harness/runtime/execution/attached.py +3 -3
- package/python/harness/runtime/execution/entries.py +189 -8
- package/python/harness/runtime/execution/kernel.py +26 -1
- package/python/harness/runtime/execution/results.py +2 -3
- package/python/harness/runtime/execution/session.py +417 -32
- package/python/harness/runtime/execution/skills.py +63 -16
- package/python/harness/runtime/learning/active.py +7 -7
- package/python/harness/runtime/learning/learner.py +3 -2
- package/python/harness/runtime/learning/library.py +9 -4
- package/python/harness/runtime/learning/reflect.py +10 -9
- package/python/harness/runtime/reactive/rules.py +4 -4
- package/python/harness/runtime/reactive/tree.py +506 -25
- package/python/harness/runtime/safety/input.py +1 -1
- package/python/harness/runtime/safety/isolation.py +3 -3
- package/python/harness/runtime/safety/shield.py +4 -50
- package/python/harness/runtime/task/delegation.py +5 -5
- package/python/harness/runtime/task/environments.py +10 -0
- package/python/harness/runtime/task/run.py +222 -27
- package/python/harness/runtime/tick.py +24 -1
- package/python/harness/sdk/__init__.py +2 -5
- package/python/harness/services/registry.py +23 -19
- package/python/harness/settings.py +9 -9
- package/python/harness/builtin/environment/constraints.txt +0 -31
- package/python/harness/builtin/prompts/shared/world.md +0 -1
|
@@ -12,18 +12,17 @@ an experiment's `model`, or `--model NAME`):
|
|
|
12
12
|
backend = "openrouter"
|
|
13
13
|
model = "x-ai/grok-4.7"
|
|
14
14
|
[vlm] questions about images (and pointing without a pointer)
|
|
15
|
-
service = "
|
|
15
|
+
service = "NAME" a role may be a service of the deploy ...
|
|
16
16
|
timeout_seconds = 300 ... with its own settings on top
|
|
17
17
|
[pointer]
|
|
18
|
-
service = "
|
|
18
|
+
service = "NAME"
|
|
19
19
|
[verifier] overrides on top of main (the success verifier)
|
|
20
20
|
[runtime] settings the model needs (context window, ...)
|
|
21
21
|
|
|
22
22
|
Resolving the agent against a deploy gives the conversation's main model, runtime and
|
|
23
23
|
prompt, and a lock: everything that decides what a run computes and nothing about
|
|
24
24
|
where it runs (URLs, GPUs, ports). The lock records the agent's VERSION, and its hash
|
|
25
|
-
is the configuration's identity; a result
|
|
26
|
-
name one.
|
|
25
|
+
is the configuration's identity; a result names one.
|
|
27
26
|
"""
|
|
28
27
|
|
|
29
28
|
from dataclasses import dataclass, field
|
|
@@ -50,18 +49,18 @@ ROLES = ("main", "vlm", "pointer", "verifier")
|
|
|
50
49
|
SERVICE_KEYS = ("segmenter", "vision", "policies", "world_models", "mcp")
|
|
51
50
|
# What one entry of each list of services is, in messages.
|
|
52
51
|
SERVICE_ENTRY = {"policies": "policy", "world_models": "world model", "mcp": "MCP server"}
|
|
53
|
-
# What
|
|
54
|
-
KNOWLEDGE_DOC = ("what the agent knows of the
|
|
52
|
+
# What a kind's `knowledge` says (runtime/execution/skills.py KNOWLEDGE).
|
|
53
|
+
KNOWLEDGE_DOC = ("what the agent knows of the environment: \"learned\" (the default: no skill written for it by hand loads, "
|
|
55
54
|
"nor is imported; what it knows is its library's, which learns with runtime.library learn = \"on\", "
|
|
56
|
-
"runtime/learning/learner.py) or \"handwritten\" (a reference baseline's skills written for
|
|
55
|
+
"runtime/learning/learner.py) or \"handwritten\" (a reference baseline's skills written for it by hand "
|
|
57
56
|
"may load)")
|
|
58
57
|
# The keys of configs/default.toml (and of an agent file that includes it).
|
|
59
58
|
AGENT_KEYS = {
|
|
60
59
|
"description": "one line, shown by harness config and the run viewer",
|
|
61
60
|
"prompt": "the prompt: a pack directory (assembled for the enabled skills) or files, in order; or a table "
|
|
62
|
-
"of them by embodiment kind
|
|
61
|
+
"of them by embodiment kind; without one, the prompt of the kind's package",
|
|
63
62
|
"mode": "program (default: the model writes programs) or direct (one action per call)",
|
|
64
|
-
"actions": "direct mode: the action vocabulary, \"module:function\" (text,
|
|
63
|
+
"actions": "direct mode: the action vocabulary, \"module:function\" (text, skills, allow_skills) -> the "
|
|
65
64
|
"program of one <action>; a ValueError names what is wrong with it",
|
|
66
65
|
"roles": "the model roles the agent uses, among main, vlm, pointer, verifier (default all)",
|
|
67
66
|
"skills": "layers = { learned, reactive, handlers }, or names = [...]; enable = [...] adds optional skills, "
|
|
@@ -71,14 +70,14 @@ AGENT_KEYS = {
|
|
|
71
70
|
"model": "the default model roles: a model file's name (configs/models/<name>.toml), or the roles inline",
|
|
72
71
|
"runtime": "runtime settings: the [runtime] keys",
|
|
73
72
|
"budget": "max_model_calls per episode (default 30)",
|
|
74
|
-
# A table for each embodiment kind
|
|
73
|
+
# A table for each embodiment kind, over what the kind's package says (Kind.agent).
|
|
75
74
|
**{name: f"the agent on {kind.what} (benches of embodiment kind {name}): roles, skills, services, runtime and "
|
|
76
75
|
"budget keys merged over the ones above (runtime also over the model file's)"
|
|
77
|
-
+ (f"; knowledge: {KNOWLEDGE_DOC}" if
|
|
78
|
-
for name, kind in KINDS.items()
|
|
76
|
+
+ (f"; knowledge: {KNOWLEDGE_DOC}" if kind.knowledge is not None else "")
|
|
77
|
+
for name, kind in KINDS.items()},
|
|
79
78
|
}
|
|
80
|
-
# The keys of a kind's table (
|
|
81
|
-
# only a
|
|
79
|
+
# The keys of a kind's table (in the agent's configuration, and in a package's kinds.toml
|
|
80
|
+
# under `agent`); `knowledge` only for a kind that declares it.
|
|
82
81
|
KIND_KEYS = ("roles", "skills", "services", "runtime", "budget", "knowledge")
|
|
83
82
|
# The model roles of a model file (configs/models/<name>.toml), besides `runtime`
|
|
84
83
|
# (settings the model needs) and `description`.
|
|
@@ -97,8 +96,8 @@ RUNTIME_KEYS = {
|
|
|
97
96
|
"opening message and after every program, as an evaluation's observation contract hands them to a "
|
|
98
97
|
"policy. Without it the model gets text every turn and looks on its own demand (look, watch)",
|
|
99
98
|
"cell_timeout_seconds": "optional deadline for a program, bounded by the episode deadline",
|
|
100
|
-
"contract": "the observation contract (embodiments.contract_filter): native (default),
|
|
101
|
-
"
|
|
99
|
+
"contract": "the observation contract (embodiments.contract_filter): native (default), or one a "
|
|
100
|
+
"package registers for the kind (contracts.toml)",
|
|
102
101
|
"default_max_seconds": "max_seconds of a bounded execution that names none",
|
|
103
102
|
"fixed_skills": "direct mode: locate, pick and place in the action vocabulary",
|
|
104
103
|
"isolation": "evaluation: run programs isolated, reaching only the installation, the source and the "
|
|
@@ -111,7 +110,7 @@ RUNTIME_KEYS = {
|
|
|
111
110
|
"searches, and optionally the plugin's engine (exa, native, ...). None: search() says there is none",
|
|
112
111
|
"viewer": "written by the session host: the model role that describes a view for a main model that reads no "
|
|
113
112
|
"images (the setting model.vision); none: such a model is told the view was not sent",
|
|
114
|
-
"knowledge": "a
|
|
113
|
+
"knowledge": "what the agent knows of the environment, for a kind that declares it: \"learned\" (default: use the agent's own library) or \"handwritten\" "
|
|
115
114
|
"(allow reference knowledge). A worker that declares this setting must report the same mode.",
|
|
116
115
|
"label": "a name the run viewer shows for the configuration",
|
|
117
116
|
"layers": "written by the resolution from [skills] layers",
|
|
@@ -135,7 +134,7 @@ RUNTIME_KEYS = {
|
|
|
135
134
|
"0: never)",
|
|
136
135
|
"mode": "written by the resolution from the agent's mode",
|
|
137
136
|
"actions": "written by the resolution: direct mode's action vocabulary (the agent's actions)",
|
|
138
|
-
"physical": "the physical-input policy of this run over the user's settings (
|
|
137
|
+
"physical": "the physical-input policy of this run over the user's settings (a package's "
|
|
139
138
|
"harness_computer/physical.py): {level, rules}; harness run EXPERIMENT sets it from --physical, "
|
|
140
139
|
"--allow-app, --deny-app, --ask-app",
|
|
141
140
|
"privacy": "what the model learns of the user's windows in this run, over the user's settings (the computer "
|
|
@@ -149,7 +148,7 @@ RUNTIME_KEYS = {
|
|
|
149
148
|
"practice sandbox its kernel gets; a task run never has one",
|
|
150
149
|
"reaction_delay_ms": "time between an observation and the action decided on it, rounded up to whole control "
|
|
151
150
|
"periods (runtime/tick.py): 0 (the default) for development and practice; 150-200 for a "
|
|
152
|
-
"person's reaction time, as in a
|
|
151
|
+
"person's reaction time, as in a recorded demo",
|
|
153
152
|
"observer": "explicit world observation: max_side (default 768), reasoning_effort (none), "
|
|
154
153
|
"max_output_tokens (1200). Legacy timer settings are accepted for old profiles.",
|
|
155
154
|
"segmenter": "written by the resolution: the segmenter's client config",
|
|
@@ -165,16 +164,20 @@ RUNTIME_KEYS = {
|
|
|
165
164
|
"turn_grace_seconds": "how long a program may hold the model's turn (default 2): one that is over by then "
|
|
166
165
|
"gives the turn back with its result, a longer one goes on as a unit and the turn comes "
|
|
167
166
|
"back with its handle",
|
|
168
|
-
"longest_wait_seconds": "
|
|
167
|
+
"longest_wait_seconds": "what a <wait> of the model without a time lasts; a wait with a time lasts that long "
|
|
169
168
|
"(default 60)",
|
|
169
|
+
"reflex_method": "whether the model's nodes of the tree of reflexes follow the method: required (the default: "
|
|
170
|
+
"the page on how a tree is made is read before the first node, every node does something, and "
|
|
171
|
+
"a node whose condition cannot be judged on the state as it is, is refused) or optional",
|
|
172
|
+
"purpose": "whether a program has to say what it is for: required (the default: a program that calls "
|
|
173
|
+
"neither until(condition) nor until(None) is not started, and the model is told) or optional",
|
|
170
174
|
"tell_budget": "whether the model is told with every call what the run has left (its model calls, its "
|
|
171
175
|
"environment steps, its live time): never (default: the host keeps the limits and ends the "
|
|
172
176
|
"turn at them) or always (a benchmark with short episodes, where the model plans its end)",
|
|
173
177
|
"thinking": "how long a call of the main model may think (runtime/conversation.py THINKING): model (the "
|
|
174
178
|
"code's default: the model's own settings), none (reasoning off, replies of at most 2000 tokens, "
|
|
175
179
|
"and the model is told to think briefly), low (reasoning effort at most low, 2000 tokens, told "
|
|
176
|
-
"likewise), medium (at most medium, 8000 tokens) or high; a model set lower keeps its own
|
|
177
|
-
"agent thinks low on games",
|
|
180
|
+
"likewise), medium (at most medium, 8000 tokens) or high; a model set lower keeps its own",
|
|
178
181
|
"max_wait_seconds": "accepted for older profiles. Use an explicit timeout to bound a program wait",
|
|
179
182
|
"strict_skills": "a configuration built in code, without skills: fail instead of dropping a skill (resolve_for)",
|
|
180
183
|
"task_descriptions": "a directory of task descriptions (<task>.md) added to the first message, or false",
|
|
@@ -203,7 +206,7 @@ def _relative(value):
|
|
|
203
206
|
from ..paths import packages
|
|
204
207
|
|
|
205
208
|
for package in packages():
|
|
206
|
-
# By whole path parts:
|
|
209
|
+
# By whole path parts: a package whose name begins with another's is not inside it.
|
|
207
210
|
if Path(value).is_relative_to(package.path):
|
|
208
211
|
return f"packages/{package.name}/{Path(value).relative_to(package.path).as_posix()}"
|
|
209
212
|
if isinstance(value, list):
|
|
@@ -239,7 +242,7 @@ class ResolvedAgent:
|
|
|
239
242
|
# placement of the services it uses: services/placement.py automatic()).
|
|
240
243
|
deploy: object = None
|
|
241
244
|
# The embodiment kind its skills and prompt were chosen for (embodiments.KINDS).
|
|
242
|
-
embodiment: str = "
|
|
245
|
+
embodiment: str = "none"
|
|
243
246
|
|
|
244
247
|
|
|
245
248
|
@dataclass
|
|
@@ -270,7 +273,7 @@ class TaskConfig:
|
|
|
270
273
|
agent_id: str | None = None
|
|
271
274
|
placement: dict | None = None
|
|
272
275
|
# The embodiment kind the worker drives (embodiments.KINDS).
|
|
273
|
-
embodiment: str = "
|
|
276
|
+
embodiment: str = "none"
|
|
274
277
|
|
|
275
278
|
def __post_init__(self):
|
|
276
279
|
if not self.base_prompt:
|
|
@@ -278,8 +281,8 @@ class TaskConfig:
|
|
|
278
281
|
|
|
279
282
|
def snapshot(self) -> dict:
|
|
280
283
|
"""The configuration as episode manifests and experiment.json record it (all but
|
|
281
|
-
`path` and `base_prompt`; the lock as `agent_lock`; the embodiment kind
|
|
282
|
-
not
|
|
284
|
+
`path` and `base_prompt`; the lock as `agent_lock`; the embodiment kind, unless the
|
|
285
|
+
kind says its locks do not name it). The frozen evaluator keeps its schema-2 agent lock."""
|
|
283
286
|
runtime = self.runtime
|
|
284
287
|
if runtime.get("mcp"):
|
|
285
288
|
runtime = {**runtime, "mcp": {name: {key: value for key, value in server.items()
|
|
@@ -292,7 +295,7 @@ class TaskConfig:
|
|
|
292
295
|
"max_wall_seconds": self.max_wall_seconds, "task_description": self.task_description,
|
|
293
296
|
"max_cost_usd": self.max_cost_usd, "agent_id": self.agent_id, "agent_lock": self.lock,
|
|
294
297
|
"placement": self.placement,
|
|
295
|
-
**({
|
|
298
|
+
**({} if _unnamed(self.embodiment) else {"embodiment": self.embodiment})}
|
|
296
299
|
|
|
297
300
|
|
|
298
301
|
def _role(spec, deploy, role: str, strict: bool):
|
|
@@ -360,7 +363,13 @@ def image_roles(services: dict) -> dict:
|
|
|
360
363
|
for role in ("vlm", "pointer", "verifier") if services.get(role)}
|
|
361
364
|
|
|
362
365
|
|
|
363
|
-
def
|
|
366
|
+
def _unnamed(embodiment) -> bool:
|
|
367
|
+
"""Whether locks and snapshots of an agent on this kind leave the kind out (Kind.unnamed_in_lock)."""
|
|
368
|
+
kind = KINDS.get(embodiment)
|
|
369
|
+
return bool(kind is not None and kind.unnamed_in_lock)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def resolve_for(runtime: dict, roles: dict, embodiment: str = "none") -> Resolution:
|
|
364
373
|
"""The skills a run of this runtime configuration enables (a conversation's, the
|
|
365
374
|
eval package's): `runtime.skills` as resolve_agent resolved them; a configuration
|
|
366
375
|
built in code without them gets those of its interface layers (runtime.layers over
|
|
@@ -393,7 +402,7 @@ def config_keys() -> dict:
|
|
|
393
402
|
|
|
394
403
|
def agent_file(value) -> Path:
|
|
395
404
|
"""An agent file: a path, or the name of a profile (configs/profiles/<name>.toml of the
|
|
396
|
-
core or a package
|
|
405
|
+
core or a package)."""
|
|
397
406
|
from ..paths import find, locate
|
|
398
407
|
|
|
399
408
|
path = locate(Path(value))
|
|
@@ -430,7 +439,7 @@ def load_model(source) -> dict:
|
|
|
430
439
|
|
|
431
440
|
|
|
432
441
|
def resolve_agent(agent=None, *, model=None, deploy=None, strict: bool = False, name: str | None = None,
|
|
433
|
-
runtime_overrides: dict | None = None, embodiment: str = "
|
|
442
|
+
runtime_overrides: dict | None = None, embodiment: str = "none",
|
|
434
443
|
channels: bool = False, package_paths=(), frozen_identity: bool = False) -> ResolvedAgent:
|
|
435
444
|
"""Resolve the agent (a path or a merged tree; default configs/default.toml) with
|
|
436
445
|
model roles from `model` (a model name, path or tree; default the agent's own
|
|
@@ -451,12 +460,12 @@ def resolve_agent(agent=None, *, model=None, deploy=None, strict: bool = False,
|
|
|
451
460
|
deploy = deploy if deploy is not None and not isinstance(deploy, (str, Path)) else load_deploy(deploy)
|
|
452
461
|
tree = dict(agent)
|
|
453
462
|
label = name or (path.stem if path else None)
|
|
454
|
-
# A kind's table changes the agent on that kind's benches; the top level
|
|
455
|
-
# Its runtime keys win over the model file's too (what
|
|
456
|
-
# depend on the model).
|
|
457
|
-
kinds = {kind: tree.pop(kind) for kind in KINDS if kind
|
|
458
|
-
# The table of a kind no package here registers (
|
|
459
|
-
#
|
|
463
|
+
# A kind's table changes the agent on that kind's benches; the top level holds for
|
|
464
|
+
# every kind. Its runtime keys win over the model file's too (what the agent sees of an
|
|
465
|
+
# environment does not depend on the model).
|
|
466
|
+
kinds = {kind: tree.pop(kind) for kind in KINDS if kind in tree}
|
|
467
|
+
# The table of a kind no package here registers (a configuration written where that
|
|
468
|
+
# package was installed) is left out: no bench of that kind runs here.
|
|
460
469
|
for key in [key for key, value in tree.items() if key not in AGENT_KEYS and isinstance(value, dict) and value
|
|
461
470
|
and set(value) <= set(KIND_KEYS)]:
|
|
462
471
|
tree.pop(key)
|
|
@@ -469,7 +478,8 @@ def resolve_agent(agent=None, *, model=None, deploy=None, strict: bool = False,
|
|
|
469
478
|
raise ValueError(f"Agent {path or label}: [{kind}] takes {KIND_KEYS}, not {sorted(unknown)}")
|
|
470
479
|
if "knowledge" in section and (KINDS[kind].knowledge is None or section["knowledge"] not in KNOWLEDGE):
|
|
471
480
|
raise ValueError(f"Agent {path or label}: knowledge is declared by its kind, one of {list(KNOWLEDGE)}")
|
|
472
|
-
|
|
481
|
+
# What the kind's package says of the agent on it, under the agent's own table for the kind.
|
|
482
|
+
kind_section = deep_merge(dict(KINDS[embodiment].agent), kinds.get(embodiment, {}))
|
|
473
483
|
kind_runtime = kind_section.pop("runtime", {})
|
|
474
484
|
default_knowledge = KINDS[embodiment].knowledge
|
|
475
485
|
knowledge = kind_section.pop("knowledge", default_knowledge)
|
|
@@ -558,7 +568,7 @@ def resolve_agent(agent=None, *, model=None, deploy=None, strict: bool = False,
|
|
|
558
568
|
+ list(skills.get("enable", [])))
|
|
559
569
|
requested = [skill for skill in dict.fromkeys(requested) if skill not in set(skills.get("disable", []))]
|
|
560
570
|
if knowledge == "learned":
|
|
561
|
-
# The agent learns the
|
|
571
|
+
# The agent learns the environment itself: none of the skills written for it by hand, and
|
|
562
572
|
# its own skill library instead.
|
|
563
573
|
named = sorted(set(handwritten()) & (set(skills.get("names") or ()) | set(skills.get("enable", ()))))
|
|
564
574
|
if named:
|
|
@@ -593,10 +603,12 @@ def resolve_agent(agent=None, *, model=None, deploy=None, strict: bool = False,
|
|
|
593
603
|
sources = tree.get("prompt", [])
|
|
594
604
|
if isinstance(sources, dict):
|
|
595
605
|
# Keyed by kind; a kind no package here registers names a prompt that is not used here.
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
606
|
+
sources = sources.get(embodiment, [])
|
|
607
|
+
if not sources:
|
|
608
|
+
# The kind's own prompt (its package's kinds.toml), or the page every agent reads.
|
|
609
|
+
from harness.paths import ROOT
|
|
610
|
+
|
|
611
|
+
sources = KINDS[embodiment].prompt or str(ROOT / "prompts" / "core")
|
|
600
612
|
sources = [sources] if isinstance(sources, str) else list(sources)
|
|
601
613
|
texts = []
|
|
602
614
|
for source in sources:
|
|
@@ -618,7 +630,7 @@ def resolve_agent(agent=None, *, model=None, deploy=None, strict: bool = False,
|
|
|
618
630
|
experience = table["experience"]
|
|
619
631
|
if experience not in modes:
|
|
620
632
|
raise ValueError("runtime.library.experience was one of read, learn, off; use runtime.library.uses")
|
|
621
|
-
if embodiment
|
|
633
|
+
if not KINDS[embodiment].library_experience:
|
|
622
634
|
table.pop("experience")
|
|
623
635
|
table.setdefault("uses", modes[experience])
|
|
624
636
|
from harness.runtime.learning.usage import USES
|
|
@@ -674,7 +686,7 @@ def resolve_agent(agent=None, *, model=None, deploy=None, strict: bool = False,
|
|
|
674
686
|
packages=package_lock([path, *package_paths, *sources, *(deploy.services[name].path for name in
|
|
675
687
|
_service_names(roles_tree, services) if name in deploy.services), *(
|
|
676
688
|
skill.path for name, skill in _skills().items() if name in runtime["skills"])]))
|
|
677
|
-
if embodiment
|
|
689
|
+
if not _unnamed(embodiment):
|
|
678
690
|
lock["embodiment"] = embodiment
|
|
679
691
|
return ResolvedAgent(name=label, path=path, main=main, verifier=verifier, runtime=runtime,
|
|
680
692
|
system_prompt=system_prompt, base_prompt=base_prompt, max_model_calls=calls, lock=lock,
|
|
@@ -739,7 +751,7 @@ def package_content(root: Path) -> str:
|
|
|
739
751
|
|
|
740
752
|
|
|
741
753
|
def overridden_agent(overrides: dict | None = None, *, agent=None, model=None, deploy=None, strict: bool = False,
|
|
742
|
-
embodiment: str = "
|
|
754
|
+
embodiment: str = "none", channels: bool = False, package_paths=(), frozen_identity: bool = False) -> ResolvedAgent:
|
|
743
755
|
"""The agent file (default configs/default.toml) with `overrides` ({"agent": ...,
|
|
744
756
|
"model": ...}, an experiment's or a conversation's changes) applied and its model roles
|
|
745
757
|
from `model` (default the agent's), resolved against `deploy` (resolve_agent)."""
|
|
@@ -756,7 +768,7 @@ def overridden_agent(overrides: dict | None = None, *, agent=None, model=None, d
|
|
|
756
768
|
|
|
757
769
|
|
|
758
770
|
def task_description(runtime: dict, task: str) -> str:
|
|
759
|
-
"""The task's description (
|
|
771
|
+
"""The task's description (a bench's own recipe for it) when the agent uses them."""
|
|
760
772
|
directory = runtime.get("task_descriptions")
|
|
761
773
|
if not directory:
|
|
762
774
|
return ""
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""Building blocks shared by the runtime, the workers and the tools: the worker
|
|
2
|
-
protocol (contracts,
|
|
2
|
+
protocol (contracts, client, worker server), traces, media, codecs, polling waits
|
|
3
3
|
and the HTTP helpers for model providers, the hashes records carry and the atomic file
|
|
4
4
|
write. Standard library only."""
|
|
5
5
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Codecs shared by benchmark workers: episode cursors, outcome tokens, action bounds, data
|
|
2
2
|
URLs.
|
|
3
3
|
|
|
4
|
-
Importable without
|
|
4
|
+
Importable without an environment's code, torch or a GPU (image_data_url imports NumPy, and
|
|
5
5
|
OpenCV or Pillow, when it is called).
|
|
6
6
|
"""
|
|
7
7
|
|
|
@@ -13,7 +13,7 @@ import zlib
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class EpisodeCursor:
|
|
16
|
-
"""Reject a repeated motion without guessing whether the
|
|
16
|
+
"""Reject a repeated motion without guessing whether the environment applied it."""
|
|
17
17
|
|
|
18
18
|
def __init__(self):
|
|
19
19
|
self.episode_id = None
|
|
@@ -102,8 +102,8 @@ def data_url(payload, mime="image/png"):
|
|
|
102
102
|
|
|
103
103
|
def image_data_url(array, fmt="jpeg", quality=95):
|
|
104
104
|
"""An HxWx3 RGB (or HxW grayscale) array as a data URL of a `fmt` (jpeg or png) image,
|
|
105
|
-
JPEG at `quality`: OpenCV encodes it, or Pillow where OpenCV is not installed (
|
|
106
|
-
environment
|
|
105
|
+
JPEG at `quality`: OpenCV encodes it, or Pillow where OpenCV is not installed (an
|
|
106
|
+
environment may have Pillow and NumPy only)."""
|
|
107
107
|
import numpy as np
|
|
108
108
|
|
|
109
109
|
array = np.asarray(array)
|
|
@@ -10,7 +10,7 @@ from .codec import decode_data_url
|
|
|
10
10
|
|
|
11
11
|
_VIDEO_PREFIX = "data:video/mp4;base64,"
|
|
12
12
|
# A first frame's time past this (2001 on) is a wall-clock time: a recording made with
|
|
13
|
-
# -use_wallclock_as_timestamps and -copyts (
|
|
13
|
+
# -use_wallclock_as_timestamps and -copyts (a screen recording).
|
|
14
14
|
_WALL_CLOCK = 1e9
|
|
15
15
|
# probe()'s start_from for a start read from the frames' own (wall-clock) times.
|
|
16
16
|
FRAMES = "its frames' times"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Ports on this machine that a process may listen on now: the session host's, a suite's
|
|
2
|
-
workers',
|
|
3
|
-
take each other's
|
|
2
|
+
workers', a package's workspaces', a service's. No bench hard-codes one, so they never
|
|
3
|
+
take each other's. Standard
|
|
4
4
|
library only."""
|
|
5
5
|
|
|
6
6
|
import os
|
|
@@ -28,7 +28,7 @@ def service_environment(environment=None, log=None) -> dict:
|
|
|
28
28
|
"""Copy the environment for a service or installer, dropping inherited library injection.
|
|
29
29
|
|
|
30
30
|
Keep library search paths and let the caller apply explicit service settings on top.
|
|
31
|
-
Report only the removed names. Worker
|
|
31
|
+
Report only the removed names. Worker launchers keep their run environment.
|
|
32
32
|
"""
|
|
33
33
|
result = dict(os.environ if environment is None else environment)
|
|
34
34
|
removed = [name for name in _LIBRARY_INJECTION if name in result]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""JSON/HTTP transport to a worker in a separate Python environment.
|
|
2
2
|
|
|
3
|
-
This is the framework's protocol,
|
|
3
|
+
This is the framework's protocol, not a benchmark's own.
|
|
4
4
|
An integration must implement Environment on the worker side.
|
|
5
5
|
"""
|
|
6
6
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""A serial worker keeps all
|
|
1
|
+
"""A serial worker keeps all of an environment's operations on the serving thread.
|
|
2
2
|
|
|
3
3
|
A worker is an embodiment's process: it serves the worker protocol (WORKER_METHODS) behind
|
|
4
4
|
its shield (runtime/safety/shield.py CommandShield), which judges every command before it moves
|
|
@@ -20,8 +20,8 @@ from .ports import OwnPort
|
|
|
20
20
|
# control rate, how it is driven (max_hz, step_seconds: runtime/tick.py driving), channels, cameras and shield),
|
|
21
21
|
# its state now, one control period's
|
|
22
22
|
# command, judged by its shield (tick), the views of its cameras or screen (observe), and
|
|
23
|
-
# the evaluator's outcome (claim_outcome, outcome). An embodiment adds its own methods
|
|
24
|
-
#
|
|
23
|
+
# the evaluator's outcome (claim_outcome, outcome). An embodiment adds its own methods
|
|
24
|
+
# (what only its kind has): `methods = (*WORKER_METHODS, ...)`.
|
|
25
25
|
#
|
|
26
26
|
# Senses (optional). A worker whose environment can run code where it is says so in its
|
|
27
27
|
# profile, "senses": True, and serves three more methods: sense(episode_id, sense, code)
|
|
@@ -43,6 +43,13 @@ from .ports import OwnPort
|
|
|
43
43
|
# resume(episode_id, held) is given that value and lets it go on. The run then pauses a
|
|
44
44
|
# call for a unit of higher priority instead of ending it (runtime/tick.py).
|
|
45
45
|
#
|
|
46
|
+
# Situation (optional). A worker may name, in its profile, the part of its state that says
|
|
47
|
+
# where the agent is or what it is in: "situation": a key of the state (the application in
|
|
48
|
+
# front, the folder, the room, the level). The first time that part takes a value it has not
|
|
49
|
+
# had in this run, the model is told so, with what it was and how many nodes the tree of
|
|
50
|
+
# reflexes has (runtime/task/run.py), as Claude Code tells of a working directory that
|
|
51
|
+
# changed.
|
|
52
|
+
#
|
|
46
53
|
# Priority (optional). Code that runs in the environment for long (a routine the agent saved
|
|
47
54
|
# there) may hold the environment for one action of it, as a program does with
|
|
48
55
|
# `with at_priority(n):`. The worker says so with its state: "priority", the priority that
|
|
@@ -90,8 +97,8 @@ class BenchWorker(Tells):
|
|
|
90
97
|
check of the views observe takes, the envelope of a tick, and the outcome behind a
|
|
91
98
|
token claimed once per episode (OutcomeToken).
|
|
92
99
|
|
|
93
|
-
A worker sets `methods` (its calls), LABEL (its name in messages), INPUT (
|
|
94
|
-
"
|
|
100
|
+
A worker sets `methods` (its calls), LABEL (its name in messages), INPUT (what its commands are called in messages,
|
|
101
|
+
"input" by default) and VIEWS (the views it has); keeps `_cursor` (EpisodeCursor);
|
|
95
102
|
and defines `_state()`, `_ended()` and `_evaluate()` (the outcome, asked once). Its
|
|
96
103
|
reset calls `new_outcome()`."""
|
|
97
104
|
|
|
@@ -130,7 +137,7 @@ class BenchWorker(Tells):
|
|
|
130
137
|
return views
|
|
131
138
|
|
|
132
139
|
def _call_observe(self, episode_id, cameras=None):
|
|
133
|
-
"""
|
|
140
|
+
"""The worker's view: its picture and its state (a worker with cameras of its own overrides it)."""
|
|
134
141
|
self._check_episode(episode_id)
|
|
135
142
|
self.views(cameras)
|
|
136
143
|
# The picture first: on a computer with several screens it picks the one the state names.
|
|
@@ -343,18 +350,17 @@ def parser():
|
|
|
343
350
|
"""The worker's arguments (python -m harness.core.worker, harness serve)."""
|
|
344
351
|
import argparse
|
|
345
352
|
|
|
346
|
-
parser = argparse.ArgumentParser(prog="harness serve", description="Serve a worker (
|
|
353
|
+
parser = argparse.ArgumentParser(prog="harness serve", description="Serve a worker (an environment) in "
|
|
347
354
|
"its own Python process")
|
|
348
355
|
parser.add_argument("--factory", required=True,
|
|
349
|
-
help="module:callable, e.g. benches.
|
|
356
|
+
help="module:callable, e.g. benches.NAME.runtime:create_environment")
|
|
350
357
|
parser.add_argument("--host", default="127.0.0.1")
|
|
351
358
|
parser.add_argument("--port", type=int, default=8765)
|
|
352
359
|
parser.add_argument("--options", help="TOML keyword arguments passed to the factory")
|
|
353
360
|
parser.add_argument("--contract", default="native",
|
|
354
361
|
help="what clients may see: native (everything, the default) or a contract a package "
|
|
355
362
|
"registers for the embodiment's kind (contracts.toml; harness.embodiments."
|
|
356
|
-
"contract_filter)
|
|
357
|
-
"screenshot")
|
|
363
|
+
"contract_filter)")
|
|
358
364
|
parser.add_argument("--embodiment", help="embodiment name (embodiments/<name>/embodiment.toml) for the contract")
|
|
359
365
|
parser.add_argument("--exit-with-stdin", action="store_true",
|
|
360
366
|
help="end once standard input closes: the process that started it (the app's workspace "
|
|
@@ -364,9 +370,9 @@ def parser():
|
|
|
364
370
|
|
|
365
371
|
|
|
366
372
|
def main(argv=None) -> int:
|
|
367
|
-
"""python -m harness.core.worker --factory module:callable: serve a worker (
|
|
368
|
-
|
|
369
|
-
|
|
373
|
+
"""python -m harness.core.worker --factory module:callable: serve a worker (an
|
|
374
|
+
environment) in its own Python process; what a suite and a package's workspaces start,
|
|
375
|
+
and the eval package's harness serve."""
|
|
370
376
|
import importlib
|
|
371
377
|
import os
|
|
372
378
|
import stat
|