@lmmslab/harness 0.0.0-stage → 0.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1308 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +221 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Building blocks shared by the runtime, the workers and the tools: the worker
|
|
2
|
+
protocol (contracts, robot client, worker server), traces, media, codecs, polling waits
|
|
3
|
+
and the HTTP helpers for model providers, the hashes records carry and the atomic file
|
|
4
|
+
write. Standard library only."""
|
|
5
|
+
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
import tempfile
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def canonical(value) -> str:
|
|
14
|
+
"""`value` as compact JSON with sorted keys (other objects as their str), so equal
|
|
15
|
+
trees give the same text."""
|
|
16
|
+
return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def sha256(value) -> str:
|
|
20
|
+
"""The hex sha256 of `value`'s canonical JSON."""
|
|
21
|
+
return hashlib.sha256(canonical(value).encode()).hexdigest()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def tree_hash(folder, paths) -> str:
|
|
25
|
+
"""A hash of files in `folder` (sorted `paths`): their names relative to it and their
|
|
26
|
+
contents."""
|
|
27
|
+
digest = hashlib.sha256()
|
|
28
|
+
for path in paths:
|
|
29
|
+
digest.update(Path(path).relative_to(folder).as_posix().encode() + b"\0")
|
|
30
|
+
digest.update(hashlib.sha256(Path(path).read_bytes()).digest())
|
|
31
|
+
return digest.hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def atomic_write(path, data, *, private=False) -> None:
|
|
35
|
+
"""Write `data` (text as UTF-8, or bytes) to `path` at once: a temporary file in the
|
|
36
|
+
same folder, then os.replace, so a reader finds the old file or the new one, never a
|
|
37
|
+
part, and writers side by side each keep a whole file. The file is readable by the
|
|
38
|
+
user only when `private` (it holds keys or tokens)."""
|
|
39
|
+
path = Path(path)
|
|
40
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
handle, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
|
42
|
+
try:
|
|
43
|
+
with os.fdopen(handle, "wb") as out:
|
|
44
|
+
out.write(data.encode("utf-8") if isinstance(data, str) else data)
|
|
45
|
+
try:
|
|
46
|
+
os.chmod(temporary, 0o600 if private else 0o644)
|
|
47
|
+
except OSError:
|
|
48
|
+
pass
|
|
49
|
+
os.replace(temporary, path)
|
|
50
|
+
except BaseException:
|
|
51
|
+
Path(temporary).unlink(missing_ok=True)
|
|
52
|
+
raise
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Run evidence: immutable payloads and a small append-only index, independent of cell replies."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Evidence:
|
|
11
|
+
"""Bounded disk evidence. Repeated payloads share bytes, never mutable target paths.
|
|
12
|
+
|
|
13
|
+
A record's reference can be resolved after the controller exits. Exhaustion is an
|
|
14
|
+
explicit error; callers must not retry an action whose result could not be recorded.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, directory, limit=256 * 1024 * 1024):
|
|
18
|
+
self.directory = Path(directory)
|
|
19
|
+
self.directory.mkdir(parents=True, exist_ok=True)
|
|
20
|
+
self.limit = limit
|
|
21
|
+
self.lock = threading.RLock()
|
|
22
|
+
self.used = sum(p.stat().st_size for p in self.directory.iterdir() if p.is_file())
|
|
23
|
+
|
|
24
|
+
def reserve(self, size):
|
|
25
|
+
with self.lock:
|
|
26
|
+
if self.used + size > self.limit:
|
|
27
|
+
raise OSError("run evidence limit reached; execution must stop")
|
|
28
|
+
self.used += size
|
|
29
|
+
|
|
30
|
+
def record(self, kind, payload, **identity):
|
|
31
|
+
raw = json.dumps(payload, ensure_ascii=False, allow_nan=False).encode("utf-8")
|
|
32
|
+
digest = hashlib.sha256(raw).hexdigest()
|
|
33
|
+
filename = digest + ".json"
|
|
34
|
+
row = (json.dumps({"kind": kind, "time": time.time(), **identity,
|
|
35
|
+
"payload": filename}, ensure_ascii=False) + "\n").encode("utf-8")
|
|
36
|
+
with self.lock:
|
|
37
|
+
path = self.directory / filename
|
|
38
|
+
needed = len(row) + (0 if path.exists() else len(raw))
|
|
39
|
+
self.reserve(needed)
|
|
40
|
+
if not path.exists():
|
|
41
|
+
path.write_bytes(raw)
|
|
42
|
+
with (self.directory / "index.jsonl").open("ab") as stream:
|
|
43
|
+
offset = stream.tell()
|
|
44
|
+
stream.write(row)
|
|
45
|
+
stream.flush()
|
|
46
|
+
return {"path": str(path), "sha256": digest, "index_offset": offset}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Codecs shared by benchmark workers: episode cursors, outcome tokens, action bounds, data
|
|
2
|
+
URLs.
|
|
3
|
+
|
|
4
|
+
Importable without a simulator, torch or a GPU (image_data_url imports NumPy, and
|
|
5
|
+
OpenCV or Pillow, when it is called).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import base64
|
|
9
|
+
import math
|
|
10
|
+
import secrets
|
|
11
|
+
import struct
|
|
12
|
+
import zlib
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class EpisodeCursor:
|
|
16
|
+
"""Reject a repeated motion without guessing whether the simulator applied it."""
|
|
17
|
+
|
|
18
|
+
def __init__(self):
|
|
19
|
+
self.episode_id = None
|
|
20
|
+
self.step = None
|
|
21
|
+
|
|
22
|
+
def start(self, episode_id: str, step: int = 0) -> None:
|
|
23
|
+
self.episode_id = episode_id
|
|
24
|
+
self.step = step
|
|
25
|
+
|
|
26
|
+
def check(self, episode_id: str, step: int) -> None:
|
|
27
|
+
if self.episode_id is None or (episode_id, step) != (self.episode_id, self.step):
|
|
28
|
+
raise ValueError("Stale or missing observation; action was not executed")
|
|
29
|
+
|
|
30
|
+
def advance(self) -> int:
|
|
31
|
+
self.step += 1
|
|
32
|
+
return self.step
|
|
33
|
+
|
|
34
|
+
def invalidate(self) -> None:
|
|
35
|
+
self.episode_id = None
|
|
36
|
+
self.step = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class OutcomeToken:
|
|
40
|
+
"""The token that reads an episode's outcome (claim_outcome, outcome): one per episode,
|
|
41
|
+
claimed once by the harness after the loop, never by a program. A second claim, an
|
|
42
|
+
outcome asked before the claim and a wrong token raise PermissionError; the token is
|
|
43
|
+
compared in constant time. A new episode gets a new OutcomeToken."""
|
|
44
|
+
|
|
45
|
+
def __init__(self):
|
|
46
|
+
self._token = None
|
|
47
|
+
|
|
48
|
+
def claim(self) -> str:
|
|
49
|
+
if self._token is not None:
|
|
50
|
+
raise PermissionError("The outcome token for this episode was already claimed")
|
|
51
|
+
self._token = secrets.token_hex(16)
|
|
52
|
+
return self._token
|
|
53
|
+
|
|
54
|
+
def check(self, token) -> None:
|
|
55
|
+
if self._token is None or not isinstance(token, str) or not secrets.compare_digest(token, self._token):
|
|
56
|
+
raise PermissionError("Invalid outcome token")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def finite_bounds(low, high):
|
|
61
|
+
bounds = []
|
|
62
|
+
for start, stop in zip(low, high, strict=True):
|
|
63
|
+
lower, upper = float(start), float(stop)
|
|
64
|
+
if not math.isfinite(lower) or not math.isfinite(upper) or lower > upper:
|
|
65
|
+
raise ValueError("Action bounds must be finite and ordered")
|
|
66
|
+
bounds.append((lower, upper))
|
|
67
|
+
return bounds
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def validate_vector(values, bounds, label="position"):
|
|
72
|
+
"""A list of finite values within its per-component bounds."""
|
|
73
|
+
if not isinstance(values, list) or len(values) != len(bounds):
|
|
74
|
+
raise ValueError(f"{label} must contain {len(bounds)} finite numbers")
|
|
75
|
+
command = []
|
|
76
|
+
for value, (lower, upper) in zip(values, bounds, strict=True):
|
|
77
|
+
if type(value) not in (int, float) or not math.isfinite(value):
|
|
78
|
+
raise ValueError(f"{label} values must be finite numbers")
|
|
79
|
+
number = float(value)
|
|
80
|
+
if number < lower or number > upper:
|
|
81
|
+
raise ValueError(f"{label} value {number} is outside [{lower}, {upper}]")
|
|
82
|
+
command.append(number)
|
|
83
|
+
return command
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def fit_into(inner, outer):
|
|
88
|
+
"""Where a picture of size `inner` (width, height) sits in one of size `outer`: (left,
|
|
89
|
+
top, width, height), as large as it fits, centred (a monitor of another shape in a
|
|
90
|
+
screenshot, a recorded screen in a replay's frame)."""
|
|
91
|
+
scale = min(outer[0] / inner[0], outer[1] / inner[1])
|
|
92
|
+
width, height = max(1, round(inner[0] * scale)), max(1, round(inner[1] * scale))
|
|
93
|
+
return (outer[0] - width) // 2, (outer[1] - height) // 2, width, height
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def data_url(payload, mime="image/png"):
|
|
97
|
+
if not payload:
|
|
98
|
+
raise ValueError("Image payload is empty")
|
|
99
|
+
encoded = base64.b64encode(payload).decode("ascii")
|
|
100
|
+
return f"data:{mime};base64,{encoded}"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def image_data_url(array, fmt="jpeg", quality=95):
|
|
104
|
+
"""An HxWx3 RGB (or HxW grayscale) array as a data URL of a `fmt` (jpeg or png) image,
|
|
105
|
+
JPEG at `quality`: OpenCV encodes it, or Pillow where OpenCV is not installed (a game's
|
|
106
|
+
environment has Pillow and NumPy only)."""
|
|
107
|
+
import numpy as np
|
|
108
|
+
|
|
109
|
+
array = np.asarray(array)
|
|
110
|
+
if not (array.ndim == 2 or (array.ndim == 3 and array.shape[-1] == 3)):
|
|
111
|
+
raise ValueError("images must be HxWx3 RGB (or HxW grayscale) arrays")
|
|
112
|
+
fmt = {"jpg": "jpeg"}.get(fmt.lower(), fmt.lower())
|
|
113
|
+
if fmt not in ("jpeg", "png"):
|
|
114
|
+
raise ValueError(f"image format {fmt!r}: jpeg or png")
|
|
115
|
+
array = array.astype(np.uint8)
|
|
116
|
+
try:
|
|
117
|
+
import cv2
|
|
118
|
+
except ImportError:
|
|
119
|
+
import io
|
|
120
|
+
|
|
121
|
+
from PIL import Image
|
|
122
|
+
|
|
123
|
+
buffer = io.BytesIO()
|
|
124
|
+
Image.fromarray(array, "RGB" if array.ndim == 3 else "L").save(
|
|
125
|
+
buffer, format=fmt.upper(), **({"quality": quality} if fmt == "jpeg" else {}))
|
|
126
|
+
return data_url(buffer.getvalue(), f"image/{fmt}")
|
|
127
|
+
ok, encoded = cv2.imencode(".jpg" if fmt == "jpeg" else ".png",
|
|
128
|
+
cv2.cvtColor(array, cv2.COLOR_RGB2BGR) if array.ndim == 3 else array,
|
|
129
|
+
[cv2.IMWRITE_JPEG_QUALITY, int(quality)] if fmt == "jpeg" else [])
|
|
130
|
+
if not ok:
|
|
131
|
+
raise ValueError("Could not encode the image")
|
|
132
|
+
return data_url(encoded.tobytes(), f"image/{fmt}")
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def decode_data_url(url) -> bytes:
|
|
136
|
+
"""The bytes a base64 data URL (data:<type>;base64,<payload>) carries."""
|
|
137
|
+
header, comma, payload = str(url).partition(",")
|
|
138
|
+
if not (header.startswith("data:") and header.endswith(";base64") and comma):
|
|
139
|
+
raise ValueError("not a base64 data URL")
|
|
140
|
+
return base64.b64decode(payload, validate=True)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def encode_png(width, height, rgb):
|
|
144
|
+
"""Encode tightly packed RGB bytes as a PNG."""
|
|
145
|
+
if width < 1 or height < 1:
|
|
146
|
+
raise ValueError("Image dimensions must be positive")
|
|
147
|
+
expected = width * height * 3
|
|
148
|
+
if len(rgb) != expected:
|
|
149
|
+
raise ValueError("RGB byte count does not match the image size")
|
|
150
|
+
raw = b"".join(
|
|
151
|
+
b"\x00" + rgb[row * width * 3:(row + 1) * width * 3]
|
|
152
|
+
for row in range(height)
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def chunk(tag, data):
|
|
156
|
+
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
|
157
|
+
|
|
158
|
+
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
|
159
|
+
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
|
|
160
|
+
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""JSON-serializable contracts shared by agents and simulator workers."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import asdict, dataclass, field
|
|
4
|
+
from typing import Any, Protocol
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class Observation:
|
|
9
|
+
episode_id: str
|
|
10
|
+
step: int
|
|
11
|
+
instruction: str
|
|
12
|
+
state: dict[str, Any]
|
|
13
|
+
# Camera name -> image data URL. This keeps remote traces self-contained.
|
|
14
|
+
images: dict[str, str] = field(default_factory=dict)
|
|
15
|
+
|
|
16
|
+
def to_dict(self) -> dict:
|
|
17
|
+
return asdict(self)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class Action:
|
|
22
|
+
name: str
|
|
23
|
+
arguments: dict[str, Any]
|
|
24
|
+
|
|
25
|
+
def to_dict(self) -> dict:
|
|
26
|
+
return asdict(self)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Transition:
|
|
31
|
+
observation: Observation
|
|
32
|
+
reward: float = 0.0
|
|
33
|
+
success: bool = False
|
|
34
|
+
terminated: bool = False
|
|
35
|
+
truncated: bool = False
|
|
36
|
+
info: dict[str, Any] = field(default_factory=dict)
|
|
37
|
+
# Camera name -> MP4 data URL of the motion produced by this action.
|
|
38
|
+
# Still images stay on the observation. Video is not sent to the model.
|
|
39
|
+
videos: dict[str, str] = field(default_factory=dict)
|
|
40
|
+
|
|
41
|
+
def to_dict(self) -> dict:
|
|
42
|
+
return asdict(self)
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def from_dict(cls, data: dict) -> "Transition":
|
|
46
|
+
data = dict(data)
|
|
47
|
+
data["observation"] = Observation(**data["observation"])
|
|
48
|
+
return cls(**data)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class Decision:
|
|
53
|
+
action: Action
|
|
54
|
+
usage: dict[str, Any] = field(default_factory=dict)
|
|
55
|
+
response_id: str | None = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Environment(Protocol):
|
|
59
|
+
"""A tick worker. Only actor-visible observations belong here.
|
|
60
|
+
|
|
61
|
+
Its tick call checks the episode and step before acting. Legacy action-tool
|
|
62
|
+
workers instead implement execute(action, episode_id, step) -> Transition.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
name: str
|
|
66
|
+
methods: tuple[str, ...]
|
|
67
|
+
|
|
68
|
+
def tools(self) -> list[dict]: ...
|
|
69
|
+
def reset(self, task: str, seed: int) -> Observation: ...
|
|
70
|
+
def call(self, name: str, arguments: dict) -> dict: ...
|
|
71
|
+
def close(self) -> None: ...
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Agent(Protocol):
|
|
75
|
+
def decide(self, request: dict) -> Decision: ...
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""The episodes under a runs directory, as the GUI lists and reads them.
|
|
2
|
+
|
|
3
|
+
An episode is a directory with an events.jsonl (runtime/conversation.py); its
|
|
4
|
+
manifest.json and summary.json say what ran and how it ended. An episode without a
|
|
5
|
+
summary is still being written (or was cut off).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
import secrets
|
|
12
|
+
import time
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Files of an episode the GUI may serve, by extension.
|
|
16
|
+
MEDIA_TYPES = {".mp4": "video/mp4", ".webm": "video/webm", ".png": "image/png", ".jpg": "image/jpeg",
|
|
17
|
+
".jpeg": "image/jpeg", ".json": "application/json", ".log": "text/plain; charset=utf-8",
|
|
18
|
+
".txt": "text/plain; charset=utf-8"}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def new_name(suffix: str = "") -> str:
|
|
22
|
+
"""A new run's directory name: "20261102-140530-3fa9c1", then "-<suffix>" when given."""
|
|
23
|
+
return f"{time.strftime('%Y%m%d-%H%M%S')}-{secrets.token_hex(3)}" + (f"-{suffix}" if suffix else "")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# How an episode ended (finished, closed, budget, step_limit, interrupted, error), and its words, where not the same.
|
|
27
|
+
ENDINGS = {"success": "finished", "budget_exhausted": "budget", "cost_exhausted": "budget",
|
|
28
|
+
"wall_time_exhausted": "budget", "episode_ended": "step_limit"}
|
|
29
|
+
ENDING_WORDS = {"closed": "closed by the user", "budget": "out of budget", "step_limit": "step limit"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def summary(directory) -> dict:
|
|
33
|
+
"""An episode's summary.json, {} when it has none or it cannot be read."""
|
|
34
|
+
value = _read_json(Path(directory) / "summary.json")
|
|
35
|
+
return value if isinstance(value, dict) else {}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def usage(source) -> dict | None:
|
|
39
|
+
"""{model_calls, cost_usd} of a summary or of a directory's (None when it has none)."""
|
|
40
|
+
found = source if isinstance(source, dict) else summary(source) or None
|
|
41
|
+
return None if found is None else {"model_calls": int(found.get("model_calls") or 0),
|
|
42
|
+
"cost_usd": round(float(found.get("cost_usd") or 0), 6)}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def ending(found: dict | None) -> str | None:
|
|
46
|
+
"""How an episode ended (its summary's status, as ENDINGS names it); None without one."""
|
|
47
|
+
status = (found or {}).get("status")
|
|
48
|
+
return ENDINGS.get(str(status), str(status)) if status else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _read_json(path: Path):
|
|
52
|
+
try:
|
|
53
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
54
|
+
except (OSError, ValueError):
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def episode_dir(root: Path, run: str) -> Path:
|
|
59
|
+
"""The episode directory `run` (a path relative to `root`); ValueError when it leaves
|
|
60
|
+
the root or has no trace."""
|
|
61
|
+
directory = (root / run).resolve()
|
|
62
|
+
if directory != root and root not in directory.parents:
|
|
63
|
+
raise ValueError("run outside the runs directory")
|
|
64
|
+
if not (directory / "events.jsonl").is_file():
|
|
65
|
+
raise ValueError(f"no episode at {run}")
|
|
66
|
+
return directory
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def describe(root: Path, directory: Path) -> dict:
|
|
70
|
+
"""One episode as the list shows it: where it is, its task, seed, status and ending,
|
|
71
|
+
when its trace last changed, and whether it is still being written."""
|
|
72
|
+
done = summary(directory)
|
|
73
|
+
manifest = _read_json(directory / "manifest.json") or {}
|
|
74
|
+
config = manifest.get("config") or {}
|
|
75
|
+
trace = directory / "events.jsonl"
|
|
76
|
+
return {
|
|
77
|
+
"run": directory.relative_to(root).as_posix(),
|
|
78
|
+
"group": directory.parent.relative_to(root).as_posix() if directory.parent != root else "",
|
|
79
|
+
"task": done.get("task") or config.get("task"),
|
|
80
|
+
"seed": done.get("seed", manifest.get("seed")),
|
|
81
|
+
"status": done.get("status"),
|
|
82
|
+
"ending": ending(done),
|
|
83
|
+
"success": done.get("success"),
|
|
84
|
+
"model": (config.get("main") or {}).get("model"),
|
|
85
|
+
"interactive": "interactive" in manifest,
|
|
86
|
+
"modified": trace.stat().st_mtime,
|
|
87
|
+
"live": not done,
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def list_episodes(root: Path, limit: int = 500) -> list[dict]:
|
|
92
|
+
"""The episodes under `root`, the most recently changed first (at most `limit`)."""
|
|
93
|
+
found = []
|
|
94
|
+
for current, directories, files in os.walk(root):
|
|
95
|
+
if "events.jsonl" in files:
|
|
96
|
+
found.append(Path(current))
|
|
97
|
+
directories[:] = []
|
|
98
|
+
continue
|
|
99
|
+
directories[:] = sorted(name for name in directories if not name.startswith("."))
|
|
100
|
+
found.sort(key=lambda path: (path / "events.jsonl").stat().st_mtime, reverse=True)
|
|
101
|
+
return [describe(root, path) for path in found[:limit]]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def read_records(directory: Path, after: int = -1) -> list[dict]:
|
|
105
|
+
"""The trace records of an episode with `seq` above `after` (a line still being
|
|
106
|
+
written is left for later)."""
|
|
107
|
+
records = []
|
|
108
|
+
with (directory / "events.jsonl").open("rb") as handle:
|
|
109
|
+
for line in handle:
|
|
110
|
+
if not line.endswith(b"\n"):
|
|
111
|
+
break
|
|
112
|
+
record = json.loads(line)
|
|
113
|
+
if record.get("seq", -1) > after:
|
|
114
|
+
records.append(record)
|
|
115
|
+
return records
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class Follow:
|
|
119
|
+
"""New records of a trace as they are written: next() returns the records added since
|
|
120
|
+
the previous call (complete lines only)."""
|
|
121
|
+
|
|
122
|
+
def __init__(self, directory: Path, after: int = -1):
|
|
123
|
+
self.path = directory / "events.jsonl"
|
|
124
|
+
self.after = after
|
|
125
|
+
self.offset = 0
|
|
126
|
+
self.partial = b""
|
|
127
|
+
|
|
128
|
+
def next(self) -> list[dict]:
|
|
129
|
+
records = []
|
|
130
|
+
if not self.path.is_file():
|
|
131
|
+
# A conversation that has not written its first record yet.
|
|
132
|
+
return records
|
|
133
|
+
with self.path.open("rb") as handle:
|
|
134
|
+
handle.seek(self.offset)
|
|
135
|
+
data = handle.read()
|
|
136
|
+
self.offset += len(data)
|
|
137
|
+
lines = (self.partial + data).split(b"\n")
|
|
138
|
+
self.partial = lines.pop()
|
|
139
|
+
for line in lines:
|
|
140
|
+
if not line.strip():
|
|
141
|
+
continue
|
|
142
|
+
record = json.loads(line)
|
|
143
|
+
if record.get("seq", -1) > self.after:
|
|
144
|
+
self.after = record["seq"]
|
|
145
|
+
records.append(record)
|
|
146
|
+
return records
|
|
147
|
+
|
|
148
|
+
def ended(self) -> bool:
|
|
149
|
+
"""Whether the episode has its summary (nothing more will be written)."""
|
|
150
|
+
return (self.path.parent / "summary.json").is_file()
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""How every frontend shows money, time and long text (ui/static/format.js: the same money and
|
|
2
|
+
time): cost(0.0342) "$0.0342", duration(64.2) "64 s", clock(75) "1:15", timestamp(epoch or ISO)."""
|
|
3
|
+
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
import math
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _number(value) -> float | None:
|
|
9
|
+
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) else None
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def clip(text, limit: int) -> str:
|
|
13
|
+
text = str(text)
|
|
14
|
+
return text if len(text) <= limit else text[:limit - 1] + "…"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def clip_lines(text, lines: int = 20, width: int = 160) -> list[str]:
|
|
18
|
+
"""`text`'s first `lines` lines, each at most `width` characters, and how many more."""
|
|
19
|
+
rows = str(text).rstrip().splitlines()
|
|
20
|
+
return [clip(row, width) for row in rows[:lines]] + ([f"… {len(rows) - lines} more lines"] if len(rows) > lines else [])
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def cost(usd) -> str:
|
|
24
|
+
usd = _number(usd)
|
|
25
|
+
return "" if usd is None else f"${usd:.4f}" if abs(usd) < 1 else f"${usd:.2f}"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def duration(seconds, whole_until: int = 600) -> str:
|
|
29
|
+
"""One decimal under 10 s, whole seconds under `whole_until`, then [h:]m:ss; "" for no number."""
|
|
30
|
+
seconds = _number(seconds)
|
|
31
|
+
if seconds is None:
|
|
32
|
+
return ""
|
|
33
|
+
seconds = max(seconds, 0.0)
|
|
34
|
+
if seconds < 10: # halves up, as JavaScript's Math.round
|
|
35
|
+
return f"{math.floor(seconds * 10 + 0.5) / 10:.1f} s"
|
|
36
|
+
if seconds < whole_until:
|
|
37
|
+
return f"{math.floor(seconds + 0.5)} s"
|
|
38
|
+
hours, rest = divmod(math.floor(seconds), 3600)
|
|
39
|
+
return f"{hours}:{rest // 60:02d}:{rest % 60:02d}" if hours else f"{rest // 60}:{rest % 60:02d}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def clock(seconds) -> str:
|
|
43
|
+
return duration(seconds, 60)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def timestamp(when) -> str:
|
|
47
|
+
try:
|
|
48
|
+
moment = datetime.fromisoformat(when) if isinstance(when, str) else datetime.fromtimestamp(_number(when))
|
|
49
|
+
return (moment.astimezone() if moment.tzinfo else moment).strftime("%Y-%m-%d %H:%M")
|
|
50
|
+
except (TypeError, ValueError, OverflowError, OSError):
|
|
51
|
+
return ""
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Shared HTTP requests, cancellation and event-stream decoding."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from http.client import HTTPConnection, HTTPSConnection
|
|
5
|
+
import socket
|
|
6
|
+
import threading
|
|
7
|
+
from urllib.request import HTTPHandler, HTTPSHandler, Request, build_opener, urlopen
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Preempted(BaseException):
|
|
11
|
+
"""The caller cancelled the request. This must bypass provider retry handling."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class RequestCancellation:
|
|
15
|
+
"""Cancel the current HTTP exchange and wake a retry wait."""
|
|
16
|
+
|
|
17
|
+
def __init__(self):
|
|
18
|
+
self.event = threading.Event()
|
|
19
|
+
self._lock = threading.Lock()
|
|
20
|
+
self._socket = None
|
|
21
|
+
|
|
22
|
+
def check(self):
|
|
23
|
+
if self.event.is_set():
|
|
24
|
+
raise Preempted()
|
|
25
|
+
|
|
26
|
+
def bind(self, sock):
|
|
27
|
+
with self._lock:
|
|
28
|
+
self._socket = sock
|
|
29
|
+
self.check()
|
|
30
|
+
|
|
31
|
+
def cancel(self):
|
|
32
|
+
with self._lock:
|
|
33
|
+
self.event.set()
|
|
34
|
+
if self._socket is not None:
|
|
35
|
+
try:
|
|
36
|
+
self._socket.shutdown(socket.SHUT_RDWR)
|
|
37
|
+
except OSError:
|
|
38
|
+
pass # The exchange already closed.
|
|
39
|
+
|
|
40
|
+
def opener(self):
|
|
41
|
+
# Bind before writing the request, including when urlopen handles a proxy or redirect.
|
|
42
|
+
# Closing the response from another thread can block on its buffered reader.
|
|
43
|
+
cancellation = self
|
|
44
|
+
|
|
45
|
+
class Connection:
|
|
46
|
+
def connect(self):
|
|
47
|
+
cancellation.check()
|
|
48
|
+
super().connect()
|
|
49
|
+
try:
|
|
50
|
+
cancellation.bind(self.sock)
|
|
51
|
+
except Preempted:
|
|
52
|
+
self.close()
|
|
53
|
+
raise
|
|
54
|
+
|
|
55
|
+
class HTTP(Connection, HTTPConnection):
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
class HTTPS(Connection, HTTPSConnection):
|
|
59
|
+
pass
|
|
60
|
+
|
|
61
|
+
class HttpHandler(HTTPHandler):
|
|
62
|
+
def http_open(self, request):
|
|
63
|
+
return self.do_open(HTTP, request)
|
|
64
|
+
|
|
65
|
+
class HttpsHandler(HTTPSHandler):
|
|
66
|
+
def https_open(self, request):
|
|
67
|
+
return self.do_open(HTTPS, request, context=self._context)
|
|
68
|
+
|
|
69
|
+
return build_opener(HttpHandler(), HttpsHandler()).open
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def open_url(url, body=None, headers=None, timeout=60.0, *, cancellation=None):
|
|
73
|
+
"""The open response (a context manager) to a request to `url`: a POST of `body` as
|
|
74
|
+
JSON (no NaN; numpy values as lists), a GET without one. urllib's HTTPError and
|
|
75
|
+
URLError go to the caller, which says what failed."""
|
|
76
|
+
data = None if body is None else json.dumps(body, allow_nan=False, default=_jsonable).encode()
|
|
77
|
+
headers = {"user-agent": "the-harness", **({} if data is None else {"content-type": "application/json"}),
|
|
78
|
+
**(headers or {})}
|
|
79
|
+
opener = urlopen if cancellation is None else cancellation.opener()
|
|
80
|
+
return opener(Request(url, data=data, headers=headers, method="GET" if data is None else "POST"), timeout=timeout)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def post_json(url, body, headers=None, timeout=60.0, *, cancellation=None):
|
|
84
|
+
"""POST `body` as JSON to `url` and return the JSON answer."""
|
|
85
|
+
with open_url(url, body, headers, timeout, **({"cancellation": cancellation} if cancellation else {})) as response:
|
|
86
|
+
return json.loads(response.read())
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def read_sse(lines):
|
|
90
|
+
"""The events of a text/event-stream as they come, as (event, data): data is the
|
|
91
|
+
event's data lines joined, event its event field ("message" without one). `lines` is
|
|
92
|
+
a response or any iterable of byte lines. A comment (": keep-alive") comes as (None,
|
|
93
|
+
its text), so a reader hears that the stream is alive."""
|
|
94
|
+
event, data = None, []
|
|
95
|
+
for raw in lines:
|
|
96
|
+
line = raw.decode("utf-8", "replace").rstrip("\r\n")
|
|
97
|
+
if not line:
|
|
98
|
+
if data or event:
|
|
99
|
+
yield event or "message", "\n".join(data)
|
|
100
|
+
event, data = None, []
|
|
101
|
+
elif line.startswith(":"):
|
|
102
|
+
yield None, line[1:].strip()
|
|
103
|
+
else:
|
|
104
|
+
name, _, value = line.partition(":")
|
|
105
|
+
value = value[1:] if value.startswith(" ") else value
|
|
106
|
+
if name == "data":
|
|
107
|
+
data.append(value)
|
|
108
|
+
elif name == "event":
|
|
109
|
+
event = value
|
|
110
|
+
if data:
|
|
111
|
+
yield event or "message", "\n".join(data)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _jsonable(value):
|
|
115
|
+
"""numpy arrays and scalars (a program's positions, say) as JSON values."""
|
|
116
|
+
if hasattr(value, "tolist"):
|
|
117
|
+
return value.tolist()
|
|
118
|
+
raise TypeError(f"{type(value).__name__} is not JSON serializable")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def redact(text, secret):
|
|
122
|
+
if secret and secret in text:
|
|
123
|
+
return text.replace(secret, "[redacted]")
|
|
124
|
+
return text
|