@lmmslab/harness 0.0.0-stage → 0.81.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +124 -2
  3. package/bin/harness.js +71 -0
  4. package/package.json +34 -4
  5. package/python/harness/README.md +72 -0
  6. package/python/harness/__init__.py +8 -0
  7. package/python/harness/__main__.py +3 -0
  8. package/python/harness/bench.py +124 -0
  9. package/python/harness/builtin/benches/README.md +5 -0
  10. package/python/harness/builtin/benches/none/bench.toml +6 -0
  11. package/python/harness/builtin/configs/README.md +26 -0
  12. package/python/harness/builtin/configs/assistant.toml +11 -0
  13. package/python/harness/builtin/configs/default.toml +33 -0
  14. package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
  15. package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
  16. package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
  17. package/python/harness/builtin/configs/models/grok-low.toml +8 -0
  18. package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
  19. package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
  20. package/python/harness/builtin/environment/README.md +53 -0
  21. package/python/harness/builtin/environment/constraints.txt +31 -0
  22. package/python/harness/builtin/model-settings.js +143 -0
  23. package/python/harness/builtin/prompts/conversation/core.md +5 -0
  24. package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
  25. package/python/harness/builtin/prompts/core/core.md +88 -0
  26. package/python/harness/builtin/prompts/core/pack.toml +9 -0
  27. package/python/harness/builtin/prompts/reflect/core.md +45 -0
  28. package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
  29. package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
  30. package/python/harness/builtin/prompts/shared/agents.md +14 -0
  31. package/python/harness/builtin/prompts/shared/checklist.md +21 -0
  32. package/python/harness/builtin/prompts/shared/entries.md +86 -0
  33. package/python/harness/builtin/prompts/shared/environments.md +21 -0
  34. package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
  35. package/python/harness/builtin/prompts/shared/learning.md +27 -0
  36. package/python/harness/builtin/prompts/shared/mcp.md +9 -0
  37. package/python/harness/builtin/prompts/shared/world.md +1 -0
  38. package/python/harness/builtin/skills/README.md +10 -0
  39. package/python/harness/builtin/skills/agents/__init__.py +88 -0
  40. package/python/harness/builtin/skills/agents/skill.toml +11 -0
  41. package/python/harness/builtin/skills/checklist/__init__.py +32 -0
  42. package/python/harness/builtin/skills/checklist/skill.toml +9 -0
  43. package/python/harness/builtin/skills/conversation/__init__.py +23 -0
  44. package/python/harness/builtin/skills/conversation/skill.toml +8 -0
  45. package/python/harness/builtin/skills/entries/__init__.py +62 -0
  46. package/python/harness/builtin/skills/entries/skill.toml +8 -0
  47. package/python/harness/builtin/skills/environments/__init__.py +69 -0
  48. package/python/harness/builtin/skills/environments/skill.toml +9 -0
  49. package/python/harness/builtin/skills/experience/__init__.py +27 -0
  50. package/python/harness/builtin/skills/experience/skill.toml +9 -0
  51. package/python/harness/builtin/skills/instructions/__init__.py +29 -0
  52. package/python/harness/builtin/skills/instructions/skill.toml +9 -0
  53. package/python/harness/builtin/skills/learning/__init__.py +189 -0
  54. package/python/harness/builtin/skills/learning/skill.toml +7 -0
  55. package/python/harness/builtin/skills/mcp/__init__.py +54 -0
  56. package/python/harness/builtin/skills/mcp/skill.toml +10 -0
  57. package/python/harness/builtin/skills/memory/__init__.py +130 -0
  58. package/python/harness/builtin/skills/memory/skill.toml +13 -0
  59. package/python/harness/builtin/skills/verify/__init__.py +31 -0
  60. package/python/harness/builtin/skills/verify/skill.toml +7 -0
  61. package/python/harness/builtin/third_party/sources.toml +40 -0
  62. package/python/harness/cli.py +1182 -0
  63. package/python/harness/config/__init__.py +224 -0
  64. package/python/harness/config/agent.py +777 -0
  65. package/python/harness/core/__init__.py +52 -0
  66. package/python/harness/core/artifacts.py +46 -0
  67. package/python/harness/core/codec.py +160 -0
  68. package/python/harness/core/contracts.py +75 -0
  69. package/python/harness/core/episodes.py +150 -0
  70. package/python/harness/core/format.py +51 -0
  71. package/python/harness/core/http.py +124 -0
  72. package/python/harness/core/lanes.py +324 -0
  73. package/python/harness/core/linux_sandbox.py +55 -0
  74. package/python/harness/core/media.py +96 -0
  75. package/python/harness/core/memory.py +147 -0
  76. package/python/harness/core/ports.py +63 -0
  77. package/python/harness/core/procs.py +227 -0
  78. package/python/harness/core/remote.py +102 -0
  79. package/python/harness/core/spans.py +166 -0
  80. package/python/harness/core/trace.py +139 -0
  81. package/python/harness/core/wait.py +12 -0
  82. package/python/harness/core/worker.py +388 -0
  83. package/python/harness/core/worker_pipe.py +119 -0
  84. package/python/harness/embodiments.py +217 -0
  85. package/python/harness/extensions.py +112 -0
  86. package/python/harness/host/__init__.py +397 -0
  87. package/python/harness/host/client.py +201 -0
  88. package/python/harness/host/http.py +720 -0
  89. package/python/harness/host/listing.py +68 -0
  90. package/python/harness/host/server.py +189 -0
  91. package/python/harness/host/sessions.py +406 -0
  92. package/python/harness/host/tasks.py +1356 -0
  93. package/python/harness/host/workers.py +257 -0
  94. package/python/harness/host/workspaces.py +256 -0
  95. package/python/harness/model_list.py +117 -0
  96. package/python/harness/packages/__init__.py +73 -0
  97. package/python/harness/packages/addons.py +121 -0
  98. package/python/harness/packages/archive.py +122 -0
  99. package/python/harness/packages/describe.py +46 -0
  100. package/python/harness/packages/environments.py +448 -0
  101. package/python/harness/packages/fetch.py +57 -0
  102. package/python/harness/packages/manager.py +587 -0
  103. package/python/harness/packages/manifest.py +232 -0
  104. package/python/harness/packages/migrate.py +68 -0
  105. package/python/harness/packages/provision.py +38 -0
  106. package/python/harness/packages/registry.py +275 -0
  107. package/python/harness/packages/resolve.py +89 -0
  108. package/python/harness/packages/sources.py +141 -0
  109. package/python/harness/packages/state.py +213 -0
  110. package/python/harness/packages/versions.py +112 -0
  111. package/python/harness/paths.py +218 -0
  112. package/python/harness/providers.py +303 -0
  113. package/python/harness/runtime/__init__.py +8 -0
  114. package/python/harness/runtime/decision/__init__.py +1 -0
  115. package/python/harness/runtime/decision/loop.py +198 -0
  116. package/python/harness/runtime/decision/messages.py +271 -0
  117. package/python/harness/runtime/decision/model.py +424 -0
  118. package/python/harness/runtime/decision/prompts.py +248 -0
  119. package/python/harness/runtime/decision/protocol.py +169 -0
  120. package/python/harness/runtime/decision/thoughts.py +199 -0
  121. package/python/harness/runtime/events.py +213 -0
  122. package/python/harness/runtime/execution/__init__.py +1 -0
  123. package/python/harness/runtime/execution/attached.py +96 -0
  124. package/python/harness/runtime/execution/control.py +211 -0
  125. package/python/harness/runtime/execution/entries.py +665 -0
  126. package/python/harness/runtime/execution/kernel.py +811 -0
  127. package/python/harness/runtime/execution/noenv.py +84 -0
  128. package/python/harness/runtime/execution/results.py +40 -0
  129. package/python/harness/runtime/execution/session.py +871 -0
  130. package/python/harness/runtime/execution/skills.py +436 -0
  131. package/python/harness/runtime/learning/__init__.py +1 -0
  132. package/python/harness/runtime/learning/active.py +314 -0
  133. package/python/harness/runtime/learning/experience.py +48 -0
  134. package/python/harness/runtime/learning/fixtures.py +167 -0
  135. package/python/harness/runtime/learning/learner.py +651 -0
  136. package/python/harness/runtime/learning/library.py +504 -0
  137. package/python/harness/runtime/learning/reflect.py +151 -0
  138. package/python/harness/runtime/learning/usage.py +30 -0
  139. package/python/harness/runtime/learning/views.py +89 -0
  140. package/python/harness/runtime/reactive/__init__.py +1 -0
  141. package/python/harness/runtime/reactive/rules.py +549 -0
  142. package/python/harness/runtime/safety/__init__.py +1 -0
  143. package/python/harness/runtime/safety/approval.py +131 -0
  144. package/python/harness/runtime/safety/input.py +62 -0
  145. package/python/harness/runtime/safety/isolation.py +419 -0
  146. package/python/harness/runtime/safety/shield.py +129 -0
  147. package/python/harness/runtime/status.py +23 -0
  148. package/python/harness/runtime/task/__init__.py +7 -0
  149. package/python/harness/runtime/task/context.py +431 -0
  150. package/python/harness/runtime/task/delegation.py +468 -0
  151. package/python/harness/runtime/task/environments.py +268 -0
  152. package/python/harness/runtime/task/run.py +2878 -0
  153. package/python/harness/runtime/tick.py +909 -0
  154. package/python/harness/sdk/__init__.py +281 -0
  155. package/python/harness/services/__init__.py +6 -0
  156. package/python/harness/services/broker.py +35 -0
  157. package/python/harness/services/connectors.py +88 -0
  158. package/python/harness/services/mcp.py +298 -0
  159. package/python/harness/services/mcp_stdio.py +131 -0
  160. package/python/harness/services/placement.py +82 -0
  161. package/python/harness/services/registry.py +156 -0
  162. package/python/harness/settings.py +536 -0
  163. package/python/harness/terminal.py +79 -0
  164. package/python/harness/transcript.py +297 -0
  165. package/python/harness/tui/__init__.py +68 -0
  166. package/python/harness/tui/chat.py +1181 -0
  167. package/python/harness/tui/pty.py +81 -0
  168. package/python/harness/tui/renderer.py +45 -0
  169. package/python/harness/tui/timeline_rows.py +58 -0
  170. package/python/harness/tui/views.py +434 -0
@@ -0,0 +1,52 @@
1
+ """Building blocks shared by the runtime, the workers and the tools: the worker
2
+ protocol (contracts, robot client, worker server), traces, media, codecs, polling waits
3
+ and the HTTP helpers for model providers, the hashes records carry and the atomic file
4
+ write. Standard library only."""
5
+
6
+ import hashlib
7
+ import json
8
+ import os
9
+ from pathlib import Path
10
+ import tempfile
11
+
12
+
13
+ def canonical(value) -> str:
14
+ """`value` as compact JSON with sorted keys (other objects as their str), so equal
15
+ trees give the same text."""
16
+ return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
17
+
18
+
19
+ def sha256(value) -> str:
20
+ """The hex sha256 of `value`'s canonical JSON."""
21
+ return hashlib.sha256(canonical(value).encode()).hexdigest()
22
+
23
+
24
+ def tree_hash(folder, paths) -> str:
25
+ """A hash of files in `folder` (sorted `paths`): their names relative to it and their
26
+ contents."""
27
+ digest = hashlib.sha256()
28
+ for path in paths:
29
+ digest.update(Path(path).relative_to(folder).as_posix().encode() + b"\0")
30
+ digest.update(hashlib.sha256(Path(path).read_bytes()).digest())
31
+ return digest.hexdigest()
32
+
33
+
34
+ def atomic_write(path, data, *, private=False) -> None:
35
+ """Write `data` (text as UTF-8, or bytes) to `path` at once: a temporary file in the
36
+ same folder, then os.replace, so a reader finds the old file or the new one, never a
37
+ part, and writers side by side each keep a whole file. The file is readable by the
38
+ user only when `private` (it holds keys or tokens)."""
39
+ path = Path(path)
40
+ path.parent.mkdir(parents=True, exist_ok=True)
41
+ handle, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
42
+ try:
43
+ with os.fdopen(handle, "wb") as out:
44
+ out.write(data.encode("utf-8") if isinstance(data, str) else data)
45
+ try:
46
+ os.chmod(temporary, 0o600 if private else 0o644)
47
+ except OSError:
48
+ pass
49
+ os.replace(temporary, path)
50
+ except BaseException:
51
+ Path(temporary).unlink(missing_ok=True)
52
+ raise
@@ -0,0 +1,46 @@
1
+ """Run evidence: immutable payloads and a small append-only index, independent of cell replies."""
2
+
3
+ import hashlib
4
+ import json
5
+ from pathlib import Path
6
+ import threading
7
+ import time
8
+
9
+
10
+ class Evidence:
11
+ """Bounded disk evidence. Repeated payloads share bytes, never mutable target paths.
12
+
13
+ A record's reference can be resolved after the controller exits. Exhaustion is an
14
+ explicit error; callers must not retry an action whose result could not be recorded.
15
+ """
16
+
17
+ def __init__(self, directory, limit=256 * 1024 * 1024):
18
+ self.directory = Path(directory)
19
+ self.directory.mkdir(parents=True, exist_ok=True)
20
+ self.limit = limit
21
+ self.lock = threading.RLock()
22
+ self.used = sum(p.stat().st_size for p in self.directory.iterdir() if p.is_file())
23
+
24
+ def reserve(self, size):
25
+ with self.lock:
26
+ if self.used + size > self.limit:
27
+ raise OSError("run evidence limit reached; execution must stop")
28
+ self.used += size
29
+
30
+ def record(self, kind, payload, **identity):
31
+ raw = json.dumps(payload, ensure_ascii=False, allow_nan=False).encode("utf-8")
32
+ digest = hashlib.sha256(raw).hexdigest()
33
+ filename = digest + ".json"
34
+ row = (json.dumps({"kind": kind, "time": time.time(), **identity,
35
+ "payload": filename}, ensure_ascii=False) + "\n").encode("utf-8")
36
+ with self.lock:
37
+ path = self.directory / filename
38
+ needed = len(row) + (0 if path.exists() else len(raw))
39
+ self.reserve(needed)
40
+ if not path.exists():
41
+ path.write_bytes(raw)
42
+ with (self.directory / "index.jsonl").open("ab") as stream:
43
+ offset = stream.tell()
44
+ stream.write(row)
45
+ stream.flush()
46
+ return {"path": str(path), "sha256": digest, "index_offset": offset}
@@ -0,0 +1,160 @@
1
+ """Codecs shared by benchmark workers: episode cursors, outcome tokens, action bounds, data
2
+ URLs.
3
+
4
+ Importable without a simulator, torch or a GPU (image_data_url imports NumPy, and
5
+ OpenCV or Pillow, when it is called).
6
+ """
7
+
8
+ import base64
9
+ import math
10
+ import secrets
11
+ import struct
12
+ import zlib
13
+
14
+
15
+ class EpisodeCursor:
16
+ """Reject a repeated motion without guessing whether the simulator applied it."""
17
+
18
+ def __init__(self):
19
+ self.episode_id = None
20
+ self.step = None
21
+
22
+ def start(self, episode_id: str, step: int = 0) -> None:
23
+ self.episode_id = episode_id
24
+ self.step = step
25
+
26
+ def check(self, episode_id: str, step: int) -> None:
27
+ if self.episode_id is None or (episode_id, step) != (self.episode_id, self.step):
28
+ raise ValueError("Stale or missing observation; action was not executed")
29
+
30
+ def advance(self) -> int:
31
+ self.step += 1
32
+ return self.step
33
+
34
+ def invalidate(self) -> None:
35
+ self.episode_id = None
36
+ self.step = None
37
+
38
+
39
+ class OutcomeToken:
40
+ """The token that reads an episode's outcome (claim_outcome, outcome): one per episode,
41
+ claimed once by the harness after the loop, never by a program. A second claim, an
42
+ outcome asked before the claim and a wrong token raise PermissionError; the token is
43
+ compared in constant time. A new episode gets a new OutcomeToken."""
44
+
45
+ def __init__(self):
46
+ self._token = None
47
+
48
+ def claim(self) -> str:
49
+ if self._token is not None:
50
+ raise PermissionError("The outcome token for this episode was already claimed")
51
+ self._token = secrets.token_hex(16)
52
+ return self._token
53
+
54
+ def check(self, token) -> None:
55
+ if self._token is None or not isinstance(token, str) or not secrets.compare_digest(token, self._token):
56
+ raise PermissionError("Invalid outcome token")
57
+
58
+
59
+
60
+ def finite_bounds(low, high):
61
+ bounds = []
62
+ for start, stop in zip(low, high, strict=True):
63
+ lower, upper = float(start), float(stop)
64
+ if not math.isfinite(lower) or not math.isfinite(upper) or lower > upper:
65
+ raise ValueError("Action bounds must be finite and ordered")
66
+ bounds.append((lower, upper))
67
+ return bounds
68
+
69
+
70
+
71
+ def validate_vector(values, bounds, label="position"):
72
+ """A list of finite values within its per-component bounds."""
73
+ if not isinstance(values, list) or len(values) != len(bounds):
74
+ raise ValueError(f"{label} must contain {len(bounds)} finite numbers")
75
+ command = []
76
+ for value, (lower, upper) in zip(values, bounds, strict=True):
77
+ if type(value) not in (int, float) or not math.isfinite(value):
78
+ raise ValueError(f"{label} values must be finite numbers")
79
+ number = float(value)
80
+ if number < lower or number > upper:
81
+ raise ValueError(f"{label} value {number} is outside [{lower}, {upper}]")
82
+ command.append(number)
83
+ return command
84
+
85
+
86
+
87
+ def fit_into(inner, outer):
88
+ """Where a picture of size `inner` (width, height) sits in one of size `outer`: (left,
89
+ top, width, height), as large as it fits, centred (a monitor of another shape in a
90
+ screenshot, a recorded screen in a replay's frame)."""
91
+ scale = min(outer[0] / inner[0], outer[1] / inner[1])
92
+ width, height = max(1, round(inner[0] * scale)), max(1, round(inner[1] * scale))
93
+ return (outer[0] - width) // 2, (outer[1] - height) // 2, width, height
94
+
95
+
96
+ def data_url(payload, mime="image/png"):
97
+ if not payload:
98
+ raise ValueError("Image payload is empty")
99
+ encoded = base64.b64encode(payload).decode("ascii")
100
+ return f"data:{mime};base64,{encoded}"
101
+
102
+
103
+ def image_data_url(array, fmt="jpeg", quality=95):
104
+ """An HxWx3 RGB (or HxW grayscale) array as a data URL of a `fmt` (jpeg or png) image,
105
+ JPEG at `quality`: OpenCV encodes it, or Pillow where OpenCV is not installed (a game's
106
+ environment has Pillow and NumPy only)."""
107
+ import numpy as np
108
+
109
+ array = np.asarray(array)
110
+ if not (array.ndim == 2 or (array.ndim == 3 and array.shape[-1] == 3)):
111
+ raise ValueError("images must be HxWx3 RGB (or HxW grayscale) arrays")
112
+ fmt = {"jpg": "jpeg"}.get(fmt.lower(), fmt.lower())
113
+ if fmt not in ("jpeg", "png"):
114
+ raise ValueError(f"image format {fmt!r}: jpeg or png")
115
+ array = array.astype(np.uint8)
116
+ try:
117
+ import cv2
118
+ except ImportError:
119
+ import io
120
+
121
+ from PIL import Image
122
+
123
+ buffer = io.BytesIO()
124
+ Image.fromarray(array, "RGB" if array.ndim == 3 else "L").save(
125
+ buffer, format=fmt.upper(), **({"quality": quality} if fmt == "jpeg" else {}))
126
+ return data_url(buffer.getvalue(), f"image/{fmt}")
127
+ ok, encoded = cv2.imencode(".jpg" if fmt == "jpeg" else ".png",
128
+ cv2.cvtColor(array, cv2.COLOR_RGB2BGR) if array.ndim == 3 else array,
129
+ [cv2.IMWRITE_JPEG_QUALITY, int(quality)] if fmt == "jpeg" else [])
130
+ if not ok:
131
+ raise ValueError("Could not encode the image")
132
+ return data_url(encoded.tobytes(), f"image/{fmt}")
133
+
134
+
135
+ def decode_data_url(url) -> bytes:
136
+ """The bytes a base64 data URL (data:<type>;base64,<payload>) carries."""
137
+ header, comma, payload = str(url).partition(",")
138
+ if not (header.startswith("data:") and header.endswith(";base64") and comma):
139
+ raise ValueError("not a base64 data URL")
140
+ return base64.b64decode(payload, validate=True)
141
+
142
+
143
+ def encode_png(width, height, rgb):
144
+ """Encode tightly packed RGB bytes as a PNG."""
145
+ if width < 1 or height < 1:
146
+ raise ValueError("Image dimensions must be positive")
147
+ expected = width * height * 3
148
+ if len(rgb) != expected:
149
+ raise ValueError("RGB byte count does not match the image size")
150
+ raw = b"".join(
151
+ b"\x00" + rgb[row * width * 3:(row + 1) * width * 3]
152
+ for row in range(height)
153
+ )
154
+
155
+ def chunk(tag, data):
156
+ return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
157
+
158
+ header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
159
+ return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
160
+
@@ -0,0 +1,75 @@
1
+ """JSON-serializable contracts shared by agents and simulator workers."""
2
+
3
+ from dataclasses import asdict, dataclass, field
4
+ from typing import Any, Protocol
5
+
6
+
7
+ @dataclass
8
+ class Observation:
9
+ episode_id: str
10
+ step: int
11
+ instruction: str
12
+ state: dict[str, Any]
13
+ # Camera name -> image data URL. This keeps remote traces self-contained.
14
+ images: dict[str, str] = field(default_factory=dict)
15
+
16
+ def to_dict(self) -> dict:
17
+ return asdict(self)
18
+
19
+
20
+ @dataclass
21
+ class Action:
22
+ name: str
23
+ arguments: dict[str, Any]
24
+
25
+ def to_dict(self) -> dict:
26
+ return asdict(self)
27
+
28
+
29
+ @dataclass
30
+ class Transition:
31
+ observation: Observation
32
+ reward: float = 0.0
33
+ success: bool = False
34
+ terminated: bool = False
35
+ truncated: bool = False
36
+ info: dict[str, Any] = field(default_factory=dict)
37
+ # Camera name -> MP4 data URL of the motion produced by this action.
38
+ # Still images stay on the observation. Video is not sent to the model.
39
+ videos: dict[str, str] = field(default_factory=dict)
40
+
41
+ def to_dict(self) -> dict:
42
+ return asdict(self)
43
+
44
+ @classmethod
45
+ def from_dict(cls, data: dict) -> "Transition":
46
+ data = dict(data)
47
+ data["observation"] = Observation(**data["observation"])
48
+ return cls(**data)
49
+
50
+
51
+ @dataclass
52
+ class Decision:
53
+ action: Action
54
+ usage: dict[str, Any] = field(default_factory=dict)
55
+ response_id: str | None = None
56
+
57
+
58
+ class Environment(Protocol):
59
+ """A tick worker. Only actor-visible observations belong here.
60
+
61
+ Its tick call checks the episode and step before acting. Legacy action-tool
62
+ workers instead implement execute(action, episode_id, step) -> Transition.
63
+ """
64
+
65
+ name: str
66
+ methods: tuple[str, ...]
67
+
68
+ def tools(self) -> list[dict]: ...
69
+ def reset(self, task: str, seed: int) -> Observation: ...
70
+ def call(self, name: str, arguments: dict) -> dict: ...
71
+ def close(self) -> None: ...
72
+
73
+
74
+ class Agent(Protocol):
75
+ def decide(self, request: dict) -> Decision: ...
@@ -0,0 +1,150 @@
1
+ """The episodes under a runs directory, as the GUI lists and reads them.
2
+
3
+ An episode is a directory with an events.jsonl (runtime/conversation.py); its
4
+ manifest.json and summary.json say what ran and how it ended. An episode without a
5
+ summary is still being written (or was cut off).
6
+ """
7
+
8
+ import json
9
+ import os
10
+ from pathlib import Path
11
+ import secrets
12
+ import time
13
+
14
+
15
+ # Files of an episode the GUI may serve, by extension.
16
+ MEDIA_TYPES = {".mp4": "video/mp4", ".webm": "video/webm", ".png": "image/png", ".jpg": "image/jpeg",
17
+ ".jpeg": "image/jpeg", ".json": "application/json", ".log": "text/plain; charset=utf-8",
18
+ ".txt": "text/plain; charset=utf-8"}
19
+
20
+
21
+ def new_name(suffix: str = "") -> str:
22
+ """A new run's directory name: "20261102-140530-3fa9c1", then "-<suffix>" when given."""
23
+ return f"{time.strftime('%Y%m%d-%H%M%S')}-{secrets.token_hex(3)}" + (f"-{suffix}" if suffix else "")
24
+
25
+
26
+ # How an episode ended (finished, closed, budget, step_limit, interrupted, error), and its words, where not the same.
27
+ ENDINGS = {"success": "finished", "budget_exhausted": "budget", "cost_exhausted": "budget",
28
+ "wall_time_exhausted": "budget", "episode_ended": "step_limit"}
29
+ ENDING_WORDS = {"closed": "closed by the user", "budget": "out of budget", "step_limit": "step limit"}
30
+
31
+
32
+ def summary(directory) -> dict:
33
+ """An episode's summary.json, {} when it has none or it cannot be read."""
34
+ value = _read_json(Path(directory) / "summary.json")
35
+ return value if isinstance(value, dict) else {}
36
+
37
+
38
+ def usage(source) -> dict | None:
39
+ """{model_calls, cost_usd} of a summary or of a directory's (None when it has none)."""
40
+ found = source if isinstance(source, dict) else summary(source) or None
41
+ return None if found is None else {"model_calls": int(found.get("model_calls") or 0),
42
+ "cost_usd": round(float(found.get("cost_usd") or 0), 6)}
43
+
44
+
45
+ def ending(found: dict | None) -> str | None:
46
+ """How an episode ended (its summary's status, as ENDINGS names it); None without one."""
47
+ status = (found or {}).get("status")
48
+ return ENDINGS.get(str(status), str(status)) if status else None
49
+
50
+
51
+ def _read_json(path: Path):
52
+ try:
53
+ return json.loads(path.read_text(encoding="utf-8"))
54
+ except (OSError, ValueError):
55
+ return None
56
+
57
+
58
+ def episode_dir(root: Path, run: str) -> Path:
59
+ """The episode directory `run` (a path relative to `root`); ValueError when it leaves
60
+ the root or has no trace."""
61
+ directory = (root / run).resolve()
62
+ if directory != root and root not in directory.parents:
63
+ raise ValueError("run outside the runs directory")
64
+ if not (directory / "events.jsonl").is_file():
65
+ raise ValueError(f"no episode at {run}")
66
+ return directory
67
+
68
+
69
+ def describe(root: Path, directory: Path) -> dict:
70
+ """One episode as the list shows it: where it is, its task, seed, status and ending,
71
+ when its trace last changed, and whether it is still being written."""
72
+ done = summary(directory)
73
+ manifest = _read_json(directory / "manifest.json") or {}
74
+ config = manifest.get("config") or {}
75
+ trace = directory / "events.jsonl"
76
+ return {
77
+ "run": directory.relative_to(root).as_posix(),
78
+ "group": directory.parent.relative_to(root).as_posix() if directory.parent != root else "",
79
+ "task": done.get("task") or config.get("task"),
80
+ "seed": done.get("seed", manifest.get("seed")),
81
+ "status": done.get("status"),
82
+ "ending": ending(done),
83
+ "success": done.get("success"),
84
+ "model": (config.get("main") or {}).get("model"),
85
+ "interactive": "interactive" in manifest,
86
+ "modified": trace.stat().st_mtime,
87
+ "live": not done,
88
+ }
89
+
90
+
91
+ def list_episodes(root: Path, limit: int = 500) -> list[dict]:
92
+ """The episodes under `root`, the most recently changed first (at most `limit`)."""
93
+ found = []
94
+ for current, directories, files in os.walk(root):
95
+ if "events.jsonl" in files:
96
+ found.append(Path(current))
97
+ directories[:] = []
98
+ continue
99
+ directories[:] = sorted(name for name in directories if not name.startswith("."))
100
+ found.sort(key=lambda path: (path / "events.jsonl").stat().st_mtime, reverse=True)
101
+ return [describe(root, path) for path in found[:limit]]
102
+
103
+
104
+ def read_records(directory: Path, after: int = -1) -> list[dict]:
105
+ """The trace records of an episode with `seq` above `after` (a line still being
106
+ written is left for later)."""
107
+ records = []
108
+ with (directory / "events.jsonl").open("rb") as handle:
109
+ for line in handle:
110
+ if not line.endswith(b"\n"):
111
+ break
112
+ record = json.loads(line)
113
+ if record.get("seq", -1) > after:
114
+ records.append(record)
115
+ return records
116
+
117
+
118
+ class Follow:
119
+ """New records of a trace as they are written: next() returns the records added since
120
+ the previous call (complete lines only)."""
121
+
122
+ def __init__(self, directory: Path, after: int = -1):
123
+ self.path = directory / "events.jsonl"
124
+ self.after = after
125
+ self.offset = 0
126
+ self.partial = b""
127
+
128
+ def next(self) -> list[dict]:
129
+ records = []
130
+ if not self.path.is_file():
131
+ # A conversation that has not written its first record yet.
132
+ return records
133
+ with self.path.open("rb") as handle:
134
+ handle.seek(self.offset)
135
+ data = handle.read()
136
+ self.offset += len(data)
137
+ lines = (self.partial + data).split(b"\n")
138
+ self.partial = lines.pop()
139
+ for line in lines:
140
+ if not line.strip():
141
+ continue
142
+ record = json.loads(line)
143
+ if record.get("seq", -1) > self.after:
144
+ self.after = record["seq"]
145
+ records.append(record)
146
+ return records
147
+
148
+ def ended(self) -> bool:
149
+ """Whether the episode has its summary (nothing more will be written)."""
150
+ return (self.path.parent / "summary.json").is_file()
@@ -0,0 +1,51 @@
1
+ """How every frontend shows money, time and long text (ui/static/format.js: the same money and
2
+ time): cost(0.0342) "$0.0342", duration(64.2) "64 s", clock(75) "1:15", timestamp(epoch or ISO)."""
3
+
4
+ from datetime import datetime
5
+ import math
6
+
7
+
8
+ def _number(value) -> float | None:
9
+ return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) else None
10
+
11
+
12
+ def clip(text, limit: int) -> str:
13
+ text = str(text)
14
+ return text if len(text) <= limit else text[:limit - 1] + "…"
15
+
16
+
17
+ def clip_lines(text, lines: int = 20, width: int = 160) -> list[str]:
18
+ """`text`'s first `lines` lines, each at most `width` characters, and how many more."""
19
+ rows = str(text).rstrip().splitlines()
20
+ return [clip(row, width) for row in rows[:lines]] + ([f"… {len(rows) - lines} more lines"] if len(rows) > lines else [])
21
+
22
+
23
+ def cost(usd) -> str:
24
+ usd = _number(usd)
25
+ return "" if usd is None else f"${usd:.4f}" if abs(usd) < 1 else f"${usd:.2f}"
26
+
27
+
28
+ def duration(seconds, whole_until: int = 600) -> str:
29
+ """One decimal under 10 s, whole seconds under `whole_until`, then [h:]m:ss; "" for no number."""
30
+ seconds = _number(seconds)
31
+ if seconds is None:
32
+ return ""
33
+ seconds = max(seconds, 0.0)
34
+ if seconds < 10: # halves up, as JavaScript's Math.round
35
+ return f"{math.floor(seconds * 10 + 0.5) / 10:.1f} s"
36
+ if seconds < whole_until:
37
+ return f"{math.floor(seconds + 0.5)} s"
38
+ hours, rest = divmod(math.floor(seconds), 3600)
39
+ return f"{hours}:{rest // 60:02d}:{rest % 60:02d}" if hours else f"{rest // 60}:{rest % 60:02d}"
40
+
41
+
42
+ def clock(seconds) -> str:
43
+ return duration(seconds, 60)
44
+
45
+
46
+ def timestamp(when) -> str:
47
+ try:
48
+ moment = datetime.fromisoformat(when) if isinstance(when, str) else datetime.fromtimestamp(_number(when))
49
+ return (moment.astimezone() if moment.tzinfo else moment).strftime("%Y-%m-%d %H:%M")
50
+ except (TypeError, ValueError, OverflowError, OSError):
51
+ return ""
@@ -0,0 +1,124 @@
1
+ """Shared HTTP requests, cancellation and event-stream decoding."""
2
+
3
+ import json
4
+ from http.client import HTTPConnection, HTTPSConnection
5
+ import socket
6
+ import threading
7
+ from urllib.request import HTTPHandler, HTTPSHandler, Request, build_opener, urlopen
8
+
9
+
10
+ class Preempted(BaseException):
11
+ """The caller cancelled the request. This must bypass provider retry handling."""
12
+
13
+
14
+ class RequestCancellation:
15
+ """Cancel the current HTTP exchange and wake a retry wait."""
16
+
17
+ def __init__(self):
18
+ self.event = threading.Event()
19
+ self._lock = threading.Lock()
20
+ self._socket = None
21
+
22
+ def check(self):
23
+ if self.event.is_set():
24
+ raise Preempted()
25
+
26
+ def bind(self, sock):
27
+ with self._lock:
28
+ self._socket = sock
29
+ self.check()
30
+
31
+ def cancel(self):
32
+ with self._lock:
33
+ self.event.set()
34
+ if self._socket is not None:
35
+ try:
36
+ self._socket.shutdown(socket.SHUT_RDWR)
37
+ except OSError:
38
+ pass # The exchange already closed.
39
+
40
+ def opener(self):
41
+ # Bind before writing the request, including when urlopen handles a proxy or redirect.
42
+ # Closing the response from another thread can block on its buffered reader.
43
+ cancellation = self
44
+
45
+ class Connection:
46
+ def connect(self):
47
+ cancellation.check()
48
+ super().connect()
49
+ try:
50
+ cancellation.bind(self.sock)
51
+ except Preempted:
52
+ self.close()
53
+ raise
54
+
55
+ class HTTP(Connection, HTTPConnection):
56
+ pass
57
+
58
+ class HTTPS(Connection, HTTPSConnection):
59
+ pass
60
+
61
+ class HttpHandler(HTTPHandler):
62
+ def http_open(self, request):
63
+ return self.do_open(HTTP, request)
64
+
65
+ class HttpsHandler(HTTPSHandler):
66
+ def https_open(self, request):
67
+ return self.do_open(HTTPS, request, context=self._context)
68
+
69
+ return build_opener(HttpHandler(), HttpsHandler()).open
70
+
71
+
72
+ def open_url(url, body=None, headers=None, timeout=60.0, *, cancellation=None):
73
+ """The open response (a context manager) to a request to `url`: a POST of `body` as
74
+ JSON (no NaN; numpy values as lists), a GET without one. urllib's HTTPError and
75
+ URLError go to the caller, which says what failed."""
76
+ data = None if body is None else json.dumps(body, allow_nan=False, default=_jsonable).encode()
77
+ headers = {"user-agent": "the-harness", **({} if data is None else {"content-type": "application/json"}),
78
+ **(headers or {})}
79
+ opener = urlopen if cancellation is None else cancellation.opener()
80
+ return opener(Request(url, data=data, headers=headers, method="GET" if data is None else "POST"), timeout=timeout)
81
+
82
+
83
+ def post_json(url, body, headers=None, timeout=60.0, *, cancellation=None):
84
+ """POST `body` as JSON to `url` and return the JSON answer."""
85
+ with open_url(url, body, headers, timeout, **({"cancellation": cancellation} if cancellation else {})) as response:
86
+ return json.loads(response.read())
87
+
88
+
89
+ def read_sse(lines):
90
+ """The events of a text/event-stream as they come, as (event, data): data is the
91
+ event's data lines joined, event its event field ("message" without one). `lines` is
92
+ a response or any iterable of byte lines. A comment (": keep-alive") comes as (None,
93
+ its text), so a reader hears that the stream is alive."""
94
+ event, data = None, []
95
+ for raw in lines:
96
+ line = raw.decode("utf-8", "replace").rstrip("\r\n")
97
+ if not line:
98
+ if data or event:
99
+ yield event or "message", "\n".join(data)
100
+ event, data = None, []
101
+ elif line.startswith(":"):
102
+ yield None, line[1:].strip()
103
+ else:
104
+ name, _, value = line.partition(":")
105
+ value = value[1:] if value.startswith(" ") else value
106
+ if name == "data":
107
+ data.append(value)
108
+ elif name == "event":
109
+ event = value
110
+ if data:
111
+ yield event or "message", "\n".join(data)
112
+
113
+
114
+ def _jsonable(value):
115
+ """numpy arrays and scalars (a program's positions, say) as JSON values."""
116
+ if hasattr(value, "tolist"):
117
+ return value.tolist()
118
+ raise TypeError(f"{type(value).__name__} is not JSON serializable")
119
+
120
+
121
+ def redact(text, secret):
122
+ if secret and secret in text:
123
+ return text.replace(secret, "[redacted]")
124
+ return text