@lmmslab/harness 0.0.0-stage → 0.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1308 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +221 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from functools import cache
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import platform
|
|
7
|
+
import subprocess
|
|
8
|
+
import threading
|
|
9
|
+
from time import time
|
|
10
|
+
|
|
11
|
+
from . import atomic_write, sha256
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def write_json(path: Path, data) -> None:
|
|
15
|
+
atomic_write(path, json.dumps(data, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def git_revision(root) -> dict:
|
|
19
|
+
"""The commit of the checkout `root` and whether its tree was dirty: {"commit", "dirty"},
|
|
20
|
+
each None where git cannot say (no git, not a checkout)."""
|
|
21
|
+
def git(*args):
|
|
22
|
+
result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, encoding="utf-8",
|
|
23
|
+
errors="replace")
|
|
24
|
+
return result.stdout.strip() if result.returncode == 0 else None
|
|
25
|
+
try:
|
|
26
|
+
commit, status = git("rev-parse", "HEAD"), git("status", "--porcelain")
|
|
27
|
+
except OSError:
|
|
28
|
+
commit, status = None, None
|
|
29
|
+
return {"commit": commit, "dirty": bool(status) if status is not None else None}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def now() -> str:
|
|
33
|
+
"""The local time, to the second, as ISO 8601 with its offset (a result's started_at)."""
|
|
34
|
+
return datetime.now().astimezone().isoformat(timespec="seconds")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def stamp() -> str:
|
|
38
|
+
"""The local time as a file name part, to the millisecond (20261103_142501_123)."""
|
|
39
|
+
return datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@cache
|
|
43
|
+
def provenance() -> dict:
|
|
44
|
+
"""The Python, git commit and the hash of each of the harness's files, once per process
|
|
45
|
+
(the code that runs is the code imported)."""
|
|
46
|
+
package = Path(__file__).resolve().parents[1]
|
|
47
|
+
files = {str(path.relative_to(package)): hashlib.sha256(path.read_bytes()).hexdigest()
|
|
48
|
+
for path in sorted(package.rglob("*.py"))}
|
|
49
|
+
revision = git_revision(package)
|
|
50
|
+
return {"python": platform.python_version(), "git_commit": revision["commit"], "git_dirty": revision["dirty"],
|
|
51
|
+
"package_sha256": sha256(files), "package_files": files}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# Who a trace record comes from (its `source`).
|
|
55
|
+
SOURCES = ("user", "agent", "program", "environment", "harness")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Trace:
|
|
59
|
+
"""The append-only record of one episode (events.jsonl): every record has its number
|
|
60
|
+
`seq` (its id), the wall-clock `time` and its kind `event`. With `kinds` ({kind:
|
|
61
|
+
{"source", ...}}, runtime/conversation.py EVENTS) a record also says its `source`, and
|
|
62
|
+
a kind not in the table is refused. Subscribers get each record as it is written;
|
|
63
|
+
records may come from several threads. `context`, when set, gives what goes into every
|
|
64
|
+
record (whose trace it is: a conversation of a session, runtime/conversation.py)."""
|
|
65
|
+
|
|
66
|
+
def __init__(self, directory: Path, kinds: dict | None = None, *, append: bool = False):
|
|
67
|
+
path = directory / "events.jsonl"
|
|
68
|
+
if append:
|
|
69
|
+
from .episodes import read_records
|
|
70
|
+
|
|
71
|
+
if not path.is_file():
|
|
72
|
+
raise ValueError("the existing session trace is missing")
|
|
73
|
+
sequence = max((record["seq"] for record in read_records(directory)), default=-1) + 1
|
|
74
|
+
else:
|
|
75
|
+
directory.mkdir(parents=True, exist_ok=False)
|
|
76
|
+
path.touch()
|
|
77
|
+
sequence = 0
|
|
78
|
+
self.directory = directory
|
|
79
|
+
self.sequence = sequence
|
|
80
|
+
self.kinds = kinds
|
|
81
|
+
self.subscribers = []
|
|
82
|
+
self.context = None
|
|
83
|
+
self._lock = threading.Lock()
|
|
84
|
+
|
|
85
|
+
def subscribe(self, callback) -> None:
|
|
86
|
+
"""Call `callback(record)` with every record written from now on."""
|
|
87
|
+
self.subscribers.append(callback)
|
|
88
|
+
|
|
89
|
+
def emit(self, event: str, **data):
|
|
90
|
+
if self.kinds is not None:
|
|
91
|
+
if event not in self.kinds:
|
|
92
|
+
raise ValueError(f"Unknown trace event {event!r}")
|
|
93
|
+
# A record may name another source than its kind's (a program's finish()), never
|
|
94
|
+
# something that is no source.
|
|
95
|
+
if data.get("source", self.kinds[event]["source"]) not in SOURCES:
|
|
96
|
+
raise ValueError(f"A record's source is one of {list(SOURCES)}, not {data['source']!r}")
|
|
97
|
+
data = {"source": self.kinds[event]["source"], **data}
|
|
98
|
+
with self._lock:
|
|
99
|
+
record = {"seq": self.sequence, "time": time(), "event": event,
|
|
100
|
+
**(self.context() if self.context is not None else {}), **data}
|
|
101
|
+
with (self.directory / "events.jsonl").open("a", encoding="utf-8") as handle:
|
|
102
|
+
handle.write(json.dumps(record, ensure_ascii=False, allow_nan=False) + "\n")
|
|
103
|
+
handle.flush()
|
|
104
|
+
self.sequence += 1
|
|
105
|
+
for callback in list(self.subscribers):
|
|
106
|
+
try:
|
|
107
|
+
callback(record)
|
|
108
|
+
except Exception:
|
|
109
|
+
# A viewer that fails never stops the episode.
|
|
110
|
+
pass
|
|
111
|
+
return record
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _text_of(content):
|
|
115
|
+
if isinstance(content, str):
|
|
116
|
+
return [content]
|
|
117
|
+
return [part.get("text", "") for part in content or [] if part.get("type") in ("text", "input_text")]
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def context_summary(messages, system=None):
|
|
121
|
+
"""What one model call received in full, one short row per message: role, opening
|
|
122
|
+
text, size, images still attached and images replaced by an omission note. The
|
|
123
|
+
trace keeps each message's content once (where it was first sent); this records
|
|
124
|
+
how the whole history looked at each call, after windowing and image pruning."""
|
|
125
|
+
rows = [{"role": "system", "text": system[:160], "chars": len(system), "images": 0, "omitted": 0}] if system else []
|
|
126
|
+
for message in messages:
|
|
127
|
+
content = message.get("content")
|
|
128
|
+
texts = _text_of(content)
|
|
129
|
+
parts = [] if isinstance(content, str) else list(content or [])
|
|
130
|
+
calls = [f"{(c.get('function') or {}).get('name')}(...)" for c in message.get("tool_calls") or []]
|
|
131
|
+
joined = " ".join(t for t in texts if t) or " ".join(calls)
|
|
132
|
+
rows.append({
|
|
133
|
+
"role": message.get("role"),
|
|
134
|
+
"text": " ".join(joined.split())[:160],
|
|
135
|
+
"chars": sum(len(t) for t in texts),
|
|
136
|
+
"images": sum(part.get("type") in ("input_image", "image_url") for part in parts),
|
|
137
|
+
"omitted": sum("image omitted" in t for t in texts),
|
|
138
|
+
})
|
|
139
|
+
return rows
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Waiting for a condition by polling it."""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def wait_until(predicate, timeout: float, interval: float = 0.1):
|
|
7
|
+
"""Call `predicate()` every `interval` seconds until it returns something true or
|
|
8
|
+
`timeout` seconds have passed; what it returned last."""
|
|
9
|
+
deadline = time.monotonic() + timeout
|
|
10
|
+
while not (value := predicate()) and time.monotonic() < deadline:
|
|
11
|
+
time.sleep(interval)
|
|
12
|
+
return value
|
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
"""A serial worker keeps all simulator operations on the serving thread.
|
|
2
|
+
|
|
3
|
+
A worker is an embodiment's process: it serves the worker protocol (WORKER_METHODS) behind
|
|
4
|
+
its shield (runtime/safety/shield.py CommandShield), which judges every command before it moves
|
|
5
|
+
anything. The session skeleton (runtime/execution/session.py EpisodeCore) needs nothing else of it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
9
|
+
import hmac
|
|
10
|
+
import json
|
|
11
|
+
|
|
12
|
+
from harness.runtime.safety.approval import APPROVAL_PIPE_FLAG, WORKER_APPROVAL_HEADER, WORKER_APPROVAL_KEY, read_worker_startup, worker_approval_proof
|
|
13
|
+
from .codec import EpisodeCursor, OutcomeToken
|
|
14
|
+
from .contracts import Action
|
|
15
|
+
from .ports import OwnPort
|
|
16
|
+
|
|
17
|
+
# The worker protocol: what every worker serves to the session skeleton and the
|
|
18
|
+
# conversation, besides describe, reset and execute: what the embodiment is (profile: its kind,
|
|
19
|
+
# control rate, clock, channels, cameras and shield), its state now, one control period's
|
|
20
|
+
# command, judged by its shield (tick), the views of its cameras or screen (observe), and
|
|
21
|
+
# the evaluator's outcome (claim_outcome, outcome). An embodiment adds its own methods (a robot's
|
|
22
|
+
# fk, ik and plan; a computer's windows; a game's world): `methods = (*WORKER_METHODS, ...)`.
|
|
23
|
+
WORKER_METHODS = ("profile", "state", "tick", "observe", "claim_outcome", "outcome")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class BenchWorker:
|
|
27
|
+
"""What a benchmark's worker shares (a mixin): the call dispatch to its `_call_<name>`
|
|
28
|
+
methods, input refused outside tick (which its shield checks), the state call, the
|
|
29
|
+
check of the views observe takes, the envelope of a tick, and the outcome behind a
|
|
30
|
+
token claimed once per episode (OutcomeToken).
|
|
31
|
+
|
|
32
|
+
A worker sets `methods` (its calls), LABEL (its name in messages), INPUT ("input", or
|
|
33
|
+
"motion" for a robot) and VIEWS (the views it has); keeps `_cursor` (EpisodeCursor);
|
|
34
|
+
and defines `_state()`, `_ended()` and `_evaluate()` (the outcome, asked once). Its
|
|
35
|
+
reset calls `new_outcome()`."""
|
|
36
|
+
|
|
37
|
+
methods = WORKER_METHODS
|
|
38
|
+
LABEL = "worker"
|
|
39
|
+
INPUT = "input"
|
|
40
|
+
VIEWS = ("screen",)
|
|
41
|
+
|
|
42
|
+
def tools(self):
|
|
43
|
+
return []
|
|
44
|
+
|
|
45
|
+
def execute(self, action, episode_id, step):
|
|
46
|
+
raise RuntimeError(f"The {self.LABEL} accepts {self.INPUT} only through call('tick'), which the shield checks")
|
|
47
|
+
|
|
48
|
+
def call(self, name, arguments):
|
|
49
|
+
if name not in self.methods:
|
|
50
|
+
raise ValueError(f"Unknown {self.LABEL} method: {name}")
|
|
51
|
+
return getattr(self, f"_call_{name}")(**arguments)
|
|
52
|
+
|
|
53
|
+
def _check_episode(self, episode_id) -> None:
|
|
54
|
+
"""The call is for the current episode (a worker may also refuse an episode it can no
|
|
55
|
+
longer score)."""
|
|
56
|
+
self._cursor.check(episode_id, self._cursor.step)
|
|
57
|
+
|
|
58
|
+
def _call_state(self, episode_id):
|
|
59
|
+
self._check_episode(episode_id)
|
|
60
|
+
return self._state()
|
|
61
|
+
|
|
62
|
+
def views(self, cameras) -> list:
|
|
63
|
+
"""The views observe is asked for (all of VIEWS without any); ValueError for one this
|
|
64
|
+
worker has not."""
|
|
65
|
+
views = list(cameras or self.VIEWS)
|
|
66
|
+
unknown = set(views) - set(self.VIEWS)
|
|
67
|
+
if unknown:
|
|
68
|
+
raise ValueError(f"Unknown views: {sorted(unknown)}; this worker has {', '.join(self.VIEWS)}")
|
|
69
|
+
return views
|
|
70
|
+
|
|
71
|
+
def _call_observe(self, episode_id, cameras=None):
|
|
72
|
+
"""A desktop's view (a robot's worker overrides it: its cameras)."""
|
|
73
|
+
self._check_episode(episode_id)
|
|
74
|
+
self.views(cameras)
|
|
75
|
+
# The picture first: on a computer with several screens it picks the one the state names.
|
|
76
|
+
images = self._screenshot()
|
|
77
|
+
return {"state": self._state(), "images": images}
|
|
78
|
+
|
|
79
|
+
# -- The envelope of a tick ------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
def tick_begins(self, episode_id, step) -> None:
|
|
82
|
+
"""A tick's checks: the episode and step are the current ones, the episode goes on."""
|
|
83
|
+
self._cursor.check(episode_id, step)
|
|
84
|
+
if self._ended():
|
|
85
|
+
raise ValueError("The episode has ended; the command was not executed")
|
|
86
|
+
|
|
87
|
+
def tick_refused(self, decision) -> dict:
|
|
88
|
+
"""The shield stopped the command: nothing was executed."""
|
|
89
|
+
return {"executed": False, "shield": decision.to_dict(), "state": self._state()}
|
|
90
|
+
|
|
91
|
+
def acted(self, act, on_error=None):
|
|
92
|
+
"""`act()` once, then the next step. When it raises, the input may have reached the
|
|
93
|
+
environment: the episode cannot go on from a state nobody observed, so the cursor is
|
|
94
|
+
invalidated (the step is never sent again) and `on_error(error)` records it."""
|
|
95
|
+
try:
|
|
96
|
+
value = act()
|
|
97
|
+
except Exception as error:
|
|
98
|
+
self._cursor.invalidate()
|
|
99
|
+
if on_error is not None:
|
|
100
|
+
on_error(error)
|
|
101
|
+
raise
|
|
102
|
+
self._cursor.advance()
|
|
103
|
+
return value
|
|
104
|
+
|
|
105
|
+
def tick_executed(self, decision, episode_id, cameras=None, **result) -> dict:
|
|
106
|
+
"""The result of an executed tick, with the views after it when `cameras` asks."""
|
|
107
|
+
envelope = {"executed": True, "shield": decision.to_dict(), "state": self._state(), **result}
|
|
108
|
+
if cameras:
|
|
109
|
+
envelope["observation"] = self._call_observe(episode_id, cameras=cameras)
|
|
110
|
+
return envelope
|
|
111
|
+
|
|
112
|
+
# -- The outcome behind a token -------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
def new_outcome(self) -> None:
|
|
115
|
+
"""A new episode: a new token (core/codec.py OutcomeToken), no outcome asked."""
|
|
116
|
+
self._outcome_token = OutcomeToken()
|
|
117
|
+
self._outcome = None
|
|
118
|
+
|
|
119
|
+
def _call_claim_outcome(self, episode_id):
|
|
120
|
+
self._check_episode(episode_id)
|
|
121
|
+
return {"token": self._outcome_token.claim()}
|
|
122
|
+
|
|
123
|
+
def _call_outcome(self, token):
|
|
124
|
+
self._outcome_token.check(token)
|
|
125
|
+
if getattr(self, "_outcome", None) is None:
|
|
126
|
+
self._outcome = self._evaluate()
|
|
127
|
+
return dict(self._outcome)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class WorkerAPI:
|
|
131
|
+
"""The worker's RPC methods over one environment: describe, reset, and call.
|
|
132
|
+
|
|
133
|
+
Action-tool workers without a tick use the legacy execute route and an EpisodeCursor
|
|
134
|
+
owned here. Tick workers own their cursor. Both use the observation contract. A worker
|
|
135
|
+
that supports managed approval declares its protected `call` names in
|
|
136
|
+
`managed_control_calls`."""
|
|
137
|
+
|
|
138
|
+
def __init__(self, environment, contract=None, *, approval_key=None, approval_from_stdin=False, host_pipe=False):
|
|
139
|
+
self.environment = environment
|
|
140
|
+
self._approval_key = approval_key
|
|
141
|
+
self._managed_approval = (approval_from_stdin and bool(approval_key)
|
|
142
|
+
and callable(getattr(environment, "_sensitive_approval", None)))
|
|
143
|
+
if host_pipe and not self._managed_approval:
|
|
144
|
+
raise ValueError("A host-owned workspace requires a managed approval worker")
|
|
145
|
+
self._host_pipe = host_pipe
|
|
146
|
+
self._managed_control_calls = frozenset()
|
|
147
|
+
if approval_from_stdin and approval_key:
|
|
148
|
+
self._managed_control_calls = frozenset(getattr(environment, "managed_control_calls", ()))
|
|
149
|
+
if not self._managed_control_calls.issubset(environment.methods) or self._managed_approval and not self._managed_control_calls:
|
|
150
|
+
raise ValueError("Managed approval requires a declaration of protected worker calls")
|
|
151
|
+
self._program_control_calls = frozenset(getattr(environment, "program_control_calls", ()))
|
|
152
|
+
if not self._program_control_calls.issubset(self._managed_control_calls) and approval_from_stdin:
|
|
153
|
+
raise ValueError("Program controls must be authenticated worker controls")
|
|
154
|
+
if self._managed_approval:
|
|
155
|
+
environment._managed_approval = True
|
|
156
|
+
# What the clients may see (embodiments.contract_filter); None reports everything.
|
|
157
|
+
self.contract = contract
|
|
158
|
+
self._legacy_cursor = None if "tick" in getattr(environment, "methods", ()) else EpisodeCursor()
|
|
159
|
+
|
|
160
|
+
def _filtered(self, method, arguments, result):
|
|
161
|
+
return self.contract.apply(method, arguments, result) if self.contract is not None else result
|
|
162
|
+
|
|
163
|
+
def dispatch(self, method: str, arguments: dict, *, approval_proof=""):
|
|
164
|
+
"""The result of one RPC: `describe` (protocol version, name, tools, methods),
|
|
165
|
+
`reset`, `execute` (only on the current episode and step) or `call` (one of the
|
|
166
|
+
environment's own methods). An unknown method is an error."""
|
|
167
|
+
if self._host_pipe:
|
|
168
|
+
request = {"method": method, "arguments": arguments}
|
|
169
|
+
if not hmac.compare_digest(approval_proof, worker_approval_proof(self._approval_key, request)):
|
|
170
|
+
raise PermissionError("Worker control requires the host proof")
|
|
171
|
+
if method == "describe":
|
|
172
|
+
description = {"protocol_version": 1, "name": self.environment.name,
|
|
173
|
+
"tools": self.environment.tools(), "snapshot_restore": False,
|
|
174
|
+
"methods": sorted(getattr(self.environment, "methods", ()))}
|
|
175
|
+
if self._managed_approval:
|
|
176
|
+
description["managed_approval"] = True
|
|
177
|
+
if self._managed_control_calls:
|
|
178
|
+
description["authenticated_controls"] = True
|
|
179
|
+
description["managed_control_calls"] = sorted(self._managed_control_calls)
|
|
180
|
+
description["program_control_calls"] = sorted(self._program_control_calls)
|
|
181
|
+
if self._host_pipe:
|
|
182
|
+
description["host_pipe"] = True
|
|
183
|
+
return description
|
|
184
|
+
control_call = method == "call" and arguments.get("name") in self._managed_control_calls
|
|
185
|
+
if method == "sensitive_approval" or self._managed_control_calls and (control_call or method in ("reset", "execute")):
|
|
186
|
+
request = {"method": method, "arguments": arguments}
|
|
187
|
+
if not self._managed_control_calls or not hmac.compare_digest(
|
|
188
|
+
approval_proof, worker_approval_proof(self._approval_key, request)):
|
|
189
|
+
raise PermissionError("Managed worker control requires proof from the host")
|
|
190
|
+
if method == "sensitive_approval":
|
|
191
|
+
approve = getattr(self.environment, "_sensitive_approval", None)
|
|
192
|
+
if approve is None:
|
|
193
|
+
raise ValueError("This worker does not support sensitive approval")
|
|
194
|
+
result = approve(**arguments)
|
|
195
|
+
return self._filtered("tick", arguments, result) if arguments["op"] == "retry" else result
|
|
196
|
+
if method == "call":
|
|
197
|
+
# Environment-specific extensions (e.g. per-tick control for the
|
|
198
|
+
# runtime). The environment enforces its own episode/step checks.
|
|
199
|
+
name = arguments["name"]
|
|
200
|
+
if name not in getattr(self.environment, "methods", ()):
|
|
201
|
+
raise ValueError(f"Unknown environment method: {name}")
|
|
202
|
+
inner = arguments.get("arguments", {})
|
|
203
|
+
if name == "tick" and not self._managed_control_calls and "execution_id" in inner:
|
|
204
|
+
raise PermissionError("Worker execution IDs require an authenticated host")
|
|
205
|
+
return self._filtered(name, inner, self.environment.call(name, inner))
|
|
206
|
+
if method == "reset":
|
|
207
|
+
cursor = self._legacy_cursor
|
|
208
|
+
if cursor is not None:
|
|
209
|
+
cursor.invalidate()
|
|
210
|
+
observation = self.environment.reset(**arguments)
|
|
211
|
+
result = self._filtered("reset", arguments, observation.to_dict())
|
|
212
|
+
if cursor is not None:
|
|
213
|
+
cursor.start(observation.episode_id, observation.step)
|
|
214
|
+
return result
|
|
215
|
+
if method == "execute":
|
|
216
|
+
cursor = self._legacy_cursor
|
|
217
|
+
if cursor is None:
|
|
218
|
+
raise RuntimeError("This worker accepts actions only through call('tick')")
|
|
219
|
+
episode_id, step = arguments["episode_id"], arguments["step"]
|
|
220
|
+
cursor.check(episode_id, step)
|
|
221
|
+
action = Action(**arguments["action"])
|
|
222
|
+
# Consume the cursor before entering the worker; never replay unknown delivery.
|
|
223
|
+
cursor.invalidate()
|
|
224
|
+
transition = self.environment.execute(action, episode_id, step)
|
|
225
|
+
result = transition.to_dict()
|
|
226
|
+
result["observation"] = self._filtered("observe", arguments, result["observation"])
|
|
227
|
+
if not (transition.terminated or transition.truncated):
|
|
228
|
+
observation = transition.observation
|
|
229
|
+
cursor.start(observation.episode_id, observation.step)
|
|
230
|
+
return result
|
|
231
|
+
raise ValueError(f"Unknown worker method: {method}")
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def create_server(environment, host="127.0.0.1", port=8765, token=None, contract=None, *, approval_key=None,
|
|
235
|
+
approval_from_stdin=False):
|
|
236
|
+
"""An HTTP server (not yet serving) that answers POST /rpc with WorkerAPI, one request
|
|
237
|
+
at a time. With a `token` (required on a non-loopback host) requests must carry it as a
|
|
238
|
+
bearer token; errors come back as {"error": ...} with status 400."""
|
|
239
|
+
if host not in ("127.0.0.1", "localhost", "::1") and not token:
|
|
240
|
+
raise ValueError("Set HARNESS_WORKER_TOKEN before binding a non-loopback interface")
|
|
241
|
+
if approval_key and approval_key == token:
|
|
242
|
+
raise ValueError("The worker approval key must be separate from the worker bearer token")
|
|
243
|
+
api = WorkerAPI(environment, contract, approval_key=approval_key, approval_from_stdin=approval_from_stdin)
|
|
244
|
+
|
|
245
|
+
class Handler(BaseHTTPRequestHandler):
|
|
246
|
+
def log_message(self, format, *args):
|
|
247
|
+
pass
|
|
248
|
+
|
|
249
|
+
def do_POST(self):
|
|
250
|
+
if self.path != "/rpc":
|
|
251
|
+
self.send_error(404)
|
|
252
|
+
return
|
|
253
|
+
if token and not hmac.compare_digest(self.headers.get("Authorization", ""), f"Bearer {token}"):
|
|
254
|
+
self.send_error(401)
|
|
255
|
+
return
|
|
256
|
+
try:
|
|
257
|
+
size = int(self.headers.get("Content-Length", "0"))
|
|
258
|
+
if not 0 < size <= 32 * 1024 * 1024:
|
|
259
|
+
raise ValueError("Invalid request size")
|
|
260
|
+
request = json.loads(self.rfile.read(size))
|
|
261
|
+
result = {"result": api.dispatch(request["method"], request.get("arguments", {}),
|
|
262
|
+
approval_proof=self.headers.get(WORKER_APPROVAL_HEADER, ""))}
|
|
263
|
+
status = 200
|
|
264
|
+
except Exception as error:
|
|
265
|
+
result = {"error": f"{type(error).__name__}: {error}"}
|
|
266
|
+
status = 400
|
|
267
|
+
body = json.dumps(result, allow_nan=False).encode()
|
|
268
|
+
self.send_response(status)
|
|
269
|
+
self.send_header("Content-Type", "application/json")
|
|
270
|
+
self.send_header("Content-Length", str(len(body)))
|
|
271
|
+
self.end_headers()
|
|
272
|
+
self.wfile.write(body)
|
|
273
|
+
|
|
274
|
+
return _WorkerServer((host, port), Handler)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
class _WorkerServer(OwnPort, HTTPServer):
|
|
278
|
+
pass
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def parser():
|
|
282
|
+
"""The worker's arguments (python -m harness.core.worker, harness serve)."""
|
|
283
|
+
import argparse
|
|
284
|
+
|
|
285
|
+
parser = argparse.ArgumentParser(prog="harness serve", description="Serve a worker (a robot or simulator) in "
|
|
286
|
+
"its own Python process")
|
|
287
|
+
parser.add_argument("--factory", required=True,
|
|
288
|
+
help="module:callable, e.g. benches.robodojo.runtime:create_environment")
|
|
289
|
+
parser.add_argument("--host", default="127.0.0.1")
|
|
290
|
+
parser.add_argument("--port", type=int, default=8765)
|
|
291
|
+
parser.add_argument("--options", help="TOML keyword arguments passed to the factory")
|
|
292
|
+
parser.add_argument("--contract", default="native",
|
|
293
|
+
help="what clients may see: native (everything, the default) or a contract a package "
|
|
294
|
+
"registers for the embodiment's kind (contracts.toml; harness.embodiments."
|
|
295
|
+
"contract_filter), e.g. the robodojo package's xpolicylab or the osomni package's "
|
|
296
|
+
"screenshot")
|
|
297
|
+
parser.add_argument("--embodiment", help="embodiment name (embodiments/<name>/embodiment.toml) for the contract")
|
|
298
|
+
parser.add_argument("--exit-with-stdin", action="store_true",
|
|
299
|
+
help="end once standard input closes: the process that started it (the app's workspace "
|
|
300
|
+
"pool) ended, even without stopping it")
|
|
301
|
+
parser.add_argument(APPROVAL_PIPE_FLAG, dest="approval_from_stdin", action="store_true", help=argparse.SUPPRESS)
|
|
302
|
+
return parser
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def main(argv=None) -> int:
|
|
306
|
+
"""python -m harness.core.worker --factory module:callable: serve a worker (a robot, a
|
|
307
|
+
simulator, this computer) in its own Python process; what a suite and the computer's
|
|
308
|
+
workspaces start, and the eval package's harness serve."""
|
|
309
|
+
import importlib
|
|
310
|
+
import os
|
|
311
|
+
import stat
|
|
312
|
+
import sys
|
|
313
|
+
import threading
|
|
314
|
+
|
|
315
|
+
args = parser().parse_args(argv)
|
|
316
|
+
# Discard the retired setting before the worker can launch browser or app processes.
|
|
317
|
+
os.environ.pop(WORKER_APPROVAL_KEY, None)
|
|
318
|
+
stdin = sys.stdin.buffer if args.exit_with_stdin else None
|
|
319
|
+
approval_key = None
|
|
320
|
+
host_pipe = False
|
|
321
|
+
output = None
|
|
322
|
+
if args.approval_from_stdin:
|
|
323
|
+
if stdin is None or args.host != "127.0.0.1" or not stat.S_ISFIFO(os.fstat(stdin.fileno()).st_mode):
|
|
324
|
+
raise SystemExit("Managed worker approval requires a local worker with a private stdin pipe")
|
|
325
|
+
try:
|
|
326
|
+
# Keep any buffered bytes after startup on a private, non-inheritable fd.
|
|
327
|
+
stdin = os.fdopen(os.dup(stdin.fileno()), "rb")
|
|
328
|
+
approval_key, host_pipe = read_worker_startup(stdin)
|
|
329
|
+
except ValueError as error:
|
|
330
|
+
raise SystemExit(str(error)) from None
|
|
331
|
+
if host_pipe:
|
|
332
|
+
# Keep library and factory output on the log. Only framed replies reach the host.
|
|
333
|
+
output = os.fdopen(os.dup(1), "wb")
|
|
334
|
+
os.dup2(2, 1)
|
|
335
|
+
# Programs launched by the worker must inherit neither end of its RPC stream.
|
|
336
|
+
with open(os.devnull, "rb") as null:
|
|
337
|
+
os.dup2(null.fileno(), 0)
|
|
338
|
+
os.environ.pop("HARNESS_WORKER_TOKEN", None)
|
|
339
|
+
if os.environ.pop("HARNESS_MANAGED_LINUX_WORKER", None):
|
|
340
|
+
from .linux_sandbox import protect_process_handles
|
|
341
|
+
|
|
342
|
+
try:
|
|
343
|
+
protect_process_handles()
|
|
344
|
+
except RuntimeError as error:
|
|
345
|
+
raise SystemExit(str(error)) from None
|
|
346
|
+
from ..config import read_toml
|
|
347
|
+
from ..packages import activate
|
|
348
|
+
|
|
349
|
+
# A bench's factory is a module of a package (benches.<name>.runtime): its folder on the path.
|
|
350
|
+
activate()
|
|
351
|
+
module, function = args.factory.split(":", 1)
|
|
352
|
+
factory = getattr(importlib.import_module(module), function)
|
|
353
|
+
contract = None
|
|
354
|
+
if args.contract != "native":
|
|
355
|
+
from ..embodiments import contract_filter, load_embodiment
|
|
356
|
+
|
|
357
|
+
if not args.embodiment:
|
|
358
|
+
raise SystemExit("--contract needs --embodiment")
|
|
359
|
+
contract = contract_filter(args.contract, load_embodiment(args.embodiment))
|
|
360
|
+
environment = factory(**(read_toml(args.options) if args.options else {}))
|
|
361
|
+
try:
|
|
362
|
+
if host_pipe:
|
|
363
|
+
from .worker_pipe import serve
|
|
364
|
+
|
|
365
|
+
with stdin, output:
|
|
366
|
+
api = WorkerAPI(environment, contract, approval_key=approval_key,
|
|
367
|
+
approval_from_stdin=True, host_pipe=True)
|
|
368
|
+
serve(api, stdin, output)
|
|
369
|
+
return 0
|
|
370
|
+
with create_server(environment, args.host, args.port, os.environ.get("HARNESS_WORKER_TOKEN"),
|
|
371
|
+
contract=contract, approval_key=approval_key,
|
|
372
|
+
approval_from_stdin=args.approval_from_stdin) as server:
|
|
373
|
+
print(json.dumps({"worker": environment.name, "address": server.server_address}), flush=True)
|
|
374
|
+
if stdin is not None:
|
|
375
|
+
def until_closed():
|
|
376
|
+
while stdin.read(4096):
|
|
377
|
+
pass
|
|
378
|
+
server.shutdown()
|
|
379
|
+
|
|
380
|
+
threading.Thread(target=until_closed, name="stdin", daemon=True).start()
|
|
381
|
+
server.serve_forever()
|
|
382
|
+
finally:
|
|
383
|
+
environment.close()
|
|
384
|
+
return 0
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
if __name__ == "__main__":
|
|
388
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Framed RPC over a workspace worker's private stdin and stdout."""
|
|
2
|
+
|
|
3
|
+
from concurrent.futures import Future
|
|
4
|
+
import json
|
|
5
|
+
from queue import Full, Queue
|
|
6
|
+
import threading
|
|
7
|
+
|
|
8
|
+
from harness.runtime.safety.approval import worker_approval_proof
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_MAX_FRAME = 32 * 1024 * 1024
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _read(pipe):
|
|
15
|
+
frame = pipe.readline(_MAX_FRAME + 1)
|
|
16
|
+
if not frame:
|
|
17
|
+
raise EOFError("The private worker pipe closed")
|
|
18
|
+
if len(frame) > _MAX_FRAME or not frame.endswith(b"\n"):
|
|
19
|
+
raise ValueError("Invalid private worker frame")
|
|
20
|
+
return json.loads(frame)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _write(pipe, value):
|
|
24
|
+
frame = json.dumps(value, allow_nan=False, separators=(",", ":")).encode() + b"\n"
|
|
25
|
+
if len(frame) > _MAX_FRAME:
|
|
26
|
+
raise ValueError("Private worker frame is too large")
|
|
27
|
+
# Unbuffered process pipes can write only part of a frame.
|
|
28
|
+
view = memoryview(frame)
|
|
29
|
+
while view:
|
|
30
|
+
count = pipe.write(view)
|
|
31
|
+
if not count:
|
|
32
|
+
raise EOFError("The private worker pipe closed while writing")
|
|
33
|
+
view = view[count:]
|
|
34
|
+
pipe.flush()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def serve(api, incoming, outgoing):
|
|
38
|
+
"""Serve one request at a time. EOF ends the worker with its host."""
|
|
39
|
+
sequence = 0
|
|
40
|
+
while True:
|
|
41
|
+
try:
|
|
42
|
+
message = _read(incoming)
|
|
43
|
+
except EOFError:
|
|
44
|
+
return
|
|
45
|
+
sequence += 1
|
|
46
|
+
if message["sequence"] != sequence:
|
|
47
|
+
raise ValueError("Unexpected private worker request sequence")
|
|
48
|
+
request = message["request"]
|
|
49
|
+
try:
|
|
50
|
+
result = {"result": api.dispatch(request["method"], request["arguments"],
|
|
51
|
+
approval_proof=message["proof"])}
|
|
52
|
+
except Exception as error:
|
|
53
|
+
result = {"error": f"{type(error).__name__}: {error}"}
|
|
54
|
+
_write(outgoing, {"sequence": sequence, **result})
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class WorkerRejected(RuntimeError):
|
|
58
|
+
"""The worker responded with an error on the expected RPC sequence."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class WorkerPipe:
|
|
62
|
+
"""Host I/O for a single workspace lease. A failed exchange closes the stream to callers."""
|
|
63
|
+
|
|
64
|
+
def __init__(self, process, key):
|
|
65
|
+
self._key = key
|
|
66
|
+
self._requests = Queue(maxsize=1)
|
|
67
|
+
self._sequence = 0
|
|
68
|
+
self._failed = False
|
|
69
|
+
self._lock = threading.Lock()
|
|
70
|
+
self._thread = threading.Thread(target=self._exchange, args=(process,), daemon=True)
|
|
71
|
+
self._thread.start()
|
|
72
|
+
|
|
73
|
+
def request(self, method, arguments, *, timeout):
|
|
74
|
+
with self._lock:
|
|
75
|
+
if self._failed:
|
|
76
|
+
raise RuntimeError("The private worker transport has ended")
|
|
77
|
+
request = {"method": method, "arguments": arguments}
|
|
78
|
+
proof = worker_approval_proof(self._key, request)
|
|
79
|
+
future = Future()
|
|
80
|
+
self._sequence += 1
|
|
81
|
+
try:
|
|
82
|
+
self._requests.put_nowait(({"sequence": self._sequence, "request": request, "proof": proof}, future))
|
|
83
|
+
except Full:
|
|
84
|
+
self._sequence -= 1
|
|
85
|
+
raise RuntimeError("The worker still has a pending request") from None
|
|
86
|
+
try:
|
|
87
|
+
response = future.result(timeout=timeout)
|
|
88
|
+
except TimeoutError:
|
|
89
|
+
# A read-only receipt query can follow the outstanding delivery.
|
|
90
|
+
raise
|
|
91
|
+
except Exception:
|
|
92
|
+
self._failed = True
|
|
93
|
+
raise
|
|
94
|
+
if "error" in response:
|
|
95
|
+
raise WorkerRejected(response["error"])
|
|
96
|
+
return response["result"]
|
|
97
|
+
|
|
98
|
+
def close(self):
|
|
99
|
+
# The process owner closes or kills the worker to wake any in-flight pipe I/O.
|
|
100
|
+
# The queue sentinel also wakes an idle exchange thread on normal release.
|
|
101
|
+
with self._lock:
|
|
102
|
+
self._failed = True
|
|
103
|
+
try:
|
|
104
|
+
self._requests.put_nowait(None)
|
|
105
|
+
except Full:
|
|
106
|
+
pass # Process termination wakes the queued exchange.
|
|
107
|
+
|
|
108
|
+
def _exchange(self, process):
|
|
109
|
+
while (job := self._requests.get()) is not None:
|
|
110
|
+
message, future = job
|
|
111
|
+
try:
|
|
112
|
+
_write(process.stdin, message)
|
|
113
|
+
response = _read(process.stdout)
|
|
114
|
+
if response["sequence"] != message["sequence"] or ("result" in response) == ("error" in response):
|
|
115
|
+
raise ValueError("Invalid private worker reply")
|
|
116
|
+
future.set_result(response)
|
|
117
|
+
except Exception as error:
|
|
118
|
+
future.set_exception(error)
|
|
119
|
+
return
|