@lmmslab/harness 0.0.0-stage → 0.81.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1356 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +232 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
"""Environments: where a service's model or a benchmark's simulator runs, installed the
|
|
2
|
+
first time a run needs it and never before.
|
|
3
|
+
|
|
4
|
+
A service manifest (services/<name>.toml) or a benchmark (benches/<name>/bench.toml)
|
|
5
|
+
describes its environment in an [environment] table:
|
|
6
|
+
|
|
7
|
+
[environment]
|
|
8
|
+
python = "3.11" the Python version
|
|
9
|
+
requirements = ["torch==2.7.0", "timm==1.0.30"] pinned requirements (pip syntax)
|
|
10
|
+
indexes = ["https://download.pytorch.org/whl/cu128"]
|
|
11
|
+
constraints = "environment/lock/pip-freeze.txt" pins every install respects
|
|
12
|
+
sources = [{ source = "sam3", install = "editable", no_deps = true }]
|
|
13
|
+
commands = [["{python}", "-m", "pip", "check"]] run after the installs, in order
|
|
14
|
+
imports = ["torch", "sam3"] the check that it works
|
|
15
|
+
|
|
16
|
+
or names another environment to run in: `environment = "openwam"` (that service's) or
|
|
17
|
+
`environment = "robodojo"` (that benchmark's). Each environment is a virtual
|
|
18
|
+
environment of its own under HOME/envs/<name>/<hash>/, the hash taken over everything
|
|
19
|
+
that decides what is installed, so a manifest whose environment changes gets a new one
|
|
20
|
+
and an unchanged one is reused. A source is checked out once at its pinned commit
|
|
21
|
+
under HOME/sources/ (third_party/sources.toml names the shared ones) and shared by every
|
|
22
|
+
environment that uses it.
|
|
23
|
+
|
|
24
|
+
uv builds them when it is on PATH (`uv venv --python X` fetches the Python version
|
|
25
|
+
itself); without it, a python<X> on PATH and pip do. A deploy (or $HARNESS_ENVIRONMENTS)
|
|
26
|
+
with environments = "project" installs none: every service and worker runs in the
|
|
27
|
+
interpreter of the harness itself, the development environment of a checkout
|
|
28
|
+
(environment/setup.sh), which has everything.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
import contextlib
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
import hashlib
|
|
34
|
+
import json
|
|
35
|
+
import os
|
|
36
|
+
import re
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
import shutil
|
|
39
|
+
import subprocess
|
|
40
|
+
import sys
|
|
41
|
+
import time
|
|
42
|
+
import tomllib
|
|
43
|
+
|
|
44
|
+
from ..core import format as formats
|
|
45
|
+
from ..core.procs import file_lock, service_environment
|
|
46
|
+
from ..paths import ROOT, home, third_party
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# The keys of an [environment] table.
|
|
50
|
+
ENVIRONMENT_KEYS = {
|
|
51
|
+
"python": "the Python version, e.g. \"3.11\"",
|
|
52
|
+
"requirements": "pinned requirements, in pip syntax (\"-r FILE\" names a file of a source)",
|
|
53
|
+
"indexes": "package indexes besides the default one (e.g. PyTorch's CUDA wheels)",
|
|
54
|
+
"constraints": "pins every install of the environment respects: a file of the harness "
|
|
55
|
+
"(environment/lock/pip-freeze.txt) or a list",
|
|
56
|
+
"sources": "checkouts at fixed commits, and how to install each: [{ source or repo + rev, nested, install, "
|
|
57
|
+
"subdirectory, no_deps, flags }]",
|
|
58
|
+
"commands": "commands run after the installs, in order: argv lists, or { run, env, cwd } "
|
|
59
|
+
"({python}, {env}, {root}, {home} and the sources' checkouts such as {robodojo} are filled in)",
|
|
60
|
+
"env": "environment variables of the installs and commands",
|
|
61
|
+
"overrides": "versions that win over what a package declares (isaacsim-kernel's websockets==12.0 next to "
|
|
62
|
+
"17.1): kept out of the constraints, forced after the installs",
|
|
63
|
+
"imports": "modules that must import once it is installed (the check)",
|
|
64
|
+
}
|
|
65
|
+
# The keys of one entry of `sources`.
|
|
66
|
+
SOURCE_KEYS = {
|
|
67
|
+
"source": "a source of third_party/sources.toml (its url, revision and nested checkouts)",
|
|
68
|
+
"repo": "or a git URL ...",
|
|
69
|
+
"rev": "... and the full commit to check out",
|
|
70
|
+
"nested": "nested checkouts to fetch at the revisions sources.toml gives (default none)",
|
|
71
|
+
"install": "none (default: only checked out), editable or package",
|
|
72
|
+
"subdirectory": "install what is in this subdirectory of the checkout",
|
|
73
|
+
"no_deps": "install without its dependencies (they are in requirements)",
|
|
74
|
+
"flags": "further pip flags, e.g. --no-build-isolation",
|
|
75
|
+
}
|
|
76
|
+
INSTALLS = ("none", "editable", "package")
|
|
77
|
+
SOURCES_FILE = ROOT / "third_party" / "sources.toml"
|
|
78
|
+
ENVIRONMENTS_ENV = "HARNESS_ENVIRONMENTS"
|
|
79
|
+
RECORD = "harness-env.json"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def validate(spec, where: str) -> None:
|
|
83
|
+
"""Raise ValueError when an [environment] table (or the name of another one) is not
|
|
84
|
+
one ENVIRONMENT_KEYS describes."""
|
|
85
|
+
if isinstance(spec, str):
|
|
86
|
+
return
|
|
87
|
+
if not isinstance(spec, dict):
|
|
88
|
+
raise ValueError(f"{where}: environment is a table or the name of another environment")
|
|
89
|
+
unknown = set(spec) - set(ENVIRONMENT_KEYS)
|
|
90
|
+
if unknown:
|
|
91
|
+
raise ValueError(f"{where}: environment has unknown keys {sorted(unknown)}")
|
|
92
|
+
if not isinstance(spec.get("python", ""), str) or not spec.get("python"):
|
|
93
|
+
raise ValueError(f"{where}: environment needs a python version (python = \"3.11\")")
|
|
94
|
+
for entry in spec.get("sources", []):
|
|
95
|
+
unknown = set(entry) - set(SOURCE_KEYS)
|
|
96
|
+
if unknown:
|
|
97
|
+
raise ValueError(f"{where}: a source has unknown keys {sorted(unknown)}")
|
|
98
|
+
if bool(entry.get("source")) == bool(entry.get("repo")):
|
|
99
|
+
raise ValueError(f"{where}: a source names either source or repo (with rev)")
|
|
100
|
+
if entry.get("repo") and len(str(entry.get("rev", ""))) != 40:
|
|
101
|
+
raise ValueError(f"{where}: a source repo needs rev, a full commit")
|
|
102
|
+
if entry.get("source") and entry["source"] not in shared_sources():
|
|
103
|
+
raise ValueError(f"{where}: no source {entry['source']!r} in {SOURCES_FILE.name}; "
|
|
104
|
+
f"known: {sorted(shared_sources())}")
|
|
105
|
+
if entry.get("install", "none") not in INSTALLS:
|
|
106
|
+
raise ValueError(f"{where}: a source's install is one of {INSTALLS}")
|
|
107
|
+
for command in spec.get("commands", []):
|
|
108
|
+
if not (isinstance(command, list) or (isinstance(command, dict) and "run" in command)):
|
|
109
|
+
raise ValueError(f"{where}: a command is an argv list or {{ run = [...], env, cwd }}")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def shared_sources() -> dict:
|
|
113
|
+
"""third_party/sources.toml: the checkouts several environments share, by name."""
|
|
114
|
+
with SOURCES_FILE.open("rb") as handle:
|
|
115
|
+
return tomllib.load(handle)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def mode(deploy=None) -> str:
|
|
119
|
+
""""own" (each service and benchmark in an environment of its own, installed on first
|
|
120
|
+
use) or "project" (everything in the harness's own interpreter): $HARNESS_ENVIRONMENTS,
|
|
121
|
+
else the deploy's `environments`, else own."""
|
|
122
|
+
value = os.environ.get(ENVIRONMENTS_ENV) or getattr(deploy, "environments", None) or "own"
|
|
123
|
+
if value not in ("own", "project"):
|
|
124
|
+
raise ValueError(f"environments must be own or project, not {value!r}")
|
|
125
|
+
return value
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# -- sources ---------------------------------------------------------------------------------
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True)
|
|
131
|
+
class Source:
|
|
132
|
+
"""A checkout: where it is, the URL and commit, and its nested checkouts (path ->
|
|
133
|
+
commit). `managed`: the harness checks it out (under HOME/sources); otherwise it is the
|
|
134
|
+
user's (a deploy's or $HARNESS_<NAME>_ROOT, or a checkout's third_party/) and is used
|
|
135
|
+
as it is."""
|
|
136
|
+
|
|
137
|
+
name: str
|
|
138
|
+
path: Path
|
|
139
|
+
url: str
|
|
140
|
+
rev: str
|
|
141
|
+
nested: tuple = ()
|
|
142
|
+
managed: bool = True
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def source(entry: dict, deploy=None) -> Source:
|
|
146
|
+
"""The checkout of one `sources` entry (or a name of sources.toml)."""
|
|
147
|
+
if isinstance(entry, str):
|
|
148
|
+
entry = {"source": entry}
|
|
149
|
+
if entry.get("repo"):
|
|
150
|
+
url, rev = entry["repo"], entry["rev"]
|
|
151
|
+
name = url.rstrip("/").rsplit("/", 1)[-1].removesuffix(".git")
|
|
152
|
+
return Source(name=name, path=home() / "sources" / f"{name}-{rev[:12]}", url=url, rev=rev)
|
|
153
|
+
name = entry["source"]
|
|
154
|
+
known = shared_sources()[name]
|
|
155
|
+
folder = Path(known.get("path", name)).name
|
|
156
|
+
wanted = set(entry.get("nested", ()))
|
|
157
|
+
nested = tuple((path, rev) for path, rev in known.get("nested", {}).items() if path in wanted)
|
|
158
|
+
path = Path(getattr(deploy, name)) if name == "robodojo" and deploy is not None else third_party(folder)
|
|
159
|
+
managed = path == home() / "sources" / folder
|
|
160
|
+
return Source(name=name, path=path, url=known["url"], rev=known["revision"], nested=nested, managed=managed)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _git(*args, cwd=None, log=None):
|
|
164
|
+
# Large files in git LFS (images of READMEs, sample data) are not code: never fetched.
|
|
165
|
+
base = service_environment(log=(lambda line: log.write(line + "\n")) if log else None)
|
|
166
|
+
result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, encoding="utf-8", errors="replace",
|
|
167
|
+
env={**base, "GIT_LFS_SKIP_SMUDGE": "1"})
|
|
168
|
+
if log:
|
|
169
|
+
log.write(f"$ git {' '.join(args)}\n{result.stdout}{result.stderr}")
|
|
170
|
+
if result.returncode != 0:
|
|
171
|
+
raise RuntimeError(f"git {' '.join(args)} failed: {result.stderr.strip()[-500:]}")
|
|
172
|
+
return result.stdout.strip()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _fetch(path: Path, url: str, rev: str, log) -> None:
|
|
176
|
+
"""Make `path` a git checkout of `url` at `rev` (fetching only that commit)."""
|
|
177
|
+
if not (path / ".git").exists():
|
|
178
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
179
|
+
_git("init", "-q", cwd=path, log=log)
|
|
180
|
+
_git("remote", "add", "origin", url, cwd=path, log=log)
|
|
181
|
+
else:
|
|
182
|
+
with contextlib.suppress(RuntimeError):
|
|
183
|
+
if _git("rev-parse", "HEAD", cwd=path, log=log) == rev:
|
|
184
|
+
return
|
|
185
|
+
for attempt in range(4):
|
|
186
|
+
try:
|
|
187
|
+
_git("fetch", "-q", "--depth", "1", "origin", rev, cwd=path, log=log)
|
|
188
|
+
break
|
|
189
|
+
except RuntimeError:
|
|
190
|
+
if attempt == 3:
|
|
191
|
+
raise
|
|
192
|
+
time.sleep(10 * (attempt + 1)) # a dropped connection: try again
|
|
193
|
+
_git("-c", "advice.detachedHead=false", "checkout", "-q", "--force", rev, cwd=path, log=log)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def checkout(item: Source, log) -> Path:
|
|
197
|
+
"""Check a source out at its commit (and its nested checkouts at theirs) unless it is
|
|
198
|
+
there already; a source that is not managed must exist and is used as it is."""
|
|
199
|
+
if not item.managed:
|
|
200
|
+
if not item.path.is_dir():
|
|
201
|
+
raise RuntimeError(f"{item.name}: no checkout at {item.path}")
|
|
202
|
+
return item.path
|
|
203
|
+
marker = item.path / ".harness-source.json"
|
|
204
|
+
state = {"url": item.url, "rev": item.rev, "nested": dict(item.nested)}
|
|
205
|
+
with _locked(home() / "locks" / f"source-{item.path.name}.lock"):
|
|
206
|
+
try:
|
|
207
|
+
done = json.loads(marker.read_text(encoding="utf-8"))
|
|
208
|
+
except (OSError, ValueError):
|
|
209
|
+
done = {}
|
|
210
|
+
if done.get("url") == item.url and done.get("rev") == item.rev and \
|
|
211
|
+
all(done.get("nested", {}).get(path) == rev for path, rev in item.nested):
|
|
212
|
+
return item.path
|
|
213
|
+
_fetch(item.path, item.url, item.rev, log)
|
|
214
|
+
for path, rev in item.nested:
|
|
215
|
+
url = _git("config", "-f", ".gitmodules", f"submodule.{path}.url", cwd=item.path, log=log)
|
|
216
|
+
_fetch(item.path / path, url, rev, log)
|
|
217
|
+
marker.write_text(json.dumps({**done, **state, "nested": {**done.get("nested", {}), **dict(item.nested)}}),
|
|
218
|
+
encoding="utf-8")
|
|
219
|
+
return item.path
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# -- environments ----------------------------------------------------------------------------
|
|
223
|
+
|
|
224
|
+
@dataclass
|
|
225
|
+
class Environment:
|
|
226
|
+
"""One environment as resolved: the name it is installed under, its table, the
|
|
227
|
+
sources it uses, and its directory under HOME/envs (named by its hash)."""
|
|
228
|
+
|
|
229
|
+
name: str
|
|
230
|
+
spec: dict
|
|
231
|
+
sources: list = field(default_factory=list)
|
|
232
|
+
hash: str = ""
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def directory(self) -> Path:
|
|
236
|
+
return home() / "envs" / self.name / self.hash
|
|
237
|
+
|
|
238
|
+
@property
|
|
239
|
+
def python(self) -> Path:
|
|
240
|
+
"""Its interpreter."""
|
|
241
|
+
return self.directory / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
|
242
|
+
|
|
243
|
+
def installed(self) -> dict | None:
|
|
244
|
+
"""The record of a finished install (version, hash, time), or None."""
|
|
245
|
+
try:
|
|
246
|
+
record = json.loads((self.directory / RECORD).read_text(encoding="utf-8"))
|
|
247
|
+
except (OSError, ValueError):
|
|
248
|
+
return None
|
|
249
|
+
return record if record.get("hash") == self.hash else None
|
|
250
|
+
|
|
251
|
+
def variables(self) -> dict:
|
|
252
|
+
"""Placeholders of its commands: python, env and each source's checkout by name."""
|
|
253
|
+
return {"python": str(self.python), "env": str(self.directory), "root": str(ROOT), "home": str(home()),
|
|
254
|
+
**{item.name: str(item.path) for item in self.sources}}
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _package(requirement: str) -> str:
|
|
258
|
+
"""The normalized package name of a requirement ("Foo_Bar[x]==1" -> "foo-bar")."""
|
|
259
|
+
return re.split(r"[^A-Za-z0-9._-]", requirement.strip(), maxsplit=1)[0].lower().replace("_", "-")
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _constraints(spec) -> list[str]:
|
|
263
|
+
"""The pins of an environment, without those of the packages it overrides."""
|
|
264
|
+
value = spec.get("constraints") or []
|
|
265
|
+
if isinstance(value, str):
|
|
266
|
+
# A file of the core or of a package (environment/lock/pip-freeze.txt is local-models').
|
|
267
|
+
from ..paths import find
|
|
268
|
+
|
|
269
|
+
found = find(value)
|
|
270
|
+
lines = (found[0] if found else ROOT / value).read_text(encoding="utf-8").splitlines()
|
|
271
|
+
value = [line.strip() for line in lines if line.strip() and not line.lstrip().startswith("#")]
|
|
272
|
+
overridden = {_package(item) for item in spec.get("overrides", [])}
|
|
273
|
+
return [line for line in value if _package(line) not in overridden]
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def resolve(name: str, spec, deploy=None, *, benches=None, services=None) -> Environment:
|
|
277
|
+
"""The environment of a service or benchmark called `name` with table `spec` (or the
|
|
278
|
+
name of another service's or benchmark's environment)."""
|
|
279
|
+
if isinstance(spec, str):
|
|
280
|
+
if services is not None and spec in services and services[spec].environment is not None:
|
|
281
|
+
target = services[spec]
|
|
282
|
+
return resolve(spec, target.environment, deploy, benches=benches, services=services)
|
|
283
|
+
from ..bench import bench_specs
|
|
284
|
+
|
|
285
|
+
benches = benches if benches is not None else bench_specs()
|
|
286
|
+
if spec in benches:
|
|
287
|
+
return resolve(spec, benches[spec].environment, deploy, benches=benches, services=services)
|
|
288
|
+
raise ValueError(f"{name}: environment {spec!r} names no service or benchmark with an environment")
|
|
289
|
+
validate(spec, name)
|
|
290
|
+
sources = [source(entry, deploy) for entry in spec.get("sources", [])]
|
|
291
|
+
digest = hashlib.sha256(json.dumps({
|
|
292
|
+
"spec": spec, "constraints": _constraints(spec),
|
|
293
|
+
"sources": [(item.url, item.rev, list(item.nested)) for item in sources],
|
|
294
|
+
}, sort_keys=True).encode()).hexdigest()[:12]
|
|
295
|
+
return Environment(name=name, spec=spec, sources=sources, hash=digest)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _render(value, variables):
|
|
299
|
+
if isinstance(value, str):
|
|
300
|
+
for key, item in variables.items():
|
|
301
|
+
value = value.replace("{" + key + "}", item)
|
|
302
|
+
return value
|
|
303
|
+
if isinstance(value, list):
|
|
304
|
+
return [_render(item, variables) for item in value]
|
|
305
|
+
if isinstance(value, dict):
|
|
306
|
+
return {key: _render(item, variables) for key, item in value.items()}
|
|
307
|
+
return value
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _interpreter(version: str) -> str | None:
|
|
311
|
+
"""A Python of `version` on this machine (without uv): this one, or python<version>."""
|
|
312
|
+
if f"{sys.version_info[0]}.{sys.version_info[1]}" == version:
|
|
313
|
+
return sys.executable
|
|
314
|
+
return shutil.which(f"python{version}")
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
@contextlib.contextmanager
|
|
318
|
+
def _locked(path: Path):
|
|
319
|
+
"""One install of an environment at a time on this machine (Windows too)."""
|
|
320
|
+
with file_lock(path):
|
|
321
|
+
yield
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def install(environment: Environment, *, log=print) -> Environment:
|
|
325
|
+
"""Install `environment` unless it is installed: check out its sources, create the
|
|
326
|
+
virtual environment, install the requirements and sources under its constraints, run
|
|
327
|
+
its commands and check its imports, then write its record. The output goes to
|
|
328
|
+
HOME/logs/install-<name>.log; a failure raises RuntimeError naming it."""
|
|
329
|
+
if environment.installed():
|
|
330
|
+
return environment
|
|
331
|
+
logs = home() / "logs"
|
|
332
|
+
logs.mkdir(parents=True, exist_ok=True)
|
|
333
|
+
path = logs / f"install-{environment.name}.log"
|
|
334
|
+
with _locked(home() / "locks" / f"env-{environment.name}.lock"), open(path, "a", encoding="utf-8") as out:
|
|
335
|
+
if environment.installed():
|
|
336
|
+
return environment
|
|
337
|
+
out.write(f"\n=== {time.strftime('%F %T')} {environment.name} {environment.hash}\n")
|
|
338
|
+
out.flush()
|
|
339
|
+
started = time.monotonic()
|
|
340
|
+
log(f"{environment.name}: installing its environment ({environment.directory}); log {path}")
|
|
341
|
+
try:
|
|
342
|
+
_install(environment, out)
|
|
343
|
+
except Exception as error:
|
|
344
|
+
out.write(f"FAILED: {error}\n")
|
|
345
|
+
raise RuntimeError(f"{environment.name}: its environment did not install ({error}); see {path}") \
|
|
346
|
+
from None
|
|
347
|
+
log(f"{environment.name}: environment installed in {formats.duration(time.monotonic() - started)}")
|
|
348
|
+
return environment
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def on_first_use(environment: Environment, what: str, *, log=print) -> Environment:
|
|
352
|
+
"""A frontend's environment (the terminal chat's prompt_toolkit and rich, pywebview, Playwright), installed when first needed with one line to `log`."""
|
|
353
|
+
if environment.installed() is None:
|
|
354
|
+
log(f"harness: installing {what} once, into {environment.directory} (log: {home() / 'logs' / f'install-{environment.name}.log'})")
|
|
355
|
+
install(environment, log=lambda message: None)
|
|
356
|
+
return environment
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _install(environment: Environment, out) -> None:
|
|
360
|
+
spec = environment.spec
|
|
361
|
+
directory = environment.directory
|
|
362
|
+
if directory.exists():
|
|
363
|
+
shutil.rmtree(directory) # an install that did not finish
|
|
364
|
+
directory.parent.mkdir(parents=True, exist_ok=True)
|
|
365
|
+
for item in environment.sources:
|
|
366
|
+
checkout(item, out)
|
|
367
|
+
variables = environment.variables()
|
|
368
|
+
base = service_environment(log=lambda line: out.write(line + "\n"))
|
|
369
|
+
run_env = {**base, **_render(spec.get("env", {}), variables), "VIRTUAL_ENV": str(directory),
|
|
370
|
+
"PATH": os.pathsep.join([str(environment.python.parent), base.get("PATH", "")])}
|
|
371
|
+
# The new environment, not the one the harness runs in (Isaac Lab's installer, for
|
|
372
|
+
# one, prefers an active conda environment).
|
|
373
|
+
for name in ("PYTHONPATH", "PYTHONHOME", "CONDA_PREFIX", "CONDA_DEFAULT_ENV"):
|
|
374
|
+
run_env.pop(name, None)
|
|
375
|
+
if spec.get("indexes"):
|
|
376
|
+
run_env["PIP_EXTRA_INDEX_URL"] = " ".join(spec["indexes"])
|
|
377
|
+
# Wheels of a GB (torch, Isaac Sim) on a slow link outlast uv's 30 s default.
|
|
378
|
+
run_env.setdefault("UV_HTTP_TIMEOUT", "300")
|
|
379
|
+
|
|
380
|
+
def run(argv, cwd=None, env=None):
|
|
381
|
+
argv = [str(item) for item in argv]
|
|
382
|
+
out.write(f"$ {' '.join(argv)}\n")
|
|
383
|
+
out.flush()
|
|
384
|
+
result = subprocess.run(argv, cwd=cwd, env=env or run_env, stdout=out, stderr=subprocess.STDOUT)
|
|
385
|
+
if result.returncode != 0:
|
|
386
|
+
raise RuntimeError(f"{argv[0]} {' '.join(argv[1:4])} ... exited with {result.returncode}")
|
|
387
|
+
|
|
388
|
+
uv = shutil.which("uv")
|
|
389
|
+
version = spec["python"]
|
|
390
|
+
if uv:
|
|
391
|
+
run([uv, "venv", "--seed", "--python", version, directory])
|
|
392
|
+
else:
|
|
393
|
+
python = _interpreter(version)
|
|
394
|
+
if python is None:
|
|
395
|
+
raise RuntimeError(f"no Python {version} here: install uv (it fetches Python versions) or "
|
|
396
|
+
f"python{version}")
|
|
397
|
+
run([python, "-m", "venv", directory])
|
|
398
|
+
constraints = directory / "constraints.txt"
|
|
399
|
+
constraints.write_text("\n".join(_constraints(spec)) + "\n", encoding="utf-8")
|
|
400
|
+
run_env["PIP_CONSTRAINT"] = str(constraints)
|
|
401
|
+
run_env["UV_CONSTRAINT"] = str(constraints)
|
|
402
|
+
|
|
403
|
+
overrides = directory / "overrides.txt"
|
|
404
|
+
overrides.write_text("".join(f"{line}\n" for line in spec.get("overrides", [])), encoding="utf-8")
|
|
405
|
+
run_env["UV_OVERRIDE"] = str(overrides) # for the commands' own uv installs too (Isaac Lab's)
|
|
406
|
+
|
|
407
|
+
def pip(*arguments):
|
|
408
|
+
indexes = [flag for url in spec.get("indexes", []) for flag in ("--extra-index-url", url)]
|
|
409
|
+
if uv:
|
|
410
|
+
argv = [uv, "pip", "install", "--python", environment.python, "--index-strategy", "unsafe-best-match",
|
|
411
|
+
"--override", overrides, *indexes, *arguments]
|
|
412
|
+
else:
|
|
413
|
+
argv = [environment.python, "-m", "pip", "install", "--disable-pip-version-check", *indexes, *arguments]
|
|
414
|
+
# An index that times out once in a while should not cost the whole install:
|
|
415
|
+
# the same installs again (what is in place stays), twice.
|
|
416
|
+
for attempt in range(3):
|
|
417
|
+
try:
|
|
418
|
+
return run(argv)
|
|
419
|
+
except RuntimeError:
|
|
420
|
+
if attempt == 2:
|
|
421
|
+
raise
|
|
422
|
+
out.write("(again in 30 s)\n")
|
|
423
|
+
time.sleep(30)
|
|
424
|
+
|
|
425
|
+
if spec.get("requirements"):
|
|
426
|
+
# "-r FILE" names a requirements file (of a source: {robodojo}/...).
|
|
427
|
+
pip(*[part for item in _render(spec["requirements"], variables)
|
|
428
|
+
for part in (item.split(None, 1) if item.startswith("-r ") else [item])])
|
|
429
|
+
for entry, item in zip(spec.get("sources", []), environment.sources):
|
|
430
|
+
kind = entry.get("install", "none")
|
|
431
|
+
if kind == "none":
|
|
432
|
+
continue
|
|
433
|
+
target = item.path / entry["subdirectory"] if entry.get("subdirectory") else item.path
|
|
434
|
+
pip(*(["--no-deps"] if entry.get("no_deps") else []), *entry.get("flags", []),
|
|
435
|
+
*(["-e"] if kind == "editable" else []), target)
|
|
436
|
+
for command in spec.get("commands", []):
|
|
437
|
+
if isinstance(command, dict):
|
|
438
|
+
run(_render(command["run"], variables), cwd=_render(command.get("cwd"), variables),
|
|
439
|
+
env={**run_env, **_render(command.get("env", {}), variables)})
|
|
440
|
+
else:
|
|
441
|
+
run(_render(command, variables))
|
|
442
|
+
if spec.get("overrides"):
|
|
443
|
+
pip("--no-deps", *spec["overrides"])
|
|
444
|
+
if spec.get("imports"):
|
|
445
|
+
run([environment.python, "-c", "; ".join(f"import {name}" for name in spec["imports"])])
|
|
446
|
+
(directory / RECORD).write_text(json.dumps({
|
|
447
|
+
"name": environment.name, "hash": environment.hash, "python": version, "installed": time.time(),
|
|
448
|
+
"sources": {item.name: item.rev for item in environment.sources}, "spec": spec}, indent=1), encoding="utf-8")
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Files fetched by their hash: a download or copy that is kept only when its hash is the
|
|
2
|
+
expected one, and the hash of a file read in blocks (never whole into memory).
|
|
3
|
+
|
|
4
|
+
digest(path, algorithm="sha256") -> hex
|
|
5
|
+
fetch_verified(source, target, expected, algorithm="sha256", log=None) -> target
|
|
6
|
+
|
|
7
|
+
`source` is a URL (http, https, file) or a local path. The bytes go to `<target>.part`,
|
|
8
|
+
hashed as they are written, and take the target's name only when the hash matches; a
|
|
9
|
+
target already there with the expected hash is not fetched again.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import os
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from ..core.http import open_url
|
|
17
|
+
|
|
18
|
+
BLOCK = 1 << 20
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def digest(path, algorithm: str = "sha256") -> str:
|
|
22
|
+
hasher = hashlib.new(algorithm)
|
|
23
|
+
with open(path, "rb") as handle:
|
|
24
|
+
for block in iter(lambda: handle.read(BLOCK), b""):
|
|
25
|
+
hasher.update(block)
|
|
26
|
+
return hasher.hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def fetch_verified(source, target, expected: str | None, algorithm: str = "sha256", log=None,
|
|
30
|
+
timeout: float = 120.0) -> Path:
|
|
31
|
+
"""`target` with the bytes of `source`, checked against `expected` (any hash when it is
|
|
32
|
+
None). A download with another hash is removed and raises ValueError."""
|
|
33
|
+
target = Path(target)
|
|
34
|
+
if expected and target.is_file() and digest(target, algorithm) == expected:
|
|
35
|
+
return target
|
|
36
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
37
|
+
partial = target.with_name(target.name + ".part")
|
|
38
|
+
hasher = hashlib.new(algorithm)
|
|
39
|
+
url = str(source).startswith(("http://", "https://", "file:"))
|
|
40
|
+
if url and log is not None:
|
|
41
|
+
log(f"downloading {source}")
|
|
42
|
+
try:
|
|
43
|
+
with (open_url(str(source), timeout=timeout) if url else open(source, "rb")) as response, \
|
|
44
|
+
partial.open("wb") as handle:
|
|
45
|
+
for block in iter(lambda: response.read(BLOCK), b""):
|
|
46
|
+
hasher.update(block)
|
|
47
|
+
handle.write(block)
|
|
48
|
+
except BaseException:
|
|
49
|
+
partial.unlink(missing_ok=True)
|
|
50
|
+
raise
|
|
51
|
+
got = hasher.hexdigest()
|
|
52
|
+
if expected and got != expected:
|
|
53
|
+
partial.unlink()
|
|
54
|
+
raise ValueError(f"{source}: {algorithm} {got} is not the pinned {expected}")
|
|
55
|
+
os.replace(partial, target)
|
|
56
|
+
return target
|
|
57
|
+
|