@lmmslab/harness 0.0.0-stage → 0.81.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +124 -2
  3. package/bin/harness.js +71 -0
  4. package/package.json +34 -4
  5. package/python/harness/README.md +72 -0
  6. package/python/harness/__init__.py +8 -0
  7. package/python/harness/__main__.py +3 -0
  8. package/python/harness/bench.py +124 -0
  9. package/python/harness/builtin/benches/README.md +5 -0
  10. package/python/harness/builtin/benches/none/bench.toml +6 -0
  11. package/python/harness/builtin/configs/README.md +26 -0
  12. package/python/harness/builtin/configs/assistant.toml +11 -0
  13. package/python/harness/builtin/configs/default.toml +33 -0
  14. package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
  15. package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
  16. package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
  17. package/python/harness/builtin/configs/models/grok-low.toml +8 -0
  18. package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
  19. package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
  20. package/python/harness/builtin/environment/README.md +53 -0
  21. package/python/harness/builtin/environment/constraints.txt +31 -0
  22. package/python/harness/builtin/model-settings.js +143 -0
  23. package/python/harness/builtin/prompts/conversation/core.md +5 -0
  24. package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
  25. package/python/harness/builtin/prompts/core/core.md +88 -0
  26. package/python/harness/builtin/prompts/core/pack.toml +9 -0
  27. package/python/harness/builtin/prompts/reflect/core.md +45 -0
  28. package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
  29. package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
  30. package/python/harness/builtin/prompts/shared/agents.md +14 -0
  31. package/python/harness/builtin/prompts/shared/checklist.md +21 -0
  32. package/python/harness/builtin/prompts/shared/entries.md +86 -0
  33. package/python/harness/builtin/prompts/shared/environments.md +21 -0
  34. package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
  35. package/python/harness/builtin/prompts/shared/learning.md +27 -0
  36. package/python/harness/builtin/prompts/shared/mcp.md +9 -0
  37. package/python/harness/builtin/prompts/shared/world.md +1 -0
  38. package/python/harness/builtin/skills/README.md +10 -0
  39. package/python/harness/builtin/skills/agents/__init__.py +88 -0
  40. package/python/harness/builtin/skills/agents/skill.toml +11 -0
  41. package/python/harness/builtin/skills/checklist/__init__.py +32 -0
  42. package/python/harness/builtin/skills/checklist/skill.toml +9 -0
  43. package/python/harness/builtin/skills/conversation/__init__.py +23 -0
  44. package/python/harness/builtin/skills/conversation/skill.toml +8 -0
  45. package/python/harness/builtin/skills/entries/__init__.py +62 -0
  46. package/python/harness/builtin/skills/entries/skill.toml +8 -0
  47. package/python/harness/builtin/skills/environments/__init__.py +69 -0
  48. package/python/harness/builtin/skills/environments/skill.toml +9 -0
  49. package/python/harness/builtin/skills/experience/__init__.py +27 -0
  50. package/python/harness/builtin/skills/experience/skill.toml +9 -0
  51. package/python/harness/builtin/skills/instructions/__init__.py +29 -0
  52. package/python/harness/builtin/skills/instructions/skill.toml +9 -0
  53. package/python/harness/builtin/skills/learning/__init__.py +189 -0
  54. package/python/harness/builtin/skills/learning/skill.toml +7 -0
  55. package/python/harness/builtin/skills/mcp/__init__.py +54 -0
  56. package/python/harness/builtin/skills/mcp/skill.toml +10 -0
  57. package/python/harness/builtin/skills/memory/__init__.py +130 -0
  58. package/python/harness/builtin/skills/memory/skill.toml +13 -0
  59. package/python/harness/builtin/skills/verify/__init__.py +31 -0
  60. package/python/harness/builtin/skills/verify/skill.toml +7 -0
  61. package/python/harness/builtin/third_party/sources.toml +40 -0
  62. package/python/harness/cli.py +1182 -0
  63. package/python/harness/config/__init__.py +224 -0
  64. package/python/harness/config/agent.py +777 -0
  65. package/python/harness/core/__init__.py +52 -0
  66. package/python/harness/core/artifacts.py +46 -0
  67. package/python/harness/core/codec.py +160 -0
  68. package/python/harness/core/contracts.py +75 -0
  69. package/python/harness/core/episodes.py +150 -0
  70. package/python/harness/core/format.py +51 -0
  71. package/python/harness/core/http.py +124 -0
  72. package/python/harness/core/lanes.py +324 -0
  73. package/python/harness/core/linux_sandbox.py +55 -0
  74. package/python/harness/core/media.py +96 -0
  75. package/python/harness/core/memory.py +147 -0
  76. package/python/harness/core/ports.py +63 -0
  77. package/python/harness/core/procs.py +227 -0
  78. package/python/harness/core/remote.py +102 -0
  79. package/python/harness/core/spans.py +166 -0
  80. package/python/harness/core/trace.py +139 -0
  81. package/python/harness/core/wait.py +12 -0
  82. package/python/harness/core/worker.py +388 -0
  83. package/python/harness/core/worker_pipe.py +119 -0
  84. package/python/harness/embodiments.py +217 -0
  85. package/python/harness/extensions.py +112 -0
  86. package/python/harness/host/__init__.py +397 -0
  87. package/python/harness/host/client.py +201 -0
  88. package/python/harness/host/http.py +720 -0
  89. package/python/harness/host/listing.py +68 -0
  90. package/python/harness/host/server.py +189 -0
  91. package/python/harness/host/sessions.py +406 -0
  92. package/python/harness/host/tasks.py +1308 -0
  93. package/python/harness/host/workers.py +257 -0
  94. package/python/harness/host/workspaces.py +256 -0
  95. package/python/harness/model_list.py +117 -0
  96. package/python/harness/packages/__init__.py +73 -0
  97. package/python/harness/packages/addons.py +121 -0
  98. package/python/harness/packages/archive.py +122 -0
  99. package/python/harness/packages/describe.py +46 -0
  100. package/python/harness/packages/environments.py +448 -0
  101. package/python/harness/packages/fetch.py +57 -0
  102. package/python/harness/packages/manager.py +587 -0
  103. package/python/harness/packages/manifest.py +221 -0
  104. package/python/harness/packages/migrate.py +68 -0
  105. package/python/harness/packages/provision.py +38 -0
  106. package/python/harness/packages/registry.py +275 -0
  107. package/python/harness/packages/resolve.py +89 -0
  108. package/python/harness/packages/sources.py +141 -0
  109. package/python/harness/packages/state.py +213 -0
  110. package/python/harness/packages/versions.py +112 -0
  111. package/python/harness/paths.py +218 -0
  112. package/python/harness/providers.py +303 -0
  113. package/python/harness/runtime/__init__.py +8 -0
  114. package/python/harness/runtime/decision/__init__.py +1 -0
  115. package/python/harness/runtime/decision/loop.py +198 -0
  116. package/python/harness/runtime/decision/messages.py +271 -0
  117. package/python/harness/runtime/decision/model.py +424 -0
  118. package/python/harness/runtime/decision/prompts.py +248 -0
  119. package/python/harness/runtime/decision/protocol.py +169 -0
  120. package/python/harness/runtime/decision/thoughts.py +199 -0
  121. package/python/harness/runtime/events.py +213 -0
  122. package/python/harness/runtime/execution/__init__.py +1 -0
  123. package/python/harness/runtime/execution/attached.py +96 -0
  124. package/python/harness/runtime/execution/control.py +211 -0
  125. package/python/harness/runtime/execution/entries.py +665 -0
  126. package/python/harness/runtime/execution/kernel.py +811 -0
  127. package/python/harness/runtime/execution/noenv.py +84 -0
  128. package/python/harness/runtime/execution/results.py +40 -0
  129. package/python/harness/runtime/execution/session.py +871 -0
  130. package/python/harness/runtime/execution/skills.py +436 -0
  131. package/python/harness/runtime/learning/__init__.py +1 -0
  132. package/python/harness/runtime/learning/active.py +314 -0
  133. package/python/harness/runtime/learning/experience.py +48 -0
  134. package/python/harness/runtime/learning/fixtures.py +167 -0
  135. package/python/harness/runtime/learning/learner.py +651 -0
  136. package/python/harness/runtime/learning/library.py +504 -0
  137. package/python/harness/runtime/learning/reflect.py +151 -0
  138. package/python/harness/runtime/learning/usage.py +30 -0
  139. package/python/harness/runtime/learning/views.py +89 -0
  140. package/python/harness/runtime/reactive/__init__.py +1 -0
  141. package/python/harness/runtime/reactive/rules.py +549 -0
  142. package/python/harness/runtime/safety/__init__.py +1 -0
  143. package/python/harness/runtime/safety/approval.py +131 -0
  144. package/python/harness/runtime/safety/input.py +62 -0
  145. package/python/harness/runtime/safety/isolation.py +419 -0
  146. package/python/harness/runtime/safety/shield.py +129 -0
  147. package/python/harness/runtime/status.py +23 -0
  148. package/python/harness/runtime/task/__init__.py +7 -0
  149. package/python/harness/runtime/task/context.py +431 -0
  150. package/python/harness/runtime/task/delegation.py +468 -0
  151. package/python/harness/runtime/task/environments.py +268 -0
  152. package/python/harness/runtime/task/run.py +2878 -0
  153. package/python/harness/runtime/tick.py +909 -0
  154. package/python/harness/sdk/__init__.py +281 -0
  155. package/python/harness/services/__init__.py +6 -0
  156. package/python/harness/services/broker.py +35 -0
  157. package/python/harness/services/connectors.py +88 -0
  158. package/python/harness/services/mcp.py +298 -0
  159. package/python/harness/services/mcp_stdio.py +131 -0
  160. package/python/harness/services/placement.py +82 -0
  161. package/python/harness/services/registry.py +156 -0
  162. package/python/harness/settings.py +536 -0
  163. package/python/harness/terminal.py +79 -0
  164. package/python/harness/transcript.py +297 -0
  165. package/python/harness/tui/__init__.py +68 -0
  166. package/python/harness/tui/chat.py +1181 -0
  167. package/python/harness/tui/pty.py +81 -0
  168. package/python/harness/tui/renderer.py +45 -0
  169. package/python/harness/tui/timeline_rows.py +58 -0
  170. package/python/harness/tui/views.py +434 -0
@@ -0,0 +1,221 @@
1
+ """package.toml: what a package is, and what it provides, found by convention.
2
+
3
+ A package is a folder with package.toml at its root:
4
+
5
+ [package]
6
+ name = "minecraft"
7
+ version = "0.1.0"
8
+ summary = "Play Minecraft Java Edition through a window of its own"
9
+ harness = ">=0.57,<2" # the harness versions it works with
10
+ depends = { game = ">=0.1,<1" } # other packages, by name and range
11
+ platforms = ["windows", "macos", "linux"]
12
+
13
+ [provides]
14
+ commands = { audit-run = "benches.minecraft.audit:main" }
15
+ host = "harness_computer.host" # what it adds to the session host (harness/extensions.py)
16
+
17
+ Everything else it provides is found in its folders and files (CONVENTION): skills/,
18
+ embodiments/, benches/, services/, prompts/, configs/profiles/, configs/experiments/, agent-skills/,
19
+ kinds.toml, settings.toml and config_keys.toml; docs/ (README, guides, CHANGELOG.md)
20
+ and tests/ go with it. [provides] may also list names of the found kinds to say
21
+ what the package is for; a listed name that is not there is an error. The registry's
22
+ index carries the result, so the harness can find the package that provides an
23
+ embodiment, a bench, a skill or a service it lacks (packages/manager.py ensure()).
24
+ """
25
+
26
+ from dataclasses import dataclass, field
27
+ from pathlib import Path
28
+ import re
29
+ import sys
30
+ import tomllib
31
+
32
+ from .versions import Version, check_range
33
+
34
+
35
+ FILE = "package.toml"
36
+ NAME = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
37
+ PLATFORMS = ("windows", "macos", "linux")
38
+ # The keys of [package].
39
+ PACKAGE_KEYS = {
40
+ "name": "the package's name: lowercase letters, digits and hyphens (its folder's name under packages/)",
41
+ "version": "its version, MAJOR.MINOR.PATCH (a published version never changes)",
42
+ "summary": "one line: what it lets the agent do",
43
+ "harness": "the harness versions it works with, a range such as \">=0.57,<2\" (default: any)",
44
+ "depends": "other packages it needs: { name = \"range\" }",
45
+ "platforms": f"where it runs, among {', '.join(PLATFORMS)} (default: all)",
46
+ "homepage": "a URL for people",
47
+ "license": "its license (SPDX), when it is not the repository's",
48
+ }
49
+ # What a package provides, and where each is found (CONVENTION) or declared ([provides]).
50
+ PROVIDES = {
51
+ "kinds": "embodiment kinds and their sessions: the tables of kinds.toml",
52
+ "embodiments": "embodiments/<name>/embodiment.toml",
53
+ "skills": "skills/<name>/skill.toml",
54
+ "benches": "benches/<name>/bench.toml",
55
+ "services": "services/<name>.toml",
56
+ "prompts": "prompts/<name>.md and prompts/<name>/",
57
+ "profiles": "configs/profiles/<name>.toml: agent configurations (an experiment's agent = \"<name>\")",
58
+ "experiments": "configs/experiments/<name>.toml",
59
+ "agent_skills": "agent-skills/<name>/SKILL.md (Agent Skills)",
60
+ "pages": "ui/static/<name>.html: the pages of the app",
61
+ "settings": "the sections of settings.toml (the app's settings)",
62
+ "config_keys": "the keys of config_keys.toml (an agent's configuration keys and what they do)",
63
+ "commands": "CLI commands, declared only: { name = \"module:function\" }",
64
+ "agents": "kinds of sub-agent that another agent runs (another harness's CLI), declared only: "
65
+ "{ name = \"module:Class\" } (runtime/task/delegation.py)",
66
+ "host": "what it adds to the session host and its frontends, declared only: a module of the package "
67
+ "(harness/extensions.py)",
68
+ }
69
+
70
+
71
+ def platform() -> str:
72
+ """This computer's platform, as PLATFORMS names it."""
73
+ return "windows" if sys.platform.startswith("win") else "macos" if sys.platform == "darwin" else "linux"
74
+
75
+
76
+ def _toml(path: Path) -> dict:
77
+ with path.open("rb") as handle:
78
+ return tomllib.load(handle)
79
+
80
+
81
+ def _names(folder: Path, pattern: str) -> list[str]:
82
+ return sorted(path.parent.name for path in folder.glob(pattern))
83
+
84
+
85
+ def discover(folder: Path) -> dict[str, list[str]]:
86
+ """What a package folder provides, by convention (PROVIDES; commands are declared)."""
87
+ folder = Path(folder)
88
+ found = {
89
+ "kinds": sorted(_toml(folder / "kinds.toml")) if (folder / "kinds.toml").is_file() else [],
90
+ "embodiments": _names(folder, "embodiments/*/embodiment.toml"),
91
+ "skills": _names(folder, "skills/*/skill.toml"),
92
+ "benches": _names(folder, "benches/*/bench.toml"),
93
+ "services": sorted(path.stem for path in folder.glob("services/*.toml")),
94
+ "prompts": sorted({path.stem for path in folder.glob("prompts/*.md")}
95
+ | {path.name for path in folder.glob("prompts/*") if path.is_dir()
96
+ and not path.name.startswith((".", "__"))}),
97
+ "profiles": sorted(path.stem for path in folder.glob("configs/profiles/*.toml")),
98
+ "experiments": sorted(path.stem for path in folder.glob("configs/experiments/*.toml")),
99
+ "agent_skills": _names(folder, "agent-skills/*/SKILL.md"),
100
+ "pages": sorted(path.stem for path in folder.glob("ui/static/*.html")),
101
+ "settings": sorted(_toml(folder / "settings.toml")) if (folder / "settings.toml").is_file() else [],
102
+ "config_keys": sorted(_toml(folder / "config_keys.toml")) if (folder / "config_keys.toml").is_file()
103
+ else [],
104
+ }
105
+ return found
106
+
107
+
108
+ @dataclass(frozen=True)
109
+ class Package:
110
+ """A package as its package.toml declares it, with what it provides (PROVIDES)."""
111
+
112
+ name: str
113
+ version: str
114
+ summary: str = ""
115
+ harness: str = ""
116
+ depends: dict = field(default_factory=dict)
117
+ platforms: tuple = PLATFORMS
118
+ homepage: str = ""
119
+ license: str = ""
120
+ provides: dict = field(default_factory=dict)
121
+ commands: dict = field(default_factory=dict)
122
+ agents: dict = field(default_factory=dict)
123
+ path: Path = field(default=None, compare=False)
124
+
125
+ def folder(self, kind: str) -> Path:
126
+ """Where the package keeps one kind of what it provides (skills/, agent-skills/...)."""
127
+ return self.path / kind.replace("_", "-")
128
+
129
+ def runs_here(self) -> bool:
130
+ return platform() in self.platforms
131
+
132
+ def index_entry(self) -> dict:
133
+ """What the registry's index says of this version (registry.py), without its file."""
134
+ return {"harness": self.harness, "depends": dict(self.depends), "platforms": list(self.platforms),
135
+ "provides": {key: list(value) for key, value in self.provides.items()}}
136
+
137
+
138
+ def load(folder) -> Package:
139
+ """The package in `folder` (its package.toml checked, its resources found); ValueError
140
+ says what is wrong."""
141
+ folder = Path(folder)
142
+ path = folder / FILE
143
+ if not path.is_file():
144
+ raise ValueError(f"{folder}: no {FILE}")
145
+ try:
146
+ data = _toml(path)
147
+ except tomllib.TOMLDecodeError as error:
148
+ raise ValueError(f"{path}: {error}") from None
149
+ unknown = set(data) - {"package", "provides"}
150
+ if unknown:
151
+ raise ValueError(f"{path}: unknown tables {sorted(unknown)} (package.toml has [package] and [provides])")
152
+ meta = data.get("package") or {}
153
+ unknown = set(meta) - set(PACKAGE_KEYS)
154
+ if unknown:
155
+ raise ValueError(f"{path}: unknown [package] keys {sorted(unknown)}")
156
+ name = str(meta.get("name", ""))
157
+ if not NAME.match(name):
158
+ raise ValueError(f"{path}: name {name!r} is not lowercase letters, digits and hyphens")
159
+ if "version" not in meta:
160
+ raise ValueError(f"{path}: no version")
161
+ version = str(Version.parse(meta["version"]))
162
+ try:
163
+ harness = check_range(meta.get("harness", ""))
164
+ depends = {str(key): check_range(value) for key, value in dict(meta.get("depends", {})).items()}
165
+ except ValueError as error:
166
+ raise ValueError(f"{path}: {error}") from None
167
+ for other in depends:
168
+ if not NAME.match(other) or other == name:
169
+ raise ValueError(f"{path}: depends names {other!r}, not another package")
170
+ platforms = tuple(meta.get("platforms", PLATFORMS))
171
+ if not platforms or set(platforms) - set(PLATFORMS):
172
+ raise ValueError(f"{path}: platforms are among {PLATFORMS}")
173
+ declared = data.get("provides") or {}
174
+ unknown = set(declared) - set(PROVIDES)
175
+ if unknown:
176
+ raise ValueError(f"{path}: unknown [provides] keys {sorted(unknown)}")
177
+ found = discover(folder)
178
+ for key, names in declared.items():
179
+ if key in ("commands", "host", "agents"):
180
+ continue
181
+ missing = sorted(set(names) - set(found[key]))
182
+ if missing:
183
+ raise ValueError(f"{path}: [provides] {key} names {missing}, which {folder.name} does not have "
184
+ f"({PROVIDES[key]})")
185
+ commands = {str(key): str(value) for key, value in dict(declared.get("commands", {})).items()}
186
+ for command, target in commands.items():
187
+ module, _, function = target.partition(":")
188
+ if not module or not function or not (folder / module.split(".")[0]).is_dir():
189
+ raise ValueError(f"{path}: command {command} is \"module:function\" with a module in the package "
190
+ f"(e.g. benches.{name.replace('-', '_')}.commands:main), not {target!r}")
191
+ agents = {str(key): str(value) for key, value in dict(declared.get("agents", {})).items()}
192
+ for kind, target in agents.items():
193
+ module, _, attribute = target.partition(":")
194
+ if not module or not attribute or not (folder / module.split(".")[0]).is_dir():
195
+ raise ValueError(f"{path}: agent {kind} is \"module:Class\" with a module in the package, not {target!r}")
196
+ host = str(declared.get("host") or "")
197
+ if host and not (folder / host.split(".")[0]).is_dir():
198
+ raise ValueError(f"{path}: host is a module of the package (e.g. harness_{name.replace('-', '_')}.host), "
199
+ f"not {host!r}")
200
+ provides = {key: tuple(found[key]) for key in found if found[key]}
201
+ if commands:
202
+ provides["commands"] = tuple(sorted(commands))
203
+ if agents:
204
+ provides["agents"] = tuple(sorted(agents))
205
+ if host:
206
+ provides["host"] = (host,)
207
+ return Package(name=name, version=version, summary=str(meta.get("summary", "")), harness=harness,
208
+ depends=depends, platforms=platforms, homepage=str(meta.get("homepage", "")),
209
+ license=str(meta.get("license", "")), provides=provides, commands=commands,
210
+ agents=agents, path=folder.resolve())
211
+
212
+
213
+ def write(folder: Path, *, name: str, version: str = "0.1.0", summary: str = "", **extra) -> Path:
214
+ """Write a package.toml (a folder of skills becoming a package of its own)."""
215
+ from ..config import dumps
216
+
217
+ package = {"name": name, "version": version, "summary": summary,
218
+ **{key: str(value) for key, value in extra.items() if value}}
219
+ path = Path(folder) / FILE
220
+ path.write_text(dumps({"package": package}), encoding="utf-8")
221
+ return path
@@ -0,0 +1,68 @@
1
+ """The first start since packages: the skills installed in the app become packages.
2
+
3
+ Before packages, the app kept the skills a user installed under <data>/addons/
4
+ (skills/ for Harness skills, agent-skills/ for Agent Skills). On the first start with
5
+ packages, each of them becomes a local package of its own name (source
6
+ "migrated:addons"), trusted, as the user installed it (a skill the user turned off in
7
+ Settings stays off). The folder stays where it was; nothing reads it any more. The
8
+ user's config.toml records that this was done (packages.addons_migrated).
9
+ """
10
+
11
+ from pathlib import Path
12
+ import shutil
13
+ import tempfile
14
+
15
+ from ..core import tree_hash
16
+ from . import archive, state
17
+ from .addons import find_skills
18
+ from .manifest import load, write
19
+
20
+
21
+ ADDONS = "addons"
22
+
23
+
24
+ def migrate(packages, *, log=lambda message: None) -> list[str]:
25
+ """Turn the app's installed skills into packages, once; returns their names."""
26
+ addons = Path(packages.data) / ADDONS
27
+ if not addons.is_dir():
28
+ return []
29
+ user = packages.user()
30
+ if user.migrated:
31
+ return []
32
+ names = []
33
+ for kind in ("skills", "agent-skills"):
34
+ folder = addons / kind
35
+ if not folder.is_dir():
36
+ continue
37
+ found, problems = find_skills(folder)
38
+ for problem in problems:
39
+ log(f"not migrated: {problem}")
40
+ for skill in found:
41
+ if skill["name"] in user.packages:
42
+ continue
43
+ with tempfile.TemporaryDirectory(prefix="harness-migrate-") as temporary:
44
+ root = Path(temporary) / skill["name"]
45
+ shutil.copytree(skill["folder"], root / kind / skill["name"],
46
+ ignore=shutil.ignore_patterns(".git", "__pycache__"))
47
+ write(root, name=skill["name"], summary=skill["summary"])
48
+ try:
49
+ package = load(root)
50
+ except ValueError as error:
51
+ log(f"not migrated: {error}")
52
+ continue
53
+ digest = tree_hash(root, archive.files(root))
54
+ target = state.store_path(package.name, package.version, digest)
55
+ if not (target / "package.toml").is_file():
56
+ # Its .deps (requirements installed for the kernel) come along.
57
+ shutil.copytree(root, target)
58
+ deps = skill["folder"] / ".deps"
59
+ if deps.is_dir():
60
+ shutil.copytree(deps, target / kind / skill["name"] / ".deps")
61
+ state.freeze(target)
62
+ user.packages[package.name] = state.Record(package.name, package.version, digest,
63
+ "migrated:addons", path=str(target))
64
+ names.append(package.name)
65
+ log(f"{package.name}: the skill installed in the app is now a package")
66
+ user.migrated = True
67
+ user.write()
68
+ return names
@@ -0,0 +1,38 @@
1
+ """Install on first use: the environment of a bench's worker and kernel when a run needs it
2
+ and this machine does not have it yet. Services (their environments and weights) are the
3
+ local-models package's (harness_local_models/provision.py); a benchmark's checkout and
4
+ assets are the eval package's (harness_eval/provision.py, `harness checkout`).
5
+
6
+ `harness install NAMES` installs a bench's environment ahead of time.
7
+
8
+ harness install sam3 simate robodojo
9
+ """
10
+
11
+ from . import environments
12
+ from ..bench import bench_specs
13
+
14
+
15
+ def bench_environment(deploy, name: str):
16
+ """The environment of bench `name`'s worker and kernel on this deploy, or None
17
+ (environments = "project", or it has none)."""
18
+ bench = bench_specs()[name]
19
+ if bench.environment is None or environments.mode(deploy) == "project":
20
+ return None
21
+ return environments.resolve(name, bench.environment, deploy)
22
+
23
+
24
+ def bench_environment_status(deploy, name: str) -> dict:
25
+ """Whether bench `name`'s environment is installed here."""
26
+ environment = bench_environment(deploy, name)
27
+ if environment is None:
28
+ return {"name": name, "what": "bench", "installed": True, "environment": "the harness's own"}
29
+ ready = environment.installed() is not None
30
+ return {"name": name, "what": "bench", "installed": ready, "environment": str(environment.directory)}
31
+
32
+
33
+ def install_bench_environment(deploy, name: str, *, log=print) -> dict:
34
+ """Install bench `name`'s environment unless it is installed."""
35
+ environment = bench_environment(deploy, name)
36
+ if environment is not None:
37
+ environments.install(environment, log=log)
38
+ return bench_environment_status(deploy, name)
@@ -0,0 +1,275 @@
1
+ """The package registry: an index of published versions and their files.
2
+
3
+ A registry is an https base or a directory on this computer, laid out as
4
+
5
+ index.json the only file that ever changes
6
+ <name>/<version>/<name>-<version>.tar.gz the package file (archive.py), never replaced
7
+ <name>/<version>/README.md, CHANGELOG.md for the site's /packages pages
8
+ core/the_harness-<version>-py3-none-any.whl pre-release builds of the harness itself
9
+ (index.json "core"), for testers:
10
+ pip install <registry>/core/<file>
11
+
12
+ index.json:
13
+
14
+ {"schema": 1, "updated": "<ISO time>",
15
+ "packages": {"<name>": {"summary": "...", "latest": "0.1.0",
16
+ "versions": {"0.1.0": {"file": "<name>/0.1.0/<name>-0.1.0.tar.gz", "sha256": "...", "size": 123,
17
+ "harness": ">=0.57,<2", "depends": {}, "platforms": [...],
18
+ "provides": {"skills": [...], ...}, "readme": "...", "changelog": "...",
19
+ "published": "<ISO time>", "yanked": false}}}}}
20
+
21
+ The official registry is https://harness.lmms-lab.com/registry (the `registry` branch of
22
+ this repository, published with the site). $HARNESS_REGISTRY (or the packages.registry
23
+ setting) names another: a mirror, a local copy for a machine without internet access,
24
+ a directory for tests or offline installs. Whichever is configured is trusted as the official one. The index is cached
25
+ under HOME/registry/ for CACHE_SECONDS and used from there when the registry cannot be
26
+ reached; a package file is checked against the index's sha256 before it is unpacked.
27
+ """
28
+
29
+ from datetime import datetime, timezone
30
+ import hashlib
31
+ import json
32
+ import os
33
+ from pathlib import Path
34
+ import shutil
35
+ import time
36
+
37
+ from ..paths import home
38
+ from . import archive
39
+ from .fetch import digest
40
+ from .versions import Version, best, latest, satisfies
41
+
42
+
43
+ OFFICIAL = "https://harness.lmms-lab.com/registry"
44
+ REGISTRY_ENV = "HARNESS_REGISTRY"
45
+ SCHEMA = 1
46
+ CACHE_SECONDS = 600
47
+ INDEX = "index.json"
48
+
49
+
50
+ class RegistryError(RuntimeError):
51
+ pass
52
+
53
+
54
+ def configured(setting: str = "") -> str:
55
+ """The registry in use: $HARNESS_REGISTRY, else the packages.registry setting, else
56
+ the official one."""
57
+ return (os.environ.get(REGISTRY_ENV) or setting or OFFICIAL).strip()
58
+
59
+
60
+ def now() -> str:
61
+ return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
62
+
63
+
64
+ class Registry:
65
+ def __init__(self, base: str | None = None):
66
+ self.base = (base or configured()).rstrip("/\\")
67
+ self.local = not self.base.startswith(("https://", "http://"))
68
+ self._index = None
69
+ key = hashlib.sha256(self.base.encode()).hexdigest()[:12]
70
+ self.cache = home() / "registry" / key
71
+
72
+ @property
73
+ def official(self) -> bool:
74
+ return self.base == OFFICIAL
75
+
76
+ def __repr__(self):
77
+ return f"Registry({self.base})"
78
+
79
+ def _read(self, relative: str, timeout: float = 60.0) -> bytes:
80
+ if self.local:
81
+ path = Path(os.path.expanduser(self.base)) / relative
82
+ if not path.is_file():
83
+ raise RegistryError(f"{path}: not there")
84
+ return path.read_bytes()
85
+ from urllib.error import URLError
86
+ from ..core.http import open_url
87
+
88
+ try:
89
+ with open_url(f"{self.base}/{relative}", timeout=timeout) as response:
90
+ return response.read()
91
+ except (URLError, OSError) as error:
92
+ raise RegistryError(f"{self.base}/{relative}: {error}") from None
93
+
94
+ def index(self, *, refresh: bool = False) -> dict:
95
+ """The registry's index (from the cache while it is fresh, and when the registry
96
+ cannot be reached)."""
97
+ if self._index is not None and not refresh:
98
+ return self._index
99
+ cached = self.cache / INDEX
100
+ fresh = cached.is_file() and time.time() - cached.stat().st_mtime < CACHE_SECONDS
101
+ if self.local or refresh or not fresh:
102
+ try:
103
+ data = json.loads(self._read(INDEX, timeout=20.0))
104
+ check_index(data, self.base)
105
+ if not self.local:
106
+ self.cache.mkdir(parents=True, exist_ok=True)
107
+ cached.write_text(json.dumps(data), encoding="utf-8")
108
+ self._index = data
109
+ return data
110
+ except (RegistryError, ValueError) as error:
111
+ if self.local or not cached.is_file():
112
+ raise RegistryError(f"the registry {self.base} cannot be read: {error}") from None
113
+ self._index = json.loads(cached.read_text(encoding="utf-8"))
114
+ return self._index
115
+
116
+ def packages(self) -> dict:
117
+ return self.index().get("packages", {})
118
+
119
+ def versions(self, name: str) -> list[str]:
120
+ """The published versions of a package that are not withdrawn (yanked), lowest first."""
121
+ entry = self.packages().get(name) or {}
122
+ return sorted((version for version, row in entry.get("versions", {}).items() if not row.get("yanked")),
123
+ key=Version.parse)
124
+
125
+ def entry(self, name: str, version: str) -> dict:
126
+ try:
127
+ return self.packages()[name]["versions"][version]
128
+ except KeyError:
129
+ raise RegistryError(f"{self.base} has no {name} {version}") from None
130
+
131
+ def pick(self, name: str, *ranges) -> str:
132
+ """The highest published version of `name` in every range."""
133
+ if name not in self.packages():
134
+ raise RegistryError(f"no package {name} in {self.base}")
135
+ version = best(self.versions(name), *ranges)
136
+ if version is None:
137
+ raise RegistryError(f"no version of {name} in {' and '.join(r or 'any' for r in ranges)} "
138
+ f"(published: {', '.join(self.versions(name)) or 'none'})")
139
+ return version
140
+
141
+ def download(self, name: str, version: str, into: Path, *, log=print) -> tuple[Path, str]:
142
+ """Unpack one version's package file into `into`, checked against the index;
143
+ returns its folder and sha256. A file once downloaded is kept by its sha256."""
144
+ row = self.entry(name, version)
145
+ expected = row["sha256"]
146
+ kept = home() / "downloads" / f"{expected}.tar.gz"
147
+ if kept.is_file() and digest(kept) == expected:
148
+ data = kept.read_bytes()
149
+ else:
150
+ log(f"downloading {name} {version}")
151
+ data = self._read(row["file"], timeout=300.0)
152
+ if hashlib.sha256(data).hexdigest() != expected:
153
+ raise RegistryError(f"{name} {version}: the file's sha256 is not the index's; not installed")
154
+ kept.parent.mkdir(parents=True, exist_ok=True)
155
+ kept.write_bytes(data)
156
+ return archive.unpack(data, Path(into)), expected
157
+
158
+ def providers(self, what: str, name: str) -> list[str]:
159
+ """The packages whose latest version provides `name` among its `what` (skills,
160
+ embodiments, benches, services, kinds, commands...)."""
161
+ out = []
162
+ for package, entry in sorted(self.packages().items()):
163
+ versions = self.versions(package)
164
+ top = latest(versions)
165
+ if top and name in entry["versions"][top].get("provides", {}).get(what, []):
166
+ out.append(package)
167
+ return out
168
+
169
+ def search(self, query: str = "") -> list[dict]:
170
+ """The packages whose name, summary or provided names have `query` in them."""
171
+ query = query.lower().strip()
172
+ rows = []
173
+ for name, entry in sorted(self.packages().items()):
174
+ top = latest(self.versions(name))
175
+ if top is None:
176
+ continue
177
+ provides = entry["versions"][top].get("provides", {})
178
+ words = " ".join([name, entry.get("summary", ""), *[" ".join(v) for v in provides.values()]]).lower()
179
+ if query in words:
180
+ rows.append({"name": name, "version": top, "summary": entry.get("summary", ""),
181
+ "provides": provides, "platforms": entry["versions"][top].get("platforms", [])})
182
+ return rows
183
+
184
+
185
+ def check_index(data, where: str = "index.json") -> None:
186
+ """ValueError when `data` is not an index of SCHEMA."""
187
+ if not isinstance(data, dict) or data.get("schema") != SCHEMA or not isinstance(data.get("packages"), dict):
188
+ raise ValueError(f"{where}: not a registry index of schema {SCHEMA}")
189
+ for name, entry in data["packages"].items():
190
+ for version, row in (entry.get("versions") or {}).items():
191
+ Version.parse(version)
192
+ for key in ("file", "sha256"):
193
+ if not row.get(key):
194
+ raise ValueError(f"{where}: {name} {version} has no {key}")
195
+
196
+
197
+ def harness_fits(row: dict, version: str) -> bool:
198
+ """Whether a published version works with harness `version` (its `harness` range)."""
199
+ return satisfies(_release(version), row.get("harness", ""))
200
+
201
+
202
+ def _release(version: str) -> str:
203
+ # A development build (0.57.0-preview, 1.0.0-rc.1) is checked as its release.
204
+ parsed = Version.parse(version)
205
+ return f"{parsed.major}.{parsed.minor}.{parsed.patch}"
206
+
207
+
208
+ def publish_core(directory: Path, wheel: Path, *, log=print) -> dict:
209
+ """Add a build of the harness itself (its wheel) to a registry directory under core/,
210
+ with its sha256 in index.json ("core": {"latest", "versions": {version: {"file",
211
+ "sha256", "size", "published"}}}). A version already published is refused."""
212
+ import re
213
+
214
+ directory, wheel = Path(directory), Path(wheel)
215
+ match = re.match(r"^the_harness-([^-]+)-py3-none-any\.whl$", wheel.name)
216
+ if not match:
217
+ raise RegistryError(f"{wheel.name}: not a wheel of the harness (the_harness-<version>-py3-none-any.whl)")
218
+ version = match.group(1)
219
+ index_path = directory / INDEX
220
+ data = json.loads(index_path.read_text(encoding="utf-8")) if index_path.is_file() else \
221
+ {"schema": SCHEMA, "updated": now(), "packages": {}}
222
+ check_index(data, str(index_path))
223
+ core = data.setdefault("core", {"versions": {}})
224
+ if version in core["versions"]:
225
+ raise RegistryError(f"the harness {version} is published already; raise its version")
226
+ target = directory / "core" / wheel.name
227
+ target.parent.mkdir(parents=True, exist_ok=True)
228
+ shutil.copyfile(wheel, target)
229
+ row = {"file": f"core/{wheel.name}", "sha256": digest(target), "size": target.stat().st_size,
230
+ "published": now()}
231
+ core["versions"][version] = row
232
+ core["latest"] = version
233
+ data["updated"] = now()
234
+ index_path.write_text(json.dumps(data, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
235
+ log(f"published the harness {version} ({row['sha256'][:12]})")
236
+ return {"version": version, **row}
237
+
238
+
239
+ def publish(directory: Path, package_folder: Path, *, log=print) -> dict:
240
+ """Add a package folder's version to a registry directory: its package file,
241
+ README and CHANGELOG under <name>/<version>/, and its entry in index.json. A version
242
+ already published is refused (published versions never change)."""
243
+ from .manifest import load
244
+
245
+ directory = Path(directory)
246
+ package = load(package_folder)
247
+ index_path = directory / INDEX
248
+ data = json.loads(index_path.read_text(encoding="utf-8")) if index_path.is_file() else \
249
+ {"schema": SCHEMA, "updated": now(), "packages": {}}
250
+ check_index(data, str(index_path))
251
+ entry = data["packages"].setdefault(package.name, {"summary": package.summary, "versions": {}})
252
+ if package.version in entry["versions"]:
253
+ raise RegistryError(f"{package.name} {package.version} is published already; raise its version")
254
+ folder = directory / package.name / package.version
255
+ if folder.exists():
256
+ raise RegistryError(f"{folder} exists already, but the index does not list it; remove it by hand")
257
+ file = folder / f"{package.name}-{package.version}.tar.gz"
258
+ digest = archive.build(package.path, file, prefix=package.name)
259
+ row = {"file": file.relative_to(directory).as_posix(), "sha256": digest, "size": file.stat().st_size,
260
+ **package.index_entry(), "published": now(), "yanked": False}
261
+ for name, key in (("README.md", "readme"), ("CHANGELOG.md", "changelog")):
262
+ source = package.path / name
263
+ if not source.is_file():
264
+ source = package.path / "docs" / name
265
+ if source.is_file():
266
+ shutil.copyfile(source, folder / name)
267
+ row[key] = (folder / name).relative_to(directory).as_posix()
268
+ entry["versions"][package.version] = row
269
+ entry["summary"] = package.summary
270
+ entry["latest"] = latest([v for v, r in entry["versions"].items() if not r.get("yanked")]) or package.version
271
+ data["updated"] = now()
272
+ data["packages"] = dict(sorted(data["packages"].items()))
273
+ index_path.write_text(json.dumps(data, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
274
+ log(f"published {package.name} {package.version} ({digest[:12]})")
275
+ return {"name": package.name, "version": package.version, **row}