@lmmslab/harness 0.0.0-stage → 0.81.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +124 -2
  3. package/bin/harness.js +71 -0
  4. package/package.json +34 -4
  5. package/python/harness/README.md +72 -0
  6. package/python/harness/__init__.py +8 -0
  7. package/python/harness/__main__.py +3 -0
  8. package/python/harness/bench.py +124 -0
  9. package/python/harness/builtin/benches/README.md +5 -0
  10. package/python/harness/builtin/benches/none/bench.toml +6 -0
  11. package/python/harness/builtin/configs/README.md +26 -0
  12. package/python/harness/builtin/configs/assistant.toml +11 -0
  13. package/python/harness/builtin/configs/default.toml +33 -0
  14. package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
  15. package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
  16. package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
  17. package/python/harness/builtin/configs/models/grok-low.toml +8 -0
  18. package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
  19. package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
  20. package/python/harness/builtin/environment/README.md +53 -0
  21. package/python/harness/builtin/environment/constraints.txt +31 -0
  22. package/python/harness/builtin/model-settings.js +143 -0
  23. package/python/harness/builtin/prompts/conversation/core.md +5 -0
  24. package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
  25. package/python/harness/builtin/prompts/core/core.md +88 -0
  26. package/python/harness/builtin/prompts/core/pack.toml +9 -0
  27. package/python/harness/builtin/prompts/reflect/core.md +45 -0
  28. package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
  29. package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
  30. package/python/harness/builtin/prompts/shared/agents.md +14 -0
  31. package/python/harness/builtin/prompts/shared/checklist.md +21 -0
  32. package/python/harness/builtin/prompts/shared/entries.md +86 -0
  33. package/python/harness/builtin/prompts/shared/environments.md +21 -0
  34. package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
  35. package/python/harness/builtin/prompts/shared/learning.md +27 -0
  36. package/python/harness/builtin/prompts/shared/mcp.md +9 -0
  37. package/python/harness/builtin/prompts/shared/world.md +1 -0
  38. package/python/harness/builtin/skills/README.md +10 -0
  39. package/python/harness/builtin/skills/agents/__init__.py +88 -0
  40. package/python/harness/builtin/skills/agents/skill.toml +11 -0
  41. package/python/harness/builtin/skills/checklist/__init__.py +32 -0
  42. package/python/harness/builtin/skills/checklist/skill.toml +9 -0
  43. package/python/harness/builtin/skills/conversation/__init__.py +23 -0
  44. package/python/harness/builtin/skills/conversation/skill.toml +8 -0
  45. package/python/harness/builtin/skills/entries/__init__.py +62 -0
  46. package/python/harness/builtin/skills/entries/skill.toml +8 -0
  47. package/python/harness/builtin/skills/environments/__init__.py +69 -0
  48. package/python/harness/builtin/skills/environments/skill.toml +9 -0
  49. package/python/harness/builtin/skills/experience/__init__.py +27 -0
  50. package/python/harness/builtin/skills/experience/skill.toml +9 -0
  51. package/python/harness/builtin/skills/instructions/__init__.py +29 -0
  52. package/python/harness/builtin/skills/instructions/skill.toml +9 -0
  53. package/python/harness/builtin/skills/learning/__init__.py +189 -0
  54. package/python/harness/builtin/skills/learning/skill.toml +7 -0
  55. package/python/harness/builtin/skills/mcp/__init__.py +54 -0
  56. package/python/harness/builtin/skills/mcp/skill.toml +10 -0
  57. package/python/harness/builtin/skills/memory/__init__.py +130 -0
  58. package/python/harness/builtin/skills/memory/skill.toml +13 -0
  59. package/python/harness/builtin/skills/verify/__init__.py +31 -0
  60. package/python/harness/builtin/skills/verify/skill.toml +7 -0
  61. package/python/harness/builtin/third_party/sources.toml +40 -0
  62. package/python/harness/cli.py +1182 -0
  63. package/python/harness/config/__init__.py +224 -0
  64. package/python/harness/config/agent.py +777 -0
  65. package/python/harness/core/__init__.py +52 -0
  66. package/python/harness/core/artifacts.py +46 -0
  67. package/python/harness/core/codec.py +160 -0
  68. package/python/harness/core/contracts.py +75 -0
  69. package/python/harness/core/episodes.py +150 -0
  70. package/python/harness/core/format.py +51 -0
  71. package/python/harness/core/http.py +124 -0
  72. package/python/harness/core/lanes.py +324 -0
  73. package/python/harness/core/linux_sandbox.py +55 -0
  74. package/python/harness/core/media.py +96 -0
  75. package/python/harness/core/memory.py +147 -0
  76. package/python/harness/core/ports.py +63 -0
  77. package/python/harness/core/procs.py +227 -0
  78. package/python/harness/core/remote.py +102 -0
  79. package/python/harness/core/spans.py +166 -0
  80. package/python/harness/core/trace.py +139 -0
  81. package/python/harness/core/wait.py +12 -0
  82. package/python/harness/core/worker.py +388 -0
  83. package/python/harness/core/worker_pipe.py +119 -0
  84. package/python/harness/embodiments.py +217 -0
  85. package/python/harness/extensions.py +112 -0
  86. package/python/harness/host/__init__.py +397 -0
  87. package/python/harness/host/client.py +201 -0
  88. package/python/harness/host/http.py +720 -0
  89. package/python/harness/host/listing.py +68 -0
  90. package/python/harness/host/server.py +189 -0
  91. package/python/harness/host/sessions.py +406 -0
  92. package/python/harness/host/tasks.py +1356 -0
  93. package/python/harness/host/workers.py +257 -0
  94. package/python/harness/host/workspaces.py +256 -0
  95. package/python/harness/model_list.py +117 -0
  96. package/python/harness/packages/__init__.py +73 -0
  97. package/python/harness/packages/addons.py +121 -0
  98. package/python/harness/packages/archive.py +122 -0
  99. package/python/harness/packages/describe.py +46 -0
  100. package/python/harness/packages/environments.py +448 -0
  101. package/python/harness/packages/fetch.py +57 -0
  102. package/python/harness/packages/manager.py +587 -0
  103. package/python/harness/packages/manifest.py +232 -0
  104. package/python/harness/packages/migrate.py +68 -0
  105. package/python/harness/packages/provision.py +38 -0
  106. package/python/harness/packages/registry.py +275 -0
  107. package/python/harness/packages/resolve.py +89 -0
  108. package/python/harness/packages/sources.py +141 -0
  109. package/python/harness/packages/state.py +213 -0
  110. package/python/harness/packages/versions.py +112 -0
  111. package/python/harness/paths.py +218 -0
  112. package/python/harness/providers.py +303 -0
  113. package/python/harness/runtime/__init__.py +8 -0
  114. package/python/harness/runtime/decision/__init__.py +1 -0
  115. package/python/harness/runtime/decision/loop.py +198 -0
  116. package/python/harness/runtime/decision/messages.py +271 -0
  117. package/python/harness/runtime/decision/model.py +424 -0
  118. package/python/harness/runtime/decision/prompts.py +248 -0
  119. package/python/harness/runtime/decision/protocol.py +169 -0
  120. package/python/harness/runtime/decision/thoughts.py +199 -0
  121. package/python/harness/runtime/events.py +213 -0
  122. package/python/harness/runtime/execution/__init__.py +1 -0
  123. package/python/harness/runtime/execution/attached.py +96 -0
  124. package/python/harness/runtime/execution/control.py +211 -0
  125. package/python/harness/runtime/execution/entries.py +665 -0
  126. package/python/harness/runtime/execution/kernel.py +811 -0
  127. package/python/harness/runtime/execution/noenv.py +84 -0
  128. package/python/harness/runtime/execution/results.py +40 -0
  129. package/python/harness/runtime/execution/session.py +871 -0
  130. package/python/harness/runtime/execution/skills.py +436 -0
  131. package/python/harness/runtime/learning/__init__.py +1 -0
  132. package/python/harness/runtime/learning/active.py +314 -0
  133. package/python/harness/runtime/learning/experience.py +48 -0
  134. package/python/harness/runtime/learning/fixtures.py +167 -0
  135. package/python/harness/runtime/learning/learner.py +651 -0
  136. package/python/harness/runtime/learning/library.py +504 -0
  137. package/python/harness/runtime/learning/reflect.py +151 -0
  138. package/python/harness/runtime/learning/usage.py +30 -0
  139. package/python/harness/runtime/learning/views.py +89 -0
  140. package/python/harness/runtime/reactive/__init__.py +1 -0
  141. package/python/harness/runtime/reactive/rules.py +549 -0
  142. package/python/harness/runtime/safety/__init__.py +1 -0
  143. package/python/harness/runtime/safety/approval.py +131 -0
  144. package/python/harness/runtime/safety/input.py +62 -0
  145. package/python/harness/runtime/safety/isolation.py +419 -0
  146. package/python/harness/runtime/safety/shield.py +129 -0
  147. package/python/harness/runtime/status.py +23 -0
  148. package/python/harness/runtime/task/__init__.py +7 -0
  149. package/python/harness/runtime/task/context.py +431 -0
  150. package/python/harness/runtime/task/delegation.py +468 -0
  151. package/python/harness/runtime/task/environments.py +268 -0
  152. package/python/harness/runtime/task/run.py +2878 -0
  153. package/python/harness/runtime/tick.py +909 -0
  154. package/python/harness/sdk/__init__.py +281 -0
  155. package/python/harness/services/__init__.py +6 -0
  156. package/python/harness/services/broker.py +35 -0
  157. package/python/harness/services/connectors.py +88 -0
  158. package/python/harness/services/mcp.py +298 -0
  159. package/python/harness/services/mcp_stdio.py +131 -0
  160. package/python/harness/services/placement.py +82 -0
  161. package/python/harness/services/registry.py +156 -0
  162. package/python/harness/settings.py +536 -0
  163. package/python/harness/terminal.py +79 -0
  164. package/python/harness/transcript.py +297 -0
  165. package/python/harness/tui/__init__.py +68 -0
  166. package/python/harness/tui/chat.py +1181 -0
  167. package/python/harness/tui/pty.py +81 -0
  168. package/python/harness/tui/renderer.py +45 -0
  169. package/python/harness/tui/timeline_rows.py +58 -0
  170. package/python/harness/tui/views.py +434 -0
@@ -0,0 +1,504 @@
1
+ """A versioned library of what the harness has learned, for any action scenario (a robot,
2
+ a desktop, a website, a game): one learner fills it (runtime/learning/learner.py), in a run (in-context:
3
+ an entry admitted takes effect at once) and between runs (self-improvement on development
4
+ seeds), and evaluation reads it frozen.
5
+
6
+ Layout of a library directory:
7
+
8
+ notes/*.md high-level notes, appended to the system prompt
9
+ memory/*.md long-term memory entries, not in the prompt; programs find
10
+ them with recall() beside the run's own (runtime/task/context.py;
11
+ layer: notes)
12
+ skills/*.py low-level skills and handlers, executed in the kernel
13
+ namespace before the first program (layer: low)
14
+ reactive/*.py guard/controller helpers, executed only when the reactive
15
+ layer is enabled (layer: reactive)
16
+ tests/<name>.py the test of skills/<name>.py or reactive/<name>.py: Python that
17
+ uses the entry and raises when it does not work
18
+ skills/<name>.files/ an entry's attachments (a trained model's weights...)
19
+ library.toml optional: `notes_title`, the line the notes stand under in the
20
+ prompt (a library written by hand holds knowledge, not lessons)
21
+ index.json per entry: its evidence, whether it was learned actively or
22
+ passively, by whom, when, its test runs and its uses (each
23
+ execution of its code in a run that learns: memory.fold_uses;
24
+ learner.py); not part of the version
25
+ admissions.jsonl one line an admission: section, file, the sha256 of its bytes,
26
+ the version after it (what an audit checks the entries against)
27
+ candidates/ entries proposed and waiting for validation on development seeds
28
+ (not loaded, not part of the version)
29
+
30
+ Each entry is one file (with its test and attachments). The library version is the sha256
31
+ over the loaded entries' relative paths and bytes, their tests and attachments included;
32
+ every episode records it, and every admission changes it. An entry's evidence grades it
33
+ (EVIDENCE): a library loads the entries at or above its `load` level, and one that failed
34
+ never. Evaluation uses a single version, and a library may be restricted to some layers
35
+ (Tab. 8 ablation).
36
+ """
37
+
38
+ import ast
39
+ import base64
40
+ import hashlib
41
+ from functools import wraps
42
+ from graphlib import TopologicalSorter
43
+ import inspect
44
+ import json
45
+ import time
46
+ from types import FunctionType
47
+ from contextlib import contextmanager, nullcontext
48
+ from dataclasses import dataclass
49
+ from pathlib import Path
50
+ import tomllib
51
+ from weakref import WeakKeyDictionary
52
+
53
+ from harness.core import atomic_write
54
+
55
+
56
+ SECTIONS = {"notes": ".md", "memory": ".md", "skills": ".py", "reactive": ".py"}
57
+ # Library layers (Tab. 8 ablation) and the sections each one enables.
58
+ LAYER_OF = {"notes": "notes", "memory": "notes", "skills": "skills", "reactive": "reactive"}
59
+ LAYERS = ("notes", "skills", "reactive")
60
+ # What the notes stand under in the prompt, unless library.toml names another title.
61
+ NOTES_TITLE = "Notes from earlier episodes:"
62
+ # How well an entry is known to work, weakest first: proposed (no check yet), provisional
63
+ # (its own test passed in a run; a note or memory entry the model admitted), replayed (its
64
+ # test passed again at the start of a later run), validated (it raised the score on
65
+ # held-out development seeds). An entry the library holds without an index row was put
66
+ # there by hand or by self-improvement before the index: validated. A failed one never loads.
67
+ EVIDENCE = ("proposed", "provisional", "replayed", "validated")
68
+ FAILED = "failed"
69
+ INDEX = "index.json"
70
+ ADMISSIONS = "admissions.jsonl"
71
+ TESTS = "tests"
72
+ CANDIDATES = "candidates"
73
+ _code_origins = WeakKeyDictionary()
74
+ _code_active = WeakKeyDictionary()
75
+
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class Revision:
80
+ key: str
81
+ source: bytes
82
+ test: bytes | None = None
83
+ attachments: tuple = ()
84
+ dependencies: tuple = ()
85
+ bindings: tuple = ()
86
+
87
+ @property
88
+ def source_sha256(self):
89
+ return hashlib.sha256(self.source).hexdigest()
90
+
91
+ @property
92
+ def revision(self):
93
+ return hashlib.sha256(self.pack()).hexdigest()
94
+
95
+ def pack(self):
96
+ encode = lambda data: base64.b64encode(data).decode("ascii")
97
+ return json.dumps({"key": self.key, "source": encode(self.source),
98
+ "test": None if self.test is None else encode(self.test),
99
+ "attachments": [[name, encode(data)] for name, data in self.attachments],
100
+ "dependencies": list(self.dependencies),
101
+ **({"bindings": list(self.bindings)} if self.bindings else {})},
102
+ sort_keys=True, separators=(",", ":")).encode("utf-8")
103
+
104
+ @classmethod
105
+ def unpack(cls, data):
106
+ row = json.loads(data)
107
+ return cls(row["key"], base64.b64decode(row["source"]),
108
+ None if row["test"] is None else base64.b64decode(row["test"]),
109
+ tuple((name, base64.b64decode(value)) for name, value in row["attachments"]),
110
+ tuple(tuple(dependency) for dependency in row["dependencies"]),
111
+ tuple(tuple(binding) for binding in row.get("bindings", ())))
112
+
113
+ def save(self, directory):
114
+ path = Path(directory) / ".revisions" / self.revision
115
+ if not path.is_file():
116
+ atomic_write(path, self.pack())
117
+ return self.revision
118
+
119
+ @classmethod
120
+ def read(cls, directory, revision):
121
+ if len(revision) != 64 or any(char not in "0123456789abcdef" for char in revision):
122
+ raise ValueError("invalid library revision")
123
+ data = (Path(directory) / ".revisions" / revision).read_bytes()
124
+ if hashlib.sha256(data).hexdigest() != revision:
125
+ raise ValueError(f"library revision {revision} has changed")
126
+ return cls.unpack(data)
127
+
128
+
129
+ @dataclass(frozen=True)
130
+ class LibrarySnapshot:
131
+ directory: Path
132
+ layers: tuple
133
+ load: str
134
+ version: str
135
+ entries: tuple
136
+ index_json: bytes
137
+ settings: bytes
138
+
139
+ @property
140
+ def index(self):
141
+ return json.loads(self.index_json)
142
+
143
+ def manifest(self):
144
+ return {"directory": str(self.directory), "layers": list(self.layers), "version": self.version,
145
+ **({"load": self.load} if self.load != "proposed" else {}),
146
+ "entries": {section: [item.key.split("/", 1)[1] for item in self.entries
147
+ if item.key.startswith(section + "/")] for section in SECTIONS}}
148
+
149
+ def notes(self):
150
+ texts = [item.source.decode("utf-8").strip() for item in self.entries if item.key.startswith("notes/")]
151
+ title = str(tomllib.loads(self.settings.decode("utf-8")).get("notes_title") or NOTES_TITLE)
152
+ return f"\n{title}\n" + "\n".join(f"- {text}" for text in texts) + "\n" if texts else ""
153
+
154
+ def api_summary(self, active):
155
+ return active_summary(item for item in self.entries if item.key in active)
156
+
157
+
158
+ def active_summary(items):
159
+ lines = [line for item in items for line in summary_lines(item.source.decode("utf-8"))]
160
+ return "\nLibrary skills (already defined in your kernel):\n" + "\n".join(lines) + "\n" if lines else ""
161
+
162
+
163
+ def executable(key):
164
+ return key.startswith(("skills/", "reactive/"))
165
+
166
+
167
+ def eligible(item, row, layers, load):
168
+ section = item.key.split("/", 1)[0]
169
+ evidence = row.get("evidence", "validated")
170
+ if not selects(section, evidence, layers, load):
171
+ return False
172
+ if not executable(item.key) or not row or row.get("origin") == "hand":
173
+ return True
174
+ proof = row.get("proof") or {}
175
+ return (evidence in ("provisional", "replayed", "validated")
176
+ and proof.get("revision") == item.revision and bool(proof.get("conditions")))
177
+
178
+
179
+
180
+ def _verified_bindings(api, bindings):
181
+ """Pin fixture actions to the original kernel API and the tested implementation."""
182
+ from types import CodeType, MethodType
183
+
184
+ scope = {}
185
+ compiled = {}
186
+ for name, module, owner, digest in bindings:
187
+ action = api.get(name)
188
+ if not isinstance(action, MethodType) or action.__module__ != module:
189
+ raise ValueError(f"verified binding {name} is unavailable")
190
+ code = action.__func__.__code__
191
+ if code.co_filename not in compiled:
192
+ data = Path(code.co_filename).read_bytes()
193
+ compiled[code.co_filename] = (hashlib.sha256(data).hexdigest(),
194
+ compile(data, code.co_filename, "exec", dont_inherit=True))
195
+ source_digest, source = compiled[code.co_filename]
196
+ if source_digest != digest or not any(
197
+ code == method for cls in source.co_consts if isinstance(cls, CodeType) and cls.co_name == owner
198
+ for method in cls.co_consts if isinstance(method, CodeType) and method.co_name == name
199
+ ):
200
+ raise ValueError(f"verified binding {name} has a different implementation")
201
+ scope[name] = action
202
+ return scope
203
+
204
+
205
+ def stage_entries(items, namespace, active, reactive, *, api=None):
206
+ from harness.runtime.execution.entries import ClosedLoop
207
+ from harness.runtime.reactive.rules import _stage_rules
208
+ from harness.core import spans
209
+
210
+ batch = {item.key: (item, hand) for item, hand in items}
211
+ targets = {key: item.revision for key, (item, _) in batch.items()}
212
+ order = TopologicalSorter({key: [dep for dep, _ in item.dependencies if dep in batch]
213
+ for key, (item, _) in batch.items()}).static_order()
214
+ revisions = {**{key: row[0] for key, row in active.items()}, **targets}
215
+ for key, (_, _, dependencies) in active.items():
216
+ if key not in targets and any(revisions.get(dep) != required for dep, required in dependencies):
217
+ raise ValueError(f"replace {key} together with its changed dependency")
218
+ scope = dict(namespace)
219
+ owners = dict(active)
220
+ for key in targets:
221
+ for name, value in owners.get(key, (None, {}))[1].items():
222
+ if scope.get(name) is value:
223
+ del scope[name]
224
+
225
+ with _stage_rules(reactive, targets) as staged_rules:
226
+ for key in order:
227
+ item, hand = batch[key]
228
+ for key, required in item.dependencies:
229
+ if revisions.get(key) != required:
230
+ raise ValueError(f"{item.key} needs {key} at {required}")
231
+ execution = _verified_bindings(api or {}, item.bindings) if item.bindings else scope
232
+ if item.bindings:
233
+ execution["__builtins__"] = {}
234
+ before = dict(execution)
235
+ origin = item.key, item.source_sha256, item.revision
236
+ tree = ast.parse(item.source, f"<library {item.key}>")
237
+ if not hand:
238
+ for node in tree.body:
239
+ if not isinstance(node, (ast.Import, ast.ImportFrom, ast.FunctionDef, ast.ClassDef,
240
+ ast.AsyncFunctionDef, ast.Assign, ast.AnnAssign, ast.Expr)):
241
+ raise ValueError(f"{item.key} has unsupported initialization")
242
+ for child in _initialization(node):
243
+ if isinstance(child, ast.Call):
244
+ if (not isinstance(child.func, ast.Name)
245
+ or child.func.id not in ("when", "watch", "on", "rule", "reflex")):
246
+ raise ValueError(f"{item.key} must put runtime actions inside a callable")
247
+ if isinstance(child, (ast.Attribute, ast.Subscript)) and isinstance(child.ctx, ast.Store):
248
+ raise ValueError(f"{item.key} cannot modify live objects during activation")
249
+ if staged_rules is not None:
250
+ staged_rules.library_origin = origin
251
+ exec(compile(tree, f"<library {item.key}>", "exec"), execution)
252
+ exports = {name: value for name, value in execution.items() if not name.startswith("__")
253
+ and (name not in before or before[name] is not value)}
254
+ if item.bindings and exports.keys() & (api or {}).keys():
255
+ raise ValueError(f"{item.key} would replace a built-in skill")
256
+ for name, value in exports.items():
257
+ if isinstance(value, type) and issubclass(value, ClosedLoop):
258
+ _code_origins[value] = origin
259
+ _code_active[value] = active
260
+ elif isinstance(value, FunctionType) and value.__globals__ is execution:
261
+ wrapped = _tracked(value, origin, active, spans)
262
+ scope[name] = exports[name] = wrapped
263
+ scope.update(exports)
264
+ owners[item.key] = (item.revision, exports, item.dependencies)
265
+ return scope, owners, staged_rules
266
+
267
+
268
+ def _initialization(node):
269
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
270
+ fields = (*node.decorator_list, *node.args.defaults, *filter(None, node.args.kw_defaults))
271
+ return (child for part in fields for child in ast.walk(part))
272
+ if isinstance(node, ast.ClassDef):
273
+ fields = (*node.bases, *node.decorator_list, *node.keywords)
274
+ return (*[child for part in fields for child in ast.walk(part)],
275
+ *[child for part in node.body for child in _initialization(part)])
276
+ return ast.walk(node)
277
+
278
+
279
+ def _tracked(function, origin, active, spans):
280
+ @contextmanager
281
+ def execution():
282
+ if active.get(origin[0], (None,))[0] != origin[2]:
283
+ raise RuntimeError(f"library revision for {origin[0]} is no longer active")
284
+ start = time.monotonic()
285
+ try:
286
+ yield
287
+ except BaseException as error:
288
+ spans.step(function.__name__, 1, "gave up", False, time.monotonic() - start,
289
+ type(error).__name__, library=origin)
290
+ raise
291
+ else:
292
+ spans.step(function.__name__, 1, "done", True, time.monotonic() - start, library=origin)
293
+
294
+ if inspect.iscoroutinefunction(function):
295
+ @wraps(function)
296
+ async def call(*args, **kwargs):
297
+ with execution():
298
+ return await function(*args, **kwargs)
299
+ elif inspect.isasyncgenfunction(function):
300
+ @wraps(function)
301
+ async def call(*args, **kwargs):
302
+ with execution():
303
+ async for result in function(*args, **kwargs):
304
+ yield result
305
+ elif inspect.isgeneratorfunction(function):
306
+ @wraps(function)
307
+ def call(*args, **kwargs):
308
+ with execution():
309
+ return (yield from function(*args, **kwargs))
310
+ else:
311
+ @wraps(function)
312
+ def call(*args, **kwargs):
313
+ with execution():
314
+ return function(*args, **kwargs)
315
+ _code_origins[call] = origin
316
+ return call
317
+
318
+
319
+ def publish_entries(staged, namespace, active, reactive):
320
+ scope, owners, rules = staged
321
+ with reactive.library_boundary if rules is not None else nullcontext():
322
+ for key, (_, exports, _) in active.items():
323
+ if key in owners and owners[key][1] is not exports:
324
+ for name, value in exports.items():
325
+ if namespace.get(name) is value and name not in scope:
326
+ del namespace[name]
327
+ namespace.update(scope)
328
+ active.update(owners)
329
+ if rules is not None:
330
+ reactive._publish_rules(rules)
331
+
332
+
333
+ def assert_active(code):
334
+ origin = _code_origins.get(code)
335
+ active = _code_active.get(code)
336
+ if origin is not None and active is not None and active.get(origin[0], (None,))[0] != origin[2]:
337
+ raise RuntimeError(f"library revision for {origin[0]} is no longer active")
338
+
339
+
340
+ def code_origin(cls):
341
+ return _code_origins.get(cls)
342
+
343
+
344
+ def rank(evidence) -> int:
345
+ """An evidence level's place in EVIDENCE (-1 for failed or unknown)."""
346
+ return EVIDENCE.index(evidence) if evidence in EVIDENCE else -1
347
+
348
+
349
+ def selects(section, evidence, layers, load) -> bool:
350
+ """Whether this revision passes a reader's library selection."""
351
+ return LAYER_OF.get(section) in layers and rank(evidence) >= rank(load) >= 0
352
+
353
+
354
+ class Library:
355
+ def __init__(self, directory, layers=("notes", "skills", "reactive"), load="proposed"):
356
+ self.directory = Path(directory).resolve()
357
+ unknown = set(layers) - set(LAYERS)
358
+ if unknown:
359
+ raise ValueError(f"Unknown library layers: {sorted(unknown)}")
360
+ if load not in EVIDENCE:
361
+ raise ValueError(f"library load is one of {list(EVIDENCE)}, not {load!r}")
362
+ self.layers = tuple(layers)
363
+ self.load = load
364
+ if not self.directory.is_dir():
365
+ raise ValueError(f"Library directory not found: {self.directory}")
366
+
367
+ def index(self) -> dict:
368
+ """{"<section>/<file>": {"evidence", "source", "origin", ...}} (learner.py)."""
369
+ path = self.directory / INDEX
370
+ try:
371
+ data = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
372
+ except (OSError, ValueError):
373
+ data = {}
374
+ return data if isinstance(data, dict) else {}
375
+
376
+ def evidence(self, section, filename, index=None) -> str:
377
+ row = (self.index() if index is None else index).get(f"{section}/{filename}")
378
+ return row.get("evidence", "validated") if isinstance(row, dict) else "validated"
379
+
380
+ def entries(self, section):
381
+ if LAYER_OF[section] not in self.layers:
382
+ return []
383
+ folder = self.directory / section
384
+ if not folder.is_dir():
385
+ return []
386
+ index = self.index()
387
+ return sorted(path for path in folder.glob(f"*{SECTIONS[section]}")
388
+ if not (index.get(f"{section}/{path.name}") or {}).get("forgotten")
389
+ and selects(section, self.evidence(section, path.name, index), self.layers, self.load))
390
+
391
+ def test(self, section, filename) -> Path | None:
392
+ """The test of a code entry, if it has one."""
393
+ path = self.directory / TESTS / f"{Path(filename).stem}.py"
394
+ return path if section in ("skills", "reactive") and path.is_file() else None
395
+
396
+ def attachments(self, section, filename) -> list[Path]:
397
+ folder = self.directory / section / f"{Path(filename).stem}.files"
398
+ return sorted(path for path in folder.rglob("*") if path.is_file()) if folder.is_dir() else []
399
+
400
+ def capture(self, *, include_unselected=False):
401
+ """Read one index and each selected payload once."""
402
+ from harness.runtime.learning.experience import MIGRATION, legacy_entries
403
+
404
+ index = self.index()
405
+ legacy = {}
406
+ if "notes" in self.layers and not index.get(MIGRATION):
407
+ for item in list(legacy_entries(self.directory, index)):
408
+ key = item["id"]
409
+ if key not in index:
410
+ legacy[key] = Revision(key, item["text"].encode("utf-8"))
411
+ index[key] = {"memory": item, "evidence": "validated", "origin": "legacy"}
412
+ settings = self.directory / "library.toml"
413
+ raw_settings = settings.read_bytes() if settings.is_file() else b""
414
+ digest = hashlib.sha256()
415
+ if raw_settings:
416
+ digest.update(b"library.toml\0" + raw_settings + b"\0")
417
+ found = []
418
+ for section, suffix in SECTIONS.items():
419
+ folder = self.directory / section
420
+ paths = {path.name: path for path in folder.glob(f"*{suffix}")}
421
+ paths.update({key.split("/", 1)[1]: folder / key.split("/", 1)[1]
422
+ for key, row in index.items() if key.startswith(section + "/") and (row.get("revision") or key in legacy)})
423
+ for name, path in sorted(paths.items()):
424
+ key = f"{section}/{name}"
425
+ row = index.get(key) or {}
426
+ selected = selects(section, row.get("evidence", "validated"), self.layers, self.load)
427
+ if row.get("forgotten") or not selected and not include_unselected:
428
+ continue
429
+ if key in legacy:
430
+ item = legacy[key]
431
+ elif row.get("revision"):
432
+ item = Revision.read(self.directory, row["revision"])
433
+ if item.key != key:
434
+ raise ValueError(f"library index points {key} at a different entry")
435
+ else:
436
+ test = self.test(section, name)
437
+ attachments = self.attachments(section, name)
438
+ item = Revision(key, path.read_bytes(), test.read_bytes() if test else None,
439
+ tuple((part.relative_to(self.directory).as_posix(), part.read_bytes())
440
+ for part in attachments))
441
+ found.append(item)
442
+ if not selected:
443
+ continue
444
+ parts = [(item.key, item.source)]
445
+ if item.test is not None:
446
+ parts.append((f"{TESTS}/{path.stem}.py", item.test))
447
+ parts.extend(item.attachments)
448
+ for relative, data in parts:
449
+ digest.update(relative.encode() + b"\0" + data + b"\0")
450
+ if row.get("memory"):
451
+ digest.update(json.dumps(row["memory"], sort_keys=True).encode("utf-8"))
452
+ if item.dependencies:
453
+ digest.update(json.dumps(item.dependencies).encode("utf-8"))
454
+ if item.bindings:
455
+ digest.update(json.dumps(item.bindings).encode("utf-8"))
456
+ return LibrarySnapshot(self.directory, self.layers, self.load, digest.hexdigest(), tuple(found),
457
+ json.dumps(index).encode("utf-8"), raw_settings)
458
+
459
+ def version(self) -> str:
460
+ return self.capture().version
461
+
462
+ def manifest(self) -> dict:
463
+ return self.capture().manifest()
464
+
465
+ def notes(self) -> str:
466
+ return self.capture().notes()
467
+
468
+ def api_summary(self, reactive_enabled: bool) -> str:
469
+ captured = self.capture()
470
+ active = {item.key for item in captured.entries if item.key.startswith("skills/")
471
+ or reactive_enabled and item.key.startswith("reactive/")}
472
+ return captured.api_summary(active)
473
+
474
+
475
+ def summary_lines(code: str) -> list[str]:
476
+ """` name(args) first docstring line` of each public function and class of `code`
477
+ (a class: its parameters are the loop's; a closed loop is called like a function)."""
478
+
479
+ lines = []
480
+ try:
481
+ tree = ast.parse(code)
482
+ except SyntaxError:
483
+ return lines
484
+ for node in tree.body:
485
+ if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and not node.name.startswith("_"):
486
+ args = ", ".join(a.arg for a in node.args.args) if isinstance(node, ast.FunctionDef) else "..."
487
+ doc = (ast.get_docstring(node) or "").strip().splitlines()
488
+ lines.append(f" {node.name}({args})" + (f" {doc[0]}" if doc else ""))
489
+ return lines
490
+
491
+
492
+ def library_for(runtime: dict, base: Path) -> Library:
493
+ """The library a [runtime.library] table names (its path relative to `base`, or under
494
+ $HARNESS_HOME); one that learns is created empty when it is not there yet."""
495
+ from harness.runtime.learning.learner import library_path, settings
496
+
497
+ config = runtime["library"]
498
+ path = library_path(config["path"])
499
+ directory = (base / path).resolve() if not path.is_absolute() else path
500
+ chosen = settings(config)
501
+ if chosen["learn"] == "on":
502
+ for section in SECTIONS:
503
+ (directory / section).mkdir(parents=True, exist_ok=True)
504
+ return Library(directory, config.get("layers", ("notes", "skills", "reactive")), chosen["load"])
@@ -0,0 +1,151 @@
1
+ """Reflection: turn a development episode into one candidate library entry.
2
+
3
+ The model reads a compact trajectory (programs, events, outcome on the dev
4
+ seed) and the current library, and may propose one entry:
5
+
6
+ <note name="...">one or two sentences</note> -> notes/<name>.md
7
+ <memory name="...">situation and what worked</memory> -> memory/<name>.md
8
+ <skill name="...">def f(...): ...</skill> -> skills/<name>.py
9
+ <reactive name="...">def g(obs): ...</reactive> -> reactive/<name>.py
10
+ <none/>
11
+
12
+ The reply gives its kind, name and text; the learner checks them as every candidate
13
+ (runtime/learning/learner.py candidate()). A candidate only enters the library after it improves
14
+ the score on held-out development seeds (harness self-improve).
15
+ """
16
+
17
+ import ast
18
+ import json
19
+ from pathlib import Path
20
+ import re
21
+
22
+ from harness.core.episodes import new_name, read_records
23
+ from harness.paths import ROOT
24
+ from harness.runtime.learning.learner import KINDS, MEMORY_MAX_CHARS, NOTE_MAX_CHARS
25
+ from harness.runtime.learning.library import Library, eligible, executable
26
+
27
+
28
+ # The request, one paragraph per embodiment (the pack's markers: the embodiment kind, the
29
+ # bench, the run's skills). A robot's is the request of the robot harness unchanged.
30
+ PACK = ROOT / "prompts" / "reflect"
31
+
32
+
33
+ def system_prompt(names=("robot",)) -> str:
34
+ """The reflection request for a run whose embodiment kind, bench and enabled skills
35
+ are `names` (prompts/reflect): its lines joined into one paragraph."""
36
+ from harness.runtime.decision.prompts import assemble_prompt
37
+
38
+ text = assemble_prompt(PACK, list(names), features=())
39
+ paragraph = " ".join(line.strip() for line in text.splitlines() if line.strip())
40
+ return paragraph.replace("{note_max_chars}", str(NOTE_MAX_CHARS)).replace(
41
+ "{memory_max_chars}", str(MEMORY_MAX_CHARS))
42
+
43
+
44
+ _ENTRY = re.compile(rf'<({"|".join(KINDS)})\s+name="([^"]*)">(.*?)</\1>', re.DOTALL)
45
+ # An outcome's keys every worker reports; the rest is what an evaluator measured besides.
46
+ OUTCOME_KEYS = {"success", "reward", "process_score", "terminated", "truncated", "steps", "unscorable"}
47
+
48
+
49
+ def summarize_episode(run: Path, max_chars: int = 12000, *, since=0) -> str:
50
+ lines = []
51
+ summary = json.loads((run / "summary.json").read_text(encoding="utf-8")) if (run / "summary.json").exists() else {}
52
+ # A trace cut off mid-line (a crash) is read up to its last whole record.
53
+ program = 0
54
+ for event in read_records(run):
55
+ kind = event["event"]
56
+ if kind == "reset":
57
+ lines.append(f"Instruction: {event['observation']['instruction']}")
58
+ elif kind == "model_reply" and program >= since and event.get("ahead", "used") == "used":
59
+ lines.append(f"[turn {event['call']}] {event['text'].strip()[:1500]}")
60
+ elif kind == "cell":
61
+ program += 1
62
+ if program <= since:
63
+ continue
64
+ result = event["result"]
65
+ text = f" -> {result['status']}"
66
+ if result.get("escalation"):
67
+ ev = result["escalation"]["event"]
68
+ text += f" {ev['kind']} from {ev['source']}: {json.dumps(ev['detail'])[:300]}"
69
+ if result.get("error"):
70
+ text += " " + result["error"].strip().splitlines()[-1][:300]
71
+ if result.get("stdout"):
72
+ text += " | out: " + result["stdout"].strip().replace("\n", " / ")[:300]
73
+ lines.append(text)
74
+ elif kind == "skill_step" and program >= since and event.get("library_revision"):
75
+ lines.append(f"Used {event.get('library_key')} at {event['library_revision']}: "
76
+ f"{event.get('what')}, ok={event.get('ok')}")
77
+ elif kind == "verify" and program >= since:
78
+ lines.append(f" verify {event['claim'][:200]!r}: {event['answer'][:200]}")
79
+ elif kind == "outcome" and set(event.get("outcome") or {}) - OUTCOME_KEYS:
80
+ # What the evaluator measured besides the score (a practice segment's time, deaths).
81
+ extra = {key: value for key, value in event["outcome"].items() if key not in OUTCOME_KEYS}
82
+ lines.append(f"Measured: {json.dumps(extra, default=str)[:600]}")
83
+ if summary:
84
+ lines.append(f"Outcome: success={summary.get('success')} "
85
+ f"score={summary.get('reward')} status={summary.get('status')}"
86
+ + (f" guards fired={summary['guard_triggers']}" if summary.get("guard_triggers") else ""))
87
+ text = "\n".join(lines)
88
+ if len(text) > max_chars:
89
+ text = text[:max_chars // 2] + "\n...\n" + text[-max_chars // 2:]
90
+ return text
91
+
92
+
93
+ def library_text(library, holdings=None):
94
+ if isinstance(library, (str, Path)):
95
+ library = Library(library)
96
+ captured = library.capture()
97
+ index = captured.index
98
+ entries = {item.key: item for item in captured.entries
99
+ if not executable(item.key) or holdings is None and eligible(
100
+ item, index.get(item.key) or {}, captured.layers, captured.load)}
101
+ if holdings is not None:
102
+ entries.update({key: item for key, (item, _) in holdings.items()})
103
+ lines = []
104
+ for key, item in entries.items():
105
+ row = index.get(key) or {}
106
+ uses = (row.get("uses_by_revision") or {}).get(item.revision) or {}
107
+ lines.append(f"--- {key} ({item.revision}, {uses.get('ok', 0)} successes, "
108
+ f"{uses.get('failed', 0)} failures)\n{item.source.decode('utf-8').strip()[:1200]}")
109
+ return "\n".join(lines)[:8000] or "(empty)"
110
+
111
+
112
+ def propose(chat, run: Path, library, embodiment="robot", *, since=0, holdings=None):
113
+ """Use the same reflection request for online and offline learning."""
114
+ names = [embodiment] if isinstance(embodiment, str) else list(embodiment)
115
+ prompt = (f"Current library:\n{library_text(library, holdings)}\n\n"
116
+ f"Episode:\n{summarize_episode(run, since=since)}\n\nPropose at most one addition.")
117
+ from harness.core.trace import Trace
118
+ from harness.runtime.decision.thoughts import think
119
+ from harness.runtime.events import EVENTS
120
+
121
+ trace = None if getattr(chat, "thinks", False) else Trace(run / "reflection" / new_name(), EVENTS)
122
+ reply = think(chat, system_prompt(names), [{"role": "user", "content": prompt}], trace=trace)
123
+ return parse_candidate(reply.text, require_test="robot" not in names), reply.text
124
+
125
+
126
+ def parse_candidate(text: str, *, require_test=True):
127
+ """The entry a reply proposes, (kind, name, text), or None: none, empty, or code that
128
+ is not the one public function with a docstring the request asks for."""
129
+ match = _ENTRY.search(text or "")
130
+ if not match:
131
+ return None
132
+ kind, name, body = match.group(1), match.group(2), match.group(3).strip()
133
+ test = None
134
+ if kind in ("skill", "reactive"): # code; a note and a memory entry are text
135
+ check = re.search(r"<test>(.*?)</test>", text[match.end():], re.DOTALL)
136
+ if check:
137
+ test = re.sub(r"^```[a-z]*\n|\n```$", "", check.group(1).strip())
138
+ if require_test and not test:
139
+ return None
140
+ body = re.sub(r"^```[a-z]*\n|\n```$", "", body).strip()
141
+ try:
142
+ tree = ast.parse(body)
143
+ except SyntaxError:
144
+ return None
145
+ functions = [node for node in tree.body if isinstance(node, ast.FunctionDef) and not node.name.startswith("_")]
146
+ if len(functions) != 1 or not ast.get_docstring(functions[0]):
147
+ return None
148
+ if not body:
149
+ return None
150
+ entry = kind, name, body + "\n"
151
+ return (*entry, test) if test else entry