@lmmslab/harness 0.0.0-stage → 0.81.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +124 -2
  3. package/bin/harness.js +71 -0
  4. package/package.json +34 -4
  5. package/python/harness/README.md +72 -0
  6. package/python/harness/__init__.py +8 -0
  7. package/python/harness/__main__.py +3 -0
  8. package/python/harness/bench.py +124 -0
  9. package/python/harness/builtin/benches/README.md +5 -0
  10. package/python/harness/builtin/benches/none/bench.toml +6 -0
  11. package/python/harness/builtin/configs/README.md +26 -0
  12. package/python/harness/builtin/configs/assistant.toml +11 -0
  13. package/python/harness/builtin/configs/default.toml +33 -0
  14. package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
  15. package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
  16. package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
  17. package/python/harness/builtin/configs/models/grok-low.toml +8 -0
  18. package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
  19. package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
  20. package/python/harness/builtin/environment/README.md +53 -0
  21. package/python/harness/builtin/environment/constraints.txt +31 -0
  22. package/python/harness/builtin/model-settings.js +143 -0
  23. package/python/harness/builtin/prompts/conversation/core.md +5 -0
  24. package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
  25. package/python/harness/builtin/prompts/core/core.md +88 -0
  26. package/python/harness/builtin/prompts/core/pack.toml +9 -0
  27. package/python/harness/builtin/prompts/reflect/core.md +45 -0
  28. package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
  29. package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
  30. package/python/harness/builtin/prompts/shared/agents.md +14 -0
  31. package/python/harness/builtin/prompts/shared/checklist.md +21 -0
  32. package/python/harness/builtin/prompts/shared/entries.md +86 -0
  33. package/python/harness/builtin/prompts/shared/environments.md +21 -0
  34. package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
  35. package/python/harness/builtin/prompts/shared/learning.md +27 -0
  36. package/python/harness/builtin/prompts/shared/mcp.md +9 -0
  37. package/python/harness/builtin/prompts/shared/world.md +1 -0
  38. package/python/harness/builtin/skills/README.md +10 -0
  39. package/python/harness/builtin/skills/agents/__init__.py +88 -0
  40. package/python/harness/builtin/skills/agents/skill.toml +11 -0
  41. package/python/harness/builtin/skills/checklist/__init__.py +32 -0
  42. package/python/harness/builtin/skills/checklist/skill.toml +9 -0
  43. package/python/harness/builtin/skills/conversation/__init__.py +23 -0
  44. package/python/harness/builtin/skills/conversation/skill.toml +8 -0
  45. package/python/harness/builtin/skills/entries/__init__.py +62 -0
  46. package/python/harness/builtin/skills/entries/skill.toml +8 -0
  47. package/python/harness/builtin/skills/environments/__init__.py +69 -0
  48. package/python/harness/builtin/skills/environments/skill.toml +9 -0
  49. package/python/harness/builtin/skills/experience/__init__.py +27 -0
  50. package/python/harness/builtin/skills/experience/skill.toml +9 -0
  51. package/python/harness/builtin/skills/instructions/__init__.py +29 -0
  52. package/python/harness/builtin/skills/instructions/skill.toml +9 -0
  53. package/python/harness/builtin/skills/learning/__init__.py +189 -0
  54. package/python/harness/builtin/skills/learning/skill.toml +7 -0
  55. package/python/harness/builtin/skills/mcp/__init__.py +54 -0
  56. package/python/harness/builtin/skills/mcp/skill.toml +10 -0
  57. package/python/harness/builtin/skills/memory/__init__.py +130 -0
  58. package/python/harness/builtin/skills/memory/skill.toml +13 -0
  59. package/python/harness/builtin/skills/verify/__init__.py +31 -0
  60. package/python/harness/builtin/skills/verify/skill.toml +7 -0
  61. package/python/harness/builtin/third_party/sources.toml +40 -0
  62. package/python/harness/cli.py +1182 -0
  63. package/python/harness/config/__init__.py +224 -0
  64. package/python/harness/config/agent.py +777 -0
  65. package/python/harness/core/__init__.py +52 -0
  66. package/python/harness/core/artifacts.py +46 -0
  67. package/python/harness/core/codec.py +160 -0
  68. package/python/harness/core/contracts.py +75 -0
  69. package/python/harness/core/episodes.py +150 -0
  70. package/python/harness/core/format.py +51 -0
  71. package/python/harness/core/http.py +124 -0
  72. package/python/harness/core/lanes.py +324 -0
  73. package/python/harness/core/linux_sandbox.py +55 -0
  74. package/python/harness/core/media.py +96 -0
  75. package/python/harness/core/memory.py +147 -0
  76. package/python/harness/core/ports.py +63 -0
  77. package/python/harness/core/procs.py +227 -0
  78. package/python/harness/core/remote.py +102 -0
  79. package/python/harness/core/spans.py +166 -0
  80. package/python/harness/core/trace.py +139 -0
  81. package/python/harness/core/wait.py +12 -0
  82. package/python/harness/core/worker.py +388 -0
  83. package/python/harness/core/worker_pipe.py +119 -0
  84. package/python/harness/embodiments.py +217 -0
  85. package/python/harness/extensions.py +112 -0
  86. package/python/harness/host/__init__.py +397 -0
  87. package/python/harness/host/client.py +201 -0
  88. package/python/harness/host/http.py +720 -0
  89. package/python/harness/host/listing.py +68 -0
  90. package/python/harness/host/server.py +189 -0
  91. package/python/harness/host/sessions.py +406 -0
  92. package/python/harness/host/tasks.py +1308 -0
  93. package/python/harness/host/workers.py +257 -0
  94. package/python/harness/host/workspaces.py +256 -0
  95. package/python/harness/model_list.py +117 -0
  96. package/python/harness/packages/__init__.py +73 -0
  97. package/python/harness/packages/addons.py +121 -0
  98. package/python/harness/packages/archive.py +122 -0
  99. package/python/harness/packages/describe.py +46 -0
  100. package/python/harness/packages/environments.py +448 -0
  101. package/python/harness/packages/fetch.py +57 -0
  102. package/python/harness/packages/manager.py +587 -0
  103. package/python/harness/packages/manifest.py +221 -0
  104. package/python/harness/packages/migrate.py +68 -0
  105. package/python/harness/packages/provision.py +38 -0
  106. package/python/harness/packages/registry.py +275 -0
  107. package/python/harness/packages/resolve.py +89 -0
  108. package/python/harness/packages/sources.py +141 -0
  109. package/python/harness/packages/state.py +213 -0
  110. package/python/harness/packages/versions.py +112 -0
  111. package/python/harness/paths.py +218 -0
  112. package/python/harness/providers.py +303 -0
  113. package/python/harness/runtime/__init__.py +8 -0
  114. package/python/harness/runtime/decision/__init__.py +1 -0
  115. package/python/harness/runtime/decision/loop.py +198 -0
  116. package/python/harness/runtime/decision/messages.py +271 -0
  117. package/python/harness/runtime/decision/model.py +424 -0
  118. package/python/harness/runtime/decision/prompts.py +248 -0
  119. package/python/harness/runtime/decision/protocol.py +169 -0
  120. package/python/harness/runtime/decision/thoughts.py +199 -0
  121. package/python/harness/runtime/events.py +213 -0
  122. package/python/harness/runtime/execution/__init__.py +1 -0
  123. package/python/harness/runtime/execution/attached.py +96 -0
  124. package/python/harness/runtime/execution/control.py +211 -0
  125. package/python/harness/runtime/execution/entries.py +665 -0
  126. package/python/harness/runtime/execution/kernel.py +811 -0
  127. package/python/harness/runtime/execution/noenv.py +84 -0
  128. package/python/harness/runtime/execution/results.py +40 -0
  129. package/python/harness/runtime/execution/session.py +871 -0
  130. package/python/harness/runtime/execution/skills.py +436 -0
  131. package/python/harness/runtime/learning/__init__.py +1 -0
  132. package/python/harness/runtime/learning/active.py +314 -0
  133. package/python/harness/runtime/learning/experience.py +48 -0
  134. package/python/harness/runtime/learning/fixtures.py +167 -0
  135. package/python/harness/runtime/learning/learner.py +651 -0
  136. package/python/harness/runtime/learning/library.py +504 -0
  137. package/python/harness/runtime/learning/reflect.py +151 -0
  138. package/python/harness/runtime/learning/usage.py +30 -0
  139. package/python/harness/runtime/learning/views.py +89 -0
  140. package/python/harness/runtime/reactive/__init__.py +1 -0
  141. package/python/harness/runtime/reactive/rules.py +549 -0
  142. package/python/harness/runtime/safety/__init__.py +1 -0
  143. package/python/harness/runtime/safety/approval.py +131 -0
  144. package/python/harness/runtime/safety/input.py +62 -0
  145. package/python/harness/runtime/safety/isolation.py +419 -0
  146. package/python/harness/runtime/safety/shield.py +129 -0
  147. package/python/harness/runtime/status.py +23 -0
  148. package/python/harness/runtime/task/__init__.py +7 -0
  149. package/python/harness/runtime/task/context.py +431 -0
  150. package/python/harness/runtime/task/delegation.py +468 -0
  151. package/python/harness/runtime/task/environments.py +268 -0
  152. package/python/harness/runtime/task/run.py +2878 -0
  153. package/python/harness/runtime/tick.py +909 -0
  154. package/python/harness/sdk/__init__.py +281 -0
  155. package/python/harness/services/__init__.py +6 -0
  156. package/python/harness/services/broker.py +35 -0
  157. package/python/harness/services/connectors.py +88 -0
  158. package/python/harness/services/mcp.py +298 -0
  159. package/python/harness/services/mcp_stdio.py +131 -0
  160. package/python/harness/services/placement.py +82 -0
  161. package/python/harness/services/registry.py +156 -0
  162. package/python/harness/settings.py +536 -0
  163. package/python/harness/terminal.py +79 -0
  164. package/python/harness/transcript.py +297 -0
  165. package/python/harness/tui/__init__.py +68 -0
  166. package/python/harness/tui/chat.py +1181 -0
  167. package/python/harness/tui/pty.py +81 -0
  168. package/python/harness/tui/renderer.py +45 -0
  169. package/python/harness/tui/timeline_rows.py +58 -0
  170. package/python/harness/tui/views.py +434 -0
@@ -0,0 +1,651 @@
1
+ """The learner: one mechanism for in-context learning and self-improvement, for any action
2
+ scenario. A candidate entry (a note, a memory entry, a skill with its test, a reactive
3
+ entry) comes from the model in the middle of a run, or from reflection on an episode after
4
+ it; graded evidence decides whether it enters the library (runtime/learning/library.py):
5
+
6
+ its own test, run in the run (fast) -> provisional
7
+ its test passing again when a later run starts (replay) -> replayed
8
+ a rise of the score on held-out development seeds (harness self-improve)
9
+ -> validated
10
+
11
+ An entry admitted in a run takes effect in that run (the conversation and the kernel load
12
+ it at once: runtime/conversation.py, skills/learning); every admission changes the library
13
+ version and the trace records it. Learning has two triggers, one path: active, a run whose
14
+ goal is to learn the environment (`trigger = "active"`), and passive, the task runs and the
15
+ reflection after them (`trigger = "passive"`). Evaluation reads a frozen library and learns
16
+ nothing (`learn = "off"`, the default).
17
+
18
+ The settings, [runtime.library] (SETTINGS):
19
+
20
+ learn "off" | "on" admit in a run (in-context) or not (frozen)
21
+ trigger "passive" | "active" what the run is: a task, or learning the environment
22
+ evidence "test" | "validated" what admits an entry in a run: its own test (a note
23
+ or memory entry has none: admitted as proposed), or
24
+ only development seeds (it waits in candidates/)
25
+ load proposed | provisional | replayed | validated
26
+ the lowest evidence an entry needs to be loaded
27
+ replay false | true at a run's start, test the provisional entries again
28
+ reflect false | true after each episode, reflection proposes one entry
29
+
30
+ Standard library only.
31
+ """
32
+
33
+ import base64
34
+ import hashlib
35
+ import json
36
+ from dataclasses import dataclass, field
37
+ from pathlib import Path
38
+ import re
39
+ import shutil
40
+ import tempfile
41
+ import threading
42
+ import time
43
+
44
+ from harness.runtime.learning.library import ADMISSIONS, CANDIDATES, EVIDENCE, FAILED, INDEX, SECTIONS, TESTS, Library, Revision, executable, rank
45
+ from harness.core import atomic_write
46
+ from harness.core.procs import file_lock
47
+ from harness.paths import home
48
+
49
+ SETTINGS = {
50
+ "learn": ("off", "on"),
51
+ "trigger": ("passive", "active"),
52
+ "evidence": ("test", "validated"),
53
+ "load": EVIDENCE,
54
+ "replay": (False, True),
55
+ "reflect": (False, True),
56
+ }
57
+ # An entry's kind and the library section it goes into (library.SECTIONS: its file suffix).
58
+ KINDS = {"note": "notes", "memory": "memory", "skill": "skills", "reactive": "reactive"}
59
+ ORIGINS = ("model", "reflection", "self_improve", "hand")
60
+ NAME = re.compile(r"^[a-z][a-z0-9_]{1,40}$")
61
+ # Notes go into every prompt, memory entries only when recall finds them: a longer entry is
62
+ # refused (the reflection request says so; before it did not, and half the proposals were lost).
63
+ NOTE_MAX_CHARS, MEMORY_MAX_CHARS = 600, 1200
64
+ # The longest situations list (memory/situations: the plan an active run learns by).
65
+ SITUATIONS_MAX_BYTES = 32 * 1024
66
+ # An attachment is at most this large in all, per proposal (a small trained model fits).
67
+ ATTACHMENT_BYTES = 64 * 1024 * 1024
68
+ # Test runs and uses kept per entry in the index.
69
+ KEPT = 20
70
+ _LOCK = threading.Lock()
71
+
72
+
73
+ def settings(runtime_library: dict | None) -> dict:
74
+ """The learning settings of a [runtime.library] table, defaults filled in; ValueError
75
+ for a value it does not take."""
76
+ given = dict(runtime_library or {})
77
+ chosen = {}
78
+ for key, values in SETTINGS.items():
79
+ value = given.get(key, values[0])
80
+ if value not in values:
81
+ raise ValueError(f"runtime.library.{key} is one of {list(values)}, not {value!r}")
82
+ chosen[key] = value
83
+ return chosen
84
+
85
+
86
+ def library_path(path) -> Path:
87
+ """A library's directory: `$HARNESS_HOME/...` is under the harness's home."""
88
+ text = str(path)
89
+ for prefix in ("$HARNESS_HOME/", "${HARNESS_HOME}/"):
90
+ if text.startswith(prefix):
91
+ return home() / text[len(prefix):]
92
+ return Path(text)
93
+
94
+
95
+ def libraries() -> list[dict]:
96
+ """The libraries under $HARNESS_HOME/libraries: [{"name", "directory", "version", "entries"}]."""
97
+ base = home() / "libraries"
98
+ rows = []
99
+ for path in sorted(base.iterdir()) if base.is_dir() else []:
100
+ if path.is_dir():
101
+ library = Library(path)
102
+ rows.append({"name": path.name, "directory": str(path), "version": library.version(),
103
+ "entries": sum(len(library.entries(section)) for section in SECTIONS)})
104
+ return rows
105
+
106
+
107
+ def entry_key(kind: str, name: str) -> str:
108
+ """Where an entry is in a library: "<section>/<name><suffix>" (skills/walk.py)."""
109
+ return f"{KINDS[kind]}/{name}{SECTIONS[KINDS[kind]]}"
110
+
111
+
112
+ @dataclass
113
+ class Candidate:
114
+ """An entry proposed for the library."""
115
+
116
+ kind: str # note, memory, skill, reactive
117
+ name: str
118
+ text: str
119
+ test: str | None = None
120
+ summary: str = ""
121
+ attachments: dict = field(default_factory=dict) # {filename: bytes}
122
+ source: str = "passive" # active, passive
123
+ origin: str = "model" # ORIGINS
124
+ run: str | None = None # the episode it came from
125
+ bindings: tuple = ()
126
+
127
+ @property
128
+ def section(self) -> str:
129
+ return KINDS[self.kind]
130
+
131
+ @property
132
+ def filename(self) -> str:
133
+ return self.name + SECTIONS[self.section]
134
+
135
+ @property
136
+ def key(self) -> str:
137
+ return entry_key(self.kind, self.name)
138
+
139
+
140
+ def revision(self, dependencies=(), bindings=None):
141
+ return Revision(self.key, self.text.encode("utf-8"),
142
+ None if self.test is None else self.test.encode("utf-8"),
143
+ tuple((f"{self.section}/{self.name}.files/{name}", data)
144
+ for name, data in sorted(self.attachments.items())),
145
+ tuple(sorted(dependencies)), tuple(sorted(self.bindings if bindings is None else bindings)))
146
+
147
+
148
+ def entry_name_problem(name) -> str | None:
149
+ """Why `name` cannot name a library entry (None when it can): the learner's one rule,
150
+ also checked by the learning skill before an entry's test acts."""
151
+ if not isinstance(name, str) or not NAME.match(name):
152
+ return f"an entry's name is lowercase letters, digits and _ (2-41, a letter first), not {name!r}"
153
+ return None
154
+
155
+
156
+ def candidate(kind, name, text, test=None, *, summary="", attachments=None, source="passive", origin="model",
157
+ run=None) -> Candidate:
158
+ """A Candidate, checked: its kind and name, a code entry that parses (and has a test,
159
+ when it is code), a note or memory entry not too long, attachments within the cap."""
160
+ if kind not in KINDS:
161
+ raise ValueError(f"an entry's kind is one of {list(KINDS)}, not {kind!r}")
162
+ problem = entry_name_problem(name)
163
+ if problem:
164
+ raise ValueError(problem)
165
+ if source not in ("active", "passive") or origin not in ORIGINS:
166
+ raise ValueError(f"source is active or passive and origin one of {list(ORIGINS)}")
167
+ text = str(text).strip() + "\n"
168
+ if kind in ("skill", "reactive"):
169
+ import ast
170
+
171
+ try:
172
+ ast.parse(text)
173
+ if test is not None:
174
+ ast.parse(test)
175
+ except SyntaxError as error:
176
+ raise ValueError(f"the code does not parse: {error}") from None
177
+ else:
178
+ # The situations list (runtime/learning/active.py) is a plan, not a lesson: it gets more room.
179
+ situations = kind == "memory" and name == "situations"
180
+ limit = SITUATIONS_MAX_BYTES if situations else (NOTE_MAX_CHARS if kind == "note" else MEMORY_MAX_CHARS) + 1
181
+ if (len(text.encode("utf-8")) if situations else len(text)) > limit:
182
+ raise ValueError(f"a {kind} is at most {limit} characters")
183
+ files = {}
184
+ for filename, value in (attachments or {}).items():
185
+ if not re.match(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,80}$", str(filename)):
186
+ raise ValueError(f"an attachment has a plain file name, not {filename!r}")
187
+ files[str(filename)] = value.encode("utf-8") if isinstance(value, str) else bytes(value)
188
+ if sum(len(value) for value in files.values()) > ATTACHMENT_BYTES:
189
+ raise ValueError(f"attachments are at most {ATTACHMENT_BYTES // (1024 * 1024)} MB in all")
190
+ return Candidate(kind, name, text, None if test is None else str(test).strip() + "\n", str(summary or ""),
191
+ files, source, origin, run)
192
+
193
+
194
+ def _now() -> float:
195
+ return round(time.time(), 3)
196
+
197
+
198
+ class Learner:
199
+ """The library at `directory` with what a run's learning settings allow."""
200
+
201
+ def __init__(self, directory, runtime_library: dict | None = None, create: bool = False):
202
+ self.directory = library_path(directory).resolve()
203
+ if create:
204
+ for section in SECTIONS:
205
+ (self.directory / section).mkdir(parents=True, exist_ok=True)
206
+ self.settings = settings(runtime_library)
207
+ self.layers = tuple((runtime_library or {}).get("layers", ("notes", "skills", "reactive")))
208
+ self.library = Library(self.directory, self.layers, self.settings["load"])
209
+
210
+ def version(self) -> str:
211
+ return self.library.version()
212
+
213
+ # -- the index ----------------------------------------------------------------------
214
+
215
+ def index(self) -> dict:
216
+ return self.library.index()
217
+
218
+ def _save_index(self, index: dict) -> None:
219
+ atomic_write(self.directory / INDEX, json.dumps(index, indent=1, sort_keys=True))
220
+
221
+ def migrate_experience(self):
222
+ from harness.runtime.learning.experience import MIGRATION, legacy_entries
223
+
224
+ index = self.index()
225
+ if index.get(MIGRATION) or not list(legacy_entries(self.directory, index)):
226
+ return
227
+ with _LOCK, file_lock(self.directory / ".library.lock"):
228
+ index = self.index()
229
+ if index.get(MIGRATION):
230
+ return
231
+ for item in list(legacy_entries(self.directory, index)):
232
+ key = item["id"]
233
+ if key not in index:
234
+ revision = Revision(key, item["text"].encode("utf-8")).save(self.directory)
235
+ index[key] = {"memory": item, "evidence": "validated", "origin": "legacy", "revision": revision}
236
+ index[MIGRATION] = True
237
+ self._save_index(index)
238
+
239
+ def store_memory(self, item, *, forgotten=False):
240
+ key = item["id"]
241
+ if not isinstance(key, str) or not re.fullmatch(r"memory/[^/\\]+\.md", key):
242
+ raise ValueError("invalid persistent memory key")
243
+ if "notes" not in self.layers or self.settings["learn"] != "on":
244
+ raise ValueError("this library is read-only")
245
+ with _LOCK, file_lock(self.directory / ".library.lock"):
246
+ index = self.index()
247
+ row = dict(index.get(key) or {})
248
+ if forgotten:
249
+ row["forgotten"] = True
250
+ else:
251
+ content = dict(item, scope="library")
252
+ data = content["text"].encode("utf-8")
253
+ if content.get("name") == "situations" and len(data) > SITUATIONS_MAX_BYTES:
254
+ raise ValueError("the situations list is limited to 32 KiB")
255
+ revision = Revision(key, data).save(self.directory)
256
+ row.update(evidence="proposed", origin="memory", memory=content, revision=revision,
257
+ forgotten=False)
258
+ index[key] = row
259
+ self._save_index(index)
260
+ return {**item, "scope": "library", "evidence": "proposed", "forgotten": forgotten}
261
+
262
+ def entries(self, *, snapshot=None) -> list[dict]:
263
+ """Every entry in the library (loaded or not) with its evidence, source, origin and
264
+ summary, and the candidates waiting: [{"section", "name", "file", "evidence",
265
+ "source", "origin", "summary", "admitted_at", "tests", "passed", "uses", "loaded"}]."""
266
+ captured = snapshot if snapshot is not None else self.library.capture()
267
+ index = captured.index
268
+ loaded = {item.key for item in captured.entries}
269
+ rows = []
270
+ for section, suffix in SECTIONS.items():
271
+ for root, waiting in ((self.directory / section, False), (self.directory / CANDIDATES / section, True)):
272
+ prefix = f"{CANDIDATES}/{section}/" if waiting else f"{section}/"
273
+ if snapshot is None:
274
+ paths = set(root.glob(f"*{suffix}"))
275
+ paths.update(root / key.removeprefix(prefix) for key, row in index.items()
276
+ if key.startswith(prefix) and row.get("revision"))
277
+ else:
278
+ paths = {root / key.removeprefix(prefix) for key in loaded if key.startswith(prefix)}
279
+ for path in sorted(paths):
280
+ key = f"{section}/{path.name}"
281
+ row = index.get(f"{CANDIDATES}/{key}" if waiting else key) or {}
282
+ if row.get("forgotten"):
283
+ continue
284
+ tests = row.get("tests") or []
285
+ uses = (row.get("uses_by_revision") or {}).get(row.get("revision")) or {}
286
+ rows.append({"section": section, "name": (row.get("memory") or {}).get("name") or path.stem, "file": key,
287
+ "evidence": row.get("evidence", "proposed" if waiting else "validated"),
288
+ "revision": row.get("revision"),
289
+ "verified": bool(row.get("proof") and row["proof"].get("revision") == row.get("revision")
290
+ and row.get("evidence") in ("provisional", "replayed", "validated")),
291
+ "waiting": waiting, "source": row.get("source", "-"),
292
+ "origin": row.get("origin", "hand"), "summary": row.get("summary", ""),
293
+ "admitted_at": row.get("admitted_at"), "tests": len(tests),
294
+ "passed": sum(1 for run in tests if run.get("ok")),
295
+ "uses": int(uses.get("ok", 0)) + int(uses.get("failed", 0)),
296
+ "success_rate": uses.get("rate"), "loaded": key in loaded and not waiting})
297
+ return rows
298
+
299
+ # -- admitting --------------------------------------------------------------------
300
+
301
+ def wanted(self, candidate: Candidate, evidence: str) -> bool:
302
+ """Whether an entry with this evidence enters the library now (else it waits)."""
303
+ if self.settings["evidence"] == "validated":
304
+ return evidence == "validated"
305
+ return rank(evidence) >= rank("proposed")
306
+
307
+ def admit(self, candidate: Candidate, evidence: str, *, test_run: dict | None = None,
308
+ gain: float | None = None, why: str = "", proof=None, dependencies=(), bindings=None,
309
+ pending=False, expected=None, check=None) -> dict:
310
+ """Put a candidate into the library with this evidence (or, when the settings want
311
+ more, into candidates/): its files written, its index row (source, origin, the test
312
+ run, the gain) kept. {"admitted", "evidence", "version_before", "version_after",
313
+ "key"}."""
314
+ if evidence not in EVIDENCE:
315
+ raise ValueError(f"evidence is one of {list(EVIDENCE)}, not {evidence!r}")
316
+ with _LOCK, file_lock(self.directory / ".library.lock"):
317
+ before = self.version()
318
+ revision = candidate.revision(dependencies, bindings)
319
+ if proof is not None and (proof.get("revision") != revision.revision or not proof.get("conditions")):
320
+ raise ValueError("verification proof does not cover this revision")
321
+ index = self.index()
322
+ waiting = index.get(f"{CANDIDATES}/{candidate.key}") or {}
323
+ if expected is not None and (waiting.get("revision") != expected
324
+ or waiting.get("evidence") == FAILED):
325
+ raise ValueError("the pending proposal changed while being verified")
326
+ if proof is not None and expected is None and waiting.get("revision") not in (
327
+ None, candidate.revision().revision, revision.revision):
328
+ raise ValueError("the pending proposal changed while being verified")
329
+ admitted = not pending and self.wanted(candidate, evidence)
330
+ stored_evidence = evidence if admitted or proof is not None else "proposed"
331
+ # Preserve the imported revision before writing the editable file mirror.
332
+ previous = index.get(candidate.key) or {}
333
+ if admitted and not previous.get("revision"):
334
+ old = next((item for item in self.library.capture().entries if item.key == candidate.key), None)
335
+ if old is not None:
336
+ index[candidate.key] = {**previous, "revision": old.save(self.directory)}
337
+ self._save_index(index)
338
+ root = self.directory if admitted else self.directory / CANDIDATES
339
+ target = root / candidate.section / candidate.filename
340
+ replaced = target.is_file() and target.read_bytes() != candidate.text.encode("utf-8")
341
+ if expected is None:
342
+ atomic_write(target, candidate.text)
343
+ if candidate.test is not None:
344
+ atomic_write(root / TESTS / f"{candidate.name}.py", candidate.test)
345
+ folder = root / candidate.section / f"{candidate.name}.files"
346
+ if candidate.attachments:
347
+ shutil.rmtree(folder, ignore_errors=True)
348
+ for filename, data in candidate.attachments.items():
349
+ atomic_write(folder / filename, data)
350
+ index_key = candidate.key if admitted else f"{CANDIDATES}/{candidate.key}"
351
+ row = dict(index.get(index_key) or {})
352
+ if admitted:
353
+ index.pop(f"{CANDIDATES}/{candidate.key}", None)
354
+ revision_id = revision.save(self.directory)
355
+ if row.get("revision") != revision_id:
356
+ row.pop("proof", None)
357
+ row.pop("tests", None)
358
+ row["revision"] = revision_id
359
+ if proof is not None:
360
+ row["proof"] = proof
361
+ history = list(row.get("history") or [])
362
+ history.append({"at": _now(), "evidence": stored_evidence,
363
+ "revision": revision_id, **({"proof": proof} if proof is not None else {}),
364
+ **({"gain": round(gain, 4)} if gain is not None else {}), **({"why": why} if why else {})})
365
+ if admitted and replaced and (row.get("uses") or {}).get("sha256"):
366
+ history[-1]["previous_uses"] = row.pop("uses")
367
+ row.update(evidence=stored_evidence, source=candidate.source,
368
+ origin=candidate.origin, summary=candidate.summary or row.get("summary", ""),
369
+ admitted_at=row.get("admitted_at") or _now(), updated_at=_now(), history=history[-KEPT:],
370
+ **({"run": candidate.run} if candidate.run else {}))
371
+ if test_run is not None:
372
+ row["tests"] = [*(row.get("tests") or []), {"at": _now(), **test_run}][-KEPT:]
373
+ index[index_key] = row
374
+ if check is not None:
375
+ check()
376
+ self._save_index(index)
377
+ if admitted:
378
+ for stale in (self.directory / CANDIDATES / candidate.section / candidate.filename,
379
+ self.directory / CANDIDATES / TESTS / f"{candidate.name}.py"):
380
+ stale.unlink(missing_ok=True)
381
+ after = self.version()
382
+ outcome = {"admitted": admitted, "evidence": row["evidence"], "version_before": before,
383
+ "version_after": after, "key": candidate.key, "row": row, "revision": revision_id,
384
+ "sha256": hashlib.sha256(candidate.text.encode("utf-8")).hexdigest()}
385
+ if admitted:
386
+ # The library's log of what entered it (the audit's evidence that an entry
387
+ # was admitted, benches/minecraft/audit.py): section, file, its bytes' sha256.
388
+ with (self.directory / ADMISSIONS).open("a", encoding="utf-8") as log:
389
+ log.write(json.dumps({**self.record_of(outcome), "source": candidate.source,
390
+ "origin": candidate.origin, "at": _now()}) + "\n")
391
+ return outcome
392
+
393
+ @staticmethod
394
+ def record_of(outcome: dict) -> dict:
395
+ """An admission as the library's log and the trace record it: section, name (the
396
+ entry's file), sha256 of its bytes, the version after it, its evidence, and whether
397
+ it entered the library (or waits)."""
398
+ section, _, filename = outcome["key"].partition("/")
399
+ return {"section": section, "name": filename, "sha256": outcome["sha256"], "version": outcome["version_after"],
400
+ "version_before": outcome["version_before"], "evidence": outcome["evidence"],
401
+ "admitted": outcome["admitted"], "revision": outcome.get("revision"),
402
+ **({"proof": outcome["row"]["proof"]} if (outcome.get("row") or {}).get("proof") else {})}
403
+
404
+ def grade(self, key: str, evidence: str, *, test_run: dict | None = None, gain: float | None = None,
405
+ why: str = "", sha256: str | None = None) -> dict | None:
406
+ """Set an entry's evidence (a replay passed: replayed; failed: failed; development
407
+ seeds: validated), recorded in its history. Its index row, or None."""
408
+ if evidence not in (*EVIDENCE, FAILED):
409
+ raise ValueError(f"evidence is one of {[*EVIDENCE, FAILED]}, not {evidence!r}")
410
+ with _LOCK, file_lock(self.directory / ".library.lock"):
411
+ if sha256 is not None:
412
+ section, _, filename = key.partition("/")
413
+ if (section not in SECTIONS or not filename.endswith(SECTIONS[section])
414
+ or any(char in filename for char in "/\\:")):
415
+ raise ValueError(f"invalid library revision key {key!r}")
416
+ path = self.directory / section / filename
417
+ if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != sha256:
418
+ raise ValueError(f"{key} changed while it was being graded")
419
+ index = self.index()
420
+ row = dict(index.get(key) or {"source": "-", "origin": "hand"})
421
+ row["evidence"] = evidence
422
+ row["history"] = [*(row.get("history") or []), {
423
+ "at": _now(), "evidence": evidence, **({"gain": round(gain, 4)} if gain is not None else {}),
424
+ **({"sha256": sha256} if sha256 is not None else {}),
425
+ **({"revision": row["revision"]} if row.get("revision") else {}),
426
+ **({"why": why} if why else {})}][-KEPT:]
427
+ if test_run is not None:
428
+ row["tests"] = [*(row.get("tests") or []), {"at": _now(), **test_run}][-KEPT:]
429
+ index[key] = row
430
+ self._save_index(index)
431
+ return row
432
+
433
+ def remove(self, key: str, why: str = "") -> None:
434
+ """Take an entry (or a waiting candidate) out of the library: its file, test and
435
+ attachments; its index row says why."""
436
+ section, _, filename = key.partition("/")
437
+ stem = Path(filename).stem
438
+ with _LOCK, file_lock(self.directory / ".library.lock"):
439
+ for root in (self.directory, self.directory / CANDIDATES):
440
+ (root / section / filename).unlink(missing_ok=True)
441
+ (root / TESTS / f"{stem}.py").unlink(missing_ok=True)
442
+ shutil.rmtree(root / section / f"{stem}.files", ignore_errors=True)
443
+ index = self.index()
444
+ if key in index:
445
+ index[key] = {**index[key], "evidence": FAILED, "removed": _now(), **({"why": why} if why else {})}
446
+ self._save_index(index)
447
+
448
+ def grade_development(self, candidate, revision, *, pending, score):
449
+ """Accept development evidence for exactly the verified candidate that ran."""
450
+ key = candidate.key
451
+ current = self.index().get(f"{CANDIDATES}/{key}" if pending else key) or {}
452
+ proof = current.get("proof") or {}
453
+ if proof.get("revision") != revision or candidate.revision().revision != revision:
454
+ return False
455
+ proof = {**proof, "development": score}
456
+ if pending:
457
+ self.admit(candidate, "validated", proof=proof, expected=revision, gain=score["gain"],
458
+ why="paired development scores")
459
+ return True
460
+ with _LOCK, file_lock(self.directory / ".library.lock"):
461
+ index = self.index()
462
+ row = index.get(key) or {}
463
+ if row != current or row.get("evidence") not in ("provisional", "replayed"):
464
+ return False
465
+ row.update(evidence="validated", proof=proof)
466
+ row["history"] = [*(row.get("history") or []),
467
+ {"at": _now(), "revision": revision, "evidence": "validated",
468
+ "development": score}][-KEPT:]
469
+ self._save_index(index)
470
+ return True
471
+
472
+ def record_trial(self, key, expected, item, *, pending, evidence, receipt, check):
473
+ """Record a verdict on the immutable revision that actually ran."""
474
+ if item.key != key or receipt["revision"] != item.revision:
475
+ raise ValueError("the trial receipt has a different revision")
476
+ with _LOCK, file_lock(self.directory / ".library.lock"):
477
+ index = self.index()
478
+ index_key = f"{CANDIDATES}/{key}" if pending else key
479
+ row = index.get(index_key)
480
+ if row is None or row.get("revision") != expected:
481
+ return False
482
+ if row.get("evidence") != ("proposed" if pending else "provisional"):
483
+ return False
484
+ revision = item.save(self.directory)
485
+ history = list(row.get("history") or [])
486
+ result = {"at": _now(), "revision": revision, "evidence": evidence, "receipt": receipt}
487
+ row.update(revision=revision, evidence=evidence, history=[*history, result][-KEPT:],
488
+ tests=[*(row.get("tests") or []), receipt][-KEPT:])
489
+ if evidence == FAILED:
490
+ row.pop("proof", None)
491
+ if not pending:
492
+ failed = {revision, *(grade.get("revision") for grade in history
493
+ if grade.get("evidence") == FAILED)}
494
+ earlier = next((grade for grade in reversed(history)
495
+ if grade.get("revision") not in failed and grade.get("proof")), None)
496
+ if earlier is not None:
497
+ previous = Revision.read(self.directory, earlier["revision"])
498
+ if previous.key == key:
499
+ index.setdefault(f"{CANDIDATES}/{key}", dict(row))
500
+ row.update(revision=previous.revision, evidence=earlier["evidence"],
501
+ proof=earlier["proof"], tests=[])
502
+ check()
503
+ self._save_index(index)
504
+ return True
505
+
506
+ def record(self, key: str, ok: bool, detail=None) -> None:
507
+ """A use of an entry in play: counted, its rate kept."""
508
+ self.record_uses(key, [(ok, detail)])
509
+
510
+ def record_uses(self, key, results, digest=None, *, revision=None, run=None):
511
+ """Record outcomes once under the version that ran, including replaced versions."""
512
+ with _LOCK, file_lock(self.directory / ".library.lock"):
513
+ index = self.index()
514
+ row = dict(index.get(key) or {"source": "-", "origin": "hand", "evidence": "validated"})
515
+ if revision is None and digest is not None:
516
+ matching = [item for item in self.library.capture().entries
517
+ if item.key == key and item.source_sha256 == digest]
518
+ if not matching:
519
+ return 0
520
+ revision = matching[0].revision
521
+ versions = row.setdefault("uses_by_revision", {})
522
+ uses = versions.setdefault(revision or "unknown", {"ok": 0, "failed": 0})
523
+ if run is not None:
524
+ processed = uses.setdefault("runs", [])
525
+ if run in processed:
526
+ return 0
527
+ processed.append(run)
528
+ for result in results:
529
+ ok, detail = result[:2]
530
+ uses["ok" if ok else "failed"] += 1
531
+ if len(result) > 2:
532
+ uses["seconds"] = round(uses.get("seconds", 0) + (result[2] or 0), 3)
533
+ if detail:
534
+ uses["last"] = str(detail)[:200]
535
+ count = uses["ok"] + uses["failed"]
536
+ uses["rate"] = round(uses["ok"] / count, 3) if count else None
537
+ index[key] = row
538
+ self._save_index(index)
539
+ return len(results)
540
+
541
+ # -- reading --------------------------------------------------------------------------
542
+
543
+ def candidate_from(self, key: str, *, source="passive", origin="hand", pending=None) -> Candidate | None:
544
+ """Read a waiting or admitted candidate from its pinned revision."""
545
+ section, _, filename = key.partition("/")
546
+ kind = {value: name for name, value in KINDS.items()}[section]
547
+ stem = Path(filename).stem
548
+ index = self.index()
549
+ for waiting in (True, False) if pending is None else (pending,):
550
+ row = index.get(f"{CANDIDATES}/{key}" if waiting else key) or {}
551
+ if row.get("forgotten"):
552
+ continue
553
+ if row.get("revision"):
554
+ item = Revision.read(self.directory, row["revision"])
555
+ if item.key != key:
556
+ raise ValueError(f"library index points {key} at a different candidate")
557
+ prefix = f"{section}/{stem}.files/"
558
+ return Candidate(kind, stem, item.source.decode("utf-8"),
559
+ item.test.decode("utf-8") if item.test is not None else None,
560
+ row.get("summary", ""), {name.removeprefix(prefix): data
561
+ for name, data in item.attachments},
562
+ row.get("source", source), row.get("origin", origin), row.get("run"), item.bindings)
563
+ root = self.directory / CANDIDATES if waiting else self.directory
564
+ path = root / section / filename
565
+ if path.is_file():
566
+ test = root / TESTS / f"{stem}.py"
567
+ folder = root / section / f"{stem}.files"
568
+ return Candidate(kind, stem, path.read_text(encoding="utf-8"),
569
+ test.read_text(encoding="utf-8") if test.is_file() else None,
570
+ row.get("summary", ""),
571
+ {p.name: p.read_bytes() for p in folder.iterdir() if p.is_file()} if folder.is_dir()
572
+ else {}, row.get("source", source), row.get("origin", origin), row.get("run"))
573
+ return None
574
+
575
+ def attachment(self, sha256: str) -> bytes:
576
+ """The bytes of the attachment with this sha256, wherever it is in the library."""
577
+ for item in self.library.capture().entries:
578
+ for _, data in item.attachments:
579
+ if hashlib.sha256(data).hexdigest() == sha256:
580
+ return data
581
+ for section in SECTIONS:
582
+ for path in sorted((self.directory / CANDIDATES / section).glob("*.files/**/*")):
583
+ if path.is_file():
584
+ data = path.read_bytes()
585
+ if hashlib.sha256(data).hexdigest() == sha256:
586
+ return data
587
+ raise KeyError(f"no attachment {sha256} in {self.directory}")
588
+
589
+ def trial(self, candidate: Candidate, root: Path | None = None, *, proof=None) -> Path:
590
+ """A copy of the library with the candidate in it, to measure it on development seeds."""
591
+ trial = Path(tempfile.mkdtemp(prefix="library-trial-", dir=root))
592
+ shutil.copytree(self.directory, trial, dirs_exist_ok=True, ignore=shutil.ignore_patterns(CANDIDATES))
593
+ copy = Learner(trial, {**self.settings, "evidence": "test", "layers": list(self.layers)})
594
+ if proof is None:
595
+ from dataclasses import replace
596
+
597
+ candidate = replace(candidate, origin="hand")
598
+ copy.admit(candidate, "provisional", proof=proof)
599
+ return trial
600
+
601
+ def without(self, key: str, root: Path | None = None) -> Path:
602
+ """A copy of the library without one entry, to measure what it adds."""
603
+ trial = Path(tempfile.mkdtemp(prefix="library-trial-", dir=root))
604
+ shutil.copytree(self.directory, trial, dirs_exist_ok=True, ignore=shutil.ignore_patterns(CANDIDATES))
605
+ Learner(trial, {"layers": list(self.layers)}).remove(key, "measured without it")
606
+ return trial
607
+
608
+ # -- a run's requests (the host answers the kernel's, runtime/conversation.py) ----------
609
+
610
+ def handle(self, request: dict, *, source: str, run: str | None = None):
611
+ """What the kernel asks: admit (a candidate whose test, if any, passed in the
612
+ kernel), grade (a replay's result), record (a use), entries, version, attachment."""
613
+ action = request.get("action")
614
+ if action == "admit":
615
+ files = {name: base64.b64decode(value) for name, value in (request.get("attachments") or {}).items()}
616
+ entry = candidate(request.get("entry") or request.get("kind"), request["name"], request["text"], request.get("test"),
617
+ summary=request.get("summary", ""), attachments=files, source=source, origin="model",
618
+ run=run)
619
+ if executable(entry.key):
620
+ if not entry.test:
621
+ raise ValueError("an executable proposal needs a verification condition")
622
+ return self.admit(entry, "proposed", pending=True)
623
+ return self.admit(entry, "proposed")
624
+ if action == "grade":
625
+ raise ValueError("verification grades require a trusted host receipt")
626
+ if action == "record":
627
+ self.record(request["key"], bool(request.get("ok")), request.get("detail"))
628
+ return None
629
+ if action == "entries":
630
+ return self.entries()
631
+ if action == "version":
632
+ return self.version()
633
+ if action == "attachment":
634
+ return base64.b64encode(self.attachment(request["sha256"])).decode("ascii")
635
+ raise ValueError(f"unknown library request {action!r}")
636
+
637
+ # -- passive: reflection after an episode -------------------------------------------
638
+
639
+ def reflect(self, chat, run: Path, names, *, since=0, holdings=None) -> tuple[Candidate | None, str]:
640
+ """One candidate from an episode (reflect.py), checked as every candidate is
641
+ (candidate()), as passive learning; (None, reply) when the model proposes none, or
642
+ one that does not pass."""
643
+ from harness.runtime.learning.reflect import propose
644
+
645
+ found, reply = propose(chat, Path(run), self.library, names, since=since, holdings=holdings)
646
+ if found is None:
647
+ return None, reply
648
+ try:
649
+ return candidate(*found, source=self.settings["trigger"], origin="reflection", run=str(run)), reply
650
+ except ValueError:
651
+ return None, reply