@lmmslab/harness 0.0.0-stage → 0.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1308 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +221 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,651 @@
|
|
|
1
|
+
"""The learner: one mechanism for in-context learning and self-improvement, for any action
|
|
2
|
+
scenario. A candidate entry (a note, a memory entry, a skill with its test, a reactive
|
|
3
|
+
entry) comes from the model in the middle of a run, or from reflection on an episode after
|
|
4
|
+
it; graded evidence decides whether it enters the library (runtime/learning/library.py):
|
|
5
|
+
|
|
6
|
+
its own test, run in the run (fast) -> provisional
|
|
7
|
+
its test passing again when a later run starts (replay) -> replayed
|
|
8
|
+
a rise of the score on held-out development seeds (harness self-improve)
|
|
9
|
+
-> validated
|
|
10
|
+
|
|
11
|
+
An entry admitted in a run takes effect in that run (the conversation and the kernel load
|
|
12
|
+
it at once: runtime/conversation.py, skills/learning); every admission changes the library
|
|
13
|
+
version and the trace records it. Learning has two triggers, one path: active, a run whose
|
|
14
|
+
goal is to learn the environment (`trigger = "active"`), and passive, the task runs and the
|
|
15
|
+
reflection after them (`trigger = "passive"`). Evaluation reads a frozen library and learns
|
|
16
|
+
nothing (`learn = "off"`, the default).
|
|
17
|
+
|
|
18
|
+
The settings, [runtime.library] (SETTINGS):
|
|
19
|
+
|
|
20
|
+
learn "off" | "on" admit in a run (in-context) or not (frozen)
|
|
21
|
+
trigger "passive" | "active" what the run is: a task, or learning the environment
|
|
22
|
+
evidence "test" | "validated" what admits an entry in a run: its own test (a note
|
|
23
|
+
or memory entry has none: admitted as proposed), or
|
|
24
|
+
only development seeds (it waits in candidates/)
|
|
25
|
+
load proposed | provisional | replayed | validated
|
|
26
|
+
the lowest evidence an entry needs to be loaded
|
|
27
|
+
replay false | true at a run's start, test the provisional entries again
|
|
28
|
+
reflect false | true after each episode, reflection proposes one entry
|
|
29
|
+
|
|
30
|
+
Standard library only.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
import base64
|
|
34
|
+
import hashlib
|
|
35
|
+
import json
|
|
36
|
+
from dataclasses import dataclass, field
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
import re
|
|
39
|
+
import shutil
|
|
40
|
+
import tempfile
|
|
41
|
+
import threading
|
|
42
|
+
import time
|
|
43
|
+
|
|
44
|
+
from harness.runtime.learning.library import ADMISSIONS, CANDIDATES, EVIDENCE, FAILED, INDEX, SECTIONS, TESTS, Library, Revision, executable, rank
|
|
45
|
+
from harness.core import atomic_write
|
|
46
|
+
from harness.core.procs import file_lock
|
|
47
|
+
from harness.paths import home
|
|
48
|
+
|
|
49
|
+
SETTINGS = {
|
|
50
|
+
"learn": ("off", "on"),
|
|
51
|
+
"trigger": ("passive", "active"),
|
|
52
|
+
"evidence": ("test", "validated"),
|
|
53
|
+
"load": EVIDENCE,
|
|
54
|
+
"replay": (False, True),
|
|
55
|
+
"reflect": (False, True),
|
|
56
|
+
}
|
|
57
|
+
# An entry's kind and the library section it goes into (library.SECTIONS: its file suffix).
|
|
58
|
+
KINDS = {"note": "notes", "memory": "memory", "skill": "skills", "reactive": "reactive"}
|
|
59
|
+
ORIGINS = ("model", "reflection", "self_improve", "hand")
|
|
60
|
+
NAME = re.compile(r"^[a-z][a-z0-9_]{1,40}$")
|
|
61
|
+
# Notes go into every prompt, memory entries only when recall finds them: a longer entry is
|
|
62
|
+
# refused (the reflection request says so; before it did not, and half the proposals were lost).
|
|
63
|
+
NOTE_MAX_CHARS, MEMORY_MAX_CHARS = 600, 1200
|
|
64
|
+
# The longest situations list (memory/situations: the plan an active run learns by).
|
|
65
|
+
SITUATIONS_MAX_BYTES = 32 * 1024
|
|
66
|
+
# An attachment is at most this large in all, per proposal (a small trained model fits).
|
|
67
|
+
ATTACHMENT_BYTES = 64 * 1024 * 1024
|
|
68
|
+
# Test runs and uses kept per entry in the index.
|
|
69
|
+
KEPT = 20
|
|
70
|
+
_LOCK = threading.Lock()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def settings(runtime_library: dict | None) -> dict:
|
|
74
|
+
"""The learning settings of a [runtime.library] table, defaults filled in; ValueError
|
|
75
|
+
for a value it does not take."""
|
|
76
|
+
given = dict(runtime_library or {})
|
|
77
|
+
chosen = {}
|
|
78
|
+
for key, values in SETTINGS.items():
|
|
79
|
+
value = given.get(key, values[0])
|
|
80
|
+
if value not in values:
|
|
81
|
+
raise ValueError(f"runtime.library.{key} is one of {list(values)}, not {value!r}")
|
|
82
|
+
chosen[key] = value
|
|
83
|
+
return chosen
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def library_path(path) -> Path:
|
|
87
|
+
"""A library's directory: `$HARNESS_HOME/...` is under the harness's home."""
|
|
88
|
+
text = str(path)
|
|
89
|
+
for prefix in ("$HARNESS_HOME/", "${HARNESS_HOME}/"):
|
|
90
|
+
if text.startswith(prefix):
|
|
91
|
+
return home() / text[len(prefix):]
|
|
92
|
+
return Path(text)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def libraries() -> list[dict]:
|
|
96
|
+
"""The libraries under $HARNESS_HOME/libraries: [{"name", "directory", "version", "entries"}]."""
|
|
97
|
+
base = home() / "libraries"
|
|
98
|
+
rows = []
|
|
99
|
+
for path in sorted(base.iterdir()) if base.is_dir() else []:
|
|
100
|
+
if path.is_dir():
|
|
101
|
+
library = Library(path)
|
|
102
|
+
rows.append({"name": path.name, "directory": str(path), "version": library.version(),
|
|
103
|
+
"entries": sum(len(library.entries(section)) for section in SECTIONS)})
|
|
104
|
+
return rows
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def entry_key(kind: str, name: str) -> str:
|
|
108
|
+
"""Where an entry is in a library: "<section>/<name><suffix>" (skills/walk.py)."""
|
|
109
|
+
return f"{KINDS[kind]}/{name}{SECTIONS[KINDS[kind]]}"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass
|
|
113
|
+
class Candidate:
|
|
114
|
+
"""An entry proposed for the library."""
|
|
115
|
+
|
|
116
|
+
kind: str # note, memory, skill, reactive
|
|
117
|
+
name: str
|
|
118
|
+
text: str
|
|
119
|
+
test: str | None = None
|
|
120
|
+
summary: str = ""
|
|
121
|
+
attachments: dict = field(default_factory=dict) # {filename: bytes}
|
|
122
|
+
source: str = "passive" # active, passive
|
|
123
|
+
origin: str = "model" # ORIGINS
|
|
124
|
+
run: str | None = None # the episode it came from
|
|
125
|
+
bindings: tuple = ()
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def section(self) -> str:
|
|
129
|
+
return KINDS[self.kind]
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def filename(self) -> str:
|
|
133
|
+
return self.name + SECTIONS[self.section]
|
|
134
|
+
|
|
135
|
+
@property
|
|
136
|
+
def key(self) -> str:
|
|
137
|
+
return entry_key(self.kind, self.name)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def revision(self, dependencies=(), bindings=None):
|
|
141
|
+
return Revision(self.key, self.text.encode("utf-8"),
|
|
142
|
+
None if self.test is None else self.test.encode("utf-8"),
|
|
143
|
+
tuple((f"{self.section}/{self.name}.files/{name}", data)
|
|
144
|
+
for name, data in sorted(self.attachments.items())),
|
|
145
|
+
tuple(sorted(dependencies)), tuple(sorted(self.bindings if bindings is None else bindings)))
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def entry_name_problem(name) -> str | None:
|
|
149
|
+
"""Why `name` cannot name a library entry (None when it can): the learner's one rule,
|
|
150
|
+
also checked by the learning skill before an entry's test acts."""
|
|
151
|
+
if not isinstance(name, str) or not NAME.match(name):
|
|
152
|
+
return f"an entry's name is lowercase letters, digits and _ (2-41, a letter first), not {name!r}"
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def candidate(kind, name, text, test=None, *, summary="", attachments=None, source="passive", origin="model",
|
|
157
|
+
run=None) -> Candidate:
|
|
158
|
+
"""A Candidate, checked: its kind and name, a code entry that parses (and has a test,
|
|
159
|
+
when it is code), a note or memory entry not too long, attachments within the cap."""
|
|
160
|
+
if kind not in KINDS:
|
|
161
|
+
raise ValueError(f"an entry's kind is one of {list(KINDS)}, not {kind!r}")
|
|
162
|
+
problem = entry_name_problem(name)
|
|
163
|
+
if problem:
|
|
164
|
+
raise ValueError(problem)
|
|
165
|
+
if source not in ("active", "passive") or origin not in ORIGINS:
|
|
166
|
+
raise ValueError(f"source is active or passive and origin one of {list(ORIGINS)}")
|
|
167
|
+
text = str(text).strip() + "\n"
|
|
168
|
+
if kind in ("skill", "reactive"):
|
|
169
|
+
import ast
|
|
170
|
+
|
|
171
|
+
try:
|
|
172
|
+
ast.parse(text)
|
|
173
|
+
if test is not None:
|
|
174
|
+
ast.parse(test)
|
|
175
|
+
except SyntaxError as error:
|
|
176
|
+
raise ValueError(f"the code does not parse: {error}") from None
|
|
177
|
+
else:
|
|
178
|
+
# The situations list (runtime/learning/active.py) is a plan, not a lesson: it gets more room.
|
|
179
|
+
situations = kind == "memory" and name == "situations"
|
|
180
|
+
limit = SITUATIONS_MAX_BYTES if situations else (NOTE_MAX_CHARS if kind == "note" else MEMORY_MAX_CHARS) + 1
|
|
181
|
+
if (len(text.encode("utf-8")) if situations else len(text)) > limit:
|
|
182
|
+
raise ValueError(f"a {kind} is at most {limit} characters")
|
|
183
|
+
files = {}
|
|
184
|
+
for filename, value in (attachments or {}).items():
|
|
185
|
+
if not re.match(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,80}$", str(filename)):
|
|
186
|
+
raise ValueError(f"an attachment has a plain file name, not {filename!r}")
|
|
187
|
+
files[str(filename)] = value.encode("utf-8") if isinstance(value, str) else bytes(value)
|
|
188
|
+
if sum(len(value) for value in files.values()) > ATTACHMENT_BYTES:
|
|
189
|
+
raise ValueError(f"attachments are at most {ATTACHMENT_BYTES // (1024 * 1024)} MB in all")
|
|
190
|
+
return Candidate(kind, name, text, None if test is None else str(test).strip() + "\n", str(summary or ""),
|
|
191
|
+
files, source, origin, run)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _now() -> float:
|
|
195
|
+
return round(time.time(), 3)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class Learner:
|
|
199
|
+
"""The library at `directory` with what a run's learning settings allow."""
|
|
200
|
+
|
|
201
|
+
def __init__(self, directory, runtime_library: dict | None = None, create: bool = False):
|
|
202
|
+
self.directory = library_path(directory).resolve()
|
|
203
|
+
if create:
|
|
204
|
+
for section in SECTIONS:
|
|
205
|
+
(self.directory / section).mkdir(parents=True, exist_ok=True)
|
|
206
|
+
self.settings = settings(runtime_library)
|
|
207
|
+
self.layers = tuple((runtime_library or {}).get("layers", ("notes", "skills", "reactive")))
|
|
208
|
+
self.library = Library(self.directory, self.layers, self.settings["load"])
|
|
209
|
+
|
|
210
|
+
def version(self) -> str:
|
|
211
|
+
return self.library.version()
|
|
212
|
+
|
|
213
|
+
# -- the index ----------------------------------------------------------------------
|
|
214
|
+
|
|
215
|
+
def index(self) -> dict:
|
|
216
|
+
return self.library.index()
|
|
217
|
+
|
|
218
|
+
def _save_index(self, index: dict) -> None:
|
|
219
|
+
atomic_write(self.directory / INDEX, json.dumps(index, indent=1, sort_keys=True))
|
|
220
|
+
|
|
221
|
+
def migrate_experience(self):
|
|
222
|
+
from harness.runtime.learning.experience import MIGRATION, legacy_entries
|
|
223
|
+
|
|
224
|
+
index = self.index()
|
|
225
|
+
if index.get(MIGRATION) or not list(legacy_entries(self.directory, index)):
|
|
226
|
+
return
|
|
227
|
+
with _LOCK, file_lock(self.directory / ".library.lock"):
|
|
228
|
+
index = self.index()
|
|
229
|
+
if index.get(MIGRATION):
|
|
230
|
+
return
|
|
231
|
+
for item in list(legacy_entries(self.directory, index)):
|
|
232
|
+
key = item["id"]
|
|
233
|
+
if key not in index:
|
|
234
|
+
revision = Revision(key, item["text"].encode("utf-8")).save(self.directory)
|
|
235
|
+
index[key] = {"memory": item, "evidence": "validated", "origin": "legacy", "revision": revision}
|
|
236
|
+
index[MIGRATION] = True
|
|
237
|
+
self._save_index(index)
|
|
238
|
+
|
|
239
|
+
def store_memory(self, item, *, forgotten=False):
|
|
240
|
+
key = item["id"]
|
|
241
|
+
if not isinstance(key, str) or not re.fullmatch(r"memory/[^/\\]+\.md", key):
|
|
242
|
+
raise ValueError("invalid persistent memory key")
|
|
243
|
+
if "notes" not in self.layers or self.settings["learn"] != "on":
|
|
244
|
+
raise ValueError("this library is read-only")
|
|
245
|
+
with _LOCK, file_lock(self.directory / ".library.lock"):
|
|
246
|
+
index = self.index()
|
|
247
|
+
row = dict(index.get(key) or {})
|
|
248
|
+
if forgotten:
|
|
249
|
+
row["forgotten"] = True
|
|
250
|
+
else:
|
|
251
|
+
content = dict(item, scope="library")
|
|
252
|
+
data = content["text"].encode("utf-8")
|
|
253
|
+
if content.get("name") == "situations" and len(data) > SITUATIONS_MAX_BYTES:
|
|
254
|
+
raise ValueError("the situations list is limited to 32 KiB")
|
|
255
|
+
revision = Revision(key, data).save(self.directory)
|
|
256
|
+
row.update(evidence="proposed", origin="memory", memory=content, revision=revision,
|
|
257
|
+
forgotten=False)
|
|
258
|
+
index[key] = row
|
|
259
|
+
self._save_index(index)
|
|
260
|
+
return {**item, "scope": "library", "evidence": "proposed", "forgotten": forgotten}
|
|
261
|
+
|
|
262
|
+
def entries(self, *, snapshot=None) -> list[dict]:
|
|
263
|
+
"""Every entry in the library (loaded or not) with its evidence, source, origin and
|
|
264
|
+
summary, and the candidates waiting: [{"section", "name", "file", "evidence",
|
|
265
|
+
"source", "origin", "summary", "admitted_at", "tests", "passed", "uses", "loaded"}]."""
|
|
266
|
+
captured = snapshot if snapshot is not None else self.library.capture()
|
|
267
|
+
index = captured.index
|
|
268
|
+
loaded = {item.key for item in captured.entries}
|
|
269
|
+
rows = []
|
|
270
|
+
for section, suffix in SECTIONS.items():
|
|
271
|
+
for root, waiting in ((self.directory / section, False), (self.directory / CANDIDATES / section, True)):
|
|
272
|
+
prefix = f"{CANDIDATES}/{section}/" if waiting else f"{section}/"
|
|
273
|
+
if snapshot is None:
|
|
274
|
+
paths = set(root.glob(f"*{suffix}"))
|
|
275
|
+
paths.update(root / key.removeprefix(prefix) for key, row in index.items()
|
|
276
|
+
if key.startswith(prefix) and row.get("revision"))
|
|
277
|
+
else:
|
|
278
|
+
paths = {root / key.removeprefix(prefix) for key in loaded if key.startswith(prefix)}
|
|
279
|
+
for path in sorted(paths):
|
|
280
|
+
key = f"{section}/{path.name}"
|
|
281
|
+
row = index.get(f"{CANDIDATES}/{key}" if waiting else key) or {}
|
|
282
|
+
if row.get("forgotten"):
|
|
283
|
+
continue
|
|
284
|
+
tests = row.get("tests") or []
|
|
285
|
+
uses = (row.get("uses_by_revision") or {}).get(row.get("revision")) or {}
|
|
286
|
+
rows.append({"section": section, "name": (row.get("memory") or {}).get("name") or path.stem, "file": key,
|
|
287
|
+
"evidence": row.get("evidence", "proposed" if waiting else "validated"),
|
|
288
|
+
"revision": row.get("revision"),
|
|
289
|
+
"verified": bool(row.get("proof") and row["proof"].get("revision") == row.get("revision")
|
|
290
|
+
and row.get("evidence") in ("provisional", "replayed", "validated")),
|
|
291
|
+
"waiting": waiting, "source": row.get("source", "-"),
|
|
292
|
+
"origin": row.get("origin", "hand"), "summary": row.get("summary", ""),
|
|
293
|
+
"admitted_at": row.get("admitted_at"), "tests": len(tests),
|
|
294
|
+
"passed": sum(1 for run in tests if run.get("ok")),
|
|
295
|
+
"uses": int(uses.get("ok", 0)) + int(uses.get("failed", 0)),
|
|
296
|
+
"success_rate": uses.get("rate"), "loaded": key in loaded and not waiting})
|
|
297
|
+
return rows
|
|
298
|
+
|
|
299
|
+
# -- admitting --------------------------------------------------------------------
|
|
300
|
+
|
|
301
|
+
def wanted(self, candidate: Candidate, evidence: str) -> bool:
|
|
302
|
+
"""Whether an entry with this evidence enters the library now (else it waits)."""
|
|
303
|
+
if self.settings["evidence"] == "validated":
|
|
304
|
+
return evidence == "validated"
|
|
305
|
+
return rank(evidence) >= rank("proposed")
|
|
306
|
+
|
|
307
|
+
def admit(self, candidate: Candidate, evidence: str, *, test_run: dict | None = None,
|
|
308
|
+
gain: float | None = None, why: str = "", proof=None, dependencies=(), bindings=None,
|
|
309
|
+
pending=False, expected=None, check=None) -> dict:
|
|
310
|
+
"""Put a candidate into the library with this evidence (or, when the settings want
|
|
311
|
+
more, into candidates/): its files written, its index row (source, origin, the test
|
|
312
|
+
run, the gain) kept. {"admitted", "evidence", "version_before", "version_after",
|
|
313
|
+
"key"}."""
|
|
314
|
+
if evidence not in EVIDENCE:
|
|
315
|
+
raise ValueError(f"evidence is one of {list(EVIDENCE)}, not {evidence!r}")
|
|
316
|
+
with _LOCK, file_lock(self.directory / ".library.lock"):
|
|
317
|
+
before = self.version()
|
|
318
|
+
revision = candidate.revision(dependencies, bindings)
|
|
319
|
+
if proof is not None and (proof.get("revision") != revision.revision or not proof.get("conditions")):
|
|
320
|
+
raise ValueError("verification proof does not cover this revision")
|
|
321
|
+
index = self.index()
|
|
322
|
+
waiting = index.get(f"{CANDIDATES}/{candidate.key}") or {}
|
|
323
|
+
if expected is not None and (waiting.get("revision") != expected
|
|
324
|
+
or waiting.get("evidence") == FAILED):
|
|
325
|
+
raise ValueError("the pending proposal changed while being verified")
|
|
326
|
+
if proof is not None and expected is None and waiting.get("revision") not in (
|
|
327
|
+
None, candidate.revision().revision, revision.revision):
|
|
328
|
+
raise ValueError("the pending proposal changed while being verified")
|
|
329
|
+
admitted = not pending and self.wanted(candidate, evidence)
|
|
330
|
+
stored_evidence = evidence if admitted or proof is not None else "proposed"
|
|
331
|
+
# Preserve the imported revision before writing the editable file mirror.
|
|
332
|
+
previous = index.get(candidate.key) or {}
|
|
333
|
+
if admitted and not previous.get("revision"):
|
|
334
|
+
old = next((item for item in self.library.capture().entries if item.key == candidate.key), None)
|
|
335
|
+
if old is not None:
|
|
336
|
+
index[candidate.key] = {**previous, "revision": old.save(self.directory)}
|
|
337
|
+
self._save_index(index)
|
|
338
|
+
root = self.directory if admitted else self.directory / CANDIDATES
|
|
339
|
+
target = root / candidate.section / candidate.filename
|
|
340
|
+
replaced = target.is_file() and target.read_bytes() != candidate.text.encode("utf-8")
|
|
341
|
+
if expected is None:
|
|
342
|
+
atomic_write(target, candidate.text)
|
|
343
|
+
if candidate.test is not None:
|
|
344
|
+
atomic_write(root / TESTS / f"{candidate.name}.py", candidate.test)
|
|
345
|
+
folder = root / candidate.section / f"{candidate.name}.files"
|
|
346
|
+
if candidate.attachments:
|
|
347
|
+
shutil.rmtree(folder, ignore_errors=True)
|
|
348
|
+
for filename, data in candidate.attachments.items():
|
|
349
|
+
atomic_write(folder / filename, data)
|
|
350
|
+
index_key = candidate.key if admitted else f"{CANDIDATES}/{candidate.key}"
|
|
351
|
+
row = dict(index.get(index_key) or {})
|
|
352
|
+
if admitted:
|
|
353
|
+
index.pop(f"{CANDIDATES}/{candidate.key}", None)
|
|
354
|
+
revision_id = revision.save(self.directory)
|
|
355
|
+
if row.get("revision") != revision_id:
|
|
356
|
+
row.pop("proof", None)
|
|
357
|
+
row.pop("tests", None)
|
|
358
|
+
row["revision"] = revision_id
|
|
359
|
+
if proof is not None:
|
|
360
|
+
row["proof"] = proof
|
|
361
|
+
history = list(row.get("history") or [])
|
|
362
|
+
history.append({"at": _now(), "evidence": stored_evidence,
|
|
363
|
+
"revision": revision_id, **({"proof": proof} if proof is not None else {}),
|
|
364
|
+
**({"gain": round(gain, 4)} if gain is not None else {}), **({"why": why} if why else {})})
|
|
365
|
+
if admitted and replaced and (row.get("uses") or {}).get("sha256"):
|
|
366
|
+
history[-1]["previous_uses"] = row.pop("uses")
|
|
367
|
+
row.update(evidence=stored_evidence, source=candidate.source,
|
|
368
|
+
origin=candidate.origin, summary=candidate.summary or row.get("summary", ""),
|
|
369
|
+
admitted_at=row.get("admitted_at") or _now(), updated_at=_now(), history=history[-KEPT:],
|
|
370
|
+
**({"run": candidate.run} if candidate.run else {}))
|
|
371
|
+
if test_run is not None:
|
|
372
|
+
row["tests"] = [*(row.get("tests") or []), {"at": _now(), **test_run}][-KEPT:]
|
|
373
|
+
index[index_key] = row
|
|
374
|
+
if check is not None:
|
|
375
|
+
check()
|
|
376
|
+
self._save_index(index)
|
|
377
|
+
if admitted:
|
|
378
|
+
for stale in (self.directory / CANDIDATES / candidate.section / candidate.filename,
|
|
379
|
+
self.directory / CANDIDATES / TESTS / f"{candidate.name}.py"):
|
|
380
|
+
stale.unlink(missing_ok=True)
|
|
381
|
+
after = self.version()
|
|
382
|
+
outcome = {"admitted": admitted, "evidence": row["evidence"], "version_before": before,
|
|
383
|
+
"version_after": after, "key": candidate.key, "row": row, "revision": revision_id,
|
|
384
|
+
"sha256": hashlib.sha256(candidate.text.encode("utf-8")).hexdigest()}
|
|
385
|
+
if admitted:
|
|
386
|
+
# The library's log of what entered it (the audit's evidence that an entry
|
|
387
|
+
# was admitted, benches/minecraft/audit.py): section, file, its bytes' sha256.
|
|
388
|
+
with (self.directory / ADMISSIONS).open("a", encoding="utf-8") as log:
|
|
389
|
+
log.write(json.dumps({**self.record_of(outcome), "source": candidate.source,
|
|
390
|
+
"origin": candidate.origin, "at": _now()}) + "\n")
|
|
391
|
+
return outcome
|
|
392
|
+
|
|
393
|
+
@staticmethod
|
|
394
|
+
def record_of(outcome: dict) -> dict:
|
|
395
|
+
"""An admission as the library's log and the trace record it: section, name (the
|
|
396
|
+
entry's file), sha256 of its bytes, the version after it, its evidence, and whether
|
|
397
|
+
it entered the library (or waits)."""
|
|
398
|
+
section, _, filename = outcome["key"].partition("/")
|
|
399
|
+
return {"section": section, "name": filename, "sha256": outcome["sha256"], "version": outcome["version_after"],
|
|
400
|
+
"version_before": outcome["version_before"], "evidence": outcome["evidence"],
|
|
401
|
+
"admitted": outcome["admitted"], "revision": outcome.get("revision"),
|
|
402
|
+
**({"proof": outcome["row"]["proof"]} if (outcome.get("row") or {}).get("proof") else {})}
|
|
403
|
+
|
|
404
|
+
def grade(self, key: str, evidence: str, *, test_run: dict | None = None, gain: float | None = None,
|
|
405
|
+
why: str = "", sha256: str | None = None) -> dict | None:
|
|
406
|
+
"""Set an entry's evidence (a replay passed: replayed; failed: failed; development
|
|
407
|
+
seeds: validated), recorded in its history. Its index row, or None."""
|
|
408
|
+
if evidence not in (*EVIDENCE, FAILED):
|
|
409
|
+
raise ValueError(f"evidence is one of {[*EVIDENCE, FAILED]}, not {evidence!r}")
|
|
410
|
+
with _LOCK, file_lock(self.directory / ".library.lock"):
|
|
411
|
+
if sha256 is not None:
|
|
412
|
+
section, _, filename = key.partition("/")
|
|
413
|
+
if (section not in SECTIONS or not filename.endswith(SECTIONS[section])
|
|
414
|
+
or any(char in filename for char in "/\\:")):
|
|
415
|
+
raise ValueError(f"invalid library revision key {key!r}")
|
|
416
|
+
path = self.directory / section / filename
|
|
417
|
+
if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != sha256:
|
|
418
|
+
raise ValueError(f"{key} changed while it was being graded")
|
|
419
|
+
index = self.index()
|
|
420
|
+
row = dict(index.get(key) or {"source": "-", "origin": "hand"})
|
|
421
|
+
row["evidence"] = evidence
|
|
422
|
+
row["history"] = [*(row.get("history") or []), {
|
|
423
|
+
"at": _now(), "evidence": evidence, **({"gain": round(gain, 4)} if gain is not None else {}),
|
|
424
|
+
**({"sha256": sha256} if sha256 is not None else {}),
|
|
425
|
+
**({"revision": row["revision"]} if row.get("revision") else {}),
|
|
426
|
+
**({"why": why} if why else {})}][-KEPT:]
|
|
427
|
+
if test_run is not None:
|
|
428
|
+
row["tests"] = [*(row.get("tests") or []), {"at": _now(), **test_run}][-KEPT:]
|
|
429
|
+
index[key] = row
|
|
430
|
+
self._save_index(index)
|
|
431
|
+
return row
|
|
432
|
+
|
|
433
|
+
def remove(self, key: str, why: str = "") -> None:
|
|
434
|
+
"""Take an entry (or a waiting candidate) out of the library: its file, test and
|
|
435
|
+
attachments; its index row says why."""
|
|
436
|
+
section, _, filename = key.partition("/")
|
|
437
|
+
stem = Path(filename).stem
|
|
438
|
+
with _LOCK, file_lock(self.directory / ".library.lock"):
|
|
439
|
+
for root in (self.directory, self.directory / CANDIDATES):
|
|
440
|
+
(root / section / filename).unlink(missing_ok=True)
|
|
441
|
+
(root / TESTS / f"{stem}.py").unlink(missing_ok=True)
|
|
442
|
+
shutil.rmtree(root / section / f"{stem}.files", ignore_errors=True)
|
|
443
|
+
index = self.index()
|
|
444
|
+
if key in index:
|
|
445
|
+
index[key] = {**index[key], "evidence": FAILED, "removed": _now(), **({"why": why} if why else {})}
|
|
446
|
+
self._save_index(index)
|
|
447
|
+
|
|
448
|
+
def grade_development(self, candidate, revision, *, pending, score):
|
|
449
|
+
"""Accept development evidence for exactly the verified candidate that ran."""
|
|
450
|
+
key = candidate.key
|
|
451
|
+
current = self.index().get(f"{CANDIDATES}/{key}" if pending else key) or {}
|
|
452
|
+
proof = current.get("proof") or {}
|
|
453
|
+
if proof.get("revision") != revision or candidate.revision().revision != revision:
|
|
454
|
+
return False
|
|
455
|
+
proof = {**proof, "development": score}
|
|
456
|
+
if pending:
|
|
457
|
+
self.admit(candidate, "validated", proof=proof, expected=revision, gain=score["gain"],
|
|
458
|
+
why="paired development scores")
|
|
459
|
+
return True
|
|
460
|
+
with _LOCK, file_lock(self.directory / ".library.lock"):
|
|
461
|
+
index = self.index()
|
|
462
|
+
row = index.get(key) or {}
|
|
463
|
+
if row != current or row.get("evidence") not in ("provisional", "replayed"):
|
|
464
|
+
return False
|
|
465
|
+
row.update(evidence="validated", proof=proof)
|
|
466
|
+
row["history"] = [*(row.get("history") or []),
|
|
467
|
+
{"at": _now(), "revision": revision, "evidence": "validated",
|
|
468
|
+
"development": score}][-KEPT:]
|
|
469
|
+
self._save_index(index)
|
|
470
|
+
return True
|
|
471
|
+
|
|
472
|
+
def record_trial(self, key, expected, item, *, pending, evidence, receipt, check):
|
|
473
|
+
"""Record a verdict on the immutable revision that actually ran."""
|
|
474
|
+
if item.key != key or receipt["revision"] != item.revision:
|
|
475
|
+
raise ValueError("the trial receipt has a different revision")
|
|
476
|
+
with _LOCK, file_lock(self.directory / ".library.lock"):
|
|
477
|
+
index = self.index()
|
|
478
|
+
index_key = f"{CANDIDATES}/{key}" if pending else key
|
|
479
|
+
row = index.get(index_key)
|
|
480
|
+
if row is None or row.get("revision") != expected:
|
|
481
|
+
return False
|
|
482
|
+
if row.get("evidence") != ("proposed" if pending else "provisional"):
|
|
483
|
+
return False
|
|
484
|
+
revision = item.save(self.directory)
|
|
485
|
+
history = list(row.get("history") or [])
|
|
486
|
+
result = {"at": _now(), "revision": revision, "evidence": evidence, "receipt": receipt}
|
|
487
|
+
row.update(revision=revision, evidence=evidence, history=[*history, result][-KEPT:],
|
|
488
|
+
tests=[*(row.get("tests") or []), receipt][-KEPT:])
|
|
489
|
+
if evidence == FAILED:
|
|
490
|
+
row.pop("proof", None)
|
|
491
|
+
if not pending:
|
|
492
|
+
failed = {revision, *(grade.get("revision") for grade in history
|
|
493
|
+
if grade.get("evidence") == FAILED)}
|
|
494
|
+
earlier = next((grade for grade in reversed(history)
|
|
495
|
+
if grade.get("revision") not in failed and grade.get("proof")), None)
|
|
496
|
+
if earlier is not None:
|
|
497
|
+
previous = Revision.read(self.directory, earlier["revision"])
|
|
498
|
+
if previous.key == key:
|
|
499
|
+
index.setdefault(f"{CANDIDATES}/{key}", dict(row))
|
|
500
|
+
row.update(revision=previous.revision, evidence=earlier["evidence"],
|
|
501
|
+
proof=earlier["proof"], tests=[])
|
|
502
|
+
check()
|
|
503
|
+
self._save_index(index)
|
|
504
|
+
return True
|
|
505
|
+
|
|
506
|
+
def record(self, key: str, ok: bool, detail=None) -> None:
|
|
507
|
+
"""A use of an entry in play: counted, its rate kept."""
|
|
508
|
+
self.record_uses(key, [(ok, detail)])
|
|
509
|
+
|
|
510
|
+
def record_uses(self, key, results, digest=None, *, revision=None, run=None):
|
|
511
|
+
"""Record outcomes once under the version that ran, including replaced versions."""
|
|
512
|
+
with _LOCK, file_lock(self.directory / ".library.lock"):
|
|
513
|
+
index = self.index()
|
|
514
|
+
row = dict(index.get(key) or {"source": "-", "origin": "hand", "evidence": "validated"})
|
|
515
|
+
if revision is None and digest is not None:
|
|
516
|
+
matching = [item for item in self.library.capture().entries
|
|
517
|
+
if item.key == key and item.source_sha256 == digest]
|
|
518
|
+
if not matching:
|
|
519
|
+
return 0
|
|
520
|
+
revision = matching[0].revision
|
|
521
|
+
versions = row.setdefault("uses_by_revision", {})
|
|
522
|
+
uses = versions.setdefault(revision or "unknown", {"ok": 0, "failed": 0})
|
|
523
|
+
if run is not None:
|
|
524
|
+
processed = uses.setdefault("runs", [])
|
|
525
|
+
if run in processed:
|
|
526
|
+
return 0
|
|
527
|
+
processed.append(run)
|
|
528
|
+
for result in results:
|
|
529
|
+
ok, detail = result[:2]
|
|
530
|
+
uses["ok" if ok else "failed"] += 1
|
|
531
|
+
if len(result) > 2:
|
|
532
|
+
uses["seconds"] = round(uses.get("seconds", 0) + (result[2] or 0), 3)
|
|
533
|
+
if detail:
|
|
534
|
+
uses["last"] = str(detail)[:200]
|
|
535
|
+
count = uses["ok"] + uses["failed"]
|
|
536
|
+
uses["rate"] = round(uses["ok"] / count, 3) if count else None
|
|
537
|
+
index[key] = row
|
|
538
|
+
self._save_index(index)
|
|
539
|
+
return len(results)
|
|
540
|
+
|
|
541
|
+
# -- reading --------------------------------------------------------------------------
|
|
542
|
+
|
|
543
|
+
def candidate_from(self, key: str, *, source="passive", origin="hand", pending=None) -> Candidate | None:
|
|
544
|
+
"""Read a waiting or admitted candidate from its pinned revision."""
|
|
545
|
+
section, _, filename = key.partition("/")
|
|
546
|
+
kind = {value: name for name, value in KINDS.items()}[section]
|
|
547
|
+
stem = Path(filename).stem
|
|
548
|
+
index = self.index()
|
|
549
|
+
for waiting in (True, False) if pending is None else (pending,):
|
|
550
|
+
row = index.get(f"{CANDIDATES}/{key}" if waiting else key) or {}
|
|
551
|
+
if row.get("forgotten"):
|
|
552
|
+
continue
|
|
553
|
+
if row.get("revision"):
|
|
554
|
+
item = Revision.read(self.directory, row["revision"])
|
|
555
|
+
if item.key != key:
|
|
556
|
+
raise ValueError(f"library index points {key} at a different candidate")
|
|
557
|
+
prefix = f"{section}/{stem}.files/"
|
|
558
|
+
return Candidate(kind, stem, item.source.decode("utf-8"),
|
|
559
|
+
item.test.decode("utf-8") if item.test is not None else None,
|
|
560
|
+
row.get("summary", ""), {name.removeprefix(prefix): data
|
|
561
|
+
for name, data in item.attachments},
|
|
562
|
+
row.get("source", source), row.get("origin", origin), row.get("run"), item.bindings)
|
|
563
|
+
root = self.directory / CANDIDATES if waiting else self.directory
|
|
564
|
+
path = root / section / filename
|
|
565
|
+
if path.is_file():
|
|
566
|
+
test = root / TESTS / f"{stem}.py"
|
|
567
|
+
folder = root / section / f"{stem}.files"
|
|
568
|
+
return Candidate(kind, stem, path.read_text(encoding="utf-8"),
|
|
569
|
+
test.read_text(encoding="utf-8") if test.is_file() else None,
|
|
570
|
+
row.get("summary", ""),
|
|
571
|
+
{p.name: p.read_bytes() for p in folder.iterdir() if p.is_file()} if folder.is_dir()
|
|
572
|
+
else {}, row.get("source", source), row.get("origin", origin), row.get("run"))
|
|
573
|
+
return None
|
|
574
|
+
|
|
575
|
+
def attachment(self, sha256: str) -> bytes:
|
|
576
|
+
"""The bytes of the attachment with this sha256, wherever it is in the library."""
|
|
577
|
+
for item in self.library.capture().entries:
|
|
578
|
+
for _, data in item.attachments:
|
|
579
|
+
if hashlib.sha256(data).hexdigest() == sha256:
|
|
580
|
+
return data
|
|
581
|
+
for section in SECTIONS:
|
|
582
|
+
for path in sorted((self.directory / CANDIDATES / section).glob("*.files/**/*")):
|
|
583
|
+
if path.is_file():
|
|
584
|
+
data = path.read_bytes()
|
|
585
|
+
if hashlib.sha256(data).hexdigest() == sha256:
|
|
586
|
+
return data
|
|
587
|
+
raise KeyError(f"no attachment {sha256} in {self.directory}")
|
|
588
|
+
|
|
589
|
+
def trial(self, candidate: Candidate, root: Path | None = None, *, proof=None) -> Path:
|
|
590
|
+
"""A copy of the library with the candidate in it, to measure it on development seeds."""
|
|
591
|
+
trial = Path(tempfile.mkdtemp(prefix="library-trial-", dir=root))
|
|
592
|
+
shutil.copytree(self.directory, trial, dirs_exist_ok=True, ignore=shutil.ignore_patterns(CANDIDATES))
|
|
593
|
+
copy = Learner(trial, {**self.settings, "evidence": "test", "layers": list(self.layers)})
|
|
594
|
+
if proof is None:
|
|
595
|
+
from dataclasses import replace
|
|
596
|
+
|
|
597
|
+
candidate = replace(candidate, origin="hand")
|
|
598
|
+
copy.admit(candidate, "provisional", proof=proof)
|
|
599
|
+
return trial
|
|
600
|
+
|
|
601
|
+
def without(self, key: str, root: Path | None = None) -> Path:
|
|
602
|
+
"""A copy of the library without one entry, to measure what it adds."""
|
|
603
|
+
trial = Path(tempfile.mkdtemp(prefix="library-trial-", dir=root))
|
|
604
|
+
shutil.copytree(self.directory, trial, dirs_exist_ok=True, ignore=shutil.ignore_patterns(CANDIDATES))
|
|
605
|
+
Learner(trial, {"layers": list(self.layers)}).remove(key, "measured without it")
|
|
606
|
+
return trial
|
|
607
|
+
|
|
608
|
+
# -- a run's requests (the host answers the kernel's, runtime/conversation.py) ----------
|
|
609
|
+
|
|
610
|
+
def handle(self, request: dict, *, source: str, run: str | None = None):
|
|
611
|
+
"""What the kernel asks: admit (a candidate whose test, if any, passed in the
|
|
612
|
+
kernel), grade (a replay's result), record (a use), entries, version, attachment."""
|
|
613
|
+
action = request.get("action")
|
|
614
|
+
if action == "admit":
|
|
615
|
+
files = {name: base64.b64decode(value) for name, value in (request.get("attachments") or {}).items()}
|
|
616
|
+
entry = candidate(request.get("entry") or request.get("kind"), request["name"], request["text"], request.get("test"),
|
|
617
|
+
summary=request.get("summary", ""), attachments=files, source=source, origin="model",
|
|
618
|
+
run=run)
|
|
619
|
+
if executable(entry.key):
|
|
620
|
+
if not entry.test:
|
|
621
|
+
raise ValueError("an executable proposal needs a verification condition")
|
|
622
|
+
return self.admit(entry, "proposed", pending=True)
|
|
623
|
+
return self.admit(entry, "proposed")
|
|
624
|
+
if action == "grade":
|
|
625
|
+
raise ValueError("verification grades require a trusted host receipt")
|
|
626
|
+
if action == "record":
|
|
627
|
+
self.record(request["key"], bool(request.get("ok")), request.get("detail"))
|
|
628
|
+
return None
|
|
629
|
+
if action == "entries":
|
|
630
|
+
return self.entries()
|
|
631
|
+
if action == "version":
|
|
632
|
+
return self.version()
|
|
633
|
+
if action == "attachment":
|
|
634
|
+
return base64.b64encode(self.attachment(request["sha256"])).decode("ascii")
|
|
635
|
+
raise ValueError(f"unknown library request {action!r}")
|
|
636
|
+
|
|
637
|
+
# -- passive: reflection after an episode -------------------------------------------
|
|
638
|
+
|
|
639
|
+
def reflect(self, chat, run: Path, names, *, since=0, holdings=None) -> tuple[Candidate | None, str]:
|
|
640
|
+
"""One candidate from an episode (reflect.py), checked as every candidate is
|
|
641
|
+
(candidate()), as passive learning; (None, reply) when the model proposes none, or
|
|
642
|
+
one that does not pass."""
|
|
643
|
+
from harness.runtime.learning.reflect import propose
|
|
644
|
+
|
|
645
|
+
found, reply = propose(chat, Path(run), self.library, names, since=since, holdings=holdings)
|
|
646
|
+
if found is None:
|
|
647
|
+
return None, reply
|
|
648
|
+
try:
|
|
649
|
+
return candidate(*found, source=self.settings["trigger"], origin="reflection", run=str(run)), reply
|
|
650
|
+
except ValueError:
|
|
651
|
+
return None, reply
|