@lmmslab/harness 0.81.1 → 0.83.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -35
- package/package.json +2 -2
- package/python/harness/README.md +3 -3
- package/python/harness/__init__.py +1 -1
- package/python/harness/builtin/configs/assistant.toml +0 -1
- package/python/harness/builtin/configs/default.toml +1 -1
- package/python/harness/builtin/prompts/conversation/core.md +2 -2
- package/python/harness/builtin/prompts/core/core.md +197 -45
- package/python/harness/builtin/prompts/core/pack.toml +2 -2
- package/python/harness/builtin/prompts/reflect/core.md +4 -13
- package/python/harness/builtin/prompts/reflect/pack.toml +3 -2
- package/python/harness/builtin/prompts/shared/learning-goal.md +5 -29
- package/python/harness/builtin/prompts/shared/learning.md +8 -10
- package/python/harness/builtin/prompts/shared/web.md +11 -0
- package/python/harness/builtin/skills/agents/__init__.py +24 -22
- package/python/harness/builtin/skills/entries/__init__.py +43 -28
- package/python/harness/builtin/skills/entries/skill.toml +2 -3
- package/python/harness/builtin/skills/environments/__init__.py +2 -9
- package/python/harness/builtin/skills/mcp/__init__.py +2 -3
- package/python/harness/builtin/skills/memory/__init__.py +2 -5
- package/python/harness/builtin/skills/web/__init__.py +92 -0
- package/python/harness/builtin/skills/web/skill.toml +9 -0
- package/python/harness/cli.py +85 -17
- package/python/harness/config/agent.py +36 -9
- package/python/harness/core/episodes.py +5 -4
- package/python/harness/core/lanes.py +3 -0
- package/python/harness/core/memory.py +5 -1
- package/python/harness/core/remote.py +4 -3
- package/python/harness/core/worker.py +58 -3
- package/python/harness/extensions.py +23 -2
- package/python/harness/host/__init__.py +43 -1
- package/python/harness/host/client.py +36 -6
- package/python/harness/host/http.py +19 -0
- package/python/harness/host/server.py +17 -3
- package/python/harness/host/sessions.py +3 -2
- package/python/harness/host/tasks.py +107 -26
- package/python/harness/host/workspaces.py +69 -7
- package/python/harness/model_list.py +1 -1
- package/python/harness/packages/manager.py +5 -1
- package/python/harness/providers.py +14 -5
- package/python/harness/runtime/decision/loop.py +105 -117
- package/python/harness/runtime/decision/messages.py +158 -55
- package/python/harness/runtime/decision/model.py +85 -22
- package/python/harness/runtime/decision/notices.py +66 -0
- package/python/harness/runtime/decision/prompts.py +15 -11
- package/python/harness/runtime/decision/protocol.py +12 -6
- package/python/harness/runtime/decision/thoughts.py +10 -16
- package/python/harness/runtime/events.py +28 -13
- package/python/harness/runtime/execution/attached.py +1 -1
- package/python/harness/runtime/execution/control.py +17 -35
- package/python/harness/runtime/execution/entries.py +204 -124
- package/python/harness/runtime/execution/kernel.py +471 -281
- package/python/harness/runtime/execution/managed.py +231 -0
- package/python/harness/runtime/execution/noenv.py +2 -2
- package/python/harness/runtime/execution/results.py +2 -0
- package/python/harness/runtime/execution/session.py +284 -115
- package/python/harness/runtime/learning/active.py +41 -50
- package/python/harness/runtime/learning/fixtures.py +2 -2
- package/python/harness/runtime/learning/learner.py +75 -2
- package/python/harness/runtime/learning/library.py +77 -10
- package/python/harness/runtime/learning/views.py +12 -1
- package/python/harness/runtime/reactive/__init__.py +1 -1
- package/python/harness/runtime/reactive/rules.py +29 -394
- package/python/harness/runtime/reactive/tree.py +611 -0
- package/python/harness/runtime/safety/approval.py +8 -3
- package/python/harness/runtime/safety/permissions.py +242 -0
- package/python/harness/runtime/status.py +7 -2
- package/python/harness/runtime/task/delegation.py +97 -58
- package/python/harness/runtime/task/environments.py +3 -4
- package/python/harness/runtime/task/inbound.py +64 -0
- package/python/harness/runtime/task/run.py +1231 -790
- package/python/harness/runtime/tick.py +257 -91
- package/python/harness/sdk/__init__.py +7 -4
- package/python/harness/settings.py +30 -5
- package/python/harness/terminal.py +10 -6
- package/python/harness/transcript.py +18 -6
- package/python/harness/tui/answer.py +51 -0
- package/python/harness/tui/chat.py +75 -26
- package/python/harness/tui/views.py +1 -0
- package/python/harness/builtin/prompts/shared/entries.md +0 -86
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
"""The active trigger of learning (runtime/learning/learner.py): a run whose goal is to get to know the
|
|
2
2
|
environment for a task, before the runs that do the task. It is an ordinary run of the
|
|
3
|
-
task's environment with `trigger = "active"`: its instruction says
|
|
4
|
-
|
|
5
|
-
stays in it as what to learn for),
|
|
3
|
+
task's environment with `trigger = "active"`: its instruction says that it is practice and
|
|
4
|
+
what counts as learned (prompts/shared/learning-goal.md of the core; the task's own
|
|
5
|
+
instruction stays in it as what to learn for), not how to learn: that is the model's to
|
|
6
|
+
decide, by the principles of the core prompt. It is bounded by live time, not by the task's
|
|
6
7
|
budget. What it learns goes into the same library, through
|
|
7
8
|
the same learner, as what the task runs learn passively.
|
|
8
9
|
|
|
@@ -22,11 +23,10 @@ embodiment's package with a [probe] table (runtime/execution/skills.py), which a
|
|
|
22
23
|
does; keeps the table in the library (keep())
|
|
23
24
|
probe_summary(table) -> text the table's lines
|
|
24
25
|
|
|
25
|
-
and its table's
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
generic lines below. A probe is bounded by its live time: Deadline(max_seconds).check()
|
|
26
|
+
and its table's `try` text (what probe() measures) is the body's line of the learning goal.
|
|
27
|
+
A game probes its keys and mouse from its frames (the game package); a robot probes its
|
|
28
|
+
joints and grippers from its state and cameras (the robot package). A body without a probe
|
|
29
|
+
has no such line. A probe is bounded by its live time: Deadline(max_seconds).check()
|
|
30
30
|
raises OutOfTime once it is over, and the probe stops with what it measured.
|
|
31
31
|
|
|
32
32
|
Standard library only.
|
|
@@ -43,13 +43,8 @@ DEFAULT_MINUTES = 30.0
|
|
|
43
43
|
# this suffix (bench.py Bench.active_task). A worker allows its sandbox only in such an
|
|
44
44
|
# episode, never in a task run.
|
|
45
45
|
LEARN = "-learn"
|
|
46
|
-
# The lines of a body without a probe (a desktop, a browser).
|
|
47
|
-
GENERIC = {
|
|
48
|
-
"try": "Propose a note for each control you are sure of (what it does, how much, how fast).",
|
|
49
|
-
"skills": "go to a place, open and close each screen or menu, read a value it shows, find and reach a thing "
|
|
50
|
-
"you can see",
|
|
51
|
-
"drills": "each with a check of its result on the screen",
|
|
52
|
-
}
|
|
46
|
+
# The lines of a body without a probe (a desktop, a browser): none.
|
|
47
|
+
GENERIC = {"try": ""}
|
|
53
48
|
|
|
54
49
|
|
|
55
50
|
class OutOfTime(Exception):
|
|
@@ -111,9 +106,9 @@ def goal(instruction: str, minutes: float, skills=(), sandbox: dict | None = Non
|
|
|
111
106
|
text = select(PROMPT.read_text(encoding="utf-8"), features).strip()
|
|
112
107
|
lines = body(skills)
|
|
113
108
|
text = text.format(task=instruction.strip(), minutes=f"{minutes:g}", probe=lines["try"],
|
|
114
|
-
|
|
109
|
+
sandbox=sandbox_lines(sandbox))
|
|
115
110
|
# A placeholder with nothing to say leaves no line.
|
|
116
|
-
return "\n".join(line for line in text.split("\n") if line.strip() or not line)
|
|
111
|
+
return "\n".join(line for line in text.split("\n") if line.strip() or not line).rstrip()
|
|
117
112
|
|
|
118
113
|
|
|
119
114
|
def sandbox_of(runtime: dict) -> dict | None:
|
|
@@ -132,7 +127,7 @@ def sandbox_lines(sandbox: dict | None) -> str:
|
|
|
132
127
|
return (f"This environment offers a practice sandbox in this run only (never in the task's runs): "
|
|
133
128
|
f"{sandbox.get('what', '').strip()} sandbox(op, **arguments) stages a scene ({ops}). Stage a scene for "
|
|
134
129
|
"your practice drills. library.propose(kind, name, code, test=..., setup=...) stores the setup "
|
|
135
|
-
"with the test for
|
|
130
|
+
"with the test for the host's verification.")
|
|
136
131
|
|
|
137
132
|
|
|
138
133
|
def keep(session, table: dict, summary: str, note: str = "") -> dict:
|
|
@@ -215,8 +210,8 @@ def progress(rows: list[dict], seconds_left: float, seconds: float, situations:
|
|
|
215
210
|
shown = show_situations(situations)
|
|
216
211
|
return {"seconds_left": round(max(0.0, seconds_left), 1), "seconds": round(float(seconds), 1),
|
|
217
212
|
"entries": len(rows),
|
|
218
|
-
"tested": [{"key": key(row), "name": row["name"], "uses": row["uses"], "success_rate": row.get("success_rate")
|
|
219
|
-
|
|
213
|
+
"tested": [{"key": key(row), "name": row["name"], "uses": row["uses"], "success_rate": row.get("success_rate"),
|
|
214
|
+
"admitted_by": row.get("admitted_by")} for row in tested],
|
|
220
215
|
"failing": [key(row) for row in code if row["evidence"] == "failed"],
|
|
221
216
|
"verified_waiting": [key(row) for row in code if row["verified"] and not row["loaded"]],
|
|
222
217
|
"situations_text": shown,
|
|
@@ -239,62 +234,58 @@ def progress_line(now: dict, short: bool = False) -> str:
|
|
|
239
234
|
+ (f"{len(now['open'])} situations open" if now["situations"] else "no situations list yet"))
|
|
240
235
|
|
|
241
236
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
237
|
+
# What pending code waits for, by the admission level in force (learner.ADMISSION).
|
|
238
|
+
WAITS_FOR = {"fixture": "an independent fixture", "own_test": "the host to run its test between programs",
|
|
239
|
+
"off": "nothing: the host admits no code here"}
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def report(rows: list[dict], seconds_left: float | None, situations: str | None = None,
|
|
243
|
+
admission: str = "fixture") -> str:
|
|
244
|
+
"""What an active run has learned so far (progress()): what the library holds, what is
|
|
245
|
+
pending and what for, and the situations entry when there is one. Facts only: what to do
|
|
246
|
+
next is the model's to decide. The lines the model reads with each call of an active
|
|
247
|
+
run; `admission` is how the host admits code here (learner.admission). `seconds_left`
|
|
248
|
+
None: the time the run has left is not said (runtime.tell_budget)."""
|
|
249
|
+
now = progress(rows, seconds_left or 0.0, 0.0, situations)
|
|
250
|
+
left = "" if seconds_left is None else f"; {now['seconds_left'] / 60:.0f} min of live time left"
|
|
247
251
|
tested = now["tested"]
|
|
248
252
|
|
|
249
253
|
def names(found):
|
|
250
254
|
return ", ".join(found) or "none"
|
|
251
255
|
|
|
252
|
-
lines = [f"Learned so far ({now['entries']} entries
|
|
256
|
+
lines = [f"Learned so far ({now['entries']} entries{left}):",
|
|
253
257
|
" skills with a passing test: " + (", ".join(
|
|
254
|
-
f"{row['name']}" + (
|
|
255
|
-
|
|
258
|
+
f"{row['name']}" + (" [own test]" if row.get("admitted_by") == "own_test" else "")
|
|
259
|
+
+ (f" ({row['uses']} uses, {row['success_rate']:.0%} ok)"
|
|
260
|
+
if row["uses"] and row.get("success_rate") is not None else "")
|
|
256
261
|
for row in tested) or "none yet"),
|
|
257
262
|
f" untested (notes and memory entries have no test): {names(now['untested'])}"]
|
|
258
263
|
if now["failing"]:
|
|
259
264
|
lines.append(f" skills whose test failed: {names(now['failing'])}")
|
|
260
265
|
if now["waiting"]:
|
|
261
|
-
lines.append(f" code waiting for
|
|
266
|
+
lines.append(f" code waiting for {WAITS_FOR[admission]}: {names(now['waiting'])}")
|
|
262
267
|
if now["verified_waiting"]:
|
|
263
268
|
lines.append(f" verified, awaiting activation: {names(now['verified_waiting'])}")
|
|
264
269
|
if situations:
|
|
265
|
-
lines.append(" your situations
|
|
266
|
-
elif tested or now["untested"]:
|
|
267
|
-
lines.append(" no situations list yet: think the task through and write it (step 3) before more skills")
|
|
268
|
-
if now["open"]:
|
|
269
|
-
lines.append(f"Next: cover the {len(now['open'])} item(s) still marked [ ] with tested skills or rules, the "
|
|
270
|
-
"hardest case first. Mark cases awaiting an independent fixture [unknown] and record drill observations.")
|
|
271
|
-
elif not tested and not now["waiting"]:
|
|
272
|
-
lines.append("Next: write the basic skills the task needs, each a ClosedLoop with a quick test, and propose "
|
|
273
|
-
"them (library.propose(\"skill\", name, code, test=...)): a note is not a skill.")
|
|
274
|
-
elif not tested:
|
|
275
|
-
lines.append("Next: practice the task and keep useful observations while code proposals wait.")
|
|
276
|
-
elif not any(row["uses"] for row in tested):
|
|
277
|
-
lines.append("Next: set drills toward the task that use these skills. The run records each result.")
|
|
278
|
-
else:
|
|
279
|
-
lines.append("Next: rewrite the skills that failed in drills and propose them again, then drill the next "
|
|
280
|
-
"step toward the task.")
|
|
270
|
+
lines.append(" your situations entry (memory/situations):\n" + now["situations_text"])
|
|
281
271
|
return "\n".join(lines)
|
|
282
272
|
|
|
283
273
|
|
|
284
274
|
def holdings(rows: list[dict], situations: str | None = None) -> str:
|
|
285
|
-
"""What the library holds, for the start of a task run: its tested skills and
|
|
286
|
-
|
|
275
|
+
"""What the library holds, for the start of a task run: its tested skills and reflexes
|
|
276
|
+
and the situations entry it was learned with."""
|
|
287
277
|
tested = [row for row in rows if row["section"] in CODE and row["passed"]]
|
|
288
278
|
if not tested and not situations:
|
|
289
279
|
return ""
|
|
290
|
-
lines = ["Your library
|
|
280
|
+
lines = ["Your library:"]
|
|
291
281
|
for section in CODE:
|
|
292
|
-
found = [row["name"] + (
|
|
282
|
+
found = [row["name"] + (" [own test]" if row.get("admitted_by") == "own_test" else "")
|
|
283
|
+
+ (f" ({row['summary']})" if row.get("summary") else "")
|
|
293
284
|
for row in tested if row["section"] == section]
|
|
294
285
|
if found:
|
|
295
286
|
lines.append(f" {section} with a passing test: " + "; ".join(found))
|
|
296
287
|
if situations:
|
|
297
|
-
lines.append(" situations
|
|
288
|
+
lines.append(" the situations entry (memory/situations):\n" + show_situations(situations))
|
|
298
289
|
return "\n".join(lines)
|
|
299
290
|
|
|
300
291
|
|
|
@@ -21,8 +21,8 @@ MAX_BUNDLE = 2 * 1024 ** 3
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class Budget:
|
|
24
|
-
def __init__(self, seconds, cancelled):
|
|
25
|
-
self.deadline = time.monotonic() + min(
|
|
24
|
+
def __init__(self, seconds, cancelled, limit=45.0):
|
|
25
|
+
self.deadline = time.monotonic() + min(limit, seconds)
|
|
26
26
|
self.cancelled = cancelled
|
|
27
27
|
|
|
28
28
|
def remaining(self, reserve=0):
|
|
@@ -27,6 +27,18 @@ The settings, [runtime.library] (SETTINGS):
|
|
|
27
27
|
replay false | true at a run's start, test the provisional entries again
|
|
28
28
|
reflect false | true after each episode, reflection proposes one entry
|
|
29
29
|
|
|
30
|
+
How a skill or a reactive entry the agent wrote itself is admitted is the host's to say, not
|
|
31
|
+
the run's (ADMISSION; admission()): the user's setting learning.admission, or
|
|
32
|
+
$HARNESS_LEARNING_ADMISSION, which wins over it.
|
|
33
|
+
|
|
34
|
+
off no learned code is admitted or loaded
|
|
35
|
+
fixture a fixture the host provisioned for the entry verifies it (fixtures.py); the default
|
|
36
|
+
own_test where the worker's profile says physical is False: the host runs the entry's
|
|
37
|
+
own test on the staged revision, in the kernel, between programs; a pass admits
|
|
38
|
+
it as provisional with a proof whose method is "own_test". A fixture for the
|
|
39
|
+
entry, where there is one, is used instead. Such an entry goes back to
|
|
40
|
+
candidates/ when it keeps failing in use (DEMOTE_USES, DEMOTE_RATE).
|
|
41
|
+
|
|
30
42
|
Standard library only.
|
|
31
43
|
"""
|
|
32
44
|
|
|
@@ -41,7 +53,8 @@ import tempfile
|
|
|
41
53
|
import threading
|
|
42
54
|
import time
|
|
43
55
|
|
|
44
|
-
from harness.runtime.learning.library import ADMISSIONS, CANDIDATES, EVIDENCE, FAILED, INDEX, SECTIONS, TESTS,
|
|
56
|
+
from harness.runtime.learning.library import (ADMISSIONS, CANDIDATES, EVIDENCE, FAILED, INDEX, OWN_TEST, SECTIONS, TESTS,
|
|
57
|
+
Library, Revision, admitted_by, executable, rank)
|
|
45
58
|
from harness.core import atomic_write
|
|
46
59
|
from harness.core.procs import file_lock
|
|
47
60
|
from harness.paths import home
|
|
@@ -67,9 +80,32 @@ SITUATIONS_MAX_BYTES = 32 * 1024
|
|
|
67
80
|
ATTACHMENT_BYTES = 64 * 1024 * 1024
|
|
68
81
|
# Test runs and uses kept per entry in the index.
|
|
69
82
|
KEPT = 20
|
|
83
|
+
# How learned code is admitted, strictest first (the module docstring).
|
|
84
|
+
ADMISSION = ("off", "fixture", OWN_TEST)
|
|
85
|
+
ADMISSION_VARIABLE = "HARNESS_LEARNING_ADMISSION"
|
|
86
|
+
# The longest an entry's own test may run when the host runs it.
|
|
87
|
+
OWN_TEST_SECONDS = 120.0
|
|
88
|
+
# A kept test that stages its scene in the practice sandbox starts with this line (skills/learning SETUP).
|
|
89
|
+
STAGED_TEST = "# setup: stages the scene in the practice sandbox"
|
|
90
|
+
# An entry admitted by its own test goes back to candidates/ once this many uses of its
|
|
91
|
+
# revision are recorded and fewer than this share of them succeeded.
|
|
92
|
+
DEMOTE_USES, DEMOTE_RATE = 3, 0.25
|
|
70
93
|
_LOCK = threading.Lock()
|
|
71
94
|
|
|
72
95
|
|
|
96
|
+
def admission(profile: dict | None, allowed: str | None = None) -> str:
|
|
97
|
+
"""How learned code is admitted on a worker with this `profile`, given what the host
|
|
98
|
+
allows (`allowed`: one of ADMISSION; None is "fixture"): "own_test" only where the host
|
|
99
|
+
allows it and the profile says physical is False, "off" where the host says so, else
|
|
100
|
+
"fixture". ValueError for a level that does not exist."""
|
|
101
|
+
level = "fixture" if allowed in (None, "") else allowed
|
|
102
|
+
if level not in ADMISSION:
|
|
103
|
+
raise ValueError(f"learning.admission is one of {list(ADMISSION)}, not {level!r}")
|
|
104
|
+
if level == OWN_TEST and (profile or {}).get("physical") is not False:
|
|
105
|
+
return "fixture"
|
|
106
|
+
return level
|
|
107
|
+
|
|
108
|
+
|
|
73
109
|
def settings(runtime_library: dict | None) -> dict:
|
|
74
110
|
"""The learning settings of a [runtime.library] table, defaults filled in; ValueError
|
|
75
111
|
for a value it does not take."""
|
|
@@ -206,6 +242,8 @@ class Learner:
|
|
|
206
242
|
self.settings = settings(runtime_library)
|
|
207
243
|
self.layers = tuple((runtime_library or {}).get("layers", ("notes", "skills", "reactive")))
|
|
208
244
|
self.library = Library(self.directory, self.layers, self.settings["load"])
|
|
245
|
+
# The entries record_uses() sent back to candidates/: [{"key", "revision", "ok", "failed"}].
|
|
246
|
+
self.demoted = []
|
|
209
247
|
|
|
210
248
|
def version(self) -> str:
|
|
211
249
|
return self.library.version()
|
|
@@ -262,7 +300,9 @@ class Learner:
|
|
|
262
300
|
def entries(self, *, snapshot=None) -> list[dict]:
|
|
263
301
|
"""Every entry in the library (loaded or not) with its evidence, source, origin and
|
|
264
302
|
summary, and the candidates waiting: [{"section", "name", "file", "evidence",
|
|
265
|
-
"source", "origin", "summary", "admitted_at", "tests", "passed", "uses", "loaded"
|
|
303
|
+
"source", "origin", "summary", "admitted_at", "tests", "passed", "uses", "loaded",
|
|
304
|
+
"admitted_by", "demoted"}] (admitted_by: what verified a code entry's revision,
|
|
305
|
+
"own_test" or "fixture", None when nothing did; demoted: it failed in use and waits)."""
|
|
266
306
|
captured = snapshot if snapshot is not None else self.library.capture()
|
|
267
307
|
index = captured.index
|
|
268
308
|
loaded = {item.key for item in captured.entries}
|
|
@@ -288,6 +328,7 @@ class Learner:
|
|
|
288
328
|
"revision": row.get("revision"),
|
|
289
329
|
"verified": bool(row.get("proof") and row["proof"].get("revision") == row.get("revision")
|
|
290
330
|
and row.get("evidence") in ("provisional", "replayed", "validated")),
|
|
331
|
+
"admitted_by": admitted_by(row), "demoted": bool(row.get("demoted")),
|
|
291
332
|
"waiting": waiting, "source": row.get("source", "-"),
|
|
292
333
|
"origin": row.get("origin", "hand"), "summary": row.get("summary", ""),
|
|
293
334
|
"admitted_at": row.get("admitted_at"), "tests": len(tests),
|
|
@@ -355,6 +396,7 @@ class Learner:
|
|
|
355
396
|
if row.get("revision") != revision_id:
|
|
356
397
|
row.pop("proof", None)
|
|
357
398
|
row.pop("tests", None)
|
|
399
|
+
row.pop("demoted", None)
|
|
358
400
|
row["revision"] = revision_id
|
|
359
401
|
if proof is not None:
|
|
360
402
|
row["proof"] = proof
|
|
@@ -535,9 +577,40 @@ class Learner:
|
|
|
535
577
|
count = uses["ok"] + uses["failed"]
|
|
536
578
|
uses["rate"] = round(uses["ok"] / count, 3) if count else None
|
|
537
579
|
index[key] = row
|
|
580
|
+
if (self.settings["learn"] == "on" and executable(key) and row.get("revision") == revision
|
|
581
|
+
and row.get("evidence") == "provisional" and admitted_by(row) == OWN_TEST
|
|
582
|
+
and count >= DEMOTE_USES and uses["ok"] / count < DEMOTE_RATE):
|
|
583
|
+
self._demote(index, key, row, uses)
|
|
538
584
|
self._save_index(index)
|
|
539
585
|
return len(results)
|
|
540
586
|
|
|
587
|
+
def _demote(self, index, key, row, uses) -> None:
|
|
588
|
+
"""An entry admitted by its own test that keeps failing in use goes back to
|
|
589
|
+
candidates/: no proof, not loaded, and its own test is not run again for this
|
|
590
|
+
revision (a rewritten one is a new proposal; a fixture may still admit this one)."""
|
|
591
|
+
revision = row["revision"]
|
|
592
|
+
item = Revision.read(self.directory, revision)
|
|
593
|
+
section, _, filename = key.partition("/")
|
|
594
|
+
stem = Path(filename).stem
|
|
595
|
+
why = f"demoted: {uses['ok']} of {uses['ok'] + uses['failed']} uses succeeded"
|
|
596
|
+
waiting = {name: value for name, value in row.items() if name != "proof"}
|
|
597
|
+
waiting.update(evidence="proposed", updated_at=_now(),
|
|
598
|
+
demoted={"at": _now(), "revision": revision, "ok": uses["ok"], "failed": uses["failed"]},
|
|
599
|
+
history=[*(row.get("history") or []),
|
|
600
|
+
{"at": _now(), "evidence": "proposed", "revision": revision, "why": why}][-KEPT:])
|
|
601
|
+
index[f"{CANDIDATES}/{key}"] = waiting
|
|
602
|
+
del index[key]
|
|
603
|
+
pending = self.directory / CANDIDATES
|
|
604
|
+
atomic_write(pending / section / filename, item.source)
|
|
605
|
+
if item.test is not None:
|
|
606
|
+
atomic_write(pending / TESTS / f"{stem}.py", item.test)
|
|
607
|
+
for relative, data in item.attachments:
|
|
608
|
+
atomic_write(pending / relative, data)
|
|
609
|
+
(self.directory / section / filename).unlink(missing_ok=True)
|
|
610
|
+
(self.directory / TESTS / f"{stem}.py").unlink(missing_ok=True)
|
|
611
|
+
shutil.rmtree(self.directory / section / f"{stem}.files", ignore_errors=True)
|
|
612
|
+
self.demoted.append({"key": key, "revision": revision, "ok": uses["ok"], "failed": uses["failed"]})
|
|
613
|
+
|
|
541
614
|
# -- reading --------------------------------------------------------------------------
|
|
542
615
|
|
|
543
616
|
def candidate_from(self, key: str, *, source="passive", origin="hand", pending=None) -> Candidate | None:
|
|
@@ -24,7 +24,8 @@ Layout of a library directory:
|
|
|
24
24
|
learner.py); not part of the version
|
|
25
25
|
admissions.jsonl one line an admission: section, file, the sha256 of its bytes,
|
|
26
26
|
the version after it (what an audit checks the entries against)
|
|
27
|
-
candidates/ entries proposed and waiting for
|
|
27
|
+
candidates/ entries proposed and waiting: for the host's verification, for
|
|
28
|
+
validation on development seeds, or sent back after failing in use
|
|
28
29
|
(not loaded, not part of the version)
|
|
29
30
|
|
|
30
31
|
Each entry is one file (with its test and attachments). The library version is the sha256
|
|
@@ -59,6 +60,11 @@ LAYER_OF = {"notes": "notes", "memory": "notes", "skills": "skills", "reactive":
|
|
|
59
60
|
LAYERS = ("notes", "skills", "reactive")
|
|
60
61
|
# What the notes stand under in the prompt, unless library.toml names another title.
|
|
61
62
|
NOTES_TITLE = "Notes from earlier episodes:"
|
|
63
|
+
# What the system prompt holds of the library: an index, within a budget. Each skill's
|
|
64
|
+
# line up to SKILL_LINE_CHARS and all of them within SKILLS_CHARS (past that, the later
|
|
65
|
+
# skills by name only: help(name) in a program shows one); the notes up to NOTES_LINES
|
|
66
|
+
# lines or NOTES_CHARS characters, then how many more there are.
|
|
67
|
+
SKILL_LINE_CHARS, SKILLS_CHARS, NOTES_LINES, NOTES_CHARS = 1536, 8000, 200, 25_000
|
|
62
68
|
# How well an entry is known to work, weakest first: proposed (no check yet), provisional
|
|
63
69
|
# (its own test passed in a run; a note or memory entry the model admitted), replayed (its
|
|
64
70
|
# test passed again at the start of a later run), validated (it raised the score on
|
|
@@ -66,12 +72,17 @@ NOTES_TITLE = "Notes from earlier episodes:"
|
|
|
66
72
|
# there by hand or by self-improvement before the index: validated. A failed one never loads.
|
|
67
73
|
EVIDENCE = ("proposed", "provisional", "replayed", "validated")
|
|
68
74
|
FAILED = "failed"
|
|
75
|
+
# The method of the proof the host writes after it ran an entry's own test (learner.py).
|
|
76
|
+
OWN_TEST = "own_test"
|
|
69
77
|
INDEX = "index.json"
|
|
70
78
|
ADMISSIONS = "admissions.jsonl"
|
|
71
79
|
TESTS = "tests"
|
|
72
80
|
CANDIDATES = "candidates"
|
|
73
81
|
_code_origins = WeakKeyDictionary()
|
|
74
82
|
_code_active = WeakKeyDictionary()
|
|
83
|
+
# {key: revision} of the entries staged and under their own test (testing_staged): their code
|
|
84
|
+
# may run although it is not active yet.
|
|
85
|
+
_under_test = {}
|
|
75
86
|
|
|
76
87
|
|
|
77
88
|
|
|
@@ -149,14 +160,31 @@ class LibrarySnapshot:
|
|
|
149
160
|
def notes(self):
|
|
150
161
|
texts = [item.source.decode("utf-8").strip() for item in self.entries if item.key.startswith("notes/")]
|
|
151
162
|
title = str(tomllib.loads(self.settings.decode("utf-8")).get("notes_title") or NOTES_TITLE)
|
|
152
|
-
|
|
163
|
+
shown, lines, size = [], 0, 0
|
|
164
|
+
for text in texts:
|
|
165
|
+
lines, size = lines + text.count("\n") + 1, size + len(text)
|
|
166
|
+
if shown and (lines > NOTES_LINES or size > NOTES_CHARS):
|
|
167
|
+
break
|
|
168
|
+
shown.append(f"- {text}")
|
|
169
|
+
if len(shown) < len(texts):
|
|
170
|
+
shown.append(f"... {len(texts) - len(shown)} more notes are not shown here")
|
|
171
|
+
return f"\n{title}\n" + "\n".join(shown) + "\n" if texts else ""
|
|
153
172
|
|
|
154
173
|
def api_summary(self, active):
|
|
155
174
|
return active_summary(item for item in self.entries if item.key in active)
|
|
156
175
|
|
|
157
176
|
|
|
158
177
|
def active_summary(items):
|
|
159
|
-
lines = [
|
|
178
|
+
lines, size, bare = [], 0, 0
|
|
179
|
+
for item in items:
|
|
180
|
+
for line in summary_lines(item.source.decode("utf-8")):
|
|
181
|
+
line = line[:SKILL_LINE_CHARS]
|
|
182
|
+
if size + len(line) > SKILLS_CHARS and ")" in line:
|
|
183
|
+
line, bare = line[:line.index(")") + 1], bare + 1
|
|
184
|
+
lines.append(line)
|
|
185
|
+
size += len(line)
|
|
186
|
+
if bare:
|
|
187
|
+
lines.append(f" ({bare} are listed by name only: help(name) in a program shows what one does)")
|
|
160
188
|
return "\nLibrary skills (already defined in your kernel):\n" + "\n".join(lines) + "\n" if lines else ""
|
|
161
189
|
|
|
162
190
|
|
|
@@ -177,6 +205,16 @@ def eligible(item, row, layers, load):
|
|
|
177
205
|
|
|
178
206
|
|
|
179
207
|
|
|
208
|
+
def admitted_by(row) -> str | None:
|
|
209
|
+
"""What verified the revision an index row names: the method of its proof ("fixture",
|
|
210
|
+
"own_test"), or None when nothing verified that revision."""
|
|
211
|
+
proof = row.get("proof") or {}
|
|
212
|
+
conditions = proof.get("conditions")
|
|
213
|
+
if not row.get("revision") or proof.get("revision") != row["revision"] or not isinstance(conditions, dict):
|
|
214
|
+
return None
|
|
215
|
+
return conditions.get("method")
|
|
216
|
+
|
|
217
|
+
|
|
180
218
|
def _verified_bindings(api, bindings):
|
|
181
219
|
"""Pin fixture actions to the original kernel API and the tested implementation."""
|
|
182
220
|
from types import CodeType, MethodType
|
|
@@ -204,7 +242,7 @@ def _verified_bindings(api, bindings):
|
|
|
204
242
|
|
|
205
243
|
def stage_entries(items, namespace, active, reactive, *, api=None):
|
|
206
244
|
from harness.runtime.execution.entries import ClosedLoop
|
|
207
|
-
from harness.runtime.reactive.
|
|
245
|
+
from harness.runtime.reactive.tree import stage as _stage_rules
|
|
208
246
|
from harness.core import spans
|
|
209
247
|
|
|
210
248
|
batch = {item.key: (item, hand) for item, hand in items}
|
|
@@ -241,8 +279,7 @@ def stage_entries(items, namespace, active, reactive, *, api=None):
|
|
|
241
279
|
raise ValueError(f"{item.key} has unsupported initialization")
|
|
242
280
|
for child in _initialization(node):
|
|
243
281
|
if isinstance(child, ast.Call):
|
|
244
|
-
if
|
|
245
|
-
or child.func.id not in ("when", "watch", "on", "rule", "reflex")):
|
|
282
|
+
if not isinstance(child.func, ast.Name) or child.func.id not in SETS:
|
|
246
283
|
raise ValueError(f"{item.key} must put runtime actions inside a callable")
|
|
247
284
|
if isinstance(child, (ast.Attribute, ast.Subscript)) and isinstance(child.ctx, ast.Store):
|
|
248
285
|
raise ValueError(f"{item.key} cannot modify live objects during activation")
|
|
@@ -265,6 +302,21 @@ def stage_entries(items, namespace, active, reactive, *, api=None):
|
|
|
265
302
|
return scope, owners, staged_rules
|
|
266
303
|
|
|
267
304
|
|
|
305
|
+
# What a learned entry may call while it is loaded: the call that puts a node into the tree
|
|
306
|
+
# of reflexes. Everything else runs later, inside a callable (a function, a lambda).
|
|
307
|
+
SETS = ("set_reflex",)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _outside_callables(node):
|
|
311
|
+
"""The nodes of an expression that are evaluated when it is: not the body of a lambda,
|
|
312
|
+
which runs when it is called."""
|
|
313
|
+
yield node
|
|
314
|
+
for child in ast.iter_child_nodes(node):
|
|
315
|
+
if isinstance(node, ast.Lambda) and child is node.body:
|
|
316
|
+
continue
|
|
317
|
+
yield from _outside_callables(child)
|
|
318
|
+
|
|
319
|
+
|
|
268
320
|
def _initialization(node):
|
|
269
321
|
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
270
322
|
fields = (*node.decorator_list, *node.args.defaults, *filter(None, node.args.kw_defaults))
|
|
@@ -273,13 +325,28 @@ def _initialization(node):
|
|
|
273
325
|
fields = (*node.bases, *node.decorator_list, *node.keywords)
|
|
274
326
|
return (*[child for part in fields for child in ast.walk(part)],
|
|
275
327
|
*[child for part in node.body for child in _initialization(part)])
|
|
276
|
-
return
|
|
328
|
+
return _outside_callables(node)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
@contextmanager
|
|
332
|
+
def testing_staged(staged):
|
|
333
|
+
"""While an entry's own test runs on what stage_entries() returned: the staged revisions
|
|
334
|
+
may execute, as the active ones do."""
|
|
335
|
+
_under_test.update({key: row[0] for key, row in staged[1].items()})
|
|
336
|
+
try:
|
|
337
|
+
yield dict(staged[0])
|
|
338
|
+
finally:
|
|
339
|
+
_under_test.clear()
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _runs(active, origin) -> bool:
|
|
343
|
+
return active.get(origin[0], (None,))[0] == origin[2] or _under_test.get(origin[0]) == origin[2]
|
|
277
344
|
|
|
278
345
|
|
|
279
346
|
def _tracked(function, origin, active, spans):
|
|
280
347
|
@contextmanager
|
|
281
348
|
def execution():
|
|
282
|
-
if
|
|
349
|
+
if not _runs(active, origin):
|
|
283
350
|
raise RuntimeError(f"library revision for {origin[0]} is no longer active")
|
|
284
351
|
start = time.monotonic()
|
|
285
352
|
try:
|
|
@@ -327,13 +394,13 @@ def publish_entries(staged, namespace, active, reactive):
|
|
|
327
394
|
namespace.update(scope)
|
|
328
395
|
active.update(owners)
|
|
329
396
|
if rules is not None:
|
|
330
|
-
reactive.
|
|
397
|
+
reactive.publish(rules)
|
|
331
398
|
|
|
332
399
|
|
|
333
400
|
def assert_active(code):
|
|
334
401
|
origin = _code_origins.get(code)
|
|
335
402
|
active = _code_active.get(code)
|
|
336
|
-
if origin is not None and active is not None and
|
|
403
|
+
if origin is not None and active is not None and not _runs(active, origin):
|
|
337
404
|
raise RuntimeError(f"library revision for {origin[0]} is no longer active")
|
|
338
405
|
|
|
339
406
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Read-only library projections used by task recall and trace viewers."""
|
|
2
2
|
import json
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
from harness.runtime.learning.library import Library, LibrarySnapshot, selects
|
|
4
|
+
from harness.runtime.learning.library import Library, LibrarySnapshot, admitted_by, executable, selects
|
|
5
5
|
from harness.runtime.task.context import entry
|
|
6
6
|
|
|
7
7
|
|
|
@@ -23,9 +23,18 @@ def _library_entry(key, text, row, version, *, evidence=None, sha256=None):
|
|
|
23
23
|
"version": version, "summary": row.get("summary", ""), "history": history}, "library")
|
|
24
24
|
if unknown:
|
|
25
25
|
shown["history_unknown_revision"] = unknown
|
|
26
|
+
if executable(key):
|
|
27
|
+
# What verified this revision of a code entry: "own_test", "fixture", or None.
|
|
28
|
+
shown["admitted_by"] = admitted_by(row)
|
|
29
|
+
if row.get("demoted"):
|
|
30
|
+
shown["demoted"] = row["demoted"]
|
|
26
31
|
return shown
|
|
27
32
|
|
|
28
33
|
|
|
34
|
+
# How a code entry's provenance reads beside its evidence, in every frontend's table.
|
|
35
|
+
ADMITTED_BY = {"own_test": "own test", "fixture": "fixture"}
|
|
36
|
+
|
|
37
|
+
|
|
29
38
|
def admitted_entry(outcome, text):
|
|
30
39
|
"""The memory text or code preview, with its evidence at admission."""
|
|
31
40
|
if not outcome["admitted"]:
|
|
@@ -53,6 +62,8 @@ def library_entries(library: Library, version=None, *, sections=("memory",)) ->
|
|
|
53
62
|
evidence = shown["evidence"]
|
|
54
63
|
if tests:
|
|
55
64
|
evidence = f"{evidence} {sum(bool(test.get('ok')) for test in tests)}/{len(tests)}"
|
|
65
|
+
if shown.get("admitted_by"):
|
|
66
|
+
evidence += f" ({ADMITTED_BY.get(shown['admitted_by'], shown['admitted_by'])})"
|
|
56
67
|
name = shown["name"] or ""
|
|
57
68
|
if not selects(section, shown["evidence"], snapshot.layers, snapshot.load):
|
|
58
69
|
name += " (not loaded)"
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Rules and event handlers."""
|