@lmmslab/harness 0.0.0-stage → 0.81.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1356 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +232 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""The active trigger of learning (runtime/learning/learner.py): a run whose goal is to get to know the
|
|
2
|
+
environment for a task, before the runs that do the task. It is an ordinary run of the
|
|
3
|
+
task's environment with `trigger = "active"`: its instruction says to learn, not to do, and
|
|
4
|
+
how (prompts/shared/learning-goal.md of the core, the lines for its skills; the task's own instruction
|
|
5
|
+
stays in it as what to learn for), and it is bounded by live time, not by the task's
|
|
6
|
+
budget. What it learns goes into the same library, through
|
|
7
|
+
the same learner, as what the task runs learn passively.
|
|
8
|
+
|
|
9
|
+
The schedule, [runtime.library]:
|
|
10
|
+
|
|
11
|
+
active_before false (default) | true before a task's runs, one active run
|
|
12
|
+
active_minutes 1 to 240 (default 30) the active run's live time
|
|
13
|
+
|
|
14
|
+
`harness run` runs it once per task (the eval package's suite), in `<output>/<task>/active-<seed>`,
|
|
15
|
+
on the first seed's environment; the task's runs follow with `trigger = "passive"`.
|
|
16
|
+
|
|
17
|
+
The run asks its body's probe, never keys or pixels: the probe is the skill of the
|
|
18
|
+
embodiment's package with a [probe] table (runtime/execution/skills.py), which a learning run turns on
|
|
19
|
+
(config/agent.py). It exports
|
|
20
|
+
|
|
21
|
+
probe(max_seconds=...) -> table tries each of the body's controls and measures what it
|
|
22
|
+
does; keeps the table in the library (keep())
|
|
23
|
+
probe_summary(table) -> text the table's lines
|
|
24
|
+
|
|
25
|
+
and its table's texts are the body's lines of the learning goal: try (what probe() measures),
|
|
26
|
+
skills (the basic skills to write first) and drills (how a drill's result is checked). A game
|
|
27
|
+
probes its keys and mouse from its frames (the game package); a robot probes its joints and
|
|
28
|
+
grippers from its state and cameras (the robot package). A body without a probe gets the
|
|
29
|
+
generic lines below. A probe is bounded by its live time: Deadline(max_seconds).check()
|
|
30
|
+
raises OutOfTime once it is over, and the probe stops with what it measured.
|
|
31
|
+
|
|
32
|
+
Standard library only.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from copy import copy
|
|
36
|
+
import time
|
|
37
|
+
|
|
38
|
+
from harness.paths import ROOT
|
|
39
|
+
|
|
40
|
+
PROMPT = ROOT / "prompts" / "shared" / "learning-goal.md"
|
|
41
|
+
DEFAULT_MINUTES = 30.0
|
|
42
|
+
# The task an active run plays where the environment has a practice sandbox: the task with
|
|
43
|
+
# this suffix (bench.py Bench.active_task). A worker allows its sandbox only in such an
|
|
44
|
+
# episode, never in a task run.
|
|
45
|
+
LEARN = "-learn"
|
|
46
|
+
# The lines of a body without a probe (a desktop, a browser).
|
|
47
|
+
GENERIC = {
|
|
48
|
+
"try": "Propose a note for each control you are sure of (what it does, how much, how fast).",
|
|
49
|
+
"skills": "go to a place, open and close each screen or menu, read a value it shows, find and reach a thing "
|
|
50
|
+
"you can see",
|
|
51
|
+
"drills": "each with a check of its result on the screen",
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class OutOfTime(Exception):
|
|
56
|
+
"""A probe's live time is over."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Deadline:
|
|
60
|
+
"""A probe's live time: `seconds` from now."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, seconds: float):
|
|
63
|
+
self.began = time.monotonic()
|
|
64
|
+
self.at = self.began + float(seconds)
|
|
65
|
+
|
|
66
|
+
def check(self) -> None:
|
|
67
|
+
if time.monotonic() > self.at:
|
|
68
|
+
raise OutOfTime()
|
|
69
|
+
|
|
70
|
+
def elapsed(self) -> float:
|
|
71
|
+
return time.monotonic() - self.began
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def schedule(runtime_library: dict | None) -> dict:
|
|
75
|
+
"""{"before", "minutes"} of a [runtime.library] table; ValueError for a value it does not take."""
|
|
76
|
+
table = dict(runtime_library or {})
|
|
77
|
+
before = table.get("active_before", False)
|
|
78
|
+
if not isinstance(before, bool):
|
|
79
|
+
raise ValueError(f"runtime.library.active_before is true or false, not {before!r}")
|
|
80
|
+
minutes = table.get("active_minutes", DEFAULT_MINUTES)
|
|
81
|
+
if isinstance(minutes, bool) or not isinstance(minutes, (int, float)) or not 1 <= float(minutes) <= 240:
|
|
82
|
+
raise ValueError(f"runtime.library.active_minutes is 1 to 240, not {minutes!r}")
|
|
83
|
+
return {"before": before, "minutes": float(minutes)}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def is_active(runtime: dict) -> bool:
|
|
87
|
+
"""Whether a run's goal is to learn its environment (learn on, trigger active)."""
|
|
88
|
+
library = runtime.get("library") or {}
|
|
89
|
+
return library.get("learn") == "on" and library.get("trigger") == "active"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def body(skills=()) -> dict:
|
|
93
|
+
"""The body's lines of the learning goal: the [probe] table of the probe among `skills`
|
|
94
|
+
(runtime/execution/skills.py), its missing keys from GENERIC."""
|
|
95
|
+
from harness.runtime.execution.skills import load_skills
|
|
96
|
+
|
|
97
|
+
known = load_skills()
|
|
98
|
+
for name in skills:
|
|
99
|
+
if name in known and known[name].probe:
|
|
100
|
+
return {**GENERIC, **dict(known[name].probe)}
|
|
101
|
+
return dict(GENERIC)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def goal(instruction: str, minutes: float, skills=(), sandbox: dict | None = None, *, legacy_memory=False) -> str:
|
|
105
|
+
"""The active run's instruction: learn the environment for the task `instruction`, the
|
|
106
|
+
lines of the prompt for the run's `skills` kept, its body's lines (body()) in it, and,
|
|
107
|
+
when the environment offers one, its practice sandbox (sandbox_lines())."""
|
|
108
|
+
from harness.runtime.decision.prompts import select
|
|
109
|
+
|
|
110
|
+
features = (*skills, "legacy_memory") if legacy_memory else skills
|
|
111
|
+
text = select(PROMPT.read_text(encoding="utf-8"), features).strip()
|
|
112
|
+
lines = body(skills)
|
|
113
|
+
text = text.format(task=instruction.strip(), minutes=f"{minutes:g}", probe=lines["try"],
|
|
114
|
+
skills=lines["skills"], drills=lines["drills"], sandbox=sandbox_lines(sandbox))
|
|
115
|
+
# A placeholder with nothing to say leaves no line.
|
|
116
|
+
return "\n".join(line for line in text.split("\n") if line.strip() or not line)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def sandbox_of(runtime: dict) -> dict | None:
|
|
120
|
+
"""The practice sandbox a run's kernel gets ({"what", "ops"}, the bench's [sandbox] that
|
|
121
|
+
the suite put into an active run's runtime): an active run's with ops, else None."""
|
|
122
|
+
sandbox = runtime.get("sandbox") or {}
|
|
123
|
+
return dict(sandbox) if is_active(runtime) and sandbox.get("ops") else None
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def sandbox_lines(sandbox: dict | None) -> str:
|
|
127
|
+
"""The learning goal's lines about the practice sandbox a bench declares ({"what",
|
|
128
|
+
"ops"}: bench.py), or none: there is none, or it stages nothing."""
|
|
129
|
+
if not sandbox or not sandbox.get("ops"):
|
|
130
|
+
return ""
|
|
131
|
+
ops = "; ".join(f"{op}: {what}" for op, what in sandbox["ops"].items())
|
|
132
|
+
return (f"This environment offers a practice sandbox in this run only (never in the task's runs): "
|
|
133
|
+
f"{sandbox.get('what', '').strip()} sandbox(op, **arguments) stages a scene ({ops}). Stage a scene for "
|
|
134
|
+
"your practice drills. library.propose(kind, name, code, test=..., setup=...) stores the setup "
|
|
135
|
+
"with the test for a separate fixture.")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def keep(session, table: dict, summary: str, note: str = "") -> dict:
|
|
139
|
+
"""Keep what a body's probe measured in the run's library, through the learner (skills/
|
|
140
|
+
learning): the memory entry "controls" (`summary`, the table attached as controls.json)
|
|
141
|
+
and, when given, the note "controls" (`note`: the few facts every later call needs).
|
|
142
|
+
{"memory": its answer, "note": its answer}; {} where the run's library does not learn."""
|
|
143
|
+
import json
|
|
144
|
+
|
|
145
|
+
from harness.runtime.learning.learner import MEMORY_MAX_CHARS, NOTE_MAX_CHARS
|
|
146
|
+
|
|
147
|
+
library = getattr(session, "_library_api", None)
|
|
148
|
+
if library is None:
|
|
149
|
+
return {}
|
|
150
|
+
library = library()
|
|
151
|
+
if not library._learning:
|
|
152
|
+
return {}
|
|
153
|
+
kept = {"memory": library.propose("memory", "controls", fit(summary, MEMORY_MAX_CHARS, "(the rest: controls.json)"),
|
|
154
|
+
summary="what each control does, measured",
|
|
155
|
+
attachments={"controls.json": json.dumps(table, default=str)})}
|
|
156
|
+
if note:
|
|
157
|
+
kept["note"] = library.propose("note", "controls", fit(note, NOTE_MAX_CHARS), summary="the controls, measured")
|
|
158
|
+
return kept
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def fit(text: str, limit: int, tail: str = "") -> str:
|
|
162
|
+
"""`text` within `limit` characters: its first lines that fit, then `tail`."""
|
|
163
|
+
if len(text) <= limit:
|
|
164
|
+
return text
|
|
165
|
+
lines, size = [], len(tail)
|
|
166
|
+
for line in text.split("\n"):
|
|
167
|
+
if size + len(line) + 1 > limit:
|
|
168
|
+
break
|
|
169
|
+
lines.append(line)
|
|
170
|
+
size += len(line) + 1
|
|
171
|
+
return "\n".join(lines + ([tail] if tail else [])) if lines else text[:limit]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
CODE = ("skills", "reactive")
|
|
175
|
+
# The situations list as the model is shown it: its open items ([ ]) first, at most this many
|
|
176
|
+
# UTF-8 bytes (it is stored up to learner.SITUATIONS_MAX_BYTES).
|
|
177
|
+
SITUATIONS_BYTES = 8 * 1024
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def show_situations(text: str | None) -> str:
|
|
181
|
+
"""The situations list as shown: its open lines ([ ]) first, then the rest in order, up
|
|
182
|
+
to SITUATIONS_BYTES; how many lines did not fit is said."""
|
|
183
|
+
lines = [line for line in (text or "").strip().splitlines() if line.strip()]
|
|
184
|
+
ordered = sorted(lines, key=lambda line: "[ ]" not in line and "[unknown]" not in line)
|
|
185
|
+
shown, size = [], 0
|
|
186
|
+
for line in ordered:
|
|
187
|
+
cost = len(line.encode("utf-8")) + 5 # four spaces and a newline
|
|
188
|
+
if size + cost > SITUATIONS_BYTES:
|
|
189
|
+
break
|
|
190
|
+
shown.append(line)
|
|
191
|
+
size += cost
|
|
192
|
+
if len(shown) < len(lines):
|
|
193
|
+
# Reserve room for the omission notice inside the same limit.
|
|
194
|
+
while True:
|
|
195
|
+
omitted = f"({len(lines) - len(shown)} more lines not shown: the memory entry situations has them)"
|
|
196
|
+
if size + len(omitted) + 5 <= SITUATIONS_BYTES:
|
|
197
|
+
break
|
|
198
|
+
size -= len(shown.pop().encode("utf-8")) + 5
|
|
199
|
+
shown.append(omitted)
|
|
200
|
+
return "\n".join(" " + line for line in shown)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def progress(rows: list[dict], seconds_left: float, seconds: float, situations: str | None = None) -> dict:
|
|
204
|
+
"""Where an active run stands, from the learner's entries (runtime/learning/learner.py entries()):
|
|
205
|
+
the live time, the code entries with a passing test (their uses), those whose test failed,
|
|
206
|
+
the code waiting for its test, the notes and memory entries (nothing tests them) and the
|
|
207
|
+
open lines of the situations list. What report() says to the model and the frontends show
|
|
208
|
+
(the app's /api/app/learning, the terminal chat's /learn, harness run --learn)."""
|
|
209
|
+
code = [row for row in rows if row["section"] in CODE]
|
|
210
|
+
tested = [row for row in code if row["loaded"] and row["verified"]]
|
|
211
|
+
|
|
212
|
+
def key(row):
|
|
213
|
+
return f"{row['section']}/{row['name']}"
|
|
214
|
+
|
|
215
|
+
shown = show_situations(situations)
|
|
216
|
+
return {"seconds_left": round(max(0.0, seconds_left), 1), "seconds": round(float(seconds), 1),
|
|
217
|
+
"entries": len(rows),
|
|
218
|
+
"tested": [{"key": key(row), "name": row["name"], "uses": row["uses"], "success_rate": row.get("success_rate")}
|
|
219
|
+
for row in tested],
|
|
220
|
+
"failing": [key(row) for row in code if row["evidence"] == "failed"],
|
|
221
|
+
"verified_waiting": [key(row) for row in code if row["verified"] and not row["loaded"]],
|
|
222
|
+
"situations_text": shown,
|
|
223
|
+
"waiting": [key(row) for row in code if not row["verified"] and not row["loaded"] and row["evidence"] != "failed"],
|
|
224
|
+
"untested": [key(row) for row in rows if row["section"] not in CODE],
|
|
225
|
+
"situations": bool(situations and situations.strip()),
|
|
226
|
+
"open": [line.strip() for line in shown.splitlines() if "[ ]" in line]}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def progress_line(now: dict, short: bool = False) -> str:
|
|
230
|
+
"""progress() in one line, as harness run --learn and the terminal chat show it (`short`:
|
|
231
|
+
its status line's few words)."""
|
|
232
|
+
if short:
|
|
233
|
+
return (f"learning {now['seconds_left'] / 60:.0f}/{now['seconds'] / 60:.0f} min · {len(now['tested'])} tested · "
|
|
234
|
+
f"{len(now['waiting'])} waiting · " + (f"{len(now['open'])} open" if now["situations"] else "no list"))
|
|
235
|
+
return (f"learning: {now['seconds_left'] / 60:.0f} of {now['seconds'] / 60:.0f} min left · "
|
|
236
|
+
f"{len(now['tested'])} tested ({sum(1 for row in now['tested'] if row['uses'])} used) · "
|
|
237
|
+
f"{len(now['failing'])} failing · {len(now['waiting'])} waiting for verification · "
|
|
238
|
+
f"{len(now['verified_waiting'])} verified and awaiting activation · {len(now['untested'])} notes · "
|
|
239
|
+
+ (f"{len(now['open'])} situations open" if now["situations"] else "no situations list yet"))
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def report(rows: list[dict], seconds_left: float, situations: str | None = None) -> str:
|
|
243
|
+
"""What an active run has learned so far (progress()), and the next step: tested skills
|
|
244
|
+
before notes, drills before rewriting. The lines the model reads after each program of an
|
|
245
|
+
active run."""
|
|
246
|
+
now = progress(rows, seconds_left, 0.0, situations)
|
|
247
|
+
tested = now["tested"]
|
|
248
|
+
|
|
249
|
+
def names(found):
|
|
250
|
+
return ", ".join(found) or "none"
|
|
251
|
+
|
|
252
|
+
lines = [f"Learned so far ({now['entries']} entries; {now['seconds_left'] / 60:.0f} min of live time left):",
|
|
253
|
+
" skills with a passing test: " + (", ".join(
|
|
254
|
+
f"{row['name']}" + (f" ({row['uses']} uses, {row['success_rate']:.0%} ok)"
|
|
255
|
+
if row["uses"] and row.get("success_rate") is not None else "")
|
|
256
|
+
for row in tested) or "none yet"),
|
|
257
|
+
f" untested (notes and memory entries have no test): {names(now['untested'])}"]
|
|
258
|
+
if now["failing"]:
|
|
259
|
+
lines.append(f" skills whose test failed: {names(now['failing'])}")
|
|
260
|
+
if now["waiting"]:
|
|
261
|
+
lines.append(f" code waiting for an independent fixture: {names(now['waiting'])}")
|
|
262
|
+
if now["verified_waiting"]:
|
|
263
|
+
lines.append(f" verified, awaiting activation: {names(now['verified_waiting'])}")
|
|
264
|
+
if situations:
|
|
265
|
+
lines.append(" your situations list (memory/situations):\n" + now["situations_text"])
|
|
266
|
+
elif tested or now["untested"]:
|
|
267
|
+
lines.append(" no situations list yet: think the task through and write it (step 3) before more skills")
|
|
268
|
+
if now["open"]:
|
|
269
|
+
lines.append(f"Next: cover the {len(now['open'])} item(s) still marked [ ] with tested skills or rules, the "
|
|
270
|
+
"hardest case first. Mark cases awaiting an independent fixture [unknown] and record drill observations.")
|
|
271
|
+
elif not tested and not now["waiting"]:
|
|
272
|
+
lines.append("Next: write the basic skills the task needs, each a ClosedLoop with a quick test, and propose "
|
|
273
|
+
"them (library.propose(\"skill\", name, code, test=...)): a note is not a skill.")
|
|
274
|
+
elif not tested:
|
|
275
|
+
lines.append("Next: practice the task and keep useful observations while code proposals wait.")
|
|
276
|
+
elif not any(row["uses"] for row in tested):
|
|
277
|
+
lines.append("Next: set drills toward the task that use these skills. The run records each result.")
|
|
278
|
+
else:
|
|
279
|
+
lines.append("Next: rewrite the skills that failed in drills and propose them again, then drill the next "
|
|
280
|
+
"step toward the task.")
|
|
281
|
+
return "\n".join(lines)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def holdings(rows: list[dict], situations: str | None = None) -> str:
|
|
285
|
+
"""What the library holds, for the start of a task run: its tested skills and rules
|
|
286
|
+
to use before writing new code, and the situations list it was learned for."""
|
|
287
|
+
tested = [row for row in rows if row["section"] in CODE and row["passed"]]
|
|
288
|
+
if not tested and not situations:
|
|
289
|
+
return ""
|
|
290
|
+
lines = ["Your library (use these first; rewrite one that falls short rather than writing new code beside it):"]
|
|
291
|
+
for section in CODE:
|
|
292
|
+
found = [row["name"] + (f" ({row['summary']})" if row.get("summary") else "")
|
|
293
|
+
for row in tested if row["section"] == section]
|
|
294
|
+
if found:
|
|
295
|
+
lines.append(f" {section} with a passing test: " + "; ".join(found))
|
|
296
|
+
if situations:
|
|
297
|
+
lines.append(" situations it was learned for (memory/situations):\n" + show_situations(situations))
|
|
298
|
+
return "\n".join(lines)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def wanted(runtime: dict) -> bool:
|
|
302
|
+
"""Whether a task's runs want an active run before them (learning on, active_before)."""
|
|
303
|
+
library = runtime.get("library") or {}
|
|
304
|
+
return library.get("learn") == "on" and schedule(library)["before"]
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def active_config(config):
|
|
308
|
+
"""A copy of a task's configuration (config/agent.py TaskConfig) for its active
|
|
309
|
+
run: trigger "active" and the active run's live time as its wall budget."""
|
|
310
|
+
plan = schedule(config.runtime.get("library"))
|
|
311
|
+
active = copy(config)
|
|
312
|
+
active.runtime = {**config.runtime, "library": {**config.runtime["library"], "trigger": "active"}}
|
|
313
|
+
active.max_wall_seconds = plan["minutes"] * 60.0
|
|
314
|
+
return active
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Import old hints and unattributed skill statistics into the library's memory format."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
MIGRATION = "@experience_imported"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def hint_key(topic, name):
|
|
12
|
+
return f"memory/experience_{_id([topic, name])}.md"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _id(fields):
|
|
16
|
+
return hashlib.sha256(json.dumps(fields, ensure_ascii=False).encode("utf-8")).hexdigest()[:24]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def legacy_entries(directory, index):
|
|
20
|
+
def read(name):
|
|
21
|
+
path = Path(directory) / "experience" / name
|
|
22
|
+
try:
|
|
23
|
+
return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
|
24
|
+
except (OSError, ValueError):
|
|
25
|
+
return {}
|
|
26
|
+
|
|
27
|
+
for topic, hints in read("hints.json").items():
|
|
28
|
+
for name, hint in hints.items():
|
|
29
|
+
yield {"id": hint_key(topic, name), "kind": f"hint:{topic}", "name": name,
|
|
30
|
+
"text": json.dumps(hint["value"], ensure_ascii=False), "seen": hint.get("seen"),
|
|
31
|
+
"source": "legacy"}
|
|
32
|
+
for skill, stats in read("skills.json").items():
|
|
33
|
+
yield {"id": f"memory/historical_use_{_id(skill)}.md", "kind": "historical_use", "name": skill,
|
|
34
|
+
"text": json.dumps(stats, ensure_ascii=False), "seen": stats.get("runs"), "source": "legacy",
|
|
35
|
+
"summary": "Historical results, revision unknown"}
|
|
36
|
+
|
|
37
|
+
for key, row in index.items():
|
|
38
|
+
if not key.startswith(("skills/", "reactive/")):
|
|
39
|
+
continue
|
|
40
|
+
old = [item["previous_uses"] for item in row.get("history", []) if item.get("previous_uses")]
|
|
41
|
+
if row.get("uses"):
|
|
42
|
+
old.append(row["uses"])
|
|
43
|
+
for number, stats in enumerate(old):
|
|
44
|
+
yield {"id": f"memory/historical_use_{_id([key, number, stats])}.md",
|
|
45
|
+
"kind": "historical_use", "name": Path(key).stem,
|
|
46
|
+
"text": json.dumps(stats, ensure_ascii=False),
|
|
47
|
+
"seen": stats.get("ok", 0) + stats.get("failed", 0),
|
|
48
|
+
"source": "legacy library index", "summary": "Historical results, revision unknown"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Host dispatch and byte snapshots for package-owned learning fixtures."""
|
|
2
|
+
|
|
3
|
+
from importlib import import_module
|
|
4
|
+
import hashlib
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path, PurePosixPath
|
|
7
|
+
import re
|
|
8
|
+
import stat
|
|
9
|
+
import tarfile
|
|
10
|
+
import tempfile
|
|
11
|
+
import time
|
|
12
|
+
import tomllib
|
|
13
|
+
|
|
14
|
+
from harness.bench import bench_spec
|
|
15
|
+
from harness.core import sha256
|
|
16
|
+
from harness.paths import PACKAGE, packages
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
BUNDLES = Path("/opt/harness-fixtures")
|
|
20
|
+
MAX_BUNDLE = 2 * 1024 ** 3
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Budget:
|
|
24
|
+
def __init__(self, seconds, cancelled):
|
|
25
|
+
self.deadline = time.monotonic() + min(45.0, seconds)
|
|
26
|
+
self.cancelled = cancelled
|
|
27
|
+
|
|
28
|
+
def remaining(self, reserve=0):
|
|
29
|
+
if self.cancelled():
|
|
30
|
+
raise RuntimeError("conversation cancelled its fixture")
|
|
31
|
+
seconds = self.deadline - time.monotonic() - reserve
|
|
32
|
+
if seconds <= 0:
|
|
33
|
+
raise TimeoutError("fixture deadline expired")
|
|
34
|
+
return seconds
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _copy(source, target, budget):
|
|
38
|
+
digest = hashlib.sha256()
|
|
39
|
+
while True:
|
|
40
|
+
budget.remaining(reserve=1)
|
|
41
|
+
block = source.read(1024 * 1024)
|
|
42
|
+
if not block:
|
|
43
|
+
return digest.hexdigest()
|
|
44
|
+
if target is not None:
|
|
45
|
+
target.write(block)
|
|
46
|
+
digest.update(block)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _unpack(bundle, root, budget):
|
|
50
|
+
"""Extract a curated, self-contained runtime. Never create links or special files."""
|
|
51
|
+
seen, size = set(), 0
|
|
52
|
+
with tarfile.open(bundle, "r:") as archive:
|
|
53
|
+
for entry in archive:
|
|
54
|
+
budget.remaining(reserve=1)
|
|
55
|
+
path = PurePosixPath(entry.name)
|
|
56
|
+
if (not path.parts or "\\" in entry.name or ":" in entry.name
|
|
57
|
+
or path.parts[0] not in ("usr", "opt", "lib", "lib64", "etc")
|
|
58
|
+
or ".." in path.parts or path in seen or not (entry.isdir() or entry.isreg())):
|
|
59
|
+
raise ValueError("unsafe member in fixture bundle")
|
|
60
|
+
seen.add(path)
|
|
61
|
+
target = root.joinpath(*path.parts)
|
|
62
|
+
if entry.isdir():
|
|
63
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
64
|
+
continue
|
|
65
|
+
size += entry.size
|
|
66
|
+
if size > MAX_BUNDLE:
|
|
67
|
+
raise ValueError("fixture runtime bundle is too large")
|
|
68
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
69
|
+
with archive.extractfile(entry) as source, target.open("xb") as output:
|
|
70
|
+
_copy(source, output, budget)
|
|
71
|
+
target.chmod(0o755 if entry.mode & 0o111 else 0o644)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _sources(roots, budget, output=None):
|
|
75
|
+
files = {}
|
|
76
|
+
for prefix, root in roots.items():
|
|
77
|
+
for path in sorted(root.rglob("*")):
|
|
78
|
+
budget.remaining(reserve=1)
|
|
79
|
+
relative = path.relative_to(root)
|
|
80
|
+
if (path.suffix not in (".py", ".toml", ".txt") or not path.is_file()
|
|
81
|
+
or any(part.startswith(".") for part in relative.parts)):
|
|
82
|
+
continue
|
|
83
|
+
if path.is_symlink() or not path.resolve().is_relative_to(root):
|
|
84
|
+
raise ValueError("fixture runtime source must be a regular file in its package")
|
|
85
|
+
key = f"{prefix}/{relative.as_posix()}"
|
|
86
|
+
with path.open("rb") as source:
|
|
87
|
+
if not stat.S_ISREG(path.stat().st_mode):
|
|
88
|
+
raise ValueError("fixture runtime source must be a regular file")
|
|
89
|
+
if output is None:
|
|
90
|
+
files[key] = _copy(source, None, budget)
|
|
91
|
+
else:
|
|
92
|
+
target = output / key
|
|
93
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
94
|
+
with target.open("xb") as snapshot:
|
|
95
|
+
files[key] = _copy(source, snapshot, budget)
|
|
96
|
+
return files
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def configured(profile):
|
|
100
|
+
"""Read the host's fixture inventory, outside the model-facing runtime settings."""
|
|
101
|
+
path = os.environ.get("HARNESS_LEARNING_FIXTURES")
|
|
102
|
+
if not path or profile.get("physical") is not False:
|
|
103
|
+
return ()
|
|
104
|
+
with Path(path).open("rb") as stream:
|
|
105
|
+
inventory = tomllib.load(stream)
|
|
106
|
+
execution = profile.get("execution")
|
|
107
|
+
rows = tuple({**config, "type": execution} for config in inventory.get(execution, ()))
|
|
108
|
+
if any(not isinstance(config.get("skill"), str) for config in rows):
|
|
109
|
+
raise ValueError("each fixture needs a skill selector")
|
|
110
|
+
return rows
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _provider(execution):
|
|
114
|
+
spec = bench_spec(execution)
|
|
115
|
+
module = spec.factory.partition(":")[0].rpartition(".")[0]
|
|
116
|
+
return spec, import_module(f"{module}.fixture")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def describe(inventory):
|
|
120
|
+
descriptions = []
|
|
121
|
+
for config in inventory:
|
|
122
|
+
try:
|
|
123
|
+
_, provider = _provider(config["type"])
|
|
124
|
+
provider.supported()
|
|
125
|
+
descriptions.append(provider.describe(config))
|
|
126
|
+
except (ImportError, ValueError, RuntimeError):
|
|
127
|
+
continue
|
|
128
|
+
return descriptions
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def verify(entry, config, budget, run):
|
|
132
|
+
"""Return the exact revision and a verdict produced by the package provider."""
|
|
133
|
+
budget.remaining()
|
|
134
|
+
spec, provider = _provider(config["type"])
|
|
135
|
+
provider.supported()
|
|
136
|
+
installed = {package.name: package for package in packages()}
|
|
137
|
+
package = next(item for item in installed.values() if spec.path.is_relative_to(item.path))
|
|
138
|
+
roots = {"core/harness": PACKAGE.resolve(),
|
|
139
|
+
**{name: installed[name].path.resolve() for name in (package.name, *package.depends)}}
|
|
140
|
+
plan = provider.prepare(config, entry)
|
|
141
|
+
bundle = Path(config["bundle"]).resolve()
|
|
142
|
+
expected = config["bundle_sha256"]
|
|
143
|
+
bundle_root = Path(os.environ.get("HARNESS_FIXTURE_BUNDLES", BUNDLES)).resolve()
|
|
144
|
+
if (not bundle.is_relative_to(bundle_root) or not bundle.is_file() or bundle.stat().st_size > MAX_BUNDLE
|
|
145
|
+
or not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{64}", expected)):
|
|
146
|
+
raise ValueError("fixture needs a host-provisioned, SHA-256 pinned runtime bundle")
|
|
147
|
+
|
|
148
|
+
with tempfile.TemporaryDirectory(prefix="harness-fixture-") as temp:
|
|
149
|
+
directory = Path(temp)
|
|
150
|
+
root = directory / "root"
|
|
151
|
+
archive = directory / "runtime.tar"
|
|
152
|
+
with bundle.open("rb") as source, archive.open("xb") as snapshot:
|
|
153
|
+
if _copy(source, snapshot, budget) != expected:
|
|
154
|
+
raise ValueError("fixture runtime bundle digest changed")
|
|
155
|
+
files = _sources(roots, budget, root / "fixture")
|
|
156
|
+
item = entry.revision(bindings=provider.bindings(plan, files))
|
|
157
|
+
identity = sha256({"fixture": config, "bundle": expected, "sources": files})
|
|
158
|
+
try:
|
|
159
|
+
_unpack(archive, root, budget)
|
|
160
|
+
except tarfile.TarError as error:
|
|
161
|
+
raise ValueError("invalid fixture runtime bundle") from error
|
|
162
|
+
result = provider.observe(config, plan, root, budget)
|
|
163
|
+
if files != _sources(roots, budget):
|
|
164
|
+
raise ValueError("fixture dependency sources changed during the trial")
|
|
165
|
+
budget.remaining()
|
|
166
|
+
return item, {**result, "run": run, "revision": item.revision,
|
|
167
|
+
"fixture_type": config["type"], "fixture_sha256": identity}
|