@lmmslab/harness 0.0.0-stage → 0.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +124 -2
- package/bin/harness.js +71 -0
- package/package.json +34 -4
- package/python/harness/README.md +72 -0
- package/python/harness/__init__.py +8 -0
- package/python/harness/__main__.py +3 -0
- package/python/harness/bench.py +124 -0
- package/python/harness/builtin/benches/README.md +5 -0
- package/python/harness/builtin/benches/none/bench.toml +6 -0
- package/python/harness/builtin/configs/README.md +26 -0
- package/python/harness/builtin/configs/assistant.toml +11 -0
- package/python/harness/builtin/configs/default.toml +33 -0
- package/python/harness/builtin/configs/deploy/example-api.toml +10 -0
- package/python/harness/builtin/configs/models/deepseek-api.toml +21 -0
- package/python/harness/builtin/configs/models/deepseek-flash.toml +12 -0
- package/python/harness/builtin/configs/models/grok-low.toml +8 -0
- package/python/harness/builtin/configs/models/grok-medium.toml +43 -0
- package/python/harness/builtin/configs/models/opus-5.5.toml +11 -0
- package/python/harness/builtin/environment/README.md +53 -0
- package/python/harness/builtin/environment/constraints.txt +31 -0
- package/python/harness/builtin/model-settings.js +143 -0
- package/python/harness/builtin/prompts/conversation/core.md +5 -0
- package/python/harness/builtin/prompts/conversation/pack.toml +12 -0
- package/python/harness/builtin/prompts/core/core.md +88 -0
- package/python/harness/builtin/prompts/core/pack.toml +9 -0
- package/python/harness/builtin/prompts/reflect/core.md +45 -0
- package/python/harness/builtin/prompts/reflect/pack.toml +7 -0
- package/python/harness/builtin/prompts/shared/agent-skills.md +5 -0
- package/python/harness/builtin/prompts/shared/agents.md +14 -0
- package/python/harness/builtin/prompts/shared/checklist.md +21 -0
- package/python/harness/builtin/prompts/shared/entries.md +86 -0
- package/python/harness/builtin/prompts/shared/environments.md +21 -0
- package/python/harness/builtin/prompts/shared/learning-goal.md +31 -0
- package/python/harness/builtin/prompts/shared/learning.md +27 -0
- package/python/harness/builtin/prompts/shared/mcp.md +9 -0
- package/python/harness/builtin/prompts/shared/world.md +1 -0
- package/python/harness/builtin/skills/README.md +10 -0
- package/python/harness/builtin/skills/agents/__init__.py +88 -0
- package/python/harness/builtin/skills/agents/skill.toml +11 -0
- package/python/harness/builtin/skills/checklist/__init__.py +32 -0
- package/python/harness/builtin/skills/checklist/skill.toml +9 -0
- package/python/harness/builtin/skills/conversation/__init__.py +23 -0
- package/python/harness/builtin/skills/conversation/skill.toml +8 -0
- package/python/harness/builtin/skills/entries/__init__.py +62 -0
- package/python/harness/builtin/skills/entries/skill.toml +8 -0
- package/python/harness/builtin/skills/environments/__init__.py +69 -0
- package/python/harness/builtin/skills/environments/skill.toml +9 -0
- package/python/harness/builtin/skills/experience/__init__.py +27 -0
- package/python/harness/builtin/skills/experience/skill.toml +9 -0
- package/python/harness/builtin/skills/instructions/__init__.py +29 -0
- package/python/harness/builtin/skills/instructions/skill.toml +9 -0
- package/python/harness/builtin/skills/learning/__init__.py +189 -0
- package/python/harness/builtin/skills/learning/skill.toml +7 -0
- package/python/harness/builtin/skills/mcp/__init__.py +54 -0
- package/python/harness/builtin/skills/mcp/skill.toml +10 -0
- package/python/harness/builtin/skills/memory/__init__.py +130 -0
- package/python/harness/builtin/skills/memory/skill.toml +13 -0
- package/python/harness/builtin/skills/verify/__init__.py +31 -0
- package/python/harness/builtin/skills/verify/skill.toml +7 -0
- package/python/harness/builtin/third_party/sources.toml +40 -0
- package/python/harness/cli.py +1182 -0
- package/python/harness/config/__init__.py +224 -0
- package/python/harness/config/agent.py +777 -0
- package/python/harness/core/__init__.py +52 -0
- package/python/harness/core/artifacts.py +46 -0
- package/python/harness/core/codec.py +160 -0
- package/python/harness/core/contracts.py +75 -0
- package/python/harness/core/episodes.py +150 -0
- package/python/harness/core/format.py +51 -0
- package/python/harness/core/http.py +124 -0
- package/python/harness/core/lanes.py +324 -0
- package/python/harness/core/linux_sandbox.py +55 -0
- package/python/harness/core/media.py +96 -0
- package/python/harness/core/memory.py +147 -0
- package/python/harness/core/ports.py +63 -0
- package/python/harness/core/procs.py +227 -0
- package/python/harness/core/remote.py +102 -0
- package/python/harness/core/spans.py +166 -0
- package/python/harness/core/trace.py +139 -0
- package/python/harness/core/wait.py +12 -0
- package/python/harness/core/worker.py +388 -0
- package/python/harness/core/worker_pipe.py +119 -0
- package/python/harness/embodiments.py +217 -0
- package/python/harness/extensions.py +112 -0
- package/python/harness/host/__init__.py +397 -0
- package/python/harness/host/client.py +201 -0
- package/python/harness/host/http.py +720 -0
- package/python/harness/host/listing.py +68 -0
- package/python/harness/host/server.py +189 -0
- package/python/harness/host/sessions.py +406 -0
- package/python/harness/host/tasks.py +1308 -0
- package/python/harness/host/workers.py +257 -0
- package/python/harness/host/workspaces.py +256 -0
- package/python/harness/model_list.py +117 -0
- package/python/harness/packages/__init__.py +73 -0
- package/python/harness/packages/addons.py +121 -0
- package/python/harness/packages/archive.py +122 -0
- package/python/harness/packages/describe.py +46 -0
- package/python/harness/packages/environments.py +448 -0
- package/python/harness/packages/fetch.py +57 -0
- package/python/harness/packages/manager.py +587 -0
- package/python/harness/packages/manifest.py +221 -0
- package/python/harness/packages/migrate.py +68 -0
- package/python/harness/packages/provision.py +38 -0
- package/python/harness/packages/registry.py +275 -0
- package/python/harness/packages/resolve.py +89 -0
- package/python/harness/packages/sources.py +141 -0
- package/python/harness/packages/state.py +213 -0
- package/python/harness/packages/versions.py +112 -0
- package/python/harness/paths.py +218 -0
- package/python/harness/providers.py +303 -0
- package/python/harness/runtime/__init__.py +8 -0
- package/python/harness/runtime/decision/__init__.py +1 -0
- package/python/harness/runtime/decision/loop.py +198 -0
- package/python/harness/runtime/decision/messages.py +271 -0
- package/python/harness/runtime/decision/model.py +424 -0
- package/python/harness/runtime/decision/prompts.py +248 -0
- package/python/harness/runtime/decision/protocol.py +169 -0
- package/python/harness/runtime/decision/thoughts.py +199 -0
- package/python/harness/runtime/events.py +213 -0
- package/python/harness/runtime/execution/__init__.py +1 -0
- package/python/harness/runtime/execution/attached.py +96 -0
- package/python/harness/runtime/execution/control.py +211 -0
- package/python/harness/runtime/execution/entries.py +665 -0
- package/python/harness/runtime/execution/kernel.py +811 -0
- package/python/harness/runtime/execution/noenv.py +84 -0
- package/python/harness/runtime/execution/results.py +40 -0
- package/python/harness/runtime/execution/session.py +871 -0
- package/python/harness/runtime/execution/skills.py +436 -0
- package/python/harness/runtime/learning/__init__.py +1 -0
- package/python/harness/runtime/learning/active.py +314 -0
- package/python/harness/runtime/learning/experience.py +48 -0
- package/python/harness/runtime/learning/fixtures.py +167 -0
- package/python/harness/runtime/learning/learner.py +651 -0
- package/python/harness/runtime/learning/library.py +504 -0
- package/python/harness/runtime/learning/reflect.py +151 -0
- package/python/harness/runtime/learning/usage.py +30 -0
- package/python/harness/runtime/learning/views.py +89 -0
- package/python/harness/runtime/reactive/__init__.py +1 -0
- package/python/harness/runtime/reactive/rules.py +549 -0
- package/python/harness/runtime/safety/__init__.py +1 -0
- package/python/harness/runtime/safety/approval.py +131 -0
- package/python/harness/runtime/safety/input.py +62 -0
- package/python/harness/runtime/safety/isolation.py +419 -0
- package/python/harness/runtime/safety/shield.py +129 -0
- package/python/harness/runtime/status.py +23 -0
- package/python/harness/runtime/task/__init__.py +7 -0
- package/python/harness/runtime/task/context.py +431 -0
- package/python/harness/runtime/task/delegation.py +468 -0
- package/python/harness/runtime/task/environments.py +268 -0
- package/python/harness/runtime/task/run.py +2878 -0
- package/python/harness/runtime/tick.py +909 -0
- package/python/harness/sdk/__init__.py +281 -0
- package/python/harness/services/__init__.py +6 -0
- package/python/harness/services/broker.py +35 -0
- package/python/harness/services/connectors.py +88 -0
- package/python/harness/services/mcp.py +298 -0
- package/python/harness/services/mcp_stdio.py +131 -0
- package/python/harness/services/placement.py +82 -0
- package/python/harness/services/registry.py +156 -0
- package/python/harness/settings.py +536 -0
- package/python/harness/terminal.py +79 -0
- package/python/harness/transcript.py +297 -0
- package/python/harness/tui/__init__.py +68 -0
- package/python/harness/tui/chat.py +1181 -0
- package/python/harness/tui/pty.py +81 -0
- package/python/harness/tui/renderer.py +45 -0
- package/python/harness/tui/timeline_rows.py +58 -0
- package/python/harness/tui/views.py +434 -0
|
@@ -0,0 +1,504 @@
|
|
|
1
|
+
"""A versioned library of what the harness has learned, for any action scenario (a robot,
|
|
2
|
+
a desktop, a website, a game): one learner fills it (runtime/learning/learner.py), in a run (in-context:
|
|
3
|
+
an entry admitted takes effect at once) and between runs (self-improvement on development
|
|
4
|
+
seeds), and evaluation reads it frozen.
|
|
5
|
+
|
|
6
|
+
Layout of a library directory:
|
|
7
|
+
|
|
8
|
+
notes/*.md high-level notes, appended to the system prompt
|
|
9
|
+
memory/*.md long-term memory entries, not in the prompt; programs find
|
|
10
|
+
them with recall() beside the run's own (runtime/task/context.py;
|
|
11
|
+
layer: notes)
|
|
12
|
+
skills/*.py low-level skills and handlers, executed in the kernel
|
|
13
|
+
namespace before the first program (layer: low)
|
|
14
|
+
reactive/*.py guard/controller helpers, executed only when the reactive
|
|
15
|
+
layer is enabled (layer: reactive)
|
|
16
|
+
tests/<name>.py the test of skills/<name>.py or reactive/<name>.py: Python that
|
|
17
|
+
uses the entry and raises when it does not work
|
|
18
|
+
skills/<name>.files/ an entry's attachments (a trained model's weights...)
|
|
19
|
+
library.toml optional: `notes_title`, the line the notes stand under in the
|
|
20
|
+
prompt (a library written by hand holds knowledge, not lessons)
|
|
21
|
+
index.json per entry: its evidence, whether it was learned actively or
|
|
22
|
+
passively, by whom, when, its test runs and its uses (each
|
|
23
|
+
execution of its code in a run that learns: memory.fold_uses;
|
|
24
|
+
learner.py); not part of the version
|
|
25
|
+
admissions.jsonl one line an admission: section, file, the sha256 of its bytes,
|
|
26
|
+
the version after it (what an audit checks the entries against)
|
|
27
|
+
candidates/ entries proposed and waiting for validation on development seeds
|
|
28
|
+
(not loaded, not part of the version)
|
|
29
|
+
|
|
30
|
+
Each entry is one file (with its test and attachments). The library version is the sha256
|
|
31
|
+
over the loaded entries' relative paths and bytes, their tests and attachments included;
|
|
32
|
+
every episode records it, and every admission changes it. An entry's evidence grades it
|
|
33
|
+
(EVIDENCE): a library loads the entries at or above its `load` level, and one that failed
|
|
34
|
+
never. Evaluation uses a single version, and a library may be restricted to some layers
|
|
35
|
+
(Tab. 8 ablation).
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
import ast
|
|
39
|
+
import base64
|
|
40
|
+
import hashlib
|
|
41
|
+
from functools import wraps
|
|
42
|
+
from graphlib import TopologicalSorter
|
|
43
|
+
import inspect
|
|
44
|
+
import json
|
|
45
|
+
import time
|
|
46
|
+
from types import FunctionType
|
|
47
|
+
from contextlib import contextmanager, nullcontext
|
|
48
|
+
from dataclasses import dataclass
|
|
49
|
+
from pathlib import Path
|
|
50
|
+
import tomllib
|
|
51
|
+
from weakref import WeakKeyDictionary
|
|
52
|
+
|
|
53
|
+
from harness.core import atomic_write
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
SECTIONS = {"notes": ".md", "memory": ".md", "skills": ".py", "reactive": ".py"}
|
|
57
|
+
# Library layers (Tab. 8 ablation) and the sections each one enables.
|
|
58
|
+
LAYER_OF = {"notes": "notes", "memory": "notes", "skills": "skills", "reactive": "reactive"}
|
|
59
|
+
LAYERS = ("notes", "skills", "reactive")
|
|
60
|
+
# What the notes stand under in the prompt, unless library.toml names another title.
|
|
61
|
+
NOTES_TITLE = "Notes from earlier episodes:"
|
|
62
|
+
# How well an entry is known to work, weakest first: proposed (no check yet), provisional
|
|
63
|
+
# (its own test passed in a run; a note or memory entry the model admitted), replayed (its
|
|
64
|
+
# test passed again at the start of a later run), validated (it raised the score on
|
|
65
|
+
# held-out development seeds). An entry the library holds without an index row was put
|
|
66
|
+
# there by hand or by self-improvement before the index: validated. A failed one never loads.
|
|
67
|
+
EVIDENCE = ("proposed", "provisional", "replayed", "validated")
|
|
68
|
+
FAILED = "failed"
|
|
69
|
+
INDEX = "index.json"
|
|
70
|
+
ADMISSIONS = "admissions.jsonl"
|
|
71
|
+
TESTS = "tests"
|
|
72
|
+
CANDIDATES = "candidates"
|
|
73
|
+
_code_origins = WeakKeyDictionary()
|
|
74
|
+
_code_active = WeakKeyDictionary()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class Revision:
|
|
80
|
+
key: str
|
|
81
|
+
source: bytes
|
|
82
|
+
test: bytes | None = None
|
|
83
|
+
attachments: tuple = ()
|
|
84
|
+
dependencies: tuple = ()
|
|
85
|
+
bindings: tuple = ()
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def source_sha256(self):
|
|
89
|
+
return hashlib.sha256(self.source).hexdigest()
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def revision(self):
|
|
93
|
+
return hashlib.sha256(self.pack()).hexdigest()
|
|
94
|
+
|
|
95
|
+
def pack(self):
|
|
96
|
+
encode = lambda data: base64.b64encode(data).decode("ascii")
|
|
97
|
+
return json.dumps({"key": self.key, "source": encode(self.source),
|
|
98
|
+
"test": None if self.test is None else encode(self.test),
|
|
99
|
+
"attachments": [[name, encode(data)] for name, data in self.attachments],
|
|
100
|
+
"dependencies": list(self.dependencies),
|
|
101
|
+
**({"bindings": list(self.bindings)} if self.bindings else {})},
|
|
102
|
+
sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
103
|
+
|
|
104
|
+
@classmethod
|
|
105
|
+
def unpack(cls, data):
|
|
106
|
+
row = json.loads(data)
|
|
107
|
+
return cls(row["key"], base64.b64decode(row["source"]),
|
|
108
|
+
None if row["test"] is None else base64.b64decode(row["test"]),
|
|
109
|
+
tuple((name, base64.b64decode(value)) for name, value in row["attachments"]),
|
|
110
|
+
tuple(tuple(dependency) for dependency in row["dependencies"]),
|
|
111
|
+
tuple(tuple(binding) for binding in row.get("bindings", ())))
|
|
112
|
+
|
|
113
|
+
def save(self, directory):
|
|
114
|
+
path = Path(directory) / ".revisions" / self.revision
|
|
115
|
+
if not path.is_file():
|
|
116
|
+
atomic_write(path, self.pack())
|
|
117
|
+
return self.revision
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def read(cls, directory, revision):
|
|
121
|
+
if len(revision) != 64 or any(char not in "0123456789abcdef" for char in revision):
|
|
122
|
+
raise ValueError("invalid library revision")
|
|
123
|
+
data = (Path(directory) / ".revisions" / revision).read_bytes()
|
|
124
|
+
if hashlib.sha256(data).hexdigest() != revision:
|
|
125
|
+
raise ValueError(f"library revision {revision} has changed")
|
|
126
|
+
return cls.unpack(data)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@dataclass(frozen=True)
|
|
130
|
+
class LibrarySnapshot:
|
|
131
|
+
directory: Path
|
|
132
|
+
layers: tuple
|
|
133
|
+
load: str
|
|
134
|
+
version: str
|
|
135
|
+
entries: tuple
|
|
136
|
+
index_json: bytes
|
|
137
|
+
settings: bytes
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def index(self):
|
|
141
|
+
return json.loads(self.index_json)
|
|
142
|
+
|
|
143
|
+
def manifest(self):
|
|
144
|
+
return {"directory": str(self.directory), "layers": list(self.layers), "version": self.version,
|
|
145
|
+
**({"load": self.load} if self.load != "proposed" else {}),
|
|
146
|
+
"entries": {section: [item.key.split("/", 1)[1] for item in self.entries
|
|
147
|
+
if item.key.startswith(section + "/")] for section in SECTIONS}}
|
|
148
|
+
|
|
149
|
+
def notes(self):
|
|
150
|
+
texts = [item.source.decode("utf-8").strip() for item in self.entries if item.key.startswith("notes/")]
|
|
151
|
+
title = str(tomllib.loads(self.settings.decode("utf-8")).get("notes_title") or NOTES_TITLE)
|
|
152
|
+
return f"\n{title}\n" + "\n".join(f"- {text}" for text in texts) + "\n" if texts else ""
|
|
153
|
+
|
|
154
|
+
def api_summary(self, active):
|
|
155
|
+
return active_summary(item for item in self.entries if item.key in active)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def active_summary(items):
|
|
159
|
+
lines = [line for item in items for line in summary_lines(item.source.decode("utf-8"))]
|
|
160
|
+
return "\nLibrary skills (already defined in your kernel):\n" + "\n".join(lines) + "\n" if lines else ""
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def executable(key):
|
|
164
|
+
return key.startswith(("skills/", "reactive/"))
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def eligible(item, row, layers, load):
|
|
168
|
+
section = item.key.split("/", 1)[0]
|
|
169
|
+
evidence = row.get("evidence", "validated")
|
|
170
|
+
if not selects(section, evidence, layers, load):
|
|
171
|
+
return False
|
|
172
|
+
if not executable(item.key) or not row or row.get("origin") == "hand":
|
|
173
|
+
return True
|
|
174
|
+
proof = row.get("proof") or {}
|
|
175
|
+
return (evidence in ("provisional", "replayed", "validated")
|
|
176
|
+
and proof.get("revision") == item.revision and bool(proof.get("conditions")))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _verified_bindings(api, bindings):
|
|
181
|
+
"""Pin fixture actions to the original kernel API and the tested implementation."""
|
|
182
|
+
from types import CodeType, MethodType
|
|
183
|
+
|
|
184
|
+
scope = {}
|
|
185
|
+
compiled = {}
|
|
186
|
+
for name, module, owner, digest in bindings:
|
|
187
|
+
action = api.get(name)
|
|
188
|
+
if not isinstance(action, MethodType) or action.__module__ != module:
|
|
189
|
+
raise ValueError(f"verified binding {name} is unavailable")
|
|
190
|
+
code = action.__func__.__code__
|
|
191
|
+
if code.co_filename not in compiled:
|
|
192
|
+
data = Path(code.co_filename).read_bytes()
|
|
193
|
+
compiled[code.co_filename] = (hashlib.sha256(data).hexdigest(),
|
|
194
|
+
compile(data, code.co_filename, "exec", dont_inherit=True))
|
|
195
|
+
source_digest, source = compiled[code.co_filename]
|
|
196
|
+
if source_digest != digest or not any(
|
|
197
|
+
code == method for cls in source.co_consts if isinstance(cls, CodeType) and cls.co_name == owner
|
|
198
|
+
for method in cls.co_consts if isinstance(method, CodeType) and method.co_name == name
|
|
199
|
+
):
|
|
200
|
+
raise ValueError(f"verified binding {name} has a different implementation")
|
|
201
|
+
scope[name] = action
|
|
202
|
+
return scope
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def stage_entries(items, namespace, active, reactive, *, api=None):
|
|
206
|
+
from harness.runtime.execution.entries import ClosedLoop
|
|
207
|
+
from harness.runtime.reactive.rules import _stage_rules
|
|
208
|
+
from harness.core import spans
|
|
209
|
+
|
|
210
|
+
batch = {item.key: (item, hand) for item, hand in items}
|
|
211
|
+
targets = {key: item.revision for key, (item, _) in batch.items()}
|
|
212
|
+
order = TopologicalSorter({key: [dep for dep, _ in item.dependencies if dep in batch]
|
|
213
|
+
for key, (item, _) in batch.items()}).static_order()
|
|
214
|
+
revisions = {**{key: row[0] for key, row in active.items()}, **targets}
|
|
215
|
+
for key, (_, _, dependencies) in active.items():
|
|
216
|
+
if key not in targets and any(revisions.get(dep) != required for dep, required in dependencies):
|
|
217
|
+
raise ValueError(f"replace {key} together with its changed dependency")
|
|
218
|
+
scope = dict(namespace)
|
|
219
|
+
owners = dict(active)
|
|
220
|
+
for key in targets:
|
|
221
|
+
for name, value in owners.get(key, (None, {}))[1].items():
|
|
222
|
+
if scope.get(name) is value:
|
|
223
|
+
del scope[name]
|
|
224
|
+
|
|
225
|
+
with _stage_rules(reactive, targets) as staged_rules:
|
|
226
|
+
for key in order:
|
|
227
|
+
item, hand = batch[key]
|
|
228
|
+
for key, required in item.dependencies:
|
|
229
|
+
if revisions.get(key) != required:
|
|
230
|
+
raise ValueError(f"{item.key} needs {key} at {required}")
|
|
231
|
+
execution = _verified_bindings(api or {}, item.bindings) if item.bindings else scope
|
|
232
|
+
if item.bindings:
|
|
233
|
+
execution["__builtins__"] = {}
|
|
234
|
+
before = dict(execution)
|
|
235
|
+
origin = item.key, item.source_sha256, item.revision
|
|
236
|
+
tree = ast.parse(item.source, f"<library {item.key}>")
|
|
237
|
+
if not hand:
|
|
238
|
+
for node in tree.body:
|
|
239
|
+
if not isinstance(node, (ast.Import, ast.ImportFrom, ast.FunctionDef, ast.ClassDef,
|
|
240
|
+
ast.AsyncFunctionDef, ast.Assign, ast.AnnAssign, ast.Expr)):
|
|
241
|
+
raise ValueError(f"{item.key} has unsupported initialization")
|
|
242
|
+
for child in _initialization(node):
|
|
243
|
+
if isinstance(child, ast.Call):
|
|
244
|
+
if (not isinstance(child.func, ast.Name)
|
|
245
|
+
or child.func.id not in ("when", "watch", "on", "rule", "reflex")):
|
|
246
|
+
raise ValueError(f"{item.key} must put runtime actions inside a callable")
|
|
247
|
+
if isinstance(child, (ast.Attribute, ast.Subscript)) and isinstance(child.ctx, ast.Store):
|
|
248
|
+
raise ValueError(f"{item.key} cannot modify live objects during activation")
|
|
249
|
+
if staged_rules is not None:
|
|
250
|
+
staged_rules.library_origin = origin
|
|
251
|
+
exec(compile(tree, f"<library {item.key}>", "exec"), execution)
|
|
252
|
+
exports = {name: value for name, value in execution.items() if not name.startswith("__")
|
|
253
|
+
and (name not in before or before[name] is not value)}
|
|
254
|
+
if item.bindings and exports.keys() & (api or {}).keys():
|
|
255
|
+
raise ValueError(f"{item.key} would replace a built-in skill")
|
|
256
|
+
for name, value in exports.items():
|
|
257
|
+
if isinstance(value, type) and issubclass(value, ClosedLoop):
|
|
258
|
+
_code_origins[value] = origin
|
|
259
|
+
_code_active[value] = active
|
|
260
|
+
elif isinstance(value, FunctionType) and value.__globals__ is execution:
|
|
261
|
+
wrapped = _tracked(value, origin, active, spans)
|
|
262
|
+
scope[name] = exports[name] = wrapped
|
|
263
|
+
scope.update(exports)
|
|
264
|
+
owners[item.key] = (item.revision, exports, item.dependencies)
|
|
265
|
+
return scope, owners, staged_rules
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _initialization(node):
|
|
269
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
270
|
+
fields = (*node.decorator_list, *node.args.defaults, *filter(None, node.args.kw_defaults))
|
|
271
|
+
return (child for part in fields for child in ast.walk(part))
|
|
272
|
+
if isinstance(node, ast.ClassDef):
|
|
273
|
+
fields = (*node.bases, *node.decorator_list, *node.keywords)
|
|
274
|
+
return (*[child for part in fields for child in ast.walk(part)],
|
|
275
|
+
*[child for part in node.body for child in _initialization(part)])
|
|
276
|
+
return ast.walk(node)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _tracked(function, origin, active, spans):
|
|
280
|
+
@contextmanager
|
|
281
|
+
def execution():
|
|
282
|
+
if active.get(origin[0], (None,))[0] != origin[2]:
|
|
283
|
+
raise RuntimeError(f"library revision for {origin[0]} is no longer active")
|
|
284
|
+
start = time.monotonic()
|
|
285
|
+
try:
|
|
286
|
+
yield
|
|
287
|
+
except BaseException as error:
|
|
288
|
+
spans.step(function.__name__, 1, "gave up", False, time.monotonic() - start,
|
|
289
|
+
type(error).__name__, library=origin)
|
|
290
|
+
raise
|
|
291
|
+
else:
|
|
292
|
+
spans.step(function.__name__, 1, "done", True, time.monotonic() - start, library=origin)
|
|
293
|
+
|
|
294
|
+
if inspect.iscoroutinefunction(function):
|
|
295
|
+
@wraps(function)
|
|
296
|
+
async def call(*args, **kwargs):
|
|
297
|
+
with execution():
|
|
298
|
+
return await function(*args, **kwargs)
|
|
299
|
+
elif inspect.isasyncgenfunction(function):
|
|
300
|
+
@wraps(function)
|
|
301
|
+
async def call(*args, **kwargs):
|
|
302
|
+
with execution():
|
|
303
|
+
async for result in function(*args, **kwargs):
|
|
304
|
+
yield result
|
|
305
|
+
elif inspect.isgeneratorfunction(function):
|
|
306
|
+
@wraps(function)
|
|
307
|
+
def call(*args, **kwargs):
|
|
308
|
+
with execution():
|
|
309
|
+
return (yield from function(*args, **kwargs))
|
|
310
|
+
else:
|
|
311
|
+
@wraps(function)
|
|
312
|
+
def call(*args, **kwargs):
|
|
313
|
+
with execution():
|
|
314
|
+
return function(*args, **kwargs)
|
|
315
|
+
_code_origins[call] = origin
|
|
316
|
+
return call
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def publish_entries(staged, namespace, active, reactive):
|
|
320
|
+
scope, owners, rules = staged
|
|
321
|
+
with reactive.library_boundary if rules is not None else nullcontext():
|
|
322
|
+
for key, (_, exports, _) in active.items():
|
|
323
|
+
if key in owners and owners[key][1] is not exports:
|
|
324
|
+
for name, value in exports.items():
|
|
325
|
+
if namespace.get(name) is value and name not in scope:
|
|
326
|
+
del namespace[name]
|
|
327
|
+
namespace.update(scope)
|
|
328
|
+
active.update(owners)
|
|
329
|
+
if rules is not None:
|
|
330
|
+
reactive._publish_rules(rules)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def assert_active(code):
|
|
334
|
+
origin = _code_origins.get(code)
|
|
335
|
+
active = _code_active.get(code)
|
|
336
|
+
if origin is not None and active is not None and active.get(origin[0], (None,))[0] != origin[2]:
|
|
337
|
+
raise RuntimeError(f"library revision for {origin[0]} is no longer active")
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def code_origin(cls):
|
|
341
|
+
return _code_origins.get(cls)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def rank(evidence) -> int:
|
|
345
|
+
"""An evidence level's place in EVIDENCE (-1 for failed or unknown)."""
|
|
346
|
+
return EVIDENCE.index(evidence) if evidence in EVIDENCE else -1
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def selects(section, evidence, layers, load) -> bool:
|
|
350
|
+
"""Whether this revision passes a reader's library selection."""
|
|
351
|
+
return LAYER_OF.get(section) in layers and rank(evidence) >= rank(load) >= 0
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
class Library:
|
|
355
|
+
def __init__(self, directory, layers=("notes", "skills", "reactive"), load="proposed"):
|
|
356
|
+
self.directory = Path(directory).resolve()
|
|
357
|
+
unknown = set(layers) - set(LAYERS)
|
|
358
|
+
if unknown:
|
|
359
|
+
raise ValueError(f"Unknown library layers: {sorted(unknown)}")
|
|
360
|
+
if load not in EVIDENCE:
|
|
361
|
+
raise ValueError(f"library load is one of {list(EVIDENCE)}, not {load!r}")
|
|
362
|
+
self.layers = tuple(layers)
|
|
363
|
+
self.load = load
|
|
364
|
+
if not self.directory.is_dir():
|
|
365
|
+
raise ValueError(f"Library directory not found: {self.directory}")
|
|
366
|
+
|
|
367
|
+
def index(self) -> dict:
|
|
368
|
+
"""{"<section>/<file>": {"evidence", "source", "origin", ...}} (learner.py)."""
|
|
369
|
+
path = self.directory / INDEX
|
|
370
|
+
try:
|
|
371
|
+
data = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
|
|
372
|
+
except (OSError, ValueError):
|
|
373
|
+
data = {}
|
|
374
|
+
return data if isinstance(data, dict) else {}
|
|
375
|
+
|
|
376
|
+
def evidence(self, section, filename, index=None) -> str:
|
|
377
|
+
row = (self.index() if index is None else index).get(f"{section}/{filename}")
|
|
378
|
+
return row.get("evidence", "validated") if isinstance(row, dict) else "validated"
|
|
379
|
+
|
|
380
|
+
def entries(self, section):
|
|
381
|
+
if LAYER_OF[section] not in self.layers:
|
|
382
|
+
return []
|
|
383
|
+
folder = self.directory / section
|
|
384
|
+
if not folder.is_dir():
|
|
385
|
+
return []
|
|
386
|
+
index = self.index()
|
|
387
|
+
return sorted(path for path in folder.glob(f"*{SECTIONS[section]}")
|
|
388
|
+
if not (index.get(f"{section}/{path.name}") or {}).get("forgotten")
|
|
389
|
+
and selects(section, self.evidence(section, path.name, index), self.layers, self.load))
|
|
390
|
+
|
|
391
|
+
def test(self, section, filename) -> Path | None:
|
|
392
|
+
"""The test of a code entry, if it has one."""
|
|
393
|
+
path = self.directory / TESTS / f"{Path(filename).stem}.py"
|
|
394
|
+
return path if section in ("skills", "reactive") and path.is_file() else None
|
|
395
|
+
|
|
396
|
+
def attachments(self, section, filename) -> list[Path]:
|
|
397
|
+
folder = self.directory / section / f"{Path(filename).stem}.files"
|
|
398
|
+
return sorted(path for path in folder.rglob("*") if path.is_file()) if folder.is_dir() else []
|
|
399
|
+
|
|
400
|
+
def capture(self, *, include_unselected=False):
|
|
401
|
+
"""Read one index and each selected payload once."""
|
|
402
|
+
from harness.runtime.learning.experience import MIGRATION, legacy_entries
|
|
403
|
+
|
|
404
|
+
index = self.index()
|
|
405
|
+
legacy = {}
|
|
406
|
+
if "notes" in self.layers and not index.get(MIGRATION):
|
|
407
|
+
for item in list(legacy_entries(self.directory, index)):
|
|
408
|
+
key = item["id"]
|
|
409
|
+
if key not in index:
|
|
410
|
+
legacy[key] = Revision(key, item["text"].encode("utf-8"))
|
|
411
|
+
index[key] = {"memory": item, "evidence": "validated", "origin": "legacy"}
|
|
412
|
+
settings = self.directory / "library.toml"
|
|
413
|
+
raw_settings = settings.read_bytes() if settings.is_file() else b""
|
|
414
|
+
digest = hashlib.sha256()
|
|
415
|
+
if raw_settings:
|
|
416
|
+
digest.update(b"library.toml\0" + raw_settings + b"\0")
|
|
417
|
+
found = []
|
|
418
|
+
for section, suffix in SECTIONS.items():
|
|
419
|
+
folder = self.directory / section
|
|
420
|
+
paths = {path.name: path for path in folder.glob(f"*{suffix}")}
|
|
421
|
+
paths.update({key.split("/", 1)[1]: folder / key.split("/", 1)[1]
|
|
422
|
+
for key, row in index.items() if key.startswith(section + "/") and (row.get("revision") or key in legacy)})
|
|
423
|
+
for name, path in sorted(paths.items()):
|
|
424
|
+
key = f"{section}/{name}"
|
|
425
|
+
row = index.get(key) or {}
|
|
426
|
+
selected = selects(section, row.get("evidence", "validated"), self.layers, self.load)
|
|
427
|
+
if row.get("forgotten") or not selected and not include_unselected:
|
|
428
|
+
continue
|
|
429
|
+
if key in legacy:
|
|
430
|
+
item = legacy[key]
|
|
431
|
+
elif row.get("revision"):
|
|
432
|
+
item = Revision.read(self.directory, row["revision"])
|
|
433
|
+
if item.key != key:
|
|
434
|
+
raise ValueError(f"library index points {key} at a different entry")
|
|
435
|
+
else:
|
|
436
|
+
test = self.test(section, name)
|
|
437
|
+
attachments = self.attachments(section, name)
|
|
438
|
+
item = Revision(key, path.read_bytes(), test.read_bytes() if test else None,
|
|
439
|
+
tuple((part.relative_to(self.directory).as_posix(), part.read_bytes())
|
|
440
|
+
for part in attachments))
|
|
441
|
+
found.append(item)
|
|
442
|
+
if not selected:
|
|
443
|
+
continue
|
|
444
|
+
parts = [(item.key, item.source)]
|
|
445
|
+
if item.test is not None:
|
|
446
|
+
parts.append((f"{TESTS}/{path.stem}.py", item.test))
|
|
447
|
+
parts.extend(item.attachments)
|
|
448
|
+
for relative, data in parts:
|
|
449
|
+
digest.update(relative.encode() + b"\0" + data + b"\0")
|
|
450
|
+
if row.get("memory"):
|
|
451
|
+
digest.update(json.dumps(row["memory"], sort_keys=True).encode("utf-8"))
|
|
452
|
+
if item.dependencies:
|
|
453
|
+
digest.update(json.dumps(item.dependencies).encode("utf-8"))
|
|
454
|
+
if item.bindings:
|
|
455
|
+
digest.update(json.dumps(item.bindings).encode("utf-8"))
|
|
456
|
+
return LibrarySnapshot(self.directory, self.layers, self.load, digest.hexdigest(), tuple(found),
|
|
457
|
+
json.dumps(index).encode("utf-8"), raw_settings)
|
|
458
|
+
|
|
459
|
+
def version(self) -> str:
|
|
460
|
+
return self.capture().version
|
|
461
|
+
|
|
462
|
+
def manifest(self) -> dict:
|
|
463
|
+
return self.capture().manifest()
|
|
464
|
+
|
|
465
|
+
def notes(self) -> str:
|
|
466
|
+
return self.capture().notes()
|
|
467
|
+
|
|
468
|
+
def api_summary(self, reactive_enabled: bool) -> str:
|
|
469
|
+
captured = self.capture()
|
|
470
|
+
active = {item.key for item in captured.entries if item.key.startswith("skills/")
|
|
471
|
+
or reactive_enabled and item.key.startswith("reactive/")}
|
|
472
|
+
return captured.api_summary(active)
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def summary_lines(code: str) -> list[str]:
|
|
476
|
+
"""` name(args) first docstring line` of each public function and class of `code`
|
|
477
|
+
(a class: its parameters are the loop's; a closed loop is called like a function)."""
|
|
478
|
+
|
|
479
|
+
lines = []
|
|
480
|
+
try:
|
|
481
|
+
tree = ast.parse(code)
|
|
482
|
+
except SyntaxError:
|
|
483
|
+
return lines
|
|
484
|
+
for node in tree.body:
|
|
485
|
+
if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and not node.name.startswith("_"):
|
|
486
|
+
args = ", ".join(a.arg for a in node.args.args) if isinstance(node, ast.FunctionDef) else "..."
|
|
487
|
+
doc = (ast.get_docstring(node) or "").strip().splitlines()
|
|
488
|
+
lines.append(f" {node.name}({args})" + (f" {doc[0]}" if doc else ""))
|
|
489
|
+
return lines
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def library_for(runtime: dict, base: Path) -> Library:
|
|
493
|
+
"""The library a [runtime.library] table names (its path relative to `base`, or under
|
|
494
|
+
$HARNESS_HOME); one that learns is created empty when it is not there yet."""
|
|
495
|
+
from harness.runtime.learning.learner import library_path, settings
|
|
496
|
+
|
|
497
|
+
config = runtime["library"]
|
|
498
|
+
path = library_path(config["path"])
|
|
499
|
+
directory = (base / path).resolve() if not path.is_absolute() else path
|
|
500
|
+
chosen = settings(config)
|
|
501
|
+
if chosen["learn"] == "on":
|
|
502
|
+
for section in SECTIONS:
|
|
503
|
+
(directory / section).mkdir(parents=True, exist_ok=True)
|
|
504
|
+
return Library(directory, config.get("layers", ("notes", "skills", "reactive")), chosen["load"])
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Reflection: turn a development episode into one candidate library entry.
|
|
2
|
+
|
|
3
|
+
The model reads a compact trajectory (programs, events, outcome on the dev
|
|
4
|
+
seed) and the current library, and may propose one entry:
|
|
5
|
+
|
|
6
|
+
<note name="...">one or two sentences</note> -> notes/<name>.md
|
|
7
|
+
<memory name="...">situation and what worked</memory> -> memory/<name>.md
|
|
8
|
+
<skill name="...">def f(...): ...</skill> -> skills/<name>.py
|
|
9
|
+
<reactive name="...">def g(obs): ...</reactive> -> reactive/<name>.py
|
|
10
|
+
<none/>
|
|
11
|
+
|
|
12
|
+
The reply gives its kind, name and text; the learner checks them as every candidate
|
|
13
|
+
(runtime/learning/learner.py candidate()). A candidate only enters the library after it improves
|
|
14
|
+
the score on held-out development seeds (harness self-improve).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import ast
|
|
18
|
+
import json
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
import re
|
|
21
|
+
|
|
22
|
+
from harness.core.episodes import new_name, read_records
|
|
23
|
+
from harness.paths import ROOT
|
|
24
|
+
from harness.runtime.learning.learner import KINDS, MEMORY_MAX_CHARS, NOTE_MAX_CHARS
|
|
25
|
+
from harness.runtime.learning.library import Library, eligible, executable
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# The request, one paragraph per embodiment (the pack's markers: the embodiment kind, the
|
|
29
|
+
# bench, the run's skills). A robot's is the request of the robot harness unchanged.
|
|
30
|
+
PACK = ROOT / "prompts" / "reflect"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def system_prompt(names=("robot",)) -> str:
|
|
34
|
+
"""The reflection request for a run whose embodiment kind, bench and enabled skills
|
|
35
|
+
are `names` (prompts/reflect): its lines joined into one paragraph."""
|
|
36
|
+
from harness.runtime.decision.prompts import assemble_prompt
|
|
37
|
+
|
|
38
|
+
text = assemble_prompt(PACK, list(names), features=())
|
|
39
|
+
paragraph = " ".join(line.strip() for line in text.splitlines() if line.strip())
|
|
40
|
+
return paragraph.replace("{note_max_chars}", str(NOTE_MAX_CHARS)).replace(
|
|
41
|
+
"{memory_max_chars}", str(MEMORY_MAX_CHARS))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
_ENTRY = re.compile(rf'<({"|".join(KINDS)})\s+name="([^"]*)">(.*?)</\1>', re.DOTALL)
|
|
45
|
+
# An outcome's keys every worker reports; the rest is what an evaluator measured besides.
|
|
46
|
+
OUTCOME_KEYS = {"success", "reward", "process_score", "terminated", "truncated", "steps", "unscorable"}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def summarize_episode(run: Path, max_chars: int = 12000, *, since=0) -> str:
|
|
50
|
+
lines = []
|
|
51
|
+
summary = json.loads((run / "summary.json").read_text(encoding="utf-8")) if (run / "summary.json").exists() else {}
|
|
52
|
+
# A trace cut off mid-line (a crash) is read up to its last whole record.
|
|
53
|
+
program = 0
|
|
54
|
+
for event in read_records(run):
|
|
55
|
+
kind = event["event"]
|
|
56
|
+
if kind == "reset":
|
|
57
|
+
lines.append(f"Instruction: {event['observation']['instruction']}")
|
|
58
|
+
elif kind == "model_reply" and program >= since and event.get("ahead", "used") == "used":
|
|
59
|
+
lines.append(f"[turn {event['call']}] {event['text'].strip()[:1500]}")
|
|
60
|
+
elif kind == "cell":
|
|
61
|
+
program += 1
|
|
62
|
+
if program <= since:
|
|
63
|
+
continue
|
|
64
|
+
result = event["result"]
|
|
65
|
+
text = f" -> {result['status']}"
|
|
66
|
+
if result.get("escalation"):
|
|
67
|
+
ev = result["escalation"]["event"]
|
|
68
|
+
text += f" {ev['kind']} from {ev['source']}: {json.dumps(ev['detail'])[:300]}"
|
|
69
|
+
if result.get("error"):
|
|
70
|
+
text += " " + result["error"].strip().splitlines()[-1][:300]
|
|
71
|
+
if result.get("stdout"):
|
|
72
|
+
text += " | out: " + result["stdout"].strip().replace("\n", " / ")[:300]
|
|
73
|
+
lines.append(text)
|
|
74
|
+
elif kind == "skill_step" and program >= since and event.get("library_revision"):
|
|
75
|
+
lines.append(f"Used {event.get('library_key')} at {event['library_revision']}: "
|
|
76
|
+
f"{event.get('what')}, ok={event.get('ok')}")
|
|
77
|
+
elif kind == "verify" and program >= since:
|
|
78
|
+
lines.append(f" verify {event['claim'][:200]!r}: {event['answer'][:200]}")
|
|
79
|
+
elif kind == "outcome" and set(event.get("outcome") or {}) - OUTCOME_KEYS:
|
|
80
|
+
# What the evaluator measured besides the score (a practice segment's time, deaths).
|
|
81
|
+
extra = {key: value for key, value in event["outcome"].items() if key not in OUTCOME_KEYS}
|
|
82
|
+
lines.append(f"Measured: {json.dumps(extra, default=str)[:600]}")
|
|
83
|
+
if summary:
|
|
84
|
+
lines.append(f"Outcome: success={summary.get('success')} "
|
|
85
|
+
f"score={summary.get('reward')} status={summary.get('status')}"
|
|
86
|
+
+ (f" guards fired={summary['guard_triggers']}" if summary.get("guard_triggers") else ""))
|
|
87
|
+
text = "\n".join(lines)
|
|
88
|
+
if len(text) > max_chars:
|
|
89
|
+
text = text[:max_chars // 2] + "\n...\n" + text[-max_chars // 2:]
|
|
90
|
+
return text
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def library_text(library, holdings=None):
|
|
94
|
+
if isinstance(library, (str, Path)):
|
|
95
|
+
library = Library(library)
|
|
96
|
+
captured = library.capture()
|
|
97
|
+
index = captured.index
|
|
98
|
+
entries = {item.key: item for item in captured.entries
|
|
99
|
+
if not executable(item.key) or holdings is None and eligible(
|
|
100
|
+
item, index.get(item.key) or {}, captured.layers, captured.load)}
|
|
101
|
+
if holdings is not None:
|
|
102
|
+
entries.update({key: item for key, (item, _) in holdings.items()})
|
|
103
|
+
lines = []
|
|
104
|
+
for key, item in entries.items():
|
|
105
|
+
row = index.get(key) or {}
|
|
106
|
+
uses = (row.get("uses_by_revision") or {}).get(item.revision) or {}
|
|
107
|
+
lines.append(f"--- {key} ({item.revision}, {uses.get('ok', 0)} successes, "
|
|
108
|
+
f"{uses.get('failed', 0)} failures)\n{item.source.decode('utf-8').strip()[:1200]}")
|
|
109
|
+
return "\n".join(lines)[:8000] or "(empty)"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def propose(chat, run: Path, library, embodiment="robot", *, since=0, holdings=None):
|
|
113
|
+
"""Use the same reflection request for online and offline learning."""
|
|
114
|
+
names = [embodiment] if isinstance(embodiment, str) else list(embodiment)
|
|
115
|
+
prompt = (f"Current library:\n{library_text(library, holdings)}\n\n"
|
|
116
|
+
f"Episode:\n{summarize_episode(run, since=since)}\n\nPropose at most one addition.")
|
|
117
|
+
from harness.core.trace import Trace
|
|
118
|
+
from harness.runtime.decision.thoughts import think
|
|
119
|
+
from harness.runtime.events import EVENTS
|
|
120
|
+
|
|
121
|
+
trace = None if getattr(chat, "thinks", False) else Trace(run / "reflection" / new_name(), EVENTS)
|
|
122
|
+
reply = think(chat, system_prompt(names), [{"role": "user", "content": prompt}], trace=trace)
|
|
123
|
+
return parse_candidate(reply.text, require_test="robot" not in names), reply.text
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def parse_candidate(text: str, *, require_test=True):
|
|
127
|
+
"""The entry a reply proposes, (kind, name, text), or None: none, empty, or code that
|
|
128
|
+
is not the one public function with a docstring the request asks for."""
|
|
129
|
+
match = _ENTRY.search(text or "")
|
|
130
|
+
if not match:
|
|
131
|
+
return None
|
|
132
|
+
kind, name, body = match.group(1), match.group(2), match.group(3).strip()
|
|
133
|
+
test = None
|
|
134
|
+
if kind in ("skill", "reactive"): # code; a note and a memory entry are text
|
|
135
|
+
check = re.search(r"<test>(.*?)</test>", text[match.end():], re.DOTALL)
|
|
136
|
+
if check:
|
|
137
|
+
test = re.sub(r"^```[a-z]*\n|\n```$", "", check.group(1).strip())
|
|
138
|
+
if require_test and not test:
|
|
139
|
+
return None
|
|
140
|
+
body = re.sub(r"^```[a-z]*\n|\n```$", "", body).strip()
|
|
141
|
+
try:
|
|
142
|
+
tree = ast.parse(body)
|
|
143
|
+
except SyntaxError:
|
|
144
|
+
return None
|
|
145
|
+
functions = [node for node in tree.body if isinstance(node, ast.FunctionDef) and not node.name.startswith("_")]
|
|
146
|
+
if len(functions) != 1 or not ast.get_docstring(functions[0]):
|
|
147
|
+
return None
|
|
148
|
+
if not body:
|
|
149
|
+
return None
|
|
150
|
+
entry = kind, name, body + "\n"
|
|
151
|
+
return (*entry, test) if test else entry
|