devlyn-cli 4.2.26 → 4.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/bin/devlyn.js +510 -0
  2. package/bin/engines.js +20 -15
  3. package/bin/harness-activation.js +1 -2
  4. package/bin/release.js +16 -3
  5. package/bin/skill-history.json +5 -0
  6. package/config/skills/_shared/harness-store.py +202 -0
  7. package/config/skills/_shared/task-completion.md +6 -3
  8. package/config/skills/_shared/warm-build-dirs.py +97 -2
  9. package/config/skills/devlyn-ideate/SKILL.md +5 -1
  10. package/config/skills/devlyn-ideate/references/loop.md +12 -2
  11. package/config/skills/devlyn-ideate/references/plan-add.md +12 -0
  12. package/config/skills/devlyn-ideate/scripts/acceptance.py +5 -0
  13. package/config/skills/devlyn-ideate/scripts/queue.py +165 -15
  14. package/harness/AGENTS.md +37 -0
  15. package/harness/CLAUDE.md +37 -0
  16. package/native/claude/.claude-plugin/plugin.json +6 -0
  17. package/native/claude/skills/_shared/.ruff.toml +2 -0
  18. package/native/claude/skills/_shared/adapters/claude.md +15 -0
  19. package/native/claude/skills/_shared/adapters/codex.md +14 -0
  20. package/native/claude/skills/_shared/adapters/omp.md +6 -0
  21. package/native/claude/skills/_shared/codex-monitored.sh +263 -0
  22. package/native/claude/skills/_shared/engine-doctor.sh +70 -0
  23. package/native/claude/skills/_shared/engine-preflight.md +29 -0
  24. package/native/claude/skills/_shared/expected-contract.py +242 -0
  25. package/native/claude/skills/_shared/expected.schema.json +120 -0
  26. package/native/claude/skills/_shared/harness-store.py +202 -0
  27. package/native/claude/skills/_shared/invocation-receipt.py +499 -0
  28. package/native/claude/skills/_shared/platform-support.py +395 -0
  29. package/native/claude/skills/_shared/role-config.py +247 -0
  30. package/native/claude/skills/_shared/runtime-principles.md +35 -0
  31. package/native/claude/skills/_shared/task-complete.py +2865 -0
  32. package/native/claude/skills/_shared/task-completion.md +249 -0
  33. package/native/claude/skills/_shared/warm-build-dirs.py +1065 -0
  34. package/native/claude/skills/devlyn-engines/SKILL.md +63 -0
  35. package/native/claude/skills/devlyn-ideate/SKILL.md +132 -0
  36. package/native/claude/skills/devlyn-ideate/references/elicitation.md +29 -0
  37. package/native/claude/skills/devlyn-ideate/references/loop.md +123 -0
  38. package/native/claude/skills/devlyn-ideate/references/package-format.md +128 -0
  39. package/native/claude/skills/devlyn-ideate/references/plan-add.md +12 -0
  40. package/native/claude/skills/devlyn-ideate/scripts/acceptance.py +882 -0
  41. package/native/claude/skills/devlyn-ideate/scripts/queue.py +2191 -0
  42. package/native/codex/.agents/plugins/marketplace.json +12 -0
  43. package/native/codex/plugins/devlyn/.codex-plugin/plugin.json +6 -0
  44. package/native/codex/plugins/devlyn/skills/_shared/.ruff.toml +2 -0
  45. package/native/codex/plugins/devlyn/skills/_shared/adapters/claude.md +15 -0
  46. package/native/codex/plugins/devlyn/skills/_shared/adapters/codex.md +14 -0
  47. package/native/codex/plugins/devlyn/skills/_shared/adapters/omp.md +6 -0
  48. package/native/codex/plugins/devlyn/skills/_shared/codex-monitored.sh +263 -0
  49. package/native/codex/plugins/devlyn/skills/_shared/engine-doctor.sh +70 -0
  50. package/native/codex/plugins/devlyn/skills/_shared/engine-preflight.md +29 -0
  51. package/native/codex/plugins/devlyn/skills/_shared/expected-contract.py +242 -0
  52. package/native/codex/plugins/devlyn/skills/_shared/expected.schema.json +120 -0
  53. package/native/codex/plugins/devlyn/skills/_shared/harness-store.py +202 -0
  54. package/native/codex/plugins/devlyn/skills/_shared/invocation-receipt.py +499 -0
  55. package/native/codex/plugins/devlyn/skills/_shared/platform-support.py +395 -0
  56. package/native/codex/plugins/devlyn/skills/_shared/role-config.py +247 -0
  57. package/native/codex/plugins/devlyn/skills/_shared/runtime-principles.md +35 -0
  58. package/native/codex/plugins/devlyn/skills/_shared/task-complete.py +2865 -0
  59. package/native/codex/plugins/devlyn/skills/_shared/task-completion.md +249 -0
  60. package/native/codex/plugins/devlyn/skills/_shared/warm-build-dirs.py +1065 -0
  61. package/native/codex/plugins/devlyn/skills/devlyn-engines/SKILL.md +63 -0
  62. package/native/codex/plugins/devlyn/skills/devlyn-ideate/SKILL.md +132 -0
  63. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/elicitation.md +29 -0
  64. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/loop.md +123 -0
  65. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/package-format.md +128 -0
  66. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/plan-add.md +12 -0
  67. package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/acceptance.py +882 -0
  68. package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/queue.py +2191 -0
  69. package/package.json +3 -2
  70. package/release.json +1630 -0
@@ -0,0 +1,2191 @@
1
+ #!/usr/bin/env python3
2
+ """Ideate loop driver: package validation, the queue, locks, reconciliation and the serial drain.
3
+
4
+ `check` validates a loop package, `add` captures it with its queue rows (which may
5
+ replace one legacy row), `status` shows the reconciled queue and `drain -- <argv>` runs each
6
+ eligible task through allocation, committed inputs, one host-supplied executor
7
+ exchange, evidence-derived acceptance, the terminal transition and delivery.
8
+ Protocol and commands: ../references/loop.md; formats: ../references/package-format.md.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import argparse
13
+ import contextlib
14
+ import datetime
15
+ import errno
16
+ import functools
17
+ import hashlib
18
+ import json
19
+ import os
20
+ from pathlib import Path
21
+ import re
22
+ import runpy
23
+ import shlex
24
+ import subprocess
25
+ import sys
26
+ import tempfile
27
+ import unittest
28
+
29
+ SKILLS = Path(__file__).resolve().parents[2]
30
+ QUEUE = "docs/specs/queue.md" # The legacy queue: read for materialize, never written.
31
+ CAPTURES = "refs/devlyn/captures/"
32
+ SENTINEL = "<!-- devlyn:verification -->"
33
+ ID = r"[a-z0-9][a-z0-9-]{0,62}"
34
+ SHA_RE = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}")
35
+ JSON_FENCE_RE = re.compile(r"(?ms)^```json[ \t]*\n(.*?)\n```[ \t]*$")
36
+ ROW_RE = re.compile(r"- \[(?P<mark>[ xF])\] (?P<text>.*)")
37
+ ITEM_RE = re.compile(rf"(?P<identity>(?P<loop>{ID})\.(?P<task>{ID})) \[(?P<title>(?:[^\]\\\n]|\\.)*)\]"
38
+ r"\(docs/specs/(?P=loop)/(?P=task)/spec\.md\)(?P<rest>(?: — .*)?)")
39
+ REPLACES_RE = re.compile(r"<!-- replaces docs/specs/queue\.md row \(occurrence ([1-9][0-9]*)\): (- \[ \] .*) -->")
40
+ REQUIREMENT_RE = re.compile(r"[-*] (R[1-9][0-9]*): \S")
41
+ META_SECTIONS = ("Intent", "Constraints and exclusions", "Tasks", "Overall acceptance", "Execution policy",
42
+ "Decisions and assumptions")
43
+ TASK_SECTIONS = ("Context and goal", "Requirements", "Scope and constraints", "Out of scope", "Prerequisite inputs",
44
+ "Verification", "Deliverable and cleanup")
45
+ MANIFEST_KEYS = {"schema_version", "loop_id", "base_ref", "base_sha", "delivery", "tasks", "integration_task_id"}
46
+ FRONTMATTER_KEYS = {"id", "loop_id", "title", "kind", "review_requirements", "complexity"}
47
+ KINDS = {"feature", "spike", "prototype"}
48
+ UNSUPPORTED = {"process_evidence": "phase process obligations", "required_risk_probe_requirements": "risk-probe obligations",
49
+ "tier_a_waivers": "benchmark scope waivers", "spec_output_files": "benchmark scope surfaces"}
50
+ SETTLED = {"LOCAL_ONLY", "COMPLETE", "FAILED"}
51
+ OBLIGATIONS = [
52
+ "Work only in this worktree on its owned branch under the installed methodology; never edit the loop package or docs/specs/queue.md.",
53
+ "Commit any source changes on the owned branch; when none are needed, use inputs_sha without creating an empty commit. Leave no uncommitted or untracked source and remove residue the change created.",
54
+ "Run `runner` on the committed candidate; review records must bind engine, model, that source and both contract digests.",
55
+ "Write `submission` and exit; report a blocker instead of weakening acceptance.",
56
+ ]
57
+
58
+
59
+ class LoopError(Exception):
60
+ pass
61
+
62
+
63
+ class Waiting(LoopError):
64
+ """The task cannot proceed now: it waits with this reason while independent work continues."""
65
+
66
+
67
+ def require(condition, message):
68
+ if not condition:
69
+ raise LoopError(message)
70
+
71
+
72
+ @functools.lru_cache(maxsize=None)
73
+ def shared(name):
74
+ return runpy.run_path(str(SKILLS / "_shared" / f"{name}.py"))
75
+
76
+
77
+ @functools.lru_cache(maxsize=None)
78
+ def acceptance():
79
+ return runpy.run_path(str(Path(__file__).with_name("acceptance.py")))
80
+
81
+
82
+ def git_run(cwd, *args, data=None, env=None, ok=(0,)):
83
+ result = subprocess.run(["git", "-C", str(cwd), *args], input=data, capture_output=True, env=env)
84
+ require(result.returncode in ok, f"git {' '.join(args[:2])}: {result.stderr.decode('utf-8', 'replace').strip()}")
85
+ return result
86
+
87
+
88
+ def git(cwd, *args, **kwargs):
89
+ return git_run(cwd, *args, **kwargs).stdout.decode("utf-8").strip()
90
+
91
+
92
+ def show(cwd, rev, path):
93
+ result = git_run(cwd, "cat-file", "blob", f"{rev}:{path}", ok=(0, 128))
94
+ return result.stdout if result.returncode == 0 else None
95
+
96
+
97
+ def ancestor(cwd, old, new):
98
+ return git_run(cwd, "merge-base", "--is-ancestor", old, new, ok=(0, 1)).returncode == 0
99
+
100
+
101
+ def read_text(path):
102
+ try:
103
+ return Path(path).read_bytes().decode("utf-8").replace("\r\n", "\n")
104
+ except (OSError, UnicodeError) as exc:
105
+ raise LoopError(f"{path}: {exc}") from exc
106
+
107
+
108
+ def read_json(path):
109
+ try:
110
+ return shared("expected-contract")["loads_strict_json"](read_text(path))
111
+ except ValueError as exc:
112
+ raise LoopError(f"{path}: {exc}") from exc
113
+
114
+
115
+ def skill_diagnostics(anchor):
116
+ running = SKILLS / ".devlyn-install.json"
117
+ markers = dict.fromkeys([running, *(anchor / host / "skills/.devlyn-install.json" for host in (".claude", ".agents"))])
118
+ versions, errors = {}, {}
119
+ for marker in markers:
120
+ try:
121
+ data = json.loads(marker.read_text(encoding="utf-8"))
122
+ if not isinstance(data, dict) or not isinstance(data.get("version"), (str, type(None))):
123
+ raise ValueError("install marker must be an object with a string or null version")
124
+ versions[marker] = data.get("version")
125
+ except FileNotFoundError:
126
+ continue # Global-only and source checkouts need no repository install marker.
127
+ except (OSError, ValueError, RecursionError) as exc:
128
+ errors[marker] = str(exc)
129
+ skill = {"root": str(SKILLS), "version": versions.get(running)}
130
+ warnings = []
131
+ for marker in markers:
132
+ if marker in errors:
133
+ detail = f"unreadable: {errors[marker]}"
134
+ elif running not in errors and marker in versions and versions[marker] != skill["version"]:
135
+ detail = f"version {json.dumps(versions[marker])}"
136
+ else:
137
+ continue
138
+ warnings.append(f"Running skill {skill['root']} (version {json.dumps(skill['version'])}); "
139
+ f"install marker {marker} ({detail}); run npx devlyn-cli@latest update.")
140
+ return {"skill": skill, "warnings": warnings}
141
+
142
+
143
+ def write_json(path, value):
144
+ acceptance()["write_json"](path, value)
145
+
146
+
147
+ def ref_value(cwd, ref):
148
+ return git_run(cwd, "rev-parse", "--verify", "--quiet", ref, ok=(0, 1)).stdout.decode("utf-8").strip()
149
+
150
+
151
+ def progress(identity, event):
152
+ print(f"devlyn-loop: {identity}: {event}", file=sys.stderr, flush=True)
153
+
154
+
155
+ def repository(path):
156
+ anchor = Path(git(Path(path), "rev-parse", "--show-toplevel")).resolve()
157
+ return anchor, Path(git(anchor, "rev-parse", "--path-format=absolute", "--git-common-dir")).resolve()
158
+
159
+
160
+ @contextlib.contextmanager
161
+ def lock(common, name, *, blocking):
162
+ """Native locks on the common Gitdir: drain.lock for a controller's lifetime, queue.lock for short mutations."""
163
+ directory = common / "devlyn-loops"
164
+ directory.mkdir(exist_ok=True)
165
+ with contextlib.ExitStack() as stack:
166
+ try:
167
+ stack.enter_context(shared("platform-support")["file_lock"](directory / name, blocking=blocking))
168
+ except BlockingIOError as exc:
169
+ raise LoopError(f"another drain is active for this repository ({directory / name})") from exc
170
+ yield
171
+
172
+
173
+ def harness_call(name, *args):
174
+ helper = shared("harness-store")
175
+ try:
176
+ return helper[name](*args)
177
+ except helper["HarnessError"] as exc:
178
+ raise LoopError(str(exc)) from exc
179
+
180
+
181
+ def enforce_binding(common, anchor=None):
182
+ binding = harness_call("read_binding", common)
183
+ try:
184
+ harness_call("check_binding", binding, harness_call("resolve", SKILLS))
185
+ except (LoopError, OSError) as exc:
186
+ # A queue that settled outside queue.py (its last delivery finished through task-complete.py) keeps a binding no
187
+ # work needs: another release drops it instead of refusing. Callers outside the queue lock pass `anchor`.
188
+ if binding and anchor is not None and release_settled_binding(anchor, common, interrupted_add=True):
189
+ return enforce_binding(common)
190
+ if binding:
191
+ argv = harness_call("recovery_argv", binding, sys.argv[1:])
192
+ raise LoopError(f"{exc}; recovery argv: {json.dumps(argv, ensure_ascii=False)}") from exc
193
+ raise
194
+ return binding
195
+
196
+
197
+ def unfinished(v):
198
+ if v["unreadable"] or counts(v)["legacy_pending"]:
199
+ return True
200
+ for state in v["states"].values():
201
+ receipt = state["receipt"]
202
+ # A blocked task waits on a failed prerequisite for good, as drain counts it: it holds no binding.
203
+ if state.get("invalid") or state["kind"] in {"pending", "active"} or receipt and (
204
+ not receipt.get("queue") or receipt.get("delivery") not in SETTLED):
205
+ return True
206
+ # An interrupted capture with no add record is still resumable work.
207
+ refs = git(v["anchor"], "for-each-ref", "--format=%(refname)", CAPTURES).splitlines()
208
+ return any(ref.removeprefix(CAPTURES) not in v["records"] for ref in refs)
209
+
210
+
211
+ def bindable(anchor, common):
212
+ """Whether a first add may bind its release: the queue reads and holds no unfinished work. A queue that cannot be read
213
+ is not proven settled, so the add captures unbound, as its skill source adds, and status and drain report why."""
214
+ try:
215
+ return not unfinished(view(anchor, common, pure=True))
216
+ except (LoopError, OSError):
217
+ return False
218
+
219
+
220
+ def release_settled_binding(anchor, common, *, interrupted_add=False):
221
+ """Remove the binding once nothing needs it. The bound release keeps one from an interrupted first add (no record
222
+ yet) so its retry rebinds the same identity; another release (`interrupted_add`) drops that too, since a binding
223
+ published before its capture captured nothing. Returns whether it was removed."""
224
+ with lock(common, "queue.lock", blocking=True):
225
+ if not harness_call("read_binding", common):
226
+ return False
227
+ v = view(anchor, common, pure=True)
228
+ if unfinished(v) or not (v["records"] or interrupted_add):
229
+ return False
230
+ (common / "devlyn-loops/harness.json").unlink()
231
+ return True
232
+
233
+
234
+ def sections(text, path):
235
+ found, current, fence = {}, None, False
236
+ for line in text.split("\n"):
237
+ fence ^= line.startswith("```")
238
+ match = None if fence else re.fullmatch(r"## +(\S.*?)\s*", line)
239
+ if match:
240
+ current = match.group(1)
241
+ require(current not in found, f"{path}: duplicate section '## {current}'")
242
+ found[current] = []
243
+ elif current is not None:
244
+ found[current].append(line)
245
+ return {name: "\n".join(lines).strip() for name, lines in found.items()}
246
+
247
+
248
+ def require_sections(found, names, path):
249
+ missing = [name for name in names if not found.get(name)]
250
+ require(not missing, f"{path}: missing or empty section(s): " + ", ".join(f"'## {name}'" for name in missing))
251
+
252
+
253
+ def frontmatter(text, path):
254
+ end = text.find("\n---\n", 3)
255
+ require(text.startswith("---\n") and end > 0, f"{path}: must start with a --- frontmatter block closed by ---")
256
+ values = {}
257
+ for line in filter(str.strip, text[4:end].split("\n")):
258
+ key, separator, value = (part.strip() for part in line.partition(":"))
259
+ require(separator and re.fullmatch(r"[a-z_]+", key) and key not in values, f"{path}: invalid or duplicate frontmatter line {line!r}")
260
+ values[key] = ([item.strip().strip("\"'") for item in value[1:-1].split(",") if item.strip()]
261
+ if value.startswith("[") and value.endswith("]") else value.strip("\"'"))
262
+ return values, text[end + 5:]
263
+
264
+
265
+ def find_cycle(graph):
266
+ state, stack = {}, []
267
+
268
+ def visit(node):
269
+ state[node] = "open"
270
+ stack.append(node)
271
+ for dep in graph[node]:
272
+ if state.get(dep) == "open":
273
+ return stack[stack.index(dep):] + [dep]
274
+ if dep not in state and (cycle := visit(dep)):
275
+ return cycle
276
+ state[node] = "done"
277
+ stack.pop()
278
+ return None
279
+
280
+ return next((cycle for node in graph if node not in state and (cycle := visit(node))), None)
281
+
282
+
283
+ def closure(graph, node):
284
+ seen, todo = set(), list(graph[node])
285
+ while todo:
286
+ dep = todo.pop()
287
+ if dep not in seen:
288
+ seen.add(dep)
289
+ todo.extend(graph[dep])
290
+ return seen
291
+
292
+
293
+ def check_manifest(anchor, manifest, loop_id, meta):
294
+ def fail(message):
295
+ raise LoopError(f"{meta}: manifest {message}")
296
+
297
+ if not isinstance(manifest, dict) or set(manifest) != MANIFEST_KEYS:
298
+ fail("keys must be exactly " + ", ".join(sorted(MANIFEST_KEYS)))
299
+ if manifest["schema_version"] != 1 or isinstance(manifest["schema_version"], bool):
300
+ fail("schema_version must be 1")
301
+ if manifest["loop_id"] != loop_id or not re.fullmatch(ID, loop_id):
302
+ fail(f"loop_id must equal its directory name and match {ID}")
303
+ base_ref, base_sha = manifest["base_ref"], manifest["base_sha"]
304
+ if not isinstance(base_ref, str) or not base_ref or git_run(anchor, "check-ref-format", "refs/heads/" + base_ref, ok=(0, 1)).returncode:
305
+ fail("base_ref must be a branch name")
306
+ if not isinstance(base_sha, str) or not SHA_RE.fullmatch(base_sha) or git_run(anchor, "cat-file", "-e", base_sha + "^{commit}", ok=(0, 1, 128)).returncode:
307
+ fail("base_sha must name an exact commit in this repository")
308
+ if manifest["delivery"] not in {"auto", "pr", "local-only"}:
309
+ fail("delivery must be auto, pr or local-only")
310
+ tasks = manifest["tasks"]
311
+ if not isinstance(tasks, list) or not tasks:
312
+ fail("tasks must be a non-empty list")
313
+ order = []
314
+ for entry in tasks:
315
+ if not isinstance(entry, dict) or set(entry) != {"id", "spec", "depends_on"}:
316
+ fail("tasks need exactly id, spec and depends_on")
317
+ task, deps = entry["id"], entry["depends_on"]
318
+ if not isinstance(task, str) or not re.fullmatch(ID, task) or task in order:
319
+ fail(f"task id {task!r} must be unique and match {ID}")
320
+ if entry["spec"] != f"{task}/spec.md":
321
+ fail(f"task {task} spec link must be {task}/spec.md")
322
+ if not isinstance(deps, list) or not all(isinstance(dep, str) for dep in deps) or len(set(deps)) != len(deps):
323
+ fail(f"task {task} depends_on must list distinct task ids")
324
+ order.append(task)
325
+ graph = {entry["id"]: entry["depends_on"] for entry in tasks}
326
+ for task, deps in graph.items():
327
+ if unknown := [dep for dep in deps if dep not in graph]:
328
+ fail(f"task {task} depends on unknown task(s): {', '.join(unknown)}")
329
+ if cycle := find_cycle(graph):
330
+ fail("dependency cycle: " + " -> ".join(cycle))
331
+ for task, deps in graph.items():
332
+ if late := [dep for dep in deps if order.index(dep) > order.index(task)]:
333
+ fail(f"task {task} must be listed after its dependencies: {', '.join(late)}")
334
+ final = manifest["integration_task_id"]
335
+ if final != order[-1] or set(order[:-1]) - closure(graph, final):
336
+ fail("integration_task_id must name the last task, which depends directly or transitively on every other task")
337
+
338
+
339
+ def package_bytes(anchor, rev, rel):
340
+ """A package file's bytes in its loop's capture `rev`, or in the checkout when the loop has no record; None if absent."""
341
+ if rev is not None:
342
+ return show(anchor, rev, rel)
343
+ return (anchor / rel).read_bytes() if (anchor / rel).is_file() else None
344
+
345
+
346
+ def package_text(anchor, rev, rel):
347
+ """A package file's text and the label that names it in errors."""
348
+ label = anchor / rel if rev is None else f"{rev}:{rel}"
349
+ data = package_bytes(anchor, rev, rel)
350
+ require(data is not None, f"{label}: no such file")
351
+ try:
352
+ return data.decode("utf-8").replace("\r\n", "\n"), label
353
+ except UnicodeError as exc:
354
+ raise LoopError(f"{label}: {exc}") from exc
355
+
356
+
357
+ def load_expected(text, path, requirements, review):
358
+ try:
359
+ data = shared("expected-contract")["loads_strict_json"](text)
360
+ except ValueError as exc:
361
+ raise LoopError(f"{path}: {exc}") from exc
362
+
363
+ def fail(message):
364
+ raise LoopError(f"{path}: {message}")
365
+
366
+ for key, label in UNSUPPORTED.items():
367
+ if isinstance(data, dict) and key in data:
368
+ fail(f"{key} ({label}) is unsupported by loop acceptance; translate it into verification_commands or review_requirements without weakening it")
369
+ if error := shared("expected-contract")["validate_expected_shape"](data):
370
+ fail(error)
371
+ commands = data.get("verification_commands", [])
372
+ for index, command in enumerate(commands):
373
+ refs = command.get("contract_refs", [])
374
+ if not refs or any(ref not in requirements for ref in refs):
375
+ fail(f"verification_commands[{index}].contract_refs must name requirement IDs of this task")
376
+ for index, pattern in enumerate(data.get("forbidden_patterns", [])):
377
+ try:
378
+ re.compile(pattern["pattern"])
379
+ except re.error as exc:
380
+ fail(f"forbidden_patterns[{index}] is not a valid regex: {exc}")
381
+ if data.get("pure_design") is True:
382
+ if commands or set(review) != set(requirements):
383
+ fail("pure_design permits no verification_commands and needs review_requirements covering every requirement")
384
+ elif not commands:
385
+ fail("runtime tasks need verification_commands; a pure_design task declares it with full review coverage")
386
+ covered = {ref for command in commands for ref in command["contract_refs"]} | set(review)
387
+ if uncovered := [req for req in requirements if req not in covered]:
388
+ fail("requirement(s) without command or review coverage: " + ", ".join(uncovered))
389
+ return data
390
+
391
+
392
+ def load_task(anchor, manifest, entry, rev):
393
+ loop_id, task = manifest["loop_id"], entry["id"]
394
+ text, path = package_text(anchor, rev, f"docs/specs/{loop_id}/{task}/spec.md")
395
+ front, body = frontmatter(text, path)
396
+
397
+ def fail(message):
398
+ raise LoopError(f"{path}: {message}")
399
+
400
+ if "status" in front:
401
+ fail("status is queue state, not contract; remove it from the frontmatter")
402
+ if unknown := sorted(set(front) - FRONTMATTER_KEYS):
403
+ fail(f"unknown frontmatter key(s): {', '.join(unknown)}")
404
+ if missing := sorted(FRONTMATTER_KEYS - {"complexity"} - set(front)):
405
+ fail(f"missing frontmatter key(s): {', '.join(missing)}")
406
+ if (front["id"], front["loop_id"]) != (task, loop_id):
407
+ fail("frontmatter id and loop_id must match the manifest")
408
+ if not isinstance(front["title"], str) or not front["title"]:
409
+ fail("title must be a non-empty line")
410
+ if not isinstance(front["kind"], str) or front["kind"] not in KINDS:
411
+ fail("kind must be feature, spike or prototype")
412
+ review = front["review_requirements"]
413
+ if not isinstance(review, list):
414
+ fail("review_requirements must be a list such as [R2] or []")
415
+ found = sections(body, path)
416
+ require_sections(found, TASK_SECTIONS, path)
417
+ if not re.search(rf"(?m)^{re.escape(SENTINEL)}\n## +Verification[ \t]*$", body):
418
+ fail(f"'## Verification' must directly follow the {SENTINEL} sentinel line")
419
+ requirements = [m.group(1) for line in found["Requirements"].split("\n") if (m := REQUIREMENT_RE.match(line))]
420
+ if not requirements or len(set(requirements)) != len(requirements):
421
+ fail("Requirements need unique stable IDs written as '- R1: <obligation>'")
422
+ if unknown := [req for req in review if req not in requirements]:
423
+ fail(f"review_requirements name unknown requirement(s): {', '.join(unknown)}")
424
+ expected, expected_path = package_text(anchor, rev, f"docs/specs/{loop_id}/{task}/spec.expected.json")
425
+ return {"id": task, "title": front["title"], "kind": front["kind"], "requirements": requirements, "review": review,
426
+ "depends_on": entry["depends_on"], "expected": load_expected(expected, expected_path, requirements, review)}
427
+
428
+
429
+ def load_manifest(anchor, loop_id, rev=None):
430
+ """The loop's meta.md text and its validated manifest, from its capture `rev` or, without a record, the checkout."""
431
+ text, meta = package_text(anchor, rev, f"docs/specs/{loop_id}/meta.md")
432
+ require_sections(sections(text, meta), META_SECTIONS, meta)
433
+ fences = JSON_FENCE_RE.findall(text)
434
+ require(len(fences) == 1, f"{meta}: needs exactly one fenced json manifest, found {len(fences)}")
435
+ try:
436
+ manifest = shared("expected-contract")["loads_strict_json"](fences[0])
437
+ except ValueError as exc:
438
+ raise LoopError(f"{meta}: manifest is not strict JSON: {exc}") from exc
439
+ check_manifest(anchor, manifest, loop_id, meta)
440
+ return text, manifest
441
+
442
+
443
+ def load_package(anchor, loop_id, rev=None):
444
+ """The manifest and each task's contract, or under `invalid` the task's validation error."""
445
+ text, manifest = load_manifest(anchor, loop_id, rev)
446
+ tasks, invalid = {}, {}
447
+ for entry in manifest["tasks"]:
448
+ try:
449
+ tasks[entry["id"]] = load_task(anchor, manifest, entry, rev)
450
+ except LoopError as exc:
451
+ invalid[entry["id"]] = str(exc)
452
+ return {"loop_id": loop_id, "meta_text": text, "manifest": manifest, "tasks": tasks, "invalid": invalid}
453
+
454
+
455
+ def package_of(anchor, meta):
456
+ meta = Path(meta).resolve()
457
+ loop_id = meta.parent.name
458
+ require(meta.name == "meta.md" and meta.parent.parent == anchor / "docs/specs", f"package must be <repo>/docs/specs/<loop-id>/meta.md: {meta}")
459
+ package = load_package(anchor, loop_id)
460
+ if package["invalid"]:
461
+ raise LoopError(next(iter(package["invalid"].values())))
462
+ return package
463
+
464
+
465
+ def escape(title):
466
+ return title.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]")
467
+
468
+
469
+ def row_line(identity, title):
470
+ loop, task = identity.split(".")
471
+ return f"- [ ] {identity} [{escape(title)}](docs/specs/{loop}/{task}/spec.md)"
472
+
473
+
474
+ def parse_queue(data, label=QUEUE):
475
+ try:
476
+ lines = data.decode("utf-8").split("\n")
477
+ except UnicodeError as exc:
478
+ raise LoopError(f"{label} is not UTF-8: {exc}") from exc
479
+ rows = []
480
+ for index, raw in enumerate(lines):
481
+ match = ROW_RE.fullmatch(raw.removesuffix("\r"))
482
+ if not match:
483
+ continue
484
+ row = {"index": index, "line": match.group(0), "mark": match["mark"], "text": match["text"].rstrip(), "identity": None}
485
+ # Only a row matching the full loop-row grammar is a loop row; any other row is a legacy row.
486
+ item = ITEM_RE.fullmatch(row["text"])
487
+ if item and bool(item["rest"]) == (match["mark"] == "F"):
488
+ row.update(identity=item["identity"], loop=item["loop"], task=item["task"], rest=item["rest"])
489
+ rows.append(row)
490
+ identities = [row["identity"] for row in rows if row["identity"]]
491
+ duplicates = sorted({identity for identity in identities if identities.count(identity) > 1})
492
+ require(not duplicates, f"{label}: duplicate queue identity: {', '.join(duplicates)}")
493
+ return rows
494
+
495
+
496
+ def with_rows(data, lines):
497
+ """`data` with each identity's row replaced by `lines[identity]`, every other byte unchanged."""
498
+ out = data.split(b"\n")
499
+ for row in parse_queue(data):
500
+ if row["identity"] in lines:
501
+ ending = b"\r" if out[row["index"]].endswith(b"\r") else b""
502
+ out[row["index"]] = lines[row["identity"]].encode("utf-8") + ending
503
+ return b"\n".join(out)
504
+
505
+
506
+ def transition(before, identity, mark, suffix=""):
507
+ """The only legal transition: the identity's pending row becomes [x] or [F] with every other byte unchanged."""
508
+ row = next((row for row in parse_queue(before) if row["identity"] == identity), None)
509
+ require(row is not None and row["mark"] == " ", f"{identity} has no pending queue row to transition")
510
+ return with_rows(before, {identity: f"- [{mark}] {row['text']}{suffix}"})
511
+
512
+
513
+ def loop_queue(loop):
514
+ return f"docs/specs/{loop}/queue.md"
515
+
516
+
517
+ def loop_rows(loop, package, target):
518
+ """The queue file add generates for a loop: the legacy row it replaces, if any, then a row per task in manifest
519
+ order, each line one blank line from the next (package-format.md, Queue rows)."""
520
+ lines = [f"<!-- replaces {QUEUE} row (occurrence {target['occurrence']}): {target['line']} -->"] if target else []
521
+ lines += [row_line(f"{loop}.{task['id']}", task["title"]) for task in package["tasks"].values()]
522
+ return ("\n\n".join(lines) + "\n").encode("utf-8")
523
+
524
+
525
+ def parse_loop(data, loop, label):
526
+ """A loop's queue file: its rows, each a task row of the loop, and the legacy row it replaces as (occurrence, line)."""
527
+ require(data is not None, f"{label}: no such file")
528
+ rows = parse_queue(data, label)
529
+ require(rows and all(row["identity"] and row["loop"] == loop for row in rows), f"{label}: every row must be a task row of loop {loop}")
530
+ targets = [(int(m[1]), m[2]) for line in data.decode("utf-8").split("\n") if (m := REPLACES_RE.fullmatch(line.removesuffix("\r")))]
531
+ require(len(targets) <= 1, f"{label}: replaces more than one legacy row")
532
+ return rows, targets[0] if targets else None
533
+
534
+
535
+ def one_line(text):
536
+ text = " ".join(text.split())
537
+ return text if len(text) <= 200 else text[:199] + "…"
538
+
539
+
540
+ def branch_of(identity):
541
+ return "devlyn/" + identity.replace(".", "/")
542
+
543
+
544
+ def receipt_path(common, identity):
545
+ return common / "devlyn-completion" / hashlib.sha256(branch_of(identity).encode()).hexdigest()[:24] / "receipt.json"
546
+
547
+
548
+ def added_path(common, loop):
549
+ return common / "devlyn-loops" / loop / "added.json"
550
+
551
+
552
+ def records(common):
553
+ """Each loop's add record, in add order."""
554
+ found = []
555
+ for path in (common / "devlyn-loops").glob("*/added.json"):
556
+ added = read_json(path)
557
+ require(isinstance(added, dict) and added.get("loop_id") == path.parent.name and type(added.get("sequence")) is int
558
+ and all(isinstance(added.get(key), str) for key in ("delivery", "commit", "origin", "branch")), f"{path}: malformed add record")
559
+ found.append(added)
560
+ return {added["loop_id"]: added for added in sorted(found, key=lambda added: added["sequence"])}
561
+
562
+
563
+ def queue_view(anchor, recorded):
564
+ """The rows status and drain read (loop.md step 1): the legacy queue's raw rows in file order, a row that one loop
565
+ replaces shown as that loop's rows in its place; then the other recorded loops in add order; then the loops without a
566
+ record, as in a fresh clone, from the checkout's tracked queue files in loop-id order. A claim matching no row replaces
567
+ nothing; a row two loops claim is hidden, neither loop taking its place."""
568
+ legacy = anchor / QUEUE
569
+ raw = [row for row in parse_queue(legacy.read_bytes() if legacy.exists() else b"") if not row["identity"]]
570
+ loops = {loop: parse_loop(show(anchor, added["commit"], loop_queue(loop)), loop, f"{CAPTURES}{loop}:{loop_queue(loop)}")
571
+ for loop, added in recorded.items()}
572
+ tracked = set(filter(None, git(anchor, "ls-files", "-z", "--", ":(glob)docs/specs/*/queue.md").split("\0")))
573
+ head = ref_value(anchor, "HEAD")
574
+ if head:
575
+ tracked.update(rel for rel in git(anchor, "ls-tree", "-r", "-z", "--name-only", head, "--", "docs/specs").split("\0")
576
+ if re.fullmatch(r"docs/specs/[^/]+/queue\.md", rel))
577
+ for rel in sorted(tracked, key=lambda rel: rel.split("/")[2]):
578
+ loop, data, read_error = rel.split("/")[2], None, None
579
+ if loop in recorded:
580
+ continue
581
+ try:
582
+ data = (anchor / rel).read_bytes() if (anchor / rel).is_file() else None
583
+ except OSError as exc:
584
+ read_error = exc
585
+ # A directory name alone proves nothing. A task row in the checkout or HEAD proves this is a loop queue; HEAD
586
+ # keeps that proof when the checkout loses its rows or cannot be read. Only a proven queue is parsed or blocks.
587
+ known = False
588
+ for candidate in (data, show(anchor, head, rel) if head else None):
589
+ for line in (candidate or b"").decode("utf-8", errors="replace").splitlines():
590
+ match = ROW_RE.fullmatch(line)
591
+ item = ITEM_RE.fullmatch(match["text"].rstrip()) if match else None
592
+ if item and item["loop"] == loop and bool(item["rest"]) == (match["mark"] == "F"):
593
+ known = True
594
+ if known:
595
+ if read_error is not None:
596
+ raise LoopError(f"{anchor / rel}: {read_error}") from read_error
597
+ loops[loop] = parse_loop(data, loop, anchor / rel)
598
+ claims = {}
599
+ for loop, (_, target) in loops.items():
600
+ if target:
601
+ claims.setdefault(target, []).append(loop)
602
+ placed = {}
603
+ for (occurrence, line), owners in claims.items():
604
+ matches = [row["index"] for row in raw if row["line"] == line]
605
+ if len(matches) >= occurrence:
606
+ placed[matches[occurrence - 1]] = owners[0] if len(owners) == 1 else None
607
+ rows = []
608
+ for row in raw:
609
+ rows += [row] if row["index"] not in placed else loops[placed[row["index"]]][0] if placed[row["index"]] else []
610
+ return rows + [row for loop, (own, _) in loops.items() if loop not in placed.values() for row in own]
611
+
612
+
613
+ def package_state(cwd, commit, loop):
614
+ """Each package file of the capture `commit` as the checkout `cwd` holds it: missing, equal (line endings aside) or
615
+ different."""
616
+ states = {}
617
+ for rel in filter(None, git(cwd, "ls-tree", "-r", "-z", "--name-only", commit).split("\0")):
618
+ if rel != loop_queue(loop):
619
+ data = (cwd / rel).read_bytes() if (cwd / rel).is_file() else None
620
+ states[rel] = ("missing" if data is None else "equal" if data.replace(b"\r\n", b"\n") == show(cwd, commit, rel).replace(b"\r\n", b"\n")
621
+ else "different")
622
+ return states
623
+
624
+
625
+ def clean(common, recorded):
626
+ """Remove each recorded loop's untracked package originals that equal its capture from the checkout that added it,
627
+ keeping and naming any other copy; returns what remains to report (package-format.md, Commands)."""
628
+ notes = []
629
+ for origin in dict.fromkeys(added["origin"] for added in recorded.values()):
630
+ root = Path(origin)
631
+ try:
632
+ if not (root / "docs/specs").is_dir():
633
+ continue
634
+ untracked = set(filter(None, git(root, "ls-files", "--others", "-z", "--", "docs/specs").split("\0")))
635
+ except (LoopError, OSError) as exc:
636
+ notes.append(f"package cleanup retained: {root}: {exc}")
637
+ continue
638
+ for loop, added in recorded.items():
639
+ package = f"docs/specs/{loop}/"
640
+ if added["origin"] != origin or not any(rel.startswith(package) for rel in untracked):
641
+ continue
642
+ try:
643
+ states = package_state(root, added["commit"], loop)
644
+ except (LoopError, OSError) as exc:
645
+ notes.append(f"package cleanup retained: {root / package}: {exc}")
646
+ continue
647
+ for rel, state in states.items():
648
+ if rel in untracked and state == "different":
649
+ notes.append(f"{root / rel}: kept, it differs from the captured package {CAPTURES}{loop}")
650
+ elif rel in untracked and state == "equal":
651
+ try:
652
+ (root / rel).unlink()
653
+ except OSError as exc:
654
+ notes.append(f"package cleanup retained: {root / rel}: {exc}")
655
+ for directory, _, _ in os.walk(root / package, topdown=False):
656
+ try:
657
+ os.rmdir(directory)
658
+ except OSError as exc:
659
+ if exc.errno not in {errno.ENOTEMPTY, errno.EEXIST, errno.ENOENT}:
660
+ notes.append(f"package cleanup retained: {directory}: {exc}")
661
+ return notes
662
+
663
+
664
+ def task_complete(action, *, exclude_receipts=(), **values):
665
+ helper = shared("task-complete")
666
+ try:
667
+ # Preserve the previous-receipt sweep, excluding only queue-invalid owners.
668
+ options = {"exclude_receipts": exclude_receipts} if action == "allocate" else {}
669
+ return helper[action](argparse.Namespace(**values), **options)
670
+ except (helper["CompletionError"], OSError, KeyError, TypeError, ValueError) as exc:
671
+ # Inherited helper containment includes these three programming-error types; the controller boundary is narrower.
672
+ raise LoopError(f"task-complete {action}: {exc}") from exc
673
+
674
+
675
+ def derive_state(anchor, common, row, claims, rev):
676
+ identity = row["identity"]
677
+ state = {"kind": {" ": "pending", "x": "accepted", "F": "failed"}[row["mark"]], "receipt": None, "path": None}
678
+ require(len(claims) <= 1, f"conflicting receipts for {identity}: {', '.join(str(path) for path, _ in claims)}")
679
+ if not claims:
680
+ path = receipt_path(common, identity)
681
+ if path.parent.exists() or path.parent.is_symlink():
682
+ state.update(kind="pending", barrier=True,
683
+ invalid=f"receipt missing at {path}; allocation ownership cannot be verified; inspect {path.parent} and any "
684
+ f"{branch_of(identity)} branch/worktree, remove only confirmed interrupted-allocation artifacts, then drain again")
685
+ return state
686
+ path, receipt = claims[0]
687
+ require(isinstance(receipt, dict), f"receipt {path} is not an object")
688
+ require(receipt.get("task") == identity and receipt.get("branch") == branch_of(identity) and path == receipt_path(common, identity)
689
+ and path == path.resolve(),
690
+ f"receipt {path} claims {identity} from branch {receipt.get('branch')!r}; conflicting ownership, inspect it")
691
+ require(receipt.get("allocation") in ("owned", "allocating"),
692
+ f"receipt {path}: missing or malformed allocation checkpoint")
693
+ if receipt["allocation"] == "allocating":
694
+ # Never adopted, and no executor ran: the task waits until the user removes what the allocation left.
695
+ state["kind"] = "pending"
696
+ state["invalid"] = (f"allocation did not finish ({path}); remove its worktree {receipt.get('worktree')} and branch "
697
+ f"{receipt['branch']} if present, delete {path.parent}, then drain again")
698
+ return state
699
+ require(all(isinstance(receipt.get(key), str) and receipt[key] and "\0" not in receipt[key] for key in
700
+ ("id", "baseline", "worktree", "common_gitdir", "recovery_ref")), f"receipt {path}: malformed allocation fields")
701
+ require(receipt["common_gitdir"] == str(common) and receipt["id"] == path.parent.name,
702
+ f"receipt {path}: conflicting common Gitdir/id binding")
703
+ require(receipt["recovery_ref"] == "refs/devlyn/completed/" + receipt["id"], f"receipt {path}: conflicting recovery ref ownership")
704
+ for key in ("delivery", "product", "pr_url", "base", "remote"):
705
+ require(key not in receipt or receipt[key] is None or isinstance(receipt[key], str), f"receipt {path}: malformed {key}")
706
+ for key in ("acceptance", "queue", "merge", "workspace_cleanup", "scratch_cleanup"):
707
+ require(key not in receipt or isinstance(receipt[key], dict), f"receipt {path}: malformed {key}")
708
+ if receipt.get("merge"):
709
+ merge = receipt["merge"].get("mergeCommit")
710
+ require(isinstance(merge, dict) and isinstance(merge.get("oid"), str) and SHA_RE.fullmatch(merge["oid"]),
711
+ f"receipt {path}: malformed merge binding")
712
+ for key in ("workspace_cleanup", "scratch_cleanup"):
713
+ if value := receipt.get(key):
714
+ require(isinstance(value.get("status"), str) and (key != "workspace_cleanup" and not value.get("reason") or
715
+ isinstance(value.get("reason"), str) and isinstance(value.get("resume"), str)), f"receipt {path}: malformed {key}")
716
+ require(not receipt.get("acceptance") or isinstance(receipt["acceptance"], dict)
717
+ and all(isinstance(receipt.get(key), str) and SHA_RE.fullmatch(receipt[key]) for key in ("source_sha", "publish_sha"))
718
+ and isinstance(receipt["acceptance"].get("inputs_sha"), str)
719
+ and isinstance(receipt["acceptance"].get("reasons"), list)
720
+ and all(isinstance(reason, str) for reason in receipt["acceptance"]["reasons"]), f"receipt {path}: malformed acceptance binding")
721
+ require(not receipt.get("queue") or isinstance(receipt["queue"], dict)
722
+ and all(isinstance(receipt["queue"].get(key), str) and "\0" not in receipt["queue"][key] for key in ("file", "commit")),
723
+ f"receipt {path}: malformed terminal binding")
724
+ if receipt.get("acceptance"):
725
+ verdict = receipt["acceptance"].get("verdict")
726
+ require(verdict in ("ACCEPTED", "FAILED"), f"receipt {path}: malformed acceptance verdict")
727
+ require(receipt.get("product") is None or receipt["product"] == verdict,
728
+ f"receipt {path}: inconsistent acceptance verdict/product")
729
+ require(isinstance(receipt.get("files"), dict) and receipt["files"], f"receipt {path}: malformed bound files")
730
+ for key in ("assumptions", "reasons"):
731
+ value = receipt["acceptance"].get(key, [])
732
+ require(isinstance(value, list) and all(isinstance(item, str) for item in value), f"receipt {path}: malformed acceptance {key}")
733
+ result = None if not receipt.get("acceptance") else "failed" if receipt["acceptance"]["verdict"] == "FAILED" else "accepted"
734
+ require(row["mark"] == " " or result == state["kind"],
735
+ f"conflicting terminal state for {identity}: queue row [{row['mark']}] but receipt {path} is {result or 'unbound'}")
736
+ if result:
737
+ recovery = ref_value(anchor, receipt["recovery_ref"])
738
+ # Or an attachment interrupted after moving the recovery ref, before saving the receipt; drain completes it.
739
+ require(recovery == receipt["publish_sha"] or not receipt.get("queue") and bool(recovery)
740
+ and is_terminal(anchor, receipt, identity, recovery),
741
+ f"{identity}: recovery ref {receipt['recovery_ref']} is {recovery or 'missing'}, receipt binds {receipt['publish_sha']}")
742
+ if receipt.get("queue"):
743
+ marks = [r["mark"] for r in parse_queue(show(anchor, receipt["publish_sha"], receipt["queue"]["file"]) or b"") if r["identity"] == identity]
744
+ require(marks == ["x" if result == "accepted" else "F"], f"{identity}: terminal commit {receipt['publish_sha']} does not carry its {result} mark")
745
+ require(result != "failed" or receipt["acceptance"]["reasons"], f"receipt {path}: failed acceptance has no reason")
746
+ packet = path.parent / "packet.json"
747
+ if packet.exists():
748
+ bound = read_json(packet)
749
+ require(isinstance(bound, dict) and not acceptance()["PACKET_KEYS"] - bound.keys(), f"{packet}: malformed task packet")
750
+ harness_call("packet_schema", bound)
751
+ require(bound["worktree"] == receipt["worktree"] and bound["task"] == identity and bound["branch"] == receipt["branch"]
752
+ and bound["allocation_base"] == receipt["baseline"],
753
+ f"{packet}: conflicting packet ownership")
754
+ for key, expected in (("receipt", path), ("scratch", path.parent / "scratch"), ("submission", path.parent / "submission.json"),
755
+ ("evidence_dir", Path(receipt["worktree"]) / ".devlyn/loop")):
756
+ require(bound.get(key) == str(expected), f"{packet}: malformed or conflicting {key} path")
757
+ require(isinstance(bound["inputs_sha"], str) and SHA_RE.fullmatch(bound["inputs_sha"]), f"{packet}: malformed inputs commit")
758
+ require(bound.get("delivery") in ("local-only", "auto", "pr"), f"{packet}: malformed delivery")
759
+ for key in ("requirements", "review_requirements"):
760
+ require(isinstance(bound[key], list) and all(isinstance(item, str) for item in bound[key]), f"{packet}: malformed {key}")
761
+ for key in ("contract", "expected", "meta"):
762
+ entry = bound[key]
763
+ require(isinstance(entry, dict) and isinstance(entry.get("path"), str) and isinstance(entry.get("sha256"), str)
764
+ and re.fullmatch(r"[0-9a-f]{64}", entry["sha256"]), f"{packet}: malformed {key} binding")
765
+ candidate = Path(entry["path"])
766
+ require("\0" not in entry["path"] and candidate.is_absolute() and candidate.is_relative_to(Path(bound["worktree"]))
767
+ and ".." not in candidate.parts,
768
+ f"{packet}: {key} path escapes its worktree: {candidate}")
769
+ rel = candidate.relative_to(bound["worktree"]).as_posix()
770
+ current = package_bytes(anchor, rev, rel) if not result else None
771
+ if not result and key != "meta" and (current is None or hashlib.sha256(current).hexdigest() != bound[key]["sha256"]):
772
+ # Drain fails this task alone; the revision is planned as a new task.
773
+ state["inputs_changed"] = f"inputs-changed: {rel} changed after its inputs were committed; plan the revision as a new task"
774
+ state.update(kind=result or "active", path=path, receipt=receipt)
775
+ return state
776
+
777
+
778
+ def view(anchor, common, local_only=False, *, pure=False):
779
+ """Reconcile the queue with every receipt and recovery ref before anything is selected."""
780
+ with contextlib.nullcontext() if pure else lock(common, "queue.lock", blocking=True):
781
+ if not pure:
782
+ enforce_binding(common)
783
+ recorded = records(common)
784
+ cleanup = [] if pure else clean(common, recorded)
785
+ rows = queue_view(anchor, recorded)
786
+ claims, unreadable, damaged = {}, [], {}
787
+ canonical = {receipt_path(common, row["identity"]): row["identity"] for row in rows if row["identity"]}
788
+ branches = {branch_of(identity): identity for identity in canonical.values()}
789
+ for path in sorted((common / "devlyn-completion").glob("*/receipt.json")):
790
+ try:
791
+ receipt = read_json(path)
792
+ require(isinstance(receipt, dict), f"{path}: malformed receipt identity")
793
+ task, branch = receipt.get("task"), receipt.get("branch")
794
+ implicated = {task if isinstance(task, str) else None, canonical.get(path),
795
+ branches.get(branch) if isinstance(branch, str) else None} - {None}
796
+ require(implicated, f"{path}: malformed receipt identity")
797
+ for identity in implicated:
798
+ claims.setdefault(identity, []).append((path, receipt))
799
+ except (LoopError, OSError) as exc:
800
+ unreadable.append(f"{path}: {exc}")
801
+ if identity := canonical.get(path):
802
+ damaged[identity] = str(exc)
803
+ v = {"anchor": anchor, "common": common, "rows": rows, "records": recorded, "packages": {}, "manifests": {}, "states": {},
804
+ "unreadable": unreadable, "cleanup": cleanup, "local_only": local_only, "pure": pure}
805
+ errors = {}
806
+ for row in v["rows"]:
807
+ if row["identity"]:
808
+ identity = row["identity"]
809
+ try:
810
+ require(identity not in damaged, damaged.get(identity))
811
+ state = derive_state(anchor, common, row, claims.get(identity, []), capture(v, row["loop"]))
812
+ except (LoopError, OSError) as exc:
813
+ # No acceptance/frontier may be inferred from conflicting evidence. Its possible writer or carrier
814
+ # still holds the loop barrier, even when no trustworthy receipt can be retained in the view.
815
+ state = {"kind": "pending", "receipt": None, "path": None, "barrier": True,
816
+ "invalid": f"recovery conflict: {exc}; inspect and restore {identity}'s verified state, then drain again"}
817
+ v["states"][identity] = state
818
+ # A changed active task needs no current package text; a validation failure stops only its own task.
819
+ if "invalid" not in state and (state["kind"] == "pending" or state["kind"] == "active" and "inputs_changed" not in state):
820
+ loop = row["loop"]
821
+ if loop not in v["packages"] and loop not in errors:
822
+ try:
823
+ v["packages"][loop] = load_package(anchor, loop, capture(v, loop))
824
+ except (LoopError, OSError) as exc:
825
+ errors[loop] = str(exc)
826
+ package = v["packages"].get(loop)
827
+ if problem := errors.get(loop) or package["invalid"].get(row["task"]) or (
828
+ None if row["task"] in package["tasks"] else f"{row['identity']} is not a task of loop {loop}"):
829
+ state["invalid"] = problem
830
+ for row in v["rows"]:
831
+ state = v["states"].get(row["identity"])
832
+ for dep in v["packages"][row["loop"]]["tasks"][row["task"]]["depends_on"] if state and state["kind"] == "pending" and "invalid" not in state else []:
833
+ dep_state = v["states"].get(f"{row['loop']}.{dep}")
834
+ if dep_state is None:
835
+ state["invalid"] = (f"{row['identity']} depends on {row['loop']}.{dep}, which is not queued; restore the authoritative "
836
+ "queue from verified state or plan the continuation as a new loop, then drain again")
837
+ break
838
+ if dep_state["kind"] in {"failed", "blocked"}:
839
+ state.update(kind="blocked", blocker=f"{row['loop']}.{dep}", root=dep_state.get("root") or dep_state)
840
+ break
841
+ v["invalid_receipts"] = {path for identity, state in v["states"].items() if "invalid" in state
842
+ for path in (receipt_path(common, identity), *(path for path, _ in claims.get(identity, [])))}
843
+ return v
844
+
845
+
846
+ def capture(v, loop):
847
+ """The commit holding the loop's captured package, or None for a loop with no record (its checkout's files)."""
848
+ return v["records"][loop]["commit"] if loop in v["records"] else None
849
+
850
+
851
+ def published(v, loop):
852
+ """Why --local-only leaves an auto/pr loop to a normal drain: a task of it already has a pushed PR (loop.md, Commands)."""
853
+ for identity, state in v["states"].items():
854
+ if identity.split(".")[0] == loop and state["receipt"] and state["receipt"].get("pushed"):
855
+ url = state["receipt"].get("pr_url") or state["receipt"]["branch"]
856
+ return f"{identity} already has a pushed PR {url}, so --local-only never runs loop {loop}; drain without --local-only"
857
+ return None
858
+
859
+
860
+ def is_local(v, loop):
861
+ added, package = v["records"].get(loop), v["packages"].get(loop)
862
+ delivery = added["delivery"] if added else package and package["manifest"]["delivery"]
863
+ return (v["local_only"] and not published(v, loop) or delivery == "local-only"
864
+ or any(s["receipt"] and s["receipt"].get("local_only") for i, s in v["states"].items() if i.split(".")[0] == loop))
865
+
866
+
867
+ def delivered(receipt):
868
+ return receipt.get("delivery") == "COMPLETE" or bool(receipt.get("merge"))
869
+
870
+
871
+ def carrier(v, loop):
872
+ """The auto/pr task whose undelivered, unfailed PR carries the loop's package, which its base lacked (loop.md step 2)."""
873
+ return None if is_local(v, loop) else next((
874
+ row["identity"] for row in v["rows"] if row.get("loop") == loop and (state := v["states"][row["identity"]])["receipt"]
875
+ and state["kind"] != "failed" and not delivered(state["receipt"]) and show(v["anchor"], state["receipt"]["baseline"], loop_queue(loop)) is None), None)
876
+
877
+
878
+ def waiting(v, row):
879
+ # Eligibility precedes selection, so its task-local reads cannot reach advance's exception boundary.
880
+ try:
881
+ return wait_reason(v, row)
882
+ except (LoopError, OSError) as exc:
883
+ return f"allocation blocked: {exc}; inspect and repair {row['identity']}'s starting state, then drain again"
884
+
885
+
886
+ def wait_reason(v, row):
887
+ if invalid := v["states"][row["identity"]].get("invalid"):
888
+ return invalid
889
+ if barrier := recovery_barrier(v, row):
890
+ return barrier
891
+ local = is_local(v, row["loop"])
892
+ if v["local_only"] and not local:
893
+ return published(v, row["loop"])
894
+ for dep in v["packages"][row["loop"]]["tasks"][row["task"]]["depends_on"]:
895
+ identity = f"{row['loop']}.{dep}"
896
+ state = v["states"][identity]
897
+ if state["kind"] != "accepted":
898
+ return f"waiting for {identity} ({state['kind']})"
899
+ if not state["receipt"]:
900
+ return f"prerequisite {identity} has no receipt-bound accepted source"
901
+ if not local and not delivered(state["receipt"]):
902
+ return f"awaiting delivery of {identity}"
903
+ if local:
904
+ return local_start(v, row["loop"])[1]
905
+ if plan := carrier(v, row["loop"]):
906
+ return f"awaiting delivery of {plan}, whose PR carries its loop's package"
907
+ return None
908
+
909
+
910
+ def recovery_barrier(v, row):
911
+ return next((f"waiting for {identity} (recovery conflict), since its possible writer or package carrier holds this loop"
912
+ for identity, state in v["states"].items() if identity != row["identity"]
913
+ and identity.split(".")[0] == row["loop"] and state.get("barrier")), None)
914
+
915
+
916
+ def next_task(v, attempted):
917
+ """Earliest unsettled receipt first, then the earliest eligible pending row in physical order."""
918
+ for row in v["rows"]:
919
+ state = v["states"].get(row["identity"])
920
+ if state and state["receipt"] and row["identity"] not in attempted and "invalid" not in state and not recovery_barrier(v, row) and (
921
+ state["kind"] == "active" or not state["receipt"].get("queue") or state["receipt"].get("delivery") not in SETTLED):
922
+ return row
923
+ return next((row for row in v["rows"] if row["identity"] not in attempted and v["states"].get(row["identity"], {}).get("kind") == "pending"
924
+ and waiting(v, row) is None), None)
925
+
926
+
927
+ def annotate(v, refused):
928
+ """Name each task's wait: a pending task's reason from waiting(), else why this drain did not advance it."""
929
+ for row in v["rows"]:
930
+ state = v["states"].get(row["identity"])
931
+ if state and (reason := refused.get(row["identity"]) or state["kind"] == "pending" and waiting(v, row)
932
+ or recovery_barrier(v, row)):
933
+ state["waiting"] = reason
934
+
935
+
936
+ def frontier(v, loop):
937
+ """The loop's accepted task whose source contains every other accepted source. A local loop builds one history, though
938
+ not always in row order: a task whose unfinished allocation was removed starts again from a later row's source. Without
939
+ one, as after a fork by an older build, it is the last accepted row, and allocate's prerequisite check names the fork."""
940
+ accepted = [v["states"][row["identity"]] for row in v["rows"] if row.get("loop") == loop
941
+ and v["states"][row["identity"]]["kind"] == "accepted" and v["states"][row["identity"]]["receipt"]]
942
+ return next((tip for tip in reversed(accepted) if all(ancestor(v["anchor"], state["receipt"]["source_sha"], tip["receipt"]["source_sha"])
943
+ for state in accepted)), accepted[-1] if accepted else None)
944
+
945
+
946
+ def local_start(v, loop):
947
+ """The commit a local loop's next task starts from, and why it waits instead (loop.md steps 2-3): none while a task of
948
+ the loop is active, so no two tasks start from one frontier; else its latest accepted source; else the start its
949
+ first allocation fixed, kept in that receipt; else the HEAD of the branch it was added on, which must descend from
950
+ the manifest base_sha."""
951
+ if active := next((identity for identity, state in v["states"].items() if identity.split(".")[0] == loop and state["kind"] == "active"), None):
952
+ return None, f"waiting for {active} (active), since a local loop runs one task at a time"
953
+ tip, added = frontier(v, loop), v["records"].get(loop)
954
+ fixed = next((state["receipt"]["baseline"] for identity, state in v["states"].items() if identity.split(".")[0] == loop
955
+ and state["receipt"] and not {"remote_url", "allocated_from"} & set(state["receipt"])), None)
956
+ remedy = "restore them to their bytes at that commit, or plan the remaining work as a new loop"
957
+ if tip or fixed:
958
+ start = tip["receipt"]["source_sha"] if tip else fixed
959
+ elif added is None:
960
+ return None, f"{loop} has no add record in this repository, so no branch to start it from; drain it where it was added"
961
+ else:
962
+ branch, base = added["branch"], manifest_of(v, loop)["base_sha"]
963
+ start, remedy = ref_value(v["anchor"], "refs/heads/" + branch), f"commit them on {branch}, or restore them to their bytes at that commit"
964
+ if not start or not ancestor(v["anchor"], base, start):
965
+ where = f"is at {start}, which does not descend from the manifest base_sha {base}" if start else "no longer exists"
966
+ return None, f"{branch}, the branch {loop} was added on, {where}; bring {base} into {branch}, or plan the work as a new loop"
967
+ if drift := package_drift(v, loop, start, branch):
968
+ return start, drift
969
+ if drift := instruction_drift(v["anchor"], start, pure=v.get("pure", False)):
970
+ return start, f"the installed instructions ({drift}) in this checkout differ from those in its start commit {start}, so it would run without them; {remedy}"
971
+ return start, None
972
+
973
+
974
+ def terminal_line(v, identity):
975
+ state = v["states"][identity]
976
+ row = next(row for row in v["rows"] if row["identity"] == identity)
977
+ if state["kind"] == "blocked":
978
+ root = state["root"]["receipt"]
979
+ return f"- [F] {row['text']} — blocked-prerequisite:{state['blocker']} (receipt {root['id'] if root else 'none'})"
980
+ if state["kind"] in {"accepted", "failed"} and state["receipt"] and state["receipt"].get("queue"):
981
+ terminal = parse_queue(show(v["anchor"], state["receipt"]["publish_sha"], state["receipt"]["queue"]["file"]) or b"")
982
+ return next(r["line"] for r in terminal if r["identity"] == identity)
983
+ return None
984
+
985
+
986
+ def build_tree(worktree, parent, files):
987
+ """The tree of `parent` (or of nothing) with `files`, built in a temporary index so no checkout changes."""
988
+ with tempfile.TemporaryDirectory(prefix="devlyn-loop-index-") as temp:
989
+ env = {**os.environ, "GIT_INDEX_FILE": str(Path(temp) / "index")}
990
+ git(worktree, "read-tree", parent or "--empty", env=env)
991
+ for rel, data in files.items():
992
+ blob = git(worktree, "hash-object", "-w", "--stdin", data=data)
993
+ git(worktree, "update-index", "--add", "--cacheinfo", f"100644,{blob},{rel}", env=env)
994
+ return git(worktree, "write-tree", env=env)
995
+
996
+
997
+ def commit_files(worktree, branch, parent, files, message, *, checkout):
998
+ """Commit `files` atop `parent` on `branch`; when `parent` already carries them there is no commit to make."""
999
+ tree = build_tree(worktree, parent, files)
1000
+ if tree == git(worktree, "rev-parse", parent + "^{tree}"):
1001
+ return parent
1002
+ commit = git(worktree, "commit-tree", tree, "-p", parent, "-m", message)
1003
+ git(worktree, "update-ref", "refs/heads/" + branch, commit, parent)
1004
+ if checkout:
1005
+ sync(worktree, branch, parent, commit)
1006
+ return commit
1007
+
1008
+
1009
+ def sync(worktree, branch, parent, commit):
1010
+ """Move a checkout that was clean at `parent` to `commit`; refuses rather than overwrite local edits."""
1011
+ if git_run(worktree, "symbolic-ref", "-q", "HEAD", ok=(0, 1)).stdout.decode("utf-8").strip() == "refs/heads/" + branch:
1012
+ git(worktree, "read-tree", "-m", "-u", parent, commit)
1013
+
1014
+
1015
+ def input_files(v, row, receipt, local):
1016
+ """The task's scoped inputs (loop.md step 4). A base lacking the loop's package gets the package directory from the
1017
+ capture, its queue file's settled rows keeping their marks; otherwise a local task gets the base's queue file with the
1018
+ loop's settled rows, and an auto/pr task nothing, so its PR changes only its own row."""
1019
+ loop, rel = row["loop"], loop_queue(row["loop"])
1020
+ data, files = show(Path(receipt["worktree"]), receipt["baseline"], rel), {}
1021
+ if data is None:
1022
+ package = [f"docs/specs/{loop}/meta.md", *(f"docs/specs/{loop}/{entry['id']}/{name}" for entry in v["packages"][loop]["manifest"]["tasks"]
1023
+ for name in ("spec.md", "spec.expected.json")), rel]
1024
+ files = {path: package_bytes(v["anchor"], capture(v, loop), path) for path in package}
1025
+ require(None not in files.values(), f"{row['identity']}: the package of loop {loop} is incomplete in this checkout")
1026
+ data = files[rel]
1027
+ if local or files:
1028
+ data = with_rows(data, {other["identity"]: line for other in v["rows"] if other.get("loop") == loop
1029
+ and other["identity"] != row["identity"] and (line := terminal_line(v, other["identity"]))})
1030
+ files[rel] = data
1031
+ return files
1032
+
1033
+
1034
+ def require_merged(v, row, base, *, allocated=False):
1035
+ """auto/pr: every prerequisite's merge commit must be in the base, before allocation and before every execution."""
1036
+ for dep in v["packages"][row["loop"]]["tasks"][row["task"]]["depends_on"]:
1037
+ receipt = v["states"][f"{row['loop']}.{dep}"]["receipt"] or {}
1038
+ merge = ((receipt.get("merge") or {}).get("mergeCommit") or {}).get("oid")
1039
+ remedy = ("retain this allocation and plan the continuation as a new loop, or perform verified recovery of a never-executed allocation"
1040
+ if allocated else f"restore that merge on origin/{manifest_of(v, row['loop'])['base_ref']}, then drain again")
1041
+ require(merge and ancestor(v["anchor"], merge, base),
1042
+ f"{row['identity']}: {'allocated baseline' if allocated else 'base'} {base} lacks prerequisite {row['loop']}.{dep}'s "
1043
+ f"delivered merge {merge}; {remedy}")
1044
+
1045
+
1046
+ def package_drift(v, loop, start, where):
1047
+ """Why a task of a recorded loop cannot start at `start` (loop.md step 3): it holds the loop's queue file, so the task
1048
+ would bind its copy of the package, and that copy differs from the capture."""
1049
+ added = v["records"].get(loop)
1050
+ if added is None or show(v["anchor"], start, loop_queue(loop)) is None:
1051
+ return None
1052
+ files = [rel for rel in git(v["anchor"], "diff", "--name-only", "--no-renames", "--diff-filter=DMT", added["commit"], start, "--",
1053
+ f"docs/specs/{loop}").splitlines() if rel != loop_queue(loop)]
1054
+ return (f"{where} holds another version of loop {loop}'s package (differing from {CAPTURES}{loop}: {', '.join(files)}), so its tasks "
1055
+ f"cannot run their captured contracts there; plan the work as a new loop, or restore those files on {where} to the captured "
1056
+ "bytes") if files else None
1057
+
1058
+
1059
+ def instruction_drift(anchor, rev, *, pure=False):
1060
+ """The installed instruction files whose content at `rev` differs from the anchor checkout's (both absent is equal): a
1061
+ task starting at `rev` would run without them. Git compares them as `git status` does: a link by its target, line endings
1062
+ under the checkout's conversion rules, and an untracked or ignored file as one still to commit."""
1063
+ names = ("CLAUDE.md", "AGENTS.md")
1064
+ # Git's porcelain diff refreshes the index independently of GIT_OPTIONAL_LOCKS.
1065
+ options = ("-c", "diff.autoRefreshIndex=false") if pure else ()
1066
+ differ = git(anchor, *options, "diff", "--name-only", "--no-renames", rev, "--", *names).split("\n")
1067
+ differ += git(anchor, "ls-files", "--others", "--", *names).split("\n")
1068
+ return ", ".join(name for name in names if name in differ)
1069
+
1070
+
1071
+ def allocate(v, row, opts):
1072
+ """Allocate the task's owned worktree, or return why it waits: origin names no GitHub repository, its base is not the
1073
+ default, or its refreshed remote base holds another version of the loop's package or lacks the checkout's instructions
1074
+ (waiting() checks a local start before selection)."""
1075
+ identity, loop, task = row["identity"], row["loop"], row["task"]
1076
+ package = v["packages"][loop]
1077
+ manifest = package["manifest"]
1078
+ anchor, common = v["anchor"], v["common"]
1079
+ deps = [v["states"][f"{loop}.{dep}"] for dep in package["tasks"][task]["depends_on"]]
1080
+ values = {"repo": str(anchor), "task": identity, "branch": branch_of(identity), "repository": None,
1081
+ "base": manifest["base_ref"], "remote": "origin", "worktree": str(opts.worktree_root / loop / task),
1082
+ "local_base": None, "from_receipt": None, "start": None}
1083
+ if is_local(v, loop):
1084
+ tip, start = frontier(v, loop), local_start(v, loop)[0]
1085
+ for dep in deps:
1086
+ require(ancestor(anchor, dep["receipt"]["source_sha"], start),
1087
+ f"{identity}: prerequisite source {dep['receipt']['source_sha']} is not in the accepted frontier {start}; plan an integration task")
1088
+ values["from_receipt" if tip else "local_base"] = str(tip["path"]) if tip else start
1089
+ else:
1090
+ helper = shared("task-complete")
1091
+ origin = git_run(anchor, "config", "--get", "remote.origin.url", ok=(0, 1)).stdout.decode("utf-8").strip()
1092
+ try:
1093
+ values["repository"] = helper["repository_from_url"](origin)
1094
+ except helper["CompletionError"]:
1095
+ return (f"{manifest['delivery']} delivery needs an origin remote of the form https://github.com/<owner>/<repo>, "
1096
+ "git@github.com:<owner>/<repo> or ssh://git@github.com/<owner>/<repo>; set one, or drain with --local-only")
1097
+ try:
1098
+ default = helper["repository_info"](values["repository"])["defaultBranchRef"]["name"]
1099
+ except (helper["CompletionError"], OSError, KeyError, TypeError, ValueError) as exc:
1100
+ return f"cannot check delivery repository: {exc}"
1101
+ if manifest["base_ref"] != default:
1102
+ return (f"{manifest['delivery']} delivery base {manifest['base_ref']} is not the repository's default branch {default}; "
1103
+ f"plan the loop on {default}, or drain with --local-only")
1104
+ try:
1105
+ start = helper["remote_base"]({"common_gitdir": str(common), "remote": "origin", "base": manifest["base_ref"]})
1106
+ except (helper["CompletionError"], OSError) as exc:
1107
+ raise LoopError(f"cannot refresh origin/{manifest['base_ref']}: {exc}; restore remote access or the base ref, "
1108
+ "then drain again; the next drain retries") from exc
1109
+ values["start"] = start
1110
+ require_merged(v, row, start)
1111
+ if drift := package_drift(v, loop, start, f"origin/{manifest['base_ref']}"):
1112
+ return drift
1113
+ if values["start"] and (drift := instruction_drift(anchor, start)):
1114
+ return (f"the installed instructions ({drift}) in this checkout differ from those in its start commit {start}, so it would run "
1115
+ f"without them; commit them and push them to origin/{manifest['base_ref']}")
1116
+ if os.name == "nt" and opts.default_worktree_root:
1117
+ try:
1118
+ opts.worktree_root.mkdir(exist_ok=False)
1119
+ except FileExistsError:
1120
+ pass # An existing root belongs to its owner; never change its attributes.
1121
+ else:
1122
+ try:
1123
+ shared("platform-support")["mark_not_content_indexed"](opts.worktree_root)
1124
+ except OSError as exc:
1125
+ opts.root_diagnostics.append(
1126
+ f"content indexing unchanged: {opts.worktree_root}: could not set FILE_ATTRIBUTE_NOT_CONTENT_INDEXED: {exc}; continuing")
1127
+ result = task_complete("allocate", exclude_receipts=v["invalid_receipts"], **values)
1128
+ progress(identity, f"allocated {result['worktree']}")
1129
+ return None
1130
+
1131
+
1132
+ def ensure_packet(v, row, path, receipt):
1133
+ """Commit this task's scoped inputs on the owned branch, then write the executor's task packet."""
1134
+ packet_path = path.parent / "packet.json"
1135
+ if packet_path.exists():
1136
+ return packet_path, read_json(packet_path)
1137
+ identity, loop, task = row["identity"], row["loop"], row["task"]
1138
+ worktree, baseline, branch = Path(receipt["worktree"]), receipt["baseline"], receipt["branch"]
1139
+ local = bool(receipt.get("local_only"))
1140
+ files = input_files(v, row, receipt, local)
1141
+ head = git(worktree, "rev-parse", "refs/heads/" + branch)
1142
+ if head == baseline:
1143
+ inputs = commit_files(worktree, branch, baseline, files, f"devlyn loop: inputs for {identity}", checkout=True)
1144
+ else:
1145
+ require(git(worktree, "rev-parse", head + "^") == baseline and git(worktree, "rev-parse", head + "^{tree}") == build_tree(worktree, baseline, files),
1146
+ f"{identity}: owned branch has commits that are not its scoped inputs; inspect {worktree}")
1147
+ inputs = head
1148
+ sync(worktree, branch, baseline, inputs)
1149
+ progress(identity, f"inputs {inputs}")
1150
+ package = v["packages"][loop]
1151
+ spec = package["tasks"][task]
1152
+
1153
+ def identity_of(rel):
1154
+ return {"path": str(worktree / rel), "sha256": hashlib.sha256(show(worktree, inputs, rel)).hexdigest()}
1155
+
1156
+ packet = {
1157
+ "schema_version": 2, "task": identity, "title": spec["title"], "kind": spec["kind"],
1158
+ "worktree": str(worktree), "branch": branch, "receipt": str(path), "scratch": str(path.parent / "scratch"),
1159
+ "allocation_base": baseline, "inputs_sha": inputs,
1160
+ "contract": identity_of(f"docs/specs/{loop}/{task}/spec.md"),
1161
+ "expected": identity_of(f"docs/specs/{loop}/{task}/spec.expected.json"),
1162
+ "meta": identity_of(f"docs/specs/{loop}/meta.md"),
1163
+ "requirements": spec["requirements"], "review_requirements": spec["review"],
1164
+ "shared_constraints": sections(package["meta_text"], "meta.md")["Constraints and exclusions"],
1165
+ "dependencies": [{"task": f"{loop}.{dep}", "receipt": str(state["path"]), "source_sha": state["receipt"]["source_sha"]}
1166
+ for dep in spec["depends_on"] if (state := v["states"][f"{loop}.{dep}"])["receipt"]],
1167
+ "methodology": [identity_of(name) for name in ("CLAUDE.md", "AGENTS.md") if show(worktree, inputs, name) is not None],
1168
+ "delivery": "local-only" if local else package["manifest"]["delivery"],
1169
+ "evidence_dir": str(worktree / ".devlyn" / "loop"),
1170
+ "submission": str(path.parent / "submission.json"),
1171
+ "runner": [sys.executable, str(Path(__file__).with_name("acceptance.py")), "run", "--packet", str(packet_path)],
1172
+ "obligations": OBLIGATIONS,
1173
+ }
1174
+ instructions = packet["methodology"]
1175
+ binding = harness_call("read_binding", v["common"])
1176
+ if binding:
1177
+ defaults = {"source": "release", **binding["defaults"]}
1178
+ else:
1179
+ # An unbound queue's executor still gets the principles: the invoking skill's own copy.
1180
+ own = SKILLS / "_shared/runtime-principles.md"
1181
+ defaults = {"source": "skill", **harness_call("file_identity", own)} if own.is_file() else None
1182
+ if defaults is None and not instructions:
1183
+ raise Waiting(f"{identity}: no principles reach its executor — neither CLAUDE.md/AGENTS.md at the inputs commit nor "
1184
+ f"{SKILLS / '_shared/runtime-principles.md'}; reinstall devlyn-cli to restore that file, then drain again")
1185
+ packet.update(harness=binding["release"] if binding else None,
1186
+ methodology={"defaults": defaults, "projectInstructions": instructions}, workerPrompt=None)
1187
+ if binding:
1188
+ harness_call("check_binding", binding, harness_call("resolve", SKILLS))
1189
+ bound_skills = Path(binding["release"]["root"]) / "config/skills"
1190
+ packet["runner"][1] = str(bound_skills / "devlyn-ideate/scripts/acceptance.py")
1191
+ prompt = path.parent / "worker-prompt.md"
1192
+ body = (f"Execute task packet {packet_path}. Work only in {worktree} on its owned branch.\n"
1193
+ f"Read {binding['defaults']['path']} and apply it as defaults; explicit user and native owner/project instructions win.\n"
1194
+ f"Set DEVLYN_SKILL_DIR={bound_skills / 'devlyn-ideate'} and DEVLYN_SHARED_DIR={bound_skills / '_shared'}.\n"
1195
+ f"Follow Executor exchange and Review records in {bound_skills / 'devlyn-ideate/references/loop.md'}.\n"
1196
+ f"Meet the packet obligations, run its runner and write {packet['submission']}. Never push, open a pull request or merge.\n")
1197
+ acceptance()["atomic_write"](prompt, body.encode("utf-8"))
1198
+ packet["workerPrompt"] = harness_call("file_identity", prompt)
1199
+ write_json(packet_path, packet)
1200
+ return packet_path, packet
1201
+
1202
+
1203
+ def record(log, event):
1204
+ """Append one execution event durably, before the drain acts on it."""
1205
+ with log.open("a", encoding="utf-8") as stream:
1206
+ stream.write(f"{datetime.datetime.now(datetime.timezone.utc).isoformat()} {event}\n")
1207
+ stream.flush()
1208
+ os.fsync(stream.fileno())
1209
+
1210
+
1211
+ def ensure_submission(identity, packet_path, packet, executor):
1212
+ """Run the executor once unless its submission exists; returns a drain-recorded failure reason, else None. An
1213
+ executor that exits without writing its submission raises Waiting: the task stays active and the next drain runs it
1214
+ again.
1215
+
1216
+ An attempt's start is recorded before the spawn and its exit after the wait, so a start without an exit may have
1217
+ left a live executor: drain observes once and waits if it is active. Only established cessation permits adoption
1218
+ or another execution. Where writers cannot be observed the task fails as interrupted-unobservable."""
1219
+ harness_call("verify_packet", packet, packet_path, SKILLS)
1220
+ submission = Path(packet["submission"])
1221
+ worktree = Path(packet["worktree"])
1222
+ log = submission.with_name("executions.log")
1223
+ try:
1224
+ events = log.read_text(encoding="utf-8").splitlines() if log.exists() else []
1225
+ except UnicodeDecodeError as exc:
1226
+ raise LoopError(f"{log} is not UTF-8: {exc}; restore the execution history before draining again") from exc
1227
+ if events and events[-1].partition(" ")[2] == "start":
1228
+ helper = shared("task-complete")
1229
+ try:
1230
+ helper["stopped_writers"](worktree)
1231
+ except helper["WritersUnobservable"] as exc:
1232
+ return f"interrupted-unobservable: an interrupted executor may still be writing ({exc})"
1233
+ except helper["WriterActive"] as exc:
1234
+ raise Waiting(f"interrupted execution still active: {exc}; wait for or stop the named writer, then drain again") from exc
1235
+ except (helper["CompletionError"], OSError) as exc:
1236
+ raise LoopError(f"cannot establish that an interrupted execution stopped: {exc}; repair writer observation, then drain again") from exc
1237
+ if submission.exists():
1238
+ return None
1239
+ defaults = packet["methodology"].get("defaults") if packet.get("schema_version") == 2 else None
1240
+ if packet.get("harness") is None and defaults and not Path(defaults["path"]).is_file():
1241
+ # An installation's own copy moves with its updates; only its absence matters, and only before a spawn.
1242
+ raise Waiting(f"{identity}: its principles file {defaults['path']} is gone; reinstall devlyn-cli to restore it, then drain again")
1243
+ argv = [part.replace("{packet}", str(packet_path)) for part in executor]
1244
+ if any("{worktree_git_dir}" in part for part in argv): # Optional: a Codex executor lists it as a writable root.
1245
+ git_dir = git(worktree, "rev-parse", "--path-format=absolute", "--git-dir")
1246
+ argv = [part.replace("{worktree_git_dir}", git_dir) for part in argv]
1247
+ output = Path(packet["evidence_dir"])
1248
+ output.mkdir(parents=True, exist_ok=True)
1249
+ record(log, "start")
1250
+ try:
1251
+ # Files, never a pipe: a wrapper such as codex-monitored.sh refuses a piped stdout; stdin is empty, never the driver's.
1252
+ with (output / "executor.stdout").open("ab") as stdout, (output / "executor.stderr").open("ab") as stderr:
1253
+ child = subprocess.Popen(shared("platform-support")["native_argv"](argv, cwd=worktree), cwd=worktree, stdin=subprocess.DEVNULL,
1254
+ stdout=stdout, stderr=stderr)
1255
+ except OSError as exc:
1256
+ record(log, f"not started: {one_line(str(exc))}")
1257
+ raise LoopError(f"executor could not start: {exc}; restore the configured executable, permissions or task workspace; "
1258
+ "the next drain retries") from exc
1259
+ progress(identity, f"executing; output in {output}")
1260
+ code = child.wait()
1261
+ record(log, f"exit {code} (pid {child.pid})")
1262
+ if not submission.exists():
1263
+ raise Waiting(f"executor exited {code} without a submission (see {output / 'executor.stderr'}); the next drain runs it again")
1264
+ return None
1265
+
1266
+
1267
+ def terminal_queue(cwd, receipt, identity):
1268
+ """The loop queue file of this bound result's only legal terminal transition: its row becomes [x] or [F] — <reason>."""
1269
+ failed = receipt["acceptance"]["verdict"] == "FAILED"
1270
+ suffix = f" — {one_line(receipt['acceptance']['reasons'][0])} (receipt {receipt['id']})" if failed else ""
1271
+ rel = loop_queue(identity.split(".")[0])
1272
+ return transition(show(cwd, receipt["acceptance"]["inputs_sha"], rel) or b"", identity, "F" if failed else "x", suffix)
1273
+
1274
+
1275
+ def is_terminal(cwd, receipt, identity, commit):
1276
+ """Whether `commit` is that transition: its sole parent is the bound source and its only change is the row."""
1277
+ source, rel = receipt["source_sha"], loop_queue(identity.split(".")[0])
1278
+ return (git(cwd, "rev-list", "--parents", "-n", "1", commit).split() == [commit, source]
1279
+ and git(cwd, "diff", "--name-only", source, commit) == rel and show(cwd, commit, rel) == terminal_queue(cwd, receipt, identity))
1280
+
1281
+
1282
+ def settle(v, row, receipt_file):
1283
+ """Create (or adopt) the queue-only terminal commit atop the bound source, then attach it."""
1284
+ identity = row["identity"]
1285
+ receipt = read_json(receipt_file)
1286
+ worktree, source, branch = Path(receipt["worktree"]), receipt["source_sha"], receipt["branch"]
1287
+ failed = receipt["acceptance"]["verdict"] == "FAILED"
1288
+ with lock(v["common"], "queue.lock", blocking=True):
1289
+ head = git(worktree, "rev-parse", "refs/heads/" + branch)
1290
+ if head == source:
1291
+ commit = commit_files(worktree, branch, source, {loop_queue(row["loop"]): terminal_queue(worktree, receipt, identity)},
1292
+ f"devlyn loop: {identity} {'failed' if failed else 'accepted'}", checkout=not failed)
1293
+ else:
1294
+ require(is_terminal(worktree, receipt, identity, head),
1295
+ f"{identity}: owned branch moved past the bound source without its terminal transition; inspect {worktree}")
1296
+ commit = head
1297
+ if not failed:
1298
+ sync(worktree, branch, source, commit)
1299
+ progress(identity, f"terminal {commit}")
1300
+ task_complete("attach", receipt=str(receipt_file), commit=commit, file=loop_queue(row["loop"]))
1301
+ progress(identity, "attached")
1302
+
1303
+
1304
+ def advance(v, row, opts):
1305
+ """Run or resume one task from its durable state; never replays a bound result. Returns why it waits instead."""
1306
+ identity = row["identity"]
1307
+ state = v["states"][identity]
1308
+ state["phase"] = "allocation"
1309
+ path = receipt_path(v["common"], identity)
1310
+ local = is_local(v, row["loop"])
1311
+ if opts.local_only and not local:
1312
+ return published(v, row["loop"])
1313
+ if not path.exists() and (refused := allocate(v, row, opts)):
1314
+ return refused
1315
+ receipt = read_json(path)
1316
+ if receipt.get("acceptance") and (path.parent / "packet.json").exists():
1317
+ harness_call("verify_packet", read_json(path.parent / "packet.json"), path.parent / "packet.json", SKILLS)
1318
+ if not receipt.get("acceptance"):
1319
+ state["phase"] = "inputs"
1320
+ failure = v["states"][identity].get("inputs_changed")
1321
+ if not failure and not receipt.get("local_only"):
1322
+ require_merged(v, row, receipt["baseline"], allocated=True)
1323
+ packet_path, packet = ensure_packet(v, row, path, receipt)
1324
+ state["phase"] = "executor"
1325
+ failure = failure or ensure_submission(identity, packet_path, packet, opts.executor)
1326
+ state["phase"] = "acceptance"
1327
+ try:
1328
+ result = acceptance()["accept"](packet_path, packet["submission"], failure)
1329
+ except acceptance()["AcceptanceError"] as exc:
1330
+ raise LoopError(f"{identity}: acceptance could not run: {exc}") from exc
1331
+ progress(identity, result["verdict"].lower() + "".join(f"; {reason}" for reason in result["reasons"][:3]))
1332
+ state["phase"] = "custody binding"
1333
+ task_complete("accept", receipt=str(path), acceptance=str(Path(packet["worktree"]) / ".devlyn/loop/acceptance.json"))
1334
+ progress(identity, "bound")
1335
+ state["phase"] = "terminal transition"
1336
+ if not read_json(path).get("queue"):
1337
+ settle(v, row, path)
1338
+ receipt = read_json(path)
1339
+ state["phase"] = "delivery"
1340
+ if receipt.get("delivery") not in SETTLED:
1341
+ packet = read_json(path.parent / "packet.json")
1342
+ # The plan's mode seeds the first delivery call; a later per-task --mode persists in the receipt.
1343
+ result = task_complete("complete", receipt=str(path), acceptance=None,
1344
+ mode=None if local or receipt.get("mode_override") else packet["delivery"], local_only=local, writers_stopped=True)
1345
+ progress(identity, f"delivery {result['status']}")
1346
+ if result.get("merge_refused"):
1347
+ return f"merge refused: {result['merge_refused']}; a person merges the PR"
1348
+ return None
1349
+
1350
+
1351
+ def summary(v, row):
1352
+ state = v["states"][row["identity"]]
1353
+ receipt = state["receipt"] or {}
1354
+ item = {"identity": row["identity"], "result": state["kind"], "receipt": str(state["path"]) if state["path"] else None}
1355
+ if state["kind"] == "failed":
1356
+ item["reason"] = row["rest"].lstrip(" —") if row["mark"] == "F" else (receipt.get("acceptance") or {}).get("reasons", ["failed"])[0]
1357
+ if state["kind"] == "blocked":
1358
+ item["reason"] = f"blocked-prerequisite:{state['blocker']}"
1359
+ if reason := state.get("waiting") or state.get("inputs_changed") or state.get("invalid"):
1360
+ item["reason"] = reason
1361
+ if receipt:
1362
+ acceptance_record = receipt.get("acceptance") or {}
1363
+ branch, worktree, resume = None, None, None
1364
+ try:
1365
+ worktree = receipt["worktree"] if receipt.get("worktree") and Path(receipt["worktree"]).exists() else None
1366
+ if receipt.get("branch") and ref_value(v["anchor"], "refs/heads/" + receipt["branch"]):
1367
+ branch = receipt["branch"]
1368
+ if receipt.get("acceptance") and receipt.get("delivery") not in SETTLED:
1369
+ if receipt.get("queue"):
1370
+ binding = harness_call("read_binding", v["common"])
1371
+ skills = Path(binding["release"]["root"]) / "config/skills" if binding else SKILLS
1372
+ argv = [sys.executable, str(skills / "_shared/task-complete.py"), "complete", "--receipt", str(state["path"])]
1373
+ if receipt.get("writers_released"):
1374
+ argv.append("--writers-stopped")
1375
+ if not receipt.get("local_only") and not receipt.get("mode_override"):
1376
+ argv += ["--mode", read_json(state["path"].parent / "packet.json")["delivery"]]
1377
+ resume = shlex.join(argv)
1378
+ else:
1379
+ resume = "drain again: the terminal commit is not attached yet"
1380
+ except (LoopError, OSError) as exc:
1381
+ diagnostic = (f"task summary unavailable: {row['identity']} worktree {receipt.get('worktree')}, "
1382
+ f"branch {receipt.get('branch')}: {exc}; repair access and drain again")
1383
+ if diagnostic not in v["cleanup"]:
1384
+ v["cleanup"].append(diagnostic)
1385
+ item.update(allocation_base=receipt.get("baseline"), **{"source" if state["kind"] == "accepted" else "candidate": receipt.get("source_sha")},
1386
+ terminal=(receipt.get("queue") or {}).get("commit"), delivery=receipt.get("delivery"), pr=receipt.get("pr_url"),
1387
+ resume=resume,
1388
+ assumptions=acceptance_record.get("assumptions", []),
1389
+ questions=[r.removeprefix("needs-review: ") for r in acceptance_record.get("reasons", []) if r.startswith("needs-review: ")],
1390
+ worktree=worktree,
1391
+ cleanup=f"{c['status']} — {c['reason']}; resume: {c['resume']}" if (c := receipt.get("workspace_cleanup")) else None,
1392
+ branch=branch,
1393
+ recovery_ref=receipt.get("recovery_ref"),
1394
+ custody=str(Path(state["path"]).parent / "custody") if receipt.get("files") else None,
1395
+ scratch=(f"{c['status']} — {c['reason']}; resume: {c['resume']}" if (c := receipt.get("scratch_cleanup") or {}).get("reason")
1396
+ else c.get("status", "NOT_CLEANED")))
1397
+ return item
1398
+
1399
+
1400
+ def manifest_of(v, loop):
1401
+ """The loop's manifest, read once per view."""
1402
+ if loop not in v["manifests"]:
1403
+ v["manifests"][loop] = v["packages"][loop]["manifest"] if loop in v["packages"] else load_manifest(v["anchor"], loop, capture(v, loop))[1]
1404
+ return v["manifests"][loop]
1405
+
1406
+
1407
+ def loop_acceptance(v, loop, items):
1408
+ """Whole-loop acceptance over the manifest's complete task set: every task, the integration task last, needs
1409
+ receipt-backed acceptance. A manifest task missing from this queue joins `items` as not yet run."""
1410
+ try:
1411
+ manifest = manifest_of(v, loop)
1412
+ except LoopError as exc:
1413
+ return f"INCOMPLETE — manifest unreadable: {exc}"
1414
+ found, gaps = {item["identity"]: item for item in items}, []
1415
+ for identity in (f"{loop}.{entry['id']}" for entry in manifest["tasks"]):
1416
+ if identity not in found:
1417
+ items.append(found.setdefault(identity, {"identity": identity, "result": "not yet run"}))
1418
+ if found[identity]["result"] != "accepted":
1419
+ gaps.append(f"{identity} {found[identity]['result']}")
1420
+ elif not found[identity].get("receipt"):
1421
+ gaps.append(f"{identity} accepted without a receipt")
1422
+ return "INCOMPLETE — " + ", ".join(gaps) if gaps else "ACCEPTED"
1423
+
1424
+
1425
+ def bring_in(v, loop):
1426
+ """The command that brings the loop's accepted work into the branch it was added on (loop.md step 11)."""
1427
+ tip = frontier(v, loop)
1428
+ if tip is None:
1429
+ return None
1430
+ receipt = tip["receipt"]
1431
+ if receipt.get("local_only"):
1432
+ # It fast-forwards when it can and merges otherwise, so the commands of several local loops apply in any order.
1433
+ return f"{v['records'][loop]['branch']}: git merge --ff {receipt['branch']}" if loop in v["records"] else None
1434
+ if not delivered(receipt):
1435
+ return None
1436
+ # The anchor holds no loop commit, so after merge, squash or rebase delivery this fast-forwards unless the branch has commits
1437
+ # of its own, and merges then; like the local form, the commands of several loops apply in any order.
1438
+ return f"{receipt['base']}: git merge --ff {receipt['remote']}/{receipt['base']}"
1439
+
1440
+
1441
+ def write_reports(v, status, reason, warnings):
1442
+ paths = []
1443
+ for loop in dict.fromkeys(row["loop"] for row in v["rows"] if row["identity"]):
1444
+ path = v["common"] / "devlyn-loops" / loop / "drain-report.md"
1445
+ try:
1446
+ items = [summary(v, row) for row in v["rows"] if row.get("loop") == loop]
1447
+ lines = [f"# Drain report — {loop}", "", f"- Generated: {datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')}",
1448
+ f"- Queue: {f'{CAPTURES}{loop}:{loop_queue(loop)}' if loop in v['records'] else v['anchor'] / loop_queue(loop)}",
1449
+ f"- Drain: {status}" + (f" — {reason}" if reason else ""), f"- Whole-loop acceptance: {loop_acceptance(v, loop, items)}",
1450
+ *([f"- Bring into {line}"] if (line := bring_in(v, loop)) else []), ""]
1451
+ if warnings:
1452
+ lines += ["## Warnings", "", *(f"- {warning}" for warning in warnings), ""]
1453
+ try:
1454
+ meta, _ = load_manifest(v["anchor"], loop, capture(v, loop))
1455
+ decisions = sections(
1456
+ JSON_FENCE_RE.sub("", meta),
1457
+ v["anchor"] / f"docs/specs/{loop}/meta.md",
1458
+ )["Decisions and assumptions"] or "None recorded."
1459
+ except (LoopError, OSError) as exc:
1460
+ decisions = f"Unavailable: {exc}"
1461
+ lines += ["## Package decisions and assumptions", "", decisions, ""]
1462
+ labels = (("result", "Product"), ("reason", "Reason"), ("receipt", "Receipt"), ("custody", "Evidence custody"),
1463
+ ("recovery_ref", "Recovery ref"), ("allocation_base", "Allocation base"), ("source", "Accepted source"), ("candidate", "Unaccepted source"),
1464
+ ("terminal", "Terminal commit"), ("delivery", "Delivery"), ("pr", "PR"), ("resume", "Resume"),
1465
+ ("assumptions", "Assumptions"), ("questions", "Unresolved questions"), ("worktree", "Retained worktree"),
1466
+ ("cleanup", "Workspace cleanup"),
1467
+ ("branch", "Branch"), ("scratch", "Scratch cleanup"))
1468
+ for item in items:
1469
+ lines += [f"## {item['identity']}", ""]
1470
+ lines += [f"- {label}: {'; '.join(value) if isinstance(value, list) else value}"
1471
+ for key, label in labels if (value := item.get(key)) not in (None, [], "")]
1472
+ lines.append("")
1473
+ acceptance()["atomic_write"](path, "\n".join(lines).encode("utf-8"))
1474
+ paths.append(str(path))
1475
+ except (LoopError, OSError) as exc:
1476
+ v["cleanup"].append(f"report unavailable: {path}: {exc}; repair access and drain again")
1477
+ return paths
1478
+
1479
+
1480
+ def counts(v):
1481
+ kinds = [state["kind"] for state in v["states"].values()]
1482
+ return {kind: kinds.count(kind) for kind in ("pending", "active", "accepted", "failed", "blocked")} | {
1483
+ "legacy_pending": sum(1 for row in v["rows"] if not row["identity"] and row["mark"] == " ")}
1484
+
1485
+
1486
+ def prune_parents(v, root):
1487
+ """Retry empty-parent cleanup for completed, verified receipts; the drain alone knows this boundary."""
1488
+ for identity, state in v["states"].items():
1489
+ receipt = state["receipt"]
1490
+ if not receipt or receipt.get("delivery") != "COMPLETE" or not receipt.get("linked"):
1491
+ continue
1492
+ work = Path(receipt["worktree"])
1493
+ expected = root.joinpath(*identity.split("."))
1494
+ if work != expected:
1495
+ continue
1496
+ parent = work.parent
1497
+ helper = shared("task-complete")
1498
+ try:
1499
+ if work.exists():
1500
+ continue
1501
+ require(work == work.resolve() and not work.is_symlink() and not parent.is_symlink(),
1502
+ f"symlink or escape at {work}")
1503
+ registered = helper["registrations"](receipt)
1504
+ while parent != root and parent.is_relative_to(root):
1505
+ require(not any(Path(path).is_relative_to(parent) for path in registered),
1506
+ f"registered worktree retains parent: {parent}")
1507
+ try:
1508
+ parent.rmdir()
1509
+ except FileNotFoundError:
1510
+ pass # A previous cleanup already removed this parent.
1511
+ except OSError as exc:
1512
+ if exc.errno in {errno.ENOTEMPTY, errno.EEXIST}:
1513
+ break
1514
+ raise
1515
+ parent = parent.parent
1516
+ except (LoopError, OSError, helper["CompletionError"]) as exc:
1517
+ v["cleanup"].append(f"parent cleanup retained: {parent}: {exc}; remove it when empty or drain again")
1518
+
1519
+
1520
+ def drain(args):
1521
+ anchor, common = repository(args.repo)
1522
+ args.repo = anchor # Preserve the repository root for failures reported by main.
1523
+ enforce_binding(common, anchor)
1524
+ diagnostics = skill_diagnostics(anchor)
1525
+ require(any("{packet}" in part for part in args.executor), "executor argv must contain {packet}, replaced by the task packet path")
1526
+ args.default_worktree_root = args.worktree_root is None
1527
+ args.root_diagnostics = []
1528
+ args.worktree_root = Path(args.worktree_root).resolve() if args.worktree_root else anchor.parent / f"{anchor.name}.devlyn.noindex"
1529
+ with lock(common, "drain.lock", blocking=False):
1530
+ attempted, last, status, reason, refused = set(), None, "DRAINED", None, {}
1531
+ try:
1532
+ while True:
1533
+ last = view(anchor, common, args.local_only)
1534
+ row = next_task(last, attempted)
1535
+ if row is None:
1536
+ break
1537
+ attempted.add(row["identity"])
1538
+ try:
1539
+ wait = advance(last, row, args)
1540
+ except Waiting as exc:
1541
+ wait = str(exc)
1542
+ except (LoopError, OSError) as exc:
1543
+ phase = last["states"][row["identity"]].get("phase", "allocation")
1544
+ wait = f"{phase} blocked: {exc}"
1545
+ if phase in {"allocation", "inputs", "acceptance", "custody binding", "terminal transition"}:
1546
+ wait += f"; inspect and repair {row['identity']}'s {phase} state, then drain again"
1547
+ if wait:
1548
+ refused[row["identity"]] = wait
1549
+ if any(state["kind"] in {"pending", "active"} for state in last["states"].values()) or counts(last)["legacy_pending"] or refused:
1550
+ status = "WAITING"
1551
+ except (LoopError, OSError) as exc:
1552
+ status, reason = "BLOCKED", str(exc)
1553
+ with contextlib.suppress(LoopError, OSError):
1554
+ last = view(anchor, common, args.local_only)
1555
+ if last is None:
1556
+ return {"status": status, "reason": reason, **diagnostics}
1557
+ last["cleanup"].extend(args.root_diagnostics)
1558
+ annotate(last, refused)
1559
+ prune_parents(last, args.worktree_root)
1560
+ try:
1561
+ release_settled_binding(anchor, common)
1562
+ except (LoopError, OSError) as exc:
1563
+ last["cleanup"].append(f"queue binding not released: {exc}; the next status or drain retries")
1564
+ return {"status": status, "reason": reason, **diagnostics,
1565
+ "reports": write_reports(last, status, reason, diagnostics["warnings"]), "counts": counts(last),
1566
+ "tasks": [summary(last, row) for row in last["rows"] if row["identity"]], "cleanup": last["cleanup"]}
1567
+
1568
+
1569
+ def status_summary(v):
1570
+ anchor = v["anchor"]
1571
+ row = next_task(v, set())
1572
+ annotate(v, {})
1573
+ tasks = [summary(v, r) for r in v["rows"] if r["identity"]]
1574
+ return {"status": "OK", **skill_diagnostics(anchor), "counts": counts(v), "next": row["identity"] if row else None,
1575
+ "blockers": [f"{t['identity']}: {t['reason']}" for t in tasks if t.get("reason") and t["result"] in {"pending", "active", "blocked"}]
1576
+ + [f"legacy row {r['index'] + 1} needs planning: {r['text']}" for r in v["rows"] if not r["identity"] and r["mark"] == " "]
1577
+ + [f"unreadable receipt {item}" for item in v["unreadable"]],
1578
+ "delivery": [t for t in tasks if t.get("resume")], "cleanup": v["cleanup"]}
1579
+
1580
+
1581
+ def status(args):
1582
+ anchor, common = repository(args.repo)
1583
+ args.repo = anchor
1584
+ enforce_binding(common, anchor)
1585
+ result = status_summary(view(anchor, common, args.local_only))
1586
+ release_settled_binding(anchor, common)
1587
+ return result
1588
+
1589
+
1590
+ def inspect(args):
1591
+ anchor, common = repository(args.repo)
1592
+ try:
1593
+ result = status_summary(view(anchor, common, pure=True))
1594
+ except (LoopError, OSError) as exc:
1595
+ result = {"status": "BLOCKED", "reason": str(exc), **skill_diagnostics(anchor)}
1596
+ binding_path = common / "devlyn-loops/harness.json"
1597
+ binding = None
1598
+ compatibility = {"compatible": True, "legacy": not (binding_path.exists() or binding_path.is_symlink()), "reasons": [], "recoveryArgv": None,
1599
+ "taskPacket": {"inspect": [1, 2], "execute": [1, 2], "write": 2}, "queueBinding": {"read": [1, 1], "write": 1}}
1600
+ try:
1601
+ binding = read_json(binding_path) if not compatibility["legacy"] else None
1602
+ enforce_binding(common)
1603
+ for packet in (common / "devlyn-completion").glob("*/packet.json"):
1604
+ harness_call("packet_schema", read_json(packet))
1605
+ except (LoopError, OSError) as exc:
1606
+ compatibility.update(compatible=False, reasons=[str(exc)])
1607
+ try:
1608
+ valid = harness_call("read_binding", common)
1609
+ if valid:
1610
+ compatibility["recoveryArgv"] = harness_call("recovery_argv", valid, ["inspect", "--repo", str(anchor), "--json"])
1611
+ except (LoopError, OSError):
1612
+ pass # The diagnostic above describes the malformed binding; no safe argv can be inferred.
1613
+ return {**result, "binding": binding, "compatibility": compatibility}
1614
+
1615
+
1616
+ def legacy_target(anchor, number):
1617
+ """The pending legacy row on line `number` of the legacy queue, as add records it: its exact text and its occurrence
1618
+ among equal rows, line endings aside; None for any other line."""
1619
+ path = anchor / QUEUE
1620
+ rows = parse_queue(path.read_bytes() if path.exists() else b"")
1621
+ row = next((row for row in rows if row["index"] == number - 1 and row["identity"] is None and row["mark"] == " "), None)
1622
+ return row and {"line": row["line"], "occurrence": sum(other["line"] == row["line"] for other in rows if other["index"] <= row["index"])}
1623
+
1624
+
1625
+ def added_result(anchor, added, notes):
1626
+ loop, ref = added["loop_id"], added["capture"]
1627
+ rows, _ = parse_loop(show(anchor, added["commit"], loop_queue(loop)), loop, f"{ref}:{loop_queue(loop)}")
1628
+ return {"status": "ADDED", "tasks": [row["identity"] for row in rows], "capture": ref, "commit": added["commit"], "cleanup": notes,
1629
+ "message": f"The package is now held immutably at {ref}; inspect it with `git show {ref}:docs/specs/{loop}/meta.md`. "
1630
+ "A revision needs a new loop id."}
1631
+
1632
+
1633
+ def add(args):
1634
+ """Capture the package and its queue rows under refs/devlyn/captures/<loop>, record the add, then remove the untracked
1635
+ originals equal to the capture (package-format.md, Commands); the branch, index and tracked files stay as they are."""
1636
+ meta = Path(args.package).resolve()
1637
+ anchor, common = repository(next(path for path in meta.parents if path.is_dir()))
1638
+ loop = meta.parent.name
1639
+ ref, package_dir = CAPTURES + loop, f"docs/specs/{loop}"
1640
+ require(meta.name == "meta.md" and meta.parent.parent == anchor / "docs/specs" and re.fullmatch(ID, loop),
1641
+ f"package must be <repo>/docs/specs/<loop-id>/meta.md: {meta}")
1642
+ enforce_binding(common, anchor)
1643
+ with lock(common, "queue.lock", blocking=True):
1644
+ enforce_binding(common)
1645
+ recorded = records(common)
1646
+ notes = clean(common, recorded)
1647
+ target = legacy_target(anchor, args.materialize) if args.materialize else None
1648
+ if loop in recorded:
1649
+ # This package's add, also one interrupted after its record or whose originals are gone, is reported as it is.
1650
+ added = recorded[loop]
1651
+ differ = [rel for rel, state in package_state(anchor, added["commit"], loop).items() if state == "different"]
1652
+ require(not differ and target == added.get("materialize"),
1653
+ f"loop id {loop} was already added with another package ({', '.join(differ) or '--materialize differs'}); it is held at "
1654
+ f"{ref}, so plan revised work as a new loop")
1655
+ return added_result(anchor, added, notes)
1656
+ interrupted = (f"{ref} holds the capture of an interrupted add of {loop} that differs from the checkout; inspect it with `git show "
1657
+ f"{ref}`, then restore that package or delete the ref with `git update-ref -d {ref}`, and add again")
1658
+ if captured := ref_value(anchor, ref):
1659
+ require(all(state == "equal" for state in package_state(anchor, captured, loop).values()), interrupted)
1660
+ package = package_of(anchor, meta)
1661
+ manifest = package["manifest"]
1662
+ local = manifest["delivery"] == "local-only"
1663
+ require(show(anchor, "HEAD", loop_queue(loop)) is None, f"loop id {loop} is already used by {loop_queue(loop)}; plan revised work as a new loop")
1664
+ if tracked := git(anchor, "diff", "--name-only", "--no-renames", "--diff-filter=MDT", "HEAD", "--", package_dir):
1665
+ raise LoopError(f"{package_dir} is committed with different content ({', '.join(tracked.splitlines())}); add never captures a "
1666
+ "revision of committed package files, so plan it as a new loop")
1667
+ branch = git_run(anchor, "symbolic-ref", "-q", "--short", "HEAD", ok=(0, 1)).stdout.decode("utf-8").strip()
1668
+ require(branch, f"add records the branch the loop is added on, but HEAD is detached in {anchor}")
1669
+ if drift := instruction_drift(anchor, head := git(anchor, "rev-parse", "HEAD")):
1670
+ # Tasks start from committed state: a local loop from this branch, auto/pr from the pushed base.
1671
+ raise LoopError(f"the installed instructions ({drift}) in this checkout differ from those in HEAD {head}, so the loop's tasks would "
1672
+ f"run without them; commit {'them' if local else 'and push them'}, then add the loop")
1673
+ if staged := git(anchor, "diff", "--cached", "--name-only", "--no-renames", "HEAD", "--", package_dir):
1674
+ raise LoopError(f"the index holds package changes HEAD lacks ({', '.join(staged.splitlines())}); add leaves the index as it is, "
1675
+ f"so unstage them with `git restore --staged -- {package_dir}` and add again")
1676
+ if args.materialize:
1677
+ pending = {row["index"] for row in queue_view(anchor, recorded) if row["identity"] is None and row["mark"] == " "}
1678
+ require(target and args.materialize - 1 in pending, f"{QUEUE} line {args.materialize} is not a pending legacy row")
1679
+ intent = " ".join(sections(package["meta_text"], "meta.md")["Intent"].split())
1680
+ require(" ".join(target["line"][6:].split()) in intent, "meta.md '## Intent' must reproduce the legacy row's intent verbatim")
1681
+ paths = [f"{package_dir}/meta.md", *(f"{package_dir}/{task}/{name}" for task in package["tasks"] for name in ("spec.md", "spec.expected.json"))]
1682
+ files = {rel: package_bytes(anchor, None, rel) for rel in paths} | {loop_queue(loop): loop_rows(loop, package, target)}
1683
+ # The lock covers both first binding and first capture; an interrupted binding is authoritative on retry.
1684
+ if harness_call("read_binding", common) is None:
1685
+ invoking = harness_call("resolve", SKILLS)
1686
+ if invoking is not None and bindable(anchor, common):
1687
+ write_json(common / "devlyn-loops/harness.json", invoking)
1688
+ tree = build_tree(anchor, None, files)
1689
+ if captured:
1690
+ require(tree == git(anchor, "rev-parse", captured + "^{tree}"), interrupted)
1691
+ else:
1692
+ # The ref keeps the commit, its tree and blobs from garbage collection once the originals are gone.
1693
+ captured = git(anchor, "commit-tree", tree, "-m", f"devlyn loop: capture {loop}")
1694
+ git(anchor, "update-ref", ref, captured, "0" * len(captured))
1695
+ added = {"schema_version": 1, "loop_id": loop, "delivery": manifest["delivery"], "capture": ref, "commit": captured, "tree": tree,
1696
+ "origin": str(anchor), "branch": branch, "sequence": max((other["sequence"] for other in recorded.values()), default=0) + 1,
1697
+ **({"materialize": target} if target else {})}
1698
+ write_json(added_path(common, loop), added)
1699
+ notes += clean(common, {loop: added})
1700
+ return added_result(anchor, added, notes)
1701
+
1702
+
1703
+ def check(args):
1704
+ anchor, _ = repository(Path(args.package).resolve().parent)
1705
+ package = package_of(anchor, args.package)
1706
+ return {"status": "VALID", "loop_id": package["loop_id"], "tasks": [f"{package['loop_id']}.{task}" for task in package["tasks"]]}
1707
+
1708
+
1709
+ # Self-test fixtures: a valid two-task package, then one mutation per rejected shape.
1710
+ META = """# Loop {loop}
1711
+
1712
+ ## Intent
1713
+
1714
+ {intent}
1715
+
1716
+ ## Constraints and exclusions
1717
+
1718
+ Standard library only.
1719
+
1720
+ ## Tasks
1721
+
1722
+ T1 builds the interface; T2 consumes it.
1723
+
1724
+ ```json
1725
+ {manifest}
1726
+ ```
1727
+
1728
+ ## Overall acceptance
1729
+
1730
+ R1 of t2 checks the assembled product.
1731
+
1732
+ ## Execution policy
1733
+
1734
+ Installed methodology; autonomous question policy; delivery as declared.
1735
+
1736
+ ## Decisions and assumptions
1737
+
1738
+ None beyond the task contracts.
1739
+ """
1740
+ SPEC = """---
1741
+ id: {task}
1742
+ loop_id: {loop}
1743
+ title: {title}
1744
+ kind: feature
1745
+ review_requirements: [{review}]
1746
+ ---
1747
+ # {title}
1748
+
1749
+ ## Context and goal
1750
+
1751
+ Fixture.
1752
+
1753
+ ## Requirements
1754
+
1755
+ - R1: {requirement}
1756
+ - R2: The code stays small.
1757
+
1758
+ ## Scope and constraints
1759
+
1760
+ Only the declared files.
1761
+
1762
+ ## Out of scope
1763
+
1764
+ Nothing else.
1765
+
1766
+ ## Prerequisite inputs
1767
+
1768
+ {inputs}
1769
+
1770
+ <!-- devlyn:verification -->
1771
+ ## Verification
1772
+
1773
+ The declared commands.
1774
+
1775
+ ## Deliverable and cleanup
1776
+
1777
+ Keep the product; no residue.
1778
+ """
1779
+
1780
+
1781
+ def write_package(anchor, loop, tasks, *, delivery="local-only", base=None, intent="Ship the fixture.", review="R2"):
1782
+ """tasks: [(task_id, depends_on, title, verification_commands)] -> meta path."""
1783
+ base = base or git(anchor, "rev-parse", "HEAD")
1784
+ manifest = {"schema_version": 1, "loop_id": loop, "base_ref": "main", "base_sha": base, "delivery": delivery,
1785
+ "tasks": [{"id": task, "spec": f"{task}/spec.md", "depends_on": deps} for task, deps, _, _ in tasks],
1786
+ "integration_task_id": tasks[-1][0]}
1787
+ root = anchor / "docs/specs" / loop
1788
+ for task, deps, title, commands in tasks:
1789
+ (root / task).mkdir(parents=True, exist_ok=True)
1790
+ (root / task / "spec.md").write_text(SPEC.format(task=task, loop=loop, title=title, review=review, requirement=f"{title} works.",
1791
+ inputs=", ".join(deps) or "None."), encoding="utf-8")
1792
+ (root / task / "spec.expected.json").write_text(json.dumps({"verification_commands": commands}, indent=2), encoding="utf-8")
1793
+ (root / "meta.md").write_text(META.format(loop=loop, intent=intent, manifest=json.dumps(manifest, indent=2)), encoding="utf-8")
1794
+ return root / "meta.md"
1795
+
1796
+
1797
+ class QueueTests(unittest.TestCase):
1798
+ def setUp(self):
1799
+ self.temp = tempfile.TemporaryDirectory(prefix="devlyn-queue ✓ ")
1800
+ self.addCleanup(self.temp.cleanup)
1801
+ self.root = Path(self.temp.name).resolve()
1802
+ self.env = {**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": str(self.root / "gitconfig"),
1803
+ "GIT_AUTHOR_NAME": "Fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid",
1804
+ "GIT_COMMITTER_NAME": "Fixture", "GIT_COMMITTER_EMAIL": "fixture@example.invalid"}
1805
+ self.anchor = self.root / "repo"
1806
+ subprocess.run(["git", "init", "-q", "--initial-branch=main", str(self.anchor)], check=True, env=self.env)
1807
+ (self.anchor / ".gitignore").write_text(".devlyn/\n", encoding="utf-8")
1808
+ subprocess.run(["git", "-C", str(self.anchor), "add", "."], check=True, env=self.env)
1809
+ subprocess.run(["git", "-C", str(self.anchor), "commit", "-qm", "base"], check=True, env=self.env)
1810
+ command = {"argv": [sys.executable, "-c", "pass"], "contract_refs": ["R1"]}
1811
+ self.tasks = [("t1", [], "Interface 인터페이스", [command]), ("t2", ["t1"], "Consumer [app]", [command])]
1812
+ self.meta = write_package(self.anchor, "inv", self.tasks)
1813
+
1814
+ def cli(self, *args, cwd=None, code=0):
1815
+ result = subprocess.run([sys.executable, str(Path(__file__).resolve()), *map(str, args)], cwd=cwd or self.anchor,
1816
+ env=self.env, capture_output=True, text=True, encoding="utf-8")
1817
+ self.assertEqual(result.returncode, code, result.stdout + result.stderr)
1818
+ return json.loads(result.stdout)
1819
+
1820
+ def rejected(self, message, mutate):
1821
+ root = self.anchor / "docs/specs/inv"
1822
+ saved = {path: path.read_bytes() for path in root.rglob("*") if path.is_file()}
1823
+ mutate(root)
1824
+ try:
1825
+ with self.assertRaises(LoopError) as caught:
1826
+ package_of(self.anchor, self.meta)
1827
+ self.assertIn(message, str(caught.exception))
1828
+ finally:
1829
+ for path in root.rglob("*"):
1830
+ if path.is_file() and path not in saved:
1831
+ path.unlink()
1832
+ for path, data in saved.items():
1833
+ path.write_bytes(data)
1834
+
1835
+ def edit(self, rel, old, new):
1836
+ def mutate(root):
1837
+ path = root / rel
1838
+ text = path.read_text(encoding="utf-8")
1839
+ self.assertIn(old, text)
1840
+ path.write_text(text.replace(old, new, 1), encoding="utf-8")
1841
+ return mutate
1842
+
1843
+ def test_package_validation(self):
1844
+ self.assertEqual(self.cli("check", self.meta)["tasks"], ["inv.t1", "inv.t2"])
1845
+ manifest = json.loads(JSON_FENCE_RE.search((self.anchor / "docs/specs/inv/meta.md").read_text(encoding="utf-8")).group(1))
1846
+ dumped = json.dumps(manifest, indent=2)
1847
+
1848
+ def manifest_edit(change):
1849
+ data = json.loads(dumped)
1850
+ change(data)
1851
+ return self.edit("meta.md", dumped, json.dumps(data, indent=2))
1852
+
1853
+ cases = [
1854
+ ("exactly one fenced json manifest", self.edit("meta.md", "## Overall acceptance", "```json\n{}\n```\n\n## Overall acceptance")),
1855
+ ("manifest is not strict JSON", self.edit("meta.md", '"schema_version": 1', '"schema_version": 1, "schema_version": 1')),
1856
+ ("keys must be exactly", manifest_edit(lambda m: m.pop("delivery"))),
1857
+ ("must be unique", manifest_edit(lambda m: m["tasks"][1].update(id="t1", spec="t1/spec.md"))),
1858
+ ("depends on unknown task(s): t9", manifest_edit(lambda m: m["tasks"][1].update(depends_on=["t9"]))),
1859
+ ("dependency cycle: t1 -> t2 -> t1", manifest_edit(lambda m: m["tasks"][0].update(depends_on=["t2"]))),
1860
+ ("spec link must be t2/spec.md", manifest_edit(lambda m: m["tasks"][1].update(spec="t1/spec.md"))),
1861
+ ("integration_task_id must name the last task", manifest_edit(lambda m: m.update(integration_task_id="t1"))),
1862
+ ("integration_task_id must name the last task", manifest_edit(lambda m: m["tasks"][1].update(depends_on=[]))),
1863
+ ("base_sha must name an exact commit", manifest_edit(lambda m: m.update(base_sha="0" * 40))),
1864
+ ("missing or empty section(s): '## Execution policy'", self.edit("meta.md", "## Execution policy", "## Policy")),
1865
+ ("status is queue state", self.edit("t1/spec.md", "kind: feature", "kind: feature\nstatus: planned")),
1866
+ ("kind must be feature, spike or prototype", self.edit("t1/spec.md", "kind: feature", "kind: [feature]")),
1867
+ ("must directly follow", self.edit("t1/spec.md", "<!-- devlyn:verification -->\n", "")),
1868
+ ("unique stable IDs", self.edit("t1/spec.md", "- R2: The code", "- R1: The code")),
1869
+ ("review_requirements name unknown requirement(s): R7", self.edit("t1/spec.md", "review_requirements: [R2]", "review_requirements: [R7]")),
1870
+ ("process_evidence (phase process obligations) is unsupported", self.edit("t1/spec.expected.json", '"verification_commands"', '"process_evidence": [], "verification_commands"')),
1871
+ ("contract_refs must name requirement IDs", self.edit("t1/spec.expected.json", '"R1"', '"R9"')),
1872
+ ("without command or review coverage: R2", self.edit("t1/spec.md", "review_requirements: [R2]", "review_requirements: []")),
1873
+ ("pure_design permits no verification_commands", self.edit("t1/spec.expected.json", '"verification_commands"', '"pure_design": true, "verification_commands"')),
1874
+ ]
1875
+ for message, mutate in cases:
1876
+ with self.subTest(message=message):
1877
+ self.rejected(message, mutate)
1878
+
1879
+ def test_rows_and_transitions(self):
1880
+ queue = (b"# Intent Queue\n\n" + f"- [x] old intent\n{row_line('a.t1', 'Same')}\n- [ ] legacy intent\n{row_line('a.t2', 'Same')}\n".encode())
1881
+ rows = parse_queue(queue)
1882
+ self.assertEqual([row["identity"] for row in rows], [None, "a.t1", None, "a.t2"])
1883
+ after = transition(queue, "a.t2", "F", " — failed: check (receipt 1)")
1884
+ self.assertEqual([r["mark"] for r in parse_queue(after)], ["x", " ", " ", "F"])
1885
+ self.assertEqual(after.replace(b"- [F] a.t2 [Same](docs/specs/a/t2/spec.md) \xe2\x80\x94 failed: check (receipt 1)", b""),
1886
+ queue.replace(row_line("a.t2", "Same").encode(), b""))
1887
+ for bad, message in ((lambda: transition(after, "a.t2", "x"), "no pending queue row"),
1888
+ (lambda: parse_queue(queue + row_line("a.t1", "Dup").encode()), "duplicate queue identity: a.t1")):
1889
+ with self.assertRaisesRegex(LoopError, re.escape(message)):
1890
+ bad()
1891
+ self.assertEqual(parse_queue(row_line("a.t1", "[x] tricky \\ title").encode())[0]["identity"], "a.t1")
1892
+ # A row that does not fully match the loop-row grammar is a legacy row; trailing whitespace is tolerated.
1893
+ legacy = [b"- [ ] package.json [bump lodash](https://github.com/x/y/issues/3)", b"- [ ] a.t3 [T](docs/specs/a/t9/spec.md)",
1894
+ b"- [x] a.t4 [T](docs/specs/a/t4/spec.md) trailing words", b"- [F] a.t5 [T](docs/specs/a/t5/spec.md)"]
1895
+ self.assertEqual([row["identity"] for row in parse_queue(b"\n".join(legacy))], [None] * 4)
1896
+ spaced = row_line("a.t1", "T").encode() + b" \t\n"
1897
+ self.assertEqual(parse_queue(spaced)[0]["identity"], "a.t1")
1898
+ self.assertEqual([(r["identity"], r["mark"]) for r in parse_queue(transition(spaced, "a.t1", "F", " — failed: x (receipt 1)"))], [("a.t1", "F")])
1899
+
1900
+ def test_add_captures_the_package_and_materializes_legacy_rows(self):
1901
+ """Q, C. Prediction: add makes no commit and leaves the legacy queue, the index and every tracked file as they were;
1902
+ it points refs/devlyn/captures/inv at a commit holding exactly the package and docs/specs/inv/queue.md, its rows one
1903
+ blank line apart, records the add and removes the untracked originals; a retry reports that capture, also after the
1904
+ originals are gone; --materialize records the replaced legacy row's exact text and its occurrence among equal rows,
1905
+ line endings aside, in the loop's queue file, and status shows the loop's rows in that row's place; a different
1906
+ package under a used loop id is refused, also once committed. Before: add appended the rows to docs/specs/queue.md
1907
+ and committed them with the package."""
1908
+ package = {path.relative_to(self.anchor).as_posix(): path.read_bytes() for path in (self.anchor / "docs/specs/inv").rglob("*") if path.is_file()}
1909
+ legacy = b"# Intent Queue\n\n- [ ] Make the report weekly\r\n- [x] done\n- [ ] Make the report weekly\n"
1910
+ (self.anchor / QUEUE).write_bytes(legacy)
1911
+ git(self.anchor, "add", QUEUE, env=self.env)
1912
+ git(self.anchor, "commit", "-qm", "legacy queue", env=self.env)
1913
+ head, common, ref = git(self.anchor, "rev-parse", "HEAD"), Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir")), "refs/devlyn/captures/inv"
1914
+ added = self.cli("add", self.meta)
1915
+ rows = f"{row_line('inv.t1', 'Interface 인터페이스')}\n\n{row_line('inv.t2', 'Consumer [app]')}\n".encode()
1916
+ self.assertEqual((added["tasks"], added["capture"], added["commit"]), (["inv.t1", "inv.t2"], ref, git(self.anchor, "rev-parse", ref)))
1917
+ self.assertIn(f"git show {ref}:docs/specs/inv/meta.md", added["message"])
1918
+ self.assertEqual({rel: show(self.anchor, ref, rel) for rel in git(self.anchor, "ls-tree", "-r", "--name-only", ref).splitlines()},
1919
+ package | {"docs/specs/inv/queue.md": rows})
1920
+ self.assertEqual((git(self.anchor, "rev-parse", "HEAD"), git(self.anchor, "status", "--porcelain", "--untracked-files=all"),
1921
+ (self.anchor / QUEUE).read_bytes(), (self.anchor / "docs/specs/inv").exists()), (head, "", legacy, False))
1922
+ self.assertEqual(read_json(added_path(common, "inv")), {
1923
+ "schema_version": 1, "loop_id": "inv", "delivery": "local-only", "capture": ref, "commit": added["commit"],
1924
+ "tree": git(self.anchor, "rev-parse", ref + "^{tree}"), "origin": str(self.anchor), "branch": "main", "sequence": 1})
1925
+ self.assertEqual(self.cli("add", self.meta)["commit"], added["commit"]) # a retry, the originals gone
1926
+ self.assertEqual(transition(rows, "inv.t1", "x"), rows.replace(b"- [ ] inv.t1", b"- [x] inv.t1"))
1927
+ second = write_package(self.anchor, "rep", [("t1", [], "Weekly", self.tasks[0][3])], intent="Wrong intent.")
1928
+ self.assertIn("reproduce the legacy row's intent", self.cli("add", second, "--materialize", 5, code=1)["reason"])
1929
+ write_package(self.anchor, "rep", [("t1", [], "Weekly", self.tasks[0][3])], intent="User asked: make the report weekly.")
1930
+ self.assertIn("not a pending legacy row", self.cli("add", second, "--materialize", 4, code=1)["reason"])
1931
+ self.assertIn("reproduce", self.cli("add", second, "--materialize", 5, code=1)["reason"])
1932
+ write_package(self.anchor, "rep", [("t1", [], "Weekly", self.tasks[0][3])], intent="User asked: Make the report weekly.")
1933
+ self.cli("add", second, "--materialize", 5)
1934
+ self.assertEqual(show(self.anchor, "refs/devlyn/captures/rep", "docs/specs/rep/queue.md"),
1935
+ b"<!-- replaces docs/specs/queue.md row (occurrence 2): - [ ] Make the report weekly -->\n\n" + row_line("rep.t1", "Weekly").encode() + b"\n")
1936
+ status = self.cli("status")
1937
+ self.assertEqual((status["next"], (self.anchor / QUEUE).read_bytes()), ("rep.t1", legacy))
1938
+ self.assertEqual([blocker for blocker in status["blockers"] if blocker.startswith("legacy")], ["legacy row 3 needs planning: Make the report weekly"])
1939
+ write_package(self.anchor, "inv", [("t9", [], "Other", self.tasks[0][3])])
1940
+ for commit in (False, True): # Revised work under a used loop id is refused, also once the revision is committed.
1941
+ if commit:
1942
+ git(self.anchor, "add", "docs/specs/inv", env=self.env)
1943
+ git(self.anchor, "commit", "-qm", "revise inv", env=self.env)
1944
+ self.assertIn("loop id inv was already added with another package (docs/specs/inv/meta.md", self.cli("add", self.meta, code=1)["reason"])
1945
+
1946
+ def test_add_refuses_without_writing(self):
1947
+ """C. Each refusal leaves HEAD, the index, the files, refs and records as they were: a revised package under a used
1948
+ loop id, a detached HEAD, package files committed with other content, a loop id a committed queue file already uses,
1949
+ and package changes in the index, which add asks to unstage rather than losing them (Prediction for the last;
1950
+ before: add committed the index's draft beside the checkout's text)."""
1951
+ self.cli("add", self.meta)
1952
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
1953
+
1954
+ def g(*args):
1955
+ subprocess.run(["git", "-C", str(self.anchor), *args], check=True, env=self.env, capture_output=True)
1956
+
1957
+ def edit_committed():
1958
+ g("add", "docs/specs/trk")
1959
+ g("commit", "-qm", "commit the package by hand")
1960
+ spec = self.anchor / "docs/specs/trk/t1/spec.md"
1961
+ spec.write_text(spec.read_text(encoding="utf-8").replace(" works.", " works for every caller."), encoding="utf-8")
1962
+
1963
+ def stage_draft():
1964
+ spec = self.anchor / "docs/specs/stg/t1/spec.md"
1965
+ final = spec.read_bytes()
1966
+ spec.write_bytes(final.replace(b"Fixture.", b"Draft."))
1967
+ g("add", "docs/specs/stg/t1/spec.md")
1968
+ spec.write_bytes(final)
1969
+
1970
+ def commit_queue():
1971
+ (self.anchor / "docs/specs/usd/queue.md").write_text(row_line("usd.t1", "Interface 인터페이스") + "\n", encoding="utf-8")
1972
+ g("add", "docs/specs/usd/queue.md")
1973
+ g("commit", "-qm", "a loop brought in from elsewhere")
1974
+
1975
+ def state():
1976
+ return (git(self.anchor, "rev-parse", "HEAD"), git(self.anchor, "ls-files", "-s"), git(self.anchor, "for-each-ref"),
1977
+ git(self.anchor, "status", "--porcelain", "--untracked-files=all"), sorted(path.name for path in (common / "devlyn-loops").iterdir()))
1978
+ for loop, tasks, prepare, message in (
1979
+ ("inv", [("t9", [], "Other", self.tasks[0][3])], lambda: None, "loop id inv was already added with another package"),
1980
+ ("dt", self.tasks[:1], lambda: g("checkout", "-q", "--detach"), "HEAD is detached"),
1981
+ ("trk", self.tasks[:1], edit_committed, "docs/specs/trk is committed with different content (docs/specs/trk/t1/spec.md)"),
1982
+ ("usd", self.tasks[:1], commit_queue, "loop id usd is already used by docs/specs/usd/queue.md"),
1983
+ ("stg", self.tasks[:1], stage_draft, "the index holds package changes HEAD lacks (docs/specs/stg/t1/spec.md); add leaves "
1984
+ "the index as it is, so unstage them with `git restore --staged -- docs/specs/stg` and add again")):
1985
+ with self.subTest(loop=loop):
1986
+ meta = write_package(self.anchor, loop, tasks)
1987
+ prepare()
1988
+ before = state()
1989
+ self.assertIn(message, self.cli("add", meta, code=1)["reason"])
1990
+ self.assertEqual(state(), before)
1991
+ g("checkout", "-q", "main")
1992
+ self.assertIn("Draft.", git(self.anchor, "show", ":docs/specs/stg/t1/spec.md"))
1993
+
1994
+ def test_cleanup_reports_what_it_cannot_remove(self):
1995
+ """C. Prediction: an original that cannot be unlinked stays and is reported as incomplete cleanup; the next cleanup
1996
+ removes it. Before: add committed the package and removed nothing."""
1997
+ from unittest import mock
1998
+ saved = {path: path.read_bytes() for path in (self.anchor / "docs/specs/inv").rglob("*") if path.is_file()}
1999
+ self.cli("add", self.meta)
2000
+ for path, data in saved.items(): # The originals as a crash before cleanup leaves them.
2001
+ path.parent.mkdir(parents=True, exist_ok=True)
2002
+ path.write_bytes(data)
2003
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2004
+ with mock.patch.object(Path, "unlink", side_effect=PermissionError("unlink denied")):
2005
+ notes = clean(common, records(common))
2006
+ self.assertEqual((sorted(notes), all(path.exists() for path in saved)), (sorted(f"package cleanup retained: {path}: unlink denied" for path in saved), True))
2007
+ self.assertEqual((clean(common, records(common)), any(path.exists() for path in saved)), ([], False))
2008
+
2009
+ def test_a_legacy_row_is_replaced_only_by_its_one_exact_claim(self):
2010
+ """Q. Prediction: a loop's queue file hides the legacy row whose exact text and occurrence it records, line endings
2011
+ aside, showing the loop's rows in its place; a claim on a missing occurrence replaces nothing, so no other row
2012
+ disappears; a row two loops claim, as two clones each materializing it leave it, is hidden with neither loop in its
2013
+ place, so materializing it once more is refused; loops without a record follow in loop-id order, two-b after two.
2014
+ Before: the first row equal to the recorded text was replaced, whatever its occurrence; two-b came first, its queue
2015
+ file's path sorting before two's since '-' sorts before '/'; the row two loops claim stayed pending, and every
2016
+ further --materialize of it was added."""
2017
+ (self.anchor / QUEUE).write_bytes(b"# Intent Queue\n\n- [ ] alpha\r\n- [ ] alpha\n- [ ] beta\n")
2018
+ for loop, occurrence, line in (("one", 2, "- [ ] alpha"), ("two", 1, "- [ ] beta"), ("two-b", 1, "- [ ] beta"), ("four", 3, "- [ ] alpha")):
2019
+ (self.anchor / "docs/specs" / loop).mkdir(parents=True)
2020
+ (self.anchor / loop_queue(loop)).write_text(f"<!-- replaces {QUEUE} row (occurrence {occurrence}): {line} -->\n\n{row_line(f'{loop}.t1', 'T')}\n",
2021
+ encoding="utf-8")
2022
+ git(self.anchor, "add", "docs/specs", env=self.env)
2023
+ self.assertEqual([(row["identity"], row["index"]) for row in queue_view(self.anchor, {})],
2024
+ [(None, 2), ("one.t1", 2), ("four.t1", 2), ("two.t1", 2), ("two-b.t1", 2)])
2025
+ again = write_package(self.anchor, "rep", self.tasks[:1], intent="User asked: beta.")
2026
+ self.assertIn(f"{QUEUE} line 5 is not a pending legacy row", self.cli("add", again, "--materialize", 5, code=1)["reason"])
2027
+
2028
+ def test_handwritten_marks_never_satisfy_dependencies(self):
2029
+ """A loop without a record here, as in a fresh clone, is read from its tracked queue file, after the recorded loops
2030
+ and in loop-id order; a hand-written [x] there is no receipt-bound acceptance, so its dependent waits and whole-loop
2031
+ acceptance names the gap."""
2032
+ self.cli("add", self.meta)
2033
+ for loop in ("hw", "gw"):
2034
+ write_package(self.anchor, loop, self.tasks)
2035
+ rows = "\n\n".join(row_line(f"{loop}.{task}", title) for task, _, title, _ in self.tasks)
2036
+ (self.anchor / loop_queue(loop)).write_text(rows.replace(f"- [ ] {loop}.t1", f"- [x] {loop}.t1") + "\n", encoding="utf-8")
2037
+ git(self.anchor, "add", "docs/specs", env=self.env)
2038
+ git(self.anchor, "commit", "-qm", "loops brought in", env=self.env)
2039
+ status = self.cli("status")
2040
+ self.assertEqual((status["next"], status["blockers"]), ("inv.t1", [
2041
+ "inv.t2: waiting for inv.t1 (pending)", "gw.t2: prerequisite gw.t1 has no receipt-bound accepted source",
2042
+ "hw.t2: prerequisite hw.t1 has no receipt-bound accepted source"]))
2043
+ self.cli("drain", "--", sys.executable, "-c", "pass", "{packet}")
2044
+ report = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir")) / "devlyn-loops/hw/drain-report.md"
2045
+ self.assertIn("- Whole-loop acceptance: INCOMPLETE — hw.t1 accepted without a receipt, hw.t2 pending", report.read_text(encoding="utf-8"))
2046
+
2047
+ def test_common_gitdir_locks_span_worktrees(self):
2048
+ linked = self.root / "linked tree"
2049
+ subprocess.run(["git", "-C", str(self.anchor), "worktree", "add", "-q", "-b", "side", str(linked)], check=True, env=self.env)
2050
+ write_package(linked, "inv", self.tasks)
2051
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2052
+ holder = [sys.executable, "-c", "import pathlib, runpy, sys; lock = runpy.run_path(sys.argv[1])['file_lock'];\n"
2053
+ "with lock(pathlib.Path(sys.argv[2]), blocking=True):\n print('held', flush=True); sys.stdin.read()",
2054
+ str(SKILLS / "_shared/platform-support.py")]
2055
+ (common / "devlyn-loops").mkdir(exist_ok=True)
2056
+ for name, args, check in (("queue.lock", ["add", linked / "docs/specs/inv/meta.md"], "waits"),
2057
+ ("drain.lock", ["drain", "--", "executor", "{packet}"], "refuses")):
2058
+ with subprocess.Popen(holder + [str(common / "devlyn-loops" / name)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) as held:
2059
+ self.assertEqual(held.stdout.readline().strip(), "held")
2060
+ with subprocess.Popen([sys.executable, str(Path(__file__).resolve()), *map(str, args)], cwd=linked, env=self.env,
2061
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") as waiter:
2062
+ if check == "waits":
2063
+ with self.assertRaises(subprocess.TimeoutExpired):
2064
+ waiter.wait(timeout=1.5)
2065
+ self.assertFalse(added_path(common, "inv").exists())
2066
+ held.stdin.close()
2067
+ self.assertEqual(waiter.wait(timeout=30), 0, waiter.stderr.read())
2068
+ else:
2069
+ out, _ = waiter.communicate(timeout=30)
2070
+ self.assertEqual(waiter.returncode, 1)
2071
+ self.assertIn("another drain is active", json.loads(out)["reason"])
2072
+ held.stdin.close()
2073
+ self.assertEqual((read_json(added_path(common, "inv"))["origin"], read_json(added_path(common, "inv"))["branch"]), (str(linked), "side"))
2074
+
2075
+ def test_a_crash_on_either_side_of_the_spawn_never_starts_a_second_executor(self):
2076
+ """The attempt's start is durable before the spawn, so a resumed drain observes writers first: it waits while
2077
+ an executor lives and adopts its submission, runs the executor once when none did, and fails the task where
2078
+ writers cannot be observed. Prediction (H2): that one run exits 0 without writing a submission, so the task waits,
2079
+ naming the exit code and executor.stderr, and no submission exists. Before: the drain wrote a stand-in submission
2080
+ with a blocked-infrastructure blocker, which made the task [F]."""
2081
+ from unittest import mock
2082
+ helper = shared("task-complete")
2083
+ work, records = self.root / "work", self.root / "records"
2084
+ work.mkdir()
2085
+ records.mkdir()
2086
+ packet = {"worktree": str(work), "submission": str(records / "submission.json"), "inputs_sha": "0" * 40,
2087
+ "evidence_dir": str(work / ".devlyn/loop")}
2088
+ spawns, checks = [], []
2089
+
2090
+ class Crash(BaseException):
2091
+ """The controller dies here."""
2092
+
2093
+ class Child:
2094
+ pid = 4242
2095
+
2096
+ def wait(self):
2097
+ return 0
2098
+
2099
+ def spawn(argv, **kwargs):
2100
+ spawns.append(argv)
2101
+ return Child()
2102
+
2103
+ def crash_after_spawn(argv, **kwargs):
2104
+ spawns.append(argv)
2105
+ raise Crash
2106
+
2107
+ def crash_before_spawn(argv, **kwargs):
2108
+ raise Crash
2109
+
2110
+ def drain(popen, observe=lambda count: None):
2111
+ def stopped_writers(path):
2112
+ checks.append(path)
2113
+ observe(len(checks))
2114
+ with mock.patch.dict(helper, {"stopped_writers": stopped_writers}), mock.patch("subprocess.Popen", popen):
2115
+ try:
2116
+ return ensure_submission("l.t", records / "packet.json", packet, ["executor", "{packet}"])
2117
+ except Waiting as exc:
2118
+ return str(exc)
2119
+
2120
+ def survivor(count):
2121
+ if count < 3:
2122
+ raise helper["WriterActive"]("active process 4242 uses task files")
2123
+ Path(packet["submission"]).write_text('{"by": "executor"}', encoding="utf-8")
2124
+
2125
+ def unobservable(count):
2126
+ raise helper["WritersUnobservable"]("writer observation unsupported on this platform; retain workspace")
2127
+
2128
+ stderr = Path(packet["evidence_dir"]) / "executor.stderr"
2129
+ for crash, observe, outcome in ((crash_after_spawn, survivor, (1, 3, None)),
2130
+ (crash_before_spawn, lambda count: None,
2131
+ (1, 1, f"executor exited 0 without a submission (see {stderr}); the next drain runs it again")),
2132
+ (crash_before_spawn, unobservable, (0, 1, "interrupted-unobservable"))):
2133
+ with self.subTest(crash=crash.__name__, observe=getattr(observe, "__name__", "nothing")):
2134
+ spawns.clear()
2135
+ checks.clear()
2136
+ for path in records.iterdir():
2137
+ path.unlink()
2138
+ with self.assertRaises(Crash):
2139
+ drain(crash)
2140
+ failure = drain(spawn, observe)
2141
+ if crash is crash_after_spawn:
2142
+ # One observation per invocation, without sleep or a second spawn while the writer lives.
2143
+ self.assertEqual((len(spawns), len(checks)), (1, 1))
2144
+ self.assertIn("interrupted execution still active", failure)
2145
+ self.assertIn("interrupted execution still active", drain(spawn, observe))
2146
+ failure = drain(spawn, observe)
2147
+ self.assertEqual((len(spawns), len(checks), failure and failure.partition(": ")[0]), outcome)
2148
+ self.assertEqual(Path(packet["submission"]).exists(), crash is crash_after_spawn)
2149
+ if crash is crash_after_spawn:
2150
+ self.assertEqual(read_json(Path(packet["submission"])), {"by": "executor"})
2151
+
2152
+
2153
+ def main():
2154
+ parser = argparse.ArgumentParser(description=__doc__)
2155
+ parser.add_argument("--self-test", action="store_true")
2156
+ actions = parser.add_subparsers(dest="action")
2157
+ for name in ("check", "add"):
2158
+ actions.add_parser(name).add_argument("package", help="<repo>/docs/specs/<loop-id>/meta.md")
2159
+ actions.choices["add"].add_argument("--materialize", type=int, metavar="LINE", help="replace this pending legacy row")
2160
+ for name in ("status", "drain"):
2161
+ action = actions.add_parser(name)
2162
+ action.add_argument("--repo", default=".")
2163
+ action.add_argument("--local-only", "--no-push", action="store_true")
2164
+ inspector = actions.add_parser("inspect")
2165
+ inspector.add_argument("--repo", default=".")
2166
+ inspector.add_argument("--json", action="store_true")
2167
+ actions.choices["drain"].add_argument("--worktree-root")
2168
+ actions.choices["drain"].add_argument("executor", nargs="+", help="host executor argv after --; {packet} becomes the packet path")
2169
+ args = parser.parse_args()
2170
+ if args.self_test:
2171
+ result = unittest.TextTestRunner(verbosity=1).run(unittest.defaultTestLoader.loadTestsFromTestCase(QueueTests))
2172
+ if result.wasSuccessful():
2173
+ print(f"queue self-test: PASS ({result.testsRun} tests)")
2174
+ return 0 if result.wasSuccessful() else 1
2175
+ if args.action is None:
2176
+ parser.error("check, add, status, inspect or drain is required")
2177
+ if args.action == "inspect":
2178
+ os.environ["GIT_OPTIONAL_LOCKS"] = "0" # git diff may otherwise refresh the index even with unchanged bytes.
2179
+ try:
2180
+ result = {"check": check, "add": add, "status": status, "drain": drain, "inspect": inspect}[args.action](args)
2181
+ except (LoopError, OSError) as exc:
2182
+ result = {"status": "BLOCKED", "reason": str(exc)}
2183
+ if args.action in {"status", "drain"}:
2184
+ result.update(skill_diagnostics(Path(args.repo)))
2185
+ print(json.dumps(result, ensure_ascii=False, sort_keys=True))
2186
+ return 1 if result["status"] == "BLOCKED" else 0
2187
+
2188
+
2189
+ if __name__ == "__main__":
2190
+ shared("platform-support")["configure_utf8"]()
2191
+ sys.exit(main())