devlyn-cli 4.3.31 → 4.3.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +4 -2
  2. package/bin/devlyn.js +109 -22
  3. package/bin/harness-activation.js +4 -1
  4. package/bin/harness-status.js +15 -8
  5. package/bin/skill-history.json +10 -0
  6. package/config/skills/_shared/bring-in.py +10 -0
  7. package/config/skills/_shared/task-complete.py +787 -87
  8. package/config/skills/_shared/task-completion.md +6 -6
  9. package/config/skills/_shared/warm-build-dirs.py +13 -0
  10. package/config/skills/devlyn-ideate/SKILL.md +3 -2
  11. package/config/skills/devlyn-ideate/references/loop.md +2 -2
  12. package/config/skills/devlyn-ideate/scripts/acceptance.py +11 -0
  13. package/config/skills/devlyn-ideate/scripts/queue.py +685 -10
  14. package/native/claude/.claude-plugin/plugin.json +1 -1
  15. package/native/claude/skills/_shared/bring-in.py +10 -0
  16. package/native/claude/skills/_shared/task-complete.py +787 -87
  17. package/native/claude/skills/_shared/task-completion.md +6 -6
  18. package/native/claude/skills/_shared/warm-build-dirs.py +13 -0
  19. package/native/claude/skills/devlyn-ideate/SKILL.md +3 -2
  20. package/native/claude/skills/devlyn-ideate/references/loop.md +2 -2
  21. package/native/claude/skills/devlyn-ideate/scripts/acceptance.py +11 -0
  22. package/native/claude/skills/devlyn-ideate/scripts/queue.py +685 -10
  23. package/native/codex/.agents/plugins/marketplace.json +1 -1
  24. package/native/codex/plugins/devlyn/.codex-plugin/plugin.json +1 -1
  25. package/native/codex/plugins/devlyn/skills/_shared/bring-in.py +10 -0
  26. package/native/codex/plugins/devlyn/skills/_shared/task-complete.py +787 -87
  27. package/native/codex/plugins/devlyn/skills/_shared/task-completion.md +6 -6
  28. package/native/codex/plugins/devlyn/skills/_shared/warm-build-dirs.py +13 -0
  29. package/native/codex/plugins/devlyn/skills/devlyn-ideate/SKILL.md +3 -2
  30. package/native/codex/plugins/devlyn/skills/devlyn-ideate/references/loop.md +2 -2
  31. package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/acceptance.py +11 -0
  32. package/native/codex/plugins/devlyn/skills/devlyn-ideate/scripts/queue.py +685 -10
  33. package/package.json +1 -1
  34. package/release.json +35 -35
@@ -2,7 +2,8 @@
2
2
  """Ideate loop driver: package validation, the queue, locks, reconciliation and the serial drain.
3
3
 
4
4
  `check` validates a loop package, `add` captures it with its queue rows (which may
5
- replace one legacy row), `status` shows the reconciled queue and `drain -- <argv>` runs each
5
+ replace one legacy row), `withdraw` takes back a queued loop none of whose tasks has
6
+ started and that replaced no legacy row, `status` shows the reconciled queue and `drain -- <argv>` runs each
6
7
  eligible task through allocation, committed inputs, one host-supplied executor
7
8
  exchange, evidence-derived acceptance, the terminal transition and delivery.
8
9
  Protocol and commands: ../references/loop.md; formats: ../references/package-format.md.
@@ -159,15 +160,15 @@ def repository(path):
159
160
 
160
161
 
161
162
  @contextlib.contextmanager
162
- def lock(common, name, *, blocking):
163
- """Native locks on the common Gitdir: drain.lock for a controller's lifetime, queue.lock for short mutations."""
163
+ def lock(common, name, *, blocking, busy="a drain or a withdraw is running in this repository"):
164
+ """Native locks on the common Gitdir: drain.lock for a controller's lifetime or a withdraw, queue.lock for short mutations."""
164
165
  directory = common / "devlyn-loops"
165
166
  directory.mkdir(exist_ok=True)
166
167
  with contextlib.ExitStack() as stack:
167
168
  try:
168
169
  stack.enter_context(shared("platform-support")["file_lock"](directory / name, blocking=blocking))
169
170
  except BlockingIOError as exc:
170
- raise LoopError(f"another drain is active for this repository ({directory / name})") from exc
171
+ raise LoopError(f"{busy} ({directory / name})") from exc
171
172
  yield
172
173
 
173
174
 
@@ -235,6 +236,19 @@ def release_settled_binding(anchor, common, *, interrupted_add=False):
235
236
  return True
236
237
 
237
238
 
239
+ def refuse_other_release(common, anchor=None):
240
+ """Withdraw's refusal of a queue another release bound, made before enforce_binding hands out that release's argv or
241
+ handoff: a release older than withdraw has none. As there, a caller outside the queue lock first drops a binding no
242
+ work needs."""
243
+ binding = harness_call("read_binding", common)
244
+ if not binding or (invoking := harness_call("resolve", SKILLS)) and invoking["release"] == binding["release"] or (
245
+ anchor is not None and release_settled_binding(anchor, common, interrupted_add=True)):
246
+ return
247
+ release = binding["release"]
248
+ raise LoopError(f"this queue is bound to devlyn-cli {release['version']} at {release['root']}; withdraw waits until that "
249
+ "release's queue settles")
250
+
251
+
238
252
  def sections(text, path):
239
253
  found, current, fence = {}, None, False
240
254
  for line in text.split("\n"):
@@ -565,6 +579,28 @@ def receipt_path(common, identity):
565
579
  return retired[-1] if retired else active
566
580
 
567
581
 
582
+ def seen(path, *, entries=False):
583
+ """What withdraw sees at `path`: whether it exists, a dangling link included, or with `entries` the paths in it. Where
584
+ it or a folder above it is missing (ENOENT, ENOTDIR) it sees nothing; any other error refuses, naming the path, where
585
+ exists, is_file and glob would answer no."""
586
+ try:
587
+ found = list(path.iterdir()) if entries else [os.lstat(path)]
588
+ except (FileNotFoundError, NotADirectoryError):
589
+ found = []
590
+ except OSError as exc:
591
+ raise LoopError(f"{path} cannot be read ({exc.strerror or exc}), so withdraw cannot tell what it holds; withdraw changes "
592
+ "nothing until it can") from exc
593
+ return found if entries else bool(found)
594
+
595
+
596
+ def task_directories(common, identity):
597
+ """Every receipt directory of the task, a link included: the active one, then retired, then moved, the order in which
598
+ task-complete.py moves one, and a path once seen is kept, so a directory moving meanwhile is seen at least once."""
599
+ root, key = common / "devlyn-completion", hashlib.sha256(branch_of(identity).encode()).hexdigest()[:24]
600
+ return [*([root / key] if seen(root / key) else []),
601
+ *(path for folder in ("retired", "moved") for path in seen(root / folder, entries=True) if path.name.startswith(key + "."))]
602
+
603
+
568
604
  def added_path(common, loop):
569
605
  return common / "devlyn-loops" / loop / "added.json"
570
606
 
@@ -960,14 +996,39 @@ def next_task(v, attempted):
960
996
 
961
997
 
962
998
  def annotate(v, refused):
963
- """Name each task's wait: a pending task's reason from waiting(), else why this drain did not advance it."""
999
+ """Name each task's wait: a pending task's reason from waiting(), else why this drain did not advance it, else what an
1000
+ active task's executor left."""
964
1001
  for row in v["rows"]:
965
1002
  state = v["states"].get(row["identity"])
966
1003
  if state and (reason := refused.get(row["identity"]) or state["kind"] == "pending" and waiting(v, row)
967
- or recovery_barrier(v, row)):
1004
+ or recovery_barrier(v, row) or executor_left_nothing(state)):
968
1005
  state["waiting"] = reason
969
1006
 
970
1007
 
1008
+ def executor_left_nothing(state):
1009
+ """What an active task's executor left when its execution log ends in a start that failed, or in an exit while no
1010
+ submission is in place and the drain retired none that run wrote; nothing for a start, which may still be running."""
1011
+ if state["kind"] != "active" or {"invalid", "inputs_changed"} & state.keys() or (state["path"].parent / "submission.json").exists():
1012
+ return None
1013
+ folder = state["path"].parent
1014
+ try:
1015
+ events = [line.partition(" ")[::2] for line in (folder / "executions.log").read_bytes().decode("utf-8").splitlines()]
1016
+ last = events[-1][1] if events else ""
1017
+ code, starts = re.fullmatch(r"exit (-?\d+) \(pid \d+\)", last), [when for when, event in events if event == "start"]
1018
+ # A retired submission covers the one run that wrote it, after that run's start.
1019
+ if code and starts and any(path.stat().st_mtime >= datetime.datetime.fromisoformat(starts[-1]).timestamp()
1020
+ for path in folder.glob("submission.retired-*.json")):
1021
+ return None
1022
+ except (OSError, UnicodeError, ValueError):
1023
+ return None # Nothing ran yet, or a history the drain refuses, naming it, when it reads it.
1024
+ if code:
1025
+ return (f"its executor last exited {code[1]} (run {sum(event.startswith('exit ') for _, event in events)}) and left nothing to "
1026
+ f"accept (output: {Path(state['receipt']['worktree']) / '.devlyn/loop'})")
1027
+ if last.startswith("not started: "):
1028
+ return f"its executor could not start ({last.removeprefix('not started: ')})"
1029
+ return None
1030
+
1031
+
971
1032
  def frontier(v, loop):
972
1033
  """The loop's accepted task whose source contains every other accepted source. A local loop builds one history, though
973
1034
  not always in row order: a task whose unfinished allocation was removed starts again from a later row's source. Without
@@ -1429,6 +1490,8 @@ def advance(v, row, opts):
1429
1490
  progress(identity, f"delivery {result['status']}")
1430
1491
  if result.get("merge_refused"):
1431
1492
  return f"merge refused: {result['merge_refused']}; a person merges the PR"
1493
+ if waits := result.get("merge_waits"):
1494
+ return f"merge waits: PR is {waits} ({'GitHub has not computed its mergeability yet' if waits == 'UNKNOWN' else 'required checks or reviews not passing yet'}); drain again later, or merge it"
1432
1495
  return None
1433
1496
 
1434
1497
 
@@ -1657,11 +1720,15 @@ def status_summary(v):
1657
1720
  row = next_task(v, set())
1658
1721
  annotate(v, {})
1659
1722
  tasks = [summary(v, r) for r in v["rows"] if r["identity"]]
1723
+ # A task without a resume is no pending delivery, so a cleanup it retained is named here with its own resume.
1724
+ retained = [f"{t['identity']}: {label} {t[key]}" for t in tasks if not t.get("resume")
1725
+ for key, field, label in (("cleanup", "workspace_cleanup", "workspace cleanup"), ("scratch", "scratch_cleanup", "scratch cleanup"))
1726
+ if ((v["states"][t["identity"]]["receipt"] or {}).get(field) or {}).get("reason")]
1660
1727
  return {"status": "OK", **skill_diagnostics(anchor), "counts": counts(v), "next": row["identity"] if row else None,
1661
1728
  "blockers": [f"{t['identity']}: {t['reason']}" for t in tasks if t.get("reason") and t["result"] in {"pending", "active", "blocked"}]
1662
1729
  + [f"legacy row {r['index'] + 1} needs planning: {r['text']}" for r in v["rows"] if not r["identity"] and r["mark"] == " "]
1663
1730
  + [f"unreadable receipt {item}" for item in v["unreadable"]],
1664
- "delivery": [t for t in tasks if t.get("resume")], "cleanup": v["cleanup"]}
1731
+ "delivery": [t for t in tasks if t.get("resume")], "cleanup": v["cleanup"] + retained}
1665
1732
 
1666
1733
 
1667
1734
  def status(args):
@@ -1790,6 +1857,121 @@ def add(args):
1790
1857
  return added_result(anchor, added, notes)
1791
1858
 
1792
1859
 
1860
+ def registered_worktree(admin):
1861
+ """A linked worktree as Git lists it: the folder whose .git the gitdir file of its administrative directory
1862
+ <common>/worktrees/<id> names, as an absolute path or one relative to that directory; while that file cannot be read
1863
+ or decoded, the directory itself."""
1864
+ try:
1865
+ gitdir = (admin / "gitdir").read_bytes().decode("utf-8").rstrip()
1866
+ except (OSError, UnicodeError):
1867
+ gitdir = ""
1868
+ if not gitdir:
1869
+ return f"the worktree registered at {admin}"
1870
+ folder = gitdir.removesuffix("/.git")
1871
+ return folder if os.path.isabs(folder) else os.path.realpath(admin / folder)
1872
+
1873
+
1874
+ def tracked_places(anchor, common, rel):
1875
+ """Where this repository tracks `rel`: the HEAD of each registered worktree, each local branch none has checked out,
1876
+ and the index of each registered worktree, read through its administrative directory, so also while its folder is
1877
+ missing, with a pathspec anchored at the top, since `git -C <common>` runs inside the work tree once core.worktree
1878
+ names the folder that holds the Git directory, and with the file system monitor off, which Git would start for the
1879
+ Git directory it takes as the work tree and wait on. An administrative directory without an index is skipped; one that
1880
+ cannot be read refuses (seen), and so does an index git cannot read, naming its worktree. The HEADs and branches are
1881
+ read refusing whatever Git writes beside its answer, since it skips a ref or an object it cannot read with only a
1882
+ warning, after each folder under refs/heads is listed, a link included, which Git follows, since for-each-ref skips
1883
+ one it cannot list without a word. The listing goes depth first, so links that loop reach the system's own link
1884
+ limit at once and are refused (seen); breadth first, two such links would never end, nor does Git's own listing of
1885
+ them, which passes over a folder it cannot list."""
1886
+
1887
+ def read(*args, **kwargs):
1888
+ result = git_run(anchor, *args, **kwargs)
1889
+ require(not result.stderr, f'git {args[0]} wrote "{" ".join(result.stderr.decode("utf-8", "replace").split())}" while reading '
1890
+ f"this repository's HEADs and branches, so withdraw cannot tell whether {rel} is tracked there; withdraw changes "
1891
+ "nothing until it can")
1892
+ return result.stdout.decode("utf-8").strip()
1893
+ # A gitdir file may hold any bytes, and the listed paths only name worktrees in a refusal.
1894
+ listing = git_run(anchor, "worktree", "list", "--porcelain", "-z").stdout.decode("utf-8", "replace")
1895
+ worktrees = [row for raw in listing.split("\0\0")
1896
+ if "worktree" in (row := dict(field.partition(" ")[::2] for field in raw.split("\0") if field))]
1897
+ checked_out = {row.get("branch") for row in worktrees}
1898
+ folders = [common / "refs/heads"] # Missing where every ref is packed or in a reftable; a ref file lists nothing.
1899
+ while folders:
1900
+ folders += seen(folders.pop(), entries=True)
1901
+ commits = [(f"HEAD of {row['worktree']}", row["HEAD"]) for row in worktrees if "HEAD" in row] + [
1902
+ (f"the branch {ref.removeprefix('refs/heads/')}", ref)
1903
+ for ref in read("for-each-ref", "--format=%(refname)", "refs/heads/").splitlines() if ref not in checked_out]
1904
+ kinds = read("cat-file", "--batch-check=%(objecttype)", data="".join(f"{rev}:{rel}\n" for _, rev in commits).encode())
1905
+ places = [place for (place, _), kind in zip(commits, kinds.split("\n")) if kind == "blob"]
1906
+ linked = sorted(admin for admin in seen(common / "worktrees", entries=True) if seen(admin / "index"))
1907
+ for name, admin in [(worktrees[0]["worktree"], common), *((registered_worktree(admin), admin) for admin in linked)]:
1908
+ listed = subprocess.run(["git", "-c", "core.fsmonitor=false", "-C", str(common), f"--git-dir={admin}", "ls-files", "--",
1909
+ f":(top,literal){rel}"], capture_output=True)
1910
+ require(listed.returncode == 0, f"the index of {name} cannot be read ({' '.join(listed.stderr.decode('utf-8', 'replace').split())}), "
1911
+ f"so withdraw cannot tell whether {rel} is tracked there; withdraw takes back only a loop held in its capture")
1912
+ if listed.stdout:
1913
+ places.append(f"the index of {name}")
1914
+ return places
1915
+
1916
+
1917
+ def withdraw(args):
1918
+ """Take back a queued loop none of whose tasks has started and that replaced no legacy row: its add record, then its
1919
+ folder in devlyn-loops, then its capture ref; a retry after an interruption finishes the rest. Under the drain lock,
1920
+ so no drain allocates it."""
1921
+ anchor, common = repository(args.repo)
1922
+ args.repo = anchor
1923
+ loop, notes = args.loop, []
1924
+ ref, rel = CAPTURES + loop, loop_queue(loop)
1925
+ require(re.fullmatch(ID, loop), f"loop id must match {ID}: {loop!r}")
1926
+ refuse_other_release(common, anchor)
1927
+ enforce_binding(common, anchor)
1928
+ busy = ("a drain or another withdraw is running in this repository; a loop cannot be withdrawn while a drain runs, and one "
1929
+ "added during a drain runs in it")
1930
+ with lock(common, "drain.lock", blocking=False, busy=busy), lock(common, "queue.lock", blocking=True):
1931
+ refuse_other_release(common)
1932
+ enforce_binding(common)
1933
+ added, captured = records(common).get(loop), ref_value(anchor, ref)
1934
+ # records() lists them with glob, which skips a folder it cannot read, so this loop's own record is looked at too.
1935
+ require(added or not seen(added_path(common, loop)), f"{common / 'devlyn-loops'} cannot be listed, so withdraw cannot read "
1936
+ f"the add record {added_path(common, loop)}; withdraw changes nothing until it can")
1937
+ commit = added["commit"] if added else captured
1938
+ rows, target = parse_loop(show(anchor, commit, rel), loop, f"{ref}:{rel}") if commit else ([], None)
1939
+ tasks = [row["identity"] for row in rows]
1940
+ if target:
1941
+ claim = f'the {QUEUE} row "{one_line(target[1])}" (occurrence {target[0]})'
1942
+ only = "withdraw takes back only a loop that replaced no legacy row"
1943
+ raise LoopError(f"loop {loop} was added in place of {claim}; {only}; it stays queued" if added else
1944
+ f"the add of loop {loop} in place of {claim} stopped before its record; {only}; its capture stays at {ref}")
1945
+ # Without its record a loop is read from a tracked queue file (queue_view), so its rows would come back.
1946
+ places = tracked_places(anchor, common, rel)
1947
+ require(not places, f"{rel} is tracked in {', '.join(places)}, so loop {loop}'s rows would return without its add record; "
1948
+ "withdraw takes back only a loop held in its capture")
1949
+ started = [f"{identity} at {path}" for identity in tasks for path in task_directories(common, identity)]
1950
+ require(not started, f"loop {loop} has a started task ({'; '.join(started)}); withdraw takes back only a loop none of whose "
1951
+ "tasks has started")
1952
+ if added:
1953
+ require(captured == commit, f"{ref} is {captured or 'missing'}, but the add record of {loop} names {commit}")
1954
+ added_path(common, loop).unlink()
1955
+ if commit:
1956
+ folder = common / "devlyn-loops" / loop
1957
+ try:
1958
+ (folder / "drain-report.md").unlink(missing_ok=True)
1959
+ with contextlib.suppress(FileNotFoundError):
1960
+ folder.rmdir()
1961
+ except OSError as exc:
1962
+ notes.append(f"loop folder retained: {folder}: {exc}; remove it")
1963
+ git(anchor, "update-ref", "-d", ref, commit)
1964
+ try:
1965
+ release_settled_binding(anchor, common, interrupted_add=True)
1966
+ except (LoopError, OSError) as exc:
1967
+ notes.append(f"queue binding not released: {exc}; withdraw {loop} again to release it")
1968
+ if not commit:
1969
+ return {"status": "BLOCKED", "reason": f"loop {loop} is not queued in this repository", "cleanup": notes}
1970
+ return {"status": "WITHDRAWN", "tasks": tasks, "commit": commit, "cleanup": notes,
1971
+ "message": f"`git show {commit}:docs/specs/{loop}/meta.md` shows the withdrawn package until Git prunes that commit. "
1972
+ "Revised work is a new loop."}
1973
+
1974
+
1793
1975
  def check(args):
1794
1976
  anchor, _ = repository(Path(args.package).resolve().parent)
1795
1977
  package = package_of(anchor, args.package)
@@ -1897,6 +2079,8 @@ class QueueTests(unittest.TestCase):
1897
2079
  "import shutil\nfrom types import SimpleNamespace\n"
1898
2080
  "shutil.disk_usage = lambda path: SimpleNamespace(total=100 * 1024**3, free=100 * 1024**3)\n",
1899
2081
  encoding="utf-8")
2082
+ # A commit's detached automatic maintenance would still write into .git while a test reads the tree.
2083
+ (self.root / "gitconfig").write_text("[maintenance]\n\tauto = false\n", encoding="utf-8")
1900
2084
  self.env = {**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": str(self.root / "gitconfig"),
1901
2085
  "GIT_AUTHOR_NAME": "Fixture", "GIT_AUTHOR_EMAIL": "fixture@example.invalid",
1902
2086
  "GIT_COMMITTER_NAME": "Fixture", "GIT_COMMITTER_EMAIL": "fixture@example.invalid"}
@@ -1940,6 +2124,14 @@ class QueueTests(unittest.TestCase):
1940
2124
  path.write_text(text.replace(old, new, 1), encoding="utf-8")
1941
2125
  return mutate
1942
2126
 
2127
+ def test_fixture_commit_starts_no_automatic_maintenance(self):
2128
+ # Its detached child would still write into .git while a test reads the tree or cleanup removes it.
2129
+ trace = self.root / "trace.json"
2130
+ subprocess.run(["git", "-C", str(self.anchor), "commit", "-q", "--allow-empty", "-m", "traced"], check=True,
2131
+ env={**self.env, "GIT_TRACE2_EVENT": str(trace)})
2132
+ children = [line for line in trace.read_text(encoding="utf-8").splitlines() if '"event":"child_start"' in line]
2133
+ self.assertEqual(children, [])
2134
+
1943
2135
  def test_package_validation(self):
1944
2136
  self.assertEqual(self.cli("check", self.meta)["tasks"], ["inv.t1", "inv.t2"])
1945
2137
  manifest = json.loads(JSON_FENCE_RE.search((self.anchor / "docs/specs/inv/meta.md").read_text(encoding="utf-8")).group(1))
@@ -2106,6 +2298,486 @@ class QueueTests(unittest.TestCase):
2106
2298
  self.assertEqual((sorted(notes), all(path.exists() for path in saved)), (sorted(f"package cleanup retained: {path}: unlink denied" for path in saved), True))
2107
2299
  self.assertEqual((clean(common, records(common)), any(path.exists() for path in saved)), ([], False))
2108
2300
 
2301
+ def test_withdraw_takes_back_a_loop_none_of_whose_tasks_started(self):
2302
+ """D1. Prediction: withdrawing inv, added and never started, prints WITHDRAWN naming inv.t1, inv.t2 and the capture
2303
+ commit, and removes the add record, the capture ref and the loop's folder with its drain report, keeping
2304
+ devlyn-loops; the queue then holds no row of inv and nothing unfinished, and a second withdraw is refused as not
2305
+ queued. Before: argparse rejected withdraw."""
2306
+ added = self.cli("add", self.meta)
2307
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2308
+ (common / "devlyn-loops/inv/drain-report.md").write_text("# Drain report — inv\n", encoding="utf-8")
2309
+ result = self.cli("withdraw", "inv")
2310
+ self.assertEqual((result["status"], result["tasks"], result["commit"]), ("WITHDRAWN", ["inv.t1", "inv.t2"], added["commit"]))
2311
+ self.assertIn(f"git show {added['commit']}:docs/specs/inv/meta.md", result["message"])
2312
+ self.assertEqual((ref_value(self.anchor, "refs/devlyn/captures/inv"), sorted(path.name for path in (common / "devlyn-loops").iterdir())),
2313
+ ("", ["drain.lock", "queue.lock"]))
2314
+ self.assertEqual((queue_view(self.anchor, records(common)), unfinished(view(self.anchor, common, pure=True))), ([], False))
2315
+ self.assertIn("loop inv is not queued in this repository", self.cli("withdraw", "inv", code=1)["reason"])
2316
+
2317
+ def test_withdraw_refuses_without_changing_anything(self):
2318
+ """D1, E6. Prediction: each refusal leaves the add record and the drain report byte for byte, the capture ref, the
2319
+ loop's folder, HEAD and the index as they were: a task directory of either task, active, retired or moved, a dangling
2320
+ link included; a running drain or another withdraw, refused in withdraw's own words; a capture ref moved off the
2321
+ recorded commit, naming both; and the loop's queue file tracked in HEAD here, in the index here while core.worktree
2322
+ names this folder, in the index of the worktree the loop was added from, in HEAD and the index of a third worktree,
2323
+ in the index of a locked worktree whose folder is gone or holds only an empty directory, on a branch no worktree has
2324
+ checked out, or on the branch it was added on once that worktree is gone. Before: argparse rejected withdraw; then
2325
+ the third worktree and the other branch were withdrawn, and the busy refusal named only a drain; then the locked
2326
+ worktree without its folder was withdrawn, and with an empty directory there git's own failure was the refusal;
2327
+ then, with core.worktree set, the index here was read from inside the Git directory, where Git took the pathspec
2328
+ as .git/docs/..., and the loop was withdrawn."""
2329
+ import shutil
2330
+ linked, third, gone = self.root / "linked", self.root / "third", self.root / "gone"
2331
+ git(self.anchor, "worktree", "add", "-q", "-b", "side", str(linked), env=self.env)
2332
+ added = self.cli("add", write_package(linked, "inv", self.tasks))
2333
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2334
+ (common / "devlyn-loops/inv/drain-report.md").write_text("# Drain report — inv\n", encoding="utf-8")
2335
+ ref, rel, completion, head = "refs/devlyn/captures/inv", "docs/specs/inv/queue.md", common / "devlyn-completion", git(self.anchor, "rev-parse", "HEAD")
2336
+ key = {identity: hashlib.sha256(branch_of(identity).encode()).hexdigest()[:24] for identity in ("inv.t1", "inv.t2")}
2337
+ active, link = completion / key["inv.t1"], completion / key["inv.t2"]
2338
+ retired, moved = completion / "retired" / f"{key['inv.t2']}.operation", completion / "moved" / f"{key['inv.t1']}.operation"
2339
+
2340
+ def state():
2341
+ return (git(self.anchor, "for-each-ref"), git(self.anchor, "rev-parse", "HEAD"), git(self.anchor, "ls-files", "-s"),
2342
+ git(linked, "ls-files", "-s") if linked.exists() else None,
2343
+ sorted((str(path.relative_to(common)), path.read_bytes() if path.is_file() else None)
2344
+ for path in (common / "devlyn-loops").rglob("*") if path.suffix != ".lock"))
2345
+
2346
+ def track(work, *commit):
2347
+ (work / rel).parent.mkdir(parents=True, exist_ok=True)
2348
+ (work / rel).write_text(row_line("inv.t1", "Interface") + "\n", encoding="utf-8")
2349
+ git(work, "add", rel, env=self.env)
2350
+ if commit:
2351
+ git(work, "commit", "-qm", "a queue file committed by hand", env=self.env)
2352
+
2353
+ def untrack(work):
2354
+ git(work, "rm", "-q", "--cached", rel, env=self.env)
2355
+ (work / rel).unlink()
2356
+
2357
+ def elsewhere(*start):
2358
+ git(self.anchor, "worktree", "add", "-q", *start, str(third), env=self.env)
2359
+ track(third, "commit")
2360
+ if start != ("--detach",):
2361
+ git(self.anchor, "worktree", "remove", "--force", str(third), env=self.env)
2362
+
2363
+ def offline(empty):
2364
+ git(self.anchor, "worktree", "add", "-q", "--detach", str(gone), env=self.env)
2365
+ track(gone)
2366
+ git(self.anchor, "worktree", "lock", str(gone), env=self.env)
2367
+ shutil.rmtree(gone)
2368
+ if empty:
2369
+ gone.mkdir()
2370
+
2371
+ def online():
2372
+ git(self.anchor, "worktree", "unlock", str(gone), env=self.env)
2373
+ if gone.exists():
2374
+ gone.rmdir()
2375
+ git(self.anchor, "worktree", "prune", env=self.env)
2376
+
2377
+ def own_folder(): # A config that names the repository's own folder as its work tree.
2378
+ git(self.anchor, "config", "core.worktree", str(self.anchor), env=self.env)
2379
+ track(self.anchor)
2380
+
2381
+ def no_worktree_config():
2382
+ untrack(self.anchor)
2383
+ git(self.anchor, "config", "--unset", "core.worktree", env=self.env)
2384
+
2385
+ cases = [("active", lambda: active.mkdir(parents=True), active.rmdir, f"inv.t1 at {active}"),
2386
+ ("retired", lambda: retired.mkdir(parents=True), retired.rmdir, f"inv.t2 at {retired}"),
2387
+ ("moved", lambda: moved.mkdir(parents=True), moved.rmdir, f"inv.t1 at {moved}"),
2388
+ ("ref", lambda: git(self.anchor, "update-ref", ref, head), lambda: git(self.anchor, "update-ref", ref, added["commit"]),
2389
+ f"{ref} is {head}, but the add record of inv names {added['commit']}"),
2390
+ ("HEAD here", lambda: track(self.anchor, "commit"), lambda: git(self.anchor, "reset", "-q", "--hard", head),
2391
+ f"{rel} is tracked in HEAD of {self.anchor}"),
2392
+ ("index here, core.worktree set", own_folder, no_worktree_config, f"{rel} is tracked in the index of {self.anchor}, so"),
2393
+ ("index of origin", lambda: track(linked), lambda: untrack(linked), f"{rel} is tracked in the index of {linked}"),
2394
+ ("third worktree", lambda: elsewhere("--detach"), lambda: git(self.anchor, "worktree", "remove", "--force", str(third), env=self.env),
2395
+ f"{rel} is tracked in HEAD of {third}, the index of {third}, so"),
2396
+ ("offline, empty folder", lambda: offline(True), online, f"{rel} is tracked in the index of {gone}, so"),
2397
+ ("offline", lambda: offline(False), online, f"{rel} is tracked in the index of {gone}, so"),
2398
+ ("other branch", lambda: elsewhere("-b", "elsewhere"), lambda: git(self.anchor, "branch", "-D", "elsewhere", env=self.env),
2399
+ f"{rel} is tracked in the branch elsewhere, so")]
2400
+ if os.name != "nt":
2401
+ cases.append(("link", lambda: link.symlink_to(self.root / "nowhere"), link.unlink, f"inv.t2 at {link}"))
2402
+ for name, prepare, undo, message in cases:
2403
+ with self.subTest(case=name):
2404
+ prepare()
2405
+ before = state()
2406
+ self.assertIn(message, self.cli("withdraw", "inv", code=1)["reason"])
2407
+ self.assertEqual(state(), before)
2408
+ undo()
2409
+ with lock(common, "drain.lock", blocking=True):
2410
+ before = state()
2411
+ self.assertIn("a drain or another withdraw is running in this repository; a loop cannot be withdrawn while a drain runs, "
2412
+ "and one added during a drain runs in it", self.cli("withdraw", "inv", code=1)["reason"])
2413
+ self.assertEqual(state(), before)
2414
+ track(linked, "commit")
2415
+ git(self.anchor, "worktree", "remove", "--force", str(linked), env=self.env)
2416
+ before = state()
2417
+ self.assertIn(f"{rel} is tracked in the branch side", self.cli("withdraw", "inv", code=1)["reason"])
2418
+ self.assertEqual(state(), before)
2419
+
2420
+ def test_a_drain_started_during_a_withdraw_says_a_withdraw_may_hold_its_lock(self):
2421
+ """D1. Prediction: a drain started while a withdraw, paused inside its locks, holds drain.lock is refused, its
2422
+ reason naming a drain or a withdraw and the lock; the withdraw then finishes WITHDRAWN. Before: the reason said
2423
+ another drain was active, though only a withdraw ran."""
2424
+ import threading
2425
+ from unittest import mock
2426
+ self.cli("add", self.meta)
2427
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2428
+ inside, resume, result, real = threading.Event(), threading.Event(), {}, tracked_places
2429
+
2430
+ def paused(*args):
2431
+ inside.set()
2432
+ resume.wait(60)
2433
+ return real(*args)
2434
+
2435
+ with mock.patch.dict(withdraw.__globals__, tracked_places=paused):
2436
+ running = threading.Thread(target=lambda: result.update(withdraw(argparse.Namespace(repo=str(self.anchor), loop="inv"))))
2437
+ running.start()
2438
+ try:
2439
+ self.assertTrue(inside.wait(60))
2440
+ reason = self.cli("drain", "--", sys.executable, "-c", "pass", "{packet}", code=1)["reason"]
2441
+ finally:
2442
+ resume.set()
2443
+ running.join(60)
2444
+ self.assertEqual(reason, f"a drain or a withdraw is running in this repository ({common / 'devlyn-loops/drain.lock'})")
2445
+ self.assertEqual(result.get("status"), "WITHDRAWN")
2446
+
2447
+ def test_withdraw_finishes_after_an_interruption_between_its_deletions(self):
2448
+ """D1. Prediction: a withdraw interrupted after it removed the add record, before the capture ref, leaves that ref,
2449
+ which still counts as unfinished work; the retry removes it and prints WITHDRAWN with the same tasks and commit.
2450
+ Before: there was no withdraw."""
2451
+ from unittest import mock
2452
+ added = self.cli("add", self.meta)
2453
+ common, real = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir")), git
2454
+
2455
+ def interrupted(cwd, *args, **kwargs):
2456
+ if args[:2] == ("update-ref", "-d"):
2457
+ raise LoopError("interrupted before the capture ref")
2458
+ return real(cwd, *args, **kwargs)
2459
+
2460
+ with mock.patch.dict(withdraw.__globals__, git=interrupted), self.assertRaisesRegex(LoopError, "interrupted before"):
2461
+ withdraw(argparse.Namespace(repo=str(self.anchor), loop="inv"))
2462
+ self.assertEqual((added_path(common, "inv").exists(), ref_value(self.anchor, "refs/devlyn/captures/inv"),
2463
+ unfinished(view(self.anchor, common, pure=True))), (False, added["commit"], True))
2464
+ result = self.cli("withdraw", "inv")
2465
+ self.assertEqual((result["status"], result["tasks"], result["commit"]), ("WITHDRAWN", ["inv.t1", "inv.t2"], added["commit"]))
2466
+ self.assertEqual(ref_value(self.anchor, "refs/devlyn/captures/inv"), "")
2467
+
2468
+ def test_withdraw_refuses_a_loop_that_replaced_a_legacy_row(self):
2469
+ """D1, E6. Prediction: a loop added with --materialize is refused, naming the row its capture replaced and that row's
2470
+ occurrence and saying the loop stays queued, while the row is in the legacy queue and once no file holds it; the
2471
+ capture an add stopped before its record left is refused, saying that add stopped before its record and where its
2472
+ capture stays; each refusal leaves the add record, the capture ref, the loop's folder, HEAD and the index byte for
2473
+ byte. A loop added without --materialize is withdrawn. Before: withdraw looked for the row only here, in the
2474
+ checkout the loop was added from and on the branch it was added on, and withdrew the loop once those no longer held
2475
+ it, though another worktree's legacy queue could still show the row, pending, to be planned again; then the refusal
2476
+ said withdrawing would bring the row back, which no file holding it, or a capture that never hid it, makes false;
2477
+ then it said that capture's loop was added and stays queued, though status shows its row pending and no loop."""
2478
+ from unittest import mock
2479
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2480
+
2481
+ def state():
2482
+ return (git(self.anchor, "for-each-ref"), git(self.anchor, "rev-parse", "HEAD"), git(self.anchor, "ls-files", "-s"),
2483
+ sorted((str(path.relative_to(common)), path.read_bytes() if path.is_file() else None)
2484
+ for path in (common / "devlyn-loops").rglob("*") if path.suffix != ".lock"))
2485
+
2486
+ def refused(loop, row, recorded=True):
2487
+ claim = f'the {QUEUE} row "{row}" (occurrence 1)'
2488
+ before = state()
2489
+ self.assertEqual(self.cli("withdraw", loop, code=1)["reason"],
2490
+ f"loop {loop} was added in place of {claim}; withdraw takes back only a loop that replaced no legacy "
2491
+ "row; it stays queued" if recorded else
2492
+ f"the add of loop {loop} in place of {claim} stopped before its record; withdraw takes back only a "
2493
+ f"loop that replaced no legacy row; its capture stays at refs/devlyn/captures/{loop}")
2494
+ self.assertEqual(state(), before)
2495
+
2496
+ (self.anchor / QUEUE).write_text("# Intent Queue\n\n- [ ] Ship the fixture.\n- [ ] Ship the second fixture.\n", encoding="utf-8")
2497
+ self.cli("add", write_package(self.anchor, "rep", self.tasks[:1], intent="User asked: Ship the fixture."), "--materialize", 3)
2498
+ (common / "devlyn-loops/rep/drain-report.md").write_text("# Drain report — rep\n", encoding="utf-8")
2499
+ refused("rep", "- [ ] Ship the fixture.")
2500
+ (self.anchor / QUEUE).write_text("# Intent Queue\n\n- [ ] Ship the second fixture.\n", encoding="utf-8")
2501
+ refused("rep", "- [ ] Ship the fixture.")
2502
+ self.cli("add", self.meta)
2503
+ self.assertEqual(self.cli("withdraw", "inv")["tasks"], ["inv.t1", "inv.t2"])
2504
+ meta, real = write_package(self.anchor, "ref", self.tasks[:1], intent="User asked: Ship the second fixture."), write_json
2505
+
2506
+ def stopped(path, value):
2507
+ if Path(path).name == "added.json":
2508
+ raise LoopError("stopped before the add record")
2509
+ real(path, value)
2510
+
2511
+ with mock.patch.dict(os.environ, self.env), mock.patch.dict(add.__globals__, write_json=stopped), \
2512
+ self.assertRaisesRegex(LoopError, "stopped before"):
2513
+ add(argparse.Namespace(package=str(meta), materialize=3))
2514
+ self.assertEqual((bool(ref_value(self.anchor, "refs/devlyn/captures/ref")), added_path(common, "ref").exists()), (True, False))
2515
+ refused("ref", "- [ ] Ship the second fixture.", recorded=False)
2516
+
2517
+ def test_withdraw_sees_a_task_directory_that_moves_while_it_looks(self):
2518
+ """D1, E6. Prediction: a task directory that task-complete moves into moved/, from the active path or from retired/,
2519
+ just after withdraw has listed moved/, still refuses the withdraw, naming where it was seen, and the add record and
2520
+ the capture ref stay. Before: withdraw listed retired/ and moved/ before it looked at the active path and dropped a
2521
+ listed path that had moved since, so it saw no directory and withdrew the loop."""
2522
+ from unittest import mock
2523
+ added = self.cli("add", self.meta)
2524
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2525
+ completion, key = common / "devlyn-completion", hashlib.sha256(branch_of("inv.t1").encode()).hexdigest()[:24]
2526
+ destination, real = completion / "moved" / f"{key}.operation", Path.iterdir
2527
+ destination.parent.mkdir(parents=True)
2528
+ for start in (completion / key, completion / "retired" / f"{key}.operation"):
2529
+ with self.subTest(start=start.parent.name):
2530
+ start.mkdir(parents=True)
2531
+
2532
+ def iterdir(path):
2533
+ found = list(real(path))
2534
+ if path.name == "moved" and start.exists():
2535
+ start.rename(destination)
2536
+ return iter(found)
2537
+
2538
+ with mock.patch.object(Path, "iterdir", iterdir), self.assertRaisesRegex(LoopError, re.escape(f"inv.t1 at {start}")):
2539
+ withdraw(argparse.Namespace(repo=str(self.anchor), loop="inv"))
2540
+ self.assertEqual((added_path(common, "inv").exists(), ref_value(self.anchor, "refs/devlyn/captures/inv")), (True, added["commit"]))
2541
+ destination.rmdir()
2542
+
2543
+ def test_withdraw_of_a_loop_it_does_not_hold_deletes_nothing(self):
2544
+ """D1. Prediction: a loop with no add record and no capture ref, committed only in a linked worktree, keeps its drain
2545
+ report and its folder under the common Gitdir: withdraw from this checkout refuses, naming that worktree's HEAD and
2546
+ index; once the worktree and its branch are gone the loop is not queued, and the report and folder still stay.
2547
+ Before: the first withdraw said not queued, and before f02e2c60 it deleted the report and the folder."""
2548
+ linked = self.root / "linked"
2549
+ git(self.anchor, "worktree", "add", "-q", "-b", "side", str(linked), env=self.env)
2550
+ write_package(linked, "rl", self.tasks[:1])
2551
+ rel = loop_queue("rl")
2552
+ (linked / rel).write_text(row_line("rl.t1", "Interface") + "\n", encoding="utf-8")
2553
+ git(linked, "add", "docs/specs/rl", env=self.env)
2554
+ git(linked, "commit", "-qm", "a loop committed without an add, as a fresh clone holds it", env=self.env)
2555
+ report = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir")) / "devlyn-loops/rl/drain-report.md"
2556
+ report.parent.mkdir(parents=True)
2557
+ report.write_text("# Drain report — rl\n", encoding="utf-8")
2558
+ self.assertIn(f"{rel} is tracked in HEAD of {linked}, the index of {linked}, so", self.cli("withdraw", "rl", code=1)["reason"])
2559
+ self.assertEqual(report.read_text(encoding="utf-8"), "# Drain report — rl\n")
2560
+ git(self.anchor, "worktree", "remove", "--force", str(linked), env=self.env)
2561
+ git(self.anchor, "branch", "-D", "side", env=self.env)
2562
+ self.assertIn("loop rl is not queued in this repository", self.cli("withdraw", "rl", code=1)["reason"])
2563
+ self.assertEqual((sorted(path.name for path in report.parent.iterdir()), report.read_text(encoding="utf-8")),
2564
+ (["drain-report.md"], "# Drain report — rl\n"))
2565
+
2566
+ def test_withdraw_reads_past_prunable_missing_and_bare_worktrees(self):
2567
+ """D1, E6. Prediction: withdraw prints WITHDRAWN beside a registered worktree whose folder stays without its .git
2568
+ file (Git lists it prunable), a locked worktree whose folder is gone, neither index holding the loop's queue
2569
+ file, a dangling symbolic ref under refs/heads, which Git lists nowhere, and, where links are made (not Windows),
2570
+ a link there that dangles, which Git skips, and one at a regular file holding a commit, which Git reads as a ref
2571
+ file, and in the linked worktree of a bare clone, whose bare row has no HEAD line and whose common Gitdir has no
2572
+ index. git ls-files run in either folder fails; without the HEAD check the bare row raises KeyError."""
2573
+ import shutil
2574
+ prunable, gone, bare, work = self.root / "prunable", self.root / "gone", self.root / "bare.git", self.root / "work"
2575
+ for path in (prunable, gone):
2576
+ git(self.anchor, "worktree", "add", "-q", "--detach", str(path), env=self.env)
2577
+ (prunable / ".git").unlink()
2578
+ git(self.anchor, "worktree", "lock", str(gone), env=self.env)
2579
+ shutil.rmtree(gone)
2580
+ git(self.anchor, "symbolic-ref", "refs/heads/dangle", "refs/heads/nowhere", env=self.env)
2581
+ if os.name != "nt":
2582
+ (self.root / "commit").write_text(git(self.anchor, "rev-parse", "main", env=self.env) + "\n", encoding="utf-8")
2583
+ for name, target in (("lost", self.root / "nowhere"), ("filed", self.root / "commit")):
2584
+ (self.anchor / ".git/refs/heads" / name).symlink_to(target)
2585
+ self.cli("add", self.meta)
2586
+ self.assertEqual(self.cli("withdraw", "inv")["status"], "WITHDRAWN")
2587
+ subprocess.run(["git", "clone", "-q", "--bare", str(self.anchor), str(bare)], check=True, env=self.env)
2588
+ git(bare, "worktree", "add", "-q", str(work), "main", env=self.env)
2589
+ self.cli("add", write_package(work, "inv", self.tasks), cwd=work)
2590
+ self.assertEqual(self.cli("withdraw", "inv", cwd=work)["status"], "WITHDRAWN")
2591
+
2592
+ def test_withdraw_reads_each_registered_index_however_its_worktree_is_named(self):
2593
+ """D1, E6. Prediction: beside a registered worktree whose administrative gitdir file is missing, or holds bytes that
2594
+ are not UTF-8, a loop tracked nowhere is withdrawn, and with the loop's queue file staged in that worktree's index
2595
+ it is refused, naming the worktree by its administrative directory; a worktree whose gitdir file holds a relative
2596
+ path is named as `git worktree list` prints it; an index git cannot read is refused in one sentence naming its
2597
+ worktree and git's message; a file or an empty directory beside the administrative directories, which Git skips,
2598
+ is skipped too. No refusal changes the add record or the capture ref. Before: the missing file stopped every
2599
+ withdraw with a bare FileNotFoundError, the bytes raised an uncaught UnicodeDecodeError, and the unreadable index
2600
+ was refused in git's words alone; then the empty directory stopped every withdraw as an index git cannot read."""
2601
+ import shutil
2602
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2603
+ rel, work = loop_queue("inv"), self.root / "work"
2604
+ admin = common / "worktrees" / work.name
2605
+ admin.parent.mkdir()
2606
+ (admin.parent / ".DS_Store").write_bytes(b"") # As a file manager leaves one.
2607
+ (admin.parent / "empty").mkdir() # Git lists no worktree for it, and git worktree prune deletes it.
2608
+
2609
+ def state():
2610
+ return (git(self.anchor, "for-each-ref"), sorted((str(path.relative_to(common)), path.read_bytes() if path.is_file() else None)
2611
+ for path in (common / "devlyn-loops").rglob("*") if path.suffix != ".lock"))
2612
+
2613
+ def by_admin():
2614
+ return f"the worktree registered at {admin}"
2615
+
2616
+ def as_listed():
2617
+ return git(self.anchor, "worktree", "list", "--porcelain").split("\n\n")[1].splitlines()[0].removeprefix("worktree ")
2618
+
2619
+ for name, options, damage, named in (("gitdir missing", (), lambda: (admin / "gitdir").unlink(), by_admin),
2620
+ ("gitdir not UTF-8", (), lambda: (admin / "gitdir").write_bytes(b"\xff\xfe/.git\n"), by_admin),
2621
+ ("gitdir relative", ("--relative-paths",), lambda: None, as_listed)):
2622
+ with self.subTest(case=name):
2623
+ if git_run(self.anchor, "worktree", "add", "-q", "--detach", *options, str(work), env=self.env, ok=(0, 129)).returncode:
2624
+ self.skipTest("this git writes no relative gitdir file (git worktree add --relative-paths came in Git 2.48)")
2625
+ try:
2626
+ damage()
2627
+ self.cli("add", write_package(self.anchor, "inv", self.tasks))
2628
+ self.assertEqual(self.cli("withdraw", "inv")["status"], "WITHDRAWN")
2629
+ self.cli("add", write_package(self.anchor, "inv", self.tasks))
2630
+ (work / rel).parent.mkdir(parents=True)
2631
+ (work / rel).write_text(row_line("inv.t1", "Interface") + "\n", encoding="utf-8")
2632
+ git(work, "add", rel, env=self.env)
2633
+ before = state()
2634
+ self.assertEqual(self.cli("withdraw", "inv", code=1)["reason"],
2635
+ f"{rel} is tracked in the index of {named()}, so loop inv's rows would return without its add "
2636
+ "record; withdraw takes back only a loop held in its capture")
2637
+ self.assertEqual(state(), before)
2638
+ finally:
2639
+ shutil.rmtree(admin)
2640
+ shutil.rmtree(work)
2641
+ git(self.anchor, "worktree", "add", "-q", "--detach", str(work), env=self.env)
2642
+ (admin / "index").write_bytes(b"not an index")
2643
+ before = state()
2644
+ self.assertRegex(self.cli("withdraw", "inv", code=1)["reason"],
2645
+ rf"^the index of {re.escape(str(work))} cannot be read \(fatal: .*index.*\), so withdraw cannot tell "
2646
+ rf"whether {re.escape(rel)} is tracked there; withdraw takes back only a loop held in its capture$")
2647
+ self.assertEqual(state(), before)
2648
+
2649
+ def test_withdraw_reads_each_index_without_a_file_system_monitor(self):
2650
+ """D1, E6. Prediction: with core.fsmonitor=true and one linked worktree, neither worktree's monitor running (as
2651
+ after a restart), withdraw prints WITHDRAWN within 60 s. Before: Git took the Git directory as the work tree of
2652
+ each index read, started a monitor for it and waited on it, so withdraw never returned and held its locks. The
2653
+ monitor is turned on after the setup, whose own checkout a starting monitor can hang. Withdraw runs as an owned
2654
+ tree (its own POSIX session or Windows job) ended whole within seconds, its output in files no monitor holds open."""
2655
+ probe = git_run(self.anchor, "fsmonitor--daemon", "status", env=self.env, ok=(0, 1, 128))
2656
+ if probe.returncode == 128:
2657
+ self.skipTest(f"no file system monitor here ({probe.stderr.decode('utf-8', 'replace').strip()})")
2658
+ linked = self.root / "linked"
2659
+ git(self.anchor, "worktree", "add", "-q", "--detach", str(linked), env=self.env)
2660
+ self.cli("add", self.meta)
2661
+ for work in (self.anchor, linked):
2662
+ self.addCleanup(git_run, work, "fsmonitor--daemon", "stop", env=self.env, ok=(0, 128))
2663
+ git(self.anchor, "config", "core.fsmonitor", "true", env=self.env)
2664
+ out, err = self.root / "withdraw.out", self.root / "withdraw.err"
2665
+ with out.open("wb") as stdout, err.open("wb") as stderr:
2666
+ code = shared("platform-support")["run_process"]([sys.executable, str(Path(__file__).resolve()), "withdraw", "inv"],
2667
+ subprocess.DEVNULL, 60, on_timeout=None, cwd=self.anchor, env=self.env,
2668
+ stdout=stdout, stderr=stderr)
2669
+ output = out.read_text(encoding="utf-8") + err.read_text(encoding="utf-8")
2670
+ self.assertIsNotNone(code, f"withdraw did not return within 60 s: {output}")
2671
+ self.assertEqual(json.loads(out.read_text(encoding="utf-8"))["status"], "WITHDRAWN", output)
2672
+
2673
+ def test_withdraw_refuses_while_it_cannot_read_a_place_that_may_hold_the_loop(self):
2674
+ """D1, E6. Prediction: withdraw refuses in one sentence naming the path, and changes nothing, while it cannot read
2675
+ the index of a linked worktree holding the staged queue file behind an administrative directory it cannot enter,
2676
+ <common>/worktrees itself, retired/ or moved/ holding a task's directory, devlyn-completion holding the active one,
2677
+ the loop's folder in devlyn-loops, or devlyn-loops, which the add records are listed from; and, for a branch that
2678
+ commits the queue file, its ref file, its ref folder, also one reached through a link, or its tree object, refused
2679
+ in Git's words or naming the folder. Before: is_file, lexists and glob each answered no on the permission error, so
2680
+ withdraw took back the loop, and in the loop folder and add records cases deleted its capture ref while its add
2681
+ record stayed; the unreadable worktrees folder alone was refused, in the system's words; then Git skipped the
2682
+ branch, with only a warning for the ref file and the tree object and silently for the folder, and the loop was
2683
+ withdrawn; then withdraw passed over a link, which Git follows, and took back the loop behind one."""
2684
+ probe = self.root / "probe"
2685
+ probe.mkdir(mode=0)
2686
+ try:
2687
+ os.listdir(probe)
2688
+ self.skipTest("this process reads a folder whose mode denies it (root, or Windows)")
2689
+ except PermissionError:
2690
+ pass
2691
+ finally:
2692
+ probe.chmod(0o700)
2693
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2694
+ rel, work, completion, loops = loop_queue("inv"), self.root / "work", common / "devlyn-completion", common / "devlyn-loops"
2695
+ key = {identity: hashlib.sha256(branch_of(identity).encode()).hexdigest()[:24] for identity in ("inv.t1", "inv.t2")}
2696
+ git(self.anchor, "worktree", "add", "-q", "--detach", str(work), env=self.env)
2697
+ (work / rel).parent.mkdir(parents=True)
2698
+ (work / rel).write_text(row_line("inv.t1", "Interface") + "\n", encoding="utf-8")
2699
+
2700
+ def state():
2701
+ return (git(self.anchor, "for-each-ref"), git(work, "ls-files", "-s"), sorted(
2702
+ (str(path.relative_to(common)), path.read_bytes() if path.is_file() else None) for path in loops.rglob("*") if path.suffix != ".lock"))
2703
+
2704
+ def restore(path, mode): # A path its case removed since has nothing to restore.
2705
+ with contextlib.suppress(FileNotFoundError):
2706
+ path.chmod(mode)
2707
+
2708
+ def unreadable(path):
2709
+ return f"{path} cannot be read ({os.strerror(errno.EACCES)}), so withdraw cannot tell what it holds; withdraw changes nothing until it can"
2710
+
2711
+ staged = (lambda: git(work, "add", rel, env=self.env), lambda: git(work, "rm", "-q", "--cached", rel, env=self.env))
2712
+ cases = [("linked index", staged, common / "worktrees" / work.name, 0, unreadable(common / "worktrees" / work.name / "index")),
2713
+ ("worktrees folder", staged, common / "worktrees", 0, unreadable(common / "worktrees"))]
2714
+ for name, folder, identity in (("retired", completion / "retired", "inv.t2"), ("moved", completion / "moved", "inv.t1"),
2715
+ ("active", completion, "inv.t1")):
2716
+ directory = completion / key[identity] if name == "active" else folder / f"{key[identity]}.operation"
2717
+ cases.append((name, (functools.partial(directory.mkdir, parents=True), directory.rmdir), folder, 0,
2718
+ unreadable(directory if name == "active" else folder)))
2719
+ cases += [("loop folder", (lambda: None, lambda: None), loops / "inv", 0, unreadable(added_path(common, "inv"))),
2720
+ ("add records", (lambda: None, lambda: None), loops, 0o300,
2721
+ f"{loops} cannot be listed, so withdraw cannot read the add record {added_path(common, 'inv')}; withdraw "
2722
+ "changes nothing until it can")]
2723
+ # A commit no worktree has checked out holding the queue file, made in a temporary index, for a branch per case.
2724
+ index = {**self.env, "GIT_INDEX_FILE": str(self.root / "held.index")}
2725
+ blob = git(self.anchor, "hash-object", "-w", "--stdin", data=(row_line("inv.t1", "Interface") + "\n").encode(), env=self.env)
2726
+ git(self.anchor, "update-index", "--add", "--cacheinfo", f"100644,{blob},{rel}", env=index)
2727
+ tree = git(self.anchor, "write-tree", env=index)
2728
+ held = git(self.anchor, "commit-tree", tree, "-m", "a queue file committed by hand", env=self.env)
2729
+
2730
+ def branch(name):
2731
+ return lambda: git(self.anchor, "branch", name, held, env=self.env), lambda: git(self.anchor, "branch", "-q", "-D", name, env=self.env)
2732
+
2733
+ def git_words(command, named): # Git's own words vary with the locale; they name the ref or the object.
2734
+ return re.compile(rf'git {command} wrote ".*{re.escape(named)}.*" while reading this repository\'s HEADs and branches, so '
2735
+ rf"withdraw cannot tell whether {re.escape(rel)} is tracked there; withdraw changes nothing until it can")
2736
+ outside, nest = self.root / "outside", common / "refs/heads/nest" # A ref folder Git reaches through a link.
2737
+ behind = (lambda: (outside.mkdir(), (outside / "held").write_text(held + "\n", encoding="utf-8"), nest.symlink_to(outside)),
2738
+ lambda: (nest.unlink(), (outside / "held").unlink(), outside.rmdir()))
2739
+ cases += [("branch ref file", branch("held"), common / "refs/heads/held", 0, git_words("for-each-ref", "refs/heads/held")),
2740
+ ("branch ref folder", branch("nest/held"), nest, 0, unreadable(nest)),
2741
+ ("branch ref folder behind a link", behind, outside, 0, unreadable(nest)),
2742
+ ("tree object", branch("held"), common / "objects" / tree[:2] / tree[2:], 0, git_words("cat-file", tree))]
2743
+ for name, (prepare, undo), denied, mode, reason in cases:
2744
+ with self.subTest(case=name):
2745
+ self.cli("add", write_package(self.anchor, "inv", self.tasks)) # A retry once added: nothing changes.
2746
+ prepare()
2747
+ before, original = state(), denied.stat().st_mode & 0o7777
2748
+ self.addCleanup(restore, denied, original) # Also when the case fails, so the folder can be removed.
2749
+ denied.chmod(mode)
2750
+ try:
2751
+ refusal = self.cli("withdraw", "inv", code=1)["reason"]
2752
+ finally: # Also when withdraw took the loop back, so the next case starts from its own state.
2753
+ denied.chmod(original)
2754
+ after = state()
2755
+ undo()
2756
+ self.assertEqual(after, before)
2757
+ (self.assertRegex if isinstance(reason, re.Pattern) else self.assertEqual)(refusal, reason)
2758
+
2759
+ def test_withdraw_reads_the_head_of_each_worktree_whose_listed_paths_decode_alike(self):
2760
+ """D1, E6. Prediction: beside two linked worktrees whose gitdir files name folders that differ only in a byte that
2761
+ is not UTF-8, the first listed with a HEAD that tracks the loop's queue file while no index holds it, withdraw is
2762
+ refused, naming that HEAD by the one name both paths decode to. Before: the two HEADs were kept under that one
2763
+ name, the second's in place of the first's, and the loop was withdrawn."""
2764
+ common = Path(git(self.anchor, "rev-parse", "--path-format=absolute", "--git-common-dir"))
2765
+ rel = loop_queue("inv")
2766
+ self.cli("add", self.meta)
2767
+ for name, byte in (("first", b"\xe8"), ("second", b"\xe9")): # Git lists linked worktrees sorted by path.
2768
+ work = self.root / name
2769
+ git(self.anchor, "worktree", "add", "-q", "--detach", str(work), env=self.env)
2770
+ if name == "first":
2771
+ (work / rel).parent.mkdir(parents=True)
2772
+ (work / rel).write_text(row_line("inv.t1", "Interface") + "\n", encoding="utf-8")
2773
+ git(work, "add", rel, env=self.env)
2774
+ git(work, "commit", "-qm", "a queue file committed by hand", env=self.env)
2775
+ git(work, "rm", "-q", "--cached", rel, env=self.env)
2776
+ (common / "worktrees" / name / "gitdir").write_bytes(b"/x/caf" + byte + b"/.git\n")
2777
+ self.assertEqual(self.cli("withdraw", "inv", code=1)["reason"],
2778
+ f"{rel} is tracked in HEAD of /x/caf�, so loop inv's rows would return without its add record; "
2779
+ "withdraw takes back only a loop held in its capture")
2780
+
2109
2781
  def test_a_legacy_row_is_replaced_only_by_its_one_exact_claim(self):
2110
2782
  """Q. Prediction: a loop's queue file hides the legacy row whose exact text and occurrence it records, line endings
2111
2783
  aside, showing the loop's rows in its place; a claim on a missing occurrence replaces nothing, so no other row
@@ -2168,7 +2840,7 @@ class QueueTests(unittest.TestCase):
2168
2840
  else:
2169
2841
  out, _ = waiter.communicate(timeout=30)
2170
2842
  self.assertEqual(waiter.returncode, 1)
2171
- self.assertIn("another drain is active", json.loads(out)["reason"])
2843
+ self.assertIn("a drain or a withdraw is running in this repository", json.loads(out)["reason"])
2172
2844
  held.stdin.close()
2173
2845
  self.assertEqual((read_json(added_path(common, "inv"))["origin"], read_json(added_path(common, "inv"))["branch"]), (str(linked), "side"))
2174
2846
 
@@ -2284,6 +2956,9 @@ def main():
2284
2956
  inspector = actions.add_parser("inspect")
2285
2957
  inspector.add_argument("--repo", default=".")
2286
2958
  inspector.add_argument("--json", action="store_true")
2959
+ remover = actions.add_parser("withdraw")
2960
+ remover.add_argument("--repo", default=".")
2961
+ remover.add_argument("loop", help="a queued loop none of whose tasks has started and that replaced no legacy row")
2287
2962
  actions.choices["drain"].add_argument("--worktree-root")
2288
2963
  actions.choices["drain"].add_argument("executor", nargs="+", help="host executor argv after --; {packet} becomes the packet path")
2289
2964
  args = parser.parse_args()
@@ -2293,11 +2968,11 @@ def main():
2293
2968
  print(f"queue self-test: PASS ({result.testsRun} tests)")
2294
2969
  return 0 if result.wasSuccessful() else 1
2295
2970
  if args.action is None:
2296
- parser.error("check, add, status, inspect or drain is required")
2971
+ parser.error("check, add, withdraw, status, inspect or drain is required")
2297
2972
  if args.action == "inspect":
2298
2973
  os.environ["GIT_OPTIONAL_LOCKS"] = "0" # git diff may otherwise refresh the index even with unchanged bytes.
2299
2974
  try:
2300
- result = {"check": check, "add": add, "status": status, "drain": drain, "inspect": inspect}[args.action](args)
2975
+ result = {"check": check, "add": add, "withdraw": withdraw, "status": status, "drain": drain, "inspect": inspect}[args.action](args)
2301
2976
  except (LoopError, OSError) as exc:
2302
2977
  result = {"status": "BLOCKED", "reason": str(exc)}
2303
2978
  if args.action in {"status", "drain"}: