@opengsd/gsd-path 1.6.2 → 1.7.0

package/scripts/build_recovery.py

@@ -144,7 +144,13 @@ def settled_tasks(repo: Path) -> dict[str, tuple[str, dict]]:
             raise state.PipelineStateError(f"invalid recovery task path: {path.name}")
         text = state._read_real_file(path, path.name)
         fields = task_fields(text, path.name)
-        if fields.get("status") not in {"pending", "done"}:
+        if fields.get("status") == "withdrawn":
+            # A cited tombstone is settled inventory; recovery skips it (#483).
+            if not state._common.withdrawn_citations(text):
+                raise state.PipelineStateError(
+                    f"withdrawn task lacks an owner citation: {path.name}"
+                )
+        elif fields.get("status") not in {"pending", "done"}:
             raise state.PipelineStateError(f"settle task ownership before recovery: {path.name}")
         if fields["status"] == "pending" and any(
             fields.get(key) != "null" for key in ("agent", "base", "worktree", "task_branch")
@@ -250,7 +256,24 @@ def validate_review_backup(repo: Path, recovery: dict, directory: Path) -> None:
               for path in directory.rglob("*")
               if path.is_file() and not state._common.is_ignored_junk(path)} if directory.exists() else {}
     if actual != expected:
-        raise state.PipelineStateError("review backup differs from the recovery base")
+        extra = sorted(set(actual) - set(expected))
+        changed = sorted(
+            name for name in set(actual) & set(expected) if actual[name] != expected[name]
+        )
+        missing = sorted(set(expected) - set(actual))
+        detail = ""
+        if extra:
+            detail += f" files in the review backup that the recovery base does not have: {', '.join(extra)}."
+        if changed:
+            detail += f" files differing from the recovery base: {', '.join(changed)}."
+        if missing:
+            detail += f" files at the recovery base missing from the review backup: {', '.join(missing)}."
+        raise state.PipelineStateError(
+            "review backup differs from the recovery base." + detail
+            + " prepare-build-recovery must run before re-planning touches any recovery"
+            " artifact; remove the extra files and restore the missing or changed ones"
+            " to the recovery base's content, then rerun prepare"
+        )
 
 
 def prepare(repo: Path) -> dict:

package/scripts/build_state.py

@@ -21,6 +21,7 @@ if __package__:
         IsolationError,
         recover as recover_isolation,
         verify_landed_task_files,
+        withdrawn_citations,
     )
     from .pipeline_git import task_commit_subject
     from .pipeline_state import PipelineStateError, load_state
@@ -33,6 +34,7 @@ else:
             IsolationError,
             recover as recover_isolation,
             verify_landed_task_files,
+            withdrawn_citations,
         )
         from pipeline_git import task_commit_subject
         from pipeline_state import PipelineStateError, load_state
@@ -45,6 +47,7 @@ else:
             IsolationError,
             recover as recover_isolation,
             verify_landed_task_files,
+            withdrawn_citations,
         )
         from scripts.pipeline_git import task_commit_subject
         from scripts.pipeline_state import PipelineStateError, load_state
@@ -63,7 +66,7 @@ REVIEW_VERDICT_RE = re.compile(
 )
 REVIEW_FINDINGS_HEADING = "Review findings"
 VERIFY_HEAVY_RE = re.compile(r"(?m)^Heavy:\s*(?P<value>yes|no)\s*(?:<!--.*-->)?\s*$")
-VALID_TASK_STATUSES = {"pending", "in-progress", "done", "failed", "blocked"}
+VALID_TASK_STATUSES = {"pending", "in-progress", "done", "failed", "blocked", "withdrawn"}
 VERIFY_RESULTS = _common.VERIFY_RESULTS
 
 
@@ -271,6 +274,16 @@ def _validate_graph(tasks: Sequence[Task]) -> None:
                     f"{task.task_id} depends on later-wave {dependency}",
                     {"task": task.task_id, "dependency": dependency},
                 )
+            if by_id[dependency].status == "withdrawn" and task.status not in (
+                "done",
+                "withdrawn",
+            ):
+                raise BuildStateError(
+                    "withdrawn-dependency",
+                    f"{task.task_id} depends on withdrawn {dependency}; re-point "
+                    "the dependency or withdraw this task too (#483)",
+                    {"task": task.task_id, "dependency": dependency},
+                )
 
     visiting: List[str] = []
     visited = set()
@@ -342,6 +355,15 @@ def _load_tasks(
                 "plan-task-mismatch",
                 f"{task.task_id} names missing PLAN.md Wave {task.wave}",
             )
+        if task.status == "withdrawn" and not withdrawn_citations(
+                path.read_text(encoding="utf-8")):
+            raise BuildStateError(
+                "uncited-withdrawn",
+                f"{task.task_id} is withdrawn without an owner citation; append "
+                "a `- Withdrawn — <YYYY-MM-DD>: <owner ruling verbatim>` Log "
+                "entry (#483)",
+                {"task": task.task_id},
+            )
         parsed.append(task)
     missing_waves = sorted(set(plan_waves) - {task.wave for task in parsed})
     if missing_waves:
@@ -431,6 +453,10 @@ def _validate_ready_metadata(project: Project) -> Dict[str, Mapping[str, object]
     seen_worktrees: Dict[str, str] = {}
     seen_branches: Dict[str, str] = {}
     for task in project.tasks:
+        if task.status == "withdrawn":
+            # A tombstone is inventory, not work: no dispatch metadata is
+            # expected and recovery reports it as none (#483).
+            continue
         if task.status == "pending":
             if any(
                 value is not None
@@ -479,7 +505,10 @@ def _validate_ready_metadata(project: Project) -> Dict[str, Mapping[str, object]
                 _invalid_state(task, f"worktree is also owned by {previous}")
             continue
         if task.status == "done":
-            if any(by_id[dependency].status != "done" for dependency in task.deps):
+            if any(
+                by_id[dependency].status not in ("done", "withdrawn")
+                for dependency in task.deps
+            ):
                 _invalid_state(task, "done task has an incomplete dependency")
             if task.agent is None or task.base is None:
                 _invalid_state(task, "done task is missing agent or base")
@@ -679,7 +708,11 @@ def _unreviewed_waves(project: Project, waves: Sequence[int]) -> List[int]:
     review_dir = project.tasks_dir.parent / "review"
     reviewed = {
         wave for wave in waves
-        if all(task.review_findings for task in project.tasks if task.wave == wave)
+        if all(
+            task.review_findings
+            for task in project.tasks
+            if task.wave == wave and task.status != "withdrawn"
+        )
     }
     if review_dir.is_dir() and not review_dir.is_symlink():
         for path in review_dir.iterdir():
@@ -696,7 +729,12 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
     project = _load_project(repo, project_dir, ("active", "done") if allow_done else ("active",))
     reports = _validate_ready_metadata(project)
     by_id = {task.task_id: task for task in project.tasks}
-    unfinished_waves = sorted({task.wave for task in project.tasks if task.status != "done"})
+    # A withdrawn tombstone carries no obligation: its wave is finished
+    # unless live work remains in it (#483).
+    unfinished_waves = sorted({
+        task.wave for task in project.tasks
+        if task.status != "done" and task.status != "withdrawn"
+    })
     current_wave = unfinished_waves[0] if unfinished_waves else None
     blocked_reviews = _blocked_wave_reviews(project)
     blocked_wave = min(
@@ -705,7 +743,10 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
             for wave in blocked_reviews
             if (current_wave is None or wave < current_wave)
             and any(task.wave == wave for task in project.tasks)
-            and all(task.status == "done" for task in project.tasks if task.wave == wave)
+            and all(
+                task.status in ("done", "withdrawn")
+                for task in project.tasks if task.wave == wave
+            )
         ),
         default=None,
     )
@@ -716,7 +757,7 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
         repairs = [
             task
             for task in project.tasks
-            if task.status != "done"
+            if task.status != "done" and task.status != "withdrawn"
             and task.wave > blocked_wave
             and task.review_findings
             and source_ids.intersection(task.deps)
@@ -785,7 +826,8 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
                 ],
             }
             for task in project.tasks
-            if task.wave == current_wave and task.status != "done"
+            if task.wave == current_wave
+            and task.status != "done" and task.status != "withdrawn"
         ]
         raise BuildStateError(
             "dependency-deadlock",
@@ -946,6 +988,8 @@ def verify_landed_tasks(
         canonical_project_dir=canonical_dir,
     )
     for task in tasks:
+        if task.status == "withdrawn":
+            continue  # A cited tombstone ships without a landing (#483)
         if task.status != "done":
             _invalid_state(task, "task must have status: done before shipping")
     try:
@@ -965,6 +1009,8 @@ def verify_landed_tasks(
     }
     evidence = []
     for task in tasks:
+        if task.status == "withdrawn":
+            continue  # A cited tombstone has no landing to prove (#483)
         report = reports.get(task.task_id)
         if report is None or report.get("verdict") not in LANDED_VERDICTS:
             _invalid_state(task, "task landing is not canonical")
@@ -1044,10 +1090,13 @@ def _ledger_entries(path: Path) -> List[Dict[str, object]]:
 
 def verify_record(repo: str, command: str, commit: str, result: str,
                   execution: Optional[Dict[str, object]] = None,
-                  member: Optional[str] = None) -> Dict[str, object]:
+                  member: Optional[str] = None,
+                  rerun: bool = False) -> Dict[str, object]:
     """Append one verify run (command, commit, result, timestamp) to the ledger.
 
     A member run names its member and a commit in that member's repository.
+    ``rerun`` marks an owner-requested re-execution at the same command and
+    commit, so the ledger shows the earlier entry was superseded, not erased.
     """
 
     repository = _repo_root(repo)
@@ -1066,6 +1115,7 @@ def verify_record(repo: str, command: str, commit: str, result: str,
         "result": result,
         "recorded_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
         **({"repo": member} if member else {}),
+        **({"rerun": True} if rerun else {}),
     }
     if execution is not None:
         if (not isinstance(execution.get("stdout"), str)
@@ -1073,6 +1123,18 @@ def verify_record(repo: str, command: str, commit: str, result: str,
                 or type(execution.get("exit_code")) is not int
                 or (execution["exit_code"] == 0) != (result == "pass")):
             raise BuildStateError("invalid-execution", "verification output and result disagree")
+        if "cwd" in execution and not isinstance(execution["cwd"], str):
+            raise BuildStateError("invalid-execution", "execution cwd must be a path string")
+        if "base" in execution and not re.fullmatch(r"[0-9a-f]{40}", str(execution["base"])):
+            raise BuildStateError("invalid-execution", "execution base must be a commit sha")
+        location = execution.get("location")
+        if isinstance(location, str) and "cwd" in execution and "worktree" in execution:
+            if location.startswith("sidecar") and execution["cwd"] == execution["worktree"]:
+                raise BuildStateError("invalid-execution",
+                                      "a sidecar Verify cannot name the task checkout as its directory")
+            if location.startswith("isolate") and execution["cwd"] != execution["worktree"]:
+                raise BuildStateError("invalid-execution",
+                                      "an isolate Verify must name its task worktree as its directory")
         entry["execution"] = execution
     path.parent.mkdir(parents=True, exist_ok=True)
     previous = path.read_text(encoding="utf-8") if path.exists() else ""
@@ -1095,6 +1157,255 @@ def verify_lookup(repo: str, command: str, commit: str, member: Optional[str] =
     }
 
 
+VERIFY_LOG_RE = re.compile(r"orchestrator Verify \((?:isolate|sidecar) [^)]+\): (pass|fail), exit \d+")
+MEMBER_RECORD_RE = re.compile(r"^Member: (\S+) ([0-9a-f]{40}) ([0-9a-f]{40})$", re.MULTILINE)
+GAP_MEMBER_RE = re.compile(r"^Member reviewed HEAD: (\S+) ([0-9a-f]{40})$", re.MULTILINE)
+GAP_HEAD_RE = re.compile(r"^Reviewed HEAD: ([0-9a-f]{40})$", re.MULTILINE)
+GAP_VERDICT_RE = re.compile(r"^Gap verdict: (pass|blocked)$", re.MULTILINE)
+
+
+def _proven_landings(
+    repository: Path,
+    done_paths: Dict[str, Path],
+    canonical_tasks_dir: str,
+    head: str,
+) -> Tuple[Dict[str, str], Dict[str, str]]:
+    """Map each provably landed task id to its landing commit and record proof failures."""
+
+    def collect(payload: Mapping[str, object]) -> Dict[str, str]:
+        proven: Dict[str, str] = {}
+        for item in payload["tasks"]:  # type: ignore[index]
+            if not isinstance(item, Mapping):
+                continue
+            if str(item.get("verdict")) in LANDED_VERDICTS and item.get("commit"):
+                proven[str(item.get("task_id"))] = str(item["commit"])
+        return proven
+
+    try:
+        batch = verify_landed_task_files(repository, list(done_paths.values()), canonical_tasks_dir, head)
+        return collect(batch), {}
+    except IsolationError:
+        pass
+    proven: Dict[str, str] = {}
+    errors: Dict[str, str] = {}
+    for task_id, path in sorted(done_paths.items()):
+        try:
+            single = verify_landed_task_files(repository, [path], canonical_tasks_dir, head)
+        except IsolationError as error:
+            errors[task_id] = str(error)
+            continue
+        found = collect(single)
+        if task_id in found:
+            proven[task_id] = found[task_id]
+    return proven, errors
+
+
+def _ledger_absence_reason(
+    repository: Path,
+    task: Task,
+    landing: str,
+    member_landing: Optional[str],
+    member_base: Optional[str],
+) -> str:
+    """Explain why a passing task has no reusable ledger row."""
+
+    if task.repo:
+        if not member_landing or not member_base:
+            return "member record does not name the member landing"
+        try:
+            checkout = _member_checkout(repository, task.repo)
+            parent = _git_output(checkout, "rev-parse", f"{member_landing}^")
+        except BuildStateError:
+            return "member checkout unavailable — the record condition cannot be re-derived"
+        return ("ledger row absent although the member landing meets the record condition"
+                if parent == member_base else
+                "member landing parent differs from the member base — row deliberately not written")
+    parent = _git_output(repository, "rev-parse", f"{landing}^")
+    return ("ledger row absent although the landing meets the record condition"
+            if parent == task.base else
+            "landing parent differs from the recorded base — row deliberately not written")
+
+
+def evidence_manifest(
+    repo: str,
+    project_dir: str,
+    head: str,
+    canonical_project_dir: str = DEFAULT_PROJECT_DIR,
+) -> Dict[str, object]:
+    """Join every task to its authoritative accepted evidence and classify the result.
+
+    States: `ledger` (a reusable verify-ledger row matches the task's Verify command
+    at its proven landing — for member tasks, at the member landing recorded in the
+    coordinator record body), `log-only` (the retained task Log records the isolated
+    Verify pass and the landing is proven, but no reusable row exists), `rejected`
+    (the Log records a failed Verify, or a ledger row at the landing failed), and
+    `superseded` / `missing` (a plan-approved supersedes retired the Verify, or no
+    durable evidence exists at this head), plus `withdrawn` (a cited owner-approved
+    tombstone that never owed a landing — distinct from every completed state). A pass is never inferred from a Log label
+    alone: done tasks require a proven first-parent landing, and member rows must
+    match the recorded member landing. The project section reports the member
+    milestone project Verify only when the project Verify gap (final-gap-1.md)
+    records a passing verdict at exactly this head (`gap-embedded`), or a reusable ledger row matches PLAN's
+    project Verify command at this head (`ledger`); a blocked gap verdict or a
+    different reviewed HEAD is listed but never accepted. Limits: failed-attempt
+    executions kept only in uncommitted dispatch records are not consulted, and a
+    noncanonical task artifact aborts with `invalid-task-file` naming the file — the
+    same strictness as every other build_state command; fix or flag the file first.
+    """
+
+    repository = _repo_root(repo)
+    artifact_dir, artifact_path = _project_path(repository, project_dir)
+    canonical_dir, _ = _project_path(repository, canonical_project_dir)
+    if not FULL_SHA_RE.fullmatch(head) or not _commit_resolves(repository, head):
+        raise BuildStateError("invalid-head", "--head must be a full, existing commit SHA")
+    if _git_output(repository, "rev-parse", f"{head}^{{commit}}") != head:
+        raise BuildStateError("invalid-head", "--head must resolve to itself")
+    tasks = _load_tasks(
+        repository, artifact_dir, artifact_path, canonical_project_dir=canonical_dir
+    )
+    superseded_by: Dict[str, str] = {}
+    for task in tasks:
+        if task.status != "done":
+            continue
+        for earlier in task.supersedes:
+            superseded_by.setdefault(earlier, task.task_id)
+    done_paths = {
+        task.task_id: artifact_path / "tasks" / Path(task.task_file).name
+        for task in tasks if task.status == "done"
+    }
+    canonical_tasks_dir = str(PurePosixPath(canonical_dir) / "tasks")
+    proven, proof_errors = (
+        _proven_landings(repository, done_paths, canonical_tasks_dir, head) if done_paths else ({}, {})
+    )
+    ledger_path = _ledger_path(repository)
+    entries = _ledger_entries(ledger_path) if ledger_path.exists() else []
+    manifest: List[Dict[str, object]] = []
+    for task in tasks:
+        text = _read_file(
+            artifact_path / "tasks" / Path(task.task_file).name,
+            f"{artifact_dir}/tasks/{Path(task.task_file).name}",
+        )
+        results = VERIFY_LOG_RE.findall(text)
+        entry: Dict[str, object] = {
+            "task": task.task_id,
+            "title": task.title,
+            "status": task.status,
+            "repo": task.repo or None,
+            "verify_command": _common.task_verify_command(text),
+            "superseded_by": superseded_by.get(task.task_id),
+        }
+        manifest.append(entry)
+        if task.status == "withdrawn":
+            # The tombstone stays distinct from completed acceptance: the
+            # manifest records the withdrawal and its owner citations (#483).
+            entry["state"] = "withdrawn"
+            entry["reason"] = "; ".join(withdrawn_citations(text))
+            continue
+        if task.status != "done":
+            entry["state"] = "rejected" if "fail" in results else "missing"
+            entry["reason"] = "verify failed" if "fail" in results else "task never landed"
+            if "fail" in results:
+                entry["receipt"] = {"source": "task-log", "path": task.task_file}
+            continue
+        landing = proven.get(task.task_id)
+        if landing is None:
+            entry["state"] = "missing"
+            entry["reason"] = ("landing proof failed: " + proof_errors[task.task_id]
+                               if task.task_id in proof_errors
+                               else "done task without a provable first-parent landing at this head")
+            continue
+        entry["landing"] = landing
+        member_landing: Optional[str] = None
+        member_base: Optional[str] = None
+        if task.repo:
+            body = _git_output(repository, "log", "-1", "--format=%b", landing)
+            record = MEMBER_RECORD_RE.search(body)
+            if record is None or record.group(1) != task.repo:
+                entry["state"] = "missing"
+                entry["reason"] = "member record commit does not name this member's landing"
+                continue
+            member_landing, member_base = record.group(2), record.group(3)
+        lookup_commit = member_landing or landing
+        if not entry["verify_command"]:
+            entry["state"] = "missing"
+            entry["reason"] = "landed task without a Verify command"
+            continue
+        try:
+            lookup = verify_lookup(str(repository), str(entry["verify_command"]), lookup_commit,
+                                   task.repo or None)
+        except BuildStateError:
+            lookup = None
+        if lookup is not None and lookup["hit"]:
+            entry["state"] = "ledger" if lookup["reuse"] else "rejected"
+            entry["receipt"] = {"source": "verify-ledger", "commit": lookup_commit}
+        elif "pass" in results:
+            entry["state"] = "log-only"
+            entry["receipt"] = {"source": "task-log", "path": task.task_file}
+            entry["reason"] = _ledger_absence_reason(
+                repository, task, landing, member_landing, member_base)
+        elif entry["superseded_by"] in proven:
+            entry["state"] = "superseded"
+            entry["reason"] = (f"verify retired by {entry['superseded_by']}; "
+                               "the superseding task carries the evidence")
+        else:
+            entry["state"] = "missing"
+            entry["reason"] = "proven landing without a recorded Verify result"
+        if entry["superseded_by"] and entry["superseded_by"] not in proven:
+            note = f"superseding task {entry['superseded_by']} is not landed at this head"
+            entry["reason"] = f"{entry['reason']}; {note}" if entry.get("reason") else note
+    gap_members: List[Dict[str, object]] = []
+    gap_accepted = False
+    gap_reasons: List[str] = []
+    gap_path = repository.joinpath(*PurePosixPath(canonical_dir).parts) / "review" / "final-gap-1.md"
+    if gap_path.is_file() and not gap_path.is_symlink():
+        text = gap_path.read_text(encoding="utf-8")
+        verdict = GAP_VERDICT_RE.findall(text)
+        reviewed = GAP_HEAD_RE.findall(text)
+        for name, sha in GAP_MEMBER_RE.findall(text):
+            gap_members.append({
+                "member": name,
+                "head": sha,
+                "verdict": verdict[-1] if verdict else None,
+                "source": f"{canonical_dir}/review/{gap_path.name}",
+            })
+        if gap_members:
+            if verdict and verdict[-1] == "pass" and reviewed and reviewed[-1] == head:
+                gap_accepted = True
+            else:
+                gap_reasons.append(
+                    f"{gap_path.name}: verdict {verdict[-1] if verdict else 'unreadable'}"
+                    f" at reviewed HEAD {reviewed[-1] if reviewed else 'unreadable'}")
+    plan = _read_file(artifact_path / "plan" / "PLAN.md", f"{artifact_dir}/plan/PLAN.md")
+    try:
+        project_command = check_handoffs._line_value(plan, "Project verify:")
+    except check_handoffs.HandoffError:
+        project_command = ""
+    project_state = "missing"
+    project_reason: Optional[str] = None
+    if gap_accepted:
+        project_state = "gap-embedded"
+    elif gap_reasons:
+        project_reason = "; ".join(gap_reasons)
+    elif project_command:
+        project_lookup = verify_lookup(str(repository), project_command, head)
+        if project_lookup["reuse"]:
+            project_state = "ledger"
+        elif project_lookup["hit"]:
+            project_reason = "the recorded project Verify row at this head failed"
+    return {
+        "command": "evidence-manifest",
+        "project_dir": artifact_dir,
+        "canonical_project_dir": canonical_dir,
+        "head": head,
+        "tasks": manifest,
+        "project": {
+            "members": gap_members or None,
+            "state": project_state,
+            **({"reason": project_reason} if project_reason else {}),
+        },
+    }
+
+
 def _parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description=__doc__)
     subcommands = parser.add_subparsers(dest="command", required=True)
@@ -1127,6 +1438,13 @@ def _parser() -> argparse.ArgumentParser:
     lookup_parser.add_argument("--repo", required=True)
     lookup_parser.add_argument("--command", required=True, dest="verify_command")
     lookup_parser.add_argument("--commit", required=True)
+    manifest_parser = subcommands.add_parser(
+        "evidence-manifest", help="join every task to its authoritative accepted evidence"
+    )
+    manifest_parser.add_argument("--repo", required=True)
+    manifest_parser.add_argument("--project-dir", default=DEFAULT_PROJECT_DIR)
+    manifest_parser.add_argument("--head", required=True)
+    manifest_parser.add_argument("--canonical-project-dir", default=DEFAULT_PROJECT_DIR)
     return parser
 
 
@@ -1143,6 +1461,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
             )
         elif arguments.command == "verify-lookup":
             result = verify_lookup(arguments.repo, arguments.verify_command, arguments.commit)
+        elif arguments.command == "evidence-manifest":
+            result = evidence_manifest(
+                arguments.repo,
+                arguments.project_dir,
+                arguments.head,
+                arguments.canonical_project_dir,
+            )
         else:
             result = verify_landed_tasks(
                 arguments.repo,