@opengsd/gsd-path 1.4.0 → 1.6.0

package/scripts/build_state.py

@@ -24,7 +24,7 @@ if __package__:
     )
     from .pipeline_git import task_commit_subject
     from .pipeline_state import PipelineStateError, load_state
-    from . import _common
+    from . import _common, check_handoffs
 else:
     try:
         from check_task_briefs import _frontmatter, _sections
@@ -37,6 +37,7 @@ else:
         from pipeline_git import task_commit_subject
         from pipeline_state import PipelineStateError, load_state
         import _common
+        import check_handoffs
     except ImportError:  # pragma: no cover - package imports used by tests
         from scripts.check_task_briefs import _frontmatter, _sections
         from scripts.isolation import (
@@ -47,7 +48,7 @@ else:
         )
         from scripts.pipeline_git import task_commit_subject
         from scripts.pipeline_state import PipelineStateError, load_state
-        from scripts import _common
+        from scripts import _common, check_handoffs
 
 
 DEFAULT_PROJECT_DIR = ".project"
@@ -57,6 +58,10 @@ FULL_SHA_RE = re.compile(r"^[0-9a-f]{40}$")
 # recover verdicts that count as landed, mapped to their evidence classification.
 LANDED_VERDICTS = {"recovered": "proven-landed", "attested": "attested"}
 WAVE_HEADING_RE = re.compile(r"(?m)^## Wave (?P<wave>\d+)\b.*$")
+REVIEW_VERDICT_RE = re.compile(
+    r"(?m)^Wave verdict:\s*(?P<verdict>pass|blocked)\s*(?:<!--.*?-->)?\s*$"
+)
+REVIEW_FINDINGS_HEADING = "Review findings"
 VERIFY_HEAVY_RE = re.compile(r"(?m)^Heavy:\s*(?P<value>yes|no)\s*(?:<!--.*-->)?\s*$")
 VALID_TASK_STATUSES = {"pending", "in-progress", "done", "failed", "blocked"}
 VERIFY_RESULTS = _common.VERIFY_RESULTS
@@ -88,6 +93,9 @@ class Task:
     task_file: str
     verify_heavy: bool
     repo: str = ""
+    review_findings: bool = False
+    verify: str = ""
+    supersedes: Tuple[str, ...] = ()
 
 
 @dataclass(frozen=True)
@@ -241,6 +249,9 @@ def _parse_task(path: Path, relative_path: str, task_file: Optional[str] = None)
         task_file=task_file or relative_path,
         verify_heavy=_verify_heavy(text, label),
         repo=str(fields.get("repo") or ""),
+        review_findings=REVIEW_FINDINGS_HEADING in _sections(text),
+        verify=_common.task_verify_command(text),
+        supersedes=_normalize_list(fields.get("supersedes", []), f"{label} supersedes", task_ids=True),
     )
 
 
@@ -413,7 +424,8 @@ def _recovery_reports(project: Project) -> Dict[str, Mapping[str, object]]:
     return reports
 
 
-def _validate_ready_metadata(project: Project) -> None:
+def _validate_ready_metadata(project: Project) -> Dict[str, Mapping[str, object]]:
+    """Validate every task against its recovery report; return the reports by task id."""
     by_id = {task.task_id: task for task in project.tasks}
     recovery = _recovery_reports(project)
     seen_worktrees: Dict[str, str] = {}
@@ -477,12 +489,204 @@ def _validate_ready_metadata(project: Project) -> None:
             if report.get("verdict") not in LANDED_VERDICTS:
                 reason = report.get("reason", "landing commit is not proven")
                 _invalid_state(task, str(reason))
+    return recovery
 
 
 def _overlap(left: Task, right: Task) -> List[str]:
     if left.repo != right.repo:
         return []
-    return sorted(set(left.files) & set(right.files))
+    return _common.overlapping_paths(left.files, right.files)
+
+
+def _superseded(tasks: Sequence[Task], task: Task) -> Dict[str, str]:
+    """Earlier task id -> the task that replaces its behavior: this task, or a `done` task."""
+
+    return {
+        earlier: other.task_id
+        for other in tasks
+        if other.status == "done" or other.task_id == task.task_id
+        for earlier in other.supersedes
+    }
+
+
+def _held(by_id: Mapping[str, Task], task: Task) -> List[Task]:
+    """The tasks whose protective role this task inherits: each task it supersedes, through a chain."""
+
+    held: Dict[str, Task] = {}
+    pending = list(task.supersedes)
+    while pending:
+        other = by_id.get(pending.pop())
+        if other is not None and other.task_id not in held:  # a cycle ends here
+            held[other.task_id] = other
+            pending.extend(other.supersedes)
+    return [by_id[name] for name in sorted(held)]
+
+
+def _live_verify(
+    by_id: Mapping[str, Task], superseded: Mapping[str, str], source: Task
+) -> Tuple[str, Optional[str], List[Tuple[str, str]]]:
+    """The Verify of a source that still runs, or the task that retired all of it, and the drops.
+
+    The Verify of a fix task is exactly the composition that fix-tasks generates from the
+    Verify of each task in its deps, so it carries a copy of each: only the copies that are
+    not retired stay in the command. Any other Verify is one command. Each drop is
+    (superseded task, superseding task); a copy whose command a live dependency still runs
+    is not a drop.
+    """
+
+    if source.task_id in superseded:
+        return "", superseded[source.task_id], [(source.task_id, superseded[source.task_id])]
+    deps = [by_id[dependency] for dependency in source.deps if dependency in by_id]
+    if not deps or source.verify != _common.fix_verify(dep.verify for dep in deps):
+        return source.verify, None, []
+    parts = [_live_verify(by_id, superseded, dep) for dep in deps]
+    live = [command for command, retired_by, _ in parts if not retired_by]
+    dropped = list(dict.fromkeys(drop for _, _, drops in parts for drop in drops
+                                 if by_id[drop[0]].verify not in live))
+    if not live:
+        return "", parts[0][1], dropped
+    return _common.fix_verify(live), None, dropped
+
+
+def _regression_sources(
+    tasks: Sequence[Task], task: Task
+) -> List[Tuple[Task, List[str], Optional[str], str, List[Tuple[str, str]]]]:
+    """Each `done` task whose Verify protects a file of this task:
+    (source, replaced, retired_by, command, dropped).
+
+    A source is a task of an earlier wave that shares a declared file with this task. A task
+    that supersedes others inherits their role: it is also a source, in any wave, for a task
+    that shares a file with a task it holds, and `replaced` names those tasks.
+    `retired_by` names the task that supersedes the source: this task, or a `done` task. A
+    retired Verify does not run, also not as a copy in the Verify of a fix task: `command`
+    is the Verify that runs and `dropped` names each retired copy (`_live_verify`). The source still counts for the one-at-a-time
+    rule, so the superseding task never runs beside a task that shares a file of the
+    superseded task.
+    """
+
+    by_id = {other.task_id: other for other in tasks}
+    superseded = _superseded(tasks, task)
+    sources = []
+    for other in tasks:
+        if other.status != "done" or other.task_id == task.task_id:
+            continue
+        replaced = [held.task_id for held in _held(by_id, other) if _overlap(task, held)]
+        if replaced or (other.wave < task.wave and _overlap(task, other)):
+            command, retired_by, dropped = _live_verify(by_id, superseded, other)
+            sources.append((other, replaced, retired_by, command, dropped))
+    return sources
+
+
+def _landing_tasks(repo: Path, task_id: str, project_dir: str) -> Tuple[List[Task], Task]:
+    tasks = [
+        _parse_task(path, path.relative_to(repo).as_posix())
+        for path in sorted((repo / project_dir / "tasks").glob("*.md"))
+    ]
+    task = next((item for item in tasks if item.task_id == task_id), None)
+    if task is None:
+        raise BuildStateError("unknown-task", f"task file is missing: {task_id}")
+    return tasks, task
+
+
+def retired_verifies(
+    repo: Path, task_id: str, project_dir: str = DEFAULT_PROJECT_DIR
+) -> List[Dict[str, str]]:
+    """Return the earlier-wave tasks whose Verify a landing of this task does not run again.
+
+    A task's plan-approved `supersedes` names them. The Verify is retired for that task, and
+    for every task that lands after that task is `done`: the Verify of the superseding task
+    then runs in its place (`regression_verifies`, key `supersedes`). A fix task whose
+    generated Verify holds only retired copies is here too. A fix task whose generated Verify
+    loses some of its copies has one entry for each, with the superseded task under `part`.
+    """
+
+    tasks, task = _landing_tasks(repo, task_id, project_dir)
+    retired: List[Dict[str, str]] = []
+    for other, _, retired_by, _, dropped in _regression_sources(tasks, task):
+        if retired_by:
+            if other.verify:
+                retired.append({"task": other.task_id, "superseded_by": retired_by})
+        else:
+            retired.extend({"task": other.task_id, "part": part, "superseded_by": by}
+                           for part, by in dropped)
+    return retired
+
+
+def regression_verifies(
+    repo: Path, task_id: str, own: str, project_dir: str = DEFAULT_PROJECT_DIR
+) -> List[Dict[str, object]]:
+    """Return the Verify commands of landed tasks that a landing of this task must also pass.
+
+    One entry per distinct command, with every source task that has it, and under
+    `supersedes` the tasks whose retired Verify it replaces for this landing. A command
+    that the task's own Verify already runs is left out: an identical command, or
+    a dependency's command when the own Verify is exactly the fix-task Verify of
+    the task's dependencies. Text that only contains a command proves nothing.
+    A retired Verify is not here, also not as a copy in the Verify of a fix task: see
+    `retired_verifies`.
+    """
+
+    tasks, task = _landing_tasks(repo, task_id, project_dir)
+    by_id = {item.task_id: item for item in tasks}
+    composed = [by_id[dependency].verify for dependency in task.deps if dependency in by_id]
+    covered = {own, *(composed if own == _common.fix_verify(composed) else ())}
+    sources: Dict[str, Dict[str, object]] = {}
+    for other, replaced, _, command, _ in _regression_sources(tasks, task):
+        if command and command not in covered:
+            entry = sources.setdefault(command, {"tasks": [], "command": command})
+            entry["tasks"].append(other.task_id)
+            if replaced:  # this Verify runs in the place of the Verify of each task it replaced
+                entry["supersedes"] = sorted({*entry.get("supersedes", ()), *replaced})
+    return list(sources.values())
+
+
+def _blocked_wave_reviews(project: Project) -> Dict[int, int]:
+    """Return waves whose highest canonical review cycle has a blocked lens."""
+
+    review_dir = project.tasks_dir.parent / "review"
+    if review_dir.is_symlink() or not review_dir.is_dir():
+        return {}
+
+    latest: Dict[int, int] = {}
+    verdicts: Dict[Tuple[int, int], List[str]] = {}
+    for path in sorted(review_dir.iterdir()):
+        named = check_handoffs.WAVE_REVIEW_NAME.fullmatch(path.name)
+        if named is None or path.is_symlink() or not path.is_file():
+            continue
+        wave, cycle = int(named.group("wave")), int(named.group("cycle"))
+        latest[wave] = max(latest.get(wave, 0), cycle)
+        try:
+            text = path.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            continue
+        verdict = REVIEW_VERDICT_RE.search(text)
+        if verdict is not None:
+            verdicts.setdefault((wave, cycle), []).append(verdict.group("verdict"))
+
+    return {
+        wave: cycle
+        for wave, cycle in latest.items()
+        if "blocked" in verdicts.get((wave, cycle), [])
+    }
+
+
+def _unreviewed_waves(project: Project, waves: Sequence[int]) -> List[int]:
+    """Return the given waves that still need a canonical review file of their own.
+
+    A repair wave is left out: the next review cycle of its source wave judges it.
+    """
+
+    review_dir = project.tasks_dir.parent / "review"
+    reviewed = {
+        wave for wave in waves
+        if all(task.review_findings for task in project.tasks if task.wave == wave)
+    }
+    if review_dir.is_dir() and not review_dir.is_symlink():
+        for path in review_dir.iterdir():
+            named = check_handoffs.WAVE_REVIEW_NAME.fullmatch(path.name)
+            if named is not None and path.is_file() and not path.is_symlink():
+                reviewed.add(int(named.group("wave")))
+    return [wave for wave in waves if wave not in reviewed]
 
 
 def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
@@ -490,10 +694,43 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
     """Return task readiness, optionally accepting build/done for completion checks."""
 
     project = _load_project(repo, project_dir, ("active", "done") if allow_done else ("active",))
-    _validate_ready_metadata(project)
+    reports = _validate_ready_metadata(project)
     by_id = {task.task_id: task for task in project.tasks}
     unfinished_waves = sorted({task.wave for task in project.tasks if task.status != "done"})
     current_wave = unfinished_waves[0] if unfinished_waves else None
+    blocked_reviews = _blocked_wave_reviews(project)
+    blocked_wave = min(
+        (
+            wave
+            for wave in blocked_reviews
+            if (current_wave is None or wave < current_wave)
+            and any(task.wave == wave for task in project.tasks)
+            and all(task.status == "done" for task in project.tasks if task.wave == wave)
+        ),
+        default=None,
+    )
+    repair_for_wave: Optional[int] = None
+    blocked_review: Optional[Dict[str, object]] = None
+    if blocked_wave is not None:
+        source_ids = {task.task_id for task in project.tasks if task.wave == blocked_wave}
+        repairs = [
+            task
+            for task in project.tasks
+            if task.status != "done"
+            and task.wave > blocked_wave
+            and task.review_findings
+            and source_ids.intersection(task.deps)
+        ]
+        if repairs:
+            repair_for_wave = blocked_wave
+            current_wave = min(task.wave for task in repairs)
+        else:
+            current_wave = blocked_wave
+            blocked_review = {
+                "wave": blocked_wave,
+                "cycle": blocked_reviews[blocked_wave],
+                "waiting_waves": unfinished_waves,
+            }
     active = [task for task in project.tasks if task.status == "in-progress"]
     if current_wave is not None and any(task.wave != current_wave for task in active):
         task = next(task for task in active if task.wave != current_wave)
@@ -536,7 +773,7 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
                 )
 
     wave_active = any(task.wave == current_wave for task in active)
-    if current_wave is not None and not selectable and not wave_active:
+    if current_wave is not None and blocked_review is None and not selectable and not wave_active:
         stalled = [
             {
                 "id": task.task_id,
@@ -556,11 +793,37 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
             {"wave": current_wave, "tasks": stalled},
         )
 
+    # Tasks that share a regression source land one at a time, each from the landing of the one
+    # before, so the source's Verify runs on their combined edits. A parallel round leaves the
+    # primary task file pending: a retained isolate shows that the task is already dispatched.
+    # A retired source still counts: its superseding task holds its role.
+    sources = {task.task_id: {other.task_id for other, _, _, _, _ in _regression_sources(project.tasks, task)}
+               for task in concurrent}
+    started = [task for task in selectable if reports[task.task_id].get("verdict") != "none"]
+    claimed = {source for task in active + started for source in sources[task.task_id]}
+    dispatchable = []
+    for task in selectable:
+        if task in started or not sources[task.task_id] & claimed:
+            dispatchable.append(task)
+            claimed |= sources[task.task_id]
+
+    typed: Dict[str, object] = {}
+    if repair_for_wave is not None:
+        typed["repair_for_wave"] = repair_for_wave
+    if blocked_review is not None:
+        typed["blocked_review"] = blocked_review
+    # Informational: every wave below the lowest unfinished wave is all done.
+    unreviewed = _unreviewed_waves(project, sorted(
+        {task.wave for task in project.tasks if unfinished_waves and task.wave < unfinished_waves[0]}
+    ))
+    if unreviewed:
+        typed["unreviewed_waves"] = unreviewed
     return {
         "command": "ready",
         "branch": project.branch,
         "head": project.head,
         "current_wave": current_wave,
+        **typed,
         "ready": [
             {
                 "id": task.task_id,
@@ -570,10 +833,13 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
                 "deps": list(task.deps),
                 "files": list(task.files),
                 "task_file": task.task_file,
-                "verify_heavy": task.verify_heavy,
+                # The landing also runs each regression Verify, so a heavy one makes the task heavy.
+                "verify_heavy": task.verify_heavy
+                or any(other.verify_heavy and not retired_by
+                       for other, _, retired_by, _, _ in _regression_sources(project.tasks, task)),
                 **({"repo": task.repo} if task.repo else {}),
             }
-            for task in selectable
+            for task in dispatchable
         ],
     }
 

package/scripts/check_docs_audit.py

@@ -8,6 +8,10 @@ with testable claims where every claim row carries a valid type, a valid
 verdict, and real evidence, a `## Descriptive docs` list for claimless docs,
 the two sets disjoint and together equal to the frozen inventory, and a
 remediation queue with one classified row per non-verified claim.
+Queue numbers are unique, positive and strictly increasing, not necessarily
+contiguous. With `--prior-audit`, persisting rows keep their number and new
+rows use a number above every prior queue number and User rulings Queue #.
+Numbers stay bound to their original (doc, claim); never reuse a retired one.
 `## User rulings` is fixed-format durable memory. Pass the pre-rewrite audit
 with `--prior-audit FILE`; every prior row, including its Planned value, must
 remain an exact ordered prefix.
@@ -96,6 +100,12 @@ def _table_rows(body: str, width: int, label: str) -> List[List[str]]:
     return rows
 
 
+def _remediation_queue_rows(body: str, label: str, required: bool = False) -> List[List[str]]:
+    if not required and "|" not in body:
+        return []
+    return _table_rows(body, 6, label)
+
+
 def _not_placeholder(value: str, label: str) -> None:
     normalized = value.strip().strip("`\"").strip()
     if (
@@ -247,6 +257,63 @@ def _check_carried(
             raise AuditError(f"{label}: doc or evidence changed since the prior audit: {', '.join(moved)}")
 
 
+def _queue_number(number: str, label: str) -> int:
+    if not re.fullmatch(r"[1-9][0-9]*", number):
+        raise AuditError(f"{label}: queue numbers must be positive integers")
+    return int(number)
+
+
+def _check_queue_number_history(
+    queue: Sequence[Sequence[str]],
+    prior_sections: Dict[str, str],
+) -> None:
+    if "Remediation queue" not in prior_sections:
+        raise AuditError("prior audit is missing section: ## Remediation queue")
+    if "User rulings" not in prior_sections:
+        raise AuditError("prior audit is missing section: ## User rulings")
+
+    prior_queue = _remediation_queue_rows(
+        prior_sections["Remediation queue"], "Prior remediation queue"
+    )
+    prior_items_by_number: Dict[int, tuple] = {}
+    prior_numbers_by_item: Dict[tuple, int] = {}
+    for number, doc, claim, *_ in prior_queue:
+        numeric_number = _queue_number(number, "Prior remediation queue")
+        identity = (doc, claim)
+        prior_items_by_number[numeric_number] = identity
+        prior_numbers_by_item[identity] = numeric_number
+
+    prior_rulings = _user_rulings(
+        prior_sections["User rulings"], "Prior user rulings"
+    )
+    prior_ruling_numbers = {int(row[0]) for row in prior_rulings}
+    prior_max = max(
+        [*prior_items_by_number, *prior_ruling_numbers],
+        default=0,
+    )
+
+    for number, doc, claim, *_ in queue:
+        numeric_number = _queue_number(number, "Remediation queue")
+        identity = (doc, claim)
+        prior_number = prior_numbers_by_item.get(identity)
+        if numeric_number in prior_ruling_numbers and prior_items_by_number.get(numeric_number) != identity:
+            raise AuditError(
+                f"Remediation queue #{number}: Queue # {number} is already retired by a prior "
+                "User rulings row for a different item; give new items a number above every "
+                "prior queue number and User rulings Queue #"
+            )
+        if prior_number is not None and numeric_number != prior_number:
+            raise AuditError(
+                f"Remediation queue #{number}: persisting item must keep its prior queue "
+                f"number #{prior_number}"
+            )
+        if prior_number is None and numeric_number <= prior_max:
+            raise AuditError(
+                f"Remediation queue #{number}: new items must use a number above every prior "
+                f"queue number and User rulings Queue # (maximum {prior_max})"
+            )
+
+
 def validate(
     repo: Path,
     audit_relative: str,
@@ -354,7 +421,9 @@ def validate(
 
     queue_body = sections["Remediation queue"]
     non_verified = sum(tallies[verdict] for verdict in VERDICTS if verdict != "verified")
-    queue = _table_rows(queue_body, 6, "Remediation queue") if non_verified or "|" in queue_body else []
+    queue = _remediation_queue_rows(
+        queue_body, "Remediation queue", required=bool(non_verified)
+    )
     expected_queue = sorted(
         (doc, claim["claim"], claim["verdict"])
         for doc, claims in docs.items()
@@ -371,9 +440,13 @@ def validate(
         _not_placeholder(action, f"Remediation queue #{number}: action")
     if len(queue) != non_verified:
         raise AuditError(f"Remediation queue has {len(queue)} rows for {non_verified} non-verified claims")
-    numbers = [row[0] for row in queue]
-    if numbers != [str(number) for number in range(1, len(queue) + 1)]:
-        raise AuditError("Remediation queue numbers must be contiguous and ordered from 1")
+    numbers = [_queue_number(row[0], "Remediation queue") for row in queue]
+    if any(later <= earlier for earlier, later in zip(numbers, numbers[1:])):
+        raise AuditError(
+            "Remediation queue numbers must be unique and strictly increasing"
+        )
+    if prior_sections is not None:
+        _check_queue_number_history(queue, prior_sections)
     actual_queue = sorted((doc, claim, verdict) for _, doc, claim, verdict, _, _ in queue)
     if actual_queue != expected_queue:
         raise AuditError(