@opengsd/gsd-path 1.3.0 → 1.4.0

package/skills/gsd-path-build/scripts/archive_milestone.py

@@ -24,13 +24,15 @@ from datetime import date
 from pathlib import Path, PurePosixPath
 from typing import Iterator, NamedTuple, Optional, Sequence
 
-try:
-    from isolation import (
+if __package__:
+    from .isolation import (
         IsolationError,
+        prune_host_scratch,
+        PROJECT_ENTRIES,
         checkpoint as isolation_checkpoint,
         verify_landed_task_files,
     )
-    from pipeline_git import (
+    from .pipeline_git import (
         bound_branch_name,
         default_branch_name,
         integrate_commit_body,
@@ -43,45 +45,75 @@ try:
         ship_commit_body,
         ship_subject,
     )
-    from review_panel import KNOWN_FAMILIES, ReviewPanelError, parse_sources
-    from pipeline_state import (
+    from .review_panel import KNOWN_FAMILIES, ReviewPanelError, parse_sources
+    from .pipeline_state import (
         PipelineState,
         PipelineStateError,
         load_state,
         transition_state,
     )
-    import _common
-except ImportError:  # pragma: no cover - package import used by tests
-    from scripts.isolation import (
-        IsolationError,
-        checkpoint as isolation_checkpoint,
-        verify_landed_task_files,
-    )
-    from scripts.pipeline_git import (
-        bound_branch_name,
-        default_branch_name,
-        integrate_commit_body,
-        integrate_subject,
-        is_bound_branch,
-        is_integrate_subject,
-        is_ship_subject,
-        milestone_id,
-        milestone_number,
-        ship_commit_body,
-        ship_subject,
-    )
-    from scripts.review_panel import KNOWN_FAMILIES, ReviewPanelError, parse_sources
-    from scripts.pipeline_state import (
-        PipelineState,
-        PipelineStateError,
-        load_state,
-        transition_state,
-    )
-    from scripts import _common
-
-if sys.platform == "win32":
-    import msvcrt
+    from . import _common
 else:
+    try:
+        from isolation import (
+            IsolationError,
+            prune_host_scratch,
+            PROJECT_ENTRIES,
+            checkpoint as isolation_checkpoint,
+            verify_landed_task_files,
+        )
+        from pipeline_git import (
+            bound_branch_name,
+            default_branch_name,
+            integrate_commit_body,
+            integrate_subject,
+            is_bound_branch,
+            is_integrate_subject,
+            is_ship_subject,
+            milestone_id,
+            milestone_number,
+            ship_commit_body,
+            ship_subject,
+        )
+        from review_panel import KNOWN_FAMILIES, ReviewPanelError, parse_sources
+        from pipeline_state import (
+            PipelineState,
+            PipelineStateError,
+            load_state,
+            transition_state,
+        )
+        import _common
+    except ImportError:  # pragma: no cover - package import used by tests
+        from scripts.isolation import (
+            IsolationError,
+            prune_host_scratch,
+            PROJECT_ENTRIES,
+            checkpoint as isolation_checkpoint,
+            verify_landed_task_files,
+        )
+        from scripts.pipeline_git import (
+            bound_branch_name,
+            default_branch_name,
+            integrate_commit_body,
+            integrate_subject,
+            is_bound_branch,
+            is_integrate_subject,
+            is_ship_subject,
+            milestone_id,
+            milestone_number,
+            ship_commit_body,
+            ship_subject,
+        )
+        from scripts.review_panel import KNOWN_FAMILIES, ReviewPanelError, parse_sources
+        from scripts.pipeline_state import (
+            PipelineState,
+            PipelineStateError,
+            load_state,
+            transition_state,
+        )
+        from scripts import _common
+
+if sys.platform != "win32":
     import fcntl
 
 
@@ -350,31 +382,25 @@ else:  # standalone script or sibling import
 @contextmanager
 def discussion_lock(active_root: Path) -> Iterator[None]:
     if sys.platform == "win32":
-        lock_value = require_git_success(
-            run_git(
-                active_root.parent,
-                "rev-parse",
-                "--git-path",
-                "gsd-path-discussion.lock",
-            ),
-            "resolve discussion lock",
+        # Windows cannot flock a directory, so lock a file in the git dir,
+        # where it never shows up as an untracked project file.
+        resolved = run_git(
+            active_root.parent,
+            "rev-parse",
+            "--git-path",
+            "gsd-path-discussion.lock",
         )
-        lock_path = Path(lock_value)
+        if resolved.returncode != 0 or not resolved.stdout.strip():
+            # Outside git, which the POSIX directory lock allows: the same
+            # named kernel mutex pipeline_state holds for this directory.
+            with _common.directory_mutex(active_root):
+                yield
+            return
+        lock_path = Path(resolved.stdout.strip())
         if not lock_path.is_absolute():
             lock_path = active_root.parent / lock_path
-        lock_path.parent.mkdir(parents=True, exist_ok=True)
-        with lock_path.open("a+b") as handle:
-            handle.seek(0, os.SEEK_END)
-            if handle.tell() == 0:
-                handle.write(b"\0")
-                handle.flush()
-            handle.seek(0)
-            msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
-            try:
-                yield
-            finally:
-                handle.seek(0)
-                msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+        with _common.exclusive_lock(lock_path):
+            yield
         return
 
     descriptor = os.open(active_root, os.O_RDONLY)
@@ -535,6 +561,8 @@ def archive_sequence_entries(project: Path) -> dict[int, Path]:
     archive_root = safe_archive_root(project)
     entries = {}
     for candidate in archive_root.iterdir():
+        if _common.is_ignored_junk(candidate):
+            continue
         match = ARCHIVE_PATTERN.fullmatch(candidate.name)
         if match is None or not candidate.is_dir():
             raise ArchiveError(f"archive root has a noncanonical entry: {candidate.name}")
@@ -620,7 +648,7 @@ def pending_ruling_count(audit: Path) -> int:
 
 
 def is_existing_carry_forward(source: Path, archived: Path) -> bool:
-    entries = list(source.iterdir()) if source.is_dir() else []
+    entries = [path for path in source.iterdir() if not _common.is_ignored_junk(path)] if source.is_dir() else []
     archived_audit = archived / "DOCS-AUDIT.md"
     if not archived_audit.is_file() or pending_ruling_count(archived_audit) == 0:
         return False
@@ -681,7 +709,7 @@ def is_real_file(path: Path) -> bool:
 def canonical_task_files(tasks: Path) -> Sequence[Path]:
     if tasks.is_symlink() or not tasks.is_dir():
         return ()
-    candidates = sorted(tasks.iterdir())
+    candidates = sorted(path for path in tasks.iterdir() if not _common.is_ignored_junk(path))
     if any(
         not TASK_FILE_PATTERN.fullmatch(path.name) or not is_real_file(path)
         for path in candidates
@@ -1001,6 +1029,7 @@ def prepare(repo: Path, slug: str) -> dict:
     project = repo.resolve()
     active_root = require_project_layout(project)
     with discussion_lock(active_root):
+        prune_host_scratch(active_root)
         return prepare_locked(project, active_root, slug)
 
 
@@ -1131,13 +1160,7 @@ def require_canonical_abandon_inputs(active_root: Path, archive: Path) -> None:
 def write_abandon_manifest(
     archive: Path, slug: str, ruling: str, abandoned_on: str
 ) -> None:
-    contents = []
-    for path in archive.rglob("*"):
-        if path.is_symlink():
-            raise ArchiveError(f"archive contents must not be symlinks: {path}")
-        if path.is_file() and path.name != "MANIFEST.md":
-            contents.append(path.relative_to(archive).as_posix())
-    contents.sort()
+    contents = archive_file_inventory(archive)
     listed_contents = "\n".join(f"- {path}" for path in contents)
     atomic_replace(
         archive / "MANIFEST.md",
@@ -1693,10 +1716,13 @@ def require_clean_older_archives(project: Path, configured: str) -> None:
     current_prefix = f"{configured}/"
     dirty = set()
     ignored_current = set()
+    current_files = {f"{configured}/{name}" for name in archive_file_inventory(project / configured)}
     for record in status.split("\0"):
         has_status = len(record) > 3 and record[2] == " "
         code = record[:2] if has_status else ""
         path = record[3:] if has_status else record
+        if code == "!!" and _common.is_ignored_junk(project / path) and path not in current_files:
+            continue  # Ignored OS junk never ships and is not an artifact.
         if code == "!!" and (
             path == ".project/archive/"
             or path == configured
@@ -1745,7 +1771,7 @@ def require_clean_active_root(active_root: Path, archive: Path) -> None:
     archived_audit = archive / "research" / "DOCS-AUDIT.md"
     carried_forward = pending_ruling_count(archived_audit)
     # Persistent project metadata ships but never archives with a milestone.
-    allowed = {"STATE.md", "REPOSITORY.md", "archive", "LESSONS.md", "next"}
+    allowed = {"STATE.md", "REPOSITORY.md", "MEMBERS.md", "archive", "LESSONS.md", "next"}
     for name in ("config.json", "model-policy.json"):
         path = active_root / name
         if path.exists() or path.is_symlink():
@@ -1760,7 +1786,9 @@ def require_clean_active_root(active_root: Path, archive: Path) -> None:
     if carried_forward:
         allowed.add("research")
         active_audit = active_research / "DOCS-AUDIT.md"
-        entries = sorted(path.name for path in active_research.iterdir()) if active_research.is_dir() else []
+        entries = sorted(
+            path.name for path in active_research.iterdir() if not _common.is_ignored_junk(path)
+        ) if active_research.is_dir() else []
         if (
             active_research.is_symlink()
             or entries != ["DOCS-AUDIT.md"]
@@ -1772,7 +1800,12 @@ def require_clean_active_root(active_root: Path, archive: Path) -> None:
     elif active_research.exists() or active_research.is_symlink():
         raise ArchiveError("active research exists without a pending carry-forward queue")
 
-    unexpected = sorted(path.name for path in active_root.iterdir() if path.name not in allowed)
+    # Ignored, untracked non-pipeline entries (.DS_Store) never ship; pipeline entries block even when ignored.
+    visible = _common.git_visible_entries(active_root)
+    unexpected = sorted(
+        path.name for path in active_root.iterdir()
+        if path.name not in allowed and (path.name in PROJECT_ENTRIES or path.name in visible)
+    )
     if unexpected:
         raise ArchiveError(f"unexpected active .project paths remain: {', '.join(unexpected)}")
 
@@ -1848,6 +1881,12 @@ def prepared_transaction(
 
 
 def preflight(repo: Path) -> dict:
+    project = repo.resolve()
+    active_root = require_project_layout(project)
+    configured = strict_state(project)[0].archive
+    current = (archive_from_state(project, configured)
+               if configured and not archive_is_committed(project, configured) else None)
+    prune_host_scratch(active_root, current)
     project, _, state, configured, archive, _, reviewed_head = prepared_transaction(
         repo,
         ("ship", "shipped"),
@@ -1937,6 +1976,28 @@ def validate_shipped_roadmap(text: str, state: PipelineState, archive: str) -> N
         raise ArchiveError("ROADMAP.md must record Status: shipped and the exact Archive")
 
 
+def member_ship_rows(project: Path, archive: Path, ship_commit: str) -> list:
+    """Ship body member lines, bound to the archived lock order and FINAL.md member heads."""
+    if __package__:
+        from . import check_handoffs
+        from .pipeline_git import ship_member_rows
+    else:
+        import check_handoffs
+        from pipeline_git import ship_member_rows
+    message = require_git_success(run_git(project, "show", "-s", "--format=%B", ship_commit), "inspect ship body")
+    rows = ship_member_rows(message)
+    lock = archive / "build" / "members.json"
+    locked = [entry["name"] for entry in json.loads(lock.read_text(encoding="utf-8"))["members"]] if lock.is_file() else []
+    try:
+        heads = check_handoffs.member_reviewed_heads((archive / "review" / "FINAL.md").read_text(encoding="utf-8"),
+                                                     "FINAL.md")
+    except check_handoffs.HandoffError as error:
+        raise ArchiveError(str(error)) from error
+    if [row["name"] for row in rows] != locked or {row["name"]: row["reviewed_head"] for row in rows} != heads:
+        raise ArchiveError("ship commit member lines must name each locked member at its FINAL.md reviewed HEAD")
+    return rows
+
+
 def validate(repo: Path, *, historical: bool = False) -> dict:
     project, _, state, configured, archive, archived_files, reviewed_head = prepared_transaction(
         repo,
@@ -2030,6 +2091,7 @@ def validate(repo: Path, *, historical: bool = False) -> dict:
         )
     require_committed_carry_forward(project, configured, archive, ship_commit)
 
+    members = member_ship_rows(project, archive, ship_commit)
     ship_parent = require_git_success(run_git(project, "rev-parse", f"{ship_commit}^"), "resolve ship parent")
     if reviewed_head != ship_parent:
         raise ArchiveError(
@@ -2039,7 +2101,7 @@ def validate(repo: Path, *, historical: bool = False) -> dict:
         project,
         ship_commit,
         ship_subject(archive.name),
-        ship_commit_body(configured, reviewed_head),
+        ship_commit_body(configured, reviewed_head, members),
         "ship",
     )
 
@@ -2077,7 +2139,10 @@ def validate(repo: Path, *, historical: bool = False) -> dict:
             + f"; run: git -C {project} revert <commits after {ship_commit[:12]} that touch .project>"
         )
 
-    return {"archive": configured, "commit": ship_commit}
+    result = {"archive": configured, "commit": ship_commit}
+    if members:
+        result["members"] = members
+    return result
 
 
 def parser() -> argparse.ArgumentParser:
@@ -2119,6 +2184,12 @@ def parser() -> argparse.ArgumentParser:
     )
     refresh_parser.add_argument("--repo", required=True, type=Path)
 
+    close_members_parser = subparsers.add_parser(
+        "close-members",
+        help="after preflight, integrate each locked member in lock order before the ship commit",
+    )
+    close_members_parser.add_argument("--repo", required=True, type=Path)
+
     integrate_parser = subparsers.add_parser(
         "integrate",
         help="resume and publish the shipped milestone integration",
@@ -2139,7 +2210,7 @@ def parser() -> argparse.ArgumentParser:
 def main(argv: Optional[Sequence[str]] = None) -> int:
     arguments = parser().parse_args(argv)
     try:
-        if arguments.command in {"validate-integrated", "refresh-origin", "integrate"}:
+        if arguments.command in {"validate-integrated", "refresh-origin", "integrate", "close-members"}:
             if __package__:
                 from . import integration
             else:
@@ -2156,6 +2227,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
             result = integration.validate_integrated(arguments.repo, arguments.slug)
         elif arguments.command == "refresh-origin":
             result = integration.refresh_origin(arguments.repo)
+        elif arguments.command == "close-members":
+            result = integration.close_members(arguments.repo)
         elif arguments.command == "integrate":
             result = integration.integrate(arguments.repo, arguments.slug)
         else:

package/skills/gsd-path-build/scripts/build_recovery.py

@@ -90,6 +90,8 @@ def settled_tasks(repo: Path) -> dict[str, tuple[str, dict]]:
         raise state.PipelineStateError("build recovery requires real task artifacts")
     tasks = {}
     for path in sorted(directory.iterdir()):
+        if state._common.is_ignored_junk(path):
+            continue
         if not re.fullmatch(r"T[0-9]{3}-[a-z0-9][a-z0-9-]*\.md", path.name):
             raise state.PipelineStateError(f"invalid recovery task path: {path.name}")
         text = state._read_real_file(path, path.name)
@@ -165,7 +167,8 @@ def validate_review_backup(repo: Path, recovery: dict, directory: Path) -> None:
         if path:
             expected[str(Path(path).relative_to(".project/review"))] = checkpoint_runtime()._git_text_at(repo, recovery["base"], path)
     actual = {str(path.relative_to(directory)): path.read_text(encoding="utf-8")
-              for path in directory.rglob("*") if path.is_file()} if directory.exists() else {}
+              for path in directory.rglob("*")
+              if path.is_file() and not state._common.is_ignored_junk(path)} if directory.exists() else {}
     if actual != expected:
         raise state.PipelineStateError("review backup differs from the recovery base")
 

package/skills/gsd-path-build/scripts/build_state.py

@@ -14,28 +14,40 @@ from datetime import datetime, timezone
 from pathlib import Path, PurePosixPath
 from typing import Dict, List, Mapping, Optional, Sequence, Tuple
 
-try:
-    from check_task_briefs import _frontmatter, _sections
-    from isolation import (
+if __package__:
+    from .check_task_briefs import _frontmatter, _sections
+    from .isolation import (
         VERIFY_LEDGER_PATH,
         IsolationError,
         recover as recover_isolation,
         verify_landed_task_files,
     )
-    from pipeline_git import task_commit_subject
-    from pipeline_state import PipelineStateError, load_state
-    import _common
-except ImportError:  # pragma: no cover - package imports used by tests
-    from scripts.check_task_briefs import _frontmatter, _sections
-    from scripts.isolation import (
-        VERIFY_LEDGER_PATH,
-        IsolationError,
-        recover as recover_isolation,
-        verify_landed_task_files,
-    )
-    from scripts.pipeline_git import task_commit_subject
-    from scripts.pipeline_state import PipelineStateError, load_state
-    from scripts import _common
+    from .pipeline_git import task_commit_subject
+    from .pipeline_state import PipelineStateError, load_state
+    from . import _common
+else:
+    try:
+        from check_task_briefs import _frontmatter, _sections
+        from isolation import (
+            VERIFY_LEDGER_PATH,
+            IsolationError,
+            recover as recover_isolation,
+            verify_landed_task_files,
+        )
+        from pipeline_git import task_commit_subject
+        from pipeline_state import PipelineStateError, load_state
+        import _common
+    except ImportError:  # pragma: no cover - package imports used by tests
+        from scripts.check_task_briefs import _frontmatter, _sections
+        from scripts.isolation import (
+            VERIFY_LEDGER_PATH,
+            IsolationError,
+            recover as recover_isolation,
+            verify_landed_task_files,
+        )
+        from scripts.pipeline_git import task_commit_subject
+        from scripts.pipeline_state import PipelineStateError, load_state
+        from scripts import _common
 
 
 DEFAULT_PROJECT_DIR = ".project"
@@ -75,6 +87,7 @@ class Task:
     task_branch: Optional[str]
     task_file: str
     verify_heavy: bool
+    repo: str = ""
 
 
 @dataclass(frozen=True)
@@ -227,6 +240,7 @@ def _parse_task(path: Path, relative_path: str, task_file: Optional[str] = None)
         task_branch=_nullable_string(fields, "task_branch", label),
         task_file=task_file or relative_path,
         verify_heavy=_verify_heavy(text, label),
+        repo=str(fields.get("repo") or ""),
     )
 
 
@@ -284,6 +298,8 @@ def _load_tasks(
         raise BuildStateError("missing-input", f"task directory is missing or unsafe: {tasks_dir}")
     task_paths: Dict[str, Path] = {}
     for path in sorted(tasks_dir.iterdir()):
+        if _common.is_ignored_junk(path):
+            continue
         match = TASK_FILE_RE.fullmatch(path.name)
         if match is None or not path.is_file() or path.is_symlink():
             raise BuildStateError(
@@ -464,6 +480,8 @@ def _validate_ready_metadata(project: Project) -> None:
 
 
 def _overlap(left: Task, right: Task) -> List[str]:
+    if left.repo != right.repo:
+        return []
     return sorted(set(left.files) & set(right.files))
 
 
@@ -553,6 +571,7 @@ def ready(repo: str, project_dir: str = DEFAULT_PROJECT_DIR, *,
                 "files": list(task.files),
                 "task_file": task.task_file,
                 "verify_heavy": task.verify_heavy,
+                **({"repo": task.repo} if task.repo else {}),
             }
             for task in selectable
         ],
@@ -726,6 +745,22 @@ def _ledger_path(repo: Path) -> Path:
     return path
 
 
+def _member_checkout(repo: Path, member: Optional[str]) -> Path:
+    if member is None:
+        return repo
+    try:
+        import members
+    except ImportError:  # pragma: no cover - package import used by tests
+        from scripts import members
+    try:
+        listed = {item["name"]: Path(item["checkout"]) for item in members.read_members(repo)}
+    except members.MembersError as error:
+        raise BuildStateError("invalid-member", str(error)) from error
+    if member not in listed:
+        raise BuildStateError("invalid-member", f"{member} is not in MEMBERS.md")
+    return listed[member]
+
+
 def _ledger_key(repo: Path, command: str, commit: str) -> Tuple[str, str]:
     if not command.strip():
         raise BuildStateError("invalid-command", "--command must not be empty")
@@ -742,13 +777,20 @@ def _ledger_entries(path: Path) -> List[Dict[str, object]]:
 
 
 def verify_record(repo: str, command: str, commit: str, result: str,
-                  execution: Optional[Dict[str, object]] = None) -> Dict[str, object]:
-    """Append one verify run (command, commit, result, timestamp) to the ledger."""
+                  execution: Optional[Dict[str, object]] = None,
+                  member: Optional[str] = None) -> Dict[str, object]:
+    """Append one verify run (command, commit, result, timestamp) to the ledger.
+
+    A member run names its member and a commit in that member's repository.
+    """
 
     repository = _repo_root(repo)
     if result not in VERIFY_RESULTS:
         raise BuildStateError("invalid-result", "--result must be pass or fail")
-    normalized, commit = _ledger_key(repository, command, commit)
+    normalized, commit = _ledger_key(_member_checkout(repository, member), command, commit)
+    ignore_error = _common.project_ignore_error(repository)
+    if ignore_error:
+        raise BuildStateError("ignored-ledger", ignore_error)
     path = _ledger_path(repository)
     _ledger_entries(path)
     entry = {
@@ -757,6 +799,7 @@ def verify_record(repo: str, command: str, commit: str, result: str,
         "commit": commit,
         "result": result,
         "recorded_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        **({"repo": member} if member else {}),
     }
     if execution is not None:
         if (not isinstance(execution.get("stdout"), str)
@@ -771,12 +814,12 @@ def verify_record(repo: str, command: str, commit: str, result: str,
     return {"command": "verify-record", "ledger": VERIFY_LEDGER_PATH, "entry": entry}
 
 
-def verify_lookup(repo: str, command: str, commit: str) -> Dict[str, object]:
+def verify_lookup(repo: str, command: str, commit: str, member: Optional[str] = None) -> Dict[str, object]:
     """Return the latest recorded run of this command at this commit, or a miss."""
 
     repository = _repo_root(repo)
-    normalized, commit = _ledger_key(repository, command, commit)
-    entry = _common.latest_verify_entry(_ledger_entries(_ledger_path(repository)), normalized, commit)
+    normalized, commit = _ledger_key(_member_checkout(repository, member), command, commit)
+    entry = _common.latest_verify_entry(_ledger_entries(_ledger_path(repository)), normalized, commit, member)
     return {
         "command": "verify-lookup",
         "ledger": VERIFY_LEDGER_PATH,