@opengsd/gsd-path 1.4.0 → 1.5.0

package/skills/gsd-path-docs-audit/scripts/detect_project.py

@@ -55,10 +55,22 @@ try:
         fcntl.flock(fd, fcntl.LOCK_EX)
 
 except ImportError:  # Windows does not provide POSIX file locks.
+    import errno
     import msvcrt
+    import time
 
     def lock_exclusive(fd: int) -> None:
-        msvcrt.locking(fd, msvcrt.LK_LOCK, 1)
+        delay = 0.01
+        while True:
+            try:
+                os.lseek(fd, 0, os.SEEK_SET)
+                msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
+                return
+            except OSError as error:
+                if error.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
+                    raise
+                time.sleep(delay)
+                delay = min(delay * 2, 0.25)
 
 
 IGNORE_DIRS = {
@@ -217,16 +229,20 @@ HTML_BLOCK_START = re.compile(
     re.IGNORECASE,
 )
 REPARSE_NAME_SURROGATE = 0x20000000
-ANCHORED_EVIDENCE_SUPPORTED = (
+_POSIX_ANCHORED_EVIDENCE_SUPPORTED = (
     hasattr(os, "O_DIRECTORY")
     and hasattr(os, "O_NOFOLLOW")
     and os.open in os.supports_dir_fd
     and os.stat in os.supports_dir_fd
     and os.stat in os.supports_follow_symlinks
 )
+WINDOWS_ANCHORED_EVIDENCE_SUPPORTED = os.name == "nt"
+ANCHORED_EVIDENCE_SUPPORTED = (
+    _POSIX_ANCHORED_EVIDENCE_SUPPORTED or WINDOWS_ANCHORED_EVIDENCE_SUPPORTED
+)
 LISTDIR_DIR_FD_SUPPORTED = os.listdir in getattr(os, "supports_fd", ())
 ANCHORED_STATE_CREATE_SUPPORTED = (
-    ANCHORED_EVIDENCE_SUPPORTED
+    _POSIX_ANCHORED_EVIDENCE_SUPPORTED
     and LISTDIR_DIR_FD_SUPPORTED
     and os.mkdir in getattr(os, "supports_dir_fd", ())
     and os.unlink in getattr(os, "supports_dir_fd", ())
@@ -269,11 +285,12 @@ def is_link_like_status(status: os.stat_result) -> bool:
 def is_link_like(path: Path, status: os.stat_result) -> bool:
     if is_link_like_status(status):
         return True
-    isjunction = getattr(os.path, "isjunction", None)
-    if os.name != "nt" or isjunction is None:
+    if os.name != "nt":
         return False
     try:
-        return bool(isjunction(path))
+        return is_link_like_status(os.lstat(path))
+    except FileNotFoundError:
+        return False
     except OSError as error:
         raise DetectError(f"cannot inspect link-like evidence: {path}: {error}") from error
 
@@ -332,7 +349,7 @@ def is_managed_pipeline_directory(relative: str, root: Path) -> bool:
     return is_verified_skill_bundle(relative, root)
 
 
-def is_verified_skill_bundle_at(relative: str, directory_fd: int) -> bool:
+def is_verified_skill_bundle_at(relative: str, directory_fd: int, root: Path) -> bool:
     parts = PurePosixPath(relative).parts
     if not parts:
         return False
@@ -342,19 +359,45 @@ def is_verified_skill_bundle_at(relative: str, directory_fd: int) -> bool:
         return False
     if is_verified_installer_bundle(parts):
         return True
+    if _POSIX_ANCHORED_EVIDENCE_SUPPORTED:
+        try:
+            status = os.stat(
+                "SKILL.md",
+                dir_fd=directory_fd,
+                follow_symlinks=False,
+            )
+        except FileNotFoundError:
+            return False
+        except OSError as error:
+            raise DetectError(
+                f"cannot inspect managed skill bundle: {relative}: {error}"
+            ) from error
+        return not is_link_like_status(status) and stat.S_ISREG(status.st_mode)
+    skill_fd = None
     try:
-        status = os.stat(
+        win = get_windows_handles()
+        skill_fd = win.open_relative(
+            directory_fd,
             "SKILL.md",
-            dir_fd=directory_fd,
-            follow_symlinks=False,
+            _FILE_READ_ATTRIBUTES,
+            _PIN_SHARE,
+            _FILE_OPEN,
+            0,
         )
+        status = os.fstat(skill_fd)
     except FileNotFoundError:
         return False
     except OSError as error:
+        if getattr(error, "winerror", None) == 2:
+            return False
         raise DetectError(
             f"cannot inspect managed skill bundle: {relative}: {error}"
         ) from error
-    return not is_link_like_status(status) and stat.S_ISREG(status.st_mode)
+    finally:
+        if skill_fd is not None:
+            os.close(skill_fd)
+    skill = root.joinpath(*parts) / "SKILL.md"
+    return not is_link_like(skill, status) and stat.S_ISREG(status.st_mode)
 
 
 def is_fixed_managed_pipeline_artifact(relative: str) -> bool:
@@ -423,13 +466,20 @@ def read_regular_evidence(
         ) from error
     if not relative.parts:
         raise DetectError(f"{evidence_name} evidence path is empty: {path}")
-    if ANCHORED_EVIDENCE_SUPPORTED:
+    if _POSIX_ANCHORED_EVIDENCE_SUPPORTED:
         return read_anchored_evidence(
             relative,
             root,
             missing_ok=missing_ok,
             evidence_name=evidence_name,
         )
+    if WINDOWS_ANCHORED_EVIDENCE_SUPPORTED:
+        return read_windows_anchored_evidence(
+            relative,
+            root,
+            missing_ok=missing_ok,
+            evidence_name=evidence_name,
+        )
     return read_lstat_open_evidence(
         relative,
         root,
@@ -447,6 +497,17 @@ def directory_flags() -> int:
     )
 
 
+def listdir_anchored(dir_fd: int) -> list[str]:
+    """List names through a fresh directory descriptor under dir_fd."""
+    fresh = os.open(".", directory_flags(), dir_fd=dir_fd)
+    try:
+        if not os.path.samestat(os.fstat(fresh), os.fstat(dir_fd)):
+            raise DetectError("anchored directory listing lost its identity")
+        return os.listdir(fresh)
+    finally:
+        os.close(fresh)
+
+
 def file_flags() -> int:
     return (
         os.O_RDONLY
@@ -766,9 +827,12 @@ def raise_walk_error(error: OSError) -> None:
 
 
 def iter_worktree_files(root: Path) -> Iterable[str]:
-    if ANCHORED_EVIDENCE_SUPPORTED and LISTDIR_DIR_FD_SUPPORTED:
+    if _POSIX_ANCHORED_EVIDENCE_SUPPORTED and LISTDIR_DIR_FD_SUPPORTED:
         yield from iter_worktree_files_anchored(root)
         return
+    if WINDOWS_ANCHORED_EVIDENCE_SUPPORTED:
+        yield from iter_worktree_files_windows_anchored(root)
+        return
     for dirpath, dirnames, filenames in os.walk(
         root,
         followlinks=False,
@@ -826,7 +890,7 @@ def iter_from_dir_fd(
     dir_fd: int, relative_dir: str, root: Path
 ) -> Iterable[str]:
     try:
-        names = os.listdir(dir_fd)
+        names = listdir_anchored(dir_fd)
     except OSError as error:
         raise DetectError(f"cannot traverse filesystem evidence: {error}") from error
     for name in names:
@@ -862,7 +926,7 @@ def iter_from_dir_fd(
                     raise DetectError(
                         f"filesystem evidence changed while traversing: {relative}"
                     )
-                if is_verified_skill_bundle_at(relative, child):
+                if is_verified_skill_bundle_at(relative, child, root):
                     continue
                 yield from iter_from_dir_fd(child, relative, root)
             finally:
@@ -1302,7 +1366,7 @@ def close_file_descriptors(
 
 def project_entries(project_fd: int, project: Path) -> set:
     """Names in .project without ignored OS junk (Finder may add .DS_Store at any time)."""
-    entries = set(os.listdir(project_fd))
+    entries = set(listdir_anchored(project_fd))
     for name in entries & _common.OS_JUNK_NAMES:
         try:
             status = os.stat(name, dir_fd=project_fd, follow_symlinks=False)
@@ -1413,12 +1477,12 @@ def write_state_anchored(
                 follow_symlinks=False,
             )
         except FileNotFoundError as error:
-            if "STATE.md" in os.listdir(project_fd):
+            if "STATE.md" in listdir_anchored(project_fd):
                 raise DetectError("STATE.md already exists") from error
             raise DetectError("temporary STATE.md disappeared") from error
         if not os.path.samestat(created_status, named_temporary):
             raise DetectError("temporary STATE.md changed before writing")
-        if "STATE.md" in os.listdir(project_fd):
+        if "STATE.md" in listdir_anchored(project_fd):
             raise DetectError("STATE.md already exists")
         os.ftruncate(state_fd, 0)
         write_all(state_fd, payload)
@@ -1653,6 +1717,167 @@ class _WindowsHandles:
             raise ctypes.WinError(ctypes.get_last_error())
 
 
+_WINDOWS_HANDLES: Optional[_WindowsHandles] = None
+
+
+def get_windows_handles() -> _WindowsHandles:
+    global _WINDOWS_HANDLES
+    if _WINDOWS_HANDLES is None:
+        _WINDOWS_HANDLES = _WindowsHandles()
+    return _WINDOWS_HANDLES
+
+
+def _windows_missing_ok(error: OSError) -> bool:
+    return getattr(error, "winerror", None) in {2, 3}
+
+
+def read_windows_anchored_evidence(
+    relative: Path,
+    root: Path,
+    *,
+    missing_ok: bool,
+    evidence_name: str,
+) -> Optional[str]:
+    win = get_windows_handles()
+    descriptors: list[int] = []
+    try:
+        descriptors.append(win.open_root(root))
+        opened_root = os.fstat(descriptors[-1])
+        if not stat.S_ISDIR(opened_root.st_mode):
+            raise DetectError(f"repo evidence is not a directory: {root}")
+        for index, part in enumerate(relative.parts):
+            current = root.joinpath(*relative.parts[: index + 1])
+            final = index == len(relative.parts) - 1
+            access = _GENERIC_READ if final else _DIRECTORY_ACCESS
+            options = _FILE_NON_DIRECTORY_FILE if final else _FILE_DIRECTORY_FILE
+            try:
+                descriptor = win.open_relative(
+                    descriptors[-1],
+                    part,
+                    access,
+                    _PIN_SHARE,
+                    _FILE_OPEN,
+                    options,
+                )
+            except OSError as error:
+                if missing_ok and _windows_missing_ok(error):
+                    return None
+                raise DetectError(
+                    f"filesystem evidence disappeared: {root / relative}"
+                ) from error
+            descriptors.append(descriptor)
+            actual = os.fstat(descriptor)
+            if is_link_like(current, actual):
+                raise DetectError(
+                    f"link-like {evidence_name} evidence is not allowed: {current}"
+                )
+            if final and not stat.S_ISREG(actual.st_mode):
+                raise DetectError(
+                    f"{evidence_name} evidence is not a regular file: {current}"
+                )
+            if not final and not stat.S_ISDIR(actual.st_mode):
+                raise DetectError(
+                    f"{evidence_name} evidence parent is not a directory: {current}"
+                )
+        with os.fdopen(descriptors.pop(), "rb") as stream:
+            data = stream.read()
+    except DetectError:
+        raise
+    except OSError as error:
+        raise DetectError(
+            f"cannot read {evidence_name} evidence: {root / relative}"
+        ) from error
+    finally:
+        for descriptor in reversed(descriptors):
+            os.close(descriptor)
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError as error:
+        raise DetectError(
+            f"cannot read {evidence_name} evidence: {root / relative}"
+        ) from error
+
+
+def iter_worktree_files_windows_anchored(root: Path) -> Iterable[str]:
+    win = get_windows_handles()
+    try:
+        root_fd = win.open_root(root)
+    except OSError as error:
+        raise DetectError(f"cannot traverse filesystem evidence: {error}") from error
+    try:
+        root_status = os.fstat(root_fd)
+        if not stat.S_ISDIR(root_status.st_mode):
+            raise DetectError(f"repo evidence is not a directory: {root}")
+        yield from iter_from_windows_dir_fd(root_fd, "", root, win)
+    finally:
+        os.close(root_fd)
+
+
+def iter_from_windows_dir_fd(
+    dir_fd: int, relative_dir: str, root: Path, win: _WindowsHandles
+) -> Iterable[str]:
+    scan_dir = root if not relative_dir else root.joinpath(*PurePosixPath(relative_dir).parts)
+    try:
+        # Windows CPython cannot listdir(dir_fd); paths come from the pinned walk.
+        names = os.listdir(scan_dir)
+    except OSError as error:
+        raise DetectError(f"cannot traverse filesystem evidence: {error}") from error
+    for name in names:
+        relative = f"{relative_dir}/{name}" if relative_dir else name
+        if is_ignored(relative) or name == ".project":
+            continue
+        path = root.joinpath(*PurePosixPath(relative).parts)
+        try:
+            child_fd = win.open_relative(
+                dir_fd,
+                name,
+                _DIRECTORY_ACCESS,
+                _PIN_SHARE,
+                _FILE_OPEN,
+                _FILE_DIRECTORY_FILE,
+            )
+        except OSError:
+            child_fd = None
+        else:
+            status = os.fstat(child_fd)
+            if is_link_like(path, status):
+                os.close(child_fd)
+                continue
+            if stat.S_ISDIR(status.st_mode):
+                try:
+                    if is_verified_skill_bundle_at(relative, child_fd, root):
+                        continue
+                    yield from iter_from_windows_dir_fd(
+                        child_fd, relative, root, win
+                    )
+                finally:
+                    os.close(child_fd)
+                continue
+            os.close(child_fd)
+        try:
+            file_fd = win.open_relative(
+                dir_fd,
+                name,
+                _FILE_READ_ATTRIBUTES,
+                _PIN_SHARE,
+                _FILE_OPEN,
+                _FILE_NON_DIRECTORY_FILE,
+            )
+        except OSError as error:
+            raise DetectError(
+                f"cannot inspect filesystem evidence: {relative}: {error}"
+            ) from error
+        try:
+            status = os.fstat(file_fd)
+            if is_link_like(path, status) or not stat.S_ISREG(status.st_mode):
+                continue
+            if is_fixed_managed_pipeline_artifact(relative):
+                continue
+            yield relative
+        finally:
+            os.close(file_fd)
+
+
 def windows_project_entries(project: Path) -> set:
     """project_entries for a .project held open without delete sharing."""
     entries = set(os.listdir(project))

package/skills/gsd-path-docs-audit/scripts/discussion_validate.py

@@ -467,15 +467,17 @@ def completed_field(lines: Sequence[str], field: str, artifact: str) -> str:
 
 
 def completed_bullet_field(lines: Sequence[str], field: str, artifact: str) -> str:
-    pattern = re.compile(rf"^- \*\*{re.escape(field)}\*\*:\s*(.*)$")
-    values = []
-    for line in lines:
-        match = pattern.fullmatch(line)
-        if match:
-            values.append(match.group(1).strip().strip("`"))
-    if len(values) != 1 or not values[0] or archive_milestone.contains_placeholder(values[0]):
+    matches = _common.find_review_bullet_field_values(lines, field)
+    if len(matches) != 1:
         raise ArchiveError(f"{artifact} requires one completed {field} field")
-    return values[0]
+    raw_value = matches[0].strip()
+    value = raw_value.strip("`")
+    # Scan the value as written. Removing an outer backtick first can unbalance
+    # every code span after a leading or trailing one. contains_placeholder
+    # already handles a wholly quoted placeholder such as `<command>`.
+    if not value or archive_milestone.contains_placeholder(raw_value):
+        raise ArchiveError(f"{artifact} requires one completed {field} field")
+    return value
 
 
 def section_lines(lines: Sequence[str], heading: str, artifact: str) -> Sequence[str]:
@@ -509,6 +511,12 @@ def archived_intent_criteria(archive: Path) -> dict[int, str]:
 
 
 def parse_final_review(archive: Path) -> tuple:
+    reviewed_head, criteria, _raw_evidence = _final_review(archive)
+    return reviewed_head, criteria
+
+
+def _final_review(archive: Path) -> tuple:
+    """Parse FINAL.md and retain evidence as written for placeholder scans."""
     if __package__:
         from . import check_handoffs
     else:
@@ -562,18 +570,21 @@ def parse_final_review(archive: Path) -> tuple:
 
     expected_fields = ("Verdict", "Check", "Observed", "Reference", "Finding", "Fix direction")
     criteria = []
+    raw_evidence = []
     for position, (heading_index, _, criterion) in enumerate(headings):
         section_start = heading_index + 1
         section_end = headings[position + 1][0] if position + 1 < len(headings) else criteria_end
+        section = lines[section_start:section_end]
         values = {}
-        for line in lines[section_start:section_end]:
-            match = re.fullmatch(r"- \*\*(Verdict|Check|Observed|Reference|Finding|Fix direction)\*\*:\s*(.*)", line)
-            if not match:
-                continue
-            key, value = match.groups()
-            if key in values:
+        raw_values = {}
+        for key in expected_fields:
+            matches = _common.find_review_bullet_field_values(section, key)
+            if len(matches) > 1:
                 raise ArchiveError(f"FINAL.md repeats {key} for {criterion}")
-            values[key] = value.strip().strip("`")
+            if len(matches) != 1:
+                raise ArchiveError(f"FINAL.md criterion is incomplete: {criterion}")
+            raw_values[key] = matches[0].strip()
+            values[key] = raw_values[key].strip("`")
         surface_values = [
             line.removeprefix("- **Surface**:").strip().strip("`")
             for line in lines[section_start:section_end]
@@ -593,18 +604,19 @@ def parse_final_review(archive: Path) -> tuple:
                     check_handoffs._surface_value(value, f"FINAL.md surface {field}")
                 except check_handoffs.HandoffError as error:
                     raise ArchiveError(str(error)) from error
-            elif not value or archive_milestone.contains_placeholder(value):
+            elif not value or archive_milestone.contains_placeholder(raw_values[field]):
                 raise ArchiveError(f"FINAL.md criterion is incomplete: {criterion}")
         if values["Verdict"] != "met":
             raise ArchiveError(f"FINAL.md criterion lacks passing evidence: {criterion}")
-        evidence = next(
-            (values[field] for field in ("Reference", "Check", "Observed") if values[field] != "none"),
+        evidence_field = next(
+            (field for field in ("Reference", "Check", "Observed") if values[field] != "none"),
             None,
         )
-        if evidence is None:
+        if evidence_field is None:
             raise ArchiveError(f"FINAL.md criterion lacks passing evidence: {criterion}")
-        criteria.append((criterion, "met", evidence))
-    return reviewed_head.lower(), criteria
+        criteria.append((criterion, "met", values[evidence_field]))
+        raw_evidence.append(raw_values[evidence_field])
+    return reviewed_head.lower(), criteria, raw_evidence
 
 
 def _member_heads(text: str, source: str) -> dict:
@@ -814,10 +826,14 @@ def validate_wave_review(
     expected_depth: str,
     expected_lens: Optional[str] = None,
     task_texts: Optional[dict[str, str]] = None,
+    *,
+    is_last_cycle: bool = True,
 ) -> Sequence[str]:
     lines = path.read_text(encoding="utf-8").splitlines()
     depth = completed_field(lines, "Depth:", path.name)
-    if depth != expected_depth:
+    if depth not in {"full", "deep", "verify-only"}:
+        raise ArchiveError(f"{path.name} has an invalid review depth")
+    if is_last_cycle and depth != expected_depth:
         raise ArchiveError(f"{path.name} has an invalid review depth")
     if expected_lens is not None:
         lens = completed_field(lines, "Lens:", path.name)
@@ -842,10 +858,32 @@ def validate_wave_review(
     normalized_expected = [
         (task_id, " ".join(title.split())) for task_id, title in expected_tasks
     ]
-    if actual_tasks != normalized_expected:
-        raise ArchiveError(
-            f"{path.name} tasks and titles do not match its wave task files in order"
-        )
+    expected_by_id = dict(normalized_expected)
+    enforce_plan_tasks = is_last_cycle and wave_verdict != "blocked"
+    if enforce_plan_tasks:
+        if actual_tasks != normalized_expected:
+            raise ArchiveError(
+                f"{path.name} tasks and titles do not match its wave task files in order"
+            )
+    else:
+        known_ids = set(expected_by_id)
+        for task_id in task_ids:
+            if task_id not in known_ids:
+                raise ArchiveError(
+                    f"{path.name} tasks and titles do not match its wave task files in order"
+                )
+        plan_order = [task_id for task_id, _title in normalized_expected]
+        reviewed_order = [task_id for task_id in plan_order if task_id in task_ids]
+        if task_ids != reviewed_order:
+            raise ArchiveError(
+                f"{path.name} tasks and titles do not match its wave task files in order"
+            )
+        for _index, task_id, title, _verdict in headings:
+            expected_title = expected_by_id.get(task_id)
+            if expected_title is not None and " ".join(title.split()) != expected_title:
+                raise ArchiveError(
+                    f"{path.name} tasks and titles do not match its wave task files in order"
+                )
 
     verdicts = []
     for position, (heading_index, _task_id, _title, verdict) in enumerate(headings):
@@ -870,6 +908,9 @@ def validate_wave_review(
             )
         verdicts.append(verdict)
 
+    if not is_last_cycle:
+        return verdicts
+
     coverage_indexes = [
         index for index, line in enumerate(lines) if line == "## Intent coverage"
     ]
@@ -1083,10 +1124,17 @@ def review_cycle_counts(archive: Path) -> Sequence[int]:
                         depth,
                         lens,
                         task_texts,
+                        is_last_cycle=cycle == max(cycles),
                     )
                 )
 
         last_cycle = max(cycles)
+        for cycle, verdicts in wave_verdicts.items():
+            if cycle != last_cycle and "blocked" not in verdicts:
+                raise ArchiveError(
+                    f"superseded review cycle {cycle} for wave {wave} "
+                    "must include a blocked Wave verdict"
+                )
         if any(verdict != "pass" for verdict in wave_verdicts[last_cycle]):
             raise ArchiveError(f"last review cycle for wave {wave} did not pass")
         if any(task_verdict != "pass" for task_verdict in task_verdicts[last_cycle]):
@@ -1220,7 +1268,7 @@ def validate_manifest(project: Path, archive: Path, state: PipelineState) -> tup
     if fields["Final verdict:"] != "all criteria met; project verify passed":
         raise ArchiveError("manifest final verdict does not record the passed final gate")
 
-    reviewed_head, final_criteria = parse_final_review(archive)
+    reviewed_head, final_criteria, final_raw_evidence = _final_review(archive)
     validate_gap_reviews(archive, reviewed_head)
     task_files, attested = archive_milestone.landed_task_evidence(project, archive / "tasks", reviewed_head)
     cycle_counts = review_cycle_counts(archive)
@@ -1264,14 +1312,23 @@ def validate_manifest(project: Path, archive: Path, state: PipelineState) -> tup
             continue
         criteria_rows.append(cells)
         criteria_line_indexes.add(index)
+
+    def evidence_as_written(position: int, row: Sequence[str]) -> str:
+        # Manifest evidence stores FINAL.md's normalized value. Scan the source
+        # text when the row matches, so its inner code spans stay balanced.
+        if position < len(final_criteria) and tuple(row) == tuple(final_criteria[position]):
+            return final_raw_evidence[position]
+        return row[2]
+
     if not criteria_rows or any(
         len(row) != 3
         or not row[0]
         or row[1] != "met"
         or not row[2]
         or row[2] == "none"
-        or any(archive_milestone.contains_placeholder(cell) for cell in row[1:])
-        for row in criteria_rows
+        or archive_milestone.contains_placeholder(row[1])
+        or archive_milestone.contains_placeholder(evidence_as_written(position, row))
+        for position, row in enumerate(criteria_rows)
     ):
         raise ArchiveError("manifest success-criteria rows are incomplete")
     if [tuple(row) for row in criteria_rows] != list(final_criteria):