claude-dev-env 2.36.0 → 4.0.0

package/scripts/repository_checks/package_inventory.py

@@ -0,0 +1,118 @@
+"""Package inventory checks for committed trees."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+from types import ModuleType
+
+from policy_lint.config.constants import PATH_SEPARATOR, UTF8_ENCODING
+
+from repository_checks.config.constants import (
+    ALL_ARCHIVED_SKILL_DIRECTORY_SEGMENTS,
+    CHECK_ID_PACKAGE_INVENTORY,
+    PACKAGE_INVENTORY_CONSTANTS_MODULE_NAME,
+    PACKAGE_INVENTORY_DETECTION_MODULE_NAME,
+    PACKAGE_INVENTORY_MESSAGE_TEMPLATE,
+    WINDOWS_PATH_SEPARATOR,
+)
+from repository_checks.hook_modules import load_hooks_module
+from repository_checks.models import RepositoryFinding
+
+
+def collect_package_inventory_findings(
+    repository_root: Path, all_tracked_paths: Sequence[str]
+) -> list[RepositoryFinding]:
+    """Return tracked production files missing from package inventories.
+
+    Args:
+        repository_root: Git repository root.
+        all_tracked_paths: Repository-relative tracked paths.
+
+    Returns:
+        Findings for production files absent from their inventories.
+    """
+    detection = load_hooks_module(PACKAGE_INVENTORY_DETECTION_MODULE_NAME)
+    inventory_constants = load_hooks_module(PACKAGE_INVENTORY_CONSTANTS_MODULE_NAME)
+    all_findings: list[RepositoryFinding] = []
+    for each_relative_path in all_tracked_paths:
+        maybe_finding = find_inventory_finding_for_path(
+            repository_root,
+            each_relative_path,
+            detection,
+            inventory_constants,
+        )
+        if maybe_finding is not None:
+            all_findings.append(maybe_finding)
+    return all_findings
+
+
+def find_inventory_finding_for_path(
+    repository_root: Path,
+    relative_path: str,
+    detection: ModuleType,
+    inventory_constants: ModuleType,
+) -> RepositoryFinding | None:
+    """Return an inventory finding for one tracked production file.
+
+    Args:
+        repository_root: Git repository root.
+        relative_path: Repository-relative production path.
+        detection: Existing inventory detector module.
+        inventory_constants: Existing detector constants module.
+
+    Returns:
+        The finding when the file is missing from its inventory.
+    """
+    if _is_archived_skill_path(relative_path):
+        return None
+    absolute_path = repository_root / relative_path
+    if not absolute_path.is_file():
+        return None
+    if not detection.is_inventoried_production_file(str(absolute_path)):
+        return None
+    _require_readable_inventory_documents(absolute_path.parent, inventory_constants)
+    if detection.find_stale_inventory(str(absolute_path)) is None:
+        return None
+    return _build_finding(relative_path)
+
+
+def _build_finding(relative_path: str) -> RepositoryFinding:
+    return RepositoryFinding(
+        CHECK_ID_PACKAGE_INVENTORY,
+        relative_path.replace(WINDOWS_PATH_SEPARATOR, PATH_SEPARATOR),
+        PACKAGE_INVENTORY_MESSAGE_TEMPLATE,
+    )
+
+
+def _is_archived_skill_path(relative_path: str) -> bool:
+    all_path_segments = relative_path.replace(
+        WINDOWS_PATH_SEPARATOR, PATH_SEPARATOR
+    ).split(PATH_SEPARATOR)
+    segment_count = len(ALL_ARCHIVED_SKILL_DIRECTORY_SEGMENTS)
+    last_start_index = len(all_path_segments) - segment_count
+    for each_start_index in range(max(last_start_index + 1, 0)):
+        compared_segments = tuple(
+            all_path_segments[each_start_index : each_start_index + segment_count]
+        )
+        if compared_segments == ALL_ARCHIVED_SKILL_DIRECTORY_SEGMENTS:
+            return True
+    return False
+
+
+def _require_readable_inventory_documents(
+    package_directory: Path, inventory_constants: ModuleType
+) -> None:
+    for each_inventory_name in inventory_constants.ALL_INVENTORY_DOCUMENT_NAMES:
+        inventory_path = package_directory / each_inventory_name
+        if not inventory_path.is_file():
+            continue
+        inventory_path.read_text(encoding=UTF8_ENCODING)
+    if package_directory.name != inventory_constants.SCRIPTS_SUBDIRECTORY_NAME:
+        return
+    skill_inventory_path = (
+        package_directory.parent / inventory_constants.SKILL_INVENTORY_DOCUMENT_NAME
+    )
+    if not skill_inventory_path.is_file():
+        return
+    skill_inventory_path.read_text(encoding=UTF8_ENCODING)

package/scripts/repository_checks/pytest_testpaths.py

@@ -0,0 +1,84 @@
+"""Pytest testpath checks for committed trees."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+from types import ModuleType
+
+import tomllib
+from policy_lint.config import constants as policy_constants
+
+from repository_checks.config import constants as repository_constants
+from repository_checks.hook_modules import load_hooks_module
+from repository_checks.models import RepositoryFinding
+
+
+def collect_pytest_testpath_findings(
+    repository_root: Path, all_tracked_paths: Sequence[str]
+) -> list[RepositoryFinding]:
+    """Return tracked tests outside explicit package testpaths.
+
+    Args:
+        repository_root: Git repository root.
+        all_tracked_paths: Repository-relative tracked paths.
+
+    Returns:
+        Findings for tests excluded by their package testpaths.
+    """
+    pytest_module = load_hooks_module(repository_constants.PYTEST_TESTPATHS_MODULE_NAME)
+    all_findings: list[RepositoryFinding] = []
+    for each_relative_path in all_tracked_paths:
+        maybe_finding = _find_testpath_finding(
+            repository_root, each_relative_path, pytest_module
+        )
+        if maybe_finding is not None:
+            all_findings.append(maybe_finding)
+    return all_findings
+
+
+def _find_testpath_finding(
+    repository_root: Path,
+    relative_path: str,
+    pytest_module: ModuleType,
+) -> RepositoryFinding | None:
+    if not pytest_module.is_test_file(relative_path):
+        return None
+    absolute_path = repository_root / relative_path
+    if not absolute_path.is_file():
+        return None
+    _require_readable_inputs(absolute_path, repository_root)
+    if pytest_module.find_unregistered_test_directory(str(absolute_path)) is None:
+        return None
+    return RepositoryFinding(
+        repository_constants.CHECK_ID_PYTEST_TESTPATHS,
+        _to_posix_path(relative_path),
+        repository_constants.PYTEST_TESTPATH_MESSAGE_TEMPLATE,
+    )
+
+
+def _to_posix_path(relative_path: str) -> str:
+    return relative_path.replace(
+        repository_constants.WINDOWS_PATH_SEPARATOR,
+        policy_constants.PATH_SEPARATOR,
+    )
+
+
+def _require_readable_inputs(test_file: Path, repository_root: Path) -> None:
+    test_file.read_bytes()
+    resolved_repository_root = repository_root.resolve()
+    for each_directory in test_file.parents:
+        if not _is_inside_repository(each_directory, resolved_repository_root):
+            break
+        pyproject_path = each_directory / repository_constants.PYPROJECT_FILENAME
+        if not pyproject_path.is_file():
+            continue
+        tomllib.loads(pyproject_path.read_text(encoding=policy_constants.UTF8_ENCODING))
+
+
+def _is_inside_repository(directory: Path, resolved_repository_root: Path) -> bool:
+    try:
+        directory.resolve().relative_to(resolved_repository_root)
+    except ValueError:
+        return False
+    return True

package/scripts/repository_checks/reporting.py

@@ -0,0 +1,39 @@
+"""Stable text reports for committed-tree repository checks."""
+
+from __future__ import annotations
+
+from repository_checks.config import constants as repository_constants
+from repository_checks.models import RepositoryCheckReport
+
+
+def render_report(report: RepositoryCheckReport) -> str:
+    """Render findings and failed checks as newline-terminated text.
+
+    Args:
+        report: Findings and failed check identifiers.
+
+    Returns:
+        The report body, or an empty string when the tree is clean.
+    """
+    all_lines = [
+        repository_constants.FINDING_LINE_TEMPLATE.format(
+            check_id=each_finding.check_id,
+            relative_path=each_finding.relative_path,
+            message=each_finding.message,
+        )
+        for each_finding in report.all_findings
+    ]
+    all_lines.extend(_failed_check_lines(report))
+    if not all_lines:
+        return repository_constants.EMPTY_REPORT_TEXT
+    return (
+        repository_constants.REPORT_LINE_SEPARATOR.join(all_lines)
+        + repository_constants.REPORT_LINE_SEPARATOR
+    )
+
+
+def _failed_check_lines(report: RepositoryCheckReport) -> list[str]:
+    return [
+        repository_constants.RULE_FAILED_LINE_TEMPLATE.format(check_id=each_check_id)
+        for each_check_id in report.all_failed_check_ids
+    ]

package/scripts/repository_checks/runner.py

@@ -0,0 +1,93 @@
+"""Fail-closed runner for committed-tree repository checks."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from pathlib import Path
+
+from repository_checks import tracked_tree
+from repository_checks.claude_md import collect_claude_md_orphan_findings
+from repository_checks.config import constants as repository_constants
+from repository_checks.env_var_documentation import (
+    collect_env_var_documentation_findings,
+)
+from repository_checks.models import RepositoryCheckReport, RepositoryFinding
+from repository_checks.package_inventory import collect_package_inventory_findings
+from repository_checks.pytest_testpaths import collect_pytest_testpath_findings
+from repository_checks.tracked_secrets import collect_tracked_secret_findings
+
+RepositoryCollector = Callable[[Path, Sequence[str]], list[RepositoryFinding]]
+
+
+def run_repository_checks(repository_root: Path) -> RepositoryCheckReport:
+    """Run every committed-tree check and fail closed on collector errors.
+
+    Args:
+        repository_root: Git repository root whose tracked files are scanned.
+
+    Returns:
+        Sorted findings and any check identifiers that failed closed.
+    """
+    try:
+        all_tracked_paths = tracked_tree.tracked_relative_paths(repository_root)
+    except repository_constants.ALL_FAIL_CLOSED_EXCEPTION_TYPES:
+        return RepositoryCheckReport((), repository_constants.ALL_CHECK_IDS)
+    all_findings, all_failed_check_ids = _run_collectors(
+        repository_root, all_tracked_paths
+    )
+    return RepositoryCheckReport(
+        _sort_findings(all_findings), tuple(all_failed_check_ids)
+    )
+
+
+def _run_collectors(
+    repository_root: Path, all_tracked_paths: Sequence[str]
+) -> tuple[list[RepositoryFinding], list[str]]:
+    all_findings: list[RepositoryFinding] = []
+    all_failed_check_ids: list[str] = []
+    for each_check_id, each_collector in _all_collectors():
+        try:
+            all_findings.extend(each_collector(repository_root, all_tracked_paths))
+        except repository_constants.ALL_FAIL_CLOSED_EXCEPTION_TYPES:
+            all_failed_check_ids.append(each_check_id)
+    return all_findings, all_failed_check_ids
+
+
+def _sort_findings(
+    all_findings: list[RepositoryFinding],
+) -> tuple[RepositoryFinding, ...]:
+    return tuple(
+        sorted(
+            all_findings,
+            key=lambda each_finding: (
+                each_finding.check_id,
+                each_finding.relative_path,
+                each_finding.message,
+            ),
+        )
+    )
+
+
+def _all_collectors() -> tuple[tuple[str, RepositoryCollector], ...]:
+    return (
+        (
+            repository_constants.CHECK_ID_CLAUDE_MD_ORPHANS,
+            collect_claude_md_orphan_findings,
+        ),
+        (
+            repository_constants.CHECK_ID_ENV_VAR_DOCUMENTATION,
+            collect_env_var_documentation_findings,
+        ),
+        (
+            repository_constants.CHECK_ID_PACKAGE_INVENTORY,
+            collect_package_inventory_findings,
+        ),
+        (
+            repository_constants.CHECK_ID_PYTEST_TESTPATHS,
+            collect_pytest_testpath_findings,
+        ),
+        (
+            repository_constants.CHECK_ID_TRACKED_PERSONAL_DATA,
+            collect_tracked_secret_findings,
+        ),
+    )

package/scripts/repository_checks/tracked_secrets.py

@@ -0,0 +1,147 @@
+"""Tracked-secret checks for committed trees."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+from pathlib import Path
+from types import ModuleType
+
+from policy_lint.config import constants as policy_constants
+
+from repository_checks.config import constants as repository_constants
+from repository_checks.hook_modules import load_hooks_module
+from repository_checks.models import RepositoryFinding
+
+
+def collect_tracked_secret_findings(
+    repository_root: Path, all_tracked_paths: Sequence[str]
+) -> list[RepositoryFinding]:
+    """Return non-exempt secrets found in tracked UTF-8 files.
+
+    Args:
+        repository_root: Git repository root.
+        all_tracked_paths: Repository-relative tracked paths.
+
+    Returns:
+        Findings with redacted previews for non-exempt secrets.
+    """
+    scanner = load_hooks_module(repository_constants.PII_SCANNER_MODULE_NAME)
+    exemption = load_hooks_module(repository_constants.REPOSITORY_EXEMPTION_MODULE_NAME)
+    all_allowlisted_literals = exemption.repository_allowlisted_values(repository_root)
+    all_findings: list[RepositoryFinding] = []
+    for each_relative_path in all_tracked_paths:
+        all_findings.extend(
+            _find_secrets_for_path(
+                repository_root,
+                each_relative_path,
+                scanner,
+                all_allowlisted_literals,
+            )
+        )
+    return all_findings
+
+
+def _find_secrets_for_path(
+    repository_root: Path,
+    relative_path: str,
+    scanner: ModuleType,
+    all_allowlisted_literals: frozenset[str],
+) -> list[RepositoryFinding]:
+    posix_relative_path = relative_path.replace(
+        repository_constants.WINDOWS_PATH_SEPARATOR,
+        policy_constants.PATH_SEPARATOR,
+    )
+    if scanner.is_path_exempt_from_pii_scan(posix_relative_path):
+        return []
+    maybe_content = _read_utf8_text(
+        repository_root,
+        repository_root / relative_path,
+    )
+    if maybe_content is None:
+        return []
+    return _find_secret_matches(
+        posix_relative_path,
+        maybe_content,
+        scanner,
+        all_allowlisted_literals,
+    )
+
+
+def _find_secret_matches(
+    posix_relative_path: str,
+    content: str,
+    scanner: ModuleType,
+    all_allowlisted_literals: frozenset[str],
+) -> list[RepositoryFinding]:
+    return [
+        _build_finding(
+            posix_relative_path,
+            category=each_match.category,
+            preview=each_match.preview,
+        )
+        for each_match in scanner.scan_text_for_pii(content)
+        if _should_report_match(
+            posix_relative_path,
+            each_match.category,
+            each_match.matched_text,
+            all_allowlisted_literals,
+        )
+    ]
+
+
+def _read_utf8_text(repository_root: Path, absolute_path: Path) -> str | None:
+    if not absolute_path.is_file():
+        return None
+    _require_path_inside_repository(repository_root, absolute_path)
+    try:
+        return absolute_path.read_text(encoding=policy_constants.UTF8_ENCODING)
+    except UnicodeDecodeError:
+        return None
+
+
+def _require_path_inside_repository(repository_root: Path, absolute_path: Path) -> None:
+    resolved_repository_root = repository_root.resolve(strict=True)
+    resolved_absolute_path = absolute_path.resolve(strict=True)
+    try:
+        resolved_absolute_path.relative_to(resolved_repository_root)
+    except ValueError as e:
+        raise OSError(absolute_path) from e
+
+
+def _build_finding(
+    posix_relative_path: str, category: str, preview: str
+) -> RepositoryFinding:
+    return RepositoryFinding(
+        repository_constants.CHECK_ID_TRACKED_PERSONAL_DATA,
+        posix_relative_path,
+        repository_constants.TRACKED_MATCH_MESSAGE_TEMPLATE.format(
+            category=category,
+            preview=preview,
+        ),
+    )
+
+
+def _should_report_match(
+    posix_relative_path: str,
+    category: str,
+    matched_text: str,
+    all_allowlisted_literals: frozenset[str],
+) -> bool:
+    exact_exemption_identity = (
+        posix_relative_path,
+        category,
+        _secret_digest(matched_text),
+    )
+    if (
+        exact_exemption_identity
+        in repository_constants.ALL_TRACKED_SECRET_EXACT_EXEMPTIONS
+    ):
+        return False
+    return matched_text not in all_allowlisted_literals
+
+
+def _secret_digest(matched_text: str) -> str:
+    return hashlib.sha256(
+        matched_text.encode(policy_constants.UTF8_ENCODING)
+    ).hexdigest()

package/scripts/repository_checks/tracked_tree.py

@@ -0,0 +1,28 @@
+"""List tracked worktree paths for the committed-tree checker."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from policy_lint.selection_git import git_bytes_for, split_nul_tokens
+
+from repository_checks.config.constants import ALL_GIT_LS_FILES_ARGUMENTS
+
+
+def tracked_relative_paths(repository_root: Path) -> tuple[str, ...]:
+    """Return tracked paths as repository-relative posix strings.
+
+    ::
+
+        git ls-files -z  ->  ("notes/CLAUDE.md", "src/app.py")
+        ok:   paths are relative
+        flag: an absolute Windows path
+
+    Args:
+        repository_root: Git repository root.
+
+    Returns:
+        Tracked relative paths in Git order.
+    """
+    raw_paths = git_bytes_for(repository_root, ALL_GIT_LS_FILES_ARGUMENTS)
+    return split_nul_tokens(raw_paths)

package/scripts/repository_policy.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Public command for the committed-tree repository checker."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Never, TextIO
+
+from policy_lint.config.constants import ALL_GIT_ROOT_ARGUMENTS, UTF8_ENCODING
+from policy_lint.selection_git import GitSelectionError, git_bytes_for
+from repository_checks.config.constants import (
+    ALL_CHECK_IDS,
+    REPOSITORY_ROOT_FLAG,
+    USAGE_EXIT_CODE,
+)
+from repository_checks.reporting import render_report
+from repository_checks.runner import run_repository_checks
+
+__all__ = ["ALL_CHECK_IDS", "main"]
+
+
+class _RepositoryPolicyUsageError(ValueError):
+    """Raised when command arguments cannot form a repository scan."""
+
+
+class _RepositoryPolicyArgumentParser(argparse.ArgumentParser):
+    def error(self, message: str) -> Never:
+        """Raise a usage error for the caller.
+
+        Args:
+            message: Parser error text.
+
+        Raises:
+            _RepositoryPolicyUsageError: Always.
+        """
+        raise _RepositoryPolicyUsageError(message)
+
+
+def _discover_repository_root(starting_directory: Path) -> Path:
+    """Find the Git root that contains the command's current directory.
+
+    Args:
+        starting_directory: Directory where the command started.
+
+    Returns:
+        The resolved Git repository root.
+
+    Raises:
+        _RepositoryPolicyUsageError: If the directory is not inside a Git repository.
+    """
+    try:
+        root_bytes = git_bytes_for(starting_directory.resolve(), ALL_GIT_ROOT_ARGUMENTS)
+    except (GitSelectionError, OSError) as error:
+        raise _RepositoryPolicyUsageError(
+            "Current directory is not inside a Git repository"
+        ) from error
+    root_text = root_bytes.decode(UTF8_ENCODING).strip()
+    if not root_text:
+        raise _RepositoryPolicyUsageError("Git returned an empty repository root")
+    return Path(root_text).resolve()
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = _RepositoryPolicyArgumentParser(prog="repository_policy")
+    parser.add_argument(
+        REPOSITORY_ROOT_FLAG,
+        dest="repository_root",
+        default=None,
+        help="Git repository root to scan.",
+    )
+    return parser
+
+
+def main(
+    all_arguments: Sequence[str],
+    *,
+    repository_root: Path,
+    stdout: TextIO = sys.stdout,
+    stderr: TextIO = sys.stderr,
+) -> int:
+    """Run committed-tree repository checks.
+
+    Args:
+        all_arguments: Command arguments.
+        repository_root: Repository root used when the flag is omitted.
+        stdout: Output stream.
+        stderr: Error stream.
+
+    Returns:
+        Zero when the tree is clean, one for findings, two for usage errors,
+        and three when a check fails closed.
+    """
+    try:
+        namespace = _build_parser().parse_args(list(all_arguments))
+    except _RepositoryPolicyUsageError as usage_error:
+        stderr.write(f"{usage_error}\n")
+        return USAGE_EXIT_CODE
+    selected_root = repository_root
+    if namespace.repository_root is not None:
+        selected_root = Path(namespace.repository_root).resolve()
+    report = run_repository_checks(selected_root)
+    stdout.write(render_report(report))
+    return report.exit_code
+
+
+if __name__ == "__main__":
+    try:
+        shell_directory = Path.cwd()
+        parsed_namespace = _build_parser().parse_args(sys.argv[1:])
+        if parsed_namespace.repository_root is not None:
+            shell_repository_root = Path(parsed_namespace.repository_root).resolve()
+        else:
+            shell_repository_root = _discover_repository_root(shell_directory)
+    except _RepositoryPolicyUsageError as usage_error:
+        sys.stderr.write(f"{usage_error}\n")
+        raise SystemExit(USAGE_EXIT_CODE) from usage_error
+    raise SystemExit(main(sys.argv[1:], repository_root=shell_repository_root))