claude-dev-env 2.36.0 → 4.1.0

package/scripts/repository_checks/tracked_secrets.py

@@ -0,0 +1,147 @@
+"""Tracked-secret checks for committed trees."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+from pathlib import Path
+from types import ModuleType
+
+from policy_lint.config import constants as policy_constants
+
+from repository_checks.config import constants as repository_constants
+from repository_checks.hook_modules import load_hooks_module
+from repository_checks.models import RepositoryFinding
+
+
+def collect_tracked_secret_findings(
+    repository_root: Path, all_tracked_paths: Sequence[str]
+) -> list[RepositoryFinding]:
+    """Return non-exempt secrets found in tracked UTF-8 files.
+
+    Args:
+        repository_root: Git repository root.
+        all_tracked_paths: Repository-relative tracked paths.
+
+    Returns:
+        Findings with redacted previews for non-exempt secrets.
+    """
+    scanner = load_hooks_module(repository_constants.PII_SCANNER_MODULE_NAME)
+    exemption = load_hooks_module(repository_constants.REPOSITORY_EXEMPTION_MODULE_NAME)
+    all_allowlisted_literals = exemption.repository_allowlisted_values(repository_root)
+    all_findings: list[RepositoryFinding] = []
+    for each_relative_path in all_tracked_paths:
+        all_findings.extend(
+            _find_secrets_for_path(
+                repository_root,
+                each_relative_path,
+                scanner,
+                all_allowlisted_literals,
+            )
+        )
+    return all_findings
+
+
+def _find_secrets_for_path(
+    repository_root: Path,
+    relative_path: str,
+    scanner: ModuleType,
+    all_allowlisted_literals: frozenset[str],
+) -> list[RepositoryFinding]:
+    posix_relative_path = relative_path.replace(
+        repository_constants.WINDOWS_PATH_SEPARATOR,
+        policy_constants.PATH_SEPARATOR,
+    )
+    if scanner.is_path_exempt_from_pii_scan(posix_relative_path):
+        return []
+    maybe_content = _read_utf8_text(
+        repository_root,
+        repository_root / relative_path,
+    )
+    if maybe_content is None:
+        return []
+    return _find_secret_matches(
+        posix_relative_path,
+        maybe_content,
+        scanner,
+        all_allowlisted_literals,
+    )
+
+
+def _find_secret_matches(
+    posix_relative_path: str,
+    content: str,
+    scanner: ModuleType,
+    all_allowlisted_literals: frozenset[str],
+) -> list[RepositoryFinding]:
+    return [
+        _build_finding(
+            posix_relative_path,
+            category=each_match.category,
+            preview=each_match.preview,
+        )
+        for each_match in scanner.scan_text_for_pii(content)
+        if _should_report_match(
+            posix_relative_path,
+            each_match.category,
+            each_match.matched_text,
+            all_allowlisted_literals,
+        )
+    ]
+
+
+def _read_utf8_text(repository_root: Path, absolute_path: Path) -> str | None:
+    if not absolute_path.is_file():
+        return None
+    _require_path_inside_repository(repository_root, absolute_path)
+    try:
+        return absolute_path.read_text(encoding=policy_constants.UTF8_ENCODING)
+    except UnicodeDecodeError:
+        return None
+
+
+def _require_path_inside_repository(repository_root: Path, absolute_path: Path) -> None:
+    resolved_repository_root = repository_root.resolve(strict=True)
+    resolved_absolute_path = absolute_path.resolve(strict=True)
+    try:
+        resolved_absolute_path.relative_to(resolved_repository_root)
+    except ValueError as e:
+        raise OSError(absolute_path) from e
+
+
+def _build_finding(
+    posix_relative_path: str, category: str, preview: str
+) -> RepositoryFinding:
+    return RepositoryFinding(
+        repository_constants.CHECK_ID_TRACKED_PERSONAL_DATA,
+        posix_relative_path,
+        repository_constants.TRACKED_MATCH_MESSAGE_TEMPLATE.format(
+            category=category,
+            preview=preview,
+        ),
+    )
+
+
+def _should_report_match(
+    posix_relative_path: str,
+    category: str,
+    matched_text: str,
+    all_allowlisted_literals: frozenset[str],
+) -> bool:
+    exact_exemption_identity = (
+        posix_relative_path,
+        category,
+        _secret_digest(matched_text),
+    )
+    if (
+        exact_exemption_identity
+        in repository_constants.ALL_TRACKED_SECRET_EXACT_EXEMPTIONS
+    ):
+        return False
+    return matched_text not in all_allowlisted_literals
+
+
+def _secret_digest(matched_text: str) -> str:
+    return hashlib.sha256(
+        matched_text.encode(policy_constants.UTF8_ENCODING)
+    ).hexdigest()

package/scripts/repository_checks/tracked_tree.py

@@ -0,0 +1,28 @@
+"""List tracked worktree paths for the committed-tree checker."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from policy_lint.selection_git import git_bytes_for, split_nul_tokens
+
+from repository_checks.config.constants import ALL_GIT_LS_FILES_ARGUMENTS
+
+
+def tracked_relative_paths(repository_root: Path) -> tuple[str, ...]:
+    """Return tracked paths as repository-relative posix strings.
+
+    ::
+
+        git ls-files -z  ->  ("notes/CLAUDE.md", "src/app.py")
+        ok:   paths are relative
+        flag: an absolute Windows path
+
+    Args:
+        repository_root: Git repository root.
+
+    Returns:
+        Tracked relative paths in Git order.
+    """
+    raw_paths = git_bytes_for(repository_root, ALL_GIT_LS_FILES_ARGUMENTS)
+    return split_nul_tokens(raw_paths)

package/scripts/repository_policy.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Public command for the committed-tree repository checker."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Never, TextIO
+
+from policy_lint.config.constants import ALL_GIT_ROOT_ARGUMENTS, UTF8_ENCODING
+from policy_lint.selection_git import GitSelectionError, git_bytes_for
+from repository_checks.config.constants import (
+    ALL_CHECK_IDS,
+    REPOSITORY_ROOT_FLAG,
+    USAGE_EXIT_CODE,
+)
+from repository_checks.reporting import render_report
+from repository_checks.runner import run_repository_checks
+
+__all__ = ["ALL_CHECK_IDS", "main"]
+
+
+class _RepositoryPolicyUsageError(ValueError):
+    """Raised when command arguments cannot form a repository scan."""
+
+
+class _RepositoryPolicyArgumentParser(argparse.ArgumentParser):
+    def error(self, message: str) -> Never:
+        """Raise a usage error for the caller.
+
+        Args:
+            message: Parser error text.
+
+        Raises:
+            _RepositoryPolicyUsageError: Always.
+        """
+        raise _RepositoryPolicyUsageError(message)
+
+
+def _discover_repository_root(starting_directory: Path) -> Path:
+    """Find the Git root that contains the command's current directory.
+
+    Args:
+        starting_directory: Directory where the command started.
+
+    Returns:
+        The resolved Git repository root.
+
+    Raises:
+        _RepositoryPolicyUsageError: If the directory is not inside a Git repository.
+    """
+    try:
+        root_bytes = git_bytes_for(starting_directory.resolve(), ALL_GIT_ROOT_ARGUMENTS)
+    except (GitSelectionError, OSError) as error:
+        raise _RepositoryPolicyUsageError(
+            "Current directory is not inside a Git repository"
+        ) from error
+    root_text = root_bytes.decode(UTF8_ENCODING).strip()
+    if not root_text:
+        raise _RepositoryPolicyUsageError("Git returned an empty repository root")
+    return Path(root_text).resolve()
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = _RepositoryPolicyArgumentParser(prog="repository_policy")
+    parser.add_argument(
+        REPOSITORY_ROOT_FLAG,
+        dest="repository_root",
+        default=None,
+        help="Git repository root to scan.",
+    )
+    return parser
+
+
+def main(
+    all_arguments: Sequence[str],
+    *,
+    repository_root: Path,
+    stdout: TextIO = sys.stdout,
+    stderr: TextIO = sys.stderr,
+) -> int:
+    """Run committed-tree repository checks.
+
+    Args:
+        all_arguments: Command arguments.
+        repository_root: Repository root used when the flag is omitted.
+        stdout: Output stream.
+        stderr: Error stream.
+
+    Returns:
+        Zero when the tree is clean, one for findings, two for usage errors,
+        and three when a check fails closed.
+    """
+    try:
+        namespace = _build_parser().parse_args(list(all_arguments))
+    except _RepositoryPolicyUsageError as usage_error:
+        stderr.write(f"{usage_error}\n")
+        return USAGE_EXIT_CODE
+    selected_root = repository_root
+    if namespace.repository_root is not None:
+        selected_root = Path(namespace.repository_root).resolve()
+    report = run_repository_checks(selected_root)
+    stdout.write(render_report(report))
+    return report.exit_code
+
+
+if __name__ == "__main__":
+    try:
+        shell_directory = Path.cwd()
+        parsed_namespace = _build_parser().parse_args(sys.argv[1:])
+        if parsed_namespace.repository_root is not None:
+            shell_repository_root = Path(parsed_namespace.repository_root).resolve()
+        else:
+            shell_repository_root = _discover_repository_root(shell_directory)
+    except _RepositoryPolicyUsageError as usage_error:
+        sys.stderr.write(f"{usage_error}\n")
+        raise SystemExit(USAGE_EXIT_CODE) from usage_error
+    raise SystemExit(main(sys.argv[1:], repository_root=shell_repository_root))

package/scripts/tests/repository_policy_test_support.py

@@ -0,0 +1,89 @@
+"""Shared fixtures for repository-policy behavior tests."""
+
+from __future__ import annotations
+
+import io
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+SCRIPTS_DIRECTORY = Path(__file__).resolve().parents[1]
+UTF8_ENCODING = "utf-8"
+
+if str(SCRIPTS_DIRECTORY) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIRECTORY))
+
+import repository_policy
+
+
+def initialize_repository(repository_root: Path) -> None:
+    repository_root.mkdir(parents=True, exist_ok=True)
+    run_git(repository_root, ("init", "--initial-branch=main"))
+
+
+def commit_tracked_files(repository_root: Path) -> None:
+    run_git(repository_root, ("add", "-A"))
+    run_git(repository_root, ("commit", "--quiet", "-m", "seed"))
+
+
+def write_text(file_path: Path, content: str) -> None:
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+    file_path.write_text(content, encoding=UTF8_ENCODING)
+
+
+def patch_unreadable_named_file(
+    monkeypatch: pytest.MonkeyPatch, filename: str, error_text: str
+) -> None:
+    original_read_text = Path.read_text
+
+    def refuse_named_file(
+        file_path: Path,
+        encoding: str | None = None,
+        errors: str | None = None,
+    ) -> str:
+        if file_path.name == filename:
+            raise OSError(error_text)
+        return original_read_text(file_path, encoding=encoding, errors=errors)
+
+    monkeypatch.setattr(Path, "read_text", refuse_named_file)
+
+
+def seed_clean_repository(repository_root: Path) -> Path:
+    initialize_repository(repository_root)
+    write_text(repository_root / "README.md", "clean tree\n")
+    commit_tracked_files(repository_root)
+    return repository_root
+
+
+def run_policy(
+    repository_root: Path, all_arguments: list[str] | None = None
+) -> tuple[int, str, str]:
+    stdout = io.StringIO()
+    stderr = io.StringIO()
+    exit_code = repository_policy.main(
+        all_arguments or [],
+        repository_root=repository_root,
+        stdout=stdout,
+        stderr=stderr,
+    )
+    return exit_code, stdout.getvalue(), stderr.getvalue()
+
+
+def run_git(repository_root: Path, all_arguments: tuple[str, ...]) -> None:
+    subprocess.run(
+        (
+            "git",
+            "-c",
+            "user.email=ci@example.com",
+            "-c",
+            "user.name=CI",
+            "-c",
+            "core.hooksPath=",
+            *all_arguments,
+        ),
+        cwd=repository_root,
+        check=True,
+        capture_output=True,
+    )

package/scripts/tests/test_cde_lint.py

@@ -304,7 +304,9 @@ def test_staged_clean_reports_keep_legacy_advisories_outside_renderer(
 
     assert json_exit_code == 0
     assert json_stderr == ""
-    assert json.loads(json_stdout)["diagnostics"] == []
+    json_report = json.loads(json_stdout)
+    assert json_report["diagnostics"] == []
+    assert "terminology-sweep" in json_report["executed_rules"]
     assert text_exit_code == 0
     assert text_stdout == ""
     assert text_stderr == ""
@@ -313,6 +315,28 @@ def test_staged_clean_reports_keep_legacy_advisories_outside_renderer(
     assert editor_stderr == ""
 
 
+def test_staged_git_failure_exits_three_with_valid_json(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _stage_oversized_test_file(tmp_path)
+    adapter_module = sys.modules["policy_lint.adapters"]
+    terminology_module = adapter_module._pr_loop_script_module("terminology_sweep")
+
+    def fail_git(
+        _repository_root: Path, _all_arguments: object
+    ) -> subprocess.CompletedProcess[str]:
+        raise RuntimeError("simulated Git failure")
+
+    monkeypatch.setattr(terminology_module, "_run_strict_git", fail_git)
+    exit_code, stdout_text, stderr_text = _run_staged_format("json", tmp_path)
+    parsed_report = json.loads(stdout_text)
+
+    assert exit_code == 3
+    assert stderr_text == ""
+    assert parsed_report["diagnostics"] == []
+    assert parsed_report["failed_rules"] == ["terminology-sweep"]
+
+
 def test_should_exit_one_when_diagnostics_exist(tmp_path: Path) -> None:
     exit_code, stdout_text, _stderr_text = _run_cli(
         ["--files", "src/app.py"],

package/scripts/tests/test_claude_md.py

@@ -0,0 +1,68 @@
+"""Behavior tests for committed CLAUDE.md references."""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+
+_TESTS_DIRECTORY = Path(__file__).resolve().parent
+if str(_TESTS_DIRECTORY) not in sys.path:
+    sys.path.insert(0, str(_TESTS_DIRECTORY))
+
+from repository_checks.config.constants import (
+    CHECK_ID_CLAUDE_MD_ORPHANS,
+    FAILED_CHECK_EXIT_CODE,
+    FINDINGS_EXIT_CODE,
+)
+from repository_policy_test_support import (
+    commit_tracked_files,
+    initialize_repository,
+    run_policy,
+    write_text,
+)
+
+
+def test_should_flag_a_broken_claude_md_file_reference(tmp_path: Path) -> None:
+    repository_root = tmp_path / "repo"
+    initialize_repository(repository_root)
+    write_text(
+        repository_root / "notes" / "CLAUDE.md",
+        "# notes\n\n"
+        "| File | Role |\n"
+        "|---|---|\n"
+        "| `absent_helper.py` | Missing helper |\n",
+    )
+    commit_tracked_files(repository_root)
+    exit_code, stdout_text, _stderr_text = run_policy(repository_root)
+    assert exit_code == FINDINGS_EXIT_CODE
+    assert CHECK_ID_CLAUDE_MD_ORPHANS in stdout_text
+    assert "notes/CLAUDE.md" in stdout_text
+    assert "absent_helper.py" in stdout_text
+
+
+def test_should_fail_closed_when_claude_md_walk_fails_after_first_entry(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    repository_root = tmp_path / "repo"
+    initialize_repository(repository_root)
+    write_text(
+        repository_root / "notes" / "CLAUDE.md",
+        "# notes\n\n"
+        "| File | Role |\n"
+        "|---|---|\n"
+        "| `absent_helper.py` | Missing helper |\n",
+    )
+    commit_tracked_files(repository_root)
+
+    def fail_after_first_entry(_file_path: Path, _pattern: str = "*") -> Iterator[Path]:
+        yield repository_root / "notes" / "CLAUDE.md"
+        raise OSError("scan root unreadable")
+
+    monkeypatch.setattr(Path, "rglob", fail_after_first_entry)
+    exit_code, stdout_text, _stderr_text = run_policy(repository_root)
+    assert exit_code == FAILED_CHECK_EXIT_CODE
+    assert CHECK_ID_CLAUDE_MD_ORPHANS in stdout_text
+    assert "error: rule failed:" in stdout_text

package/scripts/tests/test_env_var_documentation.py

@@ -0,0 +1,133 @@
+"""Behavior tests for environment-variable documentation checks."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+_TESTS_DIRECTORY = Path(__file__).resolve().parent
+if str(_TESTS_DIRECTORY) not in sys.path:
+    sys.path.insert(0, str(_TESTS_DIRECTORY))
+
+from repository_checks.config.constants import (
+    CHECK_ID_ENV_VAR_DOCUMENTATION,
+    FAILED_CHECK_EXIT_CODE,
+    FINDINGS_EXIT_CODE,
+    SUCCESS_EXIT_CODE,
+)
+from repository_policy_test_support import (
+    commit_tracked_files,
+    initialize_repository,
+    patch_unreadable_named_file,
+    run_policy,
+    write_text,
+)
+
+
+def test_should_flag_environment_variable_documentation_drift(tmp_path: Path) -> None:
+    repository_root = tmp_path / "repo"
+    initialize_repository(repository_root)
+    write_text(
+        repository_root / "auth" / "google_auth.py", "def load() -> None:\n    return\n"
+    )
+    write_text(
+        repository_root / "docs" / "configuration.md",
+        "## Summary: Environment Variables\n\n"
+        "| Variable | Used By | Purpose |\n"
+        "|----------|---------|---------|\n"
+        "| `GOOGLE_APPLICATION_CREDENTIALS` | `auth/google_auth.py` | Path to JSON |\n",
+    )
+    commit_tracked_files(repository_root)
+    exit_code, stdout_text, _stderr_text = run_policy(repository_root)
+    assert exit_code == FINDINGS_EXIT_CODE
+    assert CHECK_ID_ENV_VAR_DOCUMENTATION in stdout_text
+    assert "docs/configuration.md" in stdout_text
+    assert "GOOGLE_APPLICATION_CREDENTIALS -> auth/google_auth.py" in stdout_text
+
+
+def test_should_ignore_long_code_reference_inside_code_fence(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    repository_root = tmp_path / "repo"
+    initialize_repository(repository_root)
+    long_code_reference = ("x" * 300) + ".py"
+    write_text(
+        repository_root / "docs" / "review.md",
+        f"# Review\n\n~~~diff\n+ `{long_code_reference}`\n~~~\n",
+    )
+    commit_tracked_files(repository_root)
+    original_is_file = Path.is_file
+
+    def reject_long_filename(candidate_path: Path) -> bool:
+        if len(candidate_path.name) > 255:
+            raise OSError(candidate_path)
+        return original_is_file(candidate_path)
+
+    monkeypatch.setattr(Path, "is_file", reject_long_filename)
+    exit_code, stdout_text, _stderr_text = run_policy(repository_root)
+    assert exit_code == SUCCESS_EXIT_CODE
+    assert CHECK_ID_ENV_VAR_DOCUMENTATION not in stdout_text
+
+
+def test_should_fail_closed_when_an_env_var_code_file_cannot_be_read(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    repository_root = tmp_path / "repo"
+    initialize_repository(repository_root)
+    write_text(
+        repository_root / "packages" / "service" / "auth" / "google_auth.py",
+        "def load() -> None:\n    return\n",
+    )
+    write_text(
+        repository_root / "docs" / "configuration.md",
+        "## Summary: Environment Variables\n\n"
+        "| Variable | Used By | Purpose |\n"
+        "|----------|---------|---------|\n"
+        "| `GOOGLE_APPLICATION_CREDENTIALS` | `auth/google_auth.py` | Path to JSON |\n",
+    )
+    commit_tracked_files(repository_root)
+    patch_unreadable_named_file(monkeypatch, "google_auth.py", "code file unreadable")
+    exit_code, stdout_text, _stderr_text = run_policy(repository_root)
+    assert exit_code == FAILED_CHECK_EXIT_CODE
+    assert CHECK_ID_ENV_VAR_DOCUMENTATION in stdout_text
+    assert "error: rule failed:" in stdout_text
+
+
+def test_should_fail_closed_before_reading_env_var_code_file_outside_repository(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    repository_root = tmp_path / "repo"
+    initialize_repository(repository_root)
+    outside_path = tmp_path / "outside.py"
+    write_text(outside_path, "def load() -> None:\n    return\n")
+    write_text(
+        repository_root / "docs" / "configuration.md",
+        "## Summary: Environment Variables\n\n"
+        "| Variable | Used By | Purpose |\n"
+        "|----------|---------|---------|\n"
+        "| `EXTERNAL_TOKEN` | `../outside.py` | External consumer |\n",
+    )
+    commit_tracked_files(repository_root)
+    all_read_paths = record_read_paths(monkeypatch)
+    exit_code, stdout_text, _stderr_text = run_policy(repository_root)
+    assert exit_code == FAILED_CHECK_EXIT_CODE
+    assert CHECK_ID_ENV_VAR_DOCUMENTATION in stdout_text
+    assert outside_path not in all_read_paths
+
+
+def record_read_paths(monkeypatch: pytest.MonkeyPatch) -> list[Path]:
+    original_read_text = Path.read_text
+    all_read_paths: list[Path] = []
+
+    def record_read_path(
+        file_path: Path,
+        encoding: str | None = None,
+        errors: str | None = None,
+    ) -> str:
+        all_read_paths.append(file_path.resolve())
+        return original_read_text(file_path, encoding=encoding, errors=errors)
+
+    monkeypatch.setattr(Path, "read_text", record_read_path)
+    return all_read_paths

package/scripts/tests/test_hook_modules.py

@@ -0,0 +1,17 @@
+"""Behavior tests for repository-check hook module loading."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_SCRIPTS_DIRECTORY = Path(__file__).resolve().parents[1]
+if str(_SCRIPTS_DIRECTORY) not in sys.path:
+    sys.path.insert(0, str(_SCRIPTS_DIRECTORY))
+
+from repository_checks.hook_modules import load_hooks_module
+
+
+def test_should_load_a_hooks_module_by_its_dotted_name() -> None:
+    loaded_module = load_hooks_module("hooks_constants.pii_prevention_constants")
+    assert loaded_module.__name__ == "hooks_constants.pii_prevention_constants"

package/scripts/tests/test_models.py

@@ -0,0 +1,26 @@
+"""Behavior tests for repository-check report models."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_SCRIPTS_DIRECTORY = Path(__file__).resolve().parents[1]
+if str(_SCRIPTS_DIRECTORY) not in sys.path:
+    sys.path.insert(0, str(_SCRIPTS_DIRECTORY))
+
+from repository_checks.config.constants import (
+    FAILED_CHECK_EXIT_CODE,
+    FINDINGS_EXIT_CODE,
+    SUCCESS_EXIT_CODE,
+)
+from repository_checks.models import RepositoryCheckReport, RepositoryFinding
+
+
+def test_should_choose_exit_code_by_report_severity() -> None:
+    finding = RepositoryFinding("check", "file.py", "message")
+    assert RepositoryCheckReport((), ()).exit_code == SUCCESS_EXIT_CODE
+    assert RepositoryCheckReport((finding,), ()).exit_code == FINDINGS_EXIT_CODE
+    assert RepositoryCheckReport((finding,), ("check",)).exit_code == (
+        FAILED_CHECK_EXIT_CODE
+    )