@boyingliu01/xp-gate 0.24.0 → 0.26.0

package/adapter-common.sh

@@ -220,9 +220,58 @@ detect_mutation_testable() {
   fi
   return 1
 }
-# Detect if a Python project has mutmut installed and configured
+# Echo the pytest-xdist parallel arguments when xdist is importable, or an
+# empty string with a stderr warning when it is not (#516). NEVER fails and
+# never blocks: serial semantics are the safe fallback, the warning only tells
+# the user why Gate 5 is slow. `-n auto` uses every core; `--dist loadfile`
+# groups tests per file to avoid cross-file shared state (module-level
+# fixtures, shared sqlite) racing under parallel workers — measured on a
+# 2555-test project: 30-45min serial -> 2m23s parallel with zero flaky runs.
+# Callers must invoke unquoted and unset: `pytest $(python_xdist_args) ...`
+# (SC2086-intentional: the value is a multi-word argument list).
+python_xdist_args() {
+  if python -c "import xdist" >/dev/null 2>&1; then
+    printf '%s' "-n auto --dist loadfile"
+    return 0
+  fi
+  echo "⚠ pytest-xdist not installed — Gate 5 falls back to SERIAL (may be 10x slower). Install: pip install pytest-xdist" >&2
+  return 0
+}
+# Detect a usable Python mutation engine (#410).
+#
+# Two engines, probed in preference order:
+#   1. pytest-gremlins — mutation switching (no fork), natively cross-platform,
+#      requires Python >= 3.11. Preferred wherever it is installed.
+#   2. mutmut — fork()-based, so on Windows it is ONLY runnable under WSL.
+#
+# The original implementation returned success as soon as `mutmut --version`
+# worked, which on native Windows produced a structural deadlock: `mutmut run`
+# exits 1 with "please use the WSL", so every push was blocked by a check that
+# could never pass — while uninstalling mutmut flipped the gate to SKIP. The
+# platform branch below is what removes that deadlock; it never blocks, it only
+# decides whether Gate M runs or skips.
 detect_python_mutation_testable() {
-  # Check for mutmut installation
+  # Engine 1: pytest-gremlins (cross-platform, no fork).
+  if command -v python >/dev/null 2>&1 && python -c "import pytest_gremlins" >/dev/null 2>&1; then
+    return 0
+  fi
+
+  # Determine whether mutmut can actually RUN here, not merely whether it is
+  # installed — that distinction is the whole point of this function on Windows.
+  local _uname_s
+  _uname_s="$(uname -s 2>/dev/null || echo unknown)"
+  case "$_uname_s" in
+    MINGW*|MSYS*|CYGWIN*)
+      # Native Windows: mutmut is unusable without WSL.
+      if command -v wsl >/dev/null 2>&1 && wsl mutmut --version >/dev/null 2>&1; then
+        return 0
+      fi
+      echo "⚠ Gate M (Python): mutmut cannot run on native Windows (fork()-based) and no WSL mutmut found — SKIP. Install pytest-gremlins instead: pip install pytest-gremlins (needs Python >= 3.11)." >&2
+      return 1
+      ;;
+  esac
+
+  # Unix: mutmut is fine natively.
   if command -v mutmut >/dev/null 2>&1; then
     return 0
   fi

package/adapters/python.sh

@@ -91,12 +91,62 @@ run_static_analysis() {
       return 0
     fi
   fi
-  # 逐目标调用并聚合退出码（口径见函数头注释）。
-  require_tool mypy "mypy" || return 1
+  # #534: engine selection. mypy is ALWAYS the authoritative engine (determinism —
+  # ty is an alpha, ~70% spec conformance, config-incompatible). ty is used ONLY
+  # as an explicit opt-in accelerator when XP_GATE_TY=1 AND `ty` is installed:
+  #   - XP_GATE_TY=1 + ty present  -> `ty check <target>` (ty's CLI is `ty check`,
+  #     not `ty`; exit 1 = type errors -> BLOCK, exit 2 = config/CLI error -> SKIP)
+  #   - XP_GATE_TY=1 + ty missing   -> SKIP (return 0), never BLOCK: the opt-in
+  #     path must not route through require_tool mypy and false-block a repo that
+  #     opted into ty but lacks mypy (Delphi R1 Critical).
+  #   - XP_GATE_TY unset            -> pure mypy (authoritative), unchanged.
+  # Scope semantics (.xp-gate-mypy-scope / src/ scripts/) are engine-independent
+  # and preserved verbatim below.
+  local _engine=mypy
+  if [ "${XP_GATE_TY:-}" = "1" ]; then
+    if command -v ty >/dev/null 2>&1; then
+      _engine=ty
+      echo "  ⚙  engine: ty (Rust, explicit opt-in via XP_GATE_TY=1) — mypy remains the default"
+    elif command -v mypy >/dev/null 2>&1; then
+      # ty opted in but missing; mypy present -> fall back to the authoritative
+      # engine with a warning (never silently drop the type gate, #534 R2 arch).
+      echo "  ⚠ ty requested (XP_GATE_TY=1) but not installed — falling back to mypy"
+    else
+      echo "  ⚠ ty opted in via XP_GATE_TY=1 but neither ty nor mypy is installed — SKIPPING type check (no BLOCK)"
+      echo "    install: pip install mypy, or pip install ty (alpha)"
+      return 0
+    fi
+  fi
+  # require_tool mypy only on the authoritative (mypy) engine path: when the
+  # user opted into ty, a missing mypy must NOT false-BLOCK (Delphi R1 Critical).
+  if [ "$_engine" = "mypy" ]; then
+    require_tool mypy "mypy" || return 1
+  fi
   local _mypy_rc=0
   local _mypy_t
   for _mypy_t in "${MYPY_TARGETS[@]}"; do
-    mypy "$_mypy_t" || _mypy_rc=1
+    if [ "$_engine" = "ty" ]; then
+      # ty's CLI is `ty check <path>`, not `ty <path>`. Bare `ty`/`ty check .`
+      # without targets would recurse the whole tree (== the anti `mypy .`
+      # guard #501/#513): targets are always explicit here. Capture ty's own
+      # exit code and diagnostics via a temp file (never discard them): a BLOCK
+      # with no reason is undebuggable, and a pipe `| head` would lose the code.
+      local _ty_rc=0
+      local _ty_tmp
+      _ty_tmp="$(mktemp)"
+      ty check "$_mypy_t" > "$_ty_tmp" 2>&1 || _ty_rc=$?
+      if [ "$_ty_rc" -eq 2 ]; then
+        echo "  ⚠ ty exited $_ty_rc on target '$_mypy_t' (config/CLI error) — SKIPPING, not blocking"
+        head -30 "$_ty_tmp"
+      elif [ "$_ty_rc" -ne 0 ]; then
+        echo "  ty report for '$_mypy_t' (exit $_ty_rc):"
+        head -30 "$_ty_tmp"
+        _mypy_rc=1
+      fi
+      rm -f "$_ty_tmp"
+    else
+      mypy "$_mypy_t" || _mypy_rc=1
+    fi
   done
   return $_mypy_rc
 }
@@ -133,7 +183,12 @@ run_tests() {
 
   echo "Running Python tests..."
   # Skip e2e tests by default in pre-commit (require live API + network)
-  PYTEST_OUTPUT=$(pytest --exitfirst --tb=short -m "not e2e" 2>&1)
+  # #516: pytest-xdist parallel when available, serial fallback otherwise.
+  # Unquoted expansion is intentional (multi-word arg list): the helper
+  # returns `-n auto --dist loadfile` as SEPARATE words, or an empty string
+  # for a serial fallback. Quoting would collapse it into one bogus argument.
+  # shellcheck disable=SC2046
+  PYTEST_OUTPUT=$(pytest $(python_xdist_args) --exitfirst --tb=short -m "not e2e" 2>&1)
   PYTEST_EXIT=$?
 
   # Show the short summary tail — this is where FAILED/assert diagnostics live.
@@ -176,7 +231,10 @@ run_coverage() {
   # 阈值不在此处硬编码：pytest-cov 自动读取 pyproject.toml 的
   # [tool.coverage.report] fail_under（唯一事实源）。此前这里重复写了
   # --cov-fail-under=79.9，改 pyproject 不会让门禁跟着变，属于假配置。
-  PYTEST_OUTPUT=$(pytest --exitfirst --tb=short --cov=src -m "not e2e" 2>&1)
+  # #516: xdist 并行（可用时），pytest-cov 与 xdist 正常联动（worker 各自
+  # 测量、主进程合并），fail_under 判定不受影响。
+  # shellcheck disable=SC2046
+  PYTEST_OUTPUT=$(pytest $(python_xdist_args) --exitfirst --tb=short --cov=src -m "not e2e" 2>&1)
   PYTEST_EXIT=$?
   echo "$PYTEST_OUTPUT" | grep -E "(FAILED|PASSED|passed|failed|error|ERROR|TOTAL|assert)" | tail -10
   echo "$PYTEST_OUTPUT" | tail -5

package/hooks/adapter-common.sh

@@ -220,9 +220,58 @@ detect_mutation_testable() {
   fi
   return 1
 }
-# Detect if a Python project has mutmut installed and configured
+# Echo the pytest-xdist parallel arguments when xdist is importable, or an
+# empty string with a stderr warning when it is not (#516). NEVER fails and
+# never blocks: serial semantics are the safe fallback, the warning only tells
+# the user why Gate 5 is slow. `-n auto` uses every core; `--dist loadfile`
+# groups tests per file to avoid cross-file shared state (module-level
+# fixtures, shared sqlite) racing under parallel workers — measured on a
+# 2555-test project: 30-45min serial -> 2m23s parallel with zero flaky runs.
+# Callers must invoke unquoted and unset: `pytest $(python_xdist_args) ...`
+# (SC2086-intentional: the value is a multi-word argument list).
+python_xdist_args() {
+  if python -c "import xdist" >/dev/null 2>&1; then
+    printf '%s' "-n auto --dist loadfile"
+    return 0
+  fi
+  echo "⚠ pytest-xdist not installed — Gate 5 falls back to SERIAL (may be 10x slower). Install: pip install pytest-xdist" >&2
+  return 0
+}
+# Detect a usable Python mutation engine (#410).
+#
+# Two engines, probed in preference order:
+#   1. pytest-gremlins — mutation switching (no fork), natively cross-platform,
+#      requires Python >= 3.11. Preferred wherever it is installed.
+#   2. mutmut — fork()-based, so on Windows it is ONLY runnable under WSL.
+#
+# The original implementation returned success as soon as `mutmut --version`
+# worked, which on native Windows produced a structural deadlock: `mutmut run`
+# exits 1 with "please use the WSL", so every push was blocked by a check that
+# could never pass — while uninstalling mutmut flipped the gate to SKIP. The
+# platform branch below is what removes that deadlock; it never blocks, it only
+# decides whether Gate M runs or skips.
 detect_python_mutation_testable() {
-  # Check for mutmut installation
+  # Engine 1: pytest-gremlins (cross-platform, no fork).
+  if command -v python >/dev/null 2>&1 && python -c "import pytest_gremlins" >/dev/null 2>&1; then
+    return 0
+  fi
+
+  # Determine whether mutmut can actually RUN here, not merely whether it is
+  # installed — that distinction is the whole point of this function on Windows.
+  local _uname_s
+  _uname_s="$(uname -s 2>/dev/null || echo unknown)"
+  case "$_uname_s" in
+    MINGW*|MSYS*|CYGWIN*)
+      # Native Windows: mutmut is unusable without WSL.
+      if command -v wsl >/dev/null 2>&1 && wsl mutmut --version >/dev/null 2>&1; then
+        return 0
+      fi
+      echo "⚠ Gate M (Python): mutmut cannot run on native Windows (fork()-based) and no WSL mutmut found — SKIP. Install pytest-gremlins instead: pip install pytest-gremlins (needs Python >= 3.11)." >&2
+      return 1
+      ;;
+  esac
+
+  # Unix: mutmut is fine natively.
   if command -v mutmut >/dev/null 2>&1; then
     return 0
   fi

package/hooks/lib/coverage-total.sh

@@ -46,6 +46,16 @@ parse_coverage_total() {
 # report reads it after the Stage-2 loop (last writer wins across languages),
 # exactly as the inline branch did.
 enforce_python_coverage() {
+  # Critical-1 (#399): when the test run was skipped (PY_STRATEGY=skip, set in
+  # pre-commit before this Stage-2 function), refuse to consume a stale
+  # .coverage left by an earlier subset run — its low % is meaningless for a
+  # commit that changed no Python and must not BLOCK a docs-only change. This
+  # mirrors the TS branch's TESTS_SKIPPED guard.
+  if [ "${PY_TEST_SKIPPED:-false}" = "true" ]; then
+    echo "ℹ️  Test run was skipped — skipping Python coverage check."
+    return 0
+  fi
+
   if ! command -v coverage &> /dev/null; then
     echo "⚠️  coverage.py not available, cannot enforce Python coverage threshold."
     return 0
@@ -61,6 +71,16 @@ enforce_python_coverage() {
 
   if [ -n "$COVERAGE_PERCENT" ]; then
     if [ "$COVERAGE_PERCENT" -lt 80 ]; then
+      # #399: a subset run (smart test selection) measures only the tested
+      # files, so its global total is meaningless as a verdict — the full
+      # suite may well be >=80% even when the subset is not. Warn exactly as
+      # the TypeScript branch does (#286) instead of blocking. Full runs keep
+      # the hard BLOCK below.
+      if [ "${PARTIAL_TEST_RUN:-false}" = "true" ]; then
+        echo "⚠️  Partial coverage ${COVERAGE_PERCENT}% (subset test run, not full suite)."
+        echo "    Run the full suite for a real Python coverage check."
+        return 0
+      fi
       echo "❌ BLOCKED - Python coverage ${COVERAGE_PERCENT}% below 80% threshold"
       return 1
     fi

package/hooks/lib/python-incremental-typecheck.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+# Incremental Python type-checking helpers for the pre-commit Python branch
+# (#534). This is the FAST staged-only pass: pre-commit runs it on just the
+# .py/.pyi files that changed, so a large repo doesn't pay full-inference CPU
+# on every commit. It complements the FULL mypy pass (githooks/adapters/
+# python.sh run_static_analysis, reachable via the adapter CLI / CI), which is
+# where cross-module blind spots get caught.
+#
+# Sourced by githooks/pre-commit (the python) case. Requires CHANGED_FILES
+# (space/newline separated, repo-root relative) to be in scope, matching the
+# rest of the inline branch.
+#
+# Shellcheck: functions are the public API used by pre-commit; SC1091 sources
+# are intentionally not resolved here (adapter-common not needed).
+
+# Extract the staged .py/.pyi files from CHANGED_FILES, one per line.
+# Excludes vendored/ignored trees (.venv, venv, node_modules) that a hook
+# should never type-check. Handles whitespace in paths (echo + grep would
+# split them).
+python_staged_py_files() {
+  local f
+  while IFS= read -r f || [ -n "$f" ]; do
+    case "$f" in
+      .venv/*|venv/*|node_modules/*) continue ;;
+      *.py|*.pyi) printf '%s\n' "$f" ;;
+    esac
+  done <<< "$CHANGED_FILES"
+}
+
+# Run incremental mypy over the given staged files, returning mypy's REAL exit
+# code (never a pipe stage's). Adds --follow-imports=silent so that a change to
+# A.py that references an unchanged B.py does not surface B's own pre-existing
+# diagnostics — only caller-level signature errors are reported. Callers must
+# pass the file list as arguments (already filtered to .py/.pyi).
+#
+# Scope discipline (same .xp-gate-mypy-scope contract as python.sh): a staged
+# file OUTSIDE the declared mypy scope is skipped with a note rather than type-
+# checked, keeping the incremental pass consistent with the full pass. Files in
+# scope default to "under src/ or scripts/" when no scope file exists.
+python_typecheck_incremental() {
+  [ "$#" -gt 0 ] || return 0
+
+  # Resolve the declared scope (identical precedence to run_static_analysis):
+  #   .xp-gate-mypy-scope (one target per line) wins; else src/ and scripts/
+  #   are the implicit default. Any staged .py outside this surface is skipped.
+  local -a SCOPE_DIRS=()
+  local _declared=0 line target
+  if [ -f .xp-gate-mypy-scope ]; then
+    while IFS= read -r line || [ -n "$line" ]; do
+      line="${line%%#*}"
+      line="${line#"${line%%[![:space:]]*}"}"
+      line="${line%"${line##*[![:space:]]}"}"
+      [ -z "$line" ] && continue
+      _declared=1
+      # Normalize trailing slashes (`src/` -> `src`) so the `$target/*` prefix
+      # match below never sees a doubled slash (`src//a.py` would never match).
+      line="${line%/}"
+      SCOPE_DIRS+=("$line")
+    done < .xp-gate-mypy-scope
+  else
+    [ -d src ] && SCOPE_DIRS+=(src)
+    [ -d scripts ] && SCOPE_DIRS+=(scripts)
+  fi
+
+  # If a scope file is declared but yields nothing, every staged .py is out of
+  # scope -> SKIP (exit 0), mirroring the full pass.
+  local _in_scope=false
+  local -a TYPECHECK_FILES=()
+  local f
+  if [ "${#SCOPE_DIRS[@]}" -gt 0 ]; then
+    for f in "$@"; do
+      _in_scope=false
+      for target in "${SCOPE_DIRS[@]}"; do
+        case "$f" in
+          "$target"/*) _in_scope=true ;;
+        esac
+        [ "$_in_scope" = true ] && break
+      done
+      if [ "$_in_scope" = true ]; then
+        TYPECHECK_FILES+=("$f")
+      else
+        echo "  ℹ️  mypy incremental: '$f' is outside declared scope — skipped"
+      fi
+    done
+  fi
+
+  [ "${#TYPECHECK_FILES[@]}" -gt 0 ] || return 0
+
+  # Capture mypy's exit code directly. A pipeline like
+  #   mypy ... | head -20 | tail -10; rc=$?
+  # loses mypy's real status (the pipe stage's rc is used instead), which made
+  # the gate pass silently on real type errors. Redirect to a temp file, take
+  # mypy's own exit code, then print a truncated view.
+  local tmp
+  tmp="$(mktemp)"
+  echo "Running mypy type checking (incremental)..."
+  mypy --follow-imports=silent --ignore-missing-imports "${TYPECHECK_FILES[@]}" > "$tmp" 2>&1
+  local _mypy_rc=$?
+  head -20 "$tmp"
+  tail -10 "$tmp"
+  rm -f "$tmp"
+  return "$_mypy_rc"
+}

package/hooks/lib/python-test-selection.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+# Python smart test selection (#399) — the Python counterpart of the TypeScript
+# selector (#286). Kept in its own file so the strategy truth table and the
+# related-test lookup are unit-testable without executing the hook
+# (githooks/__tests__/issue-399-python-test-selection.test.bats).
+#
+# Why: Python pre-commit ran the FULL suite on every commit (2-5min on a
+# 1000+-test project). TypeScript already counts changed test files, resolves
+# related tests from changed sources, and only falls back to the full suite
+# for large batches. Python had no equivalent.
+#
+# Three tiers, matching the TS implementation:
+#   1. changed-tests — only test files changed: run exactly those.
+#   2. related      — source files changed, no test files: run the tests that
+#                     the path convention maps to them.
+#   3. full         — >20 test files changed, dependency files changed, or no
+#                     related tests found: run everything with the real
+#                     coverage threshold.
+# Tier 1/2 run WITHOUT the global coverage threshold (subset coverage is
+# meaningless globally); the caller marks the run partial so Stage 2 warns
+# instead of blocking.
+
+# Max changed test files before falling back to the full suite. Same value as
+# the TypeScript branch (#286) so the two languages behave alike.
+PY_MAX_CHANGED_TESTS=20
+
+# Dependency manifests: a change here can break ANY test, so the selector
+# always escalates to the full suite regardless of how few tests changed.
+PY_DEPENDENCY_PATTERNS='(^|/)(pyproject\.toml|setup\.py|setup\.cfg|requirements[^/]*\.txt)$'
+
+# Decide the strategy from three counts. Pure function — no filesystem access,
+# so the truth table is exhaustive and cheap to test.
+#
+#   $1 test_count  changed test files
+#   $2 src_count   changed non-test Python source files
+#   $3 deps_changed 1 when a dependency manifest changed, else 0
+#
+# Echoes exactly one of: full | changed-tests | related | skip
+select_python_test_strategy() {
+  local test_count="${1:-0}"
+  local src_count="${2:-0}"
+  local deps_changed="${3:-0}"
+
+  # Dependency changes can break anything -> always the full suite.
+  if [ "$deps_changed" = "1" ]; then
+    printf '%s' "full"
+    return 0
+  fi
+
+  # Large batch: the per-file selection loses its point, and the TS branch
+  # uses the same cutoff.
+  if [ "$test_count" -gt "$PY_MAX_CHANGED_TESTS" ] 2>/dev/null; then
+    printf '%s' "full"
+    return 0
+  fi
+
+  if [ "$test_count" -gt 0 ] 2>/dev/null; then
+    printf '%s' "changed-tests"
+    return 0
+  fi
+
+  if [ "$src_count" -gt 0 ] 2>/dev/null; then
+    printf '%s' "related"
+    return 0
+  fi
+
+  printf '%s' "skip"
+}
+
+# Resolve the test files related to one source file, following the conventions
+# pytest projects actually use. Echoes one path per line, deduplicated, in
+# discovery order. Echoes nothing when no candidate exists — the caller then
+# falls back to the full suite (tier 3).
+#
+#   $1 source path, e.g. src/foo/bar.py
+python_related_tests() {
+  local src_file="$1"
+  [ -n "$src_file" ] || return 0
+
+  local base="${src_file%.*}"
+  local filename="${base##*/}"
+  local dir
+  dir="$(dirname "$src_file")"
+
+  local candidates=""
+  # Directory-relative candidates first (most specific), then flat tests/,
+  # then the collocated forms. Covers both pytest naming conventions
+  # (test_<name>.py and <name>_test.py).
+  candidates="$candidates
+${dir}/test_${filename}.py
+${dir}/${filename}_test.py
+${dir}/__tests__/test_${filename}.py
+${dir}/__tests__/${filename}_test.py"
+  # tests/<dir-surviving-subpath>/… : src/foo/bar.py -> tests/foo/test_bar.py
+  local subpath="${dir#src/}"
+  [ "$subpath" = "$dir" ] && subpath="$dir"
+  candidates="$candidates
+tests/${subpath}/test_${filename}.py
+tests/${subpath}/${filename}_test.py"
+  candidates="$candidates
+tests/test_${filename}.py
+tests/${filename}_test.py"
+
+  local seen=""
+  local candidate
+  # IFS-safe iteration over newline-separated candidates.
+  while IFS= read -r candidate; do
+    [ -n "$candidate" ] || continue
+    [ -f "$candidate" ] || continue
+    # Dedupe: several patterns can resolve to the same file (e.g. a source at
+    # repo root has dir='.' so the relative and flat forms coincide).
+    case "$seen" in
+      *"|$candidate|"*) continue ;;
+    esac
+    seen="$seen|$candidate|"
+    printf '%s\n' "$candidate"
+  done <<EOF
+$candidates
+EOF
+}
+
+# True (0) when any of the given paths is a Python dependency manifest.
+python_dependency_changed() {
+  local path
+  for path in "$@"; do
+    [ -n "$path" ] || continue
+    if echo "$path" | grep -qE "$PY_DEPENDENCY_PATTERNS"; then
+      return 0
+    fi
+  done
+  return 1
+}

package/hooks/lib/typecheck.sh

@@ -3,6 +3,16 @@
 # Gate 1 TypeScript static analysis — the single place that decides HOW a project
 # is type-checked (#436).
 #
+# NOT incremental (deliberately, #534): TypeScript type-checking cannot be
+# reduced to "staged files only". `tsc` has no single-file check mode — it must
+# resolve the whole module graph to report a file's real type errors, so
+# type-checking only the changed files would silently miss "changing B.ts breaks
+# an unchanged caller in A.ts". Pre-commit therefore runs the project's full
+# typecheck (`npm run typecheck` / `npx tsc --noEmit`) every commit. This is the
+# CORRECTNESS-preserving exception to xp-gate's incremental rule: lint (ESLint /
+# Biome) IS incremental (staged files / --staged), but type-check stays full
+# because incremental type-checking would be wrong, not just slower.
+#
 # Before this file, `githooks/pre-commit` and `githooks/adapters/typescript.sh` each
 # hardcoded `npx tsc --noEmit`. Plain tsc cannot resolve `.vue` single-file
 # components, so every `import X from './Foo.vue'` became a TS2307 and a Vue project