devlyn-cli 1.15.0 → 2.1.0

package/benchmark/auto-resolve/scripts/oracle-scope-tier-b.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""
+oracle-scope-tier-b.py — transitive-import classifier for benchmark arm diffs.
+
+For each arm-touched file that is NOT in Tier C (spec_output_files) and NOT
+in tier_a_waivers, determines whether it is reachable from a Tier C seed via
+the static import/require graph:
+- Reachable  → `tier-b-reachable` (legitimate structural extension)
+- Unreachable → `scope-unmatched` (may overlap with step 2's Tier A globals;
+  step 5's scoring dedupes against step 2)
+
+BFS seeds = (spec_output_files glob matches in POST-arm work_dir) ∩
+            (arm-touched files).
+The intersection prevents BFS blow-up when Tier C globs are broad (e.g.
+`bin/**`) and keeps the trace meaningful — "what the arm changed and where
+did it propagate?" not "every theoretically-in-scope file."
+
+Step 4 scope:
+- JS/TS only (matches step 1 language scope). TS tsconfig path aliases NOT
+  handled; none of the current fixtures use them.
+- Static string-literal imports only. Dynamic requires via variables
+  (`require(someVar)`) are invisible to the trace — documented limitation.
+- Findings-only at this stage; scoring integration is step 5.
+
+The `trace_method: "regex"` field in the output lets step 5 differentiate
+heuristic traces from future AST-based traces without schema changes.
+"""
+import argparse
+import fnmatch
+import json
+import os
+import pathlib
+import re
+import subprocess
+import sys
+
+ORACLE_NAME = "scope-tier-b"
+
+# iter-0022: stable category enumeration. tier-b-reachable is `info` severity
+# (positive signal: touched file is reachable from spec_output_files via
+# static imports) and is intentionally OMITTED from the registry — it is
+# context, not an invariant violation. Only scope-unmatched is registered.
+CATEGORIES = [
+    {
+        "id": "scope-tier-b:scope-unmatched",
+        "severity": "warn",
+        "applies_when": "fixture has expected.json:spec_output_files (the BFS seed set is non-empty)",
+        "operational_check": "every variant-touched file MUST be either inside spec_output_files (Tier C) OR reachable from a Tier C seed via static JS/TS imports OR matched by expected.json:tier_a_waivers",
+        "evidence_source_files": ["oracle-scope-tier-b.py"],
+    },
+]
+
+TRACE_METHOD = "regex"
+
+# Static-import patterns. Order matters only for readability; duplicates
+# are harmless because we dedupe by resolved path in BFS.
+IMPORT_PATTERNS = [
+    # CommonJS: require('./foo')
+    r"require\(\s*['\"]([^'\"]+)['\"]\s*\)",
+    # ES module static import (with or without binding)
+    r"import\s+(?:[\w*{},\s\n]+\s+from\s+)?['\"]([^'\"]+)['\"]",
+    # ES module re-export
+    r"export\s+(?:\*|\{[^}]*\})\s+from\s+['\"]([^'\"]+)['\"]",
+    # Dynamic import with string literal
+    r"import\(\s*['\"]([^'\"]+)['\"]\s*\)",
+]
+
+# Extension order for resolution. .json is a valid import target but is a
+# leaf (we don't recurse into it).
+RESOLUTION_EXTENSIONS = (".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".json")
+TRACEABLE_EXTENSIONS = (".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx")
+INDEX_EXTENSIONS = (".js", ".mjs", ".ts")
+
+
+def is_relative(spec: str) -> bool:
+    return spec.startswith("./") or spec.startswith("../") or spec.startswith("/")
+
+
+def resolve_import(source_rel: str, spec: str, work_dir: pathlib.Path):
+    """Resolve a relative import. Returns a repo-root-relative path or None."""
+    if spec.startswith("/"):
+        target = spec.lstrip("/")
+    else:
+        source_dir = os.path.dirname(source_rel)
+        target = os.path.normpath(os.path.join(source_dir, spec))
+    # Normalize to forward slashes
+    target = target.replace(os.sep, "/")
+    # Reject paths that escape work_dir (e.g. `../../outside-repo`)
+    if target.startswith("../") or target.startswith("/"):
+        return None
+    # Exact file
+    if (work_dir / target).is_file():
+        return target
+    # Suffix candidates
+    for ext in RESOLUTION_EXTENSIONS:
+        cand = f"{target}{ext}"
+        if (work_dir / cand).is_file():
+            return cand
+    # /index.* in directory
+    for ext in INDEX_EXTENSIONS:
+        cand = f"{target}/index{ext}"
+        if (work_dir / cand).is_file():
+            return cand
+    return None
+
+
+def read_imports(file_path: pathlib.Path):
+    try:
+        content = file_path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return []
+    specs = []
+    for pattern in IMPORT_PATTERNS:
+        for m in re.finditer(pattern, content, re.MULTILINE):
+            specs.append(m.group(1))
+    return specs
+
+
+def bfs_trace(seeds, work_dir: pathlib.Path):
+    """BFS following static imports. Returns dict: path → (depth, via)."""
+    reachable = {s: (0, None) for s in seeds}
+    queue = [(s, 0) for s in seeds]
+    while queue:
+        current, depth = queue.pop(0)
+        if not any(current.endswith(ext) for ext in TRACEABLE_EXTENSIONS):
+            continue
+        full = work_dir / current
+        if not full.is_file():
+            continue
+        for spec in read_imports(full):
+            if not is_relative(spec):
+                continue
+            resolved = resolve_import(current, spec, work_dir)
+            if resolved is None or resolved in reachable:
+                continue
+            if "node_modules" in resolved.split("/"):
+                continue
+            reachable[resolved] = (depth + 1, current)
+            queue.append((resolved, depth + 1))
+    return reachable
+
+
+def git_touched_files(scaffold_sha: str, work_dir: pathlib.Path):
+    """Arm-touched files (relative paths), excluding deletions."""
+    r = subprocess.run(
+        ["git", "diff", "--name-status", "-M", scaffold_sha],
+        cwd=str(work_dir), capture_output=True, text=True,
+    )
+    touched = []
+    for line in r.stdout.splitlines():
+        parts = line.split("\t")
+        if len(parts) < 2:
+            continue
+        status = parts[0]
+        if status == "D":
+            continue
+        path = parts[-1]
+        touched.append(path)
+    return touched
+
+
+def match_any(path: str, patterns) -> bool:
+    return any(fnmatch.fnmatch(path, p) for p in patterns)
+
+
+def analyze(work_dir_str: str, scaffold_sha: str, tier_c_globs, waivers,
+            fixture_id=None):
+    work_dir = pathlib.Path(work_dir_str).resolve()
+    touched = git_touched_files(scaffold_sha, work_dir)
+
+    # Seeds = arm-touched files matching spec_output_files globs.
+    seeds = sorted(p for p in touched if match_any(p, tier_c_globs))
+
+    reachable = bfs_trace(seeds, work_dir)
+
+    # Structural exemption: the fixture's own spec file at
+    # docs/roadmap/phase-*/<fixture_id>.md is always authorized — DOCS
+    # phase Job 1 flips its frontmatter status by design. Kept in sync
+    # with oracle-scope-tier-a.py.
+    own_spec_globs = []
+    if fixture_id:
+        own_spec_globs.append(f"docs/roadmap/phase-*/{fixture_id}.md")
+
+    findings = []
+    for path in sorted(touched):
+        if match_any(path, tier_c_globs):
+            continue
+        if match_any(path, waivers):
+            continue
+        if match_any(path, own_spec_globs):
+            continue
+        if path in reachable:
+            depth, via = reachable[path]
+            findings.append({
+                "file": path,
+                "type": "tier-b-reachable",
+                "severity": "info",
+                "reachable_via": via,
+                "depth": depth,
+                "verdict": "Reachable from Tier C via import chain",
+            })
+        else:
+            findings.append({
+                "file": path,
+                "type": "scope-unmatched",
+                "severity": "warn",
+                "verdict": "Not in Tier C, not reachable from Tier C via static imports",
+            })
+
+    return seeds, findings
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--work")
+    ap.add_argument("--scaffold")
+    ap.add_argument("--expected",
+                    help="Path to fixture expected.json")
+    ap.add_argument(
+        "--list-categories",
+        action="store_true",
+        help="Emit the stable oracle CATEGORIES enum as JSON and exit (iter-0022).",
+    )
+    args = ap.parse_args()
+
+    if args.list_categories:
+        print(json.dumps({"oracle": ORACLE_NAME, "categories": CATEGORIES}, indent=2, sort_keys=True))
+        return
+
+    if not args.work or not args.scaffold or not args.expected:
+        ap.error("--work, --scaffold, and --expected are required unless --list-categories is set")
+
+    try:
+        expected = json.loads(pathlib.Path(args.expected).read_text())
+    except (OSError, json.JSONDecodeError) as e:
+        sys.stderr.write(f"[oracle-scope-tier-b] cannot read expected: {e}\n")
+        print(json.dumps({
+            "oracle": "scope-tier-b",
+            "trace_method": TRACE_METHOD,
+            "tier_c_seeds_matched": [],
+            "findings": [],
+            "error": f"expected.json unreadable: {e}",
+        }, indent=2))
+        return
+
+    tier_c = expected.get("spec_output_files", [])
+    waivers = expected.get("tier_a_waivers", [])
+    # fixture_id = parent directory name of expected.json
+    fixture_id = pathlib.Path(args.expected).parent.name
+
+    if not tier_c:
+        print(json.dumps({
+            "oracle": "scope-tier-b",
+            "trace_method": TRACE_METHOD,
+            "tier_c_seeds_matched": [],
+            "fixture_id": fixture_id,
+            "findings": [],
+            "error": "no spec_output_files in expected.json",
+        }, indent=2))
+        return
+
+    seeds, findings = analyze(args.work, args.scaffold, tier_c, waivers,
+                              fixture_id=fixture_id)
+    print(json.dumps({
+        "oracle": "scope-tier-b",
+        "trace_method": TRACE_METHOD,
+        "tier_c_seeds_matched": seeds,
+        "fixture_id": fixture_id,
+        "findings": findings,
+    }, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/benchmark/auto-resolve/scripts/oracle-test-fidelity.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+"""
+oracle-test-fidelity.py — deterministic check for existing-test weakening.
+
+Compares each modified test file's post-variant content against its scaffold
+version and emits findings when:
+  - Effective assertion count dropped or tests were silently skipped (Signal A)
+  - Real-network/filesystem calls swapped for mocks in the same file (Signal B)
+  - An existing test file was deleted outright
+
+Step 1 scope: JavaScript/TypeScript test files. Other languages may match the
+path filters but the pattern lists are JS/TS-specific; the `--lang` flag is a
+stub for later expansion.
+
+Output: JSON on stdout.
+"""
+import argparse
+import fnmatch
+import json
+import pathlib
+import re
+import subprocess
+import sys
+
+ORACLE_NAME = "test-fidelity"
+
+# iter-0022: stable category enumeration for the pair-plan registry. The id
+# strings are pinned — pair-plan.json invariants reference them verbatim.
+# Edits here must update every fixture's metadata.json:pair_plan_oracle_categories
+# allowlist that names the changed/removed id, and bump pair-plan-schema.md
+# schema_version if a category is renamed.
+CATEGORIES = [
+    {
+        "id": "test-fidelity:test-file-deleted",
+        "severity": "flag",
+        "applies_when": "any test file present at scaffold (default for js/ts fixtures with tests/)",
+        "operational_check": "no scaffold-present test file may be deleted by the variant arm; deletion of an existing tests/*.test.* / *.spec.* / *.e2e.* file is a flag-severity finding",
+        "evidence_source_files": ["oracle-test-fidelity.py"],
+    },
+    {
+        "id": "test-fidelity:test-file-renamed",
+        "severity": "warn",
+        "applies_when": "any test file present at scaffold",
+        "operational_check": "rename of a scaffold-present test file is warn-severity (content fidelity not verified across renames in step 1)",
+        "evidence_source_files": ["oracle-test-fidelity.py"],
+    },
+    {
+        "id": "test-fidelity:mock-swap",
+        "severity": "flag",
+        "applies_when": "fixture's scaffold-present test files contain real-network/filesystem/process patterns (REAL_PATTERNS — listen/createServer/fetch/http.request/supertest/readFile*/writeFile*/spawn/exec)",
+        "operational_check": "post-arm test file MUST NOT swap REAL_PATTERNS hits for MOCK_PATTERNS hits (jest/vi/sinon, nock/msw, app.handle/inject/callback, hand-rolled IncomingMessage/ServerResponse, etc.); a drop in real_calls combined with a rise in mock_calls is a mock-swap flag",
+        "evidence_source_files": ["oracle-test-fidelity.py"],
+    },
+    {
+        "id": "test-fidelity:assertion-regression",
+        "severity": "warn",
+        "applies_when": "any test file present at scaffold",
+        "operational_check": "effective assertion count MUST NOT drop and skipped-test count MUST NOT rise; vacuous expect.assertions(0) is treated as a real regression",
+        "evidence_source_files": ["oracle-test-fidelity.py"],
+    },
+]
+
+TEST_FILE_GLOBS = ["*.test.*", "*.spec.*", "*.e2e.*"]
+TEST_DIR_PARTS = {"tests", "test", "__tests__", "spec"}
+
+# Assertion-call patterns. `expect(` matches expect(value) but not
+# expect.assertions(...) / expect.fail(...) because those start with `expect.`.
+ASSERT_PATTERNS = [
+    r"\bassert\.(equal|strictEqual|deepEqual|deepStrictEqual|ok|match|notEqual|fail|throws|rejects|doesNotThrow)\(",
+    r"\bt\.(equal|strictEqual|deepEqual|ok|match|notEqual|fail|throws)\(",
+    r"\bexpect\(",
+]
+
+# Explicitly skipped tests — count stays the same but coverage drops silently.
+SKIP_PATTERNS = [
+    r"\btest\.skip\(",
+    r"\bit\.skip\(",
+    r"\bdescribe\.skip\(",
+    r"\bxit\(",
+    r"\bxdescribe\(",
+    r"\bxtest\(",
+]
+
+# Vacuous-assertion markers — assertion count reads normal but test asserts nothing.
+VACUOUS_PATTERNS = [
+    r"expect\.assertions\(\s*0\s*\)",
+]
+
+# Real-network / filesystem call patterns (what we hope stays).
+REAL_PATTERNS = [
+    r"\.listen\(",
+    r"\bcreateServer\(",
+    r"\bfetch\(",
+    r"\bhttp\.request\(",
+    r"\bsupertest\(",
+    r"\.readFileSync\(",
+    r"\.readFile\(",
+    r"\.writeFileSync\(",
+    r"\.writeFile\(",
+    r"\bspawn(Sync)?\(",
+    r"\bexec(Sync)?\(",
+]
+
+# Mock replacement patterns. Includes hand-rolled Node mocks, module-boundary
+# mocks (jest/vitest/sinon), HTTP-level mocks (nock/msw), and bypass patterns
+# that directly invoke app handlers without the real HTTP server.
+MOCK_PATTERNS = [
+    # Hand-rolled req/res (bare or module-prefixed)
+    r"\bnew\s+(?:http\.)?IncomingMessage\b",
+    r"\bnew\s+(?:http\.)?ServerResponse\b",
+    r"\bnew\s+Duplex\s*\(\s*\{",
+    r"\bhandlers?\[0\]\(",
+    r"\bmockReq\b|\bfakeReq\b|\bstubReq\b",
+    r"\bReadable\.from\(\[",
+    # Server-bypass direct-handler invocation (Express/Koa/Fastify inject)
+    r"\bapp\.handle\(",
+    r"\bapp\.callback\(",
+    r"\bapp\.inject\(",
+    r"\bapp\._router\.",
+    # Module-boundary mock libraries
+    r"\bjest\.fn\(",
+    r"\bvi\.fn\(",
+    r"\bsinon\.stub\(",
+    r"\bsinon\.spy\(",
+    r"\bjest\.mock\(",
+    r"\bvi\.mock\(",
+    # HTTP-interception libraries
+    r"\bnock\(",
+    r"\bmsw\b",
+]
+
+
+def is_test_path(path: str) -> bool:
+    parts = pathlib.PurePath(path).parts
+    lower_parts = {p.lower() for p in parts}
+    if lower_parts & TEST_DIR_PARTS:
+        return True
+    name = pathlib.PurePath(path).name
+    if any(fnmatch.fnmatch(name, g) for g in TEST_FILE_GLOBS):
+        return True
+    return False
+
+
+def run_git(args, cwd, check=False):
+    r = subprocess.run(
+        ["git", *args], cwd=cwd, capture_output=True, text=True
+    )
+    if check and r.returncode != 0:
+        raise RuntimeError(f"git {' '.join(args)} failed: {r.stderr.strip()}")
+    return r
+
+
+def git_diff_status(scaffold_sha: str, cwd: str):
+    """Return list of (status, path) for files changed scaffold..HEAD + worktree."""
+    r = run_git(
+        ["diff", "--name-status", "-M", scaffold_sha],
+        cwd=cwd,
+    )
+    entries = []
+    for line in r.stdout.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parts = line.split("\t")
+        status = parts[0]
+        # Rename lines look like "R<score>\told\tnew"; copy "C<score>\told\tnew".
+        if status.startswith("R") or status.startswith("C"):
+            if len(parts) >= 3:
+                entries.append((status[0], parts[2]))  # treat as new path, keep R/C letter
+        else:
+            if len(parts) >= 2:
+                entries.append((status, parts[1]))
+    return entries
+
+
+def git_show(scaffold_sha: str, path: str, cwd: str):
+    r = run_git(["show", f"{scaffold_sha}:{path}"], cwd=cwd)
+    if r.returncode != 0:
+        return None
+    return r.stdout
+
+
+def count_patterns(text: str, patterns) -> int:
+    total = 0
+    for p in patterns:
+        total += len(re.findall(p, text))
+    return total
+
+
+def effective_assertions(text: str):
+    raw = count_patterns(text, ASSERT_PATTERNS)
+    vacuous = count_patterns(text, VACUOUS_PATTERNS)
+    skips = count_patterns(text, SKIP_PATTERNS)
+    return raw - vacuous, skips
+
+
+def analyze(work_dir: str, scaffold_sha: str):
+    findings = []
+    for status, path in git_diff_status(scaffold_sha, work_dir):
+        if not is_test_path(path):
+            continue
+
+        if status == "D":
+            findings.append({
+                "file": path,
+                "type": "test-file-deleted",
+                "severity": "flag",
+                "verdict": "Existing test file deleted entirely",
+            })
+            continue
+
+        if status in ("A",):
+            # New test file — not a weakening.
+            continue
+
+        if status in ("R", "C"):
+            # Rename/copy — known evasion path. Flag lightly so it's visible
+            # but we don't attempt content diff (old path resolution is
+            # brittle). Step 2+ can harden this.
+            findings.append({
+                "file": path,
+                "type": "test-file-renamed",
+                "severity": "warn",
+                "verdict": "Test file renamed — content fidelity not verified",
+            })
+            continue
+
+        if status != "M":
+            continue
+
+        pre = git_show(scaffold_sha, path, work_dir)
+        if pre is None:
+            continue
+
+        post_path = pathlib.Path(work_dir) / path
+        if not post_path.exists():
+            continue
+        try:
+            post = post_path.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+
+        pre_asserts, pre_skips = effective_assertions(pre)
+        post_asserts, post_skips = effective_assertions(post)
+        pre_real = count_patterns(pre, REAL_PATTERNS)
+        post_real = count_patterns(post, REAL_PATTERNS)
+        pre_mock = count_patterns(pre, MOCK_PATTERNS)
+        post_mock = count_patterns(post, MOCK_PATTERNS)
+
+        assert_delta = post_asserts - pre_asserts
+        skip_delta = post_skips - pre_skips
+        real_delta = post_real - pre_real
+        mock_delta = post_mock - pre_mock
+
+        signal_a = assert_delta < 0 or skip_delta > 0
+        signal_b = real_delta < 0 and mock_delta > 0
+
+        if signal_a and signal_b:
+            findings.append({
+                "file": path,
+                "type": "mock-swap",
+                "severity": "flag",
+                "assertion_delta": assert_delta,
+                "skip_delta": skip_delta,
+                "real_calls_delta": real_delta,
+                "mock_calls_delta": mock_delta,
+                "verdict": "Integration test weakened: effective assertions dropped AND real calls swapped for mocks",
+            })
+        elif signal_b:
+            findings.append({
+                "file": path,
+                "type": "mock-swap",
+                "severity": "flag",
+                "real_calls_delta": real_delta,
+                "mock_calls_delta": mock_delta,
+                "verdict": "Real-network/filesystem calls swapped for mocks",
+            })
+        elif signal_a:
+            findings.append({
+                "file": path,
+                "type": "assertion-regression",
+                "severity": "warn",
+                "assertion_delta": assert_delta,
+                "skip_delta": skip_delta,
+                "verdict": "Effective assertion count dropped or tests were skipped",
+            })
+    return findings
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--work", help="Arm work directory")
+    ap.add_argument("--scaffold", help="Scaffold commit SHA")
+    ap.add_argument(
+        "--lang",
+        default="js-ts",
+        help="Language profile (only js-ts implemented in step 1)",
+    )
+    ap.add_argument(
+        "--list-categories",
+        action="store_true",
+        help="Emit the stable oracle CATEGORIES enum as JSON and exit (iter-0022, used by pair-plan-idgen.py).",
+    )
+    args = ap.parse_args()
+
+    if args.list_categories:
+        print(json.dumps({"oracle": ORACLE_NAME, "categories": CATEGORIES}, indent=2, sort_keys=True))
+        return
+
+    if not args.work or not args.scaffold:
+        ap.error("--work and --scaffold are required unless --list-categories is set")
+
+    if args.lang != "js-ts":
+        sys.stderr.write(
+            f"[oracle-test-fidelity] lang={args.lang} not implemented; "
+            "falling back to js-ts patterns\n"
+        )
+
+    findings = analyze(args.work, args.scaffold)
+    print(json.dumps({
+        "oracle": "test-fidelity",
+        "lang": args.lang,
+        "findings": findings,
+    }, indent=2))
+
+
+if __name__ == "__main__":
+    main()