devlyn-cli 3.2.1 → 3.3.0

package/benchmark/auto-resolve/scripts/oracle-scope-tier-a.py

@@ -1,256 +0,0 @@
-#!/usr/bin/env python3
-"""
-oracle-scope-tier-a.py — deterministic detector for categorical file-path
-violations. Flags touches of paths that are never legitimately needed by an
-implementation task (planning docs, CI config, dep-install output, runtime
-artifacts, env/secret files) and lockfile deletions.
-
-Complementary to oracle-test-fidelity.py, which handles weakening INSIDE
-existing test files. This oracle only cares about WHICH files are touched.
-
-Path matching uses fnmatch with normalized, repo-root-relative paths.
-`docs/**` matches `docs/<anything>` but NOT `server/docs/readme.md` — the
-anchoring is left-only, which is what we want. Per-oracle convention
-documented here; step 1's content oracle uses regex instead.
-
-Fixtures can waive any Tier A pattern via `expected.json::tier_a_waivers`
-(list of fnmatch globs). Load-bearing case: F9 e2e-ideate-to-resolve
-legitimately creates docs/VISION.md, docs/ROADMAP.md, docs/roadmap/**.
-
-Step 2 scope: findings only. Scoring integration is a later step.
-"""
-import argparse
-import fnmatch
-import json
-import os
-import pathlib
-import subprocess
-import sys
-
-from pair_evidence_contract import loads_strict_json_object
-
-ORACLE_NAME = "scope-tier-a"
-
-# iter-0022: stable category enumeration. See header comment in
-# oracle-test-fidelity.py for the edit-discipline rules. tier-a-violation is
-# ONE umbrella category covering the 5 path-glob groups (planning-doc,
-# ci-config, node-modules, test-results-or-coverage, env-secrets) plus the 2
-# basename rules (.log suffix, .env/secrets. prefix); splitting into 7 sub-
-# categories was rejected during iter-0022 R0 because the oracle emits a
-# single finding-row per touched path regardless.
-CATEGORIES = [
-    {
-        "id": "scope-tier-a:lockfile-deletion",
-        "severity": "hard",
-        "applies_when": "scaffold contains a lockfile (package-lock.json / yarn.lock / pnpm-lock.yaml / bun.lock / bun.lockb)",
-        "operational_check": "variant arm MUST NOT delete a scaffold-present lockfile",
-        "evidence_source_files": ["oracle-scope-tier-a.py"],
-    },
-    {
-        "id": "scope-tier-a:tier-a-violation",
-        "severity": "hard",
-        "applies_when": "any fixture (the protected paths exist conceptually for every JS/TS repo)",
-        "operational_check": "variant arm MUST NOT add or modify paths matching: docs/roadmap/** | docs/VISION.md | docs/ROADMAP.md | .github/** | node_modules/** | **/node_modules/** | test-results/** | coverage/** | .nyc_output/** | basename suffix .log | basename prefix .env or secrets.",
-        "evidence_source_files": ["oracle-scope-tier-a.py"],
-    },
-]
-
-# Path globs — fnmatch-style, left-anchored. `**` is treated as `*` by
-# fnmatch (no special recursive semantics), so `docs/roadmap/**` matches
-# `docs/roadmap/anything/nested/here` because `*` matches `/` in fnmatch.
-TIER_A_PATH_GLOBS = [
-    # Planning docs — implementation tasks should never edit these.
-    "docs/roadmap/**",
-    "docs/VISION.md",
-    "docs/ROADMAP.md",
-    # CI / repo-wide config.
-    ".github/**",
-    # Install outputs.
-    "node_modules/**",
-    "**/node_modules/**",
-    # Runtime / test artifacts.
-    "test-results/**",
-    "coverage/**",
-    ".nyc_output/**",
-]
-
-# Basename suffix matches (run against os.path.basename so any depth hits).
-TIER_A_BASENAME_SUFFIXES = {
-    ".log",
-}
-
-# Basename prefix matches. `.env` → `.env`, `.env.local`, `.env.production`.
-# `secrets.` → `secrets.json`, `secrets.yaml`.
-TIER_A_BASENAME_PREFIXES = {
-    ".env",
-    "secrets.",
-}
-
-# Lockfiles — modification is legitimate when deps change; deletion is not.
-# Only flag D status AND only if the file existed at scaffold.
-LOCKFILE_NAMES = {
-    "package-lock.json",
-    "yarn.lock",
-    "pnpm-lock.yaml",
-    "bun.lock",
-    "bun.lockb",
-}
-
-
-def run_git(args, cwd):
-    return subprocess.run(
-        ["git", *args], cwd=cwd, capture_output=True, text=True
-    )
-
-
-def git_diff_status(scaffold_sha, cwd):
-    r = run_git(["diff", "--name-status", "-M", scaffold_sha], cwd=cwd)
-    entries = []
-    for line in r.stdout.splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        parts = line.split("\t")
-        status = parts[0]
-        if status.startswith("R") or status.startswith("C"):
-            if len(parts) >= 3:
-                # Treat as new path; keep R/C letter for reporting.
-                entries.append((status[0], parts[2]))
-        else:
-            if len(parts) >= 2:
-                entries.append((status, parts[1]))
-    return entries
-
-
-def existed_at_scaffold(scaffold_sha, path, cwd):
-    r = run_git(["cat-file", "-e", f"{scaffold_sha}:{path}"], cwd=cwd)
-    return r.returncode == 0
-
-
-def matches_any_glob(path, patterns):
-    for p in patterns:
-        if fnmatch.fnmatch(path, p):
-            return p
-    return None
-
-
-def matches_basename(path, suffixes, prefixes):
-    base = os.path.basename(path)
-    for s in suffixes:
-        if base.endswith(s):
-            return f"*{s}"
-    for p in prefixes:
-        if base.startswith(p):
-            return f"{p}*"
-    return None
-
-
-def is_waived(path, waivers):
-    for w in waivers:
-        if fnmatch.fnmatch(path, w):
-            return True
-    return False
-
-
-def analyze(work_dir, scaffold_sha, waivers, fixture_id=None):
-    findings = []
-    entries = git_diff_status(scaffold_sha, work_dir)
-
-    for status, path in entries:
-        if is_waived(path, waivers):
-            continue
-
-        # Lockfile deletion — only when file existed at scaffold.
-        if status == "D" and os.path.basename(path) in LOCKFILE_NAMES:
-            if existed_at_scaffold(scaffold_sha, path, work_dir):
-                findings.append({
-                    "file": path,
-                    "type": "lockfile-deletion",
-                    "severity": "hard",
-                    "status": status,
-                    "verdict": "Lockfile deleted (existed at scaffold)",
-                })
-            continue
-
-        # For categorical path violations, only flag when the arm ADDED or
-        # MODIFIED (including rename/copy). Pure deletion of a non-lockfile
-        # Tier A path would also be suspicious but is rare in practice and
-        # overlaps with test-fidelity; leave for a later iteration if needed.
-        if status not in ("A", "M", "R", "C"):
-            continue
-
-        matched = matches_any_glob(path, TIER_A_PATH_GLOBS)
-        if matched is None:
-            matched = matches_basename(
-                path, TIER_A_BASENAME_SUFFIXES, TIER_A_BASENAME_PREFIXES
-            )
-        if matched is None:
-            continue
-
-        findings.append({
-            "file": path,
-            "type": "tier-a-violation",
-            "severity": "hard",
-            "status": status,
-            "matched_pattern": matched,
-            "verdict": "Touched a path categorically outside implementation scope",
-        })
-
-    return findings
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--work")
-    ap.add_argument("--scaffold")
-    ap.add_argument(
-        "--expected",
-        help="Path to fixture expected.json (for tier_a_waivers)",
-        default=None,
-    )
-    ap.add_argument(
-        "--list-categories",
-        action="store_true",
-        help="Emit the stable oracle CATEGORIES enum as JSON and exit (iter-0022).",
-    )
-    args = ap.parse_args()
-
-    if args.list_categories:
-        print(json.dumps({"oracle": ORACLE_NAME, "categories": CATEGORIES}, indent=2, sort_keys=True))
-        return
-
-    if not args.work or not args.scaffold:
-        ap.error("--work and --scaffold are required unless --list-categories is set")
-
-    waivers = []
-    fixture_id = None
-    expected_error = None
-    if args.expected:
-        exp_path = pathlib.Path(args.expected)
-        # fixture_id = parent directory name of expected.json
-        fixture_id = exp_path.parent.name
-        try:
-            expected = loads_strict_json_object(exp_path.read_text())
-            raw = expected.get("tier_a_waivers", [])
-            if isinstance(raw, list):
-                waivers = [w for w in raw if isinstance(w, str)]
-        except (OSError, json.JSONDecodeError, ValueError) as e:
-            expected_error = f"expected.json unreadable: {e}"
-            sys.stderr.write(
-                f"[oracle-scope-tier-a] could not read waivers from {args.expected}: {e}\n"
-            )
-
-    findings = analyze(args.work, args.scaffold, waivers, fixture_id=fixture_id)
-    report = {
-        "oracle": "scope-tier-a",
-        "waivers": waivers,
-        "fixture_id": fixture_id,
-        "findings": findings,
-    }
-    if expected_error:
-        report["error"] = expected_error
-    print(json.dumps(report, indent=2))
-
-
-if __name__ == "__main__":
-    main()

package/benchmark/auto-resolve/scripts/oracle-scope-tier-b.py

@@ -1,287 +0,0 @@
-#!/usr/bin/env python3
-"""
-oracle-scope-tier-b.py — transitive-import classifier for benchmark arm diffs.
-
-For each arm-touched file that is NOT in Tier C (spec_output_files) and NOT
-in tier_a_waivers, determines whether it is reachable from a Tier C seed via
-the static import/require graph:
-- Reachable  → `tier-b-reachable` (legitimate structural extension)
-- Unreachable → `scope-unmatched` (may overlap with step 2's Tier A globals;
-  step 5's scoring dedupes against step 2)
-
-BFS seeds = (spec_output_files glob matches in POST-arm work_dir) ∩
-            (arm-touched files).
-The intersection prevents BFS blow-up when Tier C globs are broad (e.g.
-`bin/**`) and keeps the trace meaningful — "what the arm changed and where
-did it propagate?" not "every theoretically-in-scope file."
-
-Step 4 scope:
-- JS/TS only (matches step 1 language scope). TS tsconfig path aliases NOT
-  handled; none of the current fixtures use them.
-- Static string-literal imports only. Dynamic requires via variables
-  (`require(someVar)`) are invisible to the trace — documented limitation.
-- Findings-only at this stage; scoring integration is step 5.
-
-The `trace_method: "regex"` field in the output lets step 5 differentiate
-heuristic traces from future AST-based traces without schema changes.
-"""
-import argparse
-import fnmatch
-import json
-import os
-import pathlib
-import re
-import subprocess
-import sys
-
-from pair_evidence_contract import loads_strict_json_object
-
-ORACLE_NAME = "scope-tier-b"
-
-# iter-0022: stable category enumeration. tier-b-reachable is `info` severity
-# (positive signal: touched file is reachable from spec_output_files via
-# static imports) and is intentionally OMITTED from the registry — it is
-# context, not an invariant violation. Only scope-unmatched is registered.
-CATEGORIES = [
-    {
-        "id": "scope-tier-b:scope-unmatched",
-        "severity": "warn",
-        "applies_when": "fixture has expected.json:spec_output_files (the BFS seed set is non-empty)",
-        "operational_check": "every variant-touched file MUST be either inside spec_output_files (Tier C) OR reachable from a Tier C seed via static JS/TS imports OR matched by expected.json:tier_a_waivers",
-        "evidence_source_files": ["oracle-scope-tier-b.py"],
-    },
-]
-
-TRACE_METHOD = "regex"
-
-# Static-import patterns. Order matters only for readability; duplicates
-# are harmless because we dedupe by resolved path in BFS.
-IMPORT_PATTERNS = [
-    # CommonJS: require('./foo')
-    r"require\(\s*['\"]([^'\"]+)['\"]\s*\)",
-    # ES module static import (with or without binding)
-    r"import\s+(?:[\w*{},\s\n]+\s+from\s+)?['\"]([^'\"]+)['\"]",
-    # ES module re-export
-    r"export\s+(?:\*|\{[^}]*\})\s+from\s+['\"]([^'\"]+)['\"]",
-    # Dynamic import with string literal
-    r"import\(\s*['\"]([^'\"]+)['\"]\s*\)",
-]
-
-# Extension order for resolution. .json is a valid import target but is a
-# leaf (we don't recurse into it).
-RESOLUTION_EXTENSIONS = (".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".json")
-TRACEABLE_EXTENSIONS = (".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx")
-INDEX_EXTENSIONS = (".js", ".mjs", ".ts")
-
-
-def is_relative(spec: str) -> bool:
-    return spec.startswith("./") or spec.startswith("../") or spec.startswith("/")
-
-
-def resolve_import(source_rel: str, spec: str, work_dir: pathlib.Path):
-    """Resolve a relative import. Returns a repo-root-relative path or None."""
-    if spec.startswith("/"):
-        target = spec.lstrip("/")
-    else:
-        source_dir = os.path.dirname(source_rel)
-        target = os.path.normpath(os.path.join(source_dir, spec))
-    # Normalize to forward slashes
-    target = target.replace(os.sep, "/")
-    # Reject paths that escape work_dir (e.g. `../../outside-repo`)
-    if target.startswith("../") or target.startswith("/"):
-        return None
-    # Exact file
-    if (work_dir / target).is_file():
-        return target
-    # Suffix candidates
-    for ext in RESOLUTION_EXTENSIONS:
-        cand = f"{target}{ext}"
-        if (work_dir / cand).is_file():
-            return cand
-    # /index.* in directory
-    for ext in INDEX_EXTENSIONS:
-        cand = f"{target}/index{ext}"
-        if (work_dir / cand).is_file():
-            return cand
-    return None
-
-
-def read_imports(file_path: pathlib.Path):
-    try:
-        content = file_path.read_text(encoding="utf-8", errors="replace")
-    except OSError:
-        return []
-    specs = []
-    for pattern in IMPORT_PATTERNS:
-        for m in re.finditer(pattern, content, re.MULTILINE):
-            specs.append(m.group(1))
-    return specs
-
-
-def bfs_trace(seeds, work_dir: pathlib.Path):
-    """BFS following static imports. Returns dict: path → (depth, via)."""
-    reachable = {s: (0, None) for s in seeds}
-    queue = [(s, 0) for s in seeds]
-    while queue:
-        current, depth = queue.pop(0)
-        if not any(current.endswith(ext) for ext in TRACEABLE_EXTENSIONS):
-            continue
-        full = work_dir / current
-        if not full.is_file():
-            continue
-        for spec in read_imports(full):
-            if not is_relative(spec):
-                continue
-            resolved = resolve_import(current, spec, work_dir)
-            if resolved is None or resolved in reachable:
-                continue
-            if "node_modules" in resolved.split("/"):
-                continue
-            reachable[resolved] = (depth + 1, current)
-            queue.append((resolved, depth + 1))
-    return reachable
-
-
-def git_touched_files(scaffold_sha: str, work_dir: pathlib.Path):
-    """Arm-touched files (relative paths), excluding deletions."""
-    r = subprocess.run(
-        ["git", "diff", "--name-status", "-M", scaffold_sha],
-        cwd=str(work_dir), capture_output=True, text=True,
-    )
-    touched = []
-    for line in r.stdout.splitlines():
-        parts = line.split("\t")
-        if len(parts) < 2:
-            continue
-        status = parts[0]
-        if status == "D":
-            continue
-        path = parts[-1]
-        touched.append(path)
-    return touched
-
-
-def match_any(path: str, patterns) -> bool:
-    return any(fnmatch.fnmatch(path, p) for p in patterns)
-
-
-def analyze(work_dir_str: str, scaffold_sha: str, tier_c_globs, waivers,
-            fixture_id=None):
-    work_dir = pathlib.Path(work_dir_str).resolve()
-    touched = git_touched_files(scaffold_sha, work_dir)
-
-    # Seeds = arm-touched files matching spec_output_files globs.
-    seeds = sorted(p for p in touched if match_any(p, tier_c_globs))
-
-    reachable = bfs_trace(seeds, work_dir)
-
-    findings = []
-    for path in sorted(touched):
-        if match_any(path, tier_c_globs):
-            continue
-        if match_any(path, waivers):
-            continue
-        if path in reachable:
-            depth, via = reachable[path]
-            findings.append({
-                "file": path,
-                "type": "tier-b-reachable",
-                "severity": "info",
-                "reachable_via": via,
-                "depth": depth,
-                "verdict": "Reachable from Tier C via import chain",
-            })
-        else:
-            findings.append({
-                "file": path,
-                "type": "scope-unmatched",
-                "severity": "warn",
-                "verdict": "Not in Tier C, not reachable from Tier C via static imports",
-            })
-
-    return seeds, findings
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--work")
-    ap.add_argument("--scaffold")
-    ap.add_argument("--expected",
-                    help="Path to fixture expected.json")
-    ap.add_argument(
-        "--list-categories",
-        action="store_true",
-        help="Emit the stable oracle CATEGORIES enum as JSON and exit (iter-0022).",
-    )
-    args = ap.parse_args()
-
-    if args.list_categories:
-        print(json.dumps({"oracle": ORACLE_NAME, "categories": CATEGORIES}, indent=2, sort_keys=True))
-        return
-
-    if not args.work or not args.scaffold or not args.expected:
-        ap.error("--work, --scaffold, and --expected are required unless --list-categories is set")
-
-    try:
-        expected = loads_strict_json_object(pathlib.Path(args.expected).read_text())
-    except (OSError, json.JSONDecodeError, ValueError) as e:
-        sys.stderr.write(f"[oracle-scope-tier-b] cannot read expected: {e}\n")
-        print(json.dumps({
-            "oracle": "scope-tier-b",
-            "trace_method": TRACE_METHOD,
-            "tier_c_seeds_matched": [],
-            "findings": [],
-            "error": f"expected.json unreadable: {e}",
-        }, indent=2))
-        return
-
-    tier_c = expected.get("spec_output_files", [])
-    waivers = expected.get("tier_a_waivers", [])
-    # fixture_id = parent directory name of expected.json
-    fixture_id = pathlib.Path(args.expected).parent.name
-
-    if not isinstance(tier_c, list) or not all(isinstance(item, str) for item in tier_c):
-        print(json.dumps({
-            "oracle": "scope-tier-b",
-            "trace_method": TRACE_METHOD,
-            "tier_c_seeds_matched": [],
-            "fixture_id": fixture_id,
-            "findings": [],
-            "error": "expected.json malformed: spec_output_files must be a string array",
-        }, indent=2))
-        return
-    if not isinstance(waivers, list) or not all(isinstance(item, str) for item in waivers):
-        print(json.dumps({
-            "oracle": "scope-tier-b",
-            "trace_method": TRACE_METHOD,
-            "tier_c_seeds_matched": [],
-            "fixture_id": fixture_id,
-            "findings": [],
-            "error": "expected.json malformed: tier_a_waivers must be a string array",
-        }, indent=2))
-        return
-
-    if not tier_c:
-        print(json.dumps({
-            "oracle": "scope-tier-b",
-            "trace_method": TRACE_METHOD,
-            "tier_c_seeds_matched": [],
-            "fixture_id": fixture_id,
-            "findings": [],
-            "error": "no spec_output_files in expected.json",
-        }, indent=2))
-        return
-
-    seeds, findings = analyze(args.work, args.scaffold, tier_c, waivers,
-                              fixture_id=fixture_id)
-    print(json.dumps({
-        "oracle": "scope-tier-b",
-        "trace_method": TRACE_METHOD,
-        "tier_c_seeds_matched": seeds,
-        "fixture_id": fixture_id,
-        "findings": findings,
-    }, indent=2))
-
-
-if __name__ == "__main__":
-    main()