@sitar_fiercer4c/skills 0.1.0

package/skills/backend-code-review/scripts/sweep.py

@@ -0,0 +1,550 @@
+#!/usr/bin/env python3
+"""Whole-tree sweeps for findings that only exist in aggregate.
+
+Reading a codebase depth-first — route to controller to service to util — is how
+you find what a single file does wrong. It is structurally blind to a second
+class of finding that exists only across files: an env var nothing consumes, a
+helper defined identically in two modules, an export with no importer, a block
+of code pasted in three places. Nobody sees those by reading carefully, because
+each individual file looks fine. You see them by counting.
+
+That is what this does. Run it before writing, and treat every row as a lead to
+confirm by opening the file — the parsing is regex-based and will mis-handle
+dynamic access, re-exports, and string-built identifiers.
+
+Usage:
+    python3 sweep.py <src-dir> [--repo-root DIR] [--json] [--section NAME]
+
+Sections: env, exports, dupes, blocks, request-inputs
+"""
+
+import argparse
+import json
+import re
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+SKIP_DIRS = {"node_modules", ".git", "dist", "build", "coverage", ".next", "__pycache__"}
+
+ENV_READ = re.compile(r"process\.env\.([A-Za-z_][A-Za-z0-9_]*)")
+ENV_READ_BRACKET = re.compile(r"process\.env\[\s*['\"]([A-Za-z_][A-Za-z0-9_]*)['\"]\s*\]")
+ENV_DECL = re.compile(r"^\s*(?:#\s*)?([A-Z_][A-Z0-9_]*)\s*=")
+
+EXPORT_NAMED = re.compile(
+    r"^export\s+(?:async\s+)?(?:function|const|let|var|class)\s+([A-Za-z_$][\w$]*)"
+)
+EXPORT_BLOCK = re.compile(r"^export\s*\{([^}]*)\}", re.MULTILINE)
+EXPORT_DEFAULT = re.compile(r"^export\s+default\b")
+
+TOPLEVEL_DEF = re.compile(
+    r"^(?:export\s+)?(?:async\s+)?(?:function|const|let|var|class)\s+([A-Za-z_$][\w$]*)"
+)
+
+IMPORT_FROM = re.compile(r"""import\s+([^;]*?)\s+from\s+['"]([^'"]+)['"]""", re.S)
+REQUIRE_DESTRUCTURE = re.compile(
+    r"""(?:const|let|var)\s*\{([^}]*)\}\s*=\s*require\(\s*['"]([^'"]+)['"]\s*\)""", re.S
+)
+
+SOURCE_SUFFIXES = (".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts")
+
+REQ_CONTAINERS = ("query", "body", "headers", "params")
+# `req.query.t` / `req.query['t']` — a key read straight off the request.
+REQ_DOT = re.compile(
+    r"\breq(?:uest)?\.(" + "|".join(REQ_CONTAINERS) + r")\.([A-Za-z_$][\w$]*)"
+)
+REQ_BRACKET = re.compile(
+    r"\breq(?:uest)?\.(" + "|".join(REQ_CONTAINERS) + r")\[\s*['\"]([A-Za-z_$][\w$]*)['\"]\s*\]"
+)
+# `const { code, state } = req.query`
+REQ_DESTRUCTURE = re.compile(
+    r"\{([^}]*)\}\s*=\s*req(?:uest)?\.(" + "|".join(REQ_CONTAINERS) + r")\b"
+)
+# `const { code, state } = query` — the same keys one layer down, after the whole
+# container was handed to a service. Name-based, so it is a lead, not proof.
+PROXY_DESTRUCTURE = re.compile(
+    r"\{([^}]*)\}\s*=\s*([A-Za-z_$][\w$]*(?:[Qq]uer(?:y|ies)|[Bb]ody|[Pp]arams?|[Hh]eaders?))\b"
+)
+# `req.query` used whole: passed as an argument, or spread into an object.
+REQ_WHOLE = re.compile(r"\breq(?:uest)?\.(" + "|".join(REQ_CONTAINERS) + r")\b(?!\s*[.\[])")
+# `emailService.handleOutlookCallback(req.query)` — the container crosses into a
+# function, and the keys are named in that function's parameter list, not here.
+REQ_PASSED_TO = re.compile(
+    r"([A-Za-z_$][\w$]*)\s*\(\s*req(?:uest)?\.(" + "|".join(REQ_CONTAINERS) + r")\s*[,)]"
+)
+# `const handleOutlookCallback = async ({ state, code, error }) => {`
+FUNC_DESTRUCT_ARROW = re.compile(
+    r"(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s+)?(?:function\s*)?\(\s*\{([^}]*)\}"
+)
+FUNC_DESTRUCT_DECL = re.compile(
+    r"(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(\s*\{([^}]*)\}"
+)
+MASK_LIST = re.compile(
+    r"(?:const|let|var)\s+([A-Za-z_$][\w$]*(?:SENSITIVE|REDACT|MASK|Sensitive|Redact|Mask)[\w$]*"
+    r"|[A-Za-z_$]*(?:SENSITIVE|REDACT|MASK)[\w$]*)\s*=\s*(?:new\s+Set\s*\(\s*)?\[([^\]]*)\]",
+    re.S,
+)
+
+
+def resolve_spec(spec: str, importer: Path):
+    """Resolve a relative import specifier to the file it actually names.
+
+    Matching on the specifier's stem alone conflates `utils/round.js` with
+    `services/round.js`, and that conflation is exactly what made the export
+    sweep report live-but-unimported identifiers as used.
+    """
+    if not spec.startswith("."):
+        return None
+    base = importer.parent / spec
+    candidates = [base]
+    for suffix in SOURCE_SUFFIXES:
+        candidates.append(Path(str(base) + suffix))
+        candidates.append(base / ("index" + suffix))
+    for c in candidates:
+        try:
+            if c.is_file():
+                return c.resolve()
+        except OSError:
+            continue
+    return None
+
+
+def import_bindings(clause: str):
+    """Split an import clause into (named, namespace?, default-name).
+
+    `import { a as b }` imports the export named `a`, so the name before `as`
+    is the one that matters here — the mirror of an export block, where the
+    name after `as` is what leaves the module.
+    """
+    named = set()
+    m = re.search(r"\{([^}]*)\}", clause)
+    if m:
+        for piece in m.group(1).split(","):
+            piece = piece.strip()
+            if not piece:
+                continue
+            orig = re.split(r"\s+as\s+", piece)[0].strip()
+            if re.fullmatch(r"[A-Za-z_$][\w$]*", orig):
+                named.add(orig)
+    namespace = bool(re.search(r"\*\s+as\s+[A-Za-z_$][\w$]*", clause))
+    head = clause.split("{")[0].strip().rstrip(",").strip()
+    default = head if head and not head.startswith("*") else None
+    return named, namespace, default
+
+
+def split_names(blob: str):
+    out = set()
+    for piece in blob.split(","):
+        piece = piece.strip()
+        if not piece or piece.startswith("..."):
+            continue
+        # `{ code: authCode }` and `{ code = '' }` both still read the key `code`.
+        key = re.split(r"[:=]", piece)[0].strip()
+        if re.fullmatch(r"[A-Za-z_$][\w$]*", key):
+            out.add(key)
+    return out
+
+
+def source_files(root: Path):
+    for p in sorted(root.rglob("*")):
+        if p.suffix not in SOURCE_SUFFIXES:
+            continue
+        if any(part in SKIP_DIRS for part in p.parts):
+            continue
+        yield p
+
+
+def strip_comments(text: str) -> str:
+    # Replace block comments with their own newlines rather than deleting them, so
+    # every line keeps its original number. The line:col this script prints ends up
+    # cited verbatim in the review, and a citation that lands two lines off the thing
+    # it claims to show is worse than no citation.
+    text = re.sub(r"/\*.*?\*/", lambda m: "\n" * m.group(0).count("\n"), text, flags=re.S)
+    # `\s` would match newlines, so a blank line before a comment gets swallowed with it.
+    text = re.sub(r"^[^\S\n]*//.*$", "", text, flags=re.M)
+    return text
+
+
+def rel(p: Path, root: Path) -> str:
+    try:
+        return str(p.relative_to(root))
+    except ValueError:
+        return str(p)
+
+
+# ---------------------------------------------------------------- env
+
+def sweep_env(src: Path, repo_root: Path):
+    reads = defaultdict(list)
+    for p in source_files(src):
+        body = p.read_text(encoding="utf-8", errors="replace")
+        for i, line in enumerate(body.splitlines(), 1):
+            if line.lstrip().startswith("//"):
+                continue
+            for m in list(ENV_READ.finditer(line)) + list(ENV_READ_BRACKET.finditer(line)):
+                reads[m.group(1)].append(f"{rel(p, repo_root)}:{i}")
+
+    declared = {}
+    for name in (".env", ".env.example", ".env.sample", ".env.local"):
+        f = repo_root / name
+        if not f.exists():
+            continue
+        for i, line in enumerate(f.read_text(encoding="utf-8", errors="replace").splitlines(), 1):
+            m = ENV_DECL.match(line)
+            if m:
+                declared.setdefault(m.group(1), []).append(f"{name}:{i}")
+
+    all_names = sorted(set(reads) | set(declared))
+    rows = []
+    for n in all_names:
+        sites = reads.get(n, [])
+        outside = [s for s in sites if "/config/" not in s.replace("\\", "/")]
+        rows.append({
+            "name": n,
+            "read_sites": sites,
+            "declared_in": declared.get(n, []),
+            "read_count": len(sites),
+            "reads_outside_config": outside,
+            "declared_never_read": bool(declared.get(n)) and not sites,
+            "read_never_declared": bool(sites) and not declared.get(n),
+        })
+    return rows
+
+
+# ---------------------------------------------------------------- exports
+
+def sweep_exports(src: Path, repo_root: Path, extra_roots):
+    files = list(source_files(src))
+    for r in extra_roots:
+        if r.exists():
+            files += list(source_files(r))
+
+    bodies = {p: strip_comments(p.read_text(encoding="utf-8", errors="replace")) for p in files}
+
+    exports = {}
+    for p, body in bodies.items():
+        names = set()
+        for line in body.splitlines():
+            m = EXPORT_NAMED.match(line)
+            if m:
+                names.add(m.group(1))
+        for m in EXPORT_BLOCK.finditer(body):
+            for piece in m.group(1).split(","):
+                piece = piece.strip()
+                if not piece:
+                    continue
+                piece = re.split(r"\s+as\s+", piece)[-1].strip()
+                if re.fullmatch(r"[A-Za-z_$][\w$]*", piece):
+                    names.add(piece)
+        if names:
+            exports[p] = names
+        if EXPORT_DEFAULT.search(body):
+            exports.setdefault(p, set())
+
+    # Index every import once: which file it resolves to, and what it pulls out.
+    # Doing this per (file, name) pair instead meant scanning every body for every
+    # exported identifier, which is both slow and how the stem collision crept in.
+    imports = defaultdict(list)  # target path -> [(importer, named, namespace, default)]
+    for q, qbody in bodies.items():
+        for m in IMPORT_FROM.finditer(qbody):
+            clause, spec = m.group(1), m.group(2)
+            target = resolve_spec(spec, q)
+            if target is None:
+                continue
+            named, namespace, default = import_bindings(clause)
+            imports[target].append((q, named, namespace, default))
+        for m in REQUIRE_DESTRUCTURE.finditer(qbody):
+            target = resolve_spec(m.group(2), q)
+            if target is None:
+                continue
+            imports[target].append((q, split_names(m.group(1)), False, None))
+
+    rows = []
+    for p, names in exports.items():
+        body = bodies[p]
+        key = p.resolve()
+        incoming = imports.get(key, [])
+
+        if EXPORT_DEFAULT.search(body):
+            default_importers = sorted({
+                rel(q, repo_root) for q, _, ns, default in incoming
+                if q != p and (default or ns)
+            })
+            rows.append({
+                "file": rel(p, repo_root),
+                "name": "(default export)",
+                "in_file_refs": None,
+                "external_importers": default_importers,
+                "same_name_elsewhere": [],
+            })
+
+        for n in sorted(names):
+            in_file = len(re.findall(rf"\b{re.escape(n)}\b", body)) - 1
+            external = sorted({
+                rel(q, repo_root) for q, named, ns, _ in incoming
+                if q != p and (n in named or ns)
+            })
+            # The old bare-name signal, kept but demoted. A name occurring in a file
+            # that never imports this module is usually a second definition of the
+            # same word, not a caller — worth seeing next to DUPES, not counted as use.
+            elsewhere = sorted({
+                rel(q, repo_root) for q, qbody in bodies.items()
+                if q != p and rel(q, repo_root) not in external
+                and re.search(rf"\b{re.escape(n)}\b", qbody)
+            })
+            rows.append({
+                "file": rel(p, repo_root),
+                "name": n,
+                "in_file_refs": max(in_file, 0),
+                "external_importers": external,
+                "same_name_elsewhere": elsewhere,
+            })
+    return rows
+
+
+# ---------------------------------------------------------------- request inputs
+
+def sweep_request_inputs(src: Path, repo_root: Path):
+    """Every key read off the request, against whatever the codebase masks.
+
+    A request logger that dumps a whole container leaks whichever keys happen to
+    ride in it, and the keys are spread across controllers and the services they
+    hand the container to — so no single grep finds them all. Counting them is the
+    only way the list comes out complete.
+    """
+    keys = defaultdict(list)      # key -> ["file:line (container)"]
+    whole = []                    # sites handing a container on intact
+    masked = {}                   # lowercased key -> declaring site
+    passed_to = {}                # callee name -> "file:line (container)"
+    destructuring = {}            # function name -> (keys, "file:line")
+
+    for p in source_files(src):
+        raw = p.read_text(encoding="utf-8", errors="replace")
+        body = strip_comments(raw)
+        for m in MASK_LIST.finditer(body):
+            line = body[: m.start()].count("\n") + 1
+            for name in split_names(re.sub(r"['\"]", "", m.group(2))):
+                masked.setdefault(name.lower(), f"{rel(p, repo_root)}:{line} ({m.group(1)})")
+
+        for i, line in enumerate(body.splitlines(), 1):
+            site = f"{rel(p, repo_root)}:{i}"
+            for m in list(REQ_DOT.finditer(line)) + list(REQ_BRACKET.finditer(line)):
+                keys[m.group(2)].append(f"{site} (req.{m.group(1)})")
+            for m in REQ_DESTRUCTURE.finditer(line):
+                for k in split_names(m.group(1)):
+                    keys[k].append(f"{site} (req.{m.group(2)} destructure)")
+            for m in PROXY_DESTRUCTURE.finditer(line):
+                for k in split_names(m.group(1)):
+                    keys[k].append(f"{site} ({m.group(2)} destructure)")
+            for m in REQ_WHOLE.finditer(line):
+                whole.append(f"{site} (req.{m.group(1)})")
+            for m in REQ_PASSED_TO.finditer(line):
+                passed_to.setdefault(m.group(1), f"{site} (req.{m.group(2)})")
+
+        for pattern in (FUNC_DESTRUCT_ARROW, FUNC_DESTRUCT_DECL):
+            for m in pattern.finditer(body):
+                line = body[: m.start()].count("\n") + 1
+                destructuring.setdefault(
+                    m.group(1), (split_names(m.group(2)), f"{rel(p, repo_root)}:{line}")
+                )
+
+    # One hop: a container handed to a function whose parameter list names its keys.
+    # Without this the OAuth callback keys are invisible — the controller forwards
+    # req.query whole and only the service ever writes `code` or `state` down.
+    for callee, call_site in passed_to.items():
+        if callee not in destructuring:
+            continue
+        param_keys, def_site = destructuring[callee]
+        for k in param_keys:
+            keys[k].append(f"{def_site} (via {callee}() param, called at {call_site})")
+
+    rows = [
+        {
+            "key": k,
+            "sites": sorted(set(v)),
+            "masked": k.lower() in masked,
+            "masked_by": masked.get(k.lower()),
+        }
+        for k, v in sorted(keys.items())
+    ]
+    return {
+        "keys": rows,
+        "whole_container_sites": sorted(set(whole)),
+        "mask_lists": sorted(set(masked.values())),
+    }
+
+
+# ---------------------------------------------------------------- duplicate definitions
+
+def sweep_dupes(src: Path, repo_root: Path):
+    defs = defaultdict(list)
+    for p in source_files(src):
+        body = strip_comments(p.read_text(encoding="utf-8", errors="replace"))
+        for i, line in enumerate(body.splitlines(), 1):
+            m = TOPLEVEL_DEF.match(line)
+            if m:
+                defs[m.group(1)].append(f"{rel(p, repo_root)}:{i}")
+    return [
+        {"name": n, "sites": s}
+        for n, s in sorted(defs.items())
+        if len({x.split(":")[0] for x in s}) > 1
+    ]
+
+
+# ---------------------------------------------------------------- repeated blocks
+
+def norm(line: str) -> str:
+    return re.sub(r"\s+", " ", line.strip())
+
+
+def sweep_blocks(src: Path, repo_root: Path, window: int, min_len: int):
+    seen = defaultdict(list)
+    for p in source_files(src):
+        raw = strip_comments(p.read_text(encoding="utf-8", errors="replace")).splitlines()
+        lines = [(i + 1, norm(l)) for i, l in enumerate(raw)]
+        lines = [(i, l) for i, l in lines if len(l) >= min_len]
+        for j in range(len(lines) - window + 1):
+            chunk = lines[j:j + window]
+            if chunk[-1][0] - chunk[0][0] > window * 3:
+                continue
+            key = "\n".join(c[1] for c in chunk)
+            seen[key].append(f"{rel(p, repo_root)}:{chunk[0][0]}-{chunk[-1][0]}")
+
+    out = []
+    for key, sites in seen.items():
+        uniq = sorted(set(sites))
+        if len(uniq) > 1:
+            out.append({"sites": uniq, "lines": key.split("\n")})
+    out.sort(key=lambda r: (-len(r["sites"]), r["sites"][0]))
+    return out[:40]
+
+
+# ---------------------------------------------------------------- render
+
+def render(res):
+    o = []
+    if "env" in res:
+        o.append("=== ENV VARS ===")
+        o.append("Every process.env read, against what the .env files declare.\n")
+        for r in res["env"]:
+            flags = []
+            if r["declared_never_read"]:
+                flags.append("DECLARED-NEVER-READ")
+            if r["read_never_declared"]:
+                flags.append("READ-NEVER-DECLARED")
+            if r["reads_outside_config"]:
+                flags.append(f"{len(r['reads_outside_config'])}-READS-OUTSIDE-CONFIG")
+            o.append(f"{r['name']}  [{' '.join(flags) or 'ok'}]")
+            if r["declared_in"]:
+                o.append(f"    declared: {', '.join(r['declared_in'])}")
+            for s in r["read_sites"]:
+                o.append(f"    read:     {s}")
+            o.append("")
+
+    if "exports" in res:
+        o.append("\n=== EXPORTS ===")
+        o.append("in-file refs vs external importers. An export with 0 external importers")
+        o.append("but >0 in-file refs is over-exported, NOT dead — narrowing the export list")
+        o.append("is safe, deleting the definition is not.")
+        o.append(f"Scanned {res.get('_files_scanned', '?')} files for importers. A DEAD verdict is")
+        o.append("only as good as that number: if a whole file type went unscanned, its importers")
+        o.append("are invisible and live exports will read as dead. Check the count looks right")
+        o.append("for the tree before believing any DEAD row.\n")
+        o.append("An importer is a file whose import specifier resolves to this module and")
+        o.append("which names this identifier. A file that merely contains the same word is")
+        o.append("listed as same-name-elsewhere — usually a second definition, not a caller.\n")
+        for r in res["exports"]:
+            ext = r["external_importers"]
+            n = r["in_file_refs"]
+            if ext:
+                tag = f"used by {len(ext)}"
+            elif n:
+                tag = f"OVER-EXPORTED (0 external, {n} in-file)"
+            else:
+                tag = "DEAD (0 external, 0 in-file)"
+            o.append(f"{r['file']}  {r['name']}  [{tag}]")
+            if ext:
+                o.append(f"    importers: {', '.join(ext)}")
+            if r.get("same_name_elsewhere"):
+                o.append(f"    same-name-elsewhere: {', '.join(r['same_name_elsewhere'])}")
+        o.append("")
+
+    if "request-inputs" in res:
+        ri = res["request-inputs"]
+        o.append("\n=== REQUEST INPUT KEYS ===")
+        o.append("Every key read off req.query/body/headers/params, plus keys destructured")
+        o.append("from a same-named parameter one layer down, against the codebase's own mask")
+        o.append("list. UNMASKED matters only where a container is logged or forwarded whole —")
+        o.append("those sites are listed underneath.\n")
+        if ri["mask_lists"]:
+            o.append(f"Mask lists found: {', '.join(ri['mask_lists'])}\n")
+        else:
+            o.append("No mask list found — nothing is being redacted.\n")
+        for r in ri["keys"]:
+            tag = "MASKED" if r["masked"] else "UNMASKED"
+            o.append(f"{r['key']}  [{tag}]")
+            for s in r["sites"]:
+                o.append(f"    {s}")
+        o.append("")
+        o.append(f"Containers passed or logged whole ({len(ri['whole_container_sites'])} sites):")
+        for s in ri["whole_container_sites"]:
+            o.append(f"    {s}")
+        o.append("")
+
+    if "dupes" in res:
+        o.append("\n=== DUPLICATE TOP-LEVEL DEFINITIONS ===")
+        o.append("Same identifier defined in more than one file.\n")
+        for r in res["dupes"]:
+            o.append(f"{r['name']}")
+            for s in r["sites"]:
+                o.append(f"    {s}")
+        o.append("")
+
+    if "blocks" in res:
+        o.append("\n=== REPEATED CODE BLOCKS ===")
+        o.append("Normalised line windows appearing verbatim in more than one place.\n")
+        for r in res["blocks"]:
+            o.append(f"x{len(r['sites'])}  {', '.join(r['sites'])}")
+            for l in r["lines"][:4]:
+                o.append(f"    | {l[:100]}")
+            o.append("")
+    return "\n".join(o)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("src")
+    ap.add_argument("--repo-root", default=None)
+    ap.add_argument("--json", action="store_true")
+    ap.add_argument("--section", action="append",
+                    choices=["env", "exports", "dupes", "blocks", "request-inputs"])
+    ap.add_argument("--window", type=int, default=4)
+    ap.add_argument("--min-line-len", type=int, default=12)
+    a = ap.parse_args()
+
+    src = Path(a.src).resolve()
+    if not src.is_dir():
+        sys.exit(f"not a directory: {src}")
+    repo_root = Path(a.repo_root).resolve() if a.repo_root else src.parent
+
+    want = a.section or ["env", "exports", "dupes", "blocks", "request-inputs"]
+    res = {}
+    if "env" in want:
+        res["env"] = sweep_env(src, repo_root)
+    if "request-inputs" in want:
+        res["request-inputs"] = sweep_request_inputs(src, repo_root)
+    if "exports" in want:
+        extra = [repo_root / "scripts", repo_root / "tests"]
+        res["exports"] = sweep_exports(src, repo_root, extra)
+        res["_files_scanned"] = len(list(source_files(src))) + sum(
+            len(list(source_files(r))) for r in extra if r.exists())
+    if "dupes" in want:
+        res["dupes"] = sweep_dupes(src, repo_root)
+    if "blocks" in want:
+        res["blocks"] = sweep_blocks(src, repo_root, a.window, a.min_line_len)
+
+    print(json.dumps(res, indent=2) if a.json else render(res))
+
+
+if __name__ == "__main__":
+    main()