@allansantos-dev/smart-tool 0.9.2 → 0.9.3

package/code_impact.py

@@ -0,0 +1,132 @@
+"""What changing a function touches, from the indexed code graph: where it is defined, who calls it, what it calls,
+which tests reach it through static calls and which files import its module, each with the first line of its
+docstring. Answers the question an agent asks before an edit without a chain of searches and file reads."""
+import os
+from collections import defaultdict
+
+import code_graph
+import doc_check
+import index_profile
+import index_scope
+
+MAX_DEPTH = 4
+NOTE = ("Static analysis of the indexed snapshot: calls made through callbacks, dynamic dispatch or names built at "
+        "runtime are not seen, so treat an empty list as 'none found', not as 'none exist'.")
+
+
+def _matches(symbols, name):
+    path, _sep, wanted = name.strip().rpartition("::")
+    path = path.replace("\\", "/").removeprefix("./")
+    pool = [s for s in symbols if s["path"] == path] if path else symbols
+    exact = [s for s in pool if s["id"] == wanted or s["name"] == wanted]
+    return exact or [s for s in pool if s["name"].split(".")[-1] == wanted]
+
+
+def _site(by_id, symbol_id, line, of=None):
+    symbol = by_id.get(symbol_id) or {}
+    site = {"id": symbol_id, "function": symbol.get("name", symbol_id), "path": symbol.get("path"), "line": line}
+    if of:
+        site["of"] = of
+    return site
+
+
+def summaries(root, symbols, wanted_ids):
+    """First docstring line of the wanted symbols, read from the working tree once per file."""
+    by_path = defaultdict(list)
+    for symbol in symbols:
+        by_path[symbol["path"]].append(symbol)
+    docs = {}
+    for path in {s["path"] for s in symbols if s["id"] in wanted_ids}:
+        try:
+            with open(os.path.join(root, path), encoding="utf-8") as stream:
+                source = stream.read()
+        except (OSError, UnicodeDecodeError):
+            continue
+        for function in doc_check.functions(path, source, by_path[path]):
+            if function["doc"]:
+                docs[(path, function["start"])] = function["doc"]
+    return {s["id"]: docs[(s["path"], s["start_line"])] for s in symbols
+            if s["id"] in wanted_ids and (s["path"], s["start_line"]) in docs}
+
+
+def impact(root, symbol, view_id=None, depth=3, limit=30):
+    if not isinstance(symbol, str) or not symbol.strip():
+        raise ValueError("Pass symbol: a function, method (Class.method) or class name.")
+    if type(depth) is not int or not 1 <= depth <= MAX_DEPTH:
+        raise ValueError(f"depth must be an integer from 1 to {MAX_DEPTH}.")
+    data = code_graph.build(root, view_id)
+    found = _matches(data.get("symbols") or [], symbol)
+    diagnostics = [d.get("reason") for d in data.get("diagnostics") or [] if d.get("reason")]
+    if not found:
+        cause = f" Analysis problems: {'; '.join(diagnostics)}" if diagnostics else ""
+        raise ValueError(f"No function or class named {symbol!r} in the indexed view; check the name or reindex.{cause}")
+    profile = index_profile.current((index_scope.load_scope(root) or {}).get("profile"))
+    by_id = {s["id"]: s for s in data["symbols"]}
+    callers, callees = defaultdict(list), defaultdict(list)
+    for call in data["calls"]:
+        callers[call["target"]].append(call)
+        callees[call["source"]].append(call)
+    targets = {s["id"] for s in found}
+    label = {s["id"]: f"{s['path']}::{s['name']}" for s in found} if len(found) > 1 else {}
+    tests = {}
+    for origin in targets:
+        seen, frontier = set(targets), {origin}
+        for hops in range(1, depth + 1):
+            reached = set()
+            for target in frontier:
+                for call in callers[target]:
+                    source = call["source"]
+                    if source in seen:
+                        continue
+                    seen.add(source)
+                    reached.add(source)
+                    caller = by_id.get(source)
+                    if caller and index_profile.kind(caller["path"], profile) == "test" and source not in tests:
+                        tests[source] = {**_site(by_id, source, caller["start_line"], label.get(origin)), "hops": hops}
+            frontier = reached
+    direct = [_site(by_id, c["source"], c["line"], label.get(t)) for t in targets for c in callers[t]
+              if c["source"] not in targets]
+    outgoing = [_site(by_id, c["target"], c["line"], label.get(t)) for t in targets for c in callees[t]
+                if c["target"] not in targets]
+    files = {s["path"] for s in found}
+    importers = sorted({d["source"] for d in data["dependencies"] if d.get("target") in files and d["source"] not in files})
+    ordered_tests = sorted(tests.values(), key=lambda t: (t["hops"], t["path"], t["line"]))
+    shown = direct[:limit] + outgoing[:limit] + ordered_tests[:limit]
+    docs = summaries(root, data["symbols"], targets | {site["id"] for site in shown})
+    for site in shown:
+        doc = docs.get(site.pop("id"))
+        if doc:
+            site["doc"] = doc
+    definitions = []
+    for s in found:
+        entry = {"name": s["name"], "kind": s.get("kind"), "path": s["path"], "lines": f"{s['start_line']}-{s['end_line']}"}
+        if docs.get(s["id"]):
+            entry["doc"] = docs[s["id"]]
+        definitions.append(entry)
+    return {
+        "symbol": symbol.strip(),
+        "definitions": definitions,
+        "callers": direct[:limit],
+        "calls": outgoing[:limit],
+        "tests": ordered_tests[:limit],
+        "imported_by": importers[:limit],
+        "counts": {"callers": len(direct), "calls": len(outgoing), "tests": len(ordered_tests),
+                   "imported_by": len(importers)},
+        "test_depth": depth,
+        "diagnostics": diagnostics,
+        "view": (data.get("selected") or {}).get("label"),
+        "note": NOTE + (f" {len(found)} definitions share this name; each entry says which one ('of'); pass "
+                        "path::name to keep one." if label else ""),
+    }
+
+
+def for_agent(data, limit):
+    """The file-focused graph without what an agent never reads (view list, display limits) and with lists capped."""
+    for key in ("symbols", "dependencies", "calls", "unresolved", "diagnostics"):
+        items = data.get(key) or []
+        if len(items) > limit:
+            data[key] = items[:limit]
+            data.setdefault("omitted", {})[key] = len(items) - limit
+    for key in ("views", "limits", "languages", "current_view", "generated_at", "cache_hit", "read_only", "source"):
+        data.pop(key, None)
+    return data

package/config.py

@@ -31,6 +31,8 @@ DEFAULT_CONFIG = {
     "site_memory_similarity_threshold": 0.35,
     "local_models": "fallback",
     "hook_mode": "redirect",
+    "doc_mode": "remind",
+    "duplicate_mode": "warn",
     "contact": "",
     "model_adapter": "",
     "model_adapter_options": {},
@@ -42,6 +44,26 @@ LOCAL_MODEL_MODES = ("fallback", "off", "prefer")
 # PreToolUse hook: redirect = deny the native tool and point to Smart Tool; advise = let it run and add the tip to the
 # agent's context; off = no routing at all.
 HOOK_MODES = ("redirect", "advise", "off")
+# Edit hook on functions without a docstring: require = deny the edit until it adds one; remind = let it run and tell
+# the agent; off = say nothing. Independent of hook_mode, which only routes searches and reads.
+DOC_MODES = ("require", "remind", "off")
+# Edit hook on functions that copy one already indexed (exact or near-identical body): warn = let the edit run and
+# point to the existing function; off = say nothing. Never blocks: near matches are leads, not defects.
+DUPLICATE_MODES = ("warn", "off")
+
+
+def doc_mode(cfg=None):
+    value = (cfg if cfg is not None else load_config()).get("doc_mode") or DEFAULT_CONFIG["doc_mode"]
+    if value not in DOC_MODES:
+        raise ValueError(f"Invalid doc_mode ({value!r}) in {CONFIG_PATH}: use require, remind or off.")
+    return value
+
+
+def duplicate_mode(cfg=None):
+    value = (cfg if cfg is not None else load_config()).get("duplicate_mode") or DEFAULT_CONFIG["duplicate_mode"]
+    if value not in DUPLICATE_MODES:
+        raise ValueError(f"Invalid duplicate_mode ({value!r}) in {CONFIG_PATH}: use warn or off.")
+    return value
 
 
 def hook_mode(cfg=None):

package/doc_check.py

@@ -0,0 +1,256 @@
+"""Function docstrings: which functions a file has, whether each one is documented, and which ones an edit touched.
+Python is read with ast; TypeScript/JavaScript and Java with the code graph analyzers, where the documentation is the
+/** */ block right above the function. Used by the edit hook (missing or possibly stale docstrings), the docstring
+coverage listing and the one-line summaries in graph and impact results."""
+import ast
+import difflib
+import hashlib
+import os
+import re
+import threading
+from collections import Counter, OrderedDict, defaultdict
+
+import code_graph
+import index_profile
+import index_scope
+
+PYTHON = (".py",)
+SCRIPT = (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts")
+JAVA = (".java",)
+SUPPORTED = PYTHON + SCRIPT + JAVA
+MAX_PARSED = 64
+_SPACE = re.compile(r"\s+")
+_PARSED = OrderedDict()
+_PARSED_LOCK = threading.Lock()
+
+
+def supported(path):
+    return str(path).lower().endswith(SUPPORTED)
+
+
+def _first_line(text):
+    return next((line.strip(" *") for line in text.strip().splitlines() if line.strip(" *")), "")
+
+
+def _public_name(name):
+    return not name.startswith("_") or name.startswith("__") and name.endswith("__")
+
+
+def _python_functions(source):
+    """Python functions with "required" following pydocstyle publicity (public name, public parents, not nested in a
+    function) and PEP 698: an @override method inherits its documentation."""
+    tree = ast.parse(source)
+    found = []
+
+    def visit(node, owner, public, nested):
+        for child in ast.iter_child_nodes(node):
+            if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                doc = ast.get_docstring(child)
+                returns = f" -> {ast.unparse(child.returns)}" if child.returns else ""
+                found.append({"name": f"{owner}{child.name}", "start": child.lineno, "end": child.end_lineno,
+                              "documented": doc is not None, "doc": _first_line(doc or ""),
+                              "signature": f"{child.name}({ast.unparse(child.args)}){returns}",
+                              "required": public and not nested and _public_name(child.name)
+                                          and not any(ast.unparse(d).split(".")[-1] == "override"
+                                                      for d in child.decorator_list)})
+                visit(child, f"{owner}{child.name}.", False, True)
+            elif isinstance(child, ast.ClassDef):
+                visit(child, f"{owner}{child.name}.", public and _public_name(child.name), nested)
+    visit(tree, "", True, False)
+    return found
+
+
+def _overload_head(lines, index, name):
+    """Line where the overload signature ending at index starts, or None when the lines above are not an overload of
+    name (a declaration ending in ';')."""
+    head = re.compile(rf"^\s*(export\s+)?(default\s+)?(declare\s+)?(public\s+|protected\s+)?(static\s+)?(async\s+)?"
+                      rf"(function\s+)?{re.escape(name)}\s*[<(]")
+    for k in range(index, max(index - 30, -1), -1):
+        line = lines[k].rstrip()
+        if k != index and (line.endswith(("*/", "}", ";")) or not line):
+            return None
+        if head.match(lines[k]):
+            return k
+    return None
+
+
+def _block_above(lines, start, name=None):
+    """Text of the /** */ block above the function starting at line start. With name (TypeScript/JavaScript), uncommented
+    overload signatures in between are skipped, as eslint-plugin-jsdoc does by default."""
+    index = start - 2
+    while index >= 0:
+        if not lines[index].strip() or lines[index].lstrip().startswith("@"):
+            index -= 1
+            continue
+        head = _overload_head(lines, index, name) if name and lines[index].rstrip().endswith(";") else None
+        if head is None:
+            break
+        index = head - 1
+    if index < 0 or not lines[index].rstrip().endswith("*/"):
+        return None
+    end = index
+    while index >= 0 and "/*" not in lines[index]:
+        index -= 1
+    if index < 0 or "/**" not in lines[index]:
+        return None
+    return "\n".join(line.strip().removeprefix("/**").removesuffix("*/").strip().removeprefix("*").strip()
+                     for line in lines[index:end + 1])
+
+
+def _annotations(lines, start, end):
+    """Annotation lines right above the function and at its first lines (the Java analyzer starts a method at its
+    annotations), and the first line that is not an annotation."""
+    index, found = start - 2, []
+    while index >= 0 and (not lines[index].strip() or lines[index].lstrip().startswith("@")):
+        found.append(lines[index].strip())
+        index -= 1
+    index = start - 1
+    while index < min(end, len(lines)) and lines[index].lstrip().startswith("@"):
+        found.append(lines[index].strip())
+        index += 1
+    return found, lines[index] if index < len(lines) else ""
+
+
+def _required(path, symbol, parent, first_line, annotations):
+    """Whether a braced-language function needs documentation by the linter defaults: Checkstyle MissingJavadocMethod
+    (no @Override, not private) and eslint-plugin-jsdoc require-jsdoc (named declarations and class methods, never
+    callbacks or functions nested in another function; private members and unexported arrow functions left out)."""
+    short = symbol["name"].split("(")[0].split(".")[-1]
+    if short in ("callback", "anonymous") or not _public_name(short) or short.startswith("#"):
+        return False
+    if re.search(r"\bprivate\b", first_line):
+        return False
+    if path.lower().endswith(JAVA):
+        return not any(a.startswith("@Override") for a in annotations)
+    if parent and parent.get("kind") in ("function", "method"):
+        return False
+    if symbol.get("kind") == "method" or parent and parent.get("kind") in ("class", "interface"):
+        return True
+    return bool(re.match(r"\s*(export\s+)?(default\s+)?(declare\s+)?(async\s+)?function\b", first_line)
+                or re.match(r"\s*export\b", first_line))
+
+
+def _braced_functions(path, source, symbols):
+    lines = source.splitlines()
+    by_id = {s.get("id"): s for s in symbols}
+    script = path.lower().endswith(SCRIPT)
+    found = []
+    for symbol in symbols:
+        if symbol.get("kind") not in ("function", "method") or "@" in symbol["name"]:
+            continue
+        start, end = symbol["start_line"], symbol["end_line"]
+        short = symbol["name"].split("(")[0].split(".")[-1]
+        block = _block_above(lines, start, short if script else None)
+        header = " ".join(lines[start - 1:min(end, start + 8)])
+        annotations, first_line = _annotations(lines, start, end)
+        required = _required(path, symbol, by_id.get(symbol.get("parent")), first_line, annotations)
+        found.append({"name": symbol["name"], "start": start, "end": end, "documented": block is not None,
+                      "doc": _first_line(block or ""), "signature": _SPACE.sub(" ", header.split("{")[0]).strip(),
+                      "required": required})
+    return found
+
+
+def functions(path, source, symbols=None):
+    """Functions of one file with start/end lines, documented flag, first doc line and normalized signature. symbols
+    are the code graph symbols of that file when already known; otherwise the analyzer runs on source."""
+    lower = path.lower()
+    if lower.endswith(PYTHON):
+        try:
+            return _python_functions(source)
+        except SyntaxError:
+            return []
+    if symbols is None:
+        symbols = analyze(path, source)[0]
+    return _braced_functions(path, source, symbols)
+
+
+def analyze(path, *sources):
+    """Analyzer symbols of several versions of one TypeScript/JavaScript/Java file, from one analyzer run for the
+    versions not seen yet (an edit hook needs the file before and after; the next edit's 'before' is this 'after')."""
+    lower, name = path.lower(), os.path.basename(path)
+    keys = [hashlib.sha1(f"{name}\0{source}".encode("utf-8")).hexdigest() for source in sources]
+    with _PARSED_LOCK:
+        missing = {key: source for key, source in zip(keys, sources) if key not in _PARSED}
+    if missing:
+        batch = {f"v{i}/{name}": source for i, source in enumerate(missing.values())}
+        analysis = (code_graph._javascript_graph(batch) if lower.endswith(SCRIPT)
+                    else code_graph._java_graph(batch) if lower.endswith(JAVA) else {})
+        found = {key: [dict(s, path=name) for s in analysis.get("symbols") or [] if s.get("path") == f"v{i}/{name}"]
+                 for i, key in enumerate(missing)}
+        with _PARSED_LOCK:
+            _PARSED.update(found)
+            while len(_PARSED) > MAX_PARSED:
+                _PARSED.popitem(last=False)
+    with _PARSED_LOCK:
+        return [_PARSED[key] if key in _PARSED else [] for key in keys]
+
+
+def _touched_lines(before, after):
+    old, new = (before or "").splitlines(), after.splitlines()
+    touched = set()
+    for tag, _i1, _i2, j1, j2 in difflib.SequenceMatcher(None, old, new, autojunk=False).get_opcodes():
+        if tag != "equal":
+            touched.update(range(j1 + 1, max(j1 + 1, j2) + 1))
+    return touched
+
+
+def written(path, before, after):
+    """Functions of the edited file after the edit ('all') and the ones the edit touched ('touched')."""
+    if path.lower().endswith(SCRIPT + JAVA):
+        analyze(path, *(source for source in (before, after) if source is not None))
+    found = functions(path, after) if supported(path) else []
+    lines = _touched_lines(before, after)
+    return {"all": found, "touched": [f for f in found if lines.intersection(range(f["start"], f["end"] + 1))]}
+
+
+def review(path, before, after, edited=None):
+    """Functions the edit from before to after touched that are left without a docstring although the language
+    convention expects one ('missing': public, not nested, not an override) or keep one while their signature changed
+    ('signature_changed'). Untouched functions and body-only edits are not reported. edited is written() of the same
+    edit when the caller already has it."""
+    if not supported(path):
+        return []
+    edited = edited or written(path, before, after)
+    if not edited["touched"]:
+        return []
+    old = {f["name"]: f for f in functions(path, before)} if before else {}
+    findings = []
+    for function in edited["touched"]:
+        previous = old.get(function["name"])
+        if not function["documented"]:
+            if function["required"]:
+                findings.append({**function, "issue": "missing"})
+        elif previous and _SPACE.sub("", previous["signature"]) != _SPACE.sub("", function["signature"]):
+            findings.append({**function, "issue": "signature_changed"})
+    return findings
+
+
+def coverage(root, include_tests=False, limit=30, view_id=None):
+    """Docstring coverage of the indexed code: functions the language convention expects documented (plus any already
+    documented) and those still missing, file by file, so an agent can document a project that started without it.
+    Tests are left out unless include_tests."""
+    data = code_graph.build(root, view_id)
+    profile = index_profile.current((index_scope.load_scope(root) or {}).get("profile"))
+    kinds = ("code", "test") if include_tests else ("code",)
+    by_path = defaultdict(list)
+    for symbol in data.get("symbols") or []:
+        if supported(symbol["path"]) and index_profile.kind(symbol["path"], profile) in kinds:
+            by_path[symbol["path"]].append(symbol)
+    total, missing = 0, []
+    for path in sorted(by_path):
+        try:
+            with open(os.path.join(root, path), encoding="utf-8") as stream:
+                source = stream.read()
+        except (OSError, UnicodeDecodeError):
+            continue
+        found = [f for f in functions(path, source, by_path[path]) if f["required"] or f["documented"]]
+        total += len(found)
+        missing += [{"path": path, "function": f["name"], "line": f["start"]} for f in found if not f["documented"]]
+    per_file = Counter(item["path"] for item in missing)
+    return {"functions": total, "documented": total - len(missing),
+            "coverage_percent": round(100 * (total - len(missing)) / total, 1) if total else None,
+            "missing_total": len(missing), "missing": missing[:limit],
+            "files_with_most_missing": [{"path": p, "missing": n} for p, n in per_file.most_common(10)],
+            "include_tests": include_tests,
+            "diagnostics": [d.get("reason") for d in data.get("diagnostics") or [] if d.get("reason")],
+            "view": (data.get("selected") or {}).get("label")}

package/duplicates.py

@@ -25,8 +25,35 @@ EMBED_SLICE = 64
 CUTOFF_NOTE = ("Default semantic cutoff 0.90: in the 2026-10-02 measurement (claude-code, independent judge) pairs above "
                "0.95 were 10/10 actionable and those from 0.80 to 0.94 only 7/20; below 0.90 most are false positives "
                "and reviewing them costs time without leading to a refactor.")
+MIN_TOKENS = 50
+NEAR_MIN_TOKENS = 70
+SHINGLE = 5
+NEAR_NOTE = ("Near = at least 0.80 of 5-token sequences shared with keywords, called names, attributes and types kept "
+             "and only local names, strings and numbers abstracted; in the 2026-10-08 measurement on Python, "
+             "JavaScript, TypeScript and Java projects it caught every exact and locally renamed copy and 95-100% of "
+             "copies with one added statement, and fired on 0-1.6% of real functions.")
 _HASH_COMMENT = {"py", "rb", "sh"}
-_TOKEN = re.compile(r"[A-Za-z_]\w*|\d+|[^\s\w]")
+_TOKEN = re.compile(r"\"(?:\\.|[^\"\\\n])*\"|'(?:\\.|[^'\\\n])*'|`(?:\\.|[^`\\])*`|[A-Za-z_$][\w$]*|\d[\w.]*|[^\s\w]")
+_KEYWORDS = {
+    "py": "False None True and as assert async await break class continue def del elif else except finally for from "
+          "global if import in is lambda nonlocal not or pass raise return try while with yield self cls",
+    "js": "await break case catch class const continue debugger default delete do else export extends false finally "
+          "for function if import in instanceof let new null of return super switch this throw true try typeof "
+          "undefined var void while with yield async",
+    "java": "abstract assert boolean break byte case catch char class const continue default do double else enum "
+            "extends final finally float for if implements import instanceof int interface long native new null "
+            "package private protected public return short static super switch synchronized this throw throws "
+            "transient try void volatile while true false var",
+}
+_KEYWORDS["ts"] = _KEYWORDS["js"] + " as interface type enum implements private public protected readonly keyof never unknown any"
+_KEYWORDS = {lang: frozenset(words.split()) for lang, words in _KEYWORDS.items()}
+_LANG = {"py": "py", "js": "js", "jsx": "js", "mjs": "js", "cjs": "js", "ts": "ts", "tsx": "ts", "mts": "ts",
+         "cts": "ts", "java": "java"}
+_CONTRACT = re.compile(r"^(__\w+__|equals|hashCode|toString|compareTo|clone|close|run|call|apply|get|set|render|"
+                       r"constructor|ng[A-Z]\w*|componentDid\w+|main)$")
+_DOCSTRING = re.compile(r"\A\s*[rRuUbB]{0,2}(\"\"\"|''')[\s\S]*?\1")
+_GENERATED = re.compile(r"@generated|DO NOT EDIT|Code generated by|auto-generated", re.I)
+_WRITE_POOLS = {}
 _CONTAINER_KINDS = {"module", "template", "class", "interface"}
 _GENERIC_NAMES = {"callback", "anonymous", "constructor"}
 _WARMING = set()
@@ -64,18 +91,48 @@ def _strip_comments(text, ext):
 
 
 def _body(text, ext):
+    """Function body without the signature and, in Python, without the leading docstring, so two copies that differ
+    only in documentation compare as identical."""
     if ext in _HASH_COMMENT:
         lines = text.splitlines()
         start = next((k for k, line in enumerate(lines) if re.match(r"\s*(async\s+)?def\s", line)), 0)
         end = next((k for k in range(start, len(lines)) if lines[k].rstrip().endswith(":")), start)
-        return "\n".join(lines[end + 1:])
+        return _DOCSTRING.sub("", "\n".join(lines[end + 1:]), count=1)
     brace = text.find("{")
     return text[brace + 1:] if brace >= 0 else text
 
 
-def _shingles(text):
-    tokens = ["ID" if re.match(r"[A-Za-z_]", t) else t for t in _TOKEN.findall(text)]
-    return frozenset(tuple(tokens[i:i + 5]) for i in range(max(len(tokens) - 4, 1)))
+def _features(body, ext, name):
+    """Near-duplicate features of a function body: 5-token shingles where keywords, called names, attribute names and
+    type names stay and local names, strings and numbers become placeholders, so parallel functions calling different
+    APIs (get_stdin/get_stdout, rotateLeft/rotateRight) do not match while a copy with renamed locals does."""
+    words = _KEYWORDS.get(_LANG.get(ext), frozenset())
+    raw = _TOKEN.findall(body)
+    tokens = []
+    for k, t in enumerate(raw):
+        if t[0] in "\"'`":
+            tokens.append("STR")
+        elif t[0].isdigit():
+            tokens.append("NUM")
+        elif re.match(r"[A-Za-z_$]", t):
+            kept = (t in words or t[0].isupper() or (k and raw[k - 1] == ".")
+                    or (k + 1 < len(raw) and raw[k + 1] == "("))
+            tokens.append(t if kept else "ID")
+        else:
+            tokens.append(t)
+    return {"shingles": frozenset(tuple(tokens[i:i + SHINGLE]) for i in range(max(len(tokens) - SHINGLE + 1, 1))),
+            "tokens": len(tokens), "lang": _LANG.get(ext, ext),
+            "near_ok": len(tokens) >= NEAR_MIN_TOKENS and not _CONTRACT.match(name)}
+
+
+def _idioms(functions):
+    """Shingles shared by so many functions that they are language idioms (if err != nil, try/except), not copies."""
+    counts = {}
+    for f in functions:
+        for shingle in f["shingles"]:
+            counts[shingle] = counts.get(shingle, 0) + 1
+    cut = max(100, int(0.005 * len(functions)))
+    return frozenset(s for s, c in counts.items() if c > cut)
 
 
 def _index_state(path):
@@ -115,10 +172,12 @@ def _functions(root, include_tests):
             text = "\n".join(file_lines[s["start_line"] - 1:s["end_line"]])
             if not text.strip():
                 continue
-            normalized = re.sub(r"\s+", " ", _strip_comments(_body(text, ext), ext)).strip()
-            functions.append({"name": s["name"].split("(")[0].split(".")[-1], "path": path, "line": s["start_line"],
+            body = _strip_comments(_body(text, ext), ext)
+            normalized = re.sub(r"\s+", " ", body).strip()
+            name = s["name"].split("(")[0].split(".")[-1]
+            functions.append({"name": name, "path": path, "line": s["start_line"],
                               "end": s["end_line"], "lines": s["end_line"] - s["start_line"] + 1, "text": text[:MAX_TEXT],
-                              "hash": hashlib.sha1(normalized.encode("utf-8")).hexdigest(), "shingles": _shingles(normalized)})
+                              "hash": hashlib.sha1(normalized.encode("utf-8")).hexdigest(), **_features(body, ext, name)})
             occurrence = occurrences[functions[-1]["hash"]] = occurrences.get(functions[-1]["hash"], 0) + 1
             functions[-1]["kind"] = kind
             functions[-1]["fp"] = hashlib.sha1(f"{path}\0{functions[-1]['hash']}\0{occurrence}".encode("utf-8")).hexdigest()
@@ -183,16 +242,24 @@ def _warm(root, view, functions, embed):
             _WARMING.discard(root)
 
 
+def _similar(fa, fb):
+    """Whether two functions may be compared as near copies: same language, not overloads of one name in one file."""
+    return fa["lang"] == fb["lang"] and fa["hash"] != fb["hash"] and not (fa["path"] == fb["path"]
+                                                                         and fa["name"] == fb["name"])
+
+
 def _near(functions):
-    order = sorted(range(len(functions)), key=lambda i: len(functions[i]["shingles"]))
+    common = _idioms(functions)
+    cut = [f["shingles"] - common if f["near_ok"] else None for f in functions]
+    order = sorted((i for i in range(len(functions)) if cut[i]), key=lambda i: len(cut[i]))
     found = []
     for position, a in enumerate(order):
-        sa = functions[a]["shingles"]
+        sa = cut[a]
         for b in order[position + 1:]:
-            sb = functions[b]["shingles"]
+            sb = cut[b]
             if len(sb) * NEAR_JACCARD > len(sa):
                 break
-            if functions[a]["hash"] == functions[b]["hash"]:
+            if not _similar(functions[a], functions[b]):
                 continue
             inter = len(sa & sb)
             score = inter / (len(sa) + len(sb) - inter)
@@ -296,7 +363,7 @@ def find(root, embed, configured_model, min_similarity=DEFAULT_MIN_SIMILARITY, i
             remembered.pop(next(iter(remembered)))
     result = {
         "view": view["label"], "functions_analyzed": len(functions), "include_tests": include_tests,
-        "min_similarity": min_similarity, "cutoff_note": CUTOFF_NOTE, "semantic_status": status, "notes": notes,
+        "min_similarity": min_similarity, "cutoff_note": CUTOFF_NOTE, "near_note": NEAR_NOTE, "semantic_status": status, "notes": notes,
         "counts": {"exact_groups": len(exact), "near_pairs": len(near), "semantic_pairs": semantic_total,
                    "dismissed_hidden": len(hidden) + len(hidden_semantic)},
         "exact": [{"id": fid, "lines": g[0]["lines"], "copies": [_ref(f) for f in g]} for fid, g in exact[:limit]],
@@ -311,3 +378,71 @@ def find(root, embed, configured_model, min_similarity=DEFAULT_MIN_SIMILARITY, i
                                    "members": [_ref(skip[fid][1]), _ref(skip[fid][2])], **dismissed[fid]}
                                   for fid in hidden_semantic])
     return result
+
+
+def _write_pool(root):
+    """Indexed production functions with their idiom shingles, rebuilt only when the index changes; None while the
+    code graph is not cached yet, so an edit hook never waits for an analysis."""
+    view_path = indexer.existing_db_path(root)
+    if not view_path or code_graph.cached_symbols(root)[0] is None:
+        return None
+    state = (view_path, *_index_state(view_path))
+    with _LOCK:
+        cached = _WRITE_POOLS.get(root)
+    if cached and cached[0] == state:
+        return cached[1]
+    functions, _notes, _view = _functions(root, False)
+    production = [f for f in functions if f["kind"] == "code"]
+    pool = (production, _idioms(production))
+    with _LOCK:
+        _WRITE_POOLS[root] = (state, pool)
+    return pool
+
+
+def on_write(root, path, before, after, written):
+    """Functions an edit writes whose body copies one already indexed: exact (same tokens, comments and spacing
+    ignored) or near (see NEAR_NOTE). written lists the touched functions as doc_check.functions returns them. The
+    function being edited in place, overloads of one name and functions the edit removes from the file are skipped."""
+    ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
+    rel = os.path.relpath(path, root).replace(os.sep, "/")
+    profile = index_profile.current((index_scope.load_scope(root) or {}).get("profile"))
+    if ext not in _LANG or index_profile.kind(rel, profile) != "code" or _GENERATED.search(after[:4000]):
+        return []
+    pool = _write_pool(root)
+    if pool is None:
+        return []
+    production, common = pool
+    lines = after.splitlines()
+    remaining = {f["name"].split("(")[0].split(".")[-1] for f in written["all"]}
+    matches = []
+    for function in written["touched"]:
+        name = function["name"].split("(")[0].split(".")[-1]
+        text = "\n".join(lines[function["start"] - 1:function["end"]])
+        body = _strip_comments(_body(text, ext), ext)
+        features = _features(body, ext, name)
+        if function["end"] - function["start"] + 1 < MIN_LINES or features["tokens"] < MIN_TOKENS \
+                or _CONTRACT.match(name):
+            continue
+        digest = hashlib.sha1(re.sub(r"\s+", " ", body).strip().encode("utf-8")).hexdigest()
+        mine = features["shingles"] - common
+        best = None
+        for other in production:
+            if other["lang"] != features["lang"] or other["path"] == rel and (other["name"] == name
+                                                                              or other["name"] not in remaining):
+                continue
+            if other["hash"] == digest:
+                best = ("exact", 1.0, other)
+                break
+            if not (features["near_ok"] and other["near_ok"]):
+                continue
+            theirs = other["shingles"] - common
+            if not mine or min(len(mine), len(theirs)) < NEAR_JACCARD * max(len(mine), len(theirs)):
+                continue
+            inter = len(mine & theirs)
+            score = inter / (len(mine) + len(theirs) - inter)
+            if score >= NEAR_JACCARD and (best is None or score > best[1]):
+                best = ("near", score, other)
+        if best:
+            matches.append({"name": name, "lines": f"{function['start']}-{function['end']}", "type": best[0],
+                            "score": round(best[1], 2), "existing": _ref(best[2])})
+    return matches