@allansantos-dev/smart-tool 0.0.0-stage → 0.9.0

package/index_inventory.py

@@ -0,0 +1,75 @@
+"""Inventário somente leitura de visões, arquivos e trechos já persistidos."""
+import re
+from pathlib import Path
+
+import indexer
+import index_views
+import project_identity
+import document_text
+
+
+def views(root):
+    key = project_identity.project_id(root)
+    current = index_views.describe(root)
+    paths = list(Path(indexer.INDEX_DIR).glob(key + '.v-*.sqlite3'))
+    paths += [Path(indexer.INDEX_DIR)/(item+'.sqlite3') for item in
+              dict.fromkeys([key, *project_identity.legacy_ids(root)])]
+    rows = []
+    for path in dict.fromkeys(paths):
+        if not path.is_file() or not re.fullmatch(r'[a-f0-9]{16}(?:\.v-[a-f0-9]{16})?\.sqlite3', path.name):
+            continue
+        conn = indexer._readonly(str(path))
+        try:
+            meta = indexer._metadata(conn)
+            view_id = meta.get('view_id') or ('legacy' if current['git'] else 'workspace')
+            rows.append({'view_id': view_id, 'label': meta.get('view_label') or 'Legacy, no identified branch',
+                         'branch': meta.get('git_branch'), 'commit': meta.get('git_commit'),
+                         'current': view_id == current['view_id'], 'path': str(path),
+                         'files': conn.execute('SELECT count(*) FROM manifest').fetchone()[0],
+                         'chunks': conn.execute('SELECT count(*) FROM chunks').fetchone()[0],
+                         'model': meta.get('model_id'), 'dimensions': meta.get('dimensions'),
+                         'updated_at': meta.get('updated_at'), 'bytes': path.stat().st_size})
+        finally:
+            conn.close()
+    return sorted(rows, key=lambda row: (not row['current'], row['label']))
+
+
+def inspect(root, view_id=None, file_path=None, storage_id=None):
+    if file_path is not None:
+        if not isinstance(file_path, str):
+            raise ValueError("file_path must be a string.")
+        file_path = file_path.replace(chr(92), "/").removeprefix("./")
+    available = views(root)
+    current = index_views.describe(root)
+    explicit_view = view_id is not None or storage_id is not None
+    view_id = view_id or current['view_id']
+    selected = next((row for row in available if (Path(row['path']).name==storage_id if storage_id else row['view_id']==view_id)), None)
+    if selected is None and available and not explicit_view:
+        selected = available[0]
+    if not selected:
+        return {'views': available, 'selected': None, 'files': [], 'chunks': [],
+                'message': 'This view has not been indexed yet. Run a search or use Index now.'}
+    conn = indexer._readonly(selected['path'])
+    try:
+        columns = {row[1] for row in conn.execute('PRAGMA table_info(manifest)')}
+        # Índices anteriores ao manifest incremental não têm size/mtime_ns.
+        # A inspeção é somente leitura; não migrar nem medir o arquivo de outra branch.
+        size_column = 'm.size' if 'size' in columns else 'NULL'
+        files = [{'path': path, 'chunks': count, 'hash': digest, 'size': size} for path, count, digest, size in
+                 conn.execute(f'SELECT m.path,count(c.id),m.hash,{size_column} FROM manifest m LEFT JOIN chunks c ON c.path=m.path GROUP BY m.path ORDER BY m.path LIMIT 1001')]
+        chunks = []
+        if file_path is not None:
+            if not conn.execute('SELECT 1 FROM manifest WHERE path=?', (file_path,)).fetchone():
+                raise ValueError(f'File is outside the indexed view: {file_path}')
+            files = []
+            chunks = [{'start_line': start, 'end_line': end, 'text': text, 'dimensions': int(size or 0)//4,
+                       'location_kind':document_text.location_kind(file_path)}
+                      for start, end, text, size in conn.execute(
+                          'SELECT start_line,end_line,text,length(embedding) FROM chunks WHERE path=? ORDER BY start_line LIMIT 100', (file_path,))]
+        return {'views': available, 'selected': selected, 'files': files[:1000],
+                'truncated': len(files)>1000, 'chunks': chunks, 'max_chunks_displayed': 100,
+                'current_view': index_views.public(current),
+                'message': '' if selected['current'] else 'The current view is not shown. You are inspecting the stored index of ' + selected['label'] + '.',
+                'graph_type': 'project_branch_directory_file', 'read_only': True}
+    finally:
+        conn.close()

package/index_profile.py

@@ -0,0 +1,167 @@
+import collections
+import fnmatch
+import json
+import os
+import re
+import time
+
+import model_client
+
+PROFILE_VERSION = 2
+KINDS = ("code", "test", "doc")
+RETRY_AFTER_S = 3600
+MAX_DIRS = 90
+PROSE = {"md", "txt", "docx", "rst", "adoc"}
+# Data and markup inside a documentation folder are reports, not code (GitHub Linguist treats docs/ as documentation).
+DOC_DATA = {"html", "htm", "json", "csv", "xml", "yaml", "yml", "svg"}
+_NAME_DOC = re.compile(r"(^|/)(readme|changelog|license|licence|contributing|install|history|authors)(\.[^/]*)?$", re.I)
+_DIR_DOC = re.compile(r"(^|/)(docs?|adrs?|documentation)/", re.I)
+_TEST = re.compile(r"(^|/)(tests?|__tests__|spec|specs|e2e|fixtures?|testdata|test-utils?)(/|$)|(^|/)test_[^/]+\.py$|"
+                   r"(^|/)conftest\.py$|_test\.(py|go)$|\.(test|spec)\.[cm]?[jt]sx?$|(^|/)[^/]*[a-z0-9](?-i:Tests?)\.java$|"
+                   r"(^|/)src/test/|(^|/)test[-_][^/]*\.[cm]?[jt]sx?$|[-_]test\.[cm]?[jt]sx?$", re.I)
+_TEST_PROSE = re.compile(r"(^|/)(tests?|__tests__|e2e|fixtures?|testdata|test-utils?)(/|$)|(^|/)src/test/", re.I)
+_TEXT_LINE = re.compile(r"[A-Za-zÀ-ú]{3,}\s+[A-Za-zÀ-ú]{3,}")
+
+_TOOL = {"type": "function", "function": {
+    "name": "submit_profile", "description": "Submit the search-index profile.",
+    "parameters": {"type": "object", "required": ["groups", "languages"], "properties": {
+        "groups": {"type": "array", "items": {"type": "object", "required": ["path", "kind"], "properties": {
+            "path": {"type": "string", "description": "directory relative to root, '.' for root, or a glob like '**/NOTES.md'"},
+            "kind": {"type": "string", "enum": list(KINDS)}}}},
+        "languages": {"type": "object", "required": ["identifiers", "comments", "doc"], "properties": {
+            "identifiers": {"type": "array", "items": {"type": "string"},
+                            "description": "natural languages of identifier names (functions, classes, variables), most prevalent first (BCP-47)"},
+            "comments": {"type": "array", "items": {"type": "string"},
+                         "description": "natural languages of code comments and docstrings, most prevalent first (BCP-47)"},
+            "doc": {"type": "array", "items": {"type": "string"},
+                    "description": "natural languages of documentation, most prevalent first (BCP-47)"}}}}}}}
+
+_PROMPT = """You are profiling a repository for a code-search index at {root}.
+Classify where things live, by directory, not file by file. Three kinds:
+- code: anything the product runs or loads: source, templates, HTML pages, config, scripts, and prompt/instruction files that the product itself executes (e.g. agent/skill/command definitions of a plugin).
+- test: automated tests and their fixtures, mocks and helpers (unit, integration, e2e), even in folders with unusual names.
+- doc: material written for people: README, CHANGELOG, docs/, ADRs, plans, guides, notes.
+Use the most specific path needed: a directory, or a glob for files that break the rule of their directory. Paths not listed inherit from their closest listed parent.
+Also report the natural languages used, most prevalent first, separately for identifier names, for code comments and docstrings, and for docs. Identifiers are often English even when comments are not. Use the text samples as evidence.
+Directories (dir, file count, extensions, sample names, text sample):
+{rows}
+(total directories with indexed files: {total}; only the {shown} largest are shown)"""
+
+
+def _ext(path):
+    name = path.rsplit("/", 1)[-1]
+    return name.rsplit(".", 1)[-1].lower() if "." in name else ""
+
+
+def _sample(root, rel):
+    try:
+        with open(os.path.join(root, rel), "rb") as stream:
+            raw = stream.read(4000).decode("utf-8", "ignore")
+    except OSError:
+        return ""
+    return " | ".join(line.strip() for line in raw.splitlines() if _TEXT_LINE.search(line))[:260]
+
+
+def _evidence(root, included):
+    by_dir = collections.defaultdict(list)
+    for rel in included:
+        by_dir[rel.rsplit("/", 1)[0] if "/" in rel else "."].append(rel)
+    rows = []
+    for rel_dir, files in sorted(by_dir.items(), key=lambda item: -len(item[1]))[:MAX_DIRS]:
+        names = [path.rsplit("/", 1)[-1] for path in files]
+        text = next((path for path in files if _ext(path) in PROSE | {"py", "js", "ts", "java", "html", "mjs", "cjs"}), None)
+        rows.append({"dir": rel_dir, "files": len(files),
+                     "ext": dict(collections.Counter(_ext(path) or "(none)" for path in files).most_common(6)),
+                     "sample": names[:5], "text_sample": _sample(root, text) if text else ""})
+    return rows, len(by_dir)
+
+
+def _valid(args):
+    groups, languages = args.get("groups"), args.get("languages")
+    return (isinstance(groups, list) and all(isinstance(g, dict) and isinstance(g.get("path"), str)
+                                             and g.get("kind") in KINDS for g in groups)
+            and isinstance(languages, dict) and all(isinstance(languages.get(key), list) and
+                                                    all(isinstance(item, str) for item in languages[key])
+                                                    for key in ("identifiers", "comments", "doc")))
+
+
+def decide(root, included, model, deadline=None):
+    rows, total = _evidence(root, included)
+    remaining = deadline - time.monotonic() if deadline is not None else 120
+    if remaining <= 0:
+        raise TimeoutError("Profile analysis deadline exceeded.")
+    response = model_client.fetch("/v1/chat/completions", model_client.get_token(), method="POST",
+                                  timeout=min(120, remaining), body={
+        "model": model, "temperature": 0, "tools": [_TOOL], "tool_choice": "required",
+        "messages": [{"role": "user", "content": _PROMPT.format(
+            root=root, rows=json.dumps(rows, ensure_ascii=False), total=total, shown=len(rows))}]})
+    calls = response["choices"][0]["message"].get("tool_calls") or []
+    try:
+        args = next((json.loads(call["function"]["arguments"] or "{}") for call in calls
+                     if call["function"]["name"] == "submit_profile"), None)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError(f"The model returned a profile with invalid JSON: {exc}") from exc
+    if not isinstance(args, dict) or not _valid(args):
+        raise RuntimeError("The model did not define a valid profile of groups and languages.")
+    return {"version": PROFILE_VERSION, "groups": args["groups"], "languages": args["languages"],
+            "model": model, "decided_at": time.time()}
+
+
+def _explicit_kind(path, groups):
+    """Kind of the most specific profile group that names this path; None when only the root group covers it."""
+    covering = [group for group in groups if _covers(path, group)]
+    return _model_kind(path, covering) if covering else None
+
+
+def _covers(path, group):
+    pattern = group["path"].replace(chr(92), "/").removeprefix("./").strip("/") or "."
+    if pattern == ".":
+        return False
+    if any(char in pattern for char in "*?["):
+        return fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(path, pattern.replace("**/", ""))
+    return path.startswith(pattern + "/")
+
+
+def _model_kind(path, groups):
+    best = None
+    for group in groups:
+        pattern = group["path"].replace(chr(92), "/").removeprefix("./").strip("/") or "."
+        if any(char in pattern for char in "*?["):
+            if fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(path, pattern.replace("**/", "")):
+                return group["kind"]
+        elif pattern == "." or path.startswith(pattern + "/"):
+            depth = 0 if pattern == "." else len(pattern)
+            if best is None or depth > best[0]:
+                best = (depth, group["kind"])
+    return best[1] if best else "doc"
+
+
+def current(profile):
+    return profile if (profile or {}).get("version") == PROFILE_VERSION else None
+
+
+def kind(path, profile):
+    groups = (current(profile) or {}).get("groups") or []
+    ext = _ext(path)
+    if ext in PROSE:
+        if _NAME_DOC.search(path) or _DIR_DOC.search(path):
+            return "doc"
+        if _TEST_PROSE.search(path):
+            return "test"
+        return _model_kind(path, groups or [{"path": ".", "kind": "doc"}])
+    if _TEST.search(path):
+        return "test"
+    if ext in DOC_DATA and _DIR_DOC.search(path) and _explicit_kind(path, groups) != "code":
+        return "doc"
+    if not ext and _NAME_DOC.search(path):
+        return "doc"
+    return "test" if groups and _model_kind(path, groups) == "test" else "code"
+
+
+def public(profile):
+    profile = current(profile)
+    if not profile:
+        return {"source": "rules", "identifier_languages": [], "comment_languages": [], "doc_languages": []}
+    languages = profile["languages"]
+    return {"source": "model", "identifier_languages": languages["identifiers"],
+            "comment_languages": languages["comments"], "doc_languages": languages["doc"]}