@allansantos-dev/smart-tool 0.0.0-stage → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/LICENSE +202 -0
- package/NOTICE +32 -0
- package/README.md +116 -2
- package/atomic_io.py +94 -0
- package/autostart.py +199 -0
- package/bin/smart-tool.mjs +86 -0
- package/browser_service_client.py +232 -0
- package/capabilities.py +63 -0
- package/client_hooks.py +192 -0
- package/code_graph.py +403 -0
- package/code_graph_java.mjs +49 -0
- package/code_graph_js.cjs +135 -0
- package/config.py +241 -0
- package/daemon_launcher.py +165 -0
- package/directory_watch.py +131 -0
- package/document_text.py +69 -0
- package/duplicates.py +313 -0
- package/embedding_cache.py +99 -0
- package/endpoint_sync.py +179 -0
- package/folder_picker.py +73 -0
- package/freshness.py +278 -0
- package/gateway.py +220 -0
- package/hook_decision.py +96 -0
- package/hooks/pretool_router.py +45 -0
- package/index_inventory.py +75 -0
- package/index_profile.py +167 -0
- package/index_scope.py +458 -0
- package/index_views.py +121 -0
- package/indexer.py +878 -0
- package/install.py +353 -0
- package/install_runtime.py +283 -0
- package/integration_metrics.py +82 -0
- package/local_embedder.py +172 -0
- package/model_adapters/__init__.py +0 -0
- package/model_adapters/openai_compatible.py +57 -0
- package/model_catalog.py +74 -0
- package/model_client.py +38 -0
- package/model_defaults.py +70 -0
- package/package.json +33 -4
- package/paths.py +6 -0
- package/project_identity.py +42 -0
- package/project_insights_ui.py +38 -0
- package/project_maps_ui.py +96 -0
- package/project_monitor.py +204 -0
- package/project_scheduler.py +61 -0
- package/project_storage.py +165 -0
- package/project_store.py +220 -0
- package/projects_ui.py +146 -0
- package/requirements.txt +8 -0
- package/research_cache.py +171 -0
- package/research_metrics.py +110 -0
- package/router.py +582 -0
- package/secret_store.py +121 -0
- package/setup_ui.py +986 -0
- package/site_memory.py +53 -0
- package/smart_tool_daemon.py +3604 -0
- package/tray.py +245 -0
- package/update_check.py +22 -0
- package/usage_meter.py +124 -0
- package/version.py +18 -0
- package/view_compare.py +68 -0
- package/web_adapters/browser/browser_service.py +302 -0
- package/web_adapters/browser/crawl4ai_search.py +236 -0
- package/web_adapters/browser/fetch_camoufox.py +28 -0
- package/web_adapters/browser/requirements.txt +2 -0
- package/web_adapters/node/adapter-common.mjs +40 -0
- package/web_adapters/node/ddg-search.mjs +89 -0
- package/web_adapters/node/open-websearch-adapter.mjs +77 -0
- package/web_adapters/node/package-lock.json +2720 -0
- package/web_adapters/node/package.json +12 -0
- package/web_adapters/node/wikipedia-search.mjs +44 -0
- package/web_document_graph.py +71 -0
- package/web_fetch.py +242 -0
- package/web_search_adapters.py +193 -0
- package/web_search_ddgs.py +42 -0
- package/web_search_external.py +288 -0
- package/web_search_health.py +88 -0
package/client_hooks.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import secrets
|
|
4
|
+
import shutil
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
import daemon_launcher
|
|
9
|
+
import endpoint_sync
|
|
10
|
+
|
|
11
|
+
HOOK_SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "hooks", "pretool_router.py")
|
|
12
|
+
MARKER = "pretool_router.py"
|
|
13
|
+
HTTP_PATH = "/hooks/pretool"
|
|
14
|
+
HTTP_TIMEOUT_S = 15
|
|
15
|
+
HOME = os.path.expanduser("~")
|
|
16
|
+
PREVIEW_TTL_S = 600
|
|
17
|
+
CLIENTS = {
|
|
18
|
+
"claude": {"label": "Claude Code", "names": {"claude-code"}, "files": [os.path.join(endpoint_sync.claude_dir(), "settings.json")],
|
|
19
|
+
"docs": "https://code.claude.com/docs/en/hooks", "event": "PreToolUse", "matcher": "Grep|Glob|Read|Bash|WebSearch|WebFetch", "http": True},
|
|
20
|
+
"codex": {"label": "Codex", "prefix": "codex", "files": [os.path.join(HOME, ".codex", "hooks.json"), os.path.join(HOME, ".codex", "config.toml")],
|
|
21
|
+
"docs": "https://learn.chatgpt.com/docs/hooks", "event": "PreToolUse", "matcher": "Bash",
|
|
22
|
+
"after_install": "Codex only runs new hooks after the user approves the definition in /hooks."},
|
|
23
|
+
"cursor": {"label": "Cursor", "prefix": "cursor", "files": [os.path.join(HOME, ".cursor", "hooks.json")],
|
|
24
|
+
"docs": "https://cursor.com/docs/agent/hooks", "event": "preToolUse", "matcher": "Shell|Read|Grep",
|
|
25
|
+
"blocked": "In Cursor, `allow` approves the tool without the user and empty output blocks: there is no documented neutral "
|
|
26
|
+
"response, so the hook is not installed automatically."},
|
|
27
|
+
}
|
|
28
|
+
_PREVIEWS = {}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def identify(client_info):
|
|
32
|
+
name = str((client_info or {}).get("name") or "").casefold()
|
|
33
|
+
return next((key for key, spec in CLIENTS.items()
|
|
34
|
+
if name in spec.get("names", ()) or (spec.get("prefix") and name.startswith(spec["prefix"]))), None)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _our_hook(hook):
|
|
38
|
+
return isinstance(hook, dict) and (MARKER in str(hook.get("command", "")) or HTTP_PATH in str(hook.get("url", "")))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _ours(group):
|
|
42
|
+
return isinstance(group, dict) and isinstance(group.get("hooks"), list) and any(map(_our_hook, group["hooks"]))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _without_ours(groups):
|
|
46
|
+
"""Drops only our hook entries; a group shared with other hooks keeps them and disappears only when emptied."""
|
|
47
|
+
kept = []
|
|
48
|
+
for group in groups:
|
|
49
|
+
if not _ours(group):
|
|
50
|
+
kept.append(group)
|
|
51
|
+
continue
|
|
52
|
+
others = [hook for hook in group["hooks"] if not _our_hook(hook)]
|
|
53
|
+
if others:
|
|
54
|
+
kept.append({**group, "hooks": others})
|
|
55
|
+
return kept
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def installed(client):
|
|
59
|
+
"""Our hook is present with the current matcher; an entry with an older matcher counts as missing."""
|
|
60
|
+
spec = CLIENTS[client]
|
|
61
|
+
for path in spec["files"]:
|
|
62
|
+
try:
|
|
63
|
+
with open(path, encoding="utf-8") as stream:
|
|
64
|
+
text = stream.read()
|
|
65
|
+
except FileNotFoundError:
|
|
66
|
+
continue
|
|
67
|
+
if MARKER not in text and HTTP_PATH not in text:
|
|
68
|
+
continue
|
|
69
|
+
if not path.endswith(".json"):
|
|
70
|
+
return True
|
|
71
|
+
data = json.loads(text)
|
|
72
|
+
hooks = data.get("hooks") if isinstance(data, dict) else None
|
|
73
|
+
groups = hooks.get(spec["event"]) if isinstance(hooks, dict) else None
|
|
74
|
+
expected = _handler(client)
|
|
75
|
+
return isinstance(groups, list) and any(
|
|
76
|
+
group.get("matcher") == spec["matcher"] and expected in group["hooks"] for group in groups if _ours(group))
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def status(client_info):
|
|
81
|
+
client = identify(client_info)
|
|
82
|
+
if client is None:
|
|
83
|
+
return {"client": None, "name": (client_info or {}).get("name"), "installed": None, "error": None}
|
|
84
|
+
spec = CLIENTS[client]
|
|
85
|
+
try:
|
|
86
|
+
state, error = installed(client), None
|
|
87
|
+
except (OSError, ValueError) as exc:
|
|
88
|
+
state, error = None, f"{type(exc).__name__}: {exc}"[:200]
|
|
89
|
+
return {"client": client, "name": client_info.get("name"), "installed": state, "error": error,
|
|
90
|
+
"files": spec["files"], "docs": spec["docs"]}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def notice(state):
|
|
94
|
+
if state["client"] is None:
|
|
95
|
+
return (f"Unrecognized MCP client ({state['name'] or 'no clientInfo'}): Smart Tool does not check the hook "
|
|
96
|
+
"that redirects Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this client.")
|
|
97
|
+
spec = CLIENTS[state["client"]]
|
|
98
|
+
if state["error"]:
|
|
99
|
+
return f"Could not check the Smart Tool hook in {spec['files'][0]}: {state['error']}"
|
|
100
|
+
if state["installed"]:
|
|
101
|
+
return None
|
|
102
|
+
action = spec.get("blocked") or ("If the user asks, use project_manage action=install_hook_preview, show the "
|
|
103
|
+
"change and confirm with action=install_hook (makes a backup).")
|
|
104
|
+
return (f"Smart Tool hook missing or outdated in {spec['label']} ({spec['files'][0]}): Grep/Read/Bash are not "
|
|
105
|
+
f"redirected to smart_search, nor WebSearch/WebFetch to web_search/web_fetch. {action} Docs: {spec['docs']}")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _shell_safe(path):
|
|
109
|
+
"""Unquoted path that runs the same in cmd, PowerShell and bash (the Codex docs don't name the shell): forward
|
|
110
|
+
slashes (bash eats backslashes), no spaces (8.3 short name), no quotes (PowerShell reads a quoted head as a string)."""
|
|
111
|
+
if " " in path and sys.platform == "win32":
|
|
112
|
+
import ctypes
|
|
113
|
+
buffer = ctypes.create_unicode_buffer(1024)
|
|
114
|
+
if ctypes.windll.kernel32.GetShortPathNameW(path, buffer, len(buffer)):
|
|
115
|
+
path = buffer.value
|
|
116
|
+
if " " in path:
|
|
117
|
+
raise ValueError(f"Path contains a space and has no 8.3 short name ({path}); the hook was not installed.")
|
|
118
|
+
return path.replace("\\", "/")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _command(client):
|
|
122
|
+
return f"{_shell_safe(sys.executable)} {_shell_safe(HOOK_SCRIPT)} --client {client}"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _handler(client):
|
|
126
|
+
"""Claude Code posts the payload straight to the warm daemon (`type: "http"`, no process per tool call); other
|
|
127
|
+
clients run the thin command that forwards to the same route."""
|
|
128
|
+
if CLIENTS[client].get("http"):
|
|
129
|
+
registry = daemon_launcher.read_registry()
|
|
130
|
+
if not registry or not registry.get("url"):
|
|
131
|
+
raise ValueError("Smart Tool daemon has no registered URL; start the daemon before installing the hook.")
|
|
132
|
+
return {"type": "http", "url": f"{registry['url']}{HTTP_PATH}?client={client}", "timeout": HTTP_TIMEOUT_S}
|
|
133
|
+
return {"type": "command", "command": _command(client), "timeout": HTTP_TIMEOUT_S}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _plan(client_info):
|
|
137
|
+
client = identify(client_info)
|
|
138
|
+
if client is None:
|
|
139
|
+
raise ValueError(f"Unrecognized MCP client ({(client_info or {}).get('name') or 'no clientInfo'}); "
|
|
140
|
+
"the hook is only installed for Claude Code or Codex, from that client's own session.")
|
|
141
|
+
spec = CLIENTS[client]
|
|
142
|
+
if spec.get("blocked"):
|
|
143
|
+
raise ValueError(spec["blocked"] + f" Configure it manually if you want: {spec['docs']}")
|
|
144
|
+
path = os.path.realpath(spec["files"][0])
|
|
145
|
+
try:
|
|
146
|
+
with open(path, encoding="utf-8") as stream:
|
|
147
|
+
data = json.load(stream)
|
|
148
|
+
except FileNotFoundError:
|
|
149
|
+
data = {}
|
|
150
|
+
if not isinstance(data, dict) or not isinstance(data.setdefault("hooks", {}), dict):
|
|
151
|
+
raise ValueError(f"{path} does not have the expected format (object with 'hooks'); nothing was changed.")
|
|
152
|
+
groups = data["hooks"].setdefault(spec["event"], [])
|
|
153
|
+
if not isinstance(groups, list):
|
|
154
|
+
raise ValueError(f"{path}: hooks.{spec['event']} is not a list; nothing was changed.")
|
|
155
|
+
groups[:] = _without_ours(groups)
|
|
156
|
+
entry = {"matcher": spec["matcher"], "hooks": [_handler(client)]}
|
|
157
|
+
return client, spec, path, data, groups, entry
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def preview(client_info):
|
|
161
|
+
client, spec, path, _data, _groups, entry = _plan(client_info)
|
|
162
|
+
if installed(client):
|
|
163
|
+
return {"client": client, "installed": True, "file": path, "change": None}
|
|
164
|
+
token = secrets.token_urlsafe(12)
|
|
165
|
+
_PREVIEWS[token] = {"client": client, "at": time.time()}
|
|
166
|
+
return {"client": client, "installed": False, "file": path, "event": spec["event"], "change": entry,
|
|
167
|
+
"token": token, "expires_s": PREVIEW_TTL_S, "after_install": spec.get("after_install")}
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def install(client_info, token, confirm):
|
|
171
|
+
planned = _PREVIEWS.pop(token or "", None)
|
|
172
|
+
client, spec, path, data, groups, entry = _plan(client_info)
|
|
173
|
+
if confirm is not True or not planned or planned["client"] != client or time.time() - planned["at"] > PREVIEW_TTL_S:
|
|
174
|
+
raise ValueError("Installing requires the install_hook_preview token from this session (valid for 10 min) and confirm=true.")
|
|
175
|
+
if installed(client):
|
|
176
|
+
return {"client": client, "installed": True, "changed": False, "file": path}
|
|
177
|
+
groups.append(entry)
|
|
178
|
+
backup = None
|
|
179
|
+
if os.path.exists(path):
|
|
180
|
+
backup = f"{path}.smart-tool-backup-{time.strftime('%Y%m%d-%H%M%S')}"
|
|
181
|
+
shutil.copy2(path, backup)
|
|
182
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
183
|
+
temporary = path + ".smart-tool-tmp"
|
|
184
|
+
with open(temporary, "w", encoding="utf-8") as stream:
|
|
185
|
+
json.dump(data, stream, indent=2, ensure_ascii=False)
|
|
186
|
+
os.replace(temporary, path)
|
|
187
|
+
if not installed(client):
|
|
188
|
+
if backup:
|
|
189
|
+
shutil.copy2(backup, path)
|
|
190
|
+
raise RuntimeError(f"The hook does not appear in {path} after writing; the file was restored.")
|
|
191
|
+
return {"client": client, "installed": True, "changed": True, "file": path, "backup": backup,
|
|
192
|
+
"after_install": spec.get("after_install")}
|
package/code_graph.py
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
"""Grafos estáticos do conteúdo indexado: cobertura, imports e chamadas resolvidas."""
|
|
2
|
+
import ast
|
|
3
|
+
import collections
|
|
4
|
+
import copy
|
|
5
|
+
import contextvars
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import posixpath
|
|
10
|
+
import subprocess
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import indexer
|
|
16
|
+
import index_inventory
|
|
17
|
+
import project_identity
|
|
18
|
+
import web_search_adapters
|
|
19
|
+
import web_document_graph
|
|
20
|
+
import document_text
|
|
21
|
+
|
|
22
|
+
VERSION = 2
|
|
23
|
+
MAX_FILES = 3000
|
|
24
|
+
MAX_TEXT = 24 * 1024 * 1024
|
|
25
|
+
MAX_SYMBOLS = 8000
|
|
26
|
+
MAX_EDGES = 20000
|
|
27
|
+
_CACHE = collections.OrderedDict()
|
|
28
|
+
_LOCK = threading.Lock()
|
|
29
|
+
_ANALYSIS_SLOTS = threading.BoundedSemaphore(2)
|
|
30
|
+
_WARM_SLOT = threading.BoundedSemaphore(1)
|
|
31
|
+
_WARMING = set()
|
|
32
|
+
_WARM_STATE = {}
|
|
33
|
+
WARM_RETRY_S = 600
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def reconstruct(rows):
|
|
37
|
+
"""Remove sobreposição de chunks e recusa buracos/conflitos; mantém linhas reais."""
|
|
38
|
+
lines = {}
|
|
39
|
+
previous = None
|
|
40
|
+
for start, end, text in rows:
|
|
41
|
+
parts = text.splitlines(keepends=True)
|
|
42
|
+
if len(parts) != end-start+1:
|
|
43
|
+
return None, 'Chunks do not reconstruct the full lines of the file.'
|
|
44
|
+
for offset, part in enumerate(parts):
|
|
45
|
+
line = start+offset
|
|
46
|
+
if line in lines and start == end == line and previous and previous[0] == previous[1] == line and not previous[2].endswith(('\n','\r')):
|
|
47
|
+
lines[line] += part
|
|
48
|
+
elif line in lines and lines[line] != part:
|
|
49
|
+
return None, 'Overlapping chunks contain conflicting versions.'
|
|
50
|
+
else:
|
|
51
|
+
lines[line] = part
|
|
52
|
+
previous = (start,end,text)
|
|
53
|
+
if not lines:
|
|
54
|
+
return '', None
|
|
55
|
+
if min(lines) != 1 or len(lines) != max(lines):
|
|
56
|
+
return None, 'The snapshot has gaps; relations were not inferred for this file.'
|
|
57
|
+
return ''.join(lines[i] for i in range(1,max(lines)+1)), None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _snapshot(path):
|
|
61
|
+
conn = indexer._readonly(path)
|
|
62
|
+
try:
|
|
63
|
+
conn.execute('BEGIN')
|
|
64
|
+
columns = {row[1] for row in conn.execute('PRAGMA table_info(manifest)')}
|
|
65
|
+
size = 'm.size' if 'size' in columns else 'NULL'
|
|
66
|
+
mtime = 'm.mtime_ns' if 'mtime_ns' in columns else 'NULL'
|
|
67
|
+
records = conn.execute(f'SELECT m.path,m.hash,{size},{mtime},count(c.id),max(c.end_line) FROM manifest m LEFT JOIN chunks c ON c.path=m.path GROUP BY m.path ORDER BY m.path LIMIT ?', (MAX_FILES+1,)).fetchall()
|
|
68
|
+
files, sources, diagnostics, used = [], {}, [], 0
|
|
69
|
+
for original, digest, byte_size, stamp, chunks, line_count in records[:MAX_FILES]:
|
|
70
|
+
file_path = original.replace('\\','/')
|
|
71
|
+
record = {'id':file_path,'path':file_path,'hash':digest,'size':byte_size,'mtime_ns':stamp,
|
|
72
|
+
'chunks':chunks,'lines':line_count or 0,'group':posixpath.dirname(file_path) or '(root)',
|
|
73
|
+
'location_kind':document_text.location_kind(file_path),'format':posixpath.splitext(file_path)[1].lower().lstrip('.') or 'text'}
|
|
74
|
+
files.append(record)
|
|
75
|
+
if not file_path.lower().endswith(('.py','.java','.js','.jsx','.ts','.tsx','.mjs','.cjs','.mts','.cts','.json','.html','.htm','.css','.scss','.sass','.less','.md','.markdown')):
|
|
76
|
+
continue
|
|
77
|
+
rows = conn.execute('SELECT start_line,end_line,text FROM chunks WHERE path=? ORDER BY start_line,id', (original,)).fetchall()
|
|
78
|
+
text, error = reconstruct(rows)
|
|
79
|
+
if error:
|
|
80
|
+
diagnostics.append({'path':file_path,'reason':error})
|
|
81
|
+
elif used+len(text) > MAX_TEXT:
|
|
82
|
+
diagnostics.append({'path':file_path,'reason':'Content limit for this analysis reached.'})
|
|
83
|
+
else:
|
|
84
|
+
used += len(text)
|
|
85
|
+
sources[file_path] = text
|
|
86
|
+
return files, sources, diagnostics, len(records)>MAX_FILES
|
|
87
|
+
finally:
|
|
88
|
+
conn.close()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _python_graph(sources):
|
|
92
|
+
trees, symbols, dependencies, calls, unresolved, diagnostics = {}, [], [], [], [], []
|
|
93
|
+
decls, aliases, scopes, classes, owners, shadow_scopes, node_symbols = {}, {}, {}, {}, {}, {}, {}
|
|
94
|
+
def bind_import(path,scope,name,binding):
|
|
95
|
+
table=aliases[path].setdefault(scope,{})
|
|
96
|
+
table[name]=binding if name not in table or table[name]==binding else None
|
|
97
|
+
def resolve_file(spec):
|
|
98
|
+
spec=posixpath.normpath(spec)
|
|
99
|
+
return next((p for p in (spec+'.py',spec+'/__init__.py','src/'+spec+'.py','src/'+spec+'/__init__.py') if p in sources), None)
|
|
100
|
+
for path, text in sources.items():
|
|
101
|
+
if not path.endswith('.py'):
|
|
102
|
+
continue
|
|
103
|
+
try:
|
|
104
|
+
tree=ast.parse(text,filename=path)
|
|
105
|
+
except (SyntaxError,ValueError,RecursionError) as exc:
|
|
106
|
+
diagnostics.append({'path':path,'reason':'Incomplete or invalid Python syntax in the snapshot.','line':getattr(exc,'lineno',None)})
|
|
107
|
+
continue
|
|
108
|
+
trees[path]=tree;aliases[path]={};decls[path]={};classes[path]={}
|
|
109
|
+
def collect(node,qual=(),owner=None):
|
|
110
|
+
if isinstance(node,(ast.FunctionDef,ast.AsyncFunctionDef,ast.ClassDef)):
|
|
111
|
+
qual=qual+(node.name,)
|
|
112
|
+
sym={'id':path+'::'+'.'.join(qual)+'@'+str(node.lineno),'path':path,'name':'.'.join(qual),
|
|
113
|
+
'start_line':node.lineno,'end_line':node.end_lineno,'kind':'class' if isinstance(node,ast.ClassDef) else 'function', 'language':'python'}
|
|
114
|
+
decls[path][qual]=sym if qual not in decls[path] else None
|
|
115
|
+
scopes[id(node)]=qual;node_symbols[id(node)]=sym
|
|
116
|
+
if isinstance(node,ast.ClassDef):classes[path][qual]=node
|
|
117
|
+
if len(symbols)<MAX_SYMBOLS:symbols.append(sym)
|
|
118
|
+
owner=sym
|
|
119
|
+
owners[id(node)]=qual
|
|
120
|
+
for child in ast.iter_child_nodes(node):collect(child,qual,owner)
|
|
121
|
+
collect(tree)
|
|
122
|
+
for node in ast.walk(tree):
|
|
123
|
+
if isinstance(node,ast.Import):
|
|
124
|
+
for item in node.names:
|
|
125
|
+
spec=item.name.replace('.','/');target=resolve_file(spec)
|
|
126
|
+
bind_import(path,owners[id(node)],item.asname or item.name,(target,None))
|
|
127
|
+
dependencies.append({'source':path,'target':target,'specifier':item.name,'line':node.lineno,'kind':'import','resolution':'resolved' if target else 'external'})
|
|
128
|
+
elif isinstance(node,ast.ImportFrom):
|
|
129
|
+
prefix=posixpath.dirname(path)
|
|
130
|
+
if node.level:
|
|
131
|
+
for _ in range(node.level-1):prefix=posixpath.dirname(prefix)
|
|
132
|
+
spec=posixpath.join(prefix,(node.module or '').replace('.','/'))
|
|
133
|
+
else:spec=(node.module or '').replace('.','/')
|
|
134
|
+
base=resolve_file(spec)
|
|
135
|
+
for item in node.names:
|
|
136
|
+
target=resolve_file(spec+'/'+item.name) if item.name!='*' else None
|
|
137
|
+
target=target or base
|
|
138
|
+
bind_import(path,owners[id(node)],item.asname or item.name,(target,None if target and target!=base else item.name))
|
|
139
|
+
dependencies.append({'source':path,'target':target,'specifier':('.'*node.level)+(node.module or '')+'.'+item.name,'line':node.lineno,'kind':'import','resolution':'resolved' if target else 'unresolved' if node.level else 'external'})
|
|
140
|
+
def chain(expr):
|
|
141
|
+
if isinstance(expr,ast.Name):return expr.id
|
|
142
|
+
if isinstance(expr,ast.Attribute):
|
|
143
|
+
base=chain(expr.value)
|
|
144
|
+
return base+'.'+expr.attr if base else None
|
|
145
|
+
return None
|
|
146
|
+
def resolve(path,qual,name,shadow,instances):
|
|
147
|
+
if not name:return None
|
|
148
|
+
parts=name.split('.')
|
|
149
|
+
if parts[0] in instances:
|
|
150
|
+
typ=instances[parts[0]]
|
|
151
|
+
return decls.get(typ['path'],{}).get(tuple(typ['name'].split('.'))+tuple(parts[1:]))
|
|
152
|
+
if parts[0] in ('self','cls') and len(parts)>1:
|
|
153
|
+
for i in range(len(qual),0,-1):
|
|
154
|
+
if qual[:i] in classes[path]:return decls[path].get(qual[:i]+tuple(parts[1:]))
|
|
155
|
+
if parts[0] in shadow:return None
|
|
156
|
+
for i in range(len(qual),-1,-1):
|
|
157
|
+
if i<len(qual) and qual[:i] in classes[path]:
|
|
158
|
+
continue # O namespace da classe não é escopo lexical de seus métodos.
|
|
159
|
+
if parts[0] in shadow_scopes.get((path,qual[:i]),set()):
|
|
160
|
+
return None
|
|
161
|
+
if qual[:i]+tuple(parts) in decls[path]:
|
|
162
|
+
return decls[path][qual[:i]+tuple(parts)]
|
|
163
|
+
for length in range(len(parts),0,-1):
|
|
164
|
+
table=aliases[path].get(qual[:i],{})
|
|
165
|
+
binding=table.get('.'.join(parts[:length]))
|
|
166
|
+
if '.'.join(parts[:length]) in table:
|
|
167
|
+
if binding is None:return None
|
|
168
|
+
target_path,imported=binding
|
|
169
|
+
rest=(([imported] if imported and imported!='*' else [])+parts[length:])
|
|
170
|
+
return decls.get(target_path,{}).get(tuple(rest))
|
|
171
|
+
return None
|
|
172
|
+
for path,tree in trees.items():
|
|
173
|
+
module={'id':path+'::<module>','path':path,'name':'<module>','start_line':1,'end_line':1,'kind':'module','language':'python'}
|
|
174
|
+
used_module=False
|
|
175
|
+
def visit_scope(node,qual=()):
|
|
176
|
+
nonlocal used_module
|
|
177
|
+
owner=node_symbols.get(id(node),module)
|
|
178
|
+
shadow=set();instances={}
|
|
179
|
+
args=getattr(node,'args',None)
|
|
180
|
+
if args:
|
|
181
|
+
params=[*args.posonlyargs,*args.args,*args.kwonlyargs]+([args.vararg] if args.vararg else [])+([args.kwarg] if args.kwarg else [])
|
|
182
|
+
shadow.update(arg.arg for arg in params)
|
|
183
|
+
for arg in params:
|
|
184
|
+
target=resolve(path,qual,chain(arg.annotation),set(),{}) if arg.annotation else None
|
|
185
|
+
if target and target['kind']=='class':instances[arg.arg]=target
|
|
186
|
+
# Levanta ligações locais sem atravessar escopos aninhados.
|
|
187
|
+
def local_nodes(n):
|
|
188
|
+
for child in ast.iter_child_nodes(n):
|
|
189
|
+
if isinstance(child,(ast.FunctionDef,ast.AsyncFunctionDef,ast.ClassDef,ast.Lambda)):
|
|
190
|
+
continue
|
|
191
|
+
yield child
|
|
192
|
+
yield from local_nodes(child)
|
|
193
|
+
local=list(local_nodes(node))
|
|
194
|
+
assigned={}
|
|
195
|
+
for n in local:
|
|
196
|
+
if isinstance(n,(ast.Assign,ast.AnnAssign)):
|
|
197
|
+
targets=n.targets if isinstance(n,ast.Assign) else [n.target]
|
|
198
|
+
for target in targets:
|
|
199
|
+
if isinstance(target,ast.Name):
|
|
200
|
+
shadow.add(target.id);assigned.setdefault(target.id,[]).append(n.value)
|
|
201
|
+
for name,values in assigned.items():
|
|
202
|
+
if len(values)==1 and isinstance(values[0],ast.Call):
|
|
203
|
+
typ=resolve(path,qual,chain(values[0].func),shadow-set([name]),instances)
|
|
204
|
+
if typ and typ['kind']=='class':instances[name]=typ
|
|
205
|
+
shadow_scopes[(path,qual)]=shadow
|
|
206
|
+
for n in local:
|
|
207
|
+
if not isinstance(n,ast.Call):continue
|
|
208
|
+
expr=chain(n.func)
|
|
209
|
+
target=resolve(path,qual,expr,shadow,instances)
|
|
210
|
+
if target:
|
|
211
|
+
if owner is module:used_module=True
|
|
212
|
+
if len(calls)<MAX_EDGES:calls.append({'source':owner['id'],'target':target['id'],'line':n.lineno,'kind':'construct' if target['kind']=='class' else 'call','resolution':'static'})
|
|
213
|
+
elif len(unresolved)<1000:
|
|
214
|
+
unresolved.append({'path':path,'line':n.lineno,'expression':expr or '<dynamic expression>','reason':'External, dynamic or unresolved target in this snapshot.'})
|
|
215
|
+
for child in ast.iter_child_nodes(node):
|
|
216
|
+
if isinstance(child,(ast.FunctionDef,ast.AsyncFunctionDef,ast.ClassDef)):
|
|
217
|
+
visit_scope(child,scopes[id(child)])
|
|
218
|
+
else:
|
|
219
|
+
# Definições em blocos if/try também podem declarar escopos.
|
|
220
|
+
def nested(n):
|
|
221
|
+
for c in ast.iter_child_nodes(n):
|
|
222
|
+
if isinstance(c,(ast.FunctionDef,ast.AsyncFunctionDef,ast.ClassDef)):visit_scope(c,scopes[id(c)])
|
|
223
|
+
else:nested(c)
|
|
224
|
+
nested(child)
|
|
225
|
+
visit_scope(tree)
|
|
226
|
+
if used_module:symbols.append(module)
|
|
227
|
+
return {'symbols':symbols,'dependencies':dependencies[:MAX_EDGES],'calls':calls,'unresolved':unresolved,
|
|
228
|
+
'diagnostics':diagnostics,'parsed':list(trees),'truncated':len(symbols)>=MAX_SYMBOLS or len(dependencies)>MAX_EDGES}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _javascript_graph(sources, all_paths=None):
|
|
232
|
+
extensions=('.js','.jsx','.ts','.tsx','.mjs','.cjs','.mts','.cts')
|
|
233
|
+
relevant={path:text for path,text in sources.items() if path.endswith(extensions+('.json','.html','.htm'))}
|
|
234
|
+
if not any(path.endswith(extensions) for path in relevant):
|
|
235
|
+
return {}
|
|
236
|
+
try:
|
|
237
|
+
process=subprocess.run([web_search_adapters.NODE_EXECUTABLE,'--max-old-space-size=512',str(Path(__file__).with_name('code_graph_js.cjs'))],
|
|
238
|
+
input=json.dumps({'files':[{'path':path,'text':text} for path,text in relevant.items()],'paths':list(all_paths or sources)},ensure_ascii=False),
|
|
239
|
+
capture_output=True,text=True,encoding='utf-8',timeout=40,
|
|
240
|
+
creationflags=getattr(subprocess,'CREATE_NO_WINDOW',0))
|
|
241
|
+
if process.returncode:
|
|
242
|
+
raise RuntimeError('The TypeScript analyzer did not finish. Check the local code-analysis runtime.')
|
|
243
|
+
return json.loads(process.stdout)
|
|
244
|
+
except (OSError,subprocess.TimeoutExpired,ValueError,RuntimeError) as exc:
|
|
245
|
+
return {'diagnostics':[{'path':'JavaScript/TypeScript','reason':str(exc)[:250]}],'parsed':[],'retryable':True}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _java_graph(sources):
|
|
249
|
+
files=[{'path':path,'text':text} for path,text in sources.items() if path.lower().endswith('.java')]
|
|
250
|
+
if not files:return {}
|
|
251
|
+
try:
|
|
252
|
+
result=subprocess.run([web_search_adapters.NODE_EXECUTABLE,'--max-old-space-size=512',str(Path(__file__).with_name('code_graph_java.mjs'))],
|
|
253
|
+
input=json.dumps({'files':files},ensure_ascii=False),capture_output=True,text=True,encoding='utf-8',timeout=40,
|
|
254
|
+
creationflags=getattr(subprocess,'CREATE_NO_WINDOW',0))
|
|
255
|
+
if result.returncode:raise RuntimeError('The Java analyzer did not finish. Check the local runtime.')
|
|
256
|
+
return json.loads(result.stdout)
|
|
257
|
+
except (OSError,ValueError,RuntimeError,subprocess.TimeoutExpired) as exc:
|
|
258
|
+
return {'diagnostics':[{'path':'Java','reason':str(exc)[:250]}],'parsed':[],'retryable':True}
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _analyze(path):
|
|
262
|
+
files,sources,diagnostics,truncated=_snapshot(path)
|
|
263
|
+
merged={'symbols':[],'dependencies':[],'calls':[],'unresolved':[],'diagnostics':diagnostics,'parsed':[]}
|
|
264
|
+
retryable=False
|
|
265
|
+
for output in (_python_graph(sources),_javascript_graph(sources,[f['path'] for f in files]),_java_graph(sources),web_document_graph.analyze(sources,[f['path'] for f in files])):
|
|
266
|
+
for key in merged:merged[key].extend(output.get(key,[]))
|
|
267
|
+
truncated |= output.get('truncated',False)
|
|
268
|
+
retryable |= output.get('retryable',False)
|
|
269
|
+
by_id={sym['id']:sym for sym in merged['symbols'][:MAX_SYMBOLS]}
|
|
270
|
+
# Relações repetidas preservam contagem e locais de chamada sem poluir o desenho.
|
|
271
|
+
def collapse(rows,fields):
|
|
272
|
+
result={}
|
|
273
|
+
for row in rows:
|
|
274
|
+
key=tuple(row.get(field) for field in fields)
|
|
275
|
+
if key not in result:result[key]={**row,'count':0,'lines':[]}
|
|
276
|
+
result[key]['count']+=1
|
|
277
|
+
if len(result[key]['lines'])<20:result[key]['lines'].append(row.get('line'))
|
|
278
|
+
return list(result.values())
|
|
279
|
+
deps=collapse(merged['dependencies'],('source','target','specifier','kind'))
|
|
280
|
+
calls=collapse([r for r in merged['calls'] if r['source'] in by_id and r['target'] in by_id],('source','target','kind'))
|
|
281
|
+
supported=set(merged['parsed'])
|
|
282
|
+
for file in files:file['analysis']='parsed' if file['path'] in supported else 'not_analyzed'
|
|
283
|
+
return {'files':files,'symbols':merged['symbols'][:MAX_SYMBOLS],
|
|
284
|
+
'dependencies':deps[:MAX_EDGES],'calls':calls[:MAX_EDGES],
|
|
285
|
+
'unresolved':merged['unresolved'][:1000],'diagnostics':merged['diagnostics'],
|
|
286
|
+
'truncated':truncated or len(deps)>MAX_EDGES or len(calls)>MAX_EDGES,'retryable':retryable,
|
|
287
|
+
'counts':{'files':len(files),'parsed_files':len(supported),'symbols':len(merged['symbols']),
|
|
288
|
+
'resolved_imports':sum(bool(r['target']) for r in deps),
|
|
289
|
+
'external_imports':sum(r['resolution']=='external' for r in deps),
|
|
290
|
+
'unresolved_calls':len(merged['unresolved']),'calls':len(calls)}}
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _focus(data,file_path):
|
|
294
|
+
if not isinstance(file_path,str):
|
|
295
|
+
raise ValueError('file_path must be a string.')
|
|
296
|
+
target=file_path.replace('\\','/').removeprefix('./')
|
|
297
|
+
files=[f for f in data['files'] if f['path']==target]
|
|
298
|
+
if not files:
|
|
299
|
+
raise ValueError(f'File is outside the indexed view: {file_path}')
|
|
300
|
+
by_id={s['id']:s for s in data['symbols']}
|
|
301
|
+
own={s['id'] for s in data['symbols'] if s.get('path')==target}
|
|
302
|
+
calls=[c for c in data['calls'] if c['source'] in own or c['target'] in own]
|
|
303
|
+
linked=own|{c['source'] for c in calls}|{c['target'] for c in calls}
|
|
304
|
+
deps=[d for d in data['dependencies'] if target in (d['source'],d['target'])]
|
|
305
|
+
paths={f['path'] for f in data['files']}
|
|
306
|
+
data.update(files=files,symbols=[by_id[i] for i in linked if i in by_id],dependencies=deps,calls=calls,
|
|
307
|
+
unresolved=[u for u in data['unresolved'] if u.get('path')==target],
|
|
308
|
+
diagnostics=[d for d in data['diagnostics'] if d.get('path')==target or d.get('path') not in paths],
|
|
309
|
+
focus={'file_path':target,'symbols':len(own),'imported_by':sum(d['target']==target for d in deps),
|
|
310
|
+
'imports':sum(d['source']==target for d in deps),
|
|
311
|
+
'incoming_calls':sum(c['target'] in own and c['source'] not in own for c in calls),
|
|
312
|
+
'outgoing_calls':sum(c['source'] in own and c['target'] not in own for c in calls)})
|
|
313
|
+
return data
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _by_file(data):
|
|
317
|
+
by_file={}
|
|
318
|
+
for symbol in data['symbols']:
|
|
319
|
+
by_file.setdefault(symbol['path'],[]).append(symbol)
|
|
320
|
+
parsed={f['path'] for f in data['files'] if f.get('analysis')=='parsed'}
|
|
321
|
+
note='Graph truncated by analysis limits; some files have no functions.' if data.get('truncated') else None
|
|
322
|
+
return {'symbols':by_file,'parsed':parsed,'note':note}
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def cached_symbols(root):
|
|
326
|
+
path=indexer.existing_db_path(root)
|
|
327
|
+
if not path:
|
|
328
|
+
return None,None
|
|
329
|
+
stat=os.stat(path);key=(path,stat.st_mtime_ns,stat.st_size,VERSION)
|
|
330
|
+
with _LOCK:
|
|
331
|
+
data=_CACHE.get(key)
|
|
332
|
+
if data is not None:
|
|
333
|
+
_CACHE.move_to_end(key)
|
|
334
|
+
state=_WARM_STATE.get(root)
|
|
335
|
+
if data is None and state and state['key']==key and (state['result'] or time.time()-state['at']<WARM_RETRY_S):
|
|
336
|
+
return state['result'],state['note']
|
|
337
|
+
start=data is None and root not in _WARMING
|
|
338
|
+
if start:_WARMING.add(root)
|
|
339
|
+
if data is not None:
|
|
340
|
+
return _by_file(data),None
|
|
341
|
+
if start:
|
|
342
|
+
context=contextvars.copy_context()
|
|
343
|
+
threading.Thread(target=context.run,args=(_warm,root,key),daemon=True).start()
|
|
344
|
+
return None,None
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _warm(root,key):
|
|
348
|
+
result,note=None,None
|
|
349
|
+
try:
|
|
350
|
+
data=build(root,background=True)
|
|
351
|
+
if data.get('retryable'):
|
|
352
|
+
result=_by_file(data)
|
|
353
|
+
reasons='; '.join(d.get('reason','') for d in data.get('diagnostics',[])[:2])
|
|
354
|
+
note=f'Partial analysis: {reasons}'[:300]
|
|
355
|
+
except Exception as exc:
|
|
356
|
+
note=f'The last analysis failed: {type(exc).__name__}: {exc}'[:300]
|
|
357
|
+
finally:
|
|
358
|
+
with _LOCK:
|
|
359
|
+
if note:_WARM_STATE[root]={'key':key,'at':time.time(),'result':result,'note':note}
|
|
360
|
+
else:_WARM_STATE.pop(root,None)
|
|
361
|
+
_WARMING.discard(root)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def build(root,view_id=None,storage_id=None,file_path=None,background=False):
|
|
365
|
+
inventory=index_inventory.inspect(root,view_id,storage_id=storage_id)
|
|
366
|
+
if not inventory['selected']:
|
|
367
|
+
return {**inventory,'symbols':[],'dependencies':[],'calls':[],'diagnostics':[],'counts':{}}
|
|
368
|
+
path=inventory['selected']['path'];stat=os.stat(path)
|
|
369
|
+
key=(path,stat.st_mtime_ns,stat.st_size,VERSION)
|
|
370
|
+
with _LOCK:
|
|
371
|
+
cached=_CACHE.get(key)
|
|
372
|
+
if cached:_CACHE.move_to_end(key)
|
|
373
|
+
was_cached=cached is not None
|
|
374
|
+
if cached is None:
|
|
375
|
+
slots=_WARM_SLOT if background else _ANALYSIS_SLOTS
|
|
376
|
+
if not slots.acquire(timeout=None if background else 3):
|
|
377
|
+
raise RuntimeError('Two map analyses are in progress. Wait for them to finish and reload.')
|
|
378
|
+
try:
|
|
379
|
+
cached=_analyze(path)
|
|
380
|
+
finally:
|
|
381
|
+
slots.release()
|
|
382
|
+
after=os.stat(path)
|
|
383
|
+
if not cached['retryable'] and (after.st_mtime_ns,after.st_size)==(stat.st_mtime_ns,stat.st_size):
|
|
384
|
+
with _LOCK:
|
|
385
|
+
_CACHE[key]=cached
|
|
386
|
+
while len(_CACHE)>3:_CACHE.popitem(last=False)
|
|
387
|
+
data=copy.deepcopy(cached)
|
|
388
|
+
# Só compara metadados do diretório de trabalho quando a visão é a atual.
|
|
389
|
+
for file in data['files']:
|
|
390
|
+
file['status']='stored'
|
|
391
|
+
if inventory['selected']['current'] and file.get('mtime_ns'):
|
|
392
|
+
candidate=os.path.join(root,file['path'])
|
|
393
|
+
if project_identity.within_root(root,candidate):
|
|
394
|
+
try:
|
|
395
|
+
current=os.stat(candidate)
|
|
396
|
+
file['status']='indexed' if (current.st_mtime_ns,current.st_size)==(file['mtime_ns'],file['size']) else 'pending'
|
|
397
|
+
except OSError:file['status']='missing'
|
|
398
|
+
file.pop('mtime_ns',None)
|
|
399
|
+
data.update(views=inventory['views'],selected=inventory['selected'],current_view=inventory['current_view'],
|
|
400
|
+
message=inventory['message'],read_only=True,source='indexed_snapshot',generated_at=time.time(),
|
|
401
|
+
limits={'files':MAX_FILES,'symbols':MAX_SYMBOLS,'edges':MAX_EDGES,'display_nodes_default':120},
|
|
402
|
+
languages=['Java','Angular','Python','JavaScript','TypeScript','JSX','TSX','HTML','CSS','Markdown'],cache_hit=was_cached)
|
|
403
|
+
return _focus(data,file_path) if file_path else data
|