@allansantos-dev/smart-tool 0.0.0-stage → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/LICENSE +202 -0
- package/NOTICE +32 -0
- package/README.md +116 -3
- package/atomic_io.py +94 -0
- package/autostart.py +199 -0
- package/bin/smart-tool.mjs +82 -0
- package/browser_service_client.py +232 -0
- package/capabilities.py +63 -0
- package/client_hooks.py +192 -0
- package/code_graph.py +403 -0
- package/code_graph_java.mjs +49 -0
- package/code_graph_js.cjs +135 -0
- package/config.py +241 -0
- package/daemon_launcher.py +165 -0
- package/directory_watch.py +131 -0
- package/document_text.py +69 -0
- package/duplicates.py +313 -0
- package/embedding_cache.py +99 -0
- package/endpoint_sync.py +179 -0
- package/folder_picker.py +73 -0
- package/freshness.py +278 -0
- package/gateway.py +220 -0
- package/hook_decision.py +96 -0
- package/hooks/pretool_router.py +45 -0
- package/index_inventory.py +75 -0
- package/index_profile.py +167 -0
- package/index_scope.py +458 -0
- package/index_views.py +121 -0
- package/indexer.py +878 -0
- package/install.py +251 -0
- package/install_runtime.py +249 -0
- package/integration_metrics.py +82 -0
- package/local_embedder.py +172 -0
- package/model_adapters/__init__.py +0 -0
- package/model_adapters/openai_compatible.py +57 -0
- package/model_catalog.py +74 -0
- package/model_client.py +38 -0
- package/model_defaults.py +70 -0
- package/package.json +35 -6
- package/paths.py +6 -0
- package/project_identity.py +42 -0
- package/project_insights_ui.py +38 -0
- package/project_maps_ui.py +96 -0
- package/project_monitor.py +204 -0
- package/project_scheduler.py +61 -0
- package/project_storage.py +165 -0
- package/project_store.py +220 -0
- package/projects_ui.py +146 -0
- package/requirements.txt +8 -0
- package/research_cache.py +171 -0
- package/research_metrics.py +110 -0
- package/router.py +582 -0
- package/secret_store.py +121 -0
- package/setup_ui.py +986 -0
- package/site_memory.py +53 -0
- package/smart_tool_daemon.py +3604 -0
- package/tray.py +245 -0
- package/update_check.py +22 -0
- package/usage_meter.py +124 -0
- package/version.py +18 -0
- package/view_compare.py +68 -0
- package/web_adapters/browser/browser_service.py +302 -0
- package/web_adapters/browser/crawl4ai_search.py +236 -0
- package/web_adapters/browser/requirements.txt +2 -0
- package/web_adapters/node/adapter-common.mjs +40 -0
- package/web_adapters/node/ddg-search.mjs +89 -0
- package/web_adapters/node/open-websearch-adapter.mjs +77 -0
- package/web_adapters/node/package-lock.json +2720 -0
- package/web_adapters/node/package.json +12 -0
- package/web_adapters/node/wikipedia-search.mjs +44 -0
- package/web_document_graph.py +71 -0
- package/web_fetch.py +242 -0
- package/web_search_adapters.py +193 -0
- package/web_search_ddgs.py +42 -0
- package/web_search_external.py +288 -0
- package/web_search_health.py +88 -0
package/duplicates.py
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
import array
|
|
2
|
+
import contextvars
|
|
3
|
+
import hashlib
|
|
4
|
+
import heapq
|
|
5
|
+
import math
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
import code_graph
|
|
12
|
+
import embedding_cache
|
|
13
|
+
import index_inventory
|
|
14
|
+
import index_profile
|
|
15
|
+
import index_scope
|
|
16
|
+
import indexer
|
|
17
|
+
|
|
18
|
+
DEFAULT_MIN_SIMILARITY = 0.90
|
|
19
|
+
NEAR_JACCARD = 0.80
|
|
20
|
+
MIN_LINES = 5
|
|
21
|
+
MAX_TEXT = 3000
|
|
22
|
+
MAX_SEMANTIC_FUNCTIONS = 4000
|
|
23
|
+
SEMANTIC_DEADLINE_S = 45
|
|
24
|
+
EMBED_SLICE = 64
|
|
25
|
+
CUTOFF_NOTE = ("Default semantic cutoff 0.90: in the 2026-10-02 measurement (claude-code, independent judge) pairs above "
|
|
26
|
+
"0.95 were 10/10 actionable and those from 0.80 to 0.94 only 7/20; below 0.90 most are false positives "
|
|
27
|
+
"and reviewing them costs time without leading to a refactor.")
|
|
28
|
+
_HASH_COMMENT = {"py", "rb", "sh"}
|
|
29
|
+
_TOKEN = re.compile(r"[A-Za-z_]\w*|\d+|[^\s\w]")
|
|
30
|
+
_CONTAINER_KINDS = {"module", "template", "class", "interface"}
|
|
31
|
+
_GENERIC_NAMES = {"callback", "anonymous", "constructor"}
|
|
32
|
+
_WARMING = set()
|
|
33
|
+
_WARM_ERRORS = {}
|
|
34
|
+
LAST_FINDINGS = {}
|
|
35
|
+
MAX_REMEMBERED_FINDINGS = 20000
|
|
36
|
+
NOTE_MAX = 500
|
|
37
|
+
TYPE_BY_PREFIX = {"e": "exact", "n": "near", "s": "semantic"}
|
|
38
|
+
DISMISS_REASONS = {"false_positive", "intentional"}
|
|
39
|
+
_LOCK = threading.Lock()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _strip_comments(text, ext):
|
|
43
|
+
hash_comments = ext in _HASH_COMMENT
|
|
44
|
+
out, i, n = [], 0, len(text)
|
|
45
|
+
while i < n:
|
|
46
|
+
ch = text[i]
|
|
47
|
+
if ch in "'\"`":
|
|
48
|
+
j = i + 1
|
|
49
|
+
while j < n and text[j] != ch:
|
|
50
|
+
j += 2 if text[j] == "\\" else 1
|
|
51
|
+
out.append(text[i:j + 1]); i = j + 1
|
|
52
|
+
elif hash_comments and ch == "#":
|
|
53
|
+
while i < n and text[i] != "\n":
|
|
54
|
+
i += 1
|
|
55
|
+
elif not hash_comments and text.startswith("//", i):
|
|
56
|
+
while i < n and text[i] != "\n":
|
|
57
|
+
i += 1
|
|
58
|
+
elif not hash_comments and text.startswith("/*", i):
|
|
59
|
+
end = text.find("*/", i + 2)
|
|
60
|
+
i = n if end < 0 else end + 2
|
|
61
|
+
else:
|
|
62
|
+
out.append(ch); i += 1
|
|
63
|
+
return "".join(out)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _body(text, ext):
|
|
67
|
+
if ext in _HASH_COMMENT:
|
|
68
|
+
lines = text.splitlines()
|
|
69
|
+
start = next((k for k, line in enumerate(lines) if re.match(r"\s*(async\s+)?def\s", line)), 0)
|
|
70
|
+
end = next((k for k in range(start, len(lines)) if lines[k].rstrip().endswith(":")), start)
|
|
71
|
+
return "\n".join(lines[end + 1:])
|
|
72
|
+
brace = text.find("{")
|
|
73
|
+
return text[brace + 1:] if brace >= 0 else text
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _shingles(text):
|
|
77
|
+
tokens = ["ID" if re.match(r"[A-Za-z_]", t) else t for t in _TOKEN.findall(text)]
|
|
78
|
+
return frozenset(tuple(tokens[i:i + 5]) for i in range(max(len(tokens) - 4, 1)))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _index_state(path):
|
|
82
|
+
stat = os.stat(path)
|
|
83
|
+
return stat.st_mtime_ns, stat.st_size
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _functions(root, include_tests):
|
|
87
|
+
inventory = index_inventory.inspect(root)
|
|
88
|
+
if not inventory["selected"]:
|
|
89
|
+
raise ValueError("Project not indexed in this view yet; run a search or index it first.")
|
|
90
|
+
view = inventory["selected"]
|
|
91
|
+
before = _index_state(view["path"])
|
|
92
|
+
graph = code_graph.build(root, storage_id=os.path.basename(view["path"]))
|
|
93
|
+
_files, sources, _diagnostics, _truncated = code_graph._snapshot(view["path"])
|
|
94
|
+
if _index_state(view["path"]) != before:
|
|
95
|
+
raise RuntimeError("The index changed during the analysis (reindex in progress); try again shortly.")
|
|
96
|
+
profile = index_profile.current((index_scope.load_scope(root) or {}).get("profile"))
|
|
97
|
+
candidates = [s for s in graph["symbols"] if s.get("kind") not in _CONTAINER_KINDS
|
|
98
|
+
and s["end_line"] - s["start_line"] + 1 >= MIN_LINES
|
|
99
|
+
and s["name"].split("(")[0].split(".")[-1] not in _GENERIC_NAMES and "@" not in s["name"]]
|
|
100
|
+
by_file = {}
|
|
101
|
+
for s in candidates:
|
|
102
|
+
by_file.setdefault(s["path"], []).append(s)
|
|
103
|
+
functions = []
|
|
104
|
+
for path, symbols in by_file.items():
|
|
105
|
+
kind = index_profile.kind(path, profile)
|
|
106
|
+
if kind == "doc" or (kind == "test" and not include_tests):
|
|
107
|
+
continue
|
|
108
|
+
ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
|
|
109
|
+
file_lines = sources.get(path, "").splitlines()
|
|
110
|
+
occurrences = {}
|
|
111
|
+
for s in sorted(symbols, key=lambda item: (item["start_line"], item["end_line"])):
|
|
112
|
+
if any(o is not s and o["start_line"] <= s["start_line"] and s["end_line"] <= o["end_line"]
|
|
113
|
+
and (o["start_line"], o["end_line"]) != (s["start_line"], s["end_line"]) for o in symbols):
|
|
114
|
+
continue
|
|
115
|
+
text = "\n".join(file_lines[s["start_line"] - 1:s["end_line"]])
|
|
116
|
+
if not text.strip():
|
|
117
|
+
continue
|
|
118
|
+
normalized = re.sub(r"\s+", " ", _strip_comments(_body(text, ext), ext)).strip()
|
|
119
|
+
functions.append({"name": s["name"].split("(")[0].split(".")[-1], "path": path, "line": s["start_line"],
|
|
120
|
+
"end": s["end_line"], "lines": s["end_line"] - s["start_line"] + 1, "text": text[:MAX_TEXT],
|
|
121
|
+
"hash": hashlib.sha1(normalized.encode("utf-8")).hexdigest(), "shingles": _shingles(normalized)})
|
|
122
|
+
occurrence = occurrences[functions[-1]["hash"]] = occurrences.get(functions[-1]["hash"], 0) + 1
|
|
123
|
+
functions[-1]["kind"] = kind
|
|
124
|
+
functions[-1]["fp"] = hashlib.sha1(f"{path}\0{functions[-1]['hash']}\0{occurrence}".encode("utf-8")).hexdigest()
|
|
125
|
+
notes = []
|
|
126
|
+
if inventory.get("message"):
|
|
127
|
+
notes.append(inventory["message"])
|
|
128
|
+
if graph.get("truncated"):
|
|
129
|
+
notes.append("Graph truncated by analysis limits; some functions were left out.")
|
|
130
|
+
if graph.get("retryable"):
|
|
131
|
+
notes.append("Partial analysis: " + "; ".join(d.get("reason", "") for d in graph.get("diagnostics", [])[:2]))
|
|
132
|
+
return functions, notes, view
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _ref(f):
|
|
136
|
+
return {"name": f["name"], "path": f["path"], "lines": f"{f['line']}-{f['end']}"}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _finding_id(prefix, members):
|
|
140
|
+
production = [f for f in members if f["kind"] != "test"]
|
|
141
|
+
basis = production if len(production) >= 2 else members
|
|
142
|
+
return prefix + hashlib.sha1("|".join(sorted(f["fp"] for f in basis)).encode("utf-8")).hexdigest()[:16]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _cache(root, view):
|
|
146
|
+
if not view.get("model") or not view.get("dimensions"):
|
|
147
|
+
raise ValueError("Index has no registered embedding model; reindex the project.")
|
|
148
|
+
return embedding_cache.VectorCache(root, indexer.INDEX_DIR, {"purpose": "function_duplicates", "model": view["model"],
|
|
149
|
+
"dimensions": view["dimensions"]})
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _cached_vectors(root, view, functions):
|
|
153
|
+
cache = _cache(root, view)
|
|
154
|
+
try:
|
|
155
|
+
vectors, missing = [], 0
|
|
156
|
+
for f in functions:
|
|
157
|
+
row = cache.db.execute("SELECT vector FROM vectors WHERE key=?", (cache.key(f["text"]),)).fetchone()
|
|
158
|
+
if row is None:
|
|
159
|
+
missing += 1
|
|
160
|
+
continue
|
|
161
|
+
vector = array.array("f"); vector.frombytes(row[0])
|
|
162
|
+
norm = math.sqrt(indexer._sumprod(vector, vector))
|
|
163
|
+
vectors.append(array.array("f", (x / norm for x in vector)) if norm else None)
|
|
164
|
+
return (vectors if not missing else None), missing
|
|
165
|
+
finally:
|
|
166
|
+
cache.close()
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _warm(root, view, functions, embed):
|
|
170
|
+
try:
|
|
171
|
+
cache = _cache(root, view)
|
|
172
|
+
try:
|
|
173
|
+
for start in range(0, len(functions), EMBED_SLICE):
|
|
174
|
+
cache.embed([f["text"] for f in functions[start:start + EMBED_SLICE]],
|
|
175
|
+
lambda texts: embed(view["model"], texts), indexer.validate_vector)
|
|
176
|
+
finally:
|
|
177
|
+
cache.close()
|
|
178
|
+
except Exception as exc:
|
|
179
|
+
with _LOCK:
|
|
180
|
+
_WARM_ERRORS[root] = f"{type(exc).__name__}: {exc}"[:300]
|
|
181
|
+
finally:
|
|
182
|
+
with _LOCK:
|
|
183
|
+
_WARMING.discard(root)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _near(functions):
|
|
187
|
+
order = sorted(range(len(functions)), key=lambda i: len(functions[i]["shingles"]))
|
|
188
|
+
found = []
|
|
189
|
+
for position, a in enumerate(order):
|
|
190
|
+
sa = functions[a]["shingles"]
|
|
191
|
+
for b in order[position + 1:]:
|
|
192
|
+
sb = functions[b]["shingles"]
|
|
193
|
+
if len(sb) * NEAR_JACCARD > len(sa):
|
|
194
|
+
break
|
|
195
|
+
if functions[a]["hash"] == functions[b]["hash"]:
|
|
196
|
+
continue
|
|
197
|
+
inter = len(sa & sb)
|
|
198
|
+
score = inter / (len(sa) + len(sb) - inter)
|
|
199
|
+
if score >= NEAR_JACCARD:
|
|
200
|
+
found.append((score, a, b))
|
|
201
|
+
found.sort(reverse=True)
|
|
202
|
+
return found
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _semantic(root, view, functions, embed, configured_model, min_similarity, limit, lexical, skip):
|
|
206
|
+
if embed is None or not configured_model:
|
|
207
|
+
return "unavailable", [], 0, ["Semantic level unavailable: no embedding_model configured."]
|
|
208
|
+
if configured_model != view.get("model"):
|
|
209
|
+
return "unavailable", [], 0, [f"Semantic level unavailable: the configured model ({configured_model}) differs from the "
|
|
210
|
+
f"index model ({view.get('model')}); reindex the project to compare vectors from the same model."]
|
|
211
|
+
pool = sorted(range(len(functions)), key=lambda i: -functions[i]["lines"])[:MAX_SEMANTIC_FUNCTIONS]
|
|
212
|
+
notes = []
|
|
213
|
+
if len(functions) > MAX_SEMANTIC_FUNCTIONS:
|
|
214
|
+
notes.append(f"Semantic level limited to the {MAX_SEMANTIC_FUNCTIONS} largest of {len(functions)} functions.")
|
|
215
|
+
chosen = [functions[i] for i in pool]
|
|
216
|
+
with _LOCK:
|
|
217
|
+
error = _WARM_ERRORS.pop(root, None)
|
|
218
|
+
vectors, missing = _cached_vectors(root, view, chosen)
|
|
219
|
+
if vectors is None:
|
|
220
|
+
with _LOCK:
|
|
221
|
+
start = root not in _WARMING
|
|
222
|
+
if start:
|
|
223
|
+
_WARMING.add(root)
|
|
224
|
+
if start:
|
|
225
|
+
context = contextvars.copy_context()
|
|
226
|
+
threading.Thread(target=context.run, args=(_warm, root, view, chosen, embed), daemon=True).start()
|
|
227
|
+
pending = f"{missing} functions without embedding" if missing else "preparation in progress"
|
|
228
|
+
notes.append(f"Semantic level being prepared ({pending}); available on the next call."
|
|
229
|
+
+ (f" The last preparation failed: {error}" if error else ""))
|
|
230
|
+
return "preparing", [], 0, notes
|
|
231
|
+
heap, total, deadline = [], 0, time.monotonic() + SEMANTIC_DEADLINE_S
|
|
232
|
+
for a in range(len(chosen)):
|
|
233
|
+
if time.monotonic() > deadline:
|
|
234
|
+
notes.append(f"Semantic comparison stopped at the {SEMANTIC_DEADLINE_S} s deadline; partial result.")
|
|
235
|
+
break
|
|
236
|
+
va, fa = vectors[a], chosen[a]
|
|
237
|
+
if va is None:
|
|
238
|
+
continue
|
|
239
|
+
for b in range(a + 1, len(chosen)):
|
|
240
|
+
vb, fb = vectors[b], chosen[b]
|
|
241
|
+
if vb is None or fa["hash"] == fb["hash"] or (pool[a], pool[b]) in lexical or (pool[b], pool[a]) in lexical:
|
|
242
|
+
continue
|
|
243
|
+
score = indexer._sumprod(va, vb)
|
|
244
|
+
if score >= min_similarity:
|
|
245
|
+
if _finding_id("s", (fa, fb)) in skip:
|
|
246
|
+
skip[_finding_id("s", (fa, fb))] = (score, fa, fb)
|
|
247
|
+
continue
|
|
248
|
+
total += 1
|
|
249
|
+
item = (score, pool[a], pool[b])
|
|
250
|
+
if len(heap) < limit:
|
|
251
|
+
heapq.heappush(heap, item)
|
|
252
|
+
elif item > heap[0]:
|
|
253
|
+
heapq.heapreplace(heap, item)
|
|
254
|
+
return "ready", sorted(heap, reverse=True), total, notes
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def find(root, embed, configured_model, min_similarity=DEFAULT_MIN_SIMILARITY, include_tests=False, limit=30,
|
|
258
|
+
dismissed=None, include_dismissed=False):
|
|
259
|
+
started = time.monotonic()
|
|
260
|
+
dismissed = dismissed or {}
|
|
261
|
+
functions, notes, view = _functions(root, include_tests)
|
|
262
|
+
groups = {}
|
|
263
|
+
for f in functions:
|
|
264
|
+
groups.setdefault(f["hash"], []).append(f)
|
|
265
|
+
exact_all = sorted((g for g in groups.values() if len(g) > 1), key=lambda g: (-g[0]["lines"], -len(g)))
|
|
266
|
+
near_all = _near(functions)
|
|
267
|
+
lexical = {(a, b) for _s, a, b in near_all}
|
|
268
|
+
hidden = []
|
|
269
|
+
exact = []
|
|
270
|
+
for g in exact_all:
|
|
271
|
+
fid = _finding_id("e", g)
|
|
272
|
+
(hidden.append(("exact", fid, None, g)) if fid in dismissed else exact.append((fid, g)))
|
|
273
|
+
near = []
|
|
274
|
+
for s, a, b in near_all:
|
|
275
|
+
fid = _finding_id("n", (functions[a], functions[b]))
|
|
276
|
+
(hidden.append(("near", fid, s, (functions[a], functions[b]))) if fid in dismissed else near.append((fid, s, a, b)))
|
|
277
|
+
skip = {fid: None for fid in dismissed if fid.startswith("s")}
|
|
278
|
+
try:
|
|
279
|
+
status, semantic, semantic_total, semantic_notes = _semantic(root, view, functions, embed, configured_model,
|
|
280
|
+
min_similarity, limit, lexical, skip)
|
|
281
|
+
except Exception as exc:
|
|
282
|
+
status, semantic, semantic_total, semantic_notes = "error", [], 0, [f"Semantic level failed: {type(exc).__name__}: {exc}"[:300]]
|
|
283
|
+
notes += semantic_notes
|
|
284
|
+
if min_similarity < DEFAULT_MIN_SIMILARITY:
|
|
285
|
+
notes.append(f"Semantic cutoff below the default ({min_similarity:.2f}): expect many false positives.")
|
|
286
|
+
semantic_rows = [(_finding_id("s", (functions[a], functions[b])), s, a, b) for s, a, b in semantic]
|
|
287
|
+
hidden_semantic = [fid for fid, score in skip.items() if score is not None]
|
|
288
|
+
with _LOCK:
|
|
289
|
+
remembered = LAST_FINDINGS.setdefault(root, {})
|
|
290
|
+
remembered.update({fid: "exact" for fid, _g in exact})
|
|
291
|
+
remembered.update({fid: "near" for fid, *_r in near})
|
|
292
|
+
remembered.update({fid: "semantic" for fid, *_r in semantic_rows})
|
|
293
|
+
remembered.update({fid: kind for kind, fid, *_r in hidden})
|
|
294
|
+
remembered.update({fid: "semantic" for fid in hidden_semantic})
|
|
295
|
+
while len(remembered) > MAX_REMEMBERED_FINDINGS:
|
|
296
|
+
remembered.pop(next(iter(remembered)))
|
|
297
|
+
result = {
|
|
298
|
+
"view": view["label"], "functions_analyzed": len(functions), "include_tests": include_tests,
|
|
299
|
+
"min_similarity": min_similarity, "cutoff_note": CUTOFF_NOTE, "semantic_status": status, "notes": notes,
|
|
300
|
+
"counts": {"exact_groups": len(exact), "near_pairs": len(near), "semantic_pairs": semantic_total,
|
|
301
|
+
"dismissed_hidden": len(hidden) + len(hidden_semantic)},
|
|
302
|
+
"exact": [{"id": fid, "lines": g[0]["lines"], "copies": [_ref(f) for f in g]} for fid, g in exact[:limit]],
|
|
303
|
+
"near": [{"id": fid, "score": round(s, 3), "a": _ref(functions[a]), "b": _ref(functions[b])} for fid, s, a, b in near[:limit]],
|
|
304
|
+
"semantic": [{"id": fid, "score": round(s, 3), "a": _ref(functions[a]), "b": _ref(functions[b])} for fid, s, a, b in semantic_rows],
|
|
305
|
+
"elapsed_s": round(time.monotonic() - started, 1),
|
|
306
|
+
}
|
|
307
|
+
if include_dismissed:
|
|
308
|
+
result["dismissed"] = ([{"id": fid, "type": kind, **({"score": round(s, 3)} if s is not None else {}),
|
|
309
|
+
"members": [_ref(f) for f in members], **dismissed[fid]} for kind, fid, s, members in hidden]
|
|
310
|
+
+ [{"id": fid, "type": "semantic", "score": round(skip[fid][0], 3),
|
|
311
|
+
"members": [_ref(skip[fid][1]), _ref(skip[fid][2])], **dismissed[fid]}
|
|
312
|
+
for fid in hidden_semantic])
|
|
313
|
+
return result
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Reúso de vetores por texto exato e contrato; não guarda texto nem faz rede."""
|
|
2
|
+
import array
|
|
3
|
+
import concurrent.futures
|
|
4
|
+
import contextvars
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import sqlite3
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
import project_identity
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class VectorCache:
|
|
16
|
+
def __init__(self, root, directory, contract):
|
|
17
|
+
os.makedirs(directory, exist_ok=True)
|
|
18
|
+
path = os.path.join(directory, project_identity.project_id(root) + '.vectors.sqlite3')
|
|
19
|
+
self.db = sqlite3.connect(path, timeout=10)
|
|
20
|
+
self.db.execute('PRAGMA secure_delete=ON')
|
|
21
|
+
self.db.execute('CREATE TABLE IF NOT EXISTS vectors(key TEXT PRIMARY KEY, vector BLOB NOT NULL, used_at REAL NOT NULL)')
|
|
22
|
+
self.contract = json.dumps(contract, sort_keys=True, ensure_ascii=False)
|
|
23
|
+
self.dimensions = contract.get('dimensions')
|
|
24
|
+
self.hits = self.misses = self.characters = 0
|
|
25
|
+
|
|
26
|
+
def key(self, text):
|
|
27
|
+
return hashlib.sha256((self.contract + '\0' + text).encode('utf-8')).hexdigest()
|
|
28
|
+
|
|
29
|
+
def put(self, text, vector):
|
|
30
|
+
if not self.dimensions or len(vector) != self.dimensions:
|
|
31
|
+
return
|
|
32
|
+
packed = array.array('f', vector)
|
|
33
|
+
if not all(math.isfinite(value) for value in packed):
|
|
34
|
+
raise ValueError('Embedding does not fit in float32; cache not written.')
|
|
35
|
+
self.db.execute('INSERT OR REPLACE INTO vectors VALUES(?,?,?)',
|
|
36
|
+
(self.key(text), packed.tobytes(), time.time()))
|
|
37
|
+
|
|
38
|
+
def embed(self, texts, embed_batch, validate, use_cache=True, split=None, workers=1):
|
|
39
|
+
"""Cache lookups and writes stay on the calling thread (sqlite); only embed_batch runs in workers."""
|
|
40
|
+
output = [None] * len(texts)
|
|
41
|
+
missing = {}
|
|
42
|
+
for i, text in enumerate(texts):
|
|
43
|
+
row = self.db.execute('SELECT vector FROM vectors WHERE key=?', (self.key(text),)).fetchone() if use_cache else None
|
|
44
|
+
if row and self.dimensions:
|
|
45
|
+
vector = array.array('f'); vector.frombytes(row[0])
|
|
46
|
+
validate(vector, self.dimensions)
|
|
47
|
+
output[i] = list(vector)
|
|
48
|
+
self.hits += 1
|
|
49
|
+
else:
|
|
50
|
+
missing.setdefault(text, []).append(i)
|
|
51
|
+
if missing:
|
|
52
|
+
unique = list(missing)
|
|
53
|
+
batches = split(unique) if split else [unique]
|
|
54
|
+
dimensions = [self.dimensions]
|
|
55
|
+
|
|
56
|
+
def store(part, vectors):
|
|
57
|
+
# Each batch is cached as soon as it returns: a later failure never discards paid embeddings.
|
|
58
|
+
if len(vectors) != len(part):
|
|
59
|
+
raise ValueError('Vector count differs from the chunks sent.')
|
|
60
|
+
dimensions[0] = dimensions[0] or (len(vectors[0]) if vectors else None)
|
|
61
|
+
for vector in vectors:
|
|
62
|
+
validate(vector, dimensions[0])
|
|
63
|
+
for text, vector in zip(part, vectors):
|
|
64
|
+
self.put(text, vector)
|
|
65
|
+
positions = missing[text]
|
|
66
|
+
for i in positions:
|
|
67
|
+
output[i] = vector
|
|
68
|
+
self.hits += len(positions) - 1
|
|
69
|
+
self.misses += len(part)
|
|
70
|
+
self.characters += sum(map(len, part))
|
|
71
|
+
self.db.commit()
|
|
72
|
+
|
|
73
|
+
if workers > 1 and len(batches) > 1:
|
|
74
|
+
pool = concurrent.futures.ThreadPoolExecutor(min(workers, len(batches)))
|
|
75
|
+
futures = {pool.submit(contextvars.copy_context().run, embed_batch, part): part for part in batches}
|
|
76
|
+
error = None
|
|
77
|
+
try:
|
|
78
|
+
for future in concurrent.futures.as_completed(futures):
|
|
79
|
+
if future.cancelled():
|
|
80
|
+
continue
|
|
81
|
+
try:
|
|
82
|
+
store(futures[future], list(future.result()))
|
|
83
|
+
except Exception as exc:
|
|
84
|
+
if error is None:
|
|
85
|
+
error = exc
|
|
86
|
+
for other in futures:
|
|
87
|
+
other.cancel()
|
|
88
|
+
finally:
|
|
89
|
+
pool.shutdown(wait=True, cancel_futures=True)
|
|
90
|
+
if error is not None:
|
|
91
|
+
raise error
|
|
92
|
+
else:
|
|
93
|
+
for part in batches:
|
|
94
|
+
store(part, list(embed_batch(part)))
|
|
95
|
+
return output
|
|
96
|
+
|
|
97
|
+
def close(self):
|
|
98
|
+
self.db.commit()
|
|
99
|
+
self.db.close()
|
package/endpoint_sync.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Aponta os clientes para a porta em que o daemon subiu, só quando ela muda.
|
|
2
|
+
|
|
3
|
+
O daemon é dono da porta: depois do bind, reescreve apenas as URLs do smart-tool que os clientes guardam (MCP do
|
|
4
|
+
Claude Code em ~/.claude.json, hook HTTP em ~/.claude/settings.json, MCP do Codex em ~/.codex/config.toml). Nada roda
|
|
5
|
+
por prompt ou por sessão; o Claude Code relê hooks editados pelo file watcher, o MCP já conectado precisa de /mcp.
|
|
6
|
+
Cada arquivo é relido logo antes da gravação atômica: o Claude Code reescreve o ~/.claude.json com frequência.
|
|
7
|
+
"""
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import shutil
|
|
12
|
+
import subprocess
|
|
13
|
+
import time
|
|
14
|
+
import urllib.parse
|
|
15
|
+
|
|
16
|
+
SERVER = "smart-tool"
|
|
17
|
+
HOOK_PATH = "/hooks/pretool"
|
|
18
|
+
_CODEX_URL_RE = re.compile(r'(\[mcp_servers\.smart-tool\][^\[]*?\burl\s*=\s*")http://127\.0\.0\.1:\d+(/mcp")')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _home(*parts):
|
|
22
|
+
return os.path.join(os.path.expanduser("~"), *parts)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def claude_dir():
|
|
26
|
+
"""Claude Code keeps settings.json in CLAUDE_CONFIG_DIR when set, else ~/.claude."""
|
|
27
|
+
return os.environ.get("CLAUDE_CONFIG_DIR") or _home(".claude")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def claude_json():
|
|
31
|
+
"""User-scope MCP servers: $CLAUDE_CONFIG_DIR/.claude.json when set, else ~/.claude.json."""
|
|
32
|
+
custom = os.environ.get("CLAUDE_CONFIG_DIR")
|
|
33
|
+
return os.path.join(custom, ".claude.json") if custom else _home(".claude.json")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _rebase(url, base):
|
|
37
|
+
parsed = urllib.parse.urlsplit(url)
|
|
38
|
+
if parsed.hostname != "127.0.0.1":
|
|
39
|
+
return url
|
|
40
|
+
return urllib.parse.urlunsplit(urllib.parse.urlsplit(base)[:2] + parsed[2:])
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _claude_mcp(text, base):
|
|
44
|
+
data = json.loads(text)
|
|
45
|
+
entry = (data.get("mcpServers") or {}).get(SERVER) if isinstance(data, dict) else None
|
|
46
|
+
if not isinstance(entry, dict) or not isinstance(entry.get("url"), str):
|
|
47
|
+
return None
|
|
48
|
+
new = _rebase(entry["url"], base)
|
|
49
|
+
if new == entry["url"]:
|
|
50
|
+
return None
|
|
51
|
+
entry["url"] = new
|
|
52
|
+
return json.dumps(data, indent=2, ensure_ascii=False)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _claude_hook(text, base):
|
|
56
|
+
data = json.loads(text)
|
|
57
|
+
changed = False
|
|
58
|
+
groups = ((data.get("hooks") or {}).get("PreToolUse") or []) if isinstance(data, dict) else []
|
|
59
|
+
for group in groups if isinstance(groups, list) else []:
|
|
60
|
+
for hook in (group.get("hooks") or []) if isinstance(group, dict) else []:
|
|
61
|
+
if isinstance(hook, dict) and HOOK_PATH in str(hook.get("url", "")):
|
|
62
|
+
new = _rebase(hook["url"], base)
|
|
63
|
+
changed, hook["url"] = changed or new != hook["url"], new
|
|
64
|
+
return json.dumps(data, indent=2, ensure_ascii=False) if changed else None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _codex_mcp(text, base):
|
|
68
|
+
port = urllib.parse.urlsplit(base).port
|
|
69
|
+
new = _CODEX_URL_RE.sub(lambda m: f"{m.group(1)}http://127.0.0.1:{port}{m.group(2)}", text, count=1)
|
|
70
|
+
return new if new != text else None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def targets():
|
|
74
|
+
return ((claude_json(), _claude_mcp), (os.path.join(claude_dir(), "settings.json"), _claude_hook),
|
|
75
|
+
(_home(".codex", "config.toml"), _codex_mcp))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
_CODEX_ANY_URL_RE = re.compile(r'\[mcp_servers\.smart-tool\][^\[]*?\burl\s*=\s*"([^"]+)"')
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def registrations():
|
|
82
|
+
"""smart-tool MCP URL kept by each client, or None when it is not registered there."""
|
|
83
|
+
found = {"claude": None, "codex": None}
|
|
84
|
+
for attempt in range(2):
|
|
85
|
+
try:
|
|
86
|
+
with open(claude_json(), encoding="utf-8") as stream:
|
|
87
|
+
entry = (json.load(stream).get("mcpServers") or {}).get(SERVER)
|
|
88
|
+
found["claude"] = entry.get("url") if isinstance(entry, dict) else None
|
|
89
|
+
break
|
|
90
|
+
except ValueError:
|
|
91
|
+
# Claude Code rewrites the file all the time; one more read gets past a half-written copy.
|
|
92
|
+
time.sleep(0.2)
|
|
93
|
+
except (OSError, AttributeError):
|
|
94
|
+
break
|
|
95
|
+
try:
|
|
96
|
+
with open(_home(".codex", "config.toml"), encoding="utf-8") as stream:
|
|
97
|
+
match = _CODEX_ANY_URL_RE.search(stream.read())
|
|
98
|
+
found["codex"] = match.group(1) if match else None
|
|
99
|
+
except OSError:
|
|
100
|
+
pass
|
|
101
|
+
return found
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
REGISTER_TIMEOUT_S = 60
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def register_command(client, base):
|
|
108
|
+
"""Official CLI command that registers the smart-tool MCP server (Claude Code: user scope, every project)."""
|
|
109
|
+
url = f"{base}/mcp"
|
|
110
|
+
if client == "claude":
|
|
111
|
+
return ["claude", "mcp", "add", "--transport", "http", SERVER, url, "--scope", "user"]
|
|
112
|
+
if client == "codex":
|
|
113
|
+
return ["codex", "mcp", "add", SERVER, "--url", url]
|
|
114
|
+
raise ValueError(f"Client without MCP registration: {client!r}")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def register(client, base):
|
|
118
|
+
"""Runs the client's own CLI (it owns its config file; ~/.claude.json is rewritten by Claude Code all the time)."""
|
|
119
|
+
command = register_command(client, base)
|
|
120
|
+
executable = shutil.which(command[0])
|
|
121
|
+
if not executable:
|
|
122
|
+
raise RuntimeError(f"CLI '{command[0]}' not found in Smart Tool's PATH. Register it manually: {' '.join(command)}")
|
|
123
|
+
flags = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0
|
|
124
|
+
try:
|
|
125
|
+
proc = subprocess.Popen([executable, *command[1:]], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
126
|
+
stdin=subprocess.DEVNULL, text=True, encoding="utf-8", errors="replace",
|
|
127
|
+
creationflags=flags)
|
|
128
|
+
except OSError as exc:
|
|
129
|
+
raise RuntimeError(f"'{' '.join(command)}' did not start: {exc}") from None
|
|
130
|
+
try:
|
|
131
|
+
stdout, stderr = proc.communicate(timeout=REGISTER_TIMEOUT_S)
|
|
132
|
+
except subprocess.TimeoutExpired:
|
|
133
|
+
# npm shims are cmd.exe -> node: killing only cmd.exe leaves node holding the pipes forever.
|
|
134
|
+
if os.name == "nt":
|
|
135
|
+
subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, creationflags=flags)
|
|
136
|
+
else:
|
|
137
|
+
proc.kill()
|
|
138
|
+
proc.communicate(timeout=10)
|
|
139
|
+
raise RuntimeError(f"'{' '.join(command)}' did not finish in {REGISTER_TIMEOUT_S}s.") from None
|
|
140
|
+
if proc.returncode != 0:
|
|
141
|
+
detail = (stderr or stdout).strip()[-400:]
|
|
142
|
+
raise RuntimeError(f"'{' '.join(command)}' failed (code {proc.returncode}): {detail}")
|
|
143
|
+
if registrations().get(client) != f"{base}/mcp":
|
|
144
|
+
raise RuntimeError(f"'{' '.join(command)}' finished, but the registration did not appear in the client's configuration.")
|
|
145
|
+
return " ".join(command)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def known_port():
|
|
149
|
+
"""Port the Claude Code MCP entry already points to, so a fallback bind can reuse it without any rewrite."""
|
|
150
|
+
try:
|
|
151
|
+
parsed = urllib.parse.urlsplit(registrations()["claude"] or "")
|
|
152
|
+
return parsed.port if parsed.hostname == "127.0.0.1" else None
|
|
153
|
+
except ValueError:
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def sync(base):
|
|
158
|
+
"""Rewrites the smart-tool URLs that point to another local port. Returns the files changed."""
|
|
159
|
+
changed = []
|
|
160
|
+
for path, rewrite in targets():
|
|
161
|
+
for _ in range(2):
|
|
162
|
+
try:
|
|
163
|
+
with open(path, encoding="utf-8") as stream:
|
|
164
|
+
text = stream.read()
|
|
165
|
+
except FileNotFoundError:
|
|
166
|
+
break
|
|
167
|
+
new = rewrite(text, base)
|
|
168
|
+
if new is None:
|
|
169
|
+
break
|
|
170
|
+
with open(path, encoding="utf-8") as stream:
|
|
171
|
+
if stream.read() != text:
|
|
172
|
+
continue
|
|
173
|
+
temporary = f"{path}.smart-tool-tmp"
|
|
174
|
+
with open(temporary, "w", encoding="utf-8", newline="") as stream:
|
|
175
|
+
stream.write(new)
|
|
176
|
+
os.replace(temporary, path)
|
|
177
|
+
changed.append(path)
|
|
178
|
+
break
|
|
179
|
+
return changed
|
package/folder_picker.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Seletor nativo de pasta, compartilhado pelo tray e pela tela de projetos."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import threading
|
|
6
|
+
import urllib.request
|
|
7
|
+
import webbrowser
|
|
8
|
+
|
|
9
|
+
_PICKER_LOCK = threading.Lock()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def choose_folder():
|
|
13
|
+
if os.name != "nt":
|
|
14
|
+
raise RuntimeError("Native folder selection is only available on Windows; enter the folder path.")
|
|
15
|
+
if not _PICKER_LOCK.acquire(blocking=False):
|
|
16
|
+
raise ValueError("A folder picker is already open.")
|
|
17
|
+
import ctypes
|
|
18
|
+
from ctypes import wintypes
|
|
19
|
+
|
|
20
|
+
class BrowseInfo(ctypes.Structure):
|
|
21
|
+
_fields_ = [("hwndOwner", wintypes.HWND), ("pidlRoot", ctypes.c_void_p),
|
|
22
|
+
("pszDisplayName", wintypes.LPWSTR), ("lpszTitle", wintypes.LPCWSTR),
|
|
23
|
+
("ulFlags", wintypes.UINT), ("lpfn", ctypes.c_void_p),
|
|
24
|
+
("lParam", wintypes.LPARAM), ("iImage", ctypes.c_int)]
|
|
25
|
+
|
|
26
|
+
shell, ole = ctypes.windll.shell32, ctypes.windll.ole32
|
|
27
|
+
shell.SHBrowseForFolderW.argtypes = [ctypes.POINTER(BrowseInfo)]
|
|
28
|
+
shell.SHBrowseForFolderW.restype = ctypes.c_void_p
|
|
29
|
+
shell.SHGetPathFromIDListEx.argtypes = [ctypes.c_void_p, wintypes.LPWSTR, wintypes.DWORD, wintypes.DWORD]
|
|
30
|
+
shell.SHGetPathFromIDListEx.restype = wintypes.BOOL
|
|
31
|
+
ole.CoTaskMemFree.argtypes = [ctypes.c_void_p]
|
|
32
|
+
ole.CoInitializeEx.argtypes = [ctypes.c_void_p, wintypes.DWORD]
|
|
33
|
+
ole.CoInitializeEx.restype = ctypes.c_long
|
|
34
|
+
initialized = ole.CoInitializeEx(None, 2) in (0, 1)
|
|
35
|
+
selected = None
|
|
36
|
+
try:
|
|
37
|
+
label = ctypes.create_unicode_buffer(32768)
|
|
38
|
+
info = BrowseInfo()
|
|
39
|
+
info.pszDisplayName = ctypes.cast(label, wintypes.LPWSTR)
|
|
40
|
+
info.lpszTitle = "Smart Tool — select the folder to index"
|
|
41
|
+
info.ulFlags = 0x0001 | 0x0040 # Diretórios reais; diálogo redimensionável.
|
|
42
|
+
selected = shell.SHBrowseForFolderW(ctypes.byref(info))
|
|
43
|
+
if not selected:
|
|
44
|
+
return None
|
|
45
|
+
path = ctypes.create_unicode_buffer(32768)
|
|
46
|
+
if not shell.SHGetPathFromIDListEx(selected, path, len(path), 0):
|
|
47
|
+
raise ValueError("The selection is not an accessible folder.")
|
|
48
|
+
return path.value
|
|
49
|
+
finally:
|
|
50
|
+
if selected:
|
|
51
|
+
ole.CoTaskMemFree(selected)
|
|
52
|
+
if initialized:
|
|
53
|
+
ole.CoUninitialize()
|
|
54
|
+
_PICKER_LOCK.release()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def register_from_tray():
|
|
58
|
+
import daemon_launcher
|
|
59
|
+
root = choose_folder()
|
|
60
|
+
if not root:
|
|
61
|
+
return
|
|
62
|
+
url = daemon_launcher.ensure_daemon_running().rstrip("/")
|
|
63
|
+
with urllib.request.urlopen(url + "/projects", timeout=10) as response:
|
|
64
|
+
page = response.read().decode("utf-8")
|
|
65
|
+
match = re.search(r'<meta name="setup-token" content="([a-zA-Z0-9_-]+)">', page)
|
|
66
|
+
if not match:
|
|
67
|
+
raise RuntimeError("Reopen Smart Tool to load the updated projects page.")
|
|
68
|
+
request = urllib.request.Request(url + "/setup/projects", method="POST",
|
|
69
|
+
headers={"Content-Type": "application/json", "X-Setup-Token": match.group(1)},
|
|
70
|
+
data=json.dumps({"action": "register", "project_root": root}).encode("utf-8"))
|
|
71
|
+
with urllib.request.urlopen(request, timeout=10) as response:
|
|
72
|
+
project = json.load(response)["project"]
|
|
73
|
+
webbrowser.open(url + "/projects#" + project["id"])
|