@allansantos-dev/smart-tool 0.9.78 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +79 -0
- package/README.md +65 -18
- package/affected_tests.py +4 -0
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_graph.py +39 -17
- package/code_graph_go.py +6 -6
- package/code_graph_java.mjs +10 -9
- package/code_graph_js.cjs +8 -8
- package/code_health.py +180 -0
- package/code_impact.py +16 -2
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/doc_scope.py +1 -1
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/duplicates.py +1 -1
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/layers.py +3 -3
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +113 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/hook_decision.py
CHANGED
|
@@ -19,6 +19,7 @@ import code_graph
|
|
|
19
19
|
import code_impact
|
|
20
20
|
import comment_check
|
|
21
21
|
import config
|
|
22
|
+
import credential_check
|
|
22
23
|
import dependency_notice
|
|
23
24
|
import doc_check
|
|
24
25
|
import doc_scope
|
|
@@ -26,6 +27,7 @@ import duplicates
|
|
|
26
27
|
import edit_preview
|
|
27
28
|
import index_profile
|
|
28
29
|
import index_scope
|
|
30
|
+
import layer_check
|
|
29
31
|
import endpoint_sync
|
|
30
32
|
import project_admission
|
|
31
33
|
import project_store
|
|
@@ -283,7 +285,22 @@ def _duplicate_lines(findings):
|
|
|
283
285
|
+ " If the copy is intentional (moving the function, a deliberate fork), go ahead.")
|
|
284
286
|
|
|
285
287
|
|
|
286
|
-
def
|
|
288
|
+
def _layer_lines(findings):
|
|
289
|
+
lines = [f"`{item['function']}` ({rel}:{item['line']}) now calls `{item['called']}` ({item['path']}), a function of the "
|
|
290
|
+
f"higher layer {item['to']}, from layer {item['from']}" for rel, item in findings]
|
|
291
|
+
return ("Smart Tool layers: " + "; ".join(lines) + ". Calls go down the declared layers (.smart-tool/layers.json); "
|
|
292
|
+
"if this one is intended, go ahead.")
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _credential_lines(findings):
|
|
296
|
+
lines = [f"{rel}:{item['line']} looks like a {item['kind']}" for rel, item in findings[:MAX_DOC_FINDINGS]]
|
|
297
|
+
if len(findings) > MAX_DOC_FINDINGS:
|
|
298
|
+
lines.append(f"{len(findings) - MAX_DOC_FINDINGS} more line(s) in the same situation.")
|
|
299
|
+
return ("Smart Tool credentials: " + "; ".join(lines) + ". Keep it out of the file: read it from an environment "
|
|
300
|
+
"variable or a secret store. If it is a fake value for a test, go ahead.")
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _edit_review(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode="off", layer_mode="off"):
|
|
287
304
|
"""Checks of an edit before it runs. Docstrings follow doc_mode, in code files of registered projects only (tests,
|
|
288
305
|
docs and files outside a project are left out, as in the docs coverage): missing ones deny the edit in require mode and become a reminder in remind mode; a
|
|
289
306
|
documented function whose signature changed always gets a reminder. Comments the edit adds in the wrong place
|
|
@@ -291,7 +308,7 @@ def _edit_review(payload, doc_mode, duplicate_mode, dependency_mode, comment_mod
|
|
|
291
308
|
functions follow duplicate_mode and only add context, never deny. An edit of a package.json, requirements*.txt or go.mod follows dependency_mode: advisories
|
|
292
309
|
of the added or changed dependencies are added as context, never a denial."""
|
|
293
310
|
with code_graph.pending_scope():
|
|
294
|
-
return _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode)
|
|
311
|
+
return _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode)
|
|
295
312
|
|
|
296
313
|
|
|
297
314
|
def _dependency_part(before, after, ecosystem):
|
|
@@ -307,19 +324,25 @@ def _dependency_part(before, after, ecosystem):
|
|
|
307
324
|
return f"{dependency_notice.PREFIX}dependency check skipped: {exc}"
|
|
308
325
|
|
|
309
326
|
|
|
310
|
-
def
|
|
311
|
-
"""
|
|
327
|
+
def _review_changes(changes, doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode="off"):
|
|
328
|
+
"""The checks of an edit over (path, before, after) triples: docstrings, comments, documentation anchors, copies and
|
|
329
|
+
dependency advisories, each following its mode.
|
|
330
|
+
|
|
331
|
+
Parameters: changes (iterable of (absolute path, text before or None, text after)); the four modes (`off` skips a check).
|
|
332
|
+
Returns: tuple (message parts, doc findings, comment findings); the parts are empty when nothing is worth saying.
|
|
333
|
+
"""
|
|
312
334
|
doc_findings, scope_findings, duplicate_findings, failures, dependency_parts = [], [], [], [], []
|
|
313
|
-
comment_findings = []
|
|
314
|
-
for path, before, after in
|
|
335
|
+
comment_findings, layer_findings, credential_findings = [], [], []
|
|
336
|
+
for path, before, after in changes:
|
|
337
|
+
root = _project_of(path)
|
|
338
|
+
rel = os.path.relpath(os.path.realpath(path), os.path.realpath(root)).replace(os.sep, "/") if root else os.path.basename(path)
|
|
339
|
+
credential_findings += [(rel, item) for item in credential_check.review(before, after)]
|
|
315
340
|
ecosystem = dependency_notice.ecosystem_of(path)
|
|
316
341
|
if dependency_mode != "off" and ecosystem:
|
|
317
342
|
dependency_parts += [part for part in (_dependency_part(before, after, ecosystem),) if part]
|
|
318
343
|
continue
|
|
319
344
|
if not doc_check.supported(path):
|
|
320
345
|
continue
|
|
321
|
-
root = _project_of(path)
|
|
322
|
-
rel = os.path.relpath(os.path.realpath(path), os.path.realpath(root)).replace(os.sep, "/") if root else os.path.basename(path)
|
|
323
346
|
edited = doc_check.written(path, before, after)
|
|
324
347
|
profile = index_profile.current((index_scope.load_scope(root) or {}).get("profile")) if root else None
|
|
325
348
|
if doc_mode != "off" and root and index_profile.kind(rel, profile) == "code":
|
|
@@ -336,10 +359,37 @@ def _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comm
|
|
|
336
359
|
duplicate_findings += [(rel, item) for item in duplicates.on_write(root, path, before, after, edited)]
|
|
337
360
|
except (ValueError, RuntimeError) as exc:
|
|
338
361
|
failures.append(f"Smart Tool duplicates check skipped: {exc}")
|
|
362
|
+
if layer_mode != "off" and root and edited["touched"]:
|
|
363
|
+
try:
|
|
364
|
+
layer_findings += [(rel, item) for item in layer_check.review(root, rel, before, after, edited)]
|
|
365
|
+
except (ValueError, RuntimeError, OSError) as exc:
|
|
366
|
+
failures.append(f"Smart Tool layers check skipped: {exc}")
|
|
339
367
|
parts = ([_doc_lines(doc_findings)] if doc_findings else []) + \
|
|
340
368
|
([_comment_lines(comment_findings)] if comment_findings else []) + \
|
|
341
369
|
([_scope_lines(scope_findings)] if scope_findings else []) + \
|
|
342
|
-
([_duplicate_lines(duplicate_findings)] if duplicate_findings else []) +
|
|
370
|
+
([_duplicate_lines(duplicate_findings)] if duplicate_findings else []) + \
|
|
371
|
+
([_layer_lines(layer_findings)] if layer_findings else []) + \
|
|
372
|
+
([_credential_lines(credential_findings)] if credential_findings else []) + failures + dependency_parts
|
|
373
|
+
return parts, doc_findings, comment_findings
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def review_diff(changes, cfg):
|
|
377
|
+
"""The notes the edit hook would have added for changes made without it (a commit, a patch applied by hand).
|
|
378
|
+
|
|
379
|
+
Parameters: changes (iterable of (absolute path, text before or None, text after)); cfg (dict) the loaded config.
|
|
380
|
+
Returns: str, the hook's message for the whole set; empty when every check is quiet or off.
|
|
381
|
+
"""
|
|
382
|
+
modes = (config.doc_mode(cfg), config.duplicate_mode(cfg), config.dependency_mode(cfg), config.comment_mode(cfg),
|
|
383
|
+
config.layer_mode(cfg))
|
|
384
|
+
with code_graph.pending_scope():
|
|
385
|
+
return " ".join(_review_changes(changes, *modes)[0])
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode="off", layer_mode="off"):
|
|
389
|
+
"""`_edit_review` inside one `code_graph.pending_scope`, so the pending-changes scan runs once per review."""
|
|
390
|
+
parts, doc_findings, comment_findings = _review_changes(
|
|
391
|
+
edit_preview.preview(payload.get("tool_name"), payload.get("tool_input"), payload.get("cwd")),
|
|
392
|
+
doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode)
|
|
343
393
|
if not parts:
|
|
344
394
|
return {}
|
|
345
395
|
message = " ".join(parts)
|
|
@@ -354,7 +404,7 @@ def _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode, comm
|
|
|
354
404
|
|
|
355
405
|
def decide(payload, client, web_route, jobs):
|
|
356
406
|
"""Hook output for one PreToolUse payload: edits follow doc_mode (require | remind | off) and duplicate_mode
|
|
357
|
-
(warn | off) and dependency_mode (warn | off), the other tools follow hook_mode (redirect | advise | off). In redirect mode the Smart Tool call that
|
|
407
|
+
(warn | off) and dependency_mode (warn | off), and every edit review also notes credentials the edit writes, the other tools follow hook_mode (redirect | advise | off). In redirect mode the Smart Tool call that
|
|
358
408
|
replaces a redirected one starts in `jobs` under the call's tool_use_id (see `_redirect`); only a search whose
|
|
359
409
|
query cannot be built, or a client that sends no tool_use_id, is denied with the exact call to make, and never in a
|
|
360
410
|
subagent whose tool list has no Smart Tool MCP tool: it has nothing to switch to."""
|
|
@@ -362,11 +412,12 @@ def decide(payload, client, web_route, jobs):
|
|
|
362
412
|
if isinstance(payload, dict):
|
|
363
413
|
requester.note_origin(payload.get("session_id"), payload.get("cwd"))
|
|
364
414
|
if isinstance(payload, dict) and payload.get("tool_name") in EDIT_TOOLS:
|
|
365
|
-
doc_mode, duplicate_mode, dependency_mode, comment_mode = (
|
|
366
|
-
config.doc_mode(cfg), config.duplicate_mode(cfg), config.dependency_mode(cfg), config.comment_mode(cfg)
|
|
367
|
-
|
|
415
|
+
doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode = (
|
|
416
|
+
config.doc_mode(cfg), config.duplicate_mode(cfg), config.dependency_mode(cfg), config.comment_mode(cfg),
|
|
417
|
+
config.layer_mode(cfg))
|
|
418
|
+
if "off" == doc_mode == duplicate_mode == dependency_mode == comment_mode == layer_mode:
|
|
368
419
|
return {}
|
|
369
|
-
return _edit_review(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode)
|
|
420
|
+
return _edit_review(payload, doc_mode, duplicate_mode, dependency_mode, comment_mode, layer_mode)
|
|
370
421
|
if isinstance(payload, dict) and str(payload.get("tool_name")).startswith(f"mcp__{SMART_TOOL_SERVER}__"):
|
|
371
422
|
if payload["tool_name"] == SMART_SEARCH_TOOL:
|
|
372
423
|
router.CLIENT.set(client)
|
package/indexer.py
CHANGED
|
@@ -226,6 +226,31 @@ def existing_db_path(root):
|
|
|
226
226
|
return None
|
|
227
227
|
|
|
228
228
|
|
|
229
|
+
def partial_index(root, model_id):
|
|
230
|
+
"""`(path, indexed_files)` of the generation being built for `model_id` that already holds chunks: the newest
|
|
231
|
+
`.building-*` file of the project's current view. `(None, set())` when no build is under way or it holds nothing yet.
|
|
232
|
+
A search may read it while the build commits file by file."""
|
|
233
|
+
prefix = os.path.basename(db_path(root))[:-len(".sqlite3")] + ".building-"
|
|
234
|
+
try:
|
|
235
|
+
names = [name for name in os.listdir(INDEX_DIR) if name.startswith(prefix) and name.endswith(".sqlite3")]
|
|
236
|
+
except OSError:
|
|
237
|
+
return None, set()
|
|
238
|
+
for path in sorted((os.path.join(INDEX_DIR, name) for name in names), key=os.path.getmtime, reverse=True):
|
|
239
|
+
try:
|
|
240
|
+
conn = _readonly(path)
|
|
241
|
+
try:
|
|
242
|
+
if _metadata(conn).get("model_id") != model_id:
|
|
243
|
+
continue
|
|
244
|
+
files = {row[0] for row in conn.execute("SELECT DISTINCT path FROM chunks")}
|
|
245
|
+
finally:
|
|
246
|
+
conn.close()
|
|
247
|
+
except (OSError, sqlite3.Error):
|
|
248
|
+
continue
|
|
249
|
+
if files:
|
|
250
|
+
return path, files
|
|
251
|
+
return None, set()
|
|
252
|
+
|
|
253
|
+
|
|
229
254
|
def _readonly(path):
|
|
230
255
|
"""Read-only connection to an index. A rollback journal left by a crashed writer (a hot journal) cannot be rolled
|
|
231
256
|
back by a read-only connection, which then fails every read with "attempt to write a readonly database": the
|
|
@@ -469,6 +494,30 @@ def chunk_file(root, rel_path, max_lines=CHUNK_LINES, max_chars=CHUNK_MAX_CHARS,
|
|
|
469
494
|
return chunks
|
|
470
495
|
|
|
471
496
|
|
|
497
|
+
def indexed_lines(root, rel_path, start, end):
|
|
498
|
+
"""`[(line_number, text)]` of lines `start..end` of rel_path as the index stored them (the extracted text of a PDF
|
|
499
|
+
or Office file, whose lines are not lines of the file), or None when the index holds none of them. Reads the chunk
|
|
500
|
+
rows only: no extraction of the document."""
|
|
501
|
+
db_file = existing_db_path(root)
|
|
502
|
+
if not db_file:
|
|
503
|
+
return None
|
|
504
|
+
try:
|
|
505
|
+
conn = _readonly(db_file)
|
|
506
|
+
try:
|
|
507
|
+
rows = conn.execute("SELECT start_line,text FROM chunks WHERE path=? AND start_line<=? AND end_line>=? "
|
|
508
|
+
"ORDER BY start_line,id", (rel_path, end, start)).fetchall()
|
|
509
|
+
finally:
|
|
510
|
+
conn.close()
|
|
511
|
+
except sqlite3.Error:
|
|
512
|
+
return None
|
|
513
|
+
found = {}
|
|
514
|
+
for first, text in rows:
|
|
515
|
+
for offset, line in enumerate(text.splitlines()):
|
|
516
|
+
found.setdefault(first + offset, line)
|
|
517
|
+
wanted = [(number, found[number]) for number in range(max(1, start), end + 1) if number in found]
|
|
518
|
+
return wanted or None
|
|
519
|
+
|
|
520
|
+
|
|
472
521
|
def _pack_vector(vector):
|
|
473
522
|
validate_vector(vector)
|
|
474
523
|
try:
|
|
@@ -795,6 +844,8 @@ def _reindex(root, included_paths, embed_batch, chunk_lines=CHUNK_LINES, on_prog
|
|
|
795
844
|
check()
|
|
796
845
|
if building and not stats["failed"]:
|
|
797
846
|
atomic_io.atomic_replace(work_path, project_identity.state_path(target, INDEX_DIR))
|
|
847
|
+
if current.get("model_id") not in (None, model_id):
|
|
848
|
+
stats["dropped_vectors"] = _drop_other_models(root, model_id, target)
|
|
798
849
|
stats["ready"] = not bool(stats["failed"])
|
|
799
850
|
stats["path"] = target if stats["ready"] else work_path
|
|
800
851
|
if building or total or stats["removed"] or full_check:
|
|
@@ -834,6 +885,43 @@ def _trim_other_views(root, target):
|
|
|
834
885
|
conn.close()
|
|
835
886
|
|
|
836
887
|
|
|
888
|
+
def _project_files(root):
|
|
889
|
+
"""Index files of the project in INDEX_DIR (every Git view, the legacy keys, builds, the vector cache)."""
|
|
890
|
+
for key in dict.fromkeys([project_identity.project_id(root), *project_identity.legacy_ids(root)]):
|
|
891
|
+
pattern = re.compile(re.escape(key) + r'((?:\.v-[a-f0-9]{16})?(?:\.previous|\.building-[a-f0-9]+|\.vectors)?\.sqlite3)$')
|
|
892
|
+
for name in os.listdir(INDEX_DIR):
|
|
893
|
+
if pattern.fullmatch(name):
|
|
894
|
+
yield os.path.join(INDEX_DIR, name)
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
def _drop_other_models(root, model_id, target):
|
|
898
|
+
"""After the index of a folder switched to `model_id`, deletes the vectors of the other models: the rows of the
|
|
899
|
+
vector cache and the other index files of the project (other Git views, legacy keys, unfinished builds) that hold
|
|
900
|
+
another model, with their saved graph analysis. Returns how many cached vectors were deleted."""
|
|
901
|
+
dropped = embedding_cache.drop_other_models(root, INDEX_DIR, model_id)
|
|
902
|
+
for path in list(_project_files(root)):
|
|
903
|
+
if path.endswith(".vectors.sqlite3") or os.path.normcase(path) == os.path.normcase(target):
|
|
904
|
+
continue
|
|
905
|
+
try:
|
|
906
|
+
conn = _readonly(path)
|
|
907
|
+
try:
|
|
908
|
+
model = _metadata(conn).get("model_id")
|
|
909
|
+
finally:
|
|
910
|
+
conn.close()
|
|
911
|
+
except sqlite3.Error as exc:
|
|
912
|
+
_notice(f"index {os.path.basename(path)}: not checked for vectors of another model ({exc})")
|
|
913
|
+
continue
|
|
914
|
+
if model != model_id:
|
|
915
|
+
for stale in (path, graph_cache_path(path)):
|
|
916
|
+
try:
|
|
917
|
+
os.remove(stale)
|
|
918
|
+
except FileNotFoundError:
|
|
919
|
+
pass
|
|
920
|
+
except OSError as exc:
|
|
921
|
+
_notice(f"index {os.path.basename(stale)}: vectors of {model} not deleted now ({exc})")
|
|
922
|
+
return dropped
|
|
923
|
+
|
|
924
|
+
|
|
837
925
|
def _compact(conn):
|
|
838
926
|
"""VACUUM when the removals left at least COMPACT_MIN_FREE_BYTES and COMPACT_MIN_FREE_SHARE of the file free."""
|
|
839
927
|
page = conn.execute("PRAGMA page_size").fetchone()[0]
|
|
@@ -995,11 +1083,12 @@ def overlay_exact_keys(overlay, query):
|
|
|
995
1083
|
if score == 1.0}
|
|
996
1084
|
|
|
997
1085
|
|
|
998
|
-
def search(root, query_vector=None, top_k=20, query=None, model_id=None, include=None, overlay=None):
|
|
1086
|
+
def search(root, query_vector=None, top_k=20, query=None, model_id=None, include=None, overlay=None, path=None):
|
|
999
1087
|
"""Vector plus lexical retrieval by fusion of relative scores, without network. `overlay` is
|
|
1000
1088
|
`{"drop": paths, "chunks": overlay_chunks(...)}`: the indexed chunks of the `drop` paths (stale or removed
|
|
1001
|
-
files) are ignored and the overlay chunks, read from disk, take part by keyword instead.
|
|
1002
|
-
|
|
1089
|
+
files) are ignored and the overlay chunks, read from disk, take part by keyword instead. `path` searches that
|
|
1090
|
+
index file (a `partial_index`) instead of the project's ready one."""
|
|
1091
|
+
path = path or existing_db_path(root)
|
|
1003
1092
|
if not path:
|
|
1004
1093
|
if not (overlay and overlay["chunks"] and query):
|
|
1005
1094
|
return []
|
package/layer_check.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Layer rule of the project applied at the moment a function is written: a call that goes up the declared layers.
|
|
2
|
+
|
|
3
|
+
The rule is the one confirmed in `.smart-tool/layers.json` (`layers`); a project without it is never checked. A call is
|
|
4
|
+
recognised by the name it names, so only a function name that is unique among the code files of the project and not
|
|
5
|
+
defined in the edited file counts, and only a call the edit adds (the name is not called in the same function before it).
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
import code_graph
|
|
10
|
+
import doc_check
|
|
11
|
+
import layers
|
|
12
|
+
|
|
13
|
+
_CALL = re.compile(r"\b([A-Za-z_]\w*)\s*\(")
|
|
14
|
+
_COMMENT = re.compile(r"^\s*(#|//|\*|/\*)")
|
|
15
|
+
MAX_FINDINGS = 5
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _called(source, function):
|
|
19
|
+
"""Names called in the lines of one function of `source`."""
|
|
20
|
+
lines = source.splitlines()[function["start"] - 1:function["end"]]
|
|
21
|
+
return {name for line in lines if not _COMMENT.match(line) for name in _CALL.findall(line)} - {function["name"]}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def review(root, rel, before, after, edited):
|
|
25
|
+
"""The calls an edit adds that go from a lower layer to a higher one.
|
|
26
|
+
|
|
27
|
+
Parameters: root (str) project folder; rel (str) project-relative path of the edited file; before (str or None) text
|
|
28
|
+
before the edit; after (str) text after it; edited (dict) `doc_check.written` of the same edit.
|
|
29
|
+
Returns: list of dicts {"function", "line", "called", "path", "from", "to"}, empty when the project declares no layers
|
|
30
|
+
or the file belongs to none. Raises: ValueError when the layers file is unreadable; RuntimeError when the graph is.
|
|
31
|
+
"""
|
|
32
|
+
rule = layers.load(root)
|
|
33
|
+
if rule is None:
|
|
34
|
+
return []
|
|
35
|
+
source_layer = rule.assign(rel, after)
|
|
36
|
+
if source_layer is None:
|
|
37
|
+
return []
|
|
38
|
+
order = {layer["name"]: index for index, layer in enumerate(rule.layers)}
|
|
39
|
+
owners = {}
|
|
40
|
+
for symbol in code_graph.build_current(root, None, shared=True).get("symbols") or []:
|
|
41
|
+
owners.setdefault(symbol["name"].split(".")[-1], set()).add(symbol["path"])
|
|
42
|
+
old = {function["name"]: function for function in doc_check.functions(rel, before)} if before else {}
|
|
43
|
+
found = []
|
|
44
|
+
for function in edited["touched"]:
|
|
45
|
+
previous = old.get(function["name"])
|
|
46
|
+
known = _called(before, previous) if previous else set()
|
|
47
|
+
for name in sorted(_called(after, function) - known):
|
|
48
|
+
paths = owners.get(name, set())
|
|
49
|
+
if len(paths) != 1 or rel in paths:
|
|
50
|
+
continue
|
|
51
|
+
path = next(iter(paths))
|
|
52
|
+
target_layer = rule.assign(path)
|
|
53
|
+
if target_layer and order[target_layer] < order[source_layer]:
|
|
54
|
+
found.append({"function": function["name"], "line": function["start"], "called": name, "path": path,
|
|
55
|
+
"from": source_layer, "to": target_layer})
|
|
56
|
+
return found[:MAX_FINDINGS]
|
|
57
|
+
|
package/layers.py
CHANGED
|
@@ -165,7 +165,7 @@ def summary(rule, graph, read):
|
|
|
165
165
|
the declared order, 'up' goes against it) and the folders no layer covers.
|
|
166
166
|
|
|
167
167
|
Parameters: rule (Rule); graph (dict) code graph with 'symbols' and 'calls'; read (callable path -> str or None).
|
|
168
|
-
Returns: dict with 'layers', 'calls_between', 'unassigned' and '
|
|
168
|
+
Returns: dict with 'layers', 'calls_between', 'unassigned', 'truncated' and 'left_out' (what the graph limits cut).
|
|
169
169
|
"""
|
|
170
170
|
order = {layer["name"]: index for index, layer in enumerate(rule.layers)}
|
|
171
171
|
paths = sorted({s["path"] for s in graph.get("symbols") or []})
|
|
@@ -185,7 +185,7 @@ def summary(rule, graph, read):
|
|
|
185
185
|
for (a, b), n in sorted(between.items(), key=lambda item: (order[item[0][0]],
|
|
186
186
|
order[item[0][1]]))],
|
|
187
187
|
"unassigned": [{"folder": folder, "files": n} for folder, n in unassigned.most_common(MAX_UNASSIGNED)],
|
|
188
|
-
"unassigned_files": files[None], "truncated": bool(graph.get("truncated"))}
|
|
188
|
+
"unassigned_files": files[None], "truncated": bool(graph.get("truncated")), "left_out": graph["left_out"]}
|
|
189
189
|
|
|
190
190
|
|
|
191
191
|
def report(root, layers=None, confirm=False, view_id=None):
|
|
@@ -229,7 +229,7 @@ def report(root, layers=None, confirm=False, view_id=None):
|
|
|
229
229
|
proposed = suggest(sorted({s["path"] for s in symbols}), read)
|
|
230
230
|
if not proposed:
|
|
231
231
|
return {"saved": False, "rule_path": path, "layers": [], "calls_between": [], "unassigned": [],
|
|
232
|
-
"truncated": bool(graph.get("truncated")),
|
|
232
|
+
"truncated": bool(graph.get("truncated")), "left_out": graph["left_out"],
|
|
233
233
|
"guidance": "No folder or annotation follows a known layer convention; declare the layers yourself "
|
|
234
234
|
"(name, paths, annotations, highest first) and confirm them."}
|
|
235
235
|
return {"saved": False, "rule_path": path, **summary(Rule(proposed), code, read),
|
package/local_embedder.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Client of the embeddings home (casa-de-embeddings): the per-machine daemon that serves the embedder and forwards
|
|
2
|
+
rerank to the host the user configured in it.
|
|
2
3
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
The home is found through its announce file. Its embedder is identified by the `model_sha256` and `dimension` of
|
|
5
|
+
`/info`, never by a model name. Measured in 2026-10-05 on the rerank gold set (42 code targets, no rerank,
|
|
6
|
+
docs/spikes/local_embedder_spike.py): granite-embedding-107m-multilingual on this client 42/35/41 (in pool / 1st /
|
|
7
|
+
top 5), MRR@5 0.892; the gateway's text-embedding-3-small 42/35/40, 0.885; lexical only 38/29/36, 0.767.
|
|
6
8
|
"""
|
|
7
9
|
import http.client
|
|
8
10
|
import json
|
|
@@ -18,37 +20,49 @@ import model_breaker
|
|
|
18
20
|
import model_health
|
|
19
21
|
|
|
20
22
|
PREFIX = "local:"
|
|
21
|
-
|
|
23
|
+
ANNOUNCE = os.path.join(os.path.expanduser("~"), ".casa-de-embeddings", "run", "daemon.json")
|
|
24
|
+
MIN_API_VERSION = 1
|
|
22
25
|
BATCH = 16
|
|
23
26
|
INFO_TTL_S = 60
|
|
24
27
|
MAX_ATTEMPTS = 3
|
|
25
28
|
REQUEST_TIMEOUT_S = 60
|
|
29
|
+
RERANK_TIMEOUT_S = 15
|
|
26
30
|
_lock = threading.Lock()
|
|
27
|
-
_info = {"at": 0.0, "value": None}
|
|
31
|
+
_info = {"at": 0.0, "value": None, "error": None}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class IdentityError(ValueError):
|
|
35
|
+
"""The home is announced or answers, but cannot be used as it is: a malformed announce file, an API version older
|
|
36
|
+
than this client needs, or an `/info` without `model_sha256` or `dimension`."""
|
|
28
37
|
|
|
29
38
|
|
|
30
39
|
def url():
|
|
31
|
-
"""
|
|
32
|
-
|
|
40
|
+
"""Base URL of the home from its announce file; None when the file does not exist (the home is not installed or
|
|
41
|
+
not running). A file that exists but cannot be read as an announce raises IdentityError. localhost becomes
|
|
42
|
+
127.0.0.1: on Windows ::1 is tried first and a daemon bound to IPv4 only costs about 2 s per connection."""
|
|
33
43
|
try:
|
|
34
|
-
with open(
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
value = value.strip()
|
|
38
|
-
if key.strip() == "tei.url":
|
|
39
|
-
if not value or value.lower() == "off":
|
|
40
|
-
return None
|
|
41
|
-
parsed = urllib.parse.urlsplit(value.rstrip("/"))
|
|
42
|
-
if parsed.hostname == "localhost":
|
|
43
|
-
parsed = parsed._replace(netloc=parsed.netloc.replace("localhost", "127.0.0.1", 1))
|
|
44
|
-
return urllib.parse.urlunsplit(parsed)
|
|
45
|
-
except OSError:
|
|
44
|
+
with open(ANNOUNCE, encoding="utf-8") as stream:
|
|
45
|
+
data = json.load(stream)
|
|
46
|
+
except FileNotFoundError:
|
|
46
47
|
return None
|
|
47
|
-
|
|
48
|
+
except (OSError, ValueError) as exc:
|
|
49
|
+
raise IdentityError(f"The embeddings home announce file {ANNOUNCE} cannot be read ({exc}).") from None
|
|
50
|
+
base = str(data.get("url") or "").strip().rstrip("/") if isinstance(data, dict) else ""
|
|
51
|
+
parsed = urllib.parse.urlsplit(base)
|
|
52
|
+
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
53
|
+
raise IdentityError(f"The embeddings home announce file {ANNOUNCE} has no http(s) url.")
|
|
54
|
+
version = data.get("apiVersion")
|
|
55
|
+
if type(version) is not int or version < MIN_API_VERSION:
|
|
56
|
+
raise IdentityError(f"The embeddings home announces API version {version!r}; Smart Tool needs "
|
|
57
|
+
f"{MIN_API_VERSION} or later. Update the embeddings home.")
|
|
58
|
+
if parsed.hostname == "localhost":
|
|
59
|
+
parsed = parsed._replace(netloc=parsed.netloc.replace("localhost", "127.0.0.1", 1))
|
|
60
|
+
return urllib.parse.urlunsplit(parsed)
|
|
48
61
|
|
|
49
62
|
|
|
50
63
|
def configured():
|
|
51
|
-
|
|
64
|
+
"""Whether an announce file of the home exists (malformed counts: it raises at use, loudly)."""
|
|
65
|
+
return os.path.isfile(ANNOUNCE)
|
|
52
66
|
|
|
53
67
|
|
|
54
68
|
def _connection(base, timeout):
|
|
@@ -63,7 +77,9 @@ def _request(conn, method, path, body=None):
|
|
|
63
77
|
response = conn.getresponse()
|
|
64
78
|
data = response.read()
|
|
65
79
|
if response.status >= 400:
|
|
66
|
-
|
|
80
|
+
detail = data[:300].decode("utf-8", "replace").strip()
|
|
81
|
+
raise urllib.error.HTTPError(f"{conn.host}:{conn.port}{path}", response.status,
|
|
82
|
+
f"{response.reason}{': ' + detail if detail else ''}", response.headers, None)
|
|
67
83
|
return json.loads(data)
|
|
68
84
|
|
|
69
85
|
|
|
@@ -73,48 +89,87 @@ def _probe(base, timeout):
|
|
|
73
89
|
data = _request(conn, "GET", "/info")
|
|
74
90
|
finally:
|
|
75
91
|
conn.close()
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
92
|
+
digest = str(data.get("model_sha256") or "").strip().lower()
|
|
93
|
+
try:
|
|
94
|
+
dimensions = int(data.get("dimension") or 0)
|
|
95
|
+
except (TypeError, ValueError):
|
|
96
|
+
dimensions = 0
|
|
97
|
+
if not digest or dimensions <= 0:
|
|
98
|
+
raise IdentityError(f"The embeddings home at {base} answered /info without model_sha256 or dimension, so its "
|
|
99
|
+
"model cannot be identified; Smart Tool will not fall back to lexical search silently. "
|
|
100
|
+
"Update the embeddings home so /info reports model_sha256 and dimension.")
|
|
101
|
+
cpu = bool(data.get("demoted_to_cpu")) or str(data.get("provider") or "").startswith("CPU")
|
|
102
|
+
rerank_host = data.get("rerank_host") if isinstance(data.get("rerank_host"), dict) else {}
|
|
103
|
+
return {"model": f"{PREFIX}{digest[:16]}-{dimensions}", "name": data.get("model_id") or "", "dimensions": dimensions,
|
|
104
|
+
"url": base, "sha256": digest, "cpu": cpu, "rerank": rerank_host.get("configured") is True}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def runs_on_cpu(model):
|
|
108
|
+
"""Whether `model` is the home's embedder and the home computes on the CPU (no GPU, or demoted to the CPU)."""
|
|
109
|
+
return is_local(model) and bool((info() or {}).get("cpu"))
|
|
80
110
|
|
|
81
111
|
|
|
82
112
|
def info(timeout=1.5):
|
|
83
|
-
"""
|
|
84
|
-
|
|
113
|
+
"""Home /info cached for INFO_TTL_S; None when it is not announced or not answering. A home that is announced or
|
|
114
|
+
answers but cannot be used raises IdentityError. The probe runs under the lock so a slow failing probe never
|
|
115
|
+
overwrites a newer good answer."""
|
|
85
116
|
with _lock:
|
|
86
|
-
if time.monotonic() - _info["at"]
|
|
87
|
-
|
|
88
|
-
base, value = url(), None
|
|
89
|
-
if base:
|
|
117
|
+
if time.monotonic() - _info["at"] >= INFO_TTL_S:
|
|
118
|
+
value, error = None, None
|
|
90
119
|
try:
|
|
91
|
-
|
|
120
|
+
base = url()
|
|
121
|
+
if base:
|
|
122
|
+
value = _probe(base, timeout)
|
|
123
|
+
except IdentityError as exc:
|
|
124
|
+
error = exc
|
|
92
125
|
except (OSError, ValueError, http.client.HTTPException):
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
126
|
+
pass
|
|
127
|
+
_info.update(at=time.monotonic(), value=value, error=error)
|
|
128
|
+
if _info["error"]:
|
|
129
|
+
raise _info["error"]
|
|
130
|
+
return _info["value"]
|
|
96
131
|
|
|
97
132
|
|
|
98
133
|
def is_local(model):
|
|
99
134
|
return bool(model) and model.startswith(PREFIX)
|
|
100
135
|
|
|
101
136
|
|
|
102
|
-
def resolve(
|
|
103
|
-
"""Embedding model
|
|
137
|
+
def resolve(cfg=None, current=None):
|
|
138
|
+
"""Embedding model the index of a folder must be built with, following config `embedder`.
|
|
104
139
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
if
|
|
111
|
-
|
|
140
|
+
`gateway` is the user's explicit choice: the gateway's `embedding_model`, or lexical without one. `home` (the
|
|
141
|
+
default) is the home's embedder. While the home is not running, an index the home can keep (its own, or lexical)
|
|
142
|
+
is kept as it is, so a restart of the home never triggers a rebuild to lexical and back; an index of another model
|
|
143
|
+
gives None: it keeps answering and is not updated, and re-embedding waits for the home. A folder without an index
|
|
144
|
+
is indexed lexically meanwhile. The gateway is never used in place of the home."""
|
|
145
|
+
cfg = cfg if cfg is not None else config.load_config()
|
|
146
|
+
if config.embedder(cfg) == "gateway":
|
|
147
|
+
return cfg.get("embedding_model") or indexer.LEXICAL_MODEL
|
|
112
148
|
available = info()
|
|
113
149
|
if available:
|
|
114
150
|
return available["model"]
|
|
115
|
-
if is_local(current):
|
|
116
|
-
return current
|
|
117
|
-
return
|
|
151
|
+
if not current or is_local(current) or current == indexer.LEXICAL_MODEL:
|
|
152
|
+
return current or indexer.LEXICAL_MODEL
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def rerank_configured():
|
|
157
|
+
"""Whether the home reports a rerank host configured in it."""
|
|
158
|
+
return bool((info() or {}).get("rerank"))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def rerank(query, documents, top_n, deadline=None):
|
|
162
|
+
"""The home's `POST /rerank` answer in the canonical form `{results: [{index, relevance_score}]}`. The home forwards
|
|
163
|
+
to the host the user configured; an error answer raises, it is never replaced by another reranker."""
|
|
164
|
+
base = url()
|
|
165
|
+
if not base:
|
|
166
|
+
raise RuntimeError(f"The embeddings home is not running (no announce file at {ANNOUNCE}).")
|
|
167
|
+
timeout = RERANK_TIMEOUT_S if deadline is None else min(RERANK_TIMEOUT_S, _left(deadline))
|
|
168
|
+
conn = _connection(base, timeout)
|
|
169
|
+
try:
|
|
170
|
+
return _request(conn, "POST", "/rerank", {"query": query, "documents": documents, "top_n": top_n})
|
|
171
|
+
finally:
|
|
172
|
+
conn.close()
|
|
118
173
|
|
|
119
174
|
|
|
120
175
|
def _left(deadline):
|
|
@@ -128,7 +183,7 @@ def _left(deadline):
|
|
|
128
183
|
|
|
129
184
|
def _retry_delay(failure, attempt):
|
|
130
185
|
"""Seconds to wait after a failed attempt: the `Retry-After` of the answer (whole seconds, at most the pause of the
|
|
131
|
-
circuit) when the
|
|
186
|
+
circuit) when the home sent one, else the attempt number."""
|
|
132
187
|
value = str(getattr(failure, "headers", None) and failure.headers.get("Retry-After") or "")
|
|
133
188
|
return min(int(value), model_breaker.PAUSE_S) if value.isdigit() else attempt
|
|
134
189
|
|
|
@@ -136,16 +191,18 @@ def _retry_delay(failure, attempt):
|
|
|
136
191
|
def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
137
192
|
base = url()
|
|
138
193
|
if not base:
|
|
139
|
-
raise RuntimeError(f"
|
|
194
|
+
raise RuntimeError(f"The embeddings home is not running (no announce file at {ANNOUNCE}).")
|
|
140
195
|
# Fresh /info before and after: /embed does not echo the model, and a cached answer could let vectors of a
|
|
141
196
|
# different model reach the persistent vector cache under this model's contract.
|
|
142
197
|
try:
|
|
143
198
|
before = _probe(base, _left(deadline))
|
|
199
|
+
except IdentityError:
|
|
200
|
+
raise
|
|
144
201
|
except (OSError, ValueError, http.client.HTTPException) as exc:
|
|
145
|
-
raise RuntimeError(f"
|
|
202
|
+
raise RuntimeError(f"The embeddings home is not responding ({base}): {exc}") from None
|
|
146
203
|
if before["model"] != model:
|
|
147
204
|
raise indexer.IndexCompatibilityError(
|
|
148
|
-
f"The
|
|
205
|
+
f"The embeddings home now serves {before['model']}; the index was built with {model} and must be rebuilt.")
|
|
149
206
|
vectors = []
|
|
150
207
|
conn = _connection(base, _left(deadline))
|
|
151
208
|
try:
|
|
@@ -165,19 +222,19 @@ def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
|
165
222
|
conn.close()
|
|
166
223
|
conn = _connection(base, _left(deadline))
|
|
167
224
|
if attempt == MAX_ATTEMPTS:
|
|
168
|
-
raise RuntimeError(f"
|
|
225
|
+
raise RuntimeError(f"Embeddings home failed after {MAX_ATTEMPTS} attempts: {exc}") from None
|
|
169
226
|
if on_retry:
|
|
170
227
|
on_retry(attempt + 1, len(part))
|
|
171
228
|
delay = _retry_delay(exc, attempt)
|
|
172
229
|
if deadline is not None and time.monotonic() + delay >= deadline:
|
|
173
|
-
raise TimeoutError("Not enough time left to retry the
|
|
230
|
+
raise TimeoutError("Not enough time left to retry the embeddings home.") from None
|
|
174
231
|
time.sleep(delay)
|
|
175
232
|
else:
|
|
176
233
|
model_breaker.note(model)
|
|
177
234
|
model_health.record("/v1/embeddings", {"model": model})
|
|
178
235
|
break
|
|
179
236
|
if not isinstance(got, list) or len(got) != len(part):
|
|
180
|
-
raise indexer.IndexVectorError("
|
|
237
|
+
raise indexer.IndexVectorError("The embeddings home returned a vector count different from the input.")
|
|
181
238
|
for vector in got:
|
|
182
239
|
indexer.validate_vector(vector, before["dimensions"])
|
|
183
240
|
vectors += got
|
|
@@ -185,5 +242,5 @@ def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
|
185
242
|
conn.close()
|
|
186
243
|
after = _probe(base, _left(deadline))
|
|
187
244
|
if (after["model"], after.get("sha256")) != (before["model"], before.get("sha256")):
|
|
188
|
-
raise indexer.IndexCompatibilityError("The
|
|
245
|
+
raise indexer.IndexCompatibilityError("The embeddings home changed models during the call; try again.")
|
|
189
246
|
return vectors
|