@allansantos-dev/smart-tool 0.9.79 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +65 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_health.py +180 -0
- package/code_impact.py +13 -1
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +111 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/indexer.py
CHANGED
|
@@ -226,6 +226,31 @@ def existing_db_path(root):
|
|
|
226
226
|
return None
|
|
227
227
|
|
|
228
228
|
|
|
229
|
+
def partial_index(root, model_id):
|
|
230
|
+
"""`(path, indexed_files)` of the generation being built for `model_id` that already holds chunks: the newest
|
|
231
|
+
`.building-*` file of the project's current view. `(None, set())` when no build is under way or it holds nothing yet.
|
|
232
|
+
A search may read it while the build commits file by file."""
|
|
233
|
+
prefix = os.path.basename(db_path(root))[:-len(".sqlite3")] + ".building-"
|
|
234
|
+
try:
|
|
235
|
+
names = [name for name in os.listdir(INDEX_DIR) if name.startswith(prefix) and name.endswith(".sqlite3")]
|
|
236
|
+
except OSError:
|
|
237
|
+
return None, set()
|
|
238
|
+
for path in sorted((os.path.join(INDEX_DIR, name) for name in names), key=os.path.getmtime, reverse=True):
|
|
239
|
+
try:
|
|
240
|
+
conn = _readonly(path)
|
|
241
|
+
try:
|
|
242
|
+
if _metadata(conn).get("model_id") != model_id:
|
|
243
|
+
continue
|
|
244
|
+
files = {row[0] for row in conn.execute("SELECT DISTINCT path FROM chunks")}
|
|
245
|
+
finally:
|
|
246
|
+
conn.close()
|
|
247
|
+
except (OSError, sqlite3.Error):
|
|
248
|
+
continue
|
|
249
|
+
if files:
|
|
250
|
+
return path, files
|
|
251
|
+
return None, set()
|
|
252
|
+
|
|
253
|
+
|
|
229
254
|
def _readonly(path):
|
|
230
255
|
"""Read-only connection to an index. A rollback journal left by a crashed writer (a hot journal) cannot be rolled
|
|
231
256
|
back by a read-only connection, which then fails every read with "attempt to write a readonly database": the
|
|
@@ -469,6 +494,30 @@ def chunk_file(root, rel_path, max_lines=CHUNK_LINES, max_chars=CHUNK_MAX_CHARS,
|
|
|
469
494
|
return chunks
|
|
470
495
|
|
|
471
496
|
|
|
497
|
+
def indexed_lines(root, rel_path, start, end):
|
|
498
|
+
"""`[(line_number, text)]` of lines `start..end` of rel_path as the index stored them (the extracted text of a PDF
|
|
499
|
+
or Office file, whose lines are not lines of the file), or None when the index holds none of them. Reads the chunk
|
|
500
|
+
rows only: no extraction of the document."""
|
|
501
|
+
db_file = existing_db_path(root)
|
|
502
|
+
if not db_file:
|
|
503
|
+
return None
|
|
504
|
+
try:
|
|
505
|
+
conn = _readonly(db_file)
|
|
506
|
+
try:
|
|
507
|
+
rows = conn.execute("SELECT start_line,text FROM chunks WHERE path=? AND start_line<=? AND end_line>=? "
|
|
508
|
+
"ORDER BY start_line,id", (rel_path, end, start)).fetchall()
|
|
509
|
+
finally:
|
|
510
|
+
conn.close()
|
|
511
|
+
except sqlite3.Error:
|
|
512
|
+
return None
|
|
513
|
+
found = {}
|
|
514
|
+
for first, text in rows:
|
|
515
|
+
for offset, line in enumerate(text.splitlines()):
|
|
516
|
+
found.setdefault(first + offset, line)
|
|
517
|
+
wanted = [(number, found[number]) for number in range(max(1, start), end + 1) if number in found]
|
|
518
|
+
return wanted or None
|
|
519
|
+
|
|
520
|
+
|
|
472
521
|
def _pack_vector(vector):
|
|
473
522
|
validate_vector(vector)
|
|
474
523
|
try:
|
|
@@ -795,6 +844,8 @@ def _reindex(root, included_paths, embed_batch, chunk_lines=CHUNK_LINES, on_prog
|
|
|
795
844
|
check()
|
|
796
845
|
if building and not stats["failed"]:
|
|
797
846
|
atomic_io.atomic_replace(work_path, project_identity.state_path(target, INDEX_DIR))
|
|
847
|
+
if current.get("model_id") not in (None, model_id):
|
|
848
|
+
stats["dropped_vectors"] = _drop_other_models(root, model_id, target)
|
|
798
849
|
stats["ready"] = not bool(stats["failed"])
|
|
799
850
|
stats["path"] = target if stats["ready"] else work_path
|
|
800
851
|
if building or total or stats["removed"] or full_check:
|
|
@@ -834,6 +885,43 @@ def _trim_other_views(root, target):
|
|
|
834
885
|
conn.close()
|
|
835
886
|
|
|
836
887
|
|
|
888
|
+
def _project_files(root):
|
|
889
|
+
"""Index files of the project in INDEX_DIR (every Git view, the legacy keys, builds, the vector cache)."""
|
|
890
|
+
for key in dict.fromkeys([project_identity.project_id(root), *project_identity.legacy_ids(root)]):
|
|
891
|
+
pattern = re.compile(re.escape(key) + r'((?:\.v-[a-f0-9]{16})?(?:\.previous|\.building-[a-f0-9]+|\.vectors)?\.sqlite3)$')
|
|
892
|
+
for name in os.listdir(INDEX_DIR):
|
|
893
|
+
if pattern.fullmatch(name):
|
|
894
|
+
yield os.path.join(INDEX_DIR, name)
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
def _drop_other_models(root, model_id, target):
|
|
898
|
+
"""After the index of a folder switched to `model_id`, deletes the vectors of the other models: the rows of the
|
|
899
|
+
vector cache and the other index files of the project (other Git views, legacy keys, unfinished builds) that hold
|
|
900
|
+
another model, with their saved graph analysis. Returns how many cached vectors were deleted."""
|
|
901
|
+
dropped = embedding_cache.drop_other_models(root, INDEX_DIR, model_id)
|
|
902
|
+
for path in list(_project_files(root)):
|
|
903
|
+
if path.endswith(".vectors.sqlite3") or os.path.normcase(path) == os.path.normcase(target):
|
|
904
|
+
continue
|
|
905
|
+
try:
|
|
906
|
+
conn = _readonly(path)
|
|
907
|
+
try:
|
|
908
|
+
model = _metadata(conn).get("model_id")
|
|
909
|
+
finally:
|
|
910
|
+
conn.close()
|
|
911
|
+
except sqlite3.Error as exc:
|
|
912
|
+
_notice(f"index {os.path.basename(path)}: not checked for vectors of another model ({exc})")
|
|
913
|
+
continue
|
|
914
|
+
if model != model_id:
|
|
915
|
+
for stale in (path, graph_cache_path(path)):
|
|
916
|
+
try:
|
|
917
|
+
os.remove(stale)
|
|
918
|
+
except FileNotFoundError:
|
|
919
|
+
pass
|
|
920
|
+
except OSError as exc:
|
|
921
|
+
_notice(f"index {os.path.basename(stale)}: vectors of {model} not deleted now ({exc})")
|
|
922
|
+
return dropped
|
|
923
|
+
|
|
924
|
+
|
|
837
925
|
def _compact(conn):
|
|
838
926
|
"""VACUUM when the removals left at least COMPACT_MIN_FREE_BYTES and COMPACT_MIN_FREE_SHARE of the file free."""
|
|
839
927
|
page = conn.execute("PRAGMA page_size").fetchone()[0]
|
|
@@ -995,11 +1083,12 @@ def overlay_exact_keys(overlay, query):
|
|
|
995
1083
|
if score == 1.0}
|
|
996
1084
|
|
|
997
1085
|
|
|
998
|
-
def search(root, query_vector=None, top_k=20, query=None, model_id=None, include=None, overlay=None):
|
|
1086
|
+
def search(root, query_vector=None, top_k=20, query=None, model_id=None, include=None, overlay=None, path=None):
|
|
999
1087
|
"""Vector plus lexical retrieval by fusion of relative scores, without network. `overlay` is
|
|
1000
1088
|
`{"drop": paths, "chunks": overlay_chunks(...)}`: the indexed chunks of the `drop` paths (stale or removed
|
|
1001
|
-
files) are ignored and the overlay chunks, read from disk, take part by keyword instead.
|
|
1002
|
-
|
|
1089
|
+
files) are ignored and the overlay chunks, read from disk, take part by keyword instead. `path` searches that
|
|
1090
|
+
index file (a `partial_index`) instead of the project's ready one."""
|
|
1091
|
+
path = path or existing_db_path(root)
|
|
1003
1092
|
if not path:
|
|
1004
1093
|
if not (overlay and overlay["chunks"] and query):
|
|
1005
1094
|
return []
|
package/layer_check.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Layer rule of the project applied at the moment a function is written: a call that goes up the declared layers.
|
|
2
|
+
|
|
3
|
+
The rule is the one confirmed in `.smart-tool/layers.json` (`layers`); a project without it is never checked. A call is
|
|
4
|
+
recognised by the name it names, so only a function name that is unique among the code files of the project and not
|
|
5
|
+
defined in the edited file counts, and only a call the edit adds (the name is not called in the same function before it).
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
import code_graph
|
|
10
|
+
import doc_check
|
|
11
|
+
import layers
|
|
12
|
+
|
|
13
|
+
_CALL = re.compile(r"\b([A-Za-z_]\w*)\s*\(")
|
|
14
|
+
_COMMENT = re.compile(r"^\s*(#|//|\*|/\*)")
|
|
15
|
+
MAX_FINDINGS = 5
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _called(source, function):
|
|
19
|
+
"""Names called in the lines of one function of `source`."""
|
|
20
|
+
lines = source.splitlines()[function["start"] - 1:function["end"]]
|
|
21
|
+
return {name for line in lines if not _COMMENT.match(line) for name in _CALL.findall(line)} - {function["name"]}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def review(root, rel, before, after, edited):
|
|
25
|
+
"""The calls an edit adds that go from a lower layer to a higher one.
|
|
26
|
+
|
|
27
|
+
Parameters: root (str) project folder; rel (str) project-relative path of the edited file; before (str or None) text
|
|
28
|
+
before the edit; after (str) text after it; edited (dict) `doc_check.written` of the same edit.
|
|
29
|
+
Returns: list of dicts {"function", "line", "called", "path", "from", "to"}, empty when the project declares no layers
|
|
30
|
+
or the file belongs to none. Raises: ValueError when the layers file is unreadable; RuntimeError when the graph is.
|
|
31
|
+
"""
|
|
32
|
+
rule = layers.load(root)
|
|
33
|
+
if rule is None:
|
|
34
|
+
return []
|
|
35
|
+
source_layer = rule.assign(rel, after)
|
|
36
|
+
if source_layer is None:
|
|
37
|
+
return []
|
|
38
|
+
order = {layer["name"]: index for index, layer in enumerate(rule.layers)}
|
|
39
|
+
owners = {}
|
|
40
|
+
for symbol in code_graph.build_current(root, None, shared=True).get("symbols") or []:
|
|
41
|
+
owners.setdefault(symbol["name"].split(".")[-1], set()).add(symbol["path"])
|
|
42
|
+
old = {function["name"]: function for function in doc_check.functions(rel, before)} if before else {}
|
|
43
|
+
found = []
|
|
44
|
+
for function in edited["touched"]:
|
|
45
|
+
previous = old.get(function["name"])
|
|
46
|
+
known = _called(before, previous) if previous else set()
|
|
47
|
+
for name in sorted(_called(after, function) - known):
|
|
48
|
+
paths = owners.get(name, set())
|
|
49
|
+
if len(paths) != 1 or rel in paths:
|
|
50
|
+
continue
|
|
51
|
+
path = next(iter(paths))
|
|
52
|
+
target_layer = rule.assign(path)
|
|
53
|
+
if target_layer and order[target_layer] < order[source_layer]:
|
|
54
|
+
found.append({"function": function["name"], "line": function["start"], "called": name, "path": path,
|
|
55
|
+
"from": source_layer, "to": target_layer})
|
|
56
|
+
return found[:MAX_FINDINGS]
|
|
57
|
+
|
package/local_embedder.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Client of the embeddings home (casa-de-embeddings): the per-machine daemon that serves the embedder and forwards
|
|
2
|
+
rerank to the host the user configured in it.
|
|
2
3
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
The home is found through its announce file. Its embedder is identified by the `model_sha256` and `dimension` of
|
|
5
|
+
`/info`, never by a model name. Measured in 2026-10-05 on the rerank gold set (42 code targets, no rerank,
|
|
6
|
+
docs/spikes/local_embedder_spike.py): granite-embedding-107m-multilingual on this client 42/35/41 (in pool / 1st /
|
|
7
|
+
top 5), MRR@5 0.892; the gateway's text-embedding-3-small 42/35/40, 0.885; lexical only 38/29/36, 0.767.
|
|
6
8
|
"""
|
|
7
9
|
import http.client
|
|
8
10
|
import json
|
|
@@ -18,37 +20,49 @@ import model_breaker
|
|
|
18
20
|
import model_health
|
|
19
21
|
|
|
20
22
|
PREFIX = "local:"
|
|
21
|
-
|
|
23
|
+
ANNOUNCE = os.path.join(os.path.expanduser("~"), ".casa-de-embeddings", "run", "daemon.json")
|
|
24
|
+
MIN_API_VERSION = 1
|
|
22
25
|
BATCH = 16
|
|
23
26
|
INFO_TTL_S = 60
|
|
24
27
|
MAX_ATTEMPTS = 3
|
|
25
28
|
REQUEST_TIMEOUT_S = 60
|
|
29
|
+
RERANK_TIMEOUT_S = 15
|
|
26
30
|
_lock = threading.Lock()
|
|
27
|
-
_info = {"at": 0.0, "value": None}
|
|
31
|
+
_info = {"at": 0.0, "value": None, "error": None}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class IdentityError(ValueError):
|
|
35
|
+
"""The home is announced or answers, but cannot be used as it is: a malformed announce file, an API version older
|
|
36
|
+
than this client needs, or an `/info` without `model_sha256` or `dimension`."""
|
|
28
37
|
|
|
29
38
|
|
|
30
39
|
def url():
|
|
31
|
-
"""
|
|
32
|
-
|
|
40
|
+
"""Base URL of the home from its announce file; None when the file does not exist (the home is not installed or
|
|
41
|
+
not running). A file that exists but cannot be read as an announce raises IdentityError. localhost becomes
|
|
42
|
+
127.0.0.1: on Windows ::1 is tried first and a daemon bound to IPv4 only costs about 2 s per connection."""
|
|
33
43
|
try:
|
|
34
|
-
with open(
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
value = value.strip()
|
|
38
|
-
if key.strip() == "tei.url":
|
|
39
|
-
if not value or value.lower() == "off":
|
|
40
|
-
return None
|
|
41
|
-
parsed = urllib.parse.urlsplit(value.rstrip("/"))
|
|
42
|
-
if parsed.hostname == "localhost":
|
|
43
|
-
parsed = parsed._replace(netloc=parsed.netloc.replace("localhost", "127.0.0.1", 1))
|
|
44
|
-
return urllib.parse.urlunsplit(parsed)
|
|
45
|
-
except OSError:
|
|
44
|
+
with open(ANNOUNCE, encoding="utf-8") as stream:
|
|
45
|
+
data = json.load(stream)
|
|
46
|
+
except FileNotFoundError:
|
|
46
47
|
return None
|
|
47
|
-
|
|
48
|
+
except (OSError, ValueError) as exc:
|
|
49
|
+
raise IdentityError(f"The embeddings home announce file {ANNOUNCE} cannot be read ({exc}).") from None
|
|
50
|
+
base = str(data.get("url") or "").strip().rstrip("/") if isinstance(data, dict) else ""
|
|
51
|
+
parsed = urllib.parse.urlsplit(base)
|
|
52
|
+
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
53
|
+
raise IdentityError(f"The embeddings home announce file {ANNOUNCE} has no http(s) url.")
|
|
54
|
+
version = data.get("apiVersion")
|
|
55
|
+
if type(version) is not int or version < MIN_API_VERSION:
|
|
56
|
+
raise IdentityError(f"The embeddings home announces API version {version!r}; Smart Tool needs "
|
|
57
|
+
f"{MIN_API_VERSION} or later. Update the embeddings home.")
|
|
58
|
+
if parsed.hostname == "localhost":
|
|
59
|
+
parsed = parsed._replace(netloc=parsed.netloc.replace("localhost", "127.0.0.1", 1))
|
|
60
|
+
return urllib.parse.urlunsplit(parsed)
|
|
48
61
|
|
|
49
62
|
|
|
50
63
|
def configured():
|
|
51
|
-
|
|
64
|
+
"""Whether an announce file of the home exists (malformed counts: it raises at use, loudly)."""
|
|
65
|
+
return os.path.isfile(ANNOUNCE)
|
|
52
66
|
|
|
53
67
|
|
|
54
68
|
def _connection(base, timeout):
|
|
@@ -63,7 +77,9 @@ def _request(conn, method, path, body=None):
|
|
|
63
77
|
response = conn.getresponse()
|
|
64
78
|
data = response.read()
|
|
65
79
|
if response.status >= 400:
|
|
66
|
-
|
|
80
|
+
detail = data[:300].decode("utf-8", "replace").strip()
|
|
81
|
+
raise urllib.error.HTTPError(f"{conn.host}:{conn.port}{path}", response.status,
|
|
82
|
+
f"{response.reason}{': ' + detail if detail else ''}", response.headers, None)
|
|
67
83
|
return json.loads(data)
|
|
68
84
|
|
|
69
85
|
|
|
@@ -73,48 +89,87 @@ def _probe(base, timeout):
|
|
|
73
89
|
data = _request(conn, "GET", "/info")
|
|
74
90
|
finally:
|
|
75
91
|
conn.close()
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
92
|
+
digest = str(data.get("model_sha256") or "").strip().lower()
|
|
93
|
+
try:
|
|
94
|
+
dimensions = int(data.get("dimension") or 0)
|
|
95
|
+
except (TypeError, ValueError):
|
|
96
|
+
dimensions = 0
|
|
97
|
+
if not digest or dimensions <= 0:
|
|
98
|
+
raise IdentityError(f"The embeddings home at {base} answered /info without model_sha256 or dimension, so its "
|
|
99
|
+
"model cannot be identified; Smart Tool will not fall back to lexical search silently. "
|
|
100
|
+
"Update the embeddings home so /info reports model_sha256 and dimension.")
|
|
101
|
+
cpu = bool(data.get("demoted_to_cpu")) or str(data.get("provider") or "").startswith("CPU")
|
|
102
|
+
rerank_host = data.get("rerank_host") if isinstance(data.get("rerank_host"), dict) else {}
|
|
103
|
+
return {"model": f"{PREFIX}{digest[:16]}-{dimensions}", "name": data.get("model_id") or "", "dimensions": dimensions,
|
|
104
|
+
"url": base, "sha256": digest, "cpu": cpu, "rerank": rerank_host.get("configured") is True}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def runs_on_cpu(model):
|
|
108
|
+
"""Whether `model` is the home's embedder and the home computes on the CPU (no GPU, or demoted to the CPU)."""
|
|
109
|
+
return is_local(model) and bool((info() or {}).get("cpu"))
|
|
80
110
|
|
|
81
111
|
|
|
82
112
|
def info(timeout=1.5):
|
|
83
|
-
"""
|
|
84
|
-
|
|
113
|
+
"""Home /info cached for INFO_TTL_S; None when it is not announced or not answering. A home that is announced or
|
|
114
|
+
answers but cannot be used raises IdentityError. The probe runs under the lock so a slow failing probe never
|
|
115
|
+
overwrites a newer good answer."""
|
|
85
116
|
with _lock:
|
|
86
|
-
if time.monotonic() - _info["at"]
|
|
87
|
-
|
|
88
|
-
base, value = url(), None
|
|
89
|
-
if base:
|
|
117
|
+
if time.monotonic() - _info["at"] >= INFO_TTL_S:
|
|
118
|
+
value, error = None, None
|
|
90
119
|
try:
|
|
91
|
-
|
|
120
|
+
base = url()
|
|
121
|
+
if base:
|
|
122
|
+
value = _probe(base, timeout)
|
|
123
|
+
except IdentityError as exc:
|
|
124
|
+
error = exc
|
|
92
125
|
except (OSError, ValueError, http.client.HTTPException):
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
126
|
+
pass
|
|
127
|
+
_info.update(at=time.monotonic(), value=value, error=error)
|
|
128
|
+
if _info["error"]:
|
|
129
|
+
raise _info["error"]
|
|
130
|
+
return _info["value"]
|
|
96
131
|
|
|
97
132
|
|
|
98
133
|
def is_local(model):
|
|
99
134
|
return bool(model) and model.startswith(PREFIX)
|
|
100
135
|
|
|
101
136
|
|
|
102
|
-
def resolve(
|
|
103
|
-
"""Embedding model
|
|
137
|
+
def resolve(cfg=None, current=None):
|
|
138
|
+
"""Embedding model the index of a folder must be built with, following config `embedder`.
|
|
104
139
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
if
|
|
111
|
-
|
|
140
|
+
`gateway` is the user's explicit choice: the gateway's `embedding_model`, or lexical without one. `home` (the
|
|
141
|
+
default) is the home's embedder. While the home is not running, an index the home can keep (its own, or lexical)
|
|
142
|
+
is kept as it is, so a restart of the home never triggers a rebuild to lexical and back; an index of another model
|
|
143
|
+
gives None: it keeps answering and is not updated, and re-embedding waits for the home. A folder without an index
|
|
144
|
+
is indexed lexically meanwhile. The gateway is never used in place of the home."""
|
|
145
|
+
cfg = cfg if cfg is not None else config.load_config()
|
|
146
|
+
if config.embedder(cfg) == "gateway":
|
|
147
|
+
return cfg.get("embedding_model") or indexer.LEXICAL_MODEL
|
|
112
148
|
available = info()
|
|
113
149
|
if available:
|
|
114
150
|
return available["model"]
|
|
115
|
-
if is_local(current):
|
|
116
|
-
return current
|
|
117
|
-
return
|
|
151
|
+
if not current or is_local(current) or current == indexer.LEXICAL_MODEL:
|
|
152
|
+
return current or indexer.LEXICAL_MODEL
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def rerank_configured():
|
|
157
|
+
"""Whether the home reports a rerank host configured in it."""
|
|
158
|
+
return bool((info() or {}).get("rerank"))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def rerank(query, documents, top_n, deadline=None):
|
|
162
|
+
"""The home's `POST /rerank` answer in the canonical form `{results: [{index, relevance_score}]}`. The home forwards
|
|
163
|
+
to the host the user configured; an error answer raises, it is never replaced by another reranker."""
|
|
164
|
+
base = url()
|
|
165
|
+
if not base:
|
|
166
|
+
raise RuntimeError(f"The embeddings home is not running (no announce file at {ANNOUNCE}).")
|
|
167
|
+
timeout = RERANK_TIMEOUT_S if deadline is None else min(RERANK_TIMEOUT_S, _left(deadline))
|
|
168
|
+
conn = _connection(base, timeout)
|
|
169
|
+
try:
|
|
170
|
+
return _request(conn, "POST", "/rerank", {"query": query, "documents": documents, "top_n": top_n})
|
|
171
|
+
finally:
|
|
172
|
+
conn.close()
|
|
118
173
|
|
|
119
174
|
|
|
120
175
|
def _left(deadline):
|
|
@@ -128,7 +183,7 @@ def _left(deadline):
|
|
|
128
183
|
|
|
129
184
|
def _retry_delay(failure, attempt):
|
|
130
185
|
"""Seconds to wait after a failed attempt: the `Retry-After` of the answer (whole seconds, at most the pause of the
|
|
131
|
-
circuit) when the
|
|
186
|
+
circuit) when the home sent one, else the attempt number."""
|
|
132
187
|
value = str(getattr(failure, "headers", None) and failure.headers.get("Retry-After") or "")
|
|
133
188
|
return min(int(value), model_breaker.PAUSE_S) if value.isdigit() else attempt
|
|
134
189
|
|
|
@@ -136,16 +191,18 @@ def _retry_delay(failure, attempt):
|
|
|
136
191
|
def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
137
192
|
base = url()
|
|
138
193
|
if not base:
|
|
139
|
-
raise RuntimeError(f"
|
|
194
|
+
raise RuntimeError(f"The embeddings home is not running (no announce file at {ANNOUNCE}).")
|
|
140
195
|
# Fresh /info before and after: /embed does not echo the model, and a cached answer could let vectors of a
|
|
141
196
|
# different model reach the persistent vector cache under this model's contract.
|
|
142
197
|
try:
|
|
143
198
|
before = _probe(base, _left(deadline))
|
|
199
|
+
except IdentityError:
|
|
200
|
+
raise
|
|
144
201
|
except (OSError, ValueError, http.client.HTTPException) as exc:
|
|
145
|
-
raise RuntimeError(f"
|
|
202
|
+
raise RuntimeError(f"The embeddings home is not responding ({base}): {exc}") from None
|
|
146
203
|
if before["model"] != model:
|
|
147
204
|
raise indexer.IndexCompatibilityError(
|
|
148
|
-
f"The
|
|
205
|
+
f"The embeddings home now serves {before['model']}; the index was built with {model} and must be rebuilt.")
|
|
149
206
|
vectors = []
|
|
150
207
|
conn = _connection(base, _left(deadline))
|
|
151
208
|
try:
|
|
@@ -165,19 +222,19 @@ def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
|
165
222
|
conn.close()
|
|
166
223
|
conn = _connection(base, _left(deadline))
|
|
167
224
|
if attempt == MAX_ATTEMPTS:
|
|
168
|
-
raise RuntimeError(f"
|
|
225
|
+
raise RuntimeError(f"Embeddings home failed after {MAX_ATTEMPTS} attempts: {exc}") from None
|
|
169
226
|
if on_retry:
|
|
170
227
|
on_retry(attempt + 1, len(part))
|
|
171
228
|
delay = _retry_delay(exc, attempt)
|
|
172
229
|
if deadline is not None and time.monotonic() + delay >= deadline:
|
|
173
|
-
raise TimeoutError("Not enough time left to retry the
|
|
230
|
+
raise TimeoutError("Not enough time left to retry the embeddings home.") from None
|
|
174
231
|
time.sleep(delay)
|
|
175
232
|
else:
|
|
176
233
|
model_breaker.note(model)
|
|
177
234
|
model_health.record("/v1/embeddings", {"model": model})
|
|
178
235
|
break
|
|
179
236
|
if not isinstance(got, list) or len(got) != len(part):
|
|
180
|
-
raise indexer.IndexVectorError("
|
|
237
|
+
raise indexer.IndexVectorError("The embeddings home returned a vector count different from the input.")
|
|
181
238
|
for vector in got:
|
|
182
239
|
indexer.validate_vector(vector, before["dimensions"])
|
|
183
240
|
vectors += got
|
|
@@ -185,5 +242,5 @@ def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
|
185
242
|
conn.close()
|
|
186
243
|
after = _probe(base, _left(deadline))
|
|
187
244
|
if (after["model"], after.get("sha256")) != (before["model"], before.get("sha256")):
|
|
188
|
-
raise indexer.IndexCompatibilityError("The
|
|
245
|
+
raise indexer.IndexCompatibilityError("The embeddings home changed models during the call; try again.")
|
|
189
246
|
return vectors
|
package/locales/pt-BR.json
CHANGED
|
@@ -61,11 +61,7 @@
|
|
|
61
61
|
"Quick search time limit (s)": "Tempo limite da busca rápida (s)",
|
|
62
62
|
"How long a quick web search waits for its sources.": "Quanto tempo uma busca rápida na web espera pelas suas fontes.",
|
|
63
63
|
"Per-site memory similarity": "Similaridade da memória por site",
|
|
64
|
-
"How close a new question must be to a remembered page of the same site to reuse it.": "Quão parecida uma nova pergunta precisa ser de uma página lembrada do mesmo site para reaproveitá-la.",
|
|
65
64
|
"Save models": "Salvar modelos",
|
|
66
|
-
"3. Local models": "3. Modelos locais",
|
|
67
|
-
"Embedding from the mcp-memory sidecar on this machine. Switching between gateway and local rebuilds the project indexes; going back to a model used before reuses its vector cache.": "Embedding do sidecar mcp-memory neste computador. Alternar entre gateway e local reconstrói os índices dos projetos; voltar a um modelo usado antes reaproveita o cache de vetores.",
|
|
68
|
-
"When to use the local model": "Quando usar o modelo local",
|
|
69
65
|
"web_search asks several providers at once. All of them work without a key on their public free tier; add your own key only if you need more volume.": "web_search consulta vários provedores ao mesmo tempo. Todos funcionam sem chave no plano gratuito público; adicione a sua própria chave só se precisar de mais volume.",
|
|
70
66
|
"Providers": "Provedores",
|
|
71
67
|
"A key is tested with a real search before it is saved, and is protected by DPAPI in this Windows profile.": "Uma chave é testada com uma busca real antes de ser salva e fica protegida por DPAPI neste perfil do Windows.",
|
|
@@ -215,16 +211,6 @@
|
|
|
215
211
|
"Saving…": "Salvando…",
|
|
216
212
|
"Models saved.": "Modelos salvos.",
|
|
217
213
|
"Recommended": "Recomendado",
|
|
218
|
-
"Only as a fallback": "Só como alternativa",
|
|
219
|
-
"Use the local model only when the gateway does not have the embedding model.": "Use o modelo local apenas quando o gateway não tiver o modelo de embedding.",
|
|
220
|
-
"Prefer local": "Preferir local",
|
|
221
|
-
"Use the local model even when the gateway has it; your code vectors stay on this machine.": "Use o modelo local mesmo quando o gateway o tiver; os vetores do seu código ficam neste computador.",
|
|
222
|
-
"Never": "Nunca",
|
|
223
|
-
"Always use the gateway.": "Usar sempre o gateway.",
|
|
224
|
-
"Saved. The indexes adjust on the next search or check.": "Salvo. Os índices se ajustam na próxima busca ou verificação.",
|
|
225
|
-
"No sidecar on this machine": "Nenhum sidecar neste computador",
|
|
226
|
-
"Sidecar answering:": "Sidecar respondendo:",
|
|
227
|
-
"Sidecar not answering (": "Sidecar não responde (",
|
|
228
214
|
"Status unavailable": "Status indisponível",
|
|
229
215
|
"Redirect behavior": "Comportamento do redirecionamento",
|
|
230
216
|
"Applies on the next tool call.": "Vale na próxima chamada de ferramenta.",
|
|
@@ -674,7 +660,6 @@
|
|
|
674
660
|
"per-site memory similarity threshold": "limiar de similaridade da memória por site",
|
|
675
661
|
"Send the adapter and the field values in a JSON object.": "Envie o adaptador e os valores dos campos em um objeto JSON.",
|
|
676
662
|
"Setup page session token missing or invalid; open {0}/setup again.": "O token de sessão da página de configuração está ausente ou é inválido; abra {0}/setup de novo.",
|
|
677
|
-
"Invalid mode: use fallback, off or prefer.": "Modo inválido: use fallback, off ou prefer.",
|
|
678
663
|
"Invalid client: use claude or codex.": "Cliente inválido: use claude ou codex.",
|
|
679
664
|
"Invalid request.": "Requisição inválida.",
|
|
680
665
|
"Send the contact as text.": "Envie o contato como texto.",
|
|
@@ -812,7 +797,6 @@
|
|
|
812
797
|
", {0} with your key.": ", {0} com a sua chave.",
|
|
813
798
|
"{0}{1} your attention.": "{0}{1} da sua atenção.",
|
|
814
799
|
"{0}{1} Protected by DPAPI.": "{0}{1} Protegido pelo DPAPI.",
|
|
815
|
-
"Sidecar not answering ({0})": "Sidecar sem resposta ({0})",
|
|
816
800
|
"Not available on this machine ({0}). Pages are still read by web_fetch.": "Indisponível neste computador ({0}). As páginas continuam sendo lidas pelo web_fetch.",
|
|
817
801
|
"Registered at {0}. Gives the agent smart_search, web_search and web_fetch.": "Registrado em {0}. Dá ao agente smart_search, web_search e web_fetch.",
|
|
818
802
|
"Registering the MCP server in {0}…": "Registrando o servidor MCP no {0}…",
|
|
@@ -910,7 +894,58 @@
|
|
|
910
894
|
"Tip only": "Só dica",
|
|
911
895
|
"Patterns you accepted (": "Padrões que você aceitou (",
|
|
912
896
|
"Patterns you accepted ({0})": "Padrões que você aceitou ({0})",
|
|
913
|
-
"{0} holds a setup audit of {1} from an older Smart Tool, whose findings have no fix: run project_manage action=setup_audit mode=run to replace it.": "{0} guarda uma auditoria da configuração de {1} feita por um Smart Tool mais antigo, cujos achados não trazem a correção: rode project_manage action=setup_audit mode=run para substituí-la."
|
|
897
|
+
"{0} holds a setup audit of {1} from an older Smart Tool, whose findings have no fix: run project_manage action=setup_audit mode=run to replace it.": "{0} guarda uma auditoria da configuração de {1} feita por um Smart Tool mais antigo, cujos achados não trazem a correção: rode project_manage action=setup_audit mode=run para substituí-la.",
|
|
898
|
+
"The embeddings home (casa-de-embeddings) is the embedding service of this machine, shared with other tools; its rerank connector forwards to the rerank host configured in it. Switching the embedder re-embeds each project in the background: the old index keeps answering until the new one is complete, then the old vectors are deleted.": "A casa de embeddings (casa-de-embeddings) é o serviço de embeddings desta máquina, compartilhado com outras ferramentas; o conector de rerank dela repassa ao host de rerank configurado nela. Trocar o embedder reembute cada projeto em segundo plano: o índice antigo continua respondendo até o novo ficar completo, e depois os vetores antigos são apagados.",
|
|
899
|
+
"Embedder for the project indexes": "Embedder dos índices dos projetos",
|
|
900
|
+
"Embeddings home": "Casa de embeddings",
|
|
901
|
+
"The embedding service of this machine; your code goes to the address it announces, not to the gateway.": "O serviço de embeddings desta máquina; seu código vai para o endereço que ela anuncia, não para o gateway.",
|
|
902
|
+
"The gateway's embedding_model and rerank_model; your code goes to the gateway.": "O embedding_model e o rerank_model do gateway; seu código vai para o gateway.",
|
|
903
|
+
"Saved. Each project is re-embedded in the background; its old index answers meanwhile.": "Salvo. Cada projeto é reembutido em segundo plano; o índice antigo responde enquanto isso.",
|
|
904
|
+
"Embeddings home not running": "Casa de embeddings não está rodando",
|
|
905
|
+
"Embeddings home not answering (": "Casa de embeddings não responde (",
|
|
906
|
+
"Gateway": "Gateway",
|
|
907
|
+
"Embeddings home answering:": "Casa de embeddings respondendo:",
|
|
908
|
+
"· rerank host configured": "· host de rerank configurado",
|
|
909
|
+
"Embeddings home not answering ({0})": "Casa de embeddings sem resposta ({0})",
|
|
910
|
+
"Invalid embedder: use home or gateway.": "Embedder inválido: use home ou gateway.",
|
|
911
|
+
"How close a new question must be to a remembered page of the same site to reuse it, with the gateway embedder; the embeddings home uses 0.62, measured on its model.": "Quão próxima uma pergunta nova precisa estar de uma página lembrada do mesmo site para reaproveitá-la, com o embedder do gateway; a casa de embeddings usa 0,62, medido no modelo dela.",
|
|
912
|
+
"Model use (last 30 days)": "Uso de modelos (últimos 30 dias)",
|
|
913
|
+
"Calls, failures and, when the gateway reports them, tokens and cost, per purpose and over every project. A purpose whose calls in one hour pass twice its own busiest hour of the previous week (and at least 100) is flagged here; it never raises a notification.": "Chamadas, falhas e, quando o gateway informa, tokens e custo, por finalidade e em todos os projetos. Uma finalidade cujas chamadas em uma hora passam do dobro da sua própria hora mais cheia da semana anterior (e de pelo menos 100) é sinalizada aqui; nunca gera notificação.",
|
|
914
|
+
"Volume alert": "Alerta de volume",
|
|
915
|
+
"Prompt tokens": "Tokens de entrada",
|
|
916
|
+
"Output tokens": "Tokens de saída",
|
|
917
|
+
"Reported cost": "Custo informado",
|
|
918
|
+
"No model call in the last 30 days.": "Nenhuma chamada de modelo nos últimos 30 dias.",
|
|
919
|
+
"volume alert": "alerta de volume",
|
|
920
|
+
"Could not read the usage meter:": "Não foi possível ler o medidor de uso:",
|
|
921
|
+
"Volume alert muted for 7 days.": "Alerta de volume silenciado por 7 dias.",
|
|
922
|
+
"Automatic updates": "Atualizações automáticas",
|
|
923
|
+
"Smart Tool checks npm at start and every 24 hours and, when this is on, installs a newer version by itself. The switch is the same as in the tray menu.": "O Smart Tool consulta o npm ao iniciar e a cada 24 horas e, com isto ligado, instala sozinho uma versão mais nova. A chave é a mesma do menu da bandeja.",
|
|
924
|
+
"Installed: {0}.": "Instalada: {0}.",
|
|
925
|
+
"Last check: {0}.": "Última verificação: {0}.",
|
|
926
|
+
"Not checked yet.": "Ainda não verificado.",
|
|
927
|
+
"Smart Tool is up to date.": "O Smart Tool está atualizado.",
|
|
928
|
+
"Updated from {0} to {1} on {2}.": "Atualizado da {0} para a {1} em {2}.",
|
|
929
|
+
"Automatic updates turned on.": "Atualizações automáticas ligadas.",
|
|
930
|
+
"Automatic updates turned off.": "Atualizações automáticas desligadas.",
|
|
931
|
+
"Send {\"enabled\": true or false}.": "Envie {\"enabled\": true ou false}.",
|
|
932
|
+
"{0}=off is set: automatic updates stay off until it is removed.": "{0}=off está definida: as atualizações automáticas ficam desligadas até removê-la.",
|
|
933
|
+
"Who asked the searches": "Quem pediu as buscas",
|
|
934
|
+
"The recent web search calls grouped by the client and project folder that made them. The text of a search is never recorded.": "As chamadas recentes de busca na web agrupadas pelo cliente e pela pasta do projeto que as fez. O texto de uma busca nunca é gravado.",
|
|
935
|
+
"Sessions": "Sessões",
|
|
936
|
+
"Attributed by": "Atribuído por",
|
|
937
|
+
"No web search recorded yet.": "Nenhuma busca na web registrada ainda.",
|
|
938
|
+
"Could not read the search log:": "Não foi possível ler o registro de buscas:",
|
|
939
|
+
"Calls that go up the layers on edit": "Chamadas que sobem as camadas na edição",
|
|
940
|
+
"When the project has confirmed layers (.smart-tool/layers.json), warns about a call an edit adds from a lower layer to a function of a higher one. A project without layers is never checked. It never blocks.": "Quando o projeto tem camadas confirmadas (.smart-tool/layers.json), avisa sobre uma chamada que a edição adiciona de uma camada inferior para uma função de uma superior. Projeto sem camadas nunca é verificado. Nunca bloqueia.",
|
|
941
|
+
"The edit runs; the agent is told which new calls go up the layers.": "A edição roda; o agente é avisado de quais chamadas novas sobem as camadas.",
|
|
942
|
+
"No layer checks.": "Sem verificação de camadas.",
|
|
943
|
+
"{0} volume alert": "{0} alerta de volume",
|
|
944
|
+
"{0} volume alerts": "{0} alertas de volume",
|
|
945
|
+
"In {0}, move each listed package to a version past its advisories; project_manage action=deps lists every advisory ID.": "Em {0}, mova cada pacote listado para uma versão além dos seus avisos; project_manage action=deps lista cada ID de aviso.",
|
|
946
|
+
"dependency advisories": "avisos de dependências",
|
|
947
|
+
"Fix the cause named in the detail (OSV unreachable, or the advisory cache file), then run the audit again.": "Corrija a causa indicada no detalhe (OSV inacessível ou o arquivo de cache de avisos) e rode a auditoria de novo.",
|
|
948
|
+
"; {0} more": "; mais {0}"
|
|
914
949
|
},
|
|
915
950
|
"ignore": [
|
|
916
951
|
"and {0} more",
|
package/low_priority.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Lowers the scheduling priority of the calling thread for a block, so a long index build on a machine without a GPU
|
|
2
|
+
leaves the processor to the user's own programs. Only Windows has a per-thread priority that can be put back; on any
|
|
3
|
+
other system the block runs unchanged (`current()` returns None)."""
|
|
4
|
+
import contextlib
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
BELOW_NORMAL = -1
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _kernel():
|
|
11
|
+
import ctypes
|
|
12
|
+
kernel = ctypes.windll.kernel32
|
|
13
|
+
kernel.GetCurrentThread.restype = ctypes.c_void_p
|
|
14
|
+
kernel.GetThreadPriority.argtypes = [ctypes.c_void_p]
|
|
15
|
+
kernel.SetThreadPriority.argtypes = [ctypes.c_void_p, ctypes.c_int]
|
|
16
|
+
return kernel
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def current():
|
|
20
|
+
"""Priority of the calling thread (0 is normal, -1 below normal); None where it cannot be read."""
|
|
21
|
+
if sys.platform != "win32":
|
|
22
|
+
return None
|
|
23
|
+
kernel = _kernel()
|
|
24
|
+
return kernel.GetThreadPriority(kernel.GetCurrentThread())
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@contextlib.contextmanager
|
|
28
|
+
def background():
|
|
29
|
+
"""Runs the block with the calling thread below normal priority and restores the previous one afterwards."""
|
|
30
|
+
if sys.platform != "win32":
|
|
31
|
+
yield
|
|
32
|
+
return
|
|
33
|
+
kernel = _kernel()
|
|
34
|
+
thread = kernel.GetCurrentThread()
|
|
35
|
+
previous = kernel.GetThreadPriority(thread)
|
|
36
|
+
if not kernel.SetThreadPriority(thread, BELOW_NORMAL):
|
|
37
|
+
raise OSError("SetThreadPriority failed; the index build cannot run at low priority.")
|
|
38
|
+
try:
|
|
39
|
+
yield
|
|
40
|
+
finally:
|
|
41
|
+
kernel.SetThreadPriority(thread, previous)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.80",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Smart Tool contributors",
|