@allansantos-dev/smart-tool 0.9.39 → 0.9.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/README.md +12 -5
- package/config.py +11 -0
- package/dependency_notice.py +280 -0
- package/document_text.py +114 -3
- package/freshness.py +64 -19
- package/hook_decision.py +41 -11
- package/index_profile.py +1 -1
- package/index_scope.py +9 -1
- package/indexer.py +10 -8
- package/model_breaker.py +70 -0
- package/model_client.py +17 -10
- package/model_health.py +46 -4
- package/package.json +1 -1
- package/project_monitor.py +17 -2
- package/scout.py +16 -3
- package/setup_audit.py +46 -4
- package/setup_ui.py +37 -10
- package/smart_tool_daemon.py +130 -68
- package/tray.py +7 -1
- package/usage_meter.py +13 -0
- package/version.py +1 -1
- package/web_fetch.py +13 -41
package/hook_decision.py
CHANGED
|
@@ -12,9 +12,11 @@ chamada não dá para montar. Nenhuma decisão aqui é sobre periculosidade, só
|
|
|
12
12
|
"""
|
|
13
13
|
import os
|
|
14
14
|
|
|
15
|
+
import affected_tests
|
|
15
16
|
import code_graph
|
|
16
17
|
import code_impact
|
|
17
18
|
import config
|
|
19
|
+
import dependency_notice
|
|
18
20
|
import doc_check
|
|
19
21
|
import duplicates
|
|
20
22
|
import edit_preview
|
|
@@ -182,7 +184,16 @@ def _impact_line(root, rel, name):
|
|
|
182
184
|
counts = found["counts"]
|
|
183
185
|
tests = ", ".join(f"{t['function']} ({t['path']})" for t in found["tests"])
|
|
184
186
|
calls = f"; tests calling it: {tests}" if tests else ""
|
|
185
|
-
return f" {counts['callers']} caller(s){calls}; project_manage affected_tests
|
|
187
|
+
return (f" {counts['callers']} caller(s){calls}{_run_command(root, found['tests'])}; project_manage affected_tests "
|
|
188
|
+
"lists every test to run after the edit.")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _run_command(root, tests):
|
|
192
|
+
"""`; run them: <command>` for the test files of `tests`, one command per runner and folder, or "" when no runner
|
|
193
|
+
is known for them."""
|
|
194
|
+
commands = affected_tests._commands(root, sorted({t["path"] for t in tests})) if tests else []
|
|
195
|
+
runnable = [f"cd {c['cwd']} && {c['command']}" if c["cwd"] != "." else c["command"] for c in commands if c["command"]]
|
|
196
|
+
return f"; run them: {' ; '.join(runnable)}" if runnable else ""
|
|
186
197
|
|
|
187
198
|
|
|
188
199
|
def _doc_lines(findings):
|
|
@@ -215,19 +226,37 @@ def _duplicate_lines(findings):
|
|
|
215
226
|
+ " If the copy is intentional (moving the function, a deliberate fork), go ahead.")
|
|
216
227
|
|
|
217
228
|
|
|
218
|
-
def _edit_review(payload, doc_mode, duplicate_mode):
|
|
229
|
+
def _edit_review(payload, doc_mode, duplicate_mode, dependency_mode):
|
|
219
230
|
"""Checks of an edit before it runs. Docstrings follow doc_mode, in code files of registered projects only (tests,
|
|
220
231
|
docs and files outside a project are left out, as in the docs coverage): missing ones deny the edit in require mode and become a reminder in remind mode; a
|
|
221
232
|
documented function whose signature changed always gets a reminder. Copies of indexed functions follow
|
|
222
|
-
duplicate_mode and only add context, never deny.
|
|
233
|
+
duplicate_mode and only add context, never deny. An edit of a package.json, requirements*.txt or go.mod follows dependency_mode: advisories
|
|
234
|
+
of the added or changed dependencies are added as context, never a denial."""
|
|
223
235
|
with code_graph.pending_scope():
|
|
224
|
-
return _edit_review_scoped(payload, doc_mode, duplicate_mode)
|
|
236
|
+
return _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode)
|
|
225
237
|
|
|
226
238
|
|
|
227
|
-
def
|
|
239
|
+
def _dependency_part(before, after, ecosystem):
|
|
240
|
+
"""The dependency notice of one manifest edit, or the "dependency check skipped" line when it cannot run.
|
|
241
|
+
|
|
242
|
+
Parameters: before (str or None) text before the edit; after (str) text after it; ecosystem (str) from
|
|
243
|
+
`dependency_notice.ecosystem_of`.
|
|
244
|
+
Returns: str, empty when there is nothing to say.
|
|
245
|
+
"""
|
|
246
|
+
try:
|
|
247
|
+
return dependency_notice.notice(before, after, ecosystem)
|
|
248
|
+
except dependency_notice.DependencyCheckError as exc:
|
|
249
|
+
return f"{dependency_notice.PREFIX}dependency check skipped: {exc}"
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _edit_review_scoped(payload, doc_mode, duplicate_mode, dependency_mode):
|
|
228
253
|
"""`_edit_review` inside one `code_graph.pending_scope`, so the pending-changes scan runs once per review."""
|
|
229
|
-
doc_findings, duplicate_findings, failures = [], [], []
|
|
254
|
+
doc_findings, duplicate_findings, failures, dependency_parts = [], [], [], []
|
|
230
255
|
for path, before, after in edit_preview.preview(payload.get("tool_name"), payload.get("tool_input"), payload.get("cwd")):
|
|
256
|
+
ecosystem = dependency_notice.ecosystem_of(path)
|
|
257
|
+
if dependency_mode != "off" and ecosystem:
|
|
258
|
+
dependency_parts += [part for part in (_dependency_part(before, after, ecosystem),) if part]
|
|
259
|
+
continue
|
|
231
260
|
if not doc_check.supported(path):
|
|
232
261
|
continue
|
|
233
262
|
root = _project_of(path)
|
|
@@ -242,7 +271,7 @@ def _edit_review_scoped(payload, doc_mode, duplicate_mode):
|
|
|
242
271
|
except (ValueError, RuntimeError) as exc:
|
|
243
272
|
failures.append(f"Smart Tool duplicates check skipped: {exc}")
|
|
244
273
|
parts = ([_doc_lines(doc_findings)] if doc_findings else []) + \
|
|
245
|
-
([_duplicate_lines(duplicate_findings)] if duplicate_findings else []) + failures
|
|
274
|
+
([_duplicate_lines(duplicate_findings)] if duplicate_findings else []) + failures + dependency_parts
|
|
246
275
|
if not parts:
|
|
247
276
|
return {}
|
|
248
277
|
message = " ".join(parts)
|
|
@@ -253,16 +282,17 @@ def _edit_review_scoped(payload, doc_mode, duplicate_mode):
|
|
|
253
282
|
|
|
254
283
|
def decide(payload, client, web_route, jobs):
|
|
255
284
|
"""Hook output for one PreToolUse payload: edits follow doc_mode (require | remind | off) and duplicate_mode
|
|
256
|
-
(warn | off), the other tools follow hook_mode (redirect | advise | off). In redirect mode the Smart Tool call that
|
|
285
|
+
(warn | off) and dependency_mode (warn | off), the other tools follow hook_mode (redirect | advise | off). In redirect mode the Smart Tool call that
|
|
257
286
|
replaces a redirected one starts in `jobs` under the call's tool_use_id (see `_redirect`); only a search whose
|
|
258
287
|
query cannot be built, or a client that sends no tool_use_id, is denied with the exact call to make, and never in a
|
|
259
288
|
subagent whose tool list has no Smart Tool MCP tool: it has nothing to switch to."""
|
|
260
289
|
cfg = config.load_config()
|
|
261
290
|
if isinstance(payload, dict) and payload.get("tool_name") in EDIT_TOOLS:
|
|
262
|
-
doc_mode, duplicate_mode = config.doc_mode(cfg), config.duplicate_mode(cfg)
|
|
263
|
-
|
|
291
|
+
doc_mode, duplicate_mode, dependency_mode = (config.doc_mode(cfg), config.duplicate_mode(cfg),
|
|
292
|
+
config.dependency_mode(cfg))
|
|
293
|
+
if doc_mode == "off" and duplicate_mode == "off" and dependency_mode == "off":
|
|
264
294
|
return {}
|
|
265
|
-
return _edit_review(payload, doc_mode, duplicate_mode)
|
|
295
|
+
return _edit_review(payload, doc_mode, duplicate_mode, dependency_mode)
|
|
266
296
|
if isinstance(payload, dict) and str(payload.get("tool_name")).startswith(f"mcp__{SMART_TOOL_SERVER}__"):
|
|
267
297
|
if payload["tool_name"] == SMART_SEARCH_TOOL:
|
|
268
298
|
router.CLIENT.set(client)
|
package/index_profile.py
CHANGED
|
@@ -11,7 +11,7 @@ PROFILE_VERSION = 2
|
|
|
11
11
|
KINDS = ("code", "test", "doc")
|
|
12
12
|
RETRY_AFTER_S = 3600
|
|
13
13
|
MAX_DIRS = 90
|
|
14
|
-
PROSE = {"md", "txt", "docx", "rst", "adoc"}
|
|
14
|
+
PROSE = {"md", "txt", "docx", "pdf", "rst", "adoc"}
|
|
15
15
|
# Data and markup inside a documentation, design or data-dump folder are reports and mockups, not code (GitHub Linguist
|
|
16
16
|
# treats docs/ as documentation; a design/ HTML or a probe's JSON result pushed hook_decision.py out of the code block).
|
|
17
17
|
DOC_DATA = {"html", "htm", "json", "csv", "xml", "yaml", "yml", "svg"}
|
package/index_scope.py
CHANGED
|
@@ -40,10 +40,13 @@ _PROJECT_TYPE_EXCLUDES = (
|
|
|
40
40
|
(("Podfile",), ("Pods",)),
|
|
41
41
|
)
|
|
42
42
|
|
|
43
|
+
# Files a tool writes while it works and replaces or deletes right after (atomic-write temp, backup, editor swap).
|
|
44
|
+
TRANSIENT_FILES = {"*.tmp", "*.bak", "*.swp", "*~"}
|
|
45
|
+
|
|
43
46
|
# Mesmo critério do `_ALWAYS_EXCLUDE`, para arquivo: o significado não varia por
|
|
44
47
|
# projeto. Lockfile é grafo de dependência resolvido por ferramenta — nunca é fonte de
|
|
45
48
|
# resposta numa busca semântica, e é grande o bastante pra dominar o índice.
|
|
46
|
-
_ALWAYS_EXCLUDE_FILES = {
|
|
49
|
+
_ALWAYS_EXCLUDE_FILES = TRANSIENT_FILES | {
|
|
47
50
|
"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "npm-shrinkwrap.json",
|
|
48
51
|
"poetry.lock", "pdm.lock", "uv.lock", "Pipfile.lock", "Cargo.lock",
|
|
49
52
|
"composer.lock", "go.sum", "Gemfile.lock", "packages.lock.json",
|
|
@@ -110,6 +113,11 @@ def normalize_include(include):
|
|
|
110
113
|
return bases, dropped
|
|
111
114
|
|
|
112
115
|
|
|
116
|
+
def is_transient(name):
|
|
117
|
+
"""Whether a file name is a temporary, backup or editor-swap file (`*.tmp`, `*.bak`, `*.swp`, `*~`)."""
|
|
118
|
+
return any(fnmatch.fnmatch(name.casefold(), pattern) for pattern in TRANSIENT_FILES)
|
|
119
|
+
|
|
120
|
+
|
|
113
121
|
def always_excluded(name):
|
|
114
122
|
"""Directory name in _ALWAYS_EXCLUDE, entries may be globs (e.g. *.egg-info)."""
|
|
115
123
|
folded = name.casefold()
|
package/indexer.py
CHANGED
|
@@ -31,8 +31,10 @@ CHUNK_MAX_CHARS = 8000
|
|
|
31
31
|
CHUNK_OVERLAP_LINES = 12
|
|
32
32
|
CHUNKER_VERSION = 3
|
|
33
33
|
SCHEMA_VERSION = 2
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
# An embeddings request holds up to 128 items and 120,000 characters (about 30,000 tokens): far under the 2048 items and
|
|
35
|
+
# 300,000 tokens per request that OpenAI and Azure accept (docs/research/2026-10-27-excesso-de-chamadas-embed-rerank.md).
|
|
36
|
+
REINDEX_BATCH_MAX_CHARS = 120_000
|
|
37
|
+
REINDEX_BATCH_MAX_ITEMS = 128
|
|
36
38
|
# Embedding requests in flight per flush. Measured 2026-10-03 (docs/spikes/embed_parallel_spike.py, 30 batches,
|
|
37
39
|
# 3 interleaved rounds): 1 worker 33-50 s, 2 workers 19-27 s, 4 workers 11-20 s (~2.8x), no 429 from the gateway.
|
|
38
40
|
EMBED_WORKERS = 4
|
|
@@ -129,7 +131,7 @@ def read_indexable(root, rel_path):
|
|
|
129
131
|
size = os.path.getsize(full)
|
|
130
132
|
if name.endswith(".svg") and size > MAX_SVG_INDEXABLE_BYTES:
|
|
131
133
|
return None, SKIP_GRAPHIC
|
|
132
|
-
limit = document_text.
|
|
134
|
+
limit = document_text.size_limit(name) if document_text.is_extracted(name) else MAX_INDEXABLE_BYTES
|
|
133
135
|
if size > limit:
|
|
134
136
|
return None, SKIP_TOO_LARGE
|
|
135
137
|
with open(full, "rb") as f:
|
|
@@ -138,9 +140,9 @@ def read_indexable(root, rel_path):
|
|
|
138
140
|
return None, SKIP_UNREADABLE
|
|
139
141
|
if len(data) > limit:
|
|
140
142
|
return None, SKIP_TOO_LARGE
|
|
141
|
-
if
|
|
143
|
+
if document_text.is_extracted(name):
|
|
142
144
|
try:
|
|
143
|
-
data = document_text.
|
|
145
|
+
data = document_text.extract(name, data)
|
|
144
146
|
except document_text.DocumentError as exc:
|
|
145
147
|
return None, str(exc)
|
|
146
148
|
prefix = data[:_SNIFF_BYTES]
|
|
@@ -507,7 +509,7 @@ def _stat_candidate(root, rel):
|
|
|
507
509
|
return None, SKIP_UNREADABLE
|
|
508
510
|
except OSError:
|
|
509
511
|
return None, SKIP_UNREADABLE
|
|
510
|
-
if stat.st_size > (document_text.
|
|
512
|
+
if stat.st_size > (document_text.size_limit(name) if document_text.is_extracted(name) else MAX_INDEXABLE_BYTES):
|
|
511
513
|
return None, SKIP_TOO_LARGE
|
|
512
514
|
if name.endswith(".svg") and stat.st_size > MAX_SVG_INDEXABLE_BYTES:
|
|
513
515
|
return None, SKIP_GRAPHIC
|
|
@@ -523,7 +525,7 @@ def _stable_read(root, rel):
|
|
|
523
525
|
if data is None:
|
|
524
526
|
return None, encoding, before
|
|
525
527
|
after, _ = _stat_candidate(root, rel)
|
|
526
|
-
if after == before and (
|
|
528
|
+
if after == before and (document_text.is_extracted(rel) or len(data) == before[1]):
|
|
527
529
|
return data, encoding, before
|
|
528
530
|
return None, "file changed while reading", None
|
|
529
531
|
|
|
@@ -621,7 +623,7 @@ def _reindex(root, included_paths, embed_batch, chunk_lines=CHUNK_LINES, on_prog
|
|
|
621
623
|
continue
|
|
622
624
|
observed[rel] = stat
|
|
623
625
|
force_file = any(rel.casefold() == p or rel.casefold().startswith(p + "/") for p in forced)
|
|
624
|
-
extractor_changed =
|
|
626
|
+
extractor_changed = document_text.is_extracted(rel) and meta.get('document_extractor_version') != document_text.VERSION
|
|
625
627
|
if (full_check or building or force_file or extractor_changed or old_state.get(rel) != stat):
|
|
626
628
|
candidates.append(rel)
|
|
627
629
|
else:
|
package/model_breaker.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Protection of the model gateway, per model id: a circuit that opens after consecutive 429/503 answers and a cap on
|
|
2
|
+
the requests in flight.
|
|
3
|
+
|
|
4
|
+
Pattern: a circuit breaker (closed, open for a pause, then one trial call decides). Retrying a gateway that answers
|
|
5
|
+
busy only adds load (AWS Bedrock throughput guide: honor Retry-After, back off with jitter); once it keeps answering
|
|
6
|
+
busy the caller stops asking for a while and serves what it can without the model.
|
|
7
|
+
"""
|
|
8
|
+
import contextlib
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
import urllib.error
|
|
12
|
+
|
|
13
|
+
FAILURES_TO_OPEN = 3
|
|
14
|
+
PAUSE_S = 30
|
|
15
|
+
MAX_IN_FLIGHT = 6
|
|
16
|
+
_BUSY_CODES = (429, 503)
|
|
17
|
+
_lock = threading.Lock()
|
|
18
|
+
_failures = {}
|
|
19
|
+
_open_until = {}
|
|
20
|
+
_slots = {}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class CircuitOpen(RuntimeError):
|
|
24
|
+
"""The model is paused after repeated busy answers; no request was sent."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def guard(model):
|
|
28
|
+
"""Raises CircuitOpen while `model` is paused. Nothing happens for a call without a model id."""
|
|
29
|
+
if not model:
|
|
30
|
+
return
|
|
31
|
+
with _lock:
|
|
32
|
+
left = _open_until.get(model, 0) - time.monotonic()
|
|
33
|
+
if left > 0:
|
|
34
|
+
raise CircuitOpen(f"Model {model} is paused for {int(left) + 1} s after {FAILURES_TO_OPEN} consecutive "
|
|
35
|
+
"HTTP 429/503 answers from the gateway; no request was sent.")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def note(model, failure=None):
|
|
39
|
+
"""Records the outcome of a request: a success closes the circuit, a 429/503 counts toward opening it, any other
|
|
40
|
+
failure changes nothing. After the pause the first failure opens it again."""
|
|
41
|
+
if not model:
|
|
42
|
+
return
|
|
43
|
+
busy = isinstance(failure, urllib.error.HTTPError) and failure.code in _BUSY_CODES
|
|
44
|
+
with _lock:
|
|
45
|
+
if failure is None:
|
|
46
|
+
_failures[model] = 0
|
|
47
|
+
elif busy:
|
|
48
|
+
_failures[model] = _failures.get(model, 0) + 1
|
|
49
|
+
if _failures[model] >= FAILURES_TO_OPEN:
|
|
50
|
+
_open_until[model] = time.monotonic() + PAUSE_S
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@contextlib.contextmanager
|
|
54
|
+
def slot(model):
|
|
55
|
+
"""Holds one of the `MAX_IN_FLIGHT` places of `model` for the duration of a request."""
|
|
56
|
+
if not model:
|
|
57
|
+
yield
|
|
58
|
+
return
|
|
59
|
+
with _lock:
|
|
60
|
+
semaphore = _slots.setdefault(model, threading.BoundedSemaphore(MAX_IN_FLIGHT))
|
|
61
|
+
with semaphore:
|
|
62
|
+
yield
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def reset():
|
|
66
|
+
"""Forgets every circuit and cap (tests)."""
|
|
67
|
+
with _lock:
|
|
68
|
+
_failures.clear()
|
|
69
|
+
_open_until.clear()
|
|
70
|
+
_slots.clear()
|
package/model_client.py
CHANGED
|
@@ -9,6 +9,7 @@ import usage_meter
|
|
|
9
9
|
|
|
10
10
|
import config
|
|
11
11
|
import gateway
|
|
12
|
+
import model_breaker
|
|
12
13
|
import model_health
|
|
13
14
|
|
|
14
15
|
OFFLINE = contextvars.ContextVar("model_offline", default=None)
|
|
@@ -26,18 +27,24 @@ def get_token():
|
|
|
26
27
|
|
|
27
28
|
|
|
28
29
|
def fetch(path, token, method="GET", body=None, timeout=15):
|
|
30
|
+
"""One gateway request; returns the parsed JSON. A call that names a model first passes that model's circuit and
|
|
31
|
+
request cap (model_breaker, CircuitOpen when paused), and every attempt is metered. HTTP errors propagate."""
|
|
29
32
|
if OFFLINE.get():
|
|
30
33
|
raise RuntimeError(OFFLINE.get())
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
34
|
+
model=body.get("model") if isinstance(body,dict) else None
|
|
35
|
+
model_breaker.guard(model)
|
|
36
|
+
with model_breaker.slot(model):
|
|
37
|
+
started=time.monotonic();result=None;headers=None;failure=None
|
|
38
|
+
try:
|
|
39
|
+
result,headers=_fetch(path,token,method,body,timeout)
|
|
40
|
+
return result
|
|
41
|
+
except Exception as exc:
|
|
42
|
+
failure=exc
|
|
43
|
+
raise
|
|
44
|
+
finally:
|
|
45
|
+
model_breaker.note(model,failure)
|
|
46
|
+
usage_meter.record(path,body,result,headers,failure,time.monotonic()-started)
|
|
47
|
+
model_health.record(path,body,failure)
|
|
41
48
|
|
|
42
49
|
|
|
43
50
|
def _fetch(path, _token, method="GET", body=None, timeout=15):
|
package/model_health.py
CHANGED
|
@@ -3,17 +3,19 @@ rerank). Never stores request content: the model, the role, the tool that was ru
|
|
|
3
3
|
text and the time.
|
|
4
4
|
|
|
5
5
|
Three consumers read it: the tool answer (a warning line when a call of that very tool failed), the setup screen
|
|
6
|
-
(rate table and
|
|
6
|
+
(rate table, failures by type and calls per purpose) and the tray (one toast per model per day when the rate passes the limit).
|
|
7
7
|
"""
|
|
8
8
|
import contextlib
|
|
9
9
|
import contextvars
|
|
10
10
|
import io
|
|
11
11
|
import os
|
|
12
|
+
import re
|
|
12
13
|
import sqlite3
|
|
13
14
|
import time
|
|
14
15
|
|
|
15
16
|
import config
|
|
16
17
|
import project_store
|
|
18
|
+
import usage_meter
|
|
17
19
|
|
|
18
20
|
WINDOW_S = 24 * 3600
|
|
19
21
|
ALERT_MIN_CALLS = 5
|
|
@@ -23,10 +25,17 @@ MUTE_S = 7 * 24 * 3600
|
|
|
23
25
|
KEEP_S = 7 * 24 * 3600
|
|
24
26
|
KEEP_FAILURES = 500
|
|
25
27
|
ERROR_CHARS = 300
|
|
28
|
+
SAMPLE_CHARS = 160
|
|
29
|
+
GROUP_ROWS = 20
|
|
30
|
+
PURPOSES = ("index_embedding", "query_embedding", "rerank", "embedding_probe", "scope")
|
|
26
31
|
_PATHS = ("/v1/chat/completions", "/v1/embeddings", "/v1/rerank")
|
|
27
32
|
_TOOL = contextvars.ContextVar("model_health_tool", default=None)
|
|
28
33
|
_COLLECTED = contextvars.ContextVar("model_health_collected", default=None)
|
|
29
34
|
_SCHEMA = set()
|
|
35
|
+
_STATUS = re.compile(r"HTTP(?: Error)? (\d{3})")
|
|
36
|
+
_TIMEOUT = re.compile(r"timed out|timeout|WinError 10060|Errno 110\b", re.I)
|
|
37
|
+
_CONNECTION = re.compile(r"URLError|ConnectionError|ConnectionRefused|ConnectionReset|refused|reset|getaddrinfo|"
|
|
38
|
+
r"WinError 1006[01]|WinError 10054|Name or service|unreachable", re.I)
|
|
30
39
|
|
|
31
40
|
|
|
32
41
|
@contextlib.contextmanager
|
|
@@ -56,6 +65,22 @@ def _tables(db):
|
|
|
56
65
|
_SCHEMA.add(project_store.DB_PATH)
|
|
57
66
|
|
|
58
67
|
|
|
68
|
+
def classify(error):
|
|
69
|
+
"""Failure type of a stored error text: '429', '503', '5xx' (any other 5xx), 'auth' (401/403), 'timeout',
|
|
70
|
+
'connection' or 'other'. An HTTP status in the text wins over words in the response body."""
|
|
71
|
+
status = _STATUS.search(error or "")
|
|
72
|
+
if status:
|
|
73
|
+
code = int(status.group(1))
|
|
74
|
+
if code in (429, 503):
|
|
75
|
+
return str(code)
|
|
76
|
+
if code in (401, 403):
|
|
77
|
+
return "auth"
|
|
78
|
+
return "5xx" if code >= 500 else "other"
|
|
79
|
+
if _TIMEOUT.search(error or ""):
|
|
80
|
+
return "timeout"
|
|
81
|
+
return "connection" if _CONNECTION.search(error or "") else "other"
|
|
82
|
+
|
|
83
|
+
|
|
59
84
|
def role_of(model, cfg=None):
|
|
60
85
|
"""Name of the config field that selects `model` ('other' when none does)."""
|
|
61
86
|
cfg = cfg if cfg is not None else config.load_config()
|
|
@@ -111,8 +136,24 @@ def warning_text(failures, setup_url):
|
|
|
111
136
|
f"{first['error']}. Pick another model: {setup_url}/setup#models")
|
|
112
137
|
|
|
113
138
|
|
|
139
|
+
def _groups(rows):
|
|
140
|
+
"""Failure rows (latest first) as one entry per (type, role): count, first and last time, the latest message cut to
|
|
141
|
+
SAMPLE_CHARS and the latest GROUP_ROWS raw rows. Largest group first."""
|
|
142
|
+
groups = {}
|
|
143
|
+
for row in rows:
|
|
144
|
+
group = groups.setdefault((classify(row["error"]), row["role"]), {
|
|
145
|
+
"type": classify(row["error"]), "role": row["role"], "count": 0, "first_at": row["at"], "last_at": row["at"],
|
|
146
|
+
"sample": row["error"][:SAMPLE_CHARS], "rows": []})
|
|
147
|
+
group["count"] += 1
|
|
148
|
+
group["first_at"] = row["at"]
|
|
149
|
+
if len(group["rows"]) < GROUP_ROWS:
|
|
150
|
+
group["rows"].append(row)
|
|
151
|
+
return sorted(groups.values(), key=lambda group: (-group["count"], -group["last_at"]))
|
|
152
|
+
|
|
153
|
+
|
|
114
154
|
def summary(now=None):
|
|
115
|
-
"""Per model and role over the last 24 h,
|
|
155
|
+
"""Per model and role over the last 24 h, every kept failure of the window grouped by type and role, and the calls
|
|
156
|
+
per purpose of the usage meter (its days are UTC, so that part covers the window's days, not its exact hours)."""
|
|
116
157
|
now = now or time.time()
|
|
117
158
|
with project_store._LOCK, _db() as db:
|
|
118
159
|
_tables(db)
|
|
@@ -121,7 +162,7 @@ def summary(now=None):
|
|
|
121
162
|
"SELECT model,role,SUM(calls) AS calls,SUM(errors) AS errors FROM model_calls WHERE hour>=? "
|
|
122
163
|
"GROUP BY model,role ORDER BY SUM(errors) DESC,SUM(calls) DESC", (int((now - WINDOW_S) // 3600),))]
|
|
123
164
|
failures = [dict(row) for row in db.execute(
|
|
124
|
-
"SELECT at,model,role,tool,path,error FROM model_failures WHERE at>=? ORDER BY at DESC
|
|
165
|
+
"SELECT at,model,role,tool,path,error FROM model_failures WHERE at>=? ORDER BY at DESC", (now - WINDOW_S,))]
|
|
125
166
|
for row in models:
|
|
126
167
|
last = db.execute("SELECT error FROM model_failures WHERE model=? AND role=? ORDER BY at DESC LIMIT 1",
|
|
127
168
|
(row["model"], row["role"])).fetchone()
|
|
@@ -129,7 +170,8 @@ def summary(now=None):
|
|
|
129
170
|
for row in models:
|
|
130
171
|
row["rate"] = round(row["errors"] / row["calls"], 3) if row["calls"] else 0.0
|
|
131
172
|
row["failing"] = row["calls"] >= ALERT_MIN_CALLS and row["rate"] >= ALERT_RATE
|
|
132
|
-
return {"window_hours": WINDOW_S // 3600, "models": models, "
|
|
173
|
+
return {"window_hours": WINDOW_S // 3600, "models": models, "groups": _groups(failures),
|
|
174
|
+
"calls_by_purpose": usage_meter.calls_by_purpose(now - WINDOW_S, PURPOSES)}
|
|
133
175
|
|
|
134
176
|
|
|
135
177
|
def alerts_due(now=None):
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.41",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Allan Santos",
|
package/project_monitor.py
CHANGED
|
@@ -20,6 +20,8 @@ IDLE_TICK_S = 5
|
|
|
20
20
|
# A folder that changes all the time (a tool's state folder) is indexed at most this often by the monitor; the changes
|
|
21
21
|
# keep accumulating and one job covers them, and a search meanwhile reads the stale files straight from disk.
|
|
22
22
|
MIN_JOB_GAP_S = 60
|
|
23
|
+
# While the changes do not stop, each job lengthens the wait for the next one; a wait that passes unused resets it.
|
|
24
|
+
JOB_GAP_STEPS_S = (MIN_JOB_GAP_S, 120, 240, 480, 900)
|
|
23
25
|
FULL_CHECK_S = 86400
|
|
24
26
|
MAX_EVENT_PATHS = 1000
|
|
25
27
|
|
|
@@ -27,7 +29,8 @@ MAX_EVENT_PATHS = 1000
|
|
|
27
29
|
def relevant_path(path):
|
|
28
30
|
parts = path.replace("\\", "/").strip("/").split("/")
|
|
29
31
|
return bool(parts and all(part not in ("", ".", "..") and not part.startswith(".")
|
|
30
|
-
and not index_scope.always_excluded(part) for part in parts)
|
|
32
|
+
and not index_scope.always_excluded(part) for part in parts)
|
|
33
|
+
and not index_scope.is_transient(parts[-1]))
|
|
31
34
|
|
|
32
35
|
|
|
33
36
|
def snapshot(root, scope):
|
|
@@ -58,6 +61,7 @@ class ProjectMonitor:
|
|
|
58
61
|
self._git_checked = {}
|
|
59
62
|
self._pending = {}
|
|
60
63
|
self._next_job = {}
|
|
64
|
+
self._gap_level = {}
|
|
61
65
|
self._filters = {}
|
|
62
66
|
self._views = {}
|
|
63
67
|
self._lock = threading.RLock()
|
|
@@ -155,6 +159,17 @@ class ProjectMonitor:
|
|
|
155
159
|
project_store.update(key, watch_backend="windows_events", watch_error="")
|
|
156
160
|
watcher.start()
|
|
157
161
|
|
|
162
|
+
def _job_gap(self, key, now):
|
|
163
|
+
"""Wait before the next job of a project, called when a job is enqueued at `now`: one step longer than the last
|
|
164
|
+
one when the previous wait was used up at once (at most twice its length went by), the first step otherwise."""
|
|
165
|
+
level, at = self._gap_level.get(key, (0, None))
|
|
166
|
+
if at is None or now - at >= 2 * JOB_GAP_STEPS_S[level]:
|
|
167
|
+
level = 0
|
|
168
|
+
else:
|
|
169
|
+
level = min(level + 1, len(JOB_GAP_STEPS_S) - 1)
|
|
170
|
+
self._gap_level[key] = (level, now)
|
|
171
|
+
return JOB_GAP_STEPS_S[level]
|
|
172
|
+
|
|
158
173
|
def tick(self):
|
|
159
174
|
now = time.monotonic()
|
|
160
175
|
projects = project_store.all_projects()
|
|
@@ -223,7 +238,7 @@ class ProjectMonitor:
|
|
|
223
238
|
except Exception as exc:
|
|
224
239
|
project_store.update(key, status="error", last_error=str(exc)[:300], next_retry_at=time.time() + 60)
|
|
225
240
|
continue
|
|
226
|
-
self._next_job[key] = now +
|
|
241
|
+
self._next_job[key] = now + self._job_gap(key, now)
|
|
227
242
|
with self._lock:
|
|
228
243
|
# Um evento recebido durante o submit continua pendente para a próxima geração.
|
|
229
244
|
if self._pending.get(key) is pending and pending.get("revision", 0) == revision:
|
package/scout.py
CHANGED
|
@@ -12,6 +12,7 @@ Market pattern copied (docs/research/2026-10-24-scout-modelo-leve-no-smart-tool.
|
|
|
12
12
|
excerpts, never an agent prompt, so no agent context is reopened per call.
|
|
13
13
|
"""
|
|
14
14
|
import os
|
|
15
|
+
import re
|
|
15
16
|
|
|
16
17
|
import document_text
|
|
17
18
|
import index_scope
|
|
@@ -35,6 +36,8 @@ PLAN_MAX_FOLDERS = 6
|
|
|
35
36
|
OUTLINE_MAX_DIRS = 300
|
|
36
37
|
OUTLINE_MAX_ROOT_FILES = 200
|
|
37
38
|
_FIELDS = {"project_root", "question", "terms", "max_files"}
|
|
39
|
+
DECISION_NOTES_MAX = 3
|
|
40
|
+
_INTENT = re.compile(r"\b(why|por\s*qu[êe]|rationale|reason|motivo|raz[ãa]o|decid\w*|decis[ãa]o)\b", re.IGNORECASE)
|
|
38
41
|
|
|
39
42
|
PLAN_SYSTEM = (
|
|
40
43
|
"You plan a keyword scan of a code project that has no search index yet. Given a question and the project's "
|
|
@@ -91,9 +94,15 @@ def arguments(raw):
|
|
|
91
94
|
return {"project_root": root, "question": question.strip(), "terms": (terms or "").strip(), "max_files": max_files}
|
|
92
95
|
|
|
93
96
|
|
|
97
|
+
def is_intent_question(question):
|
|
98
|
+
"""The question asks why something is as it is (why, reason, rationale, decided; por que, motivo, decisão): the
|
|
99
|
+
answer may sit in the notes the project keeps about its own intent, not in the code."""
|
|
100
|
+
return bool(_INTENT.search(question or ""))
|
|
101
|
+
|
|
102
|
+
|
|
94
103
|
def disk_lines(root, path, start, end):
|
|
95
104
|
"""`[(line_number, text)]` of lines `start..end` of the file as it is on disk now, or None when the file cannot be
|
|
96
|
-
read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx)."""
|
|
105
|
+
read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf)."""
|
|
97
106
|
if document_text.location_kind(path) != "source_line":
|
|
98
107
|
return None
|
|
99
108
|
data, encoding = indexer.read_indexable(root, path)
|
|
@@ -250,12 +259,13 @@ def valid_answer(raw, items):
|
|
|
250
259
|
return raw["found"], answer, citations
|
|
251
260
|
|
|
252
261
|
|
|
253
|
-
def render(question, source, items, citations=None, answer=None, found=None, model_note="", warnings=(), others=()
|
|
262
|
+
def render(question, source, items, citations=None, answer=None, found=None, model_note="", warnings=(), others=(),
|
|
263
|
+
notes=()):
|
|
254
264
|
"""The text returned to the agent. Lexical mode (`citations` None): every evidence window, within
|
|
255
265
|
LITERAL_BUDGET_CHARS, after the model's answer when there is one (a model that cited nothing never leaves the
|
|
256
266
|
answer without lines). Model mode: the model's answer, then only the cited ranges (sliced from the windows read
|
|
257
267
|
from disk), and the uncited excerpts as `path:lines` pointers. `others` are candidates not shown, listed as
|
|
258
|
-
pointers."""
|
|
268
|
+
pointers. `notes` are the dossier hits for a question about intent: ids with one excerpt each, after the evidence."""
|
|
259
269
|
out = [f"question: {question}", f"source: {source}", f"model: {model_note or 'none (literal lines only)'}"]
|
|
260
270
|
if answer is not None:
|
|
261
271
|
out.append(("answer: " if found else "answer (not found in the excerpts): ") + answer)
|
|
@@ -280,6 +290,9 @@ def render(question, source, items, citations=None, answer=None, found=None, mod
|
|
|
280
290
|
and citations is not None] + list(others)
|
|
281
291
|
if pointers:
|
|
282
292
|
out.append("other candidates (not shown): " + ", ".join(pointers))
|
|
293
|
+
if notes:
|
|
294
|
+
out.append("decision notes (project notes, not code):")
|
|
295
|
+
out += [f"- [{note['id']}] {note['source']}: {note['title']}: {note['snippet']}" for note in notes]
|
|
283
296
|
for warning in warnings:
|
|
284
297
|
out.append(f"warning: {warning}")
|
|
285
298
|
return "\n".join(out) + "\n"
|
package/setup_audit.py
CHANGED
|
@@ -865,13 +865,55 @@ def _save_locked(report):
|
|
|
865
865
|
atomic_io.write_secret_text(last_path, json.dumps(report, indent=1, ensure_ascii=False))
|
|
866
866
|
changes = report.get("changes") or {}
|
|
867
867
|
data = index()
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
868
|
+
key = index_key(report["project"])
|
|
869
|
+
entry = {"project": report["project"], "summary": report["summary"], "new": len(changes.get("new", [])),
|
|
870
|
+
"resolved": len(changes.get("resolved", [])), "checked_at": report["checked_at"],
|
|
871
|
+
"trigger": report["trigger"], "snapshot_key": report["snapshot_key"],
|
|
872
|
+
"pending": _pending_notice(data.get(key), report)}
|
|
873
|
+
data[key] = entry
|
|
872
874
|
_write_index(data)
|
|
873
875
|
|
|
874
876
|
|
|
877
|
+
def _pending_notice(previous, report):
|
|
878
|
+
"""The new errors and warnings of a scheduled run not yet told to the user, added to those a former run left
|
|
879
|
+
unannounced (by finding id). A manual run adds none: the user is looking at its result."""
|
|
880
|
+
pending = list((previous or {}).get("pending") or [])
|
|
881
|
+
if report["trigger"] == "schedule":
|
|
882
|
+
known = {item["id"] for item in pending}
|
|
883
|
+
pending += [{"id": f["id"], "severity": f["severity"], "check": f["check"], "subject": f["subject"]}
|
|
884
|
+
for f in report["findings"]
|
|
885
|
+
if f.get("new") and f["severity"] in ("error", "warn") and f["id"] not in known]
|
|
886
|
+
return pending
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
def notices_due():
|
|
890
|
+
"""[{project, errors, warnings, items}] for the projects whose scheduled audits found new errors or warnings since
|
|
891
|
+
the last call; marks them announced, so each finding is told once (the tray toast)."""
|
|
892
|
+
with _INDEX_LOCK:
|
|
893
|
+
data = index()
|
|
894
|
+
due = []
|
|
895
|
+
for entry in data.values():
|
|
896
|
+
pending = entry.get("pending") or []
|
|
897
|
+
if pending:
|
|
898
|
+
due.append({"project": entry["project"], "errors": sum(i["severity"] == "error" for i in pending),
|
|
899
|
+
"warnings": sum(i["severity"] == "warn" for i in pending), "items": pending})
|
|
900
|
+
entry["pending"] = []
|
|
901
|
+
if due:
|
|
902
|
+
_write_index(data)
|
|
903
|
+
return due
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
def toast_text(row):
|
|
907
|
+
"""(title, lines) of the toast for one `notices_due` row."""
|
|
908
|
+
total = row["errors"] + row["warnings"]
|
|
909
|
+
title = f"Setup audit: {total} new problem{'s' if total != 1 else ''} in the agent configuration"
|
|
910
|
+
lines = [os.path.basename(row["project"].rstrip("\\/")) or row["project"]]
|
|
911
|
+
lines += [f"{item['severity']}: {item['check']} - {item['subject']}"[:120] for item in row["items"][:3]]
|
|
912
|
+
if len(row["items"]) > 3:
|
|
913
|
+
lines.append(f"and {len(row['items']) - 3} more")
|
|
914
|
+
return title, lines
|
|
915
|
+
|
|
916
|
+
|
|
875
917
|
class Scheduler:
|
|
876
918
|
"""Periodic mode: audits every registered project whose folder exists, FIRST_RUN_DELAY_S after start and then
|
|
877
919
|
every SCHEDULE_S, one project at a time, reusing MCP probes younger than PROBE_MAX_AGE_S. projects() returns the
|