@allansantos-dev/smart-tool 0.9.72 → 0.9.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +3 -3
- package/local_embedder.py +21 -4
- package/model_health.py +2 -2
- package/package.json +35 -35
- package/project_storage.py +17 -0
- package/smart_tool_daemon.py +35 -2
- package/usage_meter.py +8 -5
- package/version.py +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.73 - beta
|
|
4
|
+
|
|
5
|
+
- Once a day (first pass 10 minutes after the daemon starts) the daemon removes, in each registered project without a
|
|
6
|
+
running job, the index views that are not pinned and were not updated for 30 days (`project_storage.auto_cleanup`). It
|
|
7
|
+
goes through the manual cleanup's own preview and commit, so every protection of `cleanup_commit` applies; a refused
|
|
8
|
+
project is kept and logged. A removed view is rebuilt from the cached vectors when its branch is indexed again. Tests:
|
|
9
|
+
`tests/test_storage_auto_cleanup.py`.
|
|
10
|
+
- The local embedder and the gateway models listings go through the shared model protections (circuit, in-flight cap,
|
|
11
|
+
`Retry-After`, the model health table), like the gateway calls. Tests: `tests/test_local_embedder.py`,
|
|
12
|
+
`tests/test_model_health.py`.
|
|
13
|
+
- The usage meter counts calls per purpose by hour (`usage_hourly`), so the 24 h window of the setup screen covers the
|
|
14
|
+
last 24 hours instead of following the UTC day.
|
|
15
|
+
- Product page: larger body text, 44 px touch targets, a labelled tool group, a favicon file, and the product video
|
|
16
|
+
rebuilt with the default voice and a window frame without an address bar. `release/pages.py` no longer publishes hidden
|
|
17
|
+
folders of `site/` (tool state such as `.token-guard`). The release notes section lists the three latest versions it
|
|
18
|
+
claims (it listed twelve), and `release/pages.check_site` refuses to publish otherwise. Tests:
|
|
19
|
+
`tests/test_site_quality.py`, `tests/test_pages.py`.
|
|
20
|
+
|
|
3
21
|
## 0.9.72 - beta
|
|
4
22
|
|
|
5
23
|
- A Smart Tool hook entry that is absent from the client's config is no longer reported `missing` when the Skill Manager holds
|
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ registered as the `playwright` server where this machine lets a browser be drive
|
|
|
17
17
|
An optional hook routes the agent's native Grep/Glob/Read/Bash/WebSearch/WebFetch calls to these tools when they
|
|
18
18
|
are cheaper, or just tells the agent what Smart Tool would do.
|
|
19
19
|
|
|
20
|
-
**Status:** 0.9.
|
|
20
|
+
**Status:** 0.9.73 beta. Windows only. What changed in each version: [CHANGELOG.md](CHANGELOG.md).
|
|
21
21
|
|
|
22
22
|
## How it works
|
|
23
23
|
|
|
@@ -83,7 +83,7 @@ gateway such as Ollama, nothing about your code leaves the machine.
|
|
|
83
83
|
## Install
|
|
84
84
|
|
|
85
85
|
```powershell
|
|
86
|
-
npx @allansantos-dev/smart-tool@0.9.
|
|
86
|
+
npx @allansantos-dev/smart-tool@0.9.73 install
|
|
87
87
|
```
|
|
88
88
|
|
|
89
89
|
Updates install themselves, on by default: the tray checks npm when it starts (at logon) and every 24 h, and when there
|
|
@@ -382,7 +382,7 @@ fails to load is listed in the setup screen with its error and does not affect t
|
|
|
382
382
|
## Uninstall
|
|
383
383
|
|
|
384
384
|
```powershell
|
|
385
|
-
npx @allansantos-dev/smart-tool@0.9.
|
|
385
|
+
npx @allansantos-dev/smart-tool@0.9.73 uninstall [--target DIR] [--keep-data]
|
|
386
386
|
```
|
|
387
387
|
|
|
388
388
|
It asks nothing and opens no window. It stops the tray and the daemon, removes the logon task, the hook entries of Smart
|
package/local_embedder.py
CHANGED
|
@@ -9,10 +9,13 @@ import json
|
|
|
9
9
|
import os
|
|
10
10
|
import threading
|
|
11
11
|
import time
|
|
12
|
+
import urllib.error
|
|
12
13
|
import urllib.parse
|
|
13
14
|
|
|
14
15
|
import config
|
|
15
16
|
import indexer
|
|
17
|
+
import model_breaker
|
|
18
|
+
import model_health
|
|
16
19
|
|
|
17
20
|
PREFIX = "local:"
|
|
18
21
|
PROPERTIES = os.path.join(os.path.expanduser("~"), ".mcp-memory", "embedding.properties")
|
|
@@ -60,7 +63,7 @@ def _request(conn, method, path, body=None):
|
|
|
60
63
|
response = conn.getresponse()
|
|
61
64
|
data = response.read()
|
|
62
65
|
if response.status >= 400:
|
|
63
|
-
raise
|
|
66
|
+
raise urllib.error.HTTPError(f"{conn.host}:{conn.port}{path}", response.status, response.reason, response.headers, None)
|
|
64
67
|
return json.loads(data)
|
|
65
68
|
|
|
66
69
|
|
|
@@ -123,6 +126,13 @@ def _left(deadline):
|
|
|
123
126
|
return min(REQUEST_TIMEOUT_S, left)
|
|
124
127
|
|
|
125
128
|
|
|
129
|
+
def _retry_delay(failure, attempt):
|
|
130
|
+
"""Seconds to wait after a failed attempt: the `Retry-After` of the answer (whole seconds, at most the pause of the
|
|
131
|
+
circuit) when the sidecar sent one, else the attempt number."""
|
|
132
|
+
value = str(getattr(failure, "headers", None) and failure.headers.get("Retry-After") or "")
|
|
133
|
+
return min(int(value), model_breaker.PAUSE_S) if value.isdigit() else attempt
|
|
134
|
+
|
|
135
|
+
|
|
126
136
|
def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
127
137
|
base = url()
|
|
128
138
|
if not base:
|
|
@@ -144,21 +154,28 @@ def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
|
|
|
144
154
|
for attempt in range(1, MAX_ATTEMPTS + 1):
|
|
145
155
|
if cancel_check:
|
|
146
156
|
cancel_check()
|
|
157
|
+
model_breaker.guard(model)
|
|
147
158
|
conn.timeout = _left(deadline)
|
|
148
159
|
try:
|
|
149
|
-
|
|
150
|
-
|
|
160
|
+
with model_breaker.slot(model):
|
|
161
|
+
got = _request(conn, "POST", "/embed", {"inputs": part})
|
|
151
162
|
except (OSError, http.client.HTTPException) as exc:
|
|
163
|
+
model_breaker.note(model, exc)
|
|
164
|
+
model_health.record("/v1/embeddings", {"model": model}, exc)
|
|
152
165
|
conn.close()
|
|
153
166
|
conn = _connection(base, _left(deadline))
|
|
154
167
|
if attempt == MAX_ATTEMPTS:
|
|
155
168
|
raise RuntimeError(f"Local embedder failed after {MAX_ATTEMPTS} attempts: {exc}") from None
|
|
156
169
|
if on_retry:
|
|
157
170
|
on_retry(attempt + 1, len(part))
|
|
158
|
-
delay = attempt
|
|
171
|
+
delay = _retry_delay(exc, attempt)
|
|
159
172
|
if deadline is not None and time.monotonic() + delay >= deadline:
|
|
160
173
|
raise TimeoutError("Not enough time left to retry the local embedder.") from None
|
|
161
174
|
time.sleep(delay)
|
|
175
|
+
else:
|
|
176
|
+
model_breaker.note(model)
|
|
177
|
+
model_health.record("/v1/embeddings", {"model": model})
|
|
178
|
+
break
|
|
162
179
|
if not isinstance(got, list) or len(got) != len(part):
|
|
163
180
|
raise indexer.IndexVectorError("Local embedder returned a vector count different from the input.")
|
|
164
181
|
for vector in got:
|
package/model_health.py
CHANGED
|
@@ -28,7 +28,7 @@ ERROR_CHARS = 300
|
|
|
28
28
|
SAMPLE_CHARS = 160
|
|
29
29
|
GROUP_ROWS = 20
|
|
30
30
|
PURPOSES = ("index_embedding", "query_embedding", "rerank", "embedding_probe", "scope")
|
|
31
|
-
_PATHS = ("/v1/chat/completions", "/v1/embeddings", "/v1/rerank")
|
|
31
|
+
_PATHS = ("/v1/chat/completions", "/v1/embeddings", "/v1/rerank", "/v1/models", "/v1/model/info")
|
|
32
32
|
_PURPOSE_ROLE = {"scope": "scope_model", "scout": "scout_model", "docs": "docs_model", "rerank": "rerank_model",
|
|
33
33
|
"index_embedding": "embedding_model", "query_embedding": "embedding_model"}
|
|
34
34
|
_TOOL = contextvars.ContextVar("model_health_tool", default=None)
|
|
@@ -112,7 +112,7 @@ def record(path, body, failure=None):
|
|
|
112
112
|
if path not in _PATHS:
|
|
113
113
|
return
|
|
114
114
|
try:
|
|
115
|
-
model = str(
|
|
115
|
+
model = (str(body.get("model", "")) if isinstance(body, dict) else "") or path
|
|
116
116
|
role = role_of(model, purpose=usage_meter.purpose())
|
|
117
117
|
now = time.time()
|
|
118
118
|
text = error_text(failure) if failure is not None else None
|
package/package.json
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
4
|
-
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
|
-
"license": "Apache-2.0",
|
|
6
|
-
"author": "Smart Tool contributors",
|
|
7
|
-
"homepage": "https://allansantos-dv.github.io/smart-tool/",
|
|
8
|
-
"repository": {
|
|
9
|
-
"type": "git",
|
|
10
|
-
"url": "git+https://github.com/AllanSantos-DV/smart-tool-dist.git"
|
|
11
|
-
},
|
|
12
|
-
"keywords": [
|
|
13
|
-
"mcp",
|
|
14
|
-
"claude-code",
|
|
15
|
-
"codex",
|
|
16
|
-
"semantic-search",
|
|
17
|
-
"web-search",
|
|
18
|
-
"coding-agent"
|
|
19
|
-
],
|
|
20
|
-
"bin": {
|
|
21
|
-
"smart-tool": "bin/smart-tool.mjs"
|
|
22
|
-
},
|
|
23
|
-
"os": [
|
|
24
|
-
"win32"
|
|
25
|
-
],
|
|
26
|
-
"cpu": [
|
|
27
|
-
"x64"
|
|
28
|
-
],
|
|
29
|
-
"engines": {
|
|
30
|
-
"node": ">=18"
|
|
31
|
-
},
|
|
32
|
-
"publishConfig": {
|
|
33
|
-
"access": "public"
|
|
34
|
-
}
|
|
35
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "@allansantos-dev/smart-tool",
|
|
3
|
+
"version": "0.9.73",
|
|
4
|
+
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"author": "Smart Tool contributors",
|
|
7
|
+
"homepage": "https://allansantos-dv.github.io/smart-tool/",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/AllanSantos-DV/smart-tool-dist.git"
|
|
11
|
+
},
|
|
12
|
+
"keywords": [
|
|
13
|
+
"mcp",
|
|
14
|
+
"claude-code",
|
|
15
|
+
"codex",
|
|
16
|
+
"semantic-search",
|
|
17
|
+
"web-search",
|
|
18
|
+
"coding-agent"
|
|
19
|
+
],
|
|
20
|
+
"bin": {
|
|
21
|
+
"smart-tool": "bin/smart-tool.mjs"
|
|
22
|
+
},
|
|
23
|
+
"os": [
|
|
24
|
+
"win32"
|
|
25
|
+
],
|
|
26
|
+
"cpu": [
|
|
27
|
+
"x64"
|
|
28
|
+
],
|
|
29
|
+
"engines": {
|
|
30
|
+
"node": ">=18"
|
|
31
|
+
},
|
|
32
|
+
"publishConfig": {
|
|
33
|
+
"access": "public"
|
|
34
|
+
}
|
|
35
|
+
}
|
package/project_storage.py
CHANGED
|
@@ -19,6 +19,8 @@ import project_identity
|
|
|
19
19
|
import project_store
|
|
20
20
|
|
|
21
21
|
_PLANS={}
|
|
22
|
+
# A view of a branch nobody has indexed for this long is rebuilt from the cached vectors on demand.
|
|
23
|
+
STALE_VIEW_S=30*86400
|
|
22
24
|
|
|
23
25
|
|
|
24
26
|
def _assets(root):
|
|
@@ -116,6 +118,21 @@ def preview(project,storage_ids,clear_cache=True):
|
|
|
116
118
|
return result
|
|
117
119
|
|
|
118
120
|
|
|
121
|
+
def stale_view_ids(project,now=None):
|
|
122
|
+
"""Storage ids of the views of `project` that are not protected and were last updated over STALE_VIEW_S ago."""
|
|
123
|
+
now=time.time() if now is None else now
|
|
124
|
+
return [v['storage_id'] for v in inventory(project)['views']
|
|
125
|
+
if not v['protected'] and v.get('updated_at') and now-v['updated_at']>STALE_VIEW_S]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def auto_cleanup(project,now=None):
|
|
129
|
+
"""Removes the stale views of `project` through the same preview and commit as the manual cleanup, so what goes is
|
|
130
|
+
exactly what the preview lists and every protection of the commit applies. None when no view is stale."""
|
|
131
|
+
ids=stale_view_ids(project,now)
|
|
132
|
+
if not ids:return None
|
|
133
|
+
return commit(project,preview(project,ids,True)['plan_id'])
|
|
134
|
+
|
|
135
|
+
|
|
119
136
|
def commit(project,plan_id):
|
|
120
137
|
plan=_PLANS.pop(plan_id,None)
|
|
121
138
|
if not plan or plan['project_id']!=project['id'] or plan['expires']<time.time():
|
package/smart_tool_daemon.py
CHANGED
|
@@ -44,6 +44,7 @@ import integration_metrics
|
|
|
44
44
|
import node_locator
|
|
45
45
|
import config
|
|
46
46
|
import disk_cleanup
|
|
47
|
+
import project_storage
|
|
47
48
|
import freshness
|
|
48
49
|
import model_client
|
|
49
50
|
import index_scope
|
|
@@ -2379,12 +2380,10 @@ def _project_action_locked(arguments):
|
|
|
2379
2380
|
if action=='usage':
|
|
2380
2381
|
return usage_meter.summary(key,arguments.get('days',30),arguments.get('view_id'))
|
|
2381
2382
|
if action=='storage':
|
|
2382
|
-
import project_storage
|
|
2383
2383
|
return project_storage.inventory(project)
|
|
2384
2384
|
active = _active_project_jobs(key)
|
|
2385
2385
|
if action in ('pin_view','cleanup_preview','cleanup_commit'):
|
|
2386
2386
|
if active:raise ValueError('Wait for this project\'s jobs to finish before changing storage.')
|
|
2387
|
-
import project_storage
|
|
2388
2387
|
if action=='pin_view':return project_storage.pin(project,arguments.get('storage_id'),arguments.get('pinned'))
|
|
2389
2388
|
if action=='cleanup_preview':return project_storage.preview(project,arguments.get('storage_ids'),arguments.get('clear_cache',True))
|
|
2390
2389
|
if arguments.get('confirm') is not True:raise ValueError('Confirm the cleanup preview before applying it.')
|
|
@@ -4769,6 +4768,39 @@ def _heal_browser_control():
|
|
|
4769
4768
|
_log(f"browser control not refreshed: {type(exc).__name__}: {exc}")
|
|
4770
4769
|
|
|
4771
4770
|
|
|
4771
|
+
STORAGE_CLEANUP_FIRST_S = 600
|
|
4772
|
+
STORAGE_CLEANUP_EVERY_S = 24 * 3600
|
|
4773
|
+
|
|
4774
|
+
|
|
4775
|
+
def _storage_cleanup_pass():
|
|
4776
|
+
"""Removes the stale views of every registered project that has no job running; returns how many views went. A
|
|
4777
|
+
project whose cleanup is refused (the branch changed, an index is open elsewhere) is kept and logged."""
|
|
4778
|
+
removed = 0
|
|
4779
|
+
for project in project_store.all_projects():
|
|
4780
|
+
if _active_project_jobs(project["id"]):
|
|
4781
|
+
continue
|
|
4782
|
+
try:
|
|
4783
|
+
result = project_storage.auto_cleanup(project)
|
|
4784
|
+
except (ValueError, RuntimeError, OSError) as exc:
|
|
4785
|
+
_log(f"storage cleanup: project {project['id']} kept: {exc}")
|
|
4786
|
+
continue
|
|
4787
|
+
if result:
|
|
4788
|
+
removed += result["removed_views"]
|
|
4789
|
+
_log(f"storage cleanup: {project['root']}: removed {result['removed_views']} view(s), "
|
|
4790
|
+
f"{result['removed_index_bytes']} bytes")
|
|
4791
|
+
return removed
|
|
4792
|
+
|
|
4793
|
+
|
|
4794
|
+
def _storage_cleanup_loop():
|
|
4795
|
+
time.sleep(STORAGE_CLEANUP_FIRST_S)
|
|
4796
|
+
while True:
|
|
4797
|
+
try:
|
|
4798
|
+
_storage_cleanup_pass()
|
|
4799
|
+
except Exception as exc:
|
|
4800
|
+
_log(f"storage cleanup failed: {type(exc).__name__}: {exc}")
|
|
4801
|
+
time.sleep(STORAGE_CLEANUP_EVERY_S)
|
|
4802
|
+
|
|
4803
|
+
|
|
4772
4804
|
def main():
|
|
4773
4805
|
global _PROJECT_MONITOR
|
|
4774
4806
|
# O bind vem antes de qualquer I/O de rede: enquanto a porta não abre, o
|
|
@@ -4799,6 +4831,7 @@ def main():
|
|
|
4799
4831
|
project_store.import_known_legacy(indexer.INDEX_DIR)
|
|
4800
4832
|
_PROJECT_MONITOR = project_monitor.ProjectMonitor(_enqueue_monitored_project)
|
|
4801
4833
|
_PROJECT_MONITOR.start()
|
|
4834
|
+
threading.Thread(target=_storage_cleanup_loop, daemon=True).start()
|
|
4802
4835
|
_log(f"project monitor started; {recovered} interrupted job(s) recovered")
|
|
4803
4836
|
except Exception as exc:
|
|
4804
4837
|
_log(f"project monitor unavailable: {_project_error(exc)}")
|
package/usage_meter.py
CHANGED
|
@@ -35,6 +35,7 @@ def phase(name):
|
|
|
35
35
|
|
|
36
36
|
def _tables(db):
|
|
37
37
|
db.execute('CREATE TABLE IF NOT EXISTS usage_daily(day TEXT,project_id TEXT,view_id TEXT,view_label TEXT,purpose TEXT,model TEXT,attempts INTEGER,successes INTEGER,errors INTEGER,input_items INTEGER,input_characters INTEGER,prompt_tokens INTEGER,output_tokens INTEGER,token_reports INTEGER,reported_usd REAL,price_reports INTEGER,elapsed_s REAL,PRIMARY KEY(day,project_id,view_id,purpose,model))')
|
|
38
|
+
db.execute('CREATE TABLE IF NOT EXISTS usage_hourly(hour INTEGER,purpose TEXT,attempts INTEGER,errors INTEGER,PRIMARY KEY(hour,purpose))')
|
|
38
39
|
db.execute('CREATE TABLE IF NOT EXISTS usage_index_jobs(job_id TEXT PRIMARY KEY,project_id TEXT,view_id TEXT,view_label TEXT,at REAL,kind TEXT,embedded_chunks INTEGER,reused_chunks INTEGER,unchanged_files INTEGER,embedded_characters INTEGER)')
|
|
39
40
|
db.execute('CREATE TABLE IF NOT EXISTS usage_imports(project_id TEXT PRIMARY KEY)')
|
|
40
41
|
|
|
@@ -80,6 +81,9 @@ def _record(path,body,response=None,headers=None,error=None,elapsed=0):
|
|
|
80
81
|
fields=['attempts','successes','errors','input_items','input_characters','prompt_tokens','output_tokens','token_reports','reported_usd','price_reports','elapsed_s']
|
|
81
82
|
update=','.join(f'{name}={name}+excluded.{name}' for name in fields)
|
|
82
83
|
db.execute('INSERT INTO usage_daily VALUES('+','.join('?' for _ in values)+') ON CONFLICT(day,project_id,view_id,purpose,model) DO UPDATE SET '+update,values)
|
|
84
|
+
hour=int(time.time()//3600)
|
|
85
|
+
db.execute('INSERT INTO usage_hourly VALUES(?,?,1,?) ON CONFLICT(hour,purpose) DO UPDATE SET attempts=attempts+1,errors=errors+excluded.errors',(hour,current['purpose'],int(error is not None)))
|
|
86
|
+
db.execute('DELETE FROM usage_hourly WHERE hour<?',(hour-366*24,))
|
|
83
87
|
cutoff=(datetime.datetime.now(datetime.timezone.utc)-datetime.timedelta(days=366)).date().isoformat()
|
|
84
88
|
db.execute('DELETE FROM usage_daily WHERE day<?',(cutoff,))
|
|
85
89
|
except Exception:
|
|
@@ -103,14 +107,13 @@ def _record_job(job_id,payload):
|
|
|
103
107
|
|
|
104
108
|
|
|
105
109
|
def calls_by_purpose(since,purposes):
|
|
106
|
-
"""Attempts and errors of every project per purpose in `purposes`, from the
|
|
107
|
-
|
|
108
|
-
day=datetime.datetime.fromtimestamp(since,datetime.timezone.utc).date().isoformat()
|
|
110
|
+
"""Attempts and errors of every project per purpose in `purposes`, from the hour of the epoch time `since` on.
|
|
111
|
+
Purposes with no call are left out."""
|
|
109
112
|
marks=','.join('?' for _ in purposes)
|
|
110
113
|
with project_store._LOCK,project_store._db() as db:
|
|
111
114
|
_tables(db)
|
|
112
|
-
rows=db.execute('SELECT purpose,SUM(attempts),SUM(errors) FROM
|
|
113
|
-
(
|
|
115
|
+
rows=db.execute('SELECT purpose,SUM(attempts),SUM(errors) FROM usage_hourly WHERE hour>=? AND purpose IN ('+marks+') GROUP BY purpose',
|
|
116
|
+
(int(since//3600),*purposes)).fetchall()
|
|
114
117
|
found={purpose:{'purpose':purpose,'attempts':attempts,'errors':errors} for purpose,attempts,errors in rows}
|
|
115
118
|
return [found[purpose] for purpose in purposes if purpose in found]
|
|
116
119
|
|
package/version.py
CHANGED