@allansantos-dev/smart-tool 0.9.79 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +65 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_health.py +180 -0
- package/code_impact.py +13 -1
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +111 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/update_check.py
CHANGED
|
@@ -159,6 +159,24 @@ def notices(now=None):
|
|
|
159
159
|
return found
|
|
160
160
|
|
|
161
161
|
|
|
162
|
+
def status():
|
|
163
|
+
"""What the setup screen shows about updates, from the state file and the config only (no network).
|
|
164
|
+
|
|
165
|
+
Returns: dict with installed (str), latest and checked_at (the last npm check; None before the first), available
|
|
166
|
+
(bool, latest is newer than installed), command (the update command when available, else None), updated (the record
|
|
167
|
+
{from, to, at} of the last update, or None), failed (the version an automatic update went for and did not install,
|
|
168
|
+
else None), enabled (the `auto_update` config), env_opt_out (bool) and automatic (both allow automatic updates).
|
|
169
|
+
Raises: ValueError when `auto_update` in the config is not a boolean.
|
|
170
|
+
"""
|
|
171
|
+
state = _state()
|
|
172
|
+
latest = state.get("latest")
|
|
173
|
+
available = bool(latest) and _parts(latest) > _parts(version.VERSION)
|
|
174
|
+
return {"installed": version.VERSION, "latest": latest, "checked_at": state.get("checked_at"), "available": available,
|
|
175
|
+
"command": update_command(latest) if available else None, "updated": state.get("updated"),
|
|
176
|
+
"failed": failed_update(), "enabled": config.auto_update(), "env_opt_out": env_opted_out(),
|
|
177
|
+
"automatic": automatic_enabled()}
|
|
178
|
+
|
|
179
|
+
|
|
162
180
|
def run_update(latest, automatic):
|
|
163
181
|
"""Starts the update screen (update_splash.py) detached from the tray, which the installer stops and restarts:
|
|
164
182
|
it runs the install command with no console window and shows each step and the result. npx is looked up by the
|
package/usage_meter.py
CHANGED
|
@@ -38,6 +38,84 @@ def _tables(db):
|
|
|
38
38
|
db.execute('CREATE TABLE IF NOT EXISTS usage_hourly(hour INTEGER,purpose TEXT,attempts INTEGER,errors INTEGER,PRIMARY KEY(hour,purpose))')
|
|
39
39
|
db.execute('CREATE TABLE IF NOT EXISTS usage_index_jobs(job_id TEXT PRIMARY KEY,project_id TEXT,view_id TEXT,view_label TEXT,at REAL,kind TEXT,embedded_chunks INTEGER,reused_chunks INTEGER,unchanged_files INTEGER,embedded_characters INTEGER)')
|
|
40
40
|
db.execute('CREATE TABLE IF NOT EXISTS usage_imports(project_id TEXT PRIMARY KEY)')
|
|
41
|
+
db.execute('CREATE TABLE IF NOT EXISTS usage_alerts(purpose TEXT PRIMARY KEY,muted_until REAL)')
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
VOLUME_FLOOR=100
|
|
45
|
+
VOLUME_FACTOR=2
|
|
46
|
+
VOLUME_BASELINE_HOURS=168
|
|
47
|
+
VOLUME_MUTE_S=7*86400
|
|
48
|
+
PURPOSE_MAX_CHARS=40
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _raw_volume(db,now):
|
|
52
|
+
"""Purposes whose attempts in the current UTC hour exceed their threshold: the larger of VOLUME_FLOOR and
|
|
53
|
+
VOLUME_FACTOR times the 99th percentile of that purpose's hourly attempts over the previous VOLUME_BASELINE_HOURS
|
|
54
|
+
hours (hours with calls only)."""
|
|
55
|
+
hour=int(now//3600)
|
|
56
|
+
series={}
|
|
57
|
+
for purpose,at,attempts in db.execute('SELECT purpose,hour,SUM(attempts) FROM usage_hourly WHERE hour>=? AND hour<=? GROUP BY purpose,hour',(hour-VOLUME_BASELINE_HOURS,hour)):
|
|
58
|
+
series.setdefault(purpose,{})[at]=attempts
|
|
59
|
+
found=[]
|
|
60
|
+
for purpose,hours in sorted(series.items()):
|
|
61
|
+
current=hours.get(hour,0)
|
|
62
|
+
past=sorted(value for at,value in hours.items() if at<hour)
|
|
63
|
+
baseline=past[max(0,math.ceil(len(past)*0.99)-1)] if past else 0
|
|
64
|
+
threshold=max(VOLUME_FLOOR,VOLUME_FACTOR*baseline)
|
|
65
|
+
if current>threshold:found.append({'purpose':purpose,'attempts':current,'threshold':threshold,'baseline_p99':baseline})
|
|
66
|
+
return found
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _volume(now):
|
|
70
|
+
with project_store._LOCK,project_store._db() as db:
|
|
71
|
+
_tables(db)
|
|
72
|
+
found=_raw_volume(db,now)
|
|
73
|
+
muted=dict(db.execute('SELECT purpose,muted_until FROM usage_alerts').fetchall())
|
|
74
|
+
return [{**row,'muted':muted.get(row['purpose'],0)>now} for row in found]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def volume_alerts(now=None):
|
|
78
|
+
"""Purposes whose model calls in the current hour are far above their own history and not muted.
|
|
79
|
+
|
|
80
|
+
Parameters: now (float or None) epoch seconds. Returns: list of {purpose, attempts, threshold, baseline_p99, muted}."""
|
|
81
|
+
return [row for row in _volume(now or time.time()) if not row['muted']]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def mute_volume(purpose,now=None):
|
|
85
|
+
"""Silences the volume alert of one purpose for 7 days; the numbers stay on the card.
|
|
86
|
+
|
|
87
|
+
Raises: ValueError when purpose is not a short text."""
|
|
88
|
+
if not isinstance(purpose,str) or not purpose.strip() or len(purpose)>PURPOSE_MAX_CHARS:
|
|
89
|
+
raise ValueError('Name the purpose to mute (up to %d characters).'%PURPOSE_MAX_CHARS)
|
|
90
|
+
now=now or time.time()
|
|
91
|
+
with project_store._LOCK,project_store._db() as db:
|
|
92
|
+
_tables(db)
|
|
93
|
+
db.execute('INSERT INTO usage_alerts VALUES(?,?) ON CONFLICT(purpose) DO UPDATE SET muted_until=excluded.muted_until',(purpose,now+VOLUME_MUTE_S))
|
|
94
|
+
return 'volume alerts for %s muted for 7 days'%purpose
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def overview(days=30,now=None):
|
|
98
|
+
"""Calls, errors and reported tokens and cost of every project per purpose over the last `days` UTC days, and the
|
|
99
|
+
hourly volume alerts (muted ones included, flagged).
|
|
100
|
+
|
|
101
|
+
Parameters: days (int) 1 to 366; now (float or None) epoch seconds.
|
|
102
|
+
Returns: {days, totals, by_purpose, volume, currency, notes}; reported tokens and cost are None when no call reported them."""
|
|
103
|
+
if type(days) is not int or not 1<=days<=366:raise ValueError('Choose a period from 1 to 366 days.')
|
|
104
|
+
now=now or time.time()
|
|
105
|
+
cutoff=(datetime.datetime.fromtimestamp(now,datetime.timezone.utc)-datetime.timedelta(days=days-1)).date().isoformat()
|
|
106
|
+
with project_store._LOCK,project_store._db() as db:
|
|
107
|
+
_tables(db)
|
|
108
|
+
rows=db.execute('SELECT purpose,SUM(attempts),SUM(errors),SUM(prompt_tokens),SUM(output_tokens),SUM(token_reports),SUM(reported_usd),SUM(price_reports) FROM usage_daily WHERE day>=? GROUP BY purpose ORDER BY SUM(attempts) DESC',(cutoff,)).fetchall()
|
|
109
|
+
def entry(purpose,attempts,errors,prompt,output,tokens,usd,prices):
|
|
110
|
+
return {'purpose':purpose,'attempts':attempts,'errors':errors,'prompt_tokens':prompt if tokens else None,
|
|
111
|
+
'output_tokens':output if tokens else None,'reported_usd':usd if prices else None}
|
|
112
|
+
by_purpose=[entry(*row) for row in rows]
|
|
113
|
+
totals={'attempts':sum(r['attempts'] for r in by_purpose),'errors':sum(r['errors'] for r in by_purpose),
|
|
114
|
+
'reported_usd':round(sum(r['reported_usd'] or 0 for r in by_purpose),6) if any(r['reported_usd'] is not None for r in by_purpose) else None}
|
|
115
|
+
return {'days':days,'totals':totals,'by_purpose':by_purpose,'volume':_volume(now),'currency':'USD',
|
|
116
|
+
'notes':['Calls and tokens are measured starting with the version that added the meter.',
|
|
117
|
+
'Monetary values appear only when reported by the model gateway; they are not an estimate of the full bill.',
|
|
118
|
+
'A volume alert compares the current hour with the same purpose over the previous week; it never raises a notification.']}
|
|
41
119
|
|
|
42
120
|
|
|
43
121
|
def _number(value):
|
package/version.py
CHANGED
|
@@ -7,13 +7,16 @@
|
|
|
7
7
|
import asyncio
|
|
8
8
|
import ipaddress
|
|
9
9
|
import json
|
|
10
|
+
import os
|
|
10
11
|
import socket
|
|
11
12
|
import sys
|
|
12
13
|
import urllib.error
|
|
13
14
|
import urllib.parse
|
|
14
15
|
import urllib.request
|
|
15
16
|
|
|
16
|
-
|
|
17
|
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
|
18
|
+
import chromium_flags
|
|
19
|
+
from crawl4ai import AsyncWebCrawler, BFSDeepCrawlStrategy, BrowserConfig, CacheMode, CrawlerRunConfig
|
|
17
20
|
from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
|
|
18
21
|
from crawl4ai.deep_crawling.filters import FilterChain, URLFilter
|
|
19
22
|
from playwright.async_api import Error as PlaywrightError
|
|
@@ -205,9 +208,10 @@ async def crawl(seed_url, max_pages):
|
|
|
205
208
|
cache_mode=CacheMode.BYPASS,
|
|
206
209
|
page_timeout=30000,
|
|
207
210
|
)
|
|
208
|
-
|
|
211
|
+
browser_config = BrowserConfig(extra_args=chromium_flags.python_chromium_args())
|
|
212
|
+
strategy = AsyncPlaywrightCrawlerStrategy(browser_config=browser_config)
|
|
209
213
|
strategy.set_hook("on_page_context_created", _install_ssrf_route_guard)
|
|
210
|
-
async with AsyncWebCrawler(crawler_strategy=strategy) as crawler:
|
|
214
|
+
async with AsyncWebCrawler(crawler_strategy=strategy, config=browser_config) as crawler:
|
|
211
215
|
results = await crawler.arun(url=safe_seed_url, config=config)
|
|
212
216
|
return [
|
|
213
217
|
{"url": r.url, "markdown": r.markdown.raw_markdown}
|
package/web_fetch.py
CHANGED
|
@@ -48,6 +48,10 @@ MIN_CHARS = 200
|
|
|
48
48
|
ANSWER_MAX_TOKENS = 1500
|
|
49
49
|
FAILED_TTL_S = 600
|
|
50
50
|
PLACEHOLDERS = ("carregando", "loading...", "enable javascript", "habilite o javascript", "ative o javascript")
|
|
51
|
+
OFFICE_TYPES = {"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
|
|
52
|
+
"application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
|
|
53
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx"}
|
|
54
|
+
GENERIC_BINARY_TYPES = ("application/octet-stream", "application/zip", "binary/octet-stream")
|
|
51
55
|
TEXT_TYPES = ("text/plain", "text/markdown", "text/x-markdown", "application/json", "text/csv", "application/xml",
|
|
52
56
|
"text/xml")
|
|
53
57
|
USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
@@ -226,6 +230,24 @@ def pdf_text(body, url, deadline):
|
|
|
226
230
|
raise FetchError(f"PDF protected by a password is not read by web_fetch ({url}).") from None
|
|
227
231
|
|
|
228
232
|
|
|
233
|
+
def office_extension(kind, url):
|
|
234
|
+
"""`docx`, `pptx` or `xlsx` when the response is an Office file: by its content type, or by the extension in the URL
|
|
235
|
+
path when the server sends a generic binary type; None otherwise."""
|
|
236
|
+
if kind in OFFICE_TYPES:
|
|
237
|
+
return OFFICE_TYPES[kind]
|
|
238
|
+
extension = urllib.parse.urlsplit(url).path.rsplit(".", 1)[-1].lower()
|
|
239
|
+
return extension if kind in GENERIC_BINARY_TYPES and extension in OFFICE_TYPES.values() else None
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def office_text(body, extension, url):
|
|
243
|
+
"""Text of a docx, pptx or xlsx with the markers and caps of document_text (the extractor the index uses, with its
|
|
244
|
+
archive guards); FetchError with the reason when the file cannot be read."""
|
|
245
|
+
try:
|
|
246
|
+
return document_text.extract(f"file.{extension}", body).decode("utf-8", errors="replace")
|
|
247
|
+
except document_text.DocumentError as exc:
|
|
248
|
+
raise FetchError(f"{extension} file could not be read ({url}): {exc}"[:300]) from None
|
|
249
|
+
|
|
250
|
+
|
|
229
251
|
def read_page(url, deadline, render):
|
|
230
252
|
"""Page text (Markdown) from the 24 h cache, HTTP + trafilatura, or `render(url, deadline)` when HTTP comes thin."""
|
|
231
253
|
url = normalize(url)
|
|
@@ -250,6 +272,8 @@ def read_page(url, deadline, render):
|
|
|
250
272
|
text = _decode(body, charset)
|
|
251
273
|
elif kind == "application/pdf":
|
|
252
274
|
text = pdf_text(body, final_url, deadline)
|
|
275
|
+
elif office_extension(kind, final_url):
|
|
276
|
+
text = office_text(body, office_extension(kind, final_url), final_url)
|
|
253
277
|
else:
|
|
254
278
|
raise FetchError(f"Content type {kind} is not read by web_fetch ({final_url}).")
|
|
255
279
|
if len(text.strip()) < MIN_CHARS:
|
package/web_search_health.py
CHANGED
|
@@ -77,6 +77,37 @@ def _read_recent_records():
|
|
|
77
77
|
return records
|
|
78
78
|
|
|
79
79
|
|
|
80
|
+
def by_requester():
|
|
81
|
+
"""The recent web search calls read back as who asked: one row per client and project folder.
|
|
82
|
+
|
|
83
|
+
Paused calls are not calls and are left out. The log never holds the query text.
|
|
84
|
+
Returns: dict {"rows": [{"client", "project", "sessions", "calls", "failed", "last_at", "sources"}], "calls": int,
|
|
85
|
+
"unattributed": int}; rows are busiest first, `sources` says how far the attribution got (see `requester`), and
|
|
86
|
+
`unattributed` counts the calls recorded without a requester.
|
|
87
|
+
"""
|
|
88
|
+
groups, unattributed, calls = {}, 0, 0
|
|
89
|
+
for record in _read_recent_records():
|
|
90
|
+
if record.get("outcome") == "paused":
|
|
91
|
+
continue
|
|
92
|
+
calls += 1
|
|
93
|
+
who = record.get("requester")
|
|
94
|
+
if not isinstance(who, dict):
|
|
95
|
+
unattributed += 1
|
|
96
|
+
continue
|
|
97
|
+
row = groups.setdefault((who.get("client") or "unknown", who.get("project") or ""),
|
|
98
|
+
{"sessions": set(), "calls": 0, "failed": 0, "last_at": 0, "sources": set()})
|
|
99
|
+
row["calls"] += 1
|
|
100
|
+
row["failed"] += not record.get("success")
|
|
101
|
+
row["last_at"] = max(row["last_at"], record.get("ts") or 0)
|
|
102
|
+
row["sessions"].add(who.get("session") or who.get("mcp_session") or "")
|
|
103
|
+
row["sources"].add(who.get("source") or "none")
|
|
104
|
+
rows = [{"client": client, "project": project, "sessions": len(row["sessions"]), "calls": row["calls"],
|
|
105
|
+
"failed": row["failed"], "last_at": row["last_at"], "sources": sorted(row["sources"])}
|
|
106
|
+
for (client, project), row in groups.items()]
|
|
107
|
+
rows.sort(key=lambda row: (-row["calls"], -row["last_at"]))
|
|
108
|
+
return {"rows": rows, "calls": calls, "unattributed": unattributed}
|
|
109
|
+
|
|
110
|
+
|
|
80
111
|
def recent_success_rate(tier_name, window=DEFAULT_WINDOW):
|
|
81
112
|
records = [r for r in _read_recent_records() if r.get("tier") == tier_name and r.get("outcome") != "paused"]
|
|
82
113
|
if not records:
|