@allansantos-dev/smart-tool 0.9.79 → 0.9.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/update_check.py CHANGED
@@ -159,6 +159,24 @@ def notices(now=None):
159
159
  return found
160
160
 
161
161
 
162
+ def status():
163
+ """What the setup screen shows about updates, from the state file and the config only (no network).
164
+
165
+ Returns: dict with installed (str), latest and checked_at (the last npm check; None before the first), available
166
+ (bool, latest is newer than installed), command (the update command when available, else None), updated (the record
167
+ {from, to, at} of the last update, or None), failed (the version an automatic update went for and did not install,
168
+ else None), enabled (the `auto_update` config), env_opt_out (bool) and automatic (both allow automatic updates).
169
+ Raises: ValueError when `auto_update` in the config is not a boolean.
170
+ """
171
+ state = _state()
172
+ latest = state.get("latest")
173
+ available = bool(latest) and _parts(latest) > _parts(version.VERSION)
174
+ return {"installed": version.VERSION, "latest": latest, "checked_at": state.get("checked_at"), "available": available,
175
+ "command": update_command(latest) if available else None, "updated": state.get("updated"),
176
+ "failed": failed_update(), "enabled": config.auto_update(), "env_opt_out": env_opted_out(),
177
+ "automatic": automatic_enabled()}
178
+
179
+
162
180
  def run_update(latest, automatic):
163
181
  """Starts the update screen (update_splash.py) detached from the tray, which the installer stops and restarts:
164
182
  it runs the install command with no console window and shows each step and the result. npx is looked up by the
package/usage_meter.py CHANGED
@@ -38,6 +38,84 @@ def _tables(db):
38
38
  db.execute('CREATE TABLE IF NOT EXISTS usage_hourly(hour INTEGER,purpose TEXT,attempts INTEGER,errors INTEGER,PRIMARY KEY(hour,purpose))')
39
39
  db.execute('CREATE TABLE IF NOT EXISTS usage_index_jobs(job_id TEXT PRIMARY KEY,project_id TEXT,view_id TEXT,view_label TEXT,at REAL,kind TEXT,embedded_chunks INTEGER,reused_chunks INTEGER,unchanged_files INTEGER,embedded_characters INTEGER)')
40
40
  db.execute('CREATE TABLE IF NOT EXISTS usage_imports(project_id TEXT PRIMARY KEY)')
41
+ db.execute('CREATE TABLE IF NOT EXISTS usage_alerts(purpose TEXT PRIMARY KEY,muted_until REAL)')
42
+
43
+
44
+ VOLUME_FLOOR=100
45
+ VOLUME_FACTOR=2
46
+ VOLUME_BASELINE_HOURS=168
47
+ VOLUME_MUTE_S=7*86400
48
+ PURPOSE_MAX_CHARS=40
49
+
50
+
51
+ def _raw_volume(db,now):
52
+ """Purposes whose attempts in the current UTC hour exceed their threshold: the larger of VOLUME_FLOOR and
53
+ VOLUME_FACTOR times the 99th percentile of that purpose's hourly attempts over the previous VOLUME_BASELINE_HOURS
54
+ hours (hours with calls only)."""
55
+ hour=int(now//3600)
56
+ series={}
57
+ for purpose,at,attempts in db.execute('SELECT purpose,hour,SUM(attempts) FROM usage_hourly WHERE hour>=? AND hour<=? GROUP BY purpose,hour',(hour-VOLUME_BASELINE_HOURS,hour)):
58
+ series.setdefault(purpose,{})[at]=attempts
59
+ found=[]
60
+ for purpose,hours in sorted(series.items()):
61
+ current=hours.get(hour,0)
62
+ past=sorted(value for at,value in hours.items() if at<hour)
63
+ baseline=past[max(0,math.ceil(len(past)*0.99)-1)] if past else 0
64
+ threshold=max(VOLUME_FLOOR,VOLUME_FACTOR*baseline)
65
+ if current>threshold:found.append({'purpose':purpose,'attempts':current,'threshold':threshold,'baseline_p99':baseline})
66
+ return found
67
+
68
+
69
+ def _volume(now):
70
+ with project_store._LOCK,project_store._db() as db:
71
+ _tables(db)
72
+ found=_raw_volume(db,now)
73
+ muted=dict(db.execute('SELECT purpose,muted_until FROM usage_alerts').fetchall())
74
+ return [{**row,'muted':muted.get(row['purpose'],0)>now} for row in found]
75
+
76
+
77
+ def volume_alerts(now=None):
78
+ """Purposes whose model calls in the current hour are far above their own history and not muted.
79
+
80
+ Parameters: now (float or None) epoch seconds. Returns: list of {purpose, attempts, threshold, baseline_p99, muted}."""
81
+ return [row for row in _volume(now or time.time()) if not row['muted']]
82
+
83
+
84
+ def mute_volume(purpose,now=None):
85
+ """Silences the volume alert of one purpose for 7 days; the numbers stay on the card.
86
+
87
+ Raises: ValueError when purpose is not a short text."""
88
+ if not isinstance(purpose,str) or not purpose.strip() or len(purpose)>PURPOSE_MAX_CHARS:
89
+ raise ValueError('Name the purpose to mute (up to %d characters).'%PURPOSE_MAX_CHARS)
90
+ now=now or time.time()
91
+ with project_store._LOCK,project_store._db() as db:
92
+ _tables(db)
93
+ db.execute('INSERT INTO usage_alerts VALUES(?,?) ON CONFLICT(purpose) DO UPDATE SET muted_until=excluded.muted_until',(purpose,now+VOLUME_MUTE_S))
94
+ return 'volume alerts for %s muted for 7 days'%purpose
95
+
96
+
97
+ def overview(days=30,now=None):
98
+ """Calls, errors and reported tokens and cost of every project per purpose over the last `days` UTC days, and the
99
+ hourly volume alerts (muted ones included, flagged).
100
+
101
+ Parameters: days (int) 1 to 366; now (float or None) epoch seconds.
102
+ Returns: {days, totals, by_purpose, volume, currency, notes}; reported tokens and cost are None when no call reported them."""
103
+ if type(days) is not int or not 1<=days<=366:raise ValueError('Choose a period from 1 to 366 days.')
104
+ now=now or time.time()
105
+ cutoff=(datetime.datetime.fromtimestamp(now,datetime.timezone.utc)-datetime.timedelta(days=days-1)).date().isoformat()
106
+ with project_store._LOCK,project_store._db() as db:
107
+ _tables(db)
108
+ rows=db.execute('SELECT purpose,SUM(attempts),SUM(errors),SUM(prompt_tokens),SUM(output_tokens),SUM(token_reports),SUM(reported_usd),SUM(price_reports) FROM usage_daily WHERE day>=? GROUP BY purpose ORDER BY SUM(attempts) DESC',(cutoff,)).fetchall()
109
+ def entry(purpose,attempts,errors,prompt,output,tokens,usd,prices):
110
+ return {'purpose':purpose,'attempts':attempts,'errors':errors,'prompt_tokens':prompt if tokens else None,
111
+ 'output_tokens':output if tokens else None,'reported_usd':usd if prices else None}
112
+ by_purpose=[entry(*row) for row in rows]
113
+ totals={'attempts':sum(r['attempts'] for r in by_purpose),'errors':sum(r['errors'] for r in by_purpose),
114
+ 'reported_usd':round(sum(r['reported_usd'] or 0 for r in by_purpose),6) if any(r['reported_usd'] is not None for r in by_purpose) else None}
115
+ return {'days':days,'totals':totals,'by_purpose':by_purpose,'volume':_volume(now),'currency':'USD',
116
+ 'notes':['Calls and tokens are measured starting with the version that added the meter.',
117
+ 'Monetary values appear only when reported by the model gateway; they are not an estimate of the full bill.',
118
+ 'A volume alert compares the current hour with the same purpose over the previous week; it never raises a notification.']}
41
119
 
42
120
 
43
121
  def _number(value):
package/version.py CHANGED
@@ -1,7 +1,7 @@
1
1
  """Single source of the Smart Tool version and of the User-Agent sent to public APIs."""
2
2
  import re
3
3
 
4
- VERSION = "0.9.79"
4
+ VERSION = "0.9.80"
5
5
  _CONTACT_RE = re.compile(r"^(?:[^\s@()<>;]+@[^\s@()<>;]+\.[^\s@()<>;]+|https?://[^\s()<>;]+)$")
6
6
 
7
7
 
@@ -7,13 +7,16 @@
7
7
  import asyncio
8
8
  import ipaddress
9
9
  import json
10
+ import os
10
11
  import socket
11
12
  import sys
12
13
  import urllib.error
13
14
  import urllib.parse
14
15
  import urllib.request
15
16
 
16
- from crawl4ai import AsyncWebCrawler, BFSDeepCrawlStrategy, CacheMode, CrawlerRunConfig
17
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
18
+ import chromium_flags
19
+ from crawl4ai import AsyncWebCrawler, BFSDeepCrawlStrategy, BrowserConfig, CacheMode, CrawlerRunConfig
17
20
  from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
18
21
  from crawl4ai.deep_crawling.filters import FilterChain, URLFilter
19
22
  from playwright.async_api import Error as PlaywrightError
@@ -205,9 +208,10 @@ async def crawl(seed_url, max_pages):
205
208
  cache_mode=CacheMode.BYPASS,
206
209
  page_timeout=30000,
207
210
  )
208
- strategy = AsyncPlaywrightCrawlerStrategy()
211
+ browser_config = BrowserConfig(extra_args=chromium_flags.python_chromium_args())
212
+ strategy = AsyncPlaywrightCrawlerStrategy(browser_config=browser_config)
209
213
  strategy.set_hook("on_page_context_created", _install_ssrf_route_guard)
210
- async with AsyncWebCrawler(crawler_strategy=strategy) as crawler:
214
+ async with AsyncWebCrawler(crawler_strategy=strategy, config=browser_config) as crawler:
211
215
  results = await crawler.arun(url=safe_seed_url, config=config)
212
216
  return [
213
217
  {"url": r.url, "markdown": r.markdown.raw_markdown}
package/web_fetch.py CHANGED
@@ -48,6 +48,10 @@ MIN_CHARS = 200
48
48
  ANSWER_MAX_TOKENS = 1500
49
49
  FAILED_TTL_S = 600
50
50
  PLACEHOLDERS = ("carregando", "loading...", "enable javascript", "habilite o javascript", "ative o javascript")
51
+ OFFICE_TYPES = {"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
52
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
53
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx"}
54
+ GENERIC_BINARY_TYPES = ("application/octet-stream", "application/zip", "binary/octet-stream")
51
55
  TEXT_TYPES = ("text/plain", "text/markdown", "text/x-markdown", "application/json", "text/csv", "application/xml",
52
56
  "text/xml")
53
57
  USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -226,6 +230,24 @@ def pdf_text(body, url, deadline):
226
230
  raise FetchError(f"PDF protected by a password is not read by web_fetch ({url}).") from None
227
231
 
228
232
 
233
+ def office_extension(kind, url):
234
+ """`docx`, `pptx` or `xlsx` when the response is an Office file: by its content type, or by the extension in the URL
235
+ path when the server sends a generic binary type; None otherwise."""
236
+ if kind in OFFICE_TYPES:
237
+ return OFFICE_TYPES[kind]
238
+ extension = urllib.parse.urlsplit(url).path.rsplit(".", 1)[-1].lower()
239
+ return extension if kind in GENERIC_BINARY_TYPES and extension in OFFICE_TYPES.values() else None
240
+
241
+
242
+ def office_text(body, extension, url):
243
+ """Text of a docx, pptx or xlsx with the markers and caps of document_text (the extractor the index uses, with its
244
+ archive guards); FetchError with the reason when the file cannot be read."""
245
+ try:
246
+ return document_text.extract(f"file.{extension}", body).decode("utf-8", errors="replace")
247
+ except document_text.DocumentError as exc:
248
+ raise FetchError(f"{extension} file could not be read ({url}): {exc}"[:300]) from None
249
+
250
+
229
251
  def read_page(url, deadline, render):
230
252
  """Page text (Markdown) from the 24 h cache, HTTP + trafilatura, or `render(url, deadline)` when HTTP comes thin."""
231
253
  url = normalize(url)
@@ -250,6 +272,8 @@ def read_page(url, deadline, render):
250
272
  text = _decode(body, charset)
251
273
  elif kind == "application/pdf":
252
274
  text = pdf_text(body, final_url, deadline)
275
+ elif office_extension(kind, final_url):
276
+ text = office_text(body, office_extension(kind, final_url), final_url)
253
277
  else:
254
278
  raise FetchError(f"Content type {kind} is not read by web_fetch ({final_url}).")
255
279
  if len(text.strip()) < MIN_CHARS:
@@ -77,6 +77,37 @@ def _read_recent_records():
77
77
  return records
78
78
 
79
79
 
80
+ def by_requester():
81
+ """The recent web search calls read back as who asked: one row per client and project folder.
82
+
83
+ Paused calls are not calls and are left out. The log never holds the query text.
84
+ Returns: dict {"rows": [{"client", "project", "sessions", "calls", "failed", "last_at", "sources"}], "calls": int,
85
+ "unattributed": int}; rows are busiest first, `sources` says how far the attribution got (see `requester`), and
86
+ `unattributed` counts the calls recorded without a requester.
87
+ """
88
+ groups, unattributed, calls = {}, 0, 0
89
+ for record in _read_recent_records():
90
+ if record.get("outcome") == "paused":
91
+ continue
92
+ calls += 1
93
+ who = record.get("requester")
94
+ if not isinstance(who, dict):
95
+ unattributed += 1
96
+ continue
97
+ row = groups.setdefault((who.get("client") or "unknown", who.get("project") or ""),
98
+ {"sessions": set(), "calls": 0, "failed": 0, "last_at": 0, "sources": set()})
99
+ row["calls"] += 1
100
+ row["failed"] += not record.get("success")
101
+ row["last_at"] = max(row["last_at"], record.get("ts") or 0)
102
+ row["sessions"].add(who.get("session") or who.get("mcp_session") or "")
103
+ row["sources"].add(who.get("source") or "none")
104
+ rows = [{"client": client, "project": project, "sessions": len(row["sessions"]), "calls": row["calls"],
105
+ "failed": row["failed"], "last_at": row["last_at"], "sources": sorted(row["sources"])}
106
+ for (client, project), row in groups.items()]
107
+ rows.sort(key=lambda row: (-row["calls"], -row["last_at"]))
108
+ return {"rows": rows, "calls": calls, "unattributed": unattributed}
109
+
110
+
80
111
  def recent_success_rate(tier_name, window=DEFAULT_WINDOW):
81
112
  records = [r for r in _read_recent_records() if r.get("tier") == tier_name and r.get("outcome") != "paused"]
82
113
  if not records: