@allansantos-dev/smart-tool 0.9.79 → 0.9.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +132 -0
  2. package/README.md +82 -18
  3. package/browser_control.py +10 -3
  4. package/browser_service_client.py +3 -32
  5. package/chromium_flags.py +30 -0
  6. package/client_hooks.py +10 -8
  7. package/code_health.py +182 -0
  8. package/code_impact.py +14 -2
  9. package/config.py +76 -8
  10. package/credential_check.py +38 -0
  11. package/dependency_notice.py +262 -14
  12. package/document_converter.py +221 -0
  13. package/document_read.py +100 -25
  14. package/document_text.py +194 -6
  15. package/dossier.py +71 -9
  16. package/embedding_cache.py +32 -8
  17. package/freshness.py +3 -2
  18. package/hook_decision.py +289 -18
  19. package/hooks/pretool_router.py +5 -4
  20. package/indexer.py +92 -3
  21. package/install.py +66 -27
  22. package/install_runtime.py +24 -10
  23. package/layer_check.py +57 -0
  24. package/link_check.py +69 -0
  25. package/local_embedder.py +115 -58
  26. package/locales/pt-BR.json +122 -18
  27. package/low_priority.py +41 -0
  28. package/model_client.py +1 -1
  29. package/model_health.py +24 -6
  30. package/notice_action.py +9 -0
  31. package/package.json +1 -1
  32. package/pr_comment.py +94 -0
  33. package/pr_impact.py +292 -8
  34. package/process_limits.py +108 -0
  35. package/project_admission.py +17 -15
  36. package/project_monitor.py +33 -7
  37. package/project_scheduler.py +12 -4
  38. package/removed_callers.py +146 -0
  39. package/scout.py +6 -2
  40. package/setup_audit.py +34 -1
  41. package/setup_ui.py +241 -41
  42. package/smart_tool_daemon.py +403 -91
  43. package/tray.py +29 -9
  44. package/update_check.py +18 -0
  45. package/update_splash.py +2 -2
  46. package/usage_meter.py +78 -0
  47. package/version.py +1 -1
  48. package/web_adapters/browser/crawl4ai_search.py +7 -3
  49. package/web_document_graph.py +12 -1
  50. package/web_fetch.py +24 -0
  51. package/web_search_health.py +31 -0
package/tray.py CHANGED
@@ -36,6 +36,8 @@ NOTICE_INTERVAL_S = 600
36
36
  NOTICE_RETRY_S = 15
37
37
  UPDATE_RETRY_S = 300
38
38
  NOTICE_UNREACHABLE_LOG_S = 600
39
+ BALLOON_TEXT_MAX = 255
40
+ BALLOON_TITLE_MAX = 63
39
41
  LOG_PATH = os.path.join(paths.DATA_DIR, "tray.log")
40
42
  LOG_MAX_BYTES = 256 * 1024
41
43
  SETUP_URL = "http://127.0.0.1:8765/setup"
@@ -89,13 +91,13 @@ def _set_state(health, icon=None):
89
91
  icon.title = i18n.t(_TITLES[health])
90
92
 
91
93
 
92
- def _notify(icon, text, buttons=(), **values):
94
+ def _notify(icon, text, buttons=(), tag=None, **values):
93
95
  """Windows notification, translated (text is an English catalog key; buttons are (catalog key, protocol link)). A
94
96
  toast under Smart Tool's own app id, since the tray balloon is shown as "Python"; when Windows refuses the toast,
95
- the balloon carries the text and the refusal."""
97
+ the balloon carries the text and the refusal. A toast with a tag replaces the earlier one with the same tag."""
96
98
  message = i18n.t(text, **values)
97
99
  try:
98
- toast_notice.show("Smart Tool", [message], [(i18n.t(label), uri) for label, uri in buttons])
100
+ toast_notice.show("Smart Tool", [message], [(i18n.t(label), uri) for label, uri in buttons], tag=tag)
99
101
  except Exception as exc:
100
102
  icon.notify(f"{message} ({exc})", "Smart Tool")
101
103
 
@@ -146,12 +148,17 @@ def _poll_loop(icon):
146
148
  time.sleep(POLL_INTERVAL_S)
147
149
 
148
150
 
149
- def _toast(title, lines, buttons, tag):
150
- """One notice: a toast Windows refuses goes to `tray.log` and does not keep the next notices from being shown."""
151
+ def _toast(icon, title, lines, buttons, tag):
152
+ """One notice: a toast Windows refuses goes to `tray.log` and, cut to what a balloon holds, to the tray balloon; a
153
+ balloon that fails is logged too, and neither keeps the next notices from being shown."""
151
154
  try:
152
155
  toast_notice.show(title, lines, buttons, tag=tag)
153
156
  except Exception as exc:
154
157
  _log(f"notice {tag}: {type(exc).__name__}: {exc}")
158
+ try:
159
+ icon.notify(f"{' '.join(lines)} ({exc})"[:BALLOON_TEXT_MAX], title[:BALLOON_TITLE_MAX])
160
+ except Exception as balloon_exc:
161
+ _log(f"notice {tag}: balloon failed: {type(balloon_exc).__name__}: {balloon_exc}")
155
162
 
156
163
 
157
164
  def _show_notices(icon, notices):
@@ -160,14 +167,14 @@ def _show_notices(icon, notices):
160
167
  if offer:
161
168
  lines = [f"{role}: {change['current']} -> {change['new']}" for role, change in offer["changes"].items()]
162
169
  nonce = offer["nonce"]
163
- _toast(i18n.t("Smart Tool has new recommended models"), lines, [
170
+ _toast(icon, i18n.t("Smart Tool has new recommended models"), lines, [
164
171
  (i18n.t("Apply now"), toast_notice.action_uri("models-apply", n=nonce)),
165
172
  (i18n.t("Don't show again"), toast_notice.action_uri("models-decline", n=nonce)),
166
173
  ], "models-upgrade")
167
174
  for row in notices.get("failing") or []:
168
175
  last = row.get("last_error") or i18n.t("see the failure log in the settings")
169
176
  _toast(
170
- i18n.t("{model} is failing", model=row["model"]),
177
+ icon, i18n.t("{model} is failing", model=row["model"]),
171
178
  [i18n.t("{errors} of {calls} calls failed in 24 h as {role}.", errors=row["errors"], calls=row["calls"],
172
179
  role=row["role"]), last[:160]],
173
180
  [(i18n.t("Switch model"), toast_notice.action_uri("open-setup", section="models")),
@@ -238,7 +245,7 @@ def _announce_start(icon):
238
245
  if failed:
239
246
  log = (i18n.N_("View log"), toast_notice.action_uri("open-update-log"))
240
247
  _notify(icon, "The update to {target} failed; Smart Tool {version} is still installed. It is tried again in "
241
- "24 h, or now from this menu.", [log], target=failed, version=version.VERSION)
248
+ "24 h, or now from this menu.", [log], tag="update-failed", target=failed, version=version.VERSION)
242
249
 
243
250
 
244
251
  def _update_loop(icon):
@@ -274,8 +281,9 @@ def _update_loop(icon):
274
281
  _notify(icon, "Updating Smart Tool from {old} to {new}...", old=version.VERSION, new=latest)
275
282
  update_check.run_update(latest, automatic=True)
276
283
  elif not known:
284
+ update_now = (i18n.N_("Update now"), toast_notice.action_uri("update-now", v=latest))
277
285
  _notify(icon, "Smart Tool {version} is available: use \"Update to {version}\" in this menu.",
278
- version=latest)
286
+ [update_now], version=latest)
279
287
  time.sleep(UPDATE_RETRY_S if unreachable else UPDATE_CHECK_INTERVAL_S)
280
288
 
281
289
 
@@ -348,6 +356,17 @@ def _choose_language(lang):
348
356
  return choose
349
357
 
350
358
 
359
+ def _open_log(icon, item):
360
+ """Menu action: opens `tray.log` with the program Windows associates with it, or says there is none yet."""
361
+ if not os.path.exists(LOG_PATH):
362
+ _notify(icon, "There is no tray log yet.")
363
+ return
364
+ try:
365
+ os.startfile(LOG_PATH)
366
+ except OSError as exc:
367
+ _notify(icon, "Could not open the log: {error}", error=exc)
368
+
369
+
351
370
  def _open_setup(icon, item):
352
371
  registry = daemon_launcher.read_registry()
353
372
  webbrowser.open((registry["url"].rstrip("/") + "/setup") if registry else SETUP_URL)
@@ -440,6 +459,7 @@ def main():
440
459
  pystray.MenuItem(lambda item: i18n.t("Projects and indexing…"), _open_projects, default=True),
441
460
  pystray.MenuItem(lambda item: i18n.t("Index a folder…"), _add_project),
442
461
  pystray.MenuItem(lambda item: i18n.t("Open settings"), _open_setup),
462
+ pystray.MenuItem(lambda item: i18n.t("Open tray log"), _open_log),
443
463
  pystray.MenuItem(lambda item: i18n.t("Start with Windows"), _toggle_autostart, checked=_autostart_checked),
444
464
  pystray.MenuItem(_update_label, _update_now),
445
465
  pystray.MenuItem(lambda item: i18n.t("Update automatically"), _toggle_auto_update,
package/update_check.py CHANGED
@@ -159,6 +159,24 @@ def notices(now=None):
159
159
  return found
160
160
 
161
161
 
162
+ def status():
163
+ """What the setup screen shows about updates, from the state file and the config only (no network).
164
+
165
+ Returns: dict with installed (str), latest and checked_at (the last npm check; None before the first), available
166
+ (bool, latest is newer than installed), command (the update command when available, else None), updated (the record
167
+ {from, to, at} of the last update, or None), failed (the version an automatic update went for and did not install,
168
+ else None), enabled (the `auto_update` config), env_opt_out (bool) and automatic (both allow automatic updates).
169
+ Raises: ValueError when `auto_update` in the config is not a boolean.
170
+ """
171
+ state = _state()
172
+ latest = state.get("latest")
173
+ available = bool(latest) and _parts(latest) > _parts(version.VERSION)
174
+ return {"installed": version.VERSION, "latest": latest, "checked_at": state.get("checked_at"), "available": available,
175
+ "command": update_command(latest) if available else None, "updated": state.get("updated"),
176
+ "failed": failed_update(), "enabled": config.auto_update(), "env_opt_out": env_opted_out(),
177
+ "automatic": automatic_enabled()}
178
+
179
+
162
180
  def run_update(latest, automatic):
163
181
  """Starts the update screen (update_splash.py) detached from the tray, which the installer stops and restarts:
164
182
  it runs the install command with no console window and shows each step and the result. npx is looked up by the
package/update_splash.py CHANGED
@@ -54,12 +54,12 @@ def install_command(target):
54
54
  the tray announced. `@latest` would let npx reuse an older package from its cache and install that instead;
55
55
  --prefer-online makes npm ask the registry for the package's version list, which in the first minutes after a
56
56
  publication its cache does not yet have (ETARGET); --expect-version makes the installer refuse a package that is
57
- not that version."""
57
+ not that version. The installer is asked for English: STEPS and the log are read from its English lines."""
58
58
  npx = node_locator.which("npx.cmd")
59
59
  if not npx:
60
60
  raise RuntimeError(i18n.t("Node.js (npx) was not found; install Node.js and try again."))
61
61
  return ["cmd.exe", "/d", "/c", npx, "--yes", "--prefer-online", f"{update_check.PACKAGE}@{target}", "install",
62
- "--expect-version", target], npx
62
+ "--language", "en", "--expect-version", target], npx
63
63
 
64
64
 
65
65
  def installed_version(folder=os.path.dirname(os.path.abspath(__file__))):
package/usage_meter.py CHANGED
@@ -38,6 +38,84 @@ def _tables(db):
38
38
  db.execute('CREATE TABLE IF NOT EXISTS usage_hourly(hour INTEGER,purpose TEXT,attempts INTEGER,errors INTEGER,PRIMARY KEY(hour,purpose))')
39
39
  db.execute('CREATE TABLE IF NOT EXISTS usage_index_jobs(job_id TEXT PRIMARY KEY,project_id TEXT,view_id TEXT,view_label TEXT,at REAL,kind TEXT,embedded_chunks INTEGER,reused_chunks INTEGER,unchanged_files INTEGER,embedded_characters INTEGER)')
40
40
  db.execute('CREATE TABLE IF NOT EXISTS usage_imports(project_id TEXT PRIMARY KEY)')
41
+ db.execute('CREATE TABLE IF NOT EXISTS usage_alerts(purpose TEXT PRIMARY KEY,muted_until REAL)')
42
+
43
+
44
+ VOLUME_FLOOR=100
45
+ VOLUME_FACTOR=2
46
+ VOLUME_BASELINE_HOURS=168
47
+ VOLUME_MUTE_S=7*86400
48
+ PURPOSE_MAX_CHARS=40
49
+
50
+
51
+ def _raw_volume(db,now):
52
+ """Purposes whose attempts in the current UTC hour exceed their threshold: the larger of VOLUME_FLOOR and
53
+ VOLUME_FACTOR times the 99th percentile of that purpose's hourly attempts over the previous VOLUME_BASELINE_HOURS
54
+ hours (hours with calls only)."""
55
+ hour=int(now//3600)
56
+ series={}
57
+ for purpose,at,attempts in db.execute('SELECT purpose,hour,SUM(attempts) FROM usage_hourly WHERE hour>=? AND hour<=? GROUP BY purpose,hour',(hour-VOLUME_BASELINE_HOURS,hour)):
58
+ series.setdefault(purpose,{})[at]=attempts
59
+ found=[]
60
+ for purpose,hours in sorted(series.items()):
61
+ current=hours.get(hour,0)
62
+ past=sorted(value for at,value in hours.items() if at<hour)
63
+ baseline=past[max(0,math.ceil(len(past)*0.99)-1)] if past else 0
64
+ threshold=max(VOLUME_FLOOR,VOLUME_FACTOR*baseline)
65
+ if current>threshold:found.append({'purpose':purpose,'attempts':current,'threshold':threshold,'baseline_p99':baseline})
66
+ return found
67
+
68
+
69
+ def _volume(now):
70
+ with project_store._LOCK,project_store._db() as db:
71
+ _tables(db)
72
+ found=_raw_volume(db,now)
73
+ muted=dict(db.execute('SELECT purpose,muted_until FROM usage_alerts').fetchall())
74
+ return [{**row,'muted':muted.get(row['purpose'],0)>now} for row in found]
75
+
76
+
77
+ def volume_alerts(now=None):
78
+ """Purposes whose model calls in the current hour are far above their own history and not muted.
79
+
80
+ Parameters: now (float or None) epoch seconds. Returns: list of {purpose, attempts, threshold, baseline_p99, muted}."""
81
+ return [row for row in _volume(now or time.time()) if not row['muted']]
82
+
83
+
84
+ def mute_volume(purpose,now=None):
85
+ """Silences the volume alert of one purpose for 7 days; the numbers stay on the card.
86
+
87
+ Raises: ValueError when purpose is not a short text."""
88
+ if not isinstance(purpose,str) or not purpose.strip() or len(purpose)>PURPOSE_MAX_CHARS:
89
+ raise ValueError('Name the purpose to mute (up to %d characters).'%PURPOSE_MAX_CHARS)
90
+ now=now or time.time()
91
+ with project_store._LOCK,project_store._db() as db:
92
+ _tables(db)
93
+ db.execute('INSERT INTO usage_alerts VALUES(?,?) ON CONFLICT(purpose) DO UPDATE SET muted_until=excluded.muted_until',(purpose,now+VOLUME_MUTE_S))
94
+ return 'volume alerts for %s muted for 7 days'%purpose
95
+
96
+
97
+ def overview(days=30,now=None):
98
+ """Calls, errors and reported tokens and cost of every project per purpose over the last `days` UTC days, and the
99
+ hourly volume alerts (muted ones included, flagged).
100
+
101
+ Parameters: days (int) 1 to 366; now (float or None) epoch seconds.
102
+ Returns: {days, totals, by_purpose, volume, currency, notes}; reported tokens and cost are None when no call reported them."""
103
+ if type(days) is not int or not 1<=days<=366:raise ValueError('Choose a period from 1 to 366 days.')
104
+ now=now or time.time()
105
+ cutoff=(datetime.datetime.fromtimestamp(now,datetime.timezone.utc)-datetime.timedelta(days=days-1)).date().isoformat()
106
+ with project_store._LOCK,project_store._db() as db:
107
+ _tables(db)
108
+ rows=db.execute('SELECT purpose,SUM(attempts),SUM(errors),SUM(prompt_tokens),SUM(output_tokens),SUM(token_reports),SUM(reported_usd),SUM(price_reports) FROM usage_daily WHERE day>=? GROUP BY purpose ORDER BY SUM(attempts) DESC',(cutoff,)).fetchall()
109
+ def entry(purpose,attempts,errors,prompt,output,tokens,usd,prices):
110
+ return {'purpose':purpose,'attempts':attempts,'errors':errors,'prompt_tokens':prompt if tokens else None,
111
+ 'output_tokens':output if tokens else None,'reported_usd':usd if prices else None}
112
+ by_purpose=[entry(*row) for row in rows]
113
+ totals={'attempts':sum(r['attempts'] for r in by_purpose),'errors':sum(r['errors'] for r in by_purpose),
114
+ 'reported_usd':round(sum(r['reported_usd'] or 0 for r in by_purpose),6) if any(r['reported_usd'] is not None for r in by_purpose) else None}
115
+ return {'days':days,'totals':totals,'by_purpose':by_purpose,'volume':_volume(now),'currency':'USD',
116
+ 'notes':['Calls and tokens are measured starting with the version that added the meter.',
117
+ 'Monetary values appear only when reported by the model gateway; they are not an estimate of the full bill.',
118
+ 'A volume alert compares the current hour with the same purpose over the previous week; it never raises a notification.']}
41
119
 
42
120
 
43
121
  def _number(value):
package/version.py CHANGED
@@ -1,7 +1,7 @@
1
1
  """Single source of the Smart Tool version and of the User-Agent sent to public APIs."""
2
2
  import re
3
3
 
4
- VERSION = "0.9.79"
4
+ VERSION = "0.9.81"
5
5
  _CONTACT_RE = re.compile(r"^(?:[^\s@()<>;]+@[^\s@()<>;]+\.[^\s@()<>;]+|https?://[^\s()<>;]+)$")
6
6
 
7
7
 
@@ -7,13 +7,16 @@
7
7
  import asyncio
8
8
  import ipaddress
9
9
  import json
10
+ import os
10
11
  import socket
11
12
  import sys
12
13
  import urllib.error
13
14
  import urllib.parse
14
15
  import urllib.request
15
16
 
16
- from crawl4ai import AsyncWebCrawler, BFSDeepCrawlStrategy, CacheMode, CrawlerRunConfig
17
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
18
+ import chromium_flags
19
+ from crawl4ai import AsyncWebCrawler, BFSDeepCrawlStrategy, BrowserConfig, CacheMode, CrawlerRunConfig
17
20
  from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
18
21
  from crawl4ai.deep_crawling.filters import FilterChain, URLFilter
19
22
  from playwright.async_api import Error as PlaywrightError
@@ -205,9 +208,10 @@ async def crawl(seed_url, max_pages):
205
208
  cache_mode=CacheMode.BYPASS,
206
209
  page_timeout=30000,
207
210
  )
208
- strategy = AsyncPlaywrightCrawlerStrategy()
211
+ browser_config = BrowserConfig(extra_args=chromium_flags.python_chromium_args())
212
+ strategy = AsyncPlaywrightCrawlerStrategy(browser_config=browser_config)
209
213
  strategy.set_hook("on_page_context_created", _install_ssrf_route_guard)
210
- async with AsyncWebCrawler(crawler_strategy=strategy) as crawler:
214
+ async with AsyncWebCrawler(crawler_strategy=strategy, config=browser_config) as crawler:
211
215
  results = await crawler.arun(url=safe_seed_url, config=config)
212
216
  return [
213
217
  {"url": r.url, "markdown": r.markdown.raw_markdown}
@@ -5,14 +5,25 @@ from html.parser import HTMLParser
5
5
  from urllib.parse import unquote,urlsplit
6
6
 
7
7
 
8
+ def local_target(origin,value):
9
+ """Caminho do alvo de uma referência local: relativo à pasta de `origin` (ou à raiz, se começar com '/')."""
10
+ path=urlsplit(value).path
11
+ return posixpath.normpath(posixpath.join(posixpath.dirname(origin),unquote(path))) if not path.startswith('/') else unquote(path).lstrip('/')
12
+
13
+
8
14
  def analyze(sources,all_paths):
15
+ """Referências locais e externas de HTML, CSS e Markdown.
16
+
17
+ Parâmetros: sources (dict caminho -> texto); all_paths (caminhos existentes, para resolver os alvos).
18
+ Retorna: dict com `dependencies` (source, target, specifier, line, kind, resolution), `diagnostics` e `parsed`.
19
+ """
9
20
  paths=set(all_paths);dependencies=[];diagnostics=[];parsed=[]
10
21
  def reference(origin,value,line,kind):
11
22
  if not value or value.startswith(('#','data:','javascript:','mailto:','tel:')):return
12
23
  url=urlsplit(value)
13
24
  if url.scheme or url.netloc:
14
25
  dependencies.append({'source':origin,'target':None,'specifier':value[:300],'line':line,'kind':kind,'resolution':'external'});return
15
- target=posixpath.normpath(posixpath.join(posixpath.dirname(origin),unquote(url.path))) if not url.path.startswith('/') else unquote(url.path).lstrip('/')
26
+ target=local_target(origin,value)
16
27
  if target.startswith('../') or target==origin:return
17
28
  found=target if target in paths else None
18
29
  dependencies.append({'source':origin,'target':found,'specifier':value[:300],'line':line,'kind':kind,'resolution':'resolved' if found else 'unresolved'})
package/web_fetch.py CHANGED
@@ -48,6 +48,10 @@ MIN_CHARS = 200
48
48
  ANSWER_MAX_TOKENS = 1500
49
49
  FAILED_TTL_S = 600
50
50
  PLACEHOLDERS = ("carregando", "loading...", "enable javascript", "habilite o javascript", "ative o javascript")
51
+ OFFICE_TYPES = {"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
52
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
53
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx"}
54
+ GENERIC_BINARY_TYPES = ("application/octet-stream", "application/zip", "binary/octet-stream")
51
55
  TEXT_TYPES = ("text/plain", "text/markdown", "text/x-markdown", "application/json", "text/csv", "application/xml",
52
56
  "text/xml")
53
57
  USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -226,6 +230,24 @@ def pdf_text(body, url, deadline):
226
230
  raise FetchError(f"PDF protected by a password is not read by web_fetch ({url}).") from None
227
231
 
228
232
 
233
+ def office_extension(kind, url):
234
+ """`docx`, `pptx` or `xlsx` when the response is an Office file: by its content type, or by the extension in the URL
235
+ path when the server sends a generic binary type; None otherwise."""
236
+ if kind in OFFICE_TYPES:
237
+ return OFFICE_TYPES[kind]
238
+ extension = urllib.parse.urlsplit(url).path.rsplit(".", 1)[-1].lower()
239
+ return extension if kind in GENERIC_BINARY_TYPES and extension in OFFICE_TYPES.values() else None
240
+
241
+
242
+ def office_text(body, extension, url):
243
+ """Text of a docx, pptx or xlsx with the markers and caps of document_text (the extractor the index uses, with its
244
+ archive guards); FetchError with the reason when the file cannot be read."""
245
+ try:
246
+ return document_text.extract(f"file.{extension}", body).decode("utf-8", errors="replace")
247
+ except document_text.DocumentError as exc:
248
+ raise FetchError(f"{extension} file could not be read ({url}): {exc}"[:300]) from None
249
+
250
+
229
251
  def read_page(url, deadline, render):
230
252
  """Page text (Markdown) from the 24 h cache, HTTP + trafilatura, or `render(url, deadline)` when HTTP comes thin."""
231
253
  url = normalize(url)
@@ -250,6 +272,8 @@ def read_page(url, deadline, render):
250
272
  text = _decode(body, charset)
251
273
  elif kind == "application/pdf":
252
274
  text = pdf_text(body, final_url, deadline)
275
+ elif office_extension(kind, final_url):
276
+ text = office_text(body, office_extension(kind, final_url), final_url)
253
277
  else:
254
278
  raise FetchError(f"Content type {kind} is not read by web_fetch ({final_url}).")
255
279
  if len(text.strip()) < MIN_CHARS:
@@ -77,6 +77,37 @@ def _read_recent_records():
77
77
  return records
78
78
 
79
79
 
80
+ def by_requester():
81
+ """The recent web search calls read back as who asked: one row per client and project folder.
82
+
83
+ Paused calls are not calls and are left out. The log never holds the query text.
84
+ Returns: dict {"rows": [{"client", "project", "sessions", "calls", "failed", "last_at", "sources"}], "calls": int,
85
+ "unattributed": int}; rows are busiest first, `sources` says how far the attribution got (see `requester`), and
86
+ `unattributed` counts the calls recorded without a requester.
87
+ """
88
+ groups, unattributed, calls = {}, 0, 0
89
+ for record in _read_recent_records():
90
+ if record.get("outcome") == "paused":
91
+ continue
92
+ calls += 1
93
+ who = record.get("requester")
94
+ if not isinstance(who, dict):
95
+ unattributed += 1
96
+ continue
97
+ row = groups.setdefault((who.get("client") or "unknown", who.get("project") or ""),
98
+ {"sessions": set(), "calls": 0, "failed": 0, "last_at": 0, "sources": set()})
99
+ row["calls"] += 1
100
+ row["failed"] += not record.get("success")
101
+ row["last_at"] = max(row["last_at"], record.get("ts") or 0)
102
+ row["sessions"].add(who.get("session") or who.get("mcp_session") or "")
103
+ row["sources"].add(who.get("source") or "none")
104
+ rows = [{"client": client, "project": project, "sessions": len(row["sessions"]), "calls": row["calls"],
105
+ "failed": row["failed"], "last_at": row["last_at"], "sources": sorted(row["sources"])}
106
+ for (client, project), row in groups.items()]
107
+ rows.sort(key=lambda row: (-row["calls"], -row["last_at"]))
108
+ return {"rows": rows, "calls": calls, "unattributed": unattributed}
109
+
110
+
80
111
  def recent_success_rate(tier_name, window=DEFAULT_WINDOW):
81
112
  records = [r for r in _read_recent_records() if r.get("tier") == tier_name and r.get("outcome") != "paused"]
82
113
  if not records: