@allansantos-dev/smart-tool 0.9.2 → 0.9.3

package/web_adapters/browser/fetch_camoufox.py

@@ -1,24 +1,26 @@
 """`camoufox fetch` for the installer. camoufox 0.5.6 prints a failed download and still exits 0, so the error is
-caught here and turned into a failing exit, and the browser executable must exist afterwards. Its downloads have no
-timeout (requests passes None, which also overrides socket.setdefaulttimeout), so every request gets one here: a
-stalled download fails after 60 s instead of hanging. Progress is printed as plain lines (the rich bar stays
-invisible in a pipe)."""
+caught here and turned into a failing exit, and the browser executable must exist afterwards. The 1.3 GB download
+goes through ranged_download (Range requests that resume after a dropped or stalled connection; camoufox still checks
+the sha256), its other requests get a timeout (requests passes None, which also overrides socket.setdefaulttimeout),
+and progress is printed as plain lines (the rich bar stays invisible in a pipe)."""
 import sys
+from io import BytesIO
 from pathlib import Path
 
 import camoufox.__main__ as camoufox_cli
 import requests
 from camoufox import pkgman
 
+import ranged_download
+
 TIMEOUT_S = (30, 60)
 REPORT_EVERY_MB = 50
-_webdl = pkgman.webdl
 _update = camoufox_cli.CamoufoxUpdate.update
 _get = requests.get
 failures = []
 
 
-def _reporting_webdl(url, desc=None, buffer=None, bar=True, progress_callback=None):
+def _reporter():
     reported = {"mb": -REPORT_EVERY_MB}
 
     def report(done, total):
@@ -27,7 +29,13 @@ def _reporting_webdl(url, desc=None, buffer=None, bar=True, progress_callback=No
             reported["mb"] = mb
             print(f"downloaded {mb} of {total // 1048576} MB", flush=True)
 
-    return _webdl(url, desc=desc, buffer=buffer, bar=False, progress_callback=progress_callback or report)
+    return report
+
+
+def _ranged_webdl(url, desc=None, buffer=None, bar=True, progress_callback=None):
+    buffer = BytesIO() if buffer is None else buffer
+    ranged_download.download(url, buffer, progress=progress_callback or _reporter())
+    return buffer
 
 
 def _get_with_timeout(*args, **kwargs):
@@ -53,7 +61,7 @@ def installed_executable():
 
 def main():
     requests.get = _get_with_timeout
-    pkgman.webdl = _reporting_webdl
+    pkgman.webdl = _ranged_webdl
     camoufox_cli.CamoufoxUpdate.update = _recording_update
     camoufox_cli.cli.main(["fetch"], standalone_mode=False)
     if failures:

package/web_adapters/browser/ranged_download.py

@@ -0,0 +1,78 @@
+"""Downloads a file in HTTP Range requests of CHUNK_BYTES, each on a new connection, resuming from the last byte
+received when a connection drops or stalls. Some networks (proxies, TLS inspection) cut every connection after a few
+hundred MB, so a single long transfer can never finish there. It gives up only after MAX_IDLE_ATTEMPTS attempts in a
+row without a single new byte. Standard library only: on Windows urllib trusts the system certificate store, including
+a corporate inspection CA, and honors HTTPS_PROXY."""
+import http.client
+import time
+import urllib.error
+import urllib.request
+
+CHUNK_BYTES = 32 * 1024 * 1024
+READ_BYTES = 256 * 1024
+TIMEOUT_S = 60
+MAX_IDLE_ATTEMPTS = 6
+MAX_FULL_RESTARTS = 3
+RETRYABLE_HTTP = {408, 429}
+
+
+class DownloadError(RuntimeError):
+    pass
+
+
+def _open(url, start, end, timeout):
+    request = urllib.request.Request(url, headers={"Range": f"bytes={start}-{end}", "User-Agent": "SmartTool-installer"})
+    return urllib.request.urlopen(request, timeout=timeout)
+
+
+def download(url, out, progress=None, chunk_bytes=CHUNK_BYTES, timeout=TIMEOUT_S, attempts=MAX_IDLE_ATTEMPTS,
+             sleep=time.sleep):
+    """Writes url into the seekable binary file out and returns its size; progress(done, total) after every read."""
+    done, total, idle, restarts = 0, None, 0, 0
+    while total is None or done < total:
+        end = done + chunk_bytes - 1 if total is None else min(done + chunk_bytes, total) - 1
+        before = done
+        try:
+            with _open(url, done, end, timeout) as response:
+                if response.status == 200:
+                    if done:
+                        restarts += 1
+                        if restarts > MAX_FULL_RESTARTS:
+                            raise DownloadError(f"{url} ignores Range requests and broke {restarts} times; giving up.")
+                        out.seek(0)
+                        out.truncate()
+                        done = before = 0
+                    length = response.headers.get("Content-Length")
+                    total = int(length) if length else None
+                elif response.status == 206:
+                    total = int(response.headers["Content-Range"].rsplit("/", 1)[1])
+                else:
+                    raise DownloadError(f"{url} answered HTTP {response.status} to a Range request.")
+                out.seek(done)
+                while data := response.read1(READ_BYTES):
+                    out.write(data)
+                    done += len(data)
+                    if progress:
+                        progress(done, total or 0)
+                if response.status == 200 and total is None:
+                    total = done
+        except urllib.error.HTTPError as exc:
+            exc.close()
+            if exc.code < 500 and exc.code not in RETRYABLE_HTTP:
+                raise DownloadError(f"{url} answered HTTP {exc.code}.") from exc
+            error = exc
+        except (OSError, http.client.HTTPException) as exc:
+            error = exc
+        else:
+            error = None
+        if done > before:
+            idle = 0
+        elif error is not None or done < (total or 0):
+            idle += 1
+            if idle >= attempts:
+                raise DownloadError(f"Download stopped at {done // 1048576} of {(total or 0) // 1048576} MB after "
+                                    f"{attempts} attempts without progress: {error or 'empty response'}")
+            sleep(min(2 ** idle, 30))
+    out.flush()
+    out.seek(0)
+    return done

package/web_fetch.py

@@ -6,7 +6,14 @@ e do OpenCode (`packages/opencode/src/tool/webfetch.ts`: limite de 5 MB, User-Ag
 renderizada: melhor resposta 18 × 18 (1 empate), respondeu certo 22 × 16, caracteres devolvidos 35 mil × 72 mil;
 3,6 s de mediana, navegador em 7 de 37 (páginas montadas por JavaScript). Repetições da mesma URL nos transcripts:
 92 de 124 dentro de 24 h (PAGE_TTL_S).
+
+Páginas acima de PAGE_MAX_CHARS (14,6% do cache, 10,4% das chamadas reais em 2026-10-08) não são mais cortadas no
+começo: o texto inteiro fica no cache e o modelo lê os pedaços mais parecidos com o pedido, até SELECT_BUDGET_CHARS, na
+ordem da página. Medido em 18 páginas reais e 41 perguntas: trechos depois do corte 0/28 → 13/28, cabeça 6/13 → 5/13,
+latência mediana 3,3 s → 3,1 s. Abaixo do limite a página vai inteira (seleção ali empatou em qualidade e piorou a
+cauda de latência; docs/BACKLOG.md).
 """
+import math
 import codecs
 import http.client
 import ipaddress
@@ -25,6 +32,10 @@ import research_cache
 CACHE_NAMESPACE = "web_fetch"
 PAGE_TTL_S = 86400
 PAGE_MAX_CHARS = 60_000
+PAGE_STORE_MAX_CHARS = 1_000_000
+SELECT_BUDGET_CHARS = 15_000
+CHUNK_CHARS = 2_000
+PAGE_FORMAT = 2
 MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024
 MAX_REDIRECTS = 5
 HTTP_TIMEOUT_S = 20
@@ -201,7 +212,8 @@ def read_page(url, deadline, render):
     """Page text (Markdown) from the 24 h cache, HTTP + trafilatura, or `render(url, deadline)` when HTTP comes thin."""
     url = normalize(url)
     for entry in research_cache.candidates(CACHE_NAMESPACE, url, limit=1):
-        if entry.get("query") == url and time.time() - entry["saved_at"] < PAGE_TTL_S:
+        if (entry.get("query") == url and time.time() - entry["saved_at"] < PAGE_TTL_S
+                and entry["result"].get("format") == PAGE_FORMAT):
             return {**entry["result"], "cached": True, "fetched_at": entry["saved_at"]}
     final_url, kind, charset, body = fetch_http(url, deadline)
     rendered, render_error = False, None
@@ -223,12 +235,57 @@ def read_page(url, deadline, render):
     if len(text.strip()) < MIN_CHARS:
         detail = f"; browser failed: {render_error}" if render_error else ", not even when rendered in the browser"
         raise FetchError(f"Page without readable text ({final_url}){detail}.")
-    page = {"url": url, "final_url": final_url, "text": text[:PAGE_MAX_CHARS], "truncated": len(text) > PAGE_MAX_CHARS,
-            "rendered": rendered}
+    page = {"url": url, "final_url": final_url, "text": text[:PAGE_STORE_MAX_CHARS], "chars": len(text),
+            "truncated": len(text) > PAGE_STORE_MAX_CHARS, "rendered": rendered, "format": PAGE_FORMAT}
     saved_at = research_cache.put(CACHE_NAMESPACE, url, page, {})
     return {**page, "cached": False, "fetched_at": saved_at or time.time(), "render_error": render_error}
 
 
+def chunks(text, size=CHUNK_CHARS):
+    """Consecutive pieces of about size characters, cut at a line break in the second half of each piece."""
+    parts, start = [], 0
+    while start < len(text):
+        end = min(len(text), start + size)
+        if end < len(text):
+            cut = text.rfind("\n", start + size // 2, end)
+            end = cut + 1 if cut > start else end
+        parts.append(text[start:end])
+        start = end
+    return parts
+
+
+def _cosine(a, b):
+    norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
+    return sum(x * y for x, y in zip(a, b)) / norm if norm else 0.0
+
+
+def select(text, prompt, embed, budget=SELECT_BUDGET_CHARS):
+    """Text the model reads: the whole page up to PAGE_MAX_CHARS; above it, the pieces most similar to the prompt up
+    to budget characters, in page order, with [...] where pieces were skipped. embed(texts) returns one vector per
+    text and raises when the embedding model is unavailable. Returns (text, number of pieces kept or None)."""
+    if len(text) <= PAGE_MAX_CHARS:
+        return text, None
+    parts = chunks(text)
+    vectors = embed(parts + [prompt])
+    if len(vectors) != len(parts) + 1:
+        raise RuntimeError("The embedding model returned a different number of vectors than page pieces.")
+    query = vectors[-1]
+    ranked = sorted(range(len(parts)), key=lambda i: -_cosine(vectors[i], query))
+    kept, size = [], 0
+    for i in ranked:
+        if size + len(parts[i]) <= budget:
+            kept.append(i)
+            size += len(parts[i])
+    kept.sort()
+    pieces, last = [], None
+    for i in kept:
+        if last is not None and i != last + 1:
+            pieces.append("\n\n[...]\n\n")
+        pieces.append(parts[i])
+        last = i
+    return "".join(pieces), len(kept)
+
+
 def answer(model, prompt, page, deadline):
     response = model_client.fetch("/v1/chat/completions", model_client.get_token(), method="POST",
                                   timeout=min(90, _left(deadline)), body={

package/web_search_health.py

@@ -21,9 +21,17 @@ _MAX_LINES_BEFORE_ROTATE = 5000
 _LOCK = threading.Lock()
 
 
-def record(tier_name, success):
+OUTCOMES = ("ok", "failed", "rate_limited", "captcha", "paused")
+
+
+def record(tier_name, success, outcome=None):
+    """One tier call: outcome tells a provider 429 (rate_limited), a captcha and a self-imposed quota pause (paused,
+    left out of success rates) apart from an ordinary failure, so quota pressure can be read from the log."""
+    outcome = outcome or ("ok" if success else "failed")
+    if outcome not in OUTCOMES:
+        raise ValueError(f"Unknown web search outcome: {outcome}")
     os.makedirs(os.path.dirname(HEALTH_PATH), exist_ok=True)
-    entry = {"ts": time.time(), "tier": tier_name, "success": bool(success)}
+    entry = {"ts": time.time(), "tier": tier_name, "success": bool(success), "outcome": outcome}
     with _LOCK:
         with open(HEALTH_PATH, "a", encoding="utf-8") as f:
             f.write(json.dumps(entry, ensure_ascii=False) + "\n")
@@ -67,7 +75,7 @@ def _read_recent_records():
 
 
 def recent_success_rate(tier_name, window=DEFAULT_WINDOW):
-    records = [r for r in _read_recent_records() if r.get("tier") == tier_name]
+    records = [r for r in _read_recent_records() if r.get("tier") == tier_name and r.get("outcome") != "paused"]
     if not records:
         return None
     records = records[-window:]
@@ -79,7 +87,7 @@ def summary_for_tiers(tier_names, window=DEFAULT_WINDOW):
     all_records = _read_recent_records()
     summary = {}
     for name in tier_names:
-        records = [r for r in all_records if r.get("tier") == name][-window:]
+        records = [r for r in all_records if r.get("tier") == name and r.get("outcome") != "paused"][-window:]
         if not records:
             summary[name] = None
         else: