@allansantos-dev/smart-tool 0.9.1 → 0.9.3

package/smart_tool_daemon.py

@@ -48,6 +48,7 @@ import index_views
 import index_inventory
 import usage_meter
 import document_text
+import embedding_cache
 import model_defaults
 import project_identity
 import project_store
@@ -291,7 +292,7 @@ def _smart_search_job_payload(job_id):
     else:
         payload["error"] = job.get("error") or "Search did not complete."
         payload["resumable"] = bool(job.get("arguments"))
-    if job.get("stats"):
+    if job.get("stats") and job["stats"] != payload.get("result"):
         payload["stats"] = job["stats"]
     if job.get("failure_phase"):
         payload["failure_phase"] = job["failure_phase"]
@@ -302,10 +303,10 @@ def _smart_search_job_payload(job_id):
 TOOLS = [
     {
         "name": "project_manage",
-        "description": "Manages folders and indexing. list/status show project_id, storage, scope and jobs. graph reads the indexed view: coverage, references and static calls in Java, Angular, Python and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls. inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. Jobs via smart_search_result.",
+        "description": "Manages folders and indexing. Every action takes project_root (the folder) or project_id. list gives a short line per project; status shows storage, scope and jobs. graph with symbol (function, Class.method, class name, or path::name when the name repeats) answers what an edit touches: definition, callers with line, calls, tests that reach it through static calls (depth hops, default 3) and files importing it; use it before changing a function to know what to update. affected_tests answers which tests to run after editing: every test file importing a changed file (git diff against base, default HEAD, untracked included) plus changed tests, likeliest failures first, with the command to run them and run_all when config or unanalyzed code changed. graph without symbol reads the indexed view: coverage, references and static calls in Java, Angular, Python and JS/TS/JSX/TSX; includes HTML/CSS/Markdown references and TXT/DOCX coverage; with file_path, narrows to that file's symbols, imports and calls (lists capped by limit). inspect lists the files of a view_id; with file_path, returns only that file's chunks. usage shows consumption and reuse; docs lists public functions without a docstring (Python docstring, JSDoc, Javadoc; private, nested and override functions excluded) with coverage percent, excluding tests unless include_tests=true, for documenting a project that started without it; duplicates lists duplicated functions (identical, near-identical and semantic bodies, min_similarity default 0.90; below that most pairs measured were false positives), excluding tests unless include_tests=true; fix real duplicates; only dismiss with duplicates_dismiss (finding_id, reason=false_positive|intentional, note) what is not a duplicate or is a copy kept on purpose: the finding stays hidden until the code changes, and duplicates_restore reopens it; integration shows the MCP client, hook and tool usage metrics; install_hook_preview shows the change that would install the hook redirecting Grep/Read to smart_search and WebSearch/WebFetch to web_search/web_fetch in this session's client and returns a plan_id; install_hook writes it with that plan_id and confirm=true, with a backup, only after the user approves the change; search_limits saves the project's default top_k per block ({code, test, doc}, each 1-20, total at most 20); storage shows disk usage and views; pin_view protects a view; cleanup_preview and cleanup_commit remove selected views after confirmation; compare compares two stored views, without checkout or embeddings. register adds a folder; preview checks the scope; index/rebuild update; pause/cancel keep checkpoints; resume continues; policy picks on_search/eager; scope adjusts the scope; remove deletes only the local index. Jobs via smart_search_result.",
         "inputSchema": {
             "type": "object", "properties": {
-                "action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore"]},
+                "action": {"type": "string", "enum": ["list", "status", "register", "preview", "index", "rebuild", "pause", "cancel", "resume", "watch", "scope", "remove", "relocate", "probe", "policy", "inspect", "graph", "usage", "storage", "pin_view", "cleanup_preview", "cleanup_commit", "compare", "search_limits", "integration", "install_hook_preview", "install_hook", "duplicates", "duplicates_dismiss", "duplicates_restore", "docs", "affected_tests"]},
                 "finding_id": {"type": "string"},
                 "reason": {"type": "string", "enum": ["false_positive", "intentional"]},
                 "note": {"type": "string"},
@@ -324,6 +325,9 @@ TOOLS = [
                 "left_view":{"type":"string"},"right_view":{"type":"string"},"relations":{"type":"boolean"},
                 "update_mode": {"type": "string", "enum": ["on_search", "eager"]},
                 "view_id": {"type": "string"}, "file_path": {"type": "string"},
+                "base": {"type": "string", "default": "HEAD", "description": "affected_tests: git revision the changes are compared with (HEAD = uncommitted work; main or origin/main for a branch)"},
+                "symbol": {"type": "string", "description": "graph: function, Class.method, class name or path::name to get its callers, calls, tests and importers"},
+                "depth": {"type": "integer", "minimum": 1, "maximum": 4, "default": 3, "description": "graph with symbol: call hops searched for tests"},
                 "project_root": {"type": "string"}, "project_id": {"type": "string"},
                 "job_id": {"type": "string"}, "watch": {"type": "boolean"},
                 "manual": {"type": "boolean"}, "force_scope": {"type": "boolean"},
@@ -1414,27 +1418,69 @@ def _find_duplicates(root, arguments, dismissed):
     return duplicates.find(root, embed, configured_model, float(similarity), include_tests, limit, dismissed, include_dismissed)
 
 
+def _with_project_id(arguments):
+    """Agents know the folder, not the id: project_root resolves to the registered project's project_id."""
+    root = arguments.get("project_root")
+    if arguments.get("project_id") or not isinstance(root, str) or arguments.get("action") in ("register", "relocate"):
+        return arguments
+    wanted = project_identity.canonical_root(root)
+    match = next((p for p in project_store.all_projects() if project_identity.canonical_root(p["root"]) == wanted), None)
+    if not match:
+        raise ValueError(f"No registered project at {root}; register it first (action=register).")
+    return {**arguments, "project_id": match["id"]}
+
+
+def _project_summary(project):
+    data = _project_payload(project)
+    index = data.get("index") or {}
+    jobs = data.get("jobs") or []
+    return {"project_id": data["id"], "name": data.get("name"), "root": data["root"], "status": data.get("status"),
+            "enabled": data.get("enabled"), "paused": data.get("paused"), "update_mode": data.get("update_mode"),
+            "files": index.get("files"), "chunks": index.get("chunks"), "last_indexed": data.get("last_indexed"),
+            "view": (data.get("view") or {}).get("label"), "active_jobs": len(data.get("active_jobs") or []),
+            "last_job": {k: jobs[0].get(k) for k in ("job_id", "status", "kind")} if jobs else None,
+            "last_error": (data.get("last_error") or None) and str(data["last_error"])[:200]}
+
+
 def _project_action(arguments):
     if not isinstance(arguments, dict):
         raise ValueError("Pass the action in a JSON object.")
+    arguments = _with_project_id(arguments)
     action = arguments.get("action", "list")
     if action == "probe":
         return _probe_index_models()
-    if action in ('graph','compare','duplicates','duplicates_dismiss','duplicates_restore'):
+    if action in ('graph','compare','duplicates','duplicates_dismiss','duplicates_restore','docs','affected_tests'):
         # Análise somente leitura pode demorar; não prende o lock global de ações.
         key=arguments.get('project_id')
         if not isinstance(key,str) or not re.fullmatch(r'[a-f0-9]{16}',key):
-            raise ValueError('Pass a valid project_id.')
+            raise ValueError('Pass project_root (the folder) or a valid project_id.')
         project=project_store.get(key)
         if not project:
             raise ValueError('Project not registered.')
         if action=='duplicates':
             return _find_duplicates(project['root'], arguments, project.get('dismissed_duplicates') or {})
+        if action=='docs':
+            import doc_check
+            include_tests=arguments.get('include_tests',False)
+            if type(include_tests) is not bool:
+                raise ValueError('include_tests must be true or false.')
+            limit=_parse_bounded_int(arguments.get('limit'),default=30,min_v=1,max_v=200,field_name='limit')
+            return doc_check.coverage(project['root'],include_tests,limit,arguments.get('view_id'))
+        if action=='affected_tests':
+            import affected_tests
+            limit=_parse_bounded_int(arguments.get('limit'),default=30,min_v=1,max_v=200,field_name='limit')
+            return affected_tests.affected(project['root'],arguments.get('base','HEAD'),limit,arguments.get('view_id'))
         if action in ('duplicates_dismiss','duplicates_restore'):
             return _change_dismissal(key, project, arguments, action == 'duplicates_dismiss')
         if action=='graph':
             import code_graph
-            return code_graph.build(project['root'],arguments.get('view_id'),file_path=arguments.get('file_path'))
+            import code_impact
+            limit=_parse_bounded_int(arguments.get('limit'),default=30,min_v=1,max_v=200,field_name='limit')
+            if arguments.get('symbol') is not None:
+                return code_impact.impact(project['root'],arguments['symbol'],arguments.get('view_id'),
+                                          arguments.get('depth',3),limit)
+            graph=code_graph.build(project['root'],arguments.get('view_id'),file_path=arguments.get('file_path'))
+            return code_impact.for_agent(graph,limit) if arguments.get('file_path') else graph
         import view_compare
         return view_compare.compare(project['root'],arguments.get('left_view'),arguments.get('right_view'),
                                     arguments.get('file_path'),arguments.get('relations',True))
@@ -1513,6 +1559,9 @@ def _project_action_locked(arguments):
         _metric("install_hook", target=result["client"], changed=result["changed"])
         return result
     if action == "list":
+        if not arguments.get("full"):
+            return {"projects": [_project_summary(p) for p in project_store.all_projects()],
+                    "detail": "Use action=status with project_id or project_root for scope, jobs and index details."}
         return {"projects": [_project_payload(p) for p in project_store.all_projects()],
                 "storage_dir": indexer.INDEX_DIR,
                 "gateway": dict(_GATEWAY_STATUS),
@@ -1526,7 +1575,7 @@ def _project_action_locked(arguments):
         return {"project": _project_payload(project)}
     key = arguments.get("project_id")
     if not isinstance(key, str) or not re.fullmatch(r"[a-f0-9]{16}", key):
-        raise ValueError("Pass a valid project_id; use the list action to look it up.")
+        raise ValueError("Pass project_root (the folder) or a valid project_id.")
     project = project_store.get(key)
     if not project:
         raise ValueError("Project not registered.")
@@ -1995,10 +2044,8 @@ def _fan_out_web_search(query, num, budget_s=None, site_domains=()):
             try:
                 fut.result()
                 _record_tier_health(tier_name, success=True)
-            except (TierSkipped, browser_service_client.BrowserSkipped):
-                pass
-            except Exception:
-                _record_tier_health(tier_name, success=False)
+            except Exception as exc:
+                _record_tier_failure(tier_name, exc)
         return _cb
 
     for fut, tier_name in futures.items():
@@ -2243,8 +2290,7 @@ def _search_web_once(query, num, scope, deadline=None, assess_semantic=False,
             if not raw_results:
                 raise RuntimeError("zero results in the requested domain")
         except Exception as exc:
-            if not isinstance(exc, TierSkipped):
-                _record_tier_health(tier_name, success=False)
+            _record_tier_failure(tier_name, exc)
             errors.append(_sanitize_text(f"{tier_name}: {type(exc).__name__}: {exc}"))
             continue
         _record_tier_health(tier_name, success=True)
@@ -2278,8 +2324,7 @@ def _search_web_once(query, num, scope, deadline=None, assess_semantic=False,
             if not raw_results:
                 raise RuntimeError("zero results in the requested domain")
         except Exception as exc:
-            if not isinstance(exc, (TierSkipped, browser_service_client.BrowserSkipped)):
-                _record_tier_health(tier_name, success=False)
+            _record_tier_failure(tier_name, exc)
             errors.append(_sanitize_text(f"{tier_name}: {type(exc).__name__}: {exc}"))
             continue
         _record_tier_health(tier_name, success=True)
@@ -2588,13 +2633,26 @@ def _annotate_quick_results(rows, classification, cache=None):
     return result
 
 
-def _record_tier_health(tier_name, success):
+def _record_tier_health(tier_name, success, outcome=None):
     try:
-        web_search_health.record(tier_name, success)
+        web_search_health.record(tier_name, success, outcome)
     except Exception:
         pass  # telemetria é best-effort — nunca pode derrubar uma busca já resolvida
 
 
+def _record_tier_failure(tier_name, exc):
+    """Health entry for a failed tier call, keeping a 429, a captcha and a quota pause apart from other errors."""
+    if isinstance(exc, (TierSkipped, browser_service_client.BrowserSkipped)):
+        outcome = "paused"
+    elif isinstance(exc, WebProviderRateLimited):
+        outcome = "rate_limited"
+    elif isinstance(exc, browser_service_client.BrowserCaptcha):
+        outcome = "captcha"
+    else:
+        outcome = "failed"
+    _record_tier_health(tier_name, False, outcome)
+
+
 _is_safe_crawl_target = web_fetch.is_safe_target
 
 
@@ -3073,6 +3131,7 @@ def _tool_result(text, is_error=False):
 
 WEB_FETCH_TIMEOUT_S = 90
 WEB_FETCH_RENDER_TIMEOUT_S = 60
+WEB_FETCH_VECTOR_ROOT = os.path.join(paths.DATA_DIR, "web-fetch")
 
 
 def _render_page(url, deadline):
@@ -3085,6 +3144,30 @@ def _render_page(url, deadline):
     return (pages[0].get("markdown") or "") if pages else ""
 
 
+def _web_fetch_embed(texts, deadline):
+    """Vectors for the pieces of a long page and the prompt, with the pieces cached per embedding model. Fails loudly
+    without an embedding model: a long page is never cut back to its start in silence."""
+    model = local_embedder.resolve(config.load_config().get("embedding_model"), None)
+    if not model or model == indexer.LEXICAL_MODEL:
+        raise RuntimeError("This page is longer than 60,000 characters and choosing its relevant parts needs an "
+                           "embedding_model, which is not configured")
+    try:
+        token = model_client.get_token()
+        probe = _embed(model, texts[-1:], token, deadline=deadline)
+        cache = embedding_cache.VectorCache(WEB_FETCH_VECTOR_ROOT, indexer.INDEX_DIR, {
+            "purpose": "web_fetch_pieces", "model": model, "dimensions": len(probe[0])})
+        try:
+            pieces = cache.embed(texts[:-1], lambda batch: _embed(model, batch, token, deadline=deadline),
+                                 indexer.validate_vector, split=lambda unique: list(_embed_batches(unique)),
+                                 workers=indexer.EMBED_WORKERS)
+        finally:
+            cache.db.close()
+    except Exception as exc:
+        raise RuntimeError(f"This page is longer than 60,000 characters and the embedding model ({model}) that picks "
+                           f"its relevant parts failed: {type(exc).__name__}: {exc}"[:400]) from None
+    return pieces + probe
+
+
 def _web_fetch_unavailable():
     cfg = config.load_config()
     if not cfg.get("router_model"):
@@ -3105,18 +3188,22 @@ def _handle_web_fetch(arguments):
             raise RuntimeError(f"Gateway model unavailable ({problem}).")
         deadline = started + WEB_FETCH_TIMEOUT_S
         page = web_fetch.read_page(url, deadline, _render_page)
-        text = web_fetch.answer(config.load_config()["router_model"], prompt, page, deadline)
+        selected, pieces = web_fetch.select(page["text"], prompt, lambda texts: _web_fetch_embed(texts, deadline))
+        text = web_fetch.answer(config.load_config()["router_model"], prompt, {**page, "text": selected}, deadline)
     except Exception as exc:
         web_fetch.mark_failed(str(url))
         _metric("web_fetch", error=type(exc).__name__, elapsed_s=round(time.monotonic() - started, 2))
         raise RuntimeError(f"web_fetch failed: {str(exc).rstrip('.')}. The native WebFetch is allowed for this URL for "
                            f"{web_fetch.FAILED_TTL_S // 60} min.") from None
     _metric("web_fetch", cached=page["cached"], rendered=page["rendered"], render_error=page.get("render_error"),
-            page_chars=len(page["text"]),
+            page_chars=page["chars"], read_chars=len(selected), pieces=pieces,
             answer_chars=len(text), elapsed_s=round(time.monotonic() - started, 2))
     origin = "from cache" if page["cached"] else "rendered in the browser" if page["rendered"] else "downloaded now"
     read_at = datetime.datetime.fromtimestamp(page["fetched_at"]).strftime("%Y-%m-%d %H:%M")
-    cut = "; page cut to the first 60,000 characters" if page["truncated"] else ""
+    cut = (f"; page of {page['chars']:,} characters: the {pieces} parts most related to the request were read"
+           if pieces else "")
+    if page["truncated"]:
+        cut += f"; only the first {web_fetch.PAGE_STORE_MAX_CHARS:,} characters were kept"
     return _sanitize_text(f"Source: {page['final_url']} (read at {read_at}, {origin}{cut})\n\n{text}")
 
 
@@ -3395,7 +3482,7 @@ class MCPHandler(BaseHTTPRequestHandler):
         if self.path == "/setup/projects":
             try:
                 setup_ui.require_token(self.headers)
-                self._write_json(200, _project_action({"action": "list"}))
+                self._write_json(200, _project_action({"action": "list", "full": True}))
             except setup_ui.SetupError as exc:
                 self._write_json(403, {"error": str(exc)})
             except Exception as exc:

package/version.py

@@ -1,7 +1,7 @@
 """Single source of the Smart Tool version and of the User-Agent sent to public APIs."""
 import re
 
-VERSION = "0.9.1"
+VERSION = "0.9.3"
 _CONTACT_RE = re.compile(r"^(?:[^\s@()<>;]+@[^\s@()<>;]+\.[^\s@()<>;]+|https?://[^\s()<>;]+)$")
 
 

package/web_adapters/browser/fetch_camoufox.py

@@ -1,16 +1,26 @@
-"""`camoufox fetch` for the installer: a stalled download fails after 60 s instead of hanging, and progress is printed
-as plain lines (the rich bar stays invisible when the output is a pipe)."""
-import socket
+"""`camoufox fetch` for the installer. camoufox 0.5.6 prints a failed download and still exits 0, so the error is
+caught here and turned into a failing exit, and the browser executable must exist afterwards. The 1.3 GB download
+goes through ranged_download (Range requests that resume after a dropped or stalled connection; camoufox still checks
+the sha256), its other requests get a timeout (requests passes None, which also overrides socket.setdefaulttimeout),
+and progress is printed as plain lines (the rich bar stays invisible in a pipe)."""
+import sys
+from io import BytesIO
+from pathlib import Path
 
+import camoufox.__main__ as camoufox_cli
+import requests
 from camoufox import pkgman
-from camoufox.__main__ import cli
 
-socket.setdefaulttimeout(60)
+import ranged_download
+
+TIMEOUT_S = (30, 60)
 REPORT_EVERY_MB = 50
-_webdl = pkgman.webdl
+_update = camoufox_cli.CamoufoxUpdate.update
+_get = requests.get
+failures = []
 
 
-def _reporting_webdl(url, desc=None, buffer=None, bar=True, progress_callback=None):
+def _reporter():
     reported = {"mb": -REPORT_EVERY_MB}
 
     def report(done, total):
@@ -19,10 +29,49 @@ def _reporting_webdl(url, desc=None, buffer=None, bar=True, progress_callback=No
             reported["mb"] = mb
             print(f"downloaded {mb} of {total // 1048576} MB", flush=True)
 
-    return _webdl(url, desc=desc, buffer=buffer, bar=False, progress_callback=progress_callback or report)
+    return report
+
+
+def _ranged_webdl(url, desc=None, buffer=None, bar=True, progress_callback=None):
+    buffer = BytesIO() if buffer is None else buffer
+    ranged_download.download(url, buffer, progress=progress_callback or _reporter())
+    return buffer
+
+
+def _get_with_timeout(*args, **kwargs):
+    kwargs.setdefault("timeout", TIMEOUT_S)
+    return _get(*args, **kwargs)
+
+
+def _recording_update(self, *args, **kwargs):
+    try:
+        return _update(self, *args, **kwargs)
+    except Exception as exc:
+        failures.append(exc)
+        raise
+
+
+def installed_executable():
+    """The browser executable, without the download that launch_path() starts when it is missing."""
+    executable = Path(pkgman.camoufox_path(download_if_missing=False)) / pkgman.LAUNCH_FILE[pkgman.OS_NAME]
+    if not executable.is_file():
+        raise FileNotFoundError(executable)
+    return executable
+
 
+def main():
+    requests.get = _get_with_timeout
+    pkgman.webdl = _ranged_webdl
+    camoufox_cli.CamoufoxUpdate.update = _recording_update
+    camoufox_cli.cli.main(["fetch"], standalone_mode=False)
+    if failures:
+        sys.exit(f"Camoufox download failed: {failures[-1]}")
+    try:
+        executable = installed_executable()
+    except Exception as exc:
+        sys.exit(f"Camoufox is not installed after the download: {exc}")
+    print(f"Camoufox {pkgman.installed_verstr()} ready: {executable}", flush=True)
 
-pkgman.webdl = _reporting_webdl
 
 if __name__ == "__main__":
-    cli(["fetch"])
+    main()

package/web_adapters/browser/ranged_download.py

@@ -0,0 +1,78 @@
+"""Downloads a file in HTTP Range requests of CHUNK_BYTES, each on a new connection, resuming from the last byte
+received when a connection drops or stalls. Some networks (proxies, TLS inspection) cut every connection after a few
+hundred MB, so a single long transfer can never finish there. It gives up only after MAX_IDLE_ATTEMPTS attempts in a
+row without a single new byte. Standard library only: on Windows urllib trusts the system certificate store, including
+a corporate inspection CA, and honors HTTPS_PROXY."""
+import http.client
+import time
+import urllib.error
+import urllib.request
+
+CHUNK_BYTES = 32 * 1024 * 1024
+READ_BYTES = 256 * 1024
+TIMEOUT_S = 60
+MAX_IDLE_ATTEMPTS = 6
+MAX_FULL_RESTARTS = 3
+RETRYABLE_HTTP = {408, 429}
+
+
+class DownloadError(RuntimeError):
+    pass
+
+
+def _open(url, start, end, timeout):
+    request = urllib.request.Request(url, headers={"Range": f"bytes={start}-{end}", "User-Agent": "SmartTool-installer"})
+    return urllib.request.urlopen(request, timeout=timeout)
+
+
+def download(url, out, progress=None, chunk_bytes=CHUNK_BYTES, timeout=TIMEOUT_S, attempts=MAX_IDLE_ATTEMPTS,
+             sleep=time.sleep):
+    """Writes url into the seekable binary file out and returns its size; progress(done, total) after every read."""
+    done, total, idle, restarts = 0, None, 0, 0
+    while total is None or done < total:
+        end = done + chunk_bytes - 1 if total is None else min(done + chunk_bytes, total) - 1
+        before = done
+        try:
+            with _open(url, done, end, timeout) as response:
+                if response.status == 200:
+                    if done:
+                        restarts += 1
+                        if restarts > MAX_FULL_RESTARTS:
+                            raise DownloadError(f"{url} ignores Range requests and broke {restarts} times; giving up.")
+                        out.seek(0)
+                        out.truncate()
+                        done = before = 0
+                    length = response.headers.get("Content-Length")
+                    total = int(length) if length else None
+                elif response.status == 206:
+                    total = int(response.headers["Content-Range"].rsplit("/", 1)[1])
+                else:
+                    raise DownloadError(f"{url} answered HTTP {response.status} to a Range request.")
+                out.seek(done)
+                while data := response.read1(READ_BYTES):
+                    out.write(data)
+                    done += len(data)
+                    if progress:
+                        progress(done, total or 0)
+                if response.status == 200 and total is None:
+                    total = done
+        except urllib.error.HTTPError as exc:
+            exc.close()
+            if exc.code < 500 and exc.code not in RETRYABLE_HTTP:
+                raise DownloadError(f"{url} answered HTTP {exc.code}.") from exc
+            error = exc
+        except (OSError, http.client.HTTPException) as exc:
+            error = exc
+        else:
+            error = None
+        if done > before:
+            idle = 0
+        elif error is not None or done < (total or 0):
+            idle += 1
+            if idle >= attempts:
+                raise DownloadError(f"Download stopped at {done // 1048576} of {(total or 0) // 1048576} MB after "
+                                    f"{attempts} attempts without progress: {error or 'empty response'}")
+            sleep(min(2 ** idle, 30))
+    out.flush()
+    out.seek(0)
+    return done

package/web_fetch.py

@@ -6,7 +6,14 @@ e do OpenCode (`packages/opencode/src/tool/webfetch.ts`: limite de 5 MB, User-Ag
 renderizada: melhor resposta 18 × 18 (1 empate), respondeu certo 22 × 16, caracteres devolvidos 35 mil × 72 mil;
 3,6 s de mediana, navegador em 7 de 37 (páginas montadas por JavaScript). Repetições da mesma URL nos transcripts:
 92 de 124 dentro de 24 h (PAGE_TTL_S).
+
+Páginas acima de PAGE_MAX_CHARS (14,6% do cache, 10,4% das chamadas reais em 2026-10-08) não são mais cortadas no
+começo: o texto inteiro fica no cache e o modelo lê os pedaços mais parecidos com o pedido, até SELECT_BUDGET_CHARS, na
+ordem da página. Medido em 18 páginas reais e 41 perguntas: trechos depois do corte 0/28 → 13/28, cabeça 6/13 → 5/13,
+latência mediana 3,3 s → 3,1 s. Abaixo do limite a página vai inteira (seleção ali empatou em qualidade e piorou a
+cauda de latência; docs/BACKLOG.md).
 """
+import math
 import codecs
 import http.client
 import ipaddress
@@ -25,6 +32,10 @@ import research_cache
 CACHE_NAMESPACE = "web_fetch"
 PAGE_TTL_S = 86400
 PAGE_MAX_CHARS = 60_000
+PAGE_STORE_MAX_CHARS = 1_000_000
+SELECT_BUDGET_CHARS = 15_000
+CHUNK_CHARS = 2_000
+PAGE_FORMAT = 2
 MAX_DOWNLOAD_BYTES = 5 * 1024 * 1024
 MAX_REDIRECTS = 5
 HTTP_TIMEOUT_S = 20
@@ -201,7 +212,8 @@ def read_page(url, deadline, render):
     """Page text (Markdown) from the 24 h cache, HTTP + trafilatura, or `render(url, deadline)` when HTTP comes thin."""
     url = normalize(url)
     for entry in research_cache.candidates(CACHE_NAMESPACE, url, limit=1):
-        if entry.get("query") == url and time.time() - entry["saved_at"] < PAGE_TTL_S:
+        if (entry.get("query") == url and time.time() - entry["saved_at"] < PAGE_TTL_S
+                and entry["result"].get("format") == PAGE_FORMAT):
             return {**entry["result"], "cached": True, "fetched_at": entry["saved_at"]}
     final_url, kind, charset, body = fetch_http(url, deadline)
     rendered, render_error = False, None
@@ -223,12 +235,57 @@ def read_page(url, deadline, render):
     if len(text.strip()) < MIN_CHARS:
         detail = f"; browser failed: {render_error}" if render_error else ", not even when rendered in the browser"
         raise FetchError(f"Page without readable text ({final_url}){detail}.")
-    page = {"url": url, "final_url": final_url, "text": text[:PAGE_MAX_CHARS], "truncated": len(text) > PAGE_MAX_CHARS,
-            "rendered": rendered}
+    page = {"url": url, "final_url": final_url, "text": text[:PAGE_STORE_MAX_CHARS], "chars": len(text),
+            "truncated": len(text) > PAGE_STORE_MAX_CHARS, "rendered": rendered, "format": PAGE_FORMAT}
     saved_at = research_cache.put(CACHE_NAMESPACE, url, page, {})
     return {**page, "cached": False, "fetched_at": saved_at or time.time(), "render_error": render_error}
 
 
+def chunks(text, size=CHUNK_CHARS):
+    """Consecutive pieces of about size characters, cut at a line break in the second half of each piece."""
+    parts, start = [], 0
+    while start < len(text):
+        end = min(len(text), start + size)
+        if end < len(text):
+            cut = text.rfind("\n", start + size // 2, end)
+            end = cut + 1 if cut > start else end
+        parts.append(text[start:end])
+        start = end
+    return parts
+
+
+def _cosine(a, b):
+    norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
+    return sum(x * y for x, y in zip(a, b)) / norm if norm else 0.0
+
+
+def select(text, prompt, embed, budget=SELECT_BUDGET_CHARS):
+    """Text the model reads: the whole page up to PAGE_MAX_CHARS; above it, the pieces most similar to the prompt up
+    to budget characters, in page order, with [...] where pieces were skipped. embed(texts) returns one vector per
+    text and raises when the embedding model is unavailable. Returns (text, number of pieces kept or None)."""
+    if len(text) <= PAGE_MAX_CHARS:
+        return text, None
+    parts = chunks(text)
+    vectors = embed(parts + [prompt])
+    if len(vectors) != len(parts) + 1:
+        raise RuntimeError("The embedding model returned a different number of vectors than page pieces.")
+    query = vectors[-1]
+    ranked = sorted(range(len(parts)), key=lambda i: -_cosine(vectors[i], query))
+    kept, size = [], 0
+    for i in ranked:
+        if size + len(parts[i]) <= budget:
+            kept.append(i)
+            size += len(parts[i])
+    kept.sort()
+    pieces, last = [], None
+    for i in kept:
+        if last is not None and i != last + 1:
+            pieces.append("\n\n[...]\n\n")
+        pieces.append(parts[i])
+        last = i
+    return "".join(pieces), len(kept)
+
+
 def answer(model, prompt, page, deadline):
     response = model_client.fetch("/v1/chat/completions", model_client.get_token(), method="POST",
                                   timeout=min(90, _left(deadline)), body={

package/web_search_health.py

@@ -21,9 +21,17 @@ _MAX_LINES_BEFORE_ROTATE = 5000
 _LOCK = threading.Lock()
 
 
-def record(tier_name, success):
+OUTCOMES = ("ok", "failed", "rate_limited", "captcha", "paused")
+
+
+def record(tier_name, success, outcome=None):
+    """One tier call: outcome tells a provider 429 (rate_limited), a captcha and a self-imposed quota pause (paused,
+    left out of success rates) apart from an ordinary failure, so quota pressure can be read from the log."""
+    outcome = outcome or ("ok" if success else "failed")
+    if outcome not in OUTCOMES:
+        raise ValueError(f"Unknown web search outcome: {outcome}")
     os.makedirs(os.path.dirname(HEALTH_PATH), exist_ok=True)
-    entry = {"ts": time.time(), "tier": tier_name, "success": bool(success)}
+    entry = {"ts": time.time(), "tier": tier_name, "success": bool(success), "outcome": outcome}
     with _LOCK:
         with open(HEALTH_PATH, "a", encoding="utf-8") as f:
             f.write(json.dumps(entry, ensure_ascii=False) + "\n")
@@ -67,7 +75,7 @@ def _read_recent_records():
 
 
 def recent_success_rate(tier_name, window=DEFAULT_WINDOW):
-    records = [r for r in _read_recent_records() if r.get("tier") == tier_name]
+    records = [r for r in _read_recent_records() if r.get("tier") == tier_name and r.get("outcome") != "paused"]
     if not records:
         return None
     records = records[-window:]
@@ -79,7 +87,7 @@ def summary_for_tiers(tier_names, window=DEFAULT_WINDOW):
     all_records = _read_recent_records()
     summary = {}
     for name in tier_names:
-        records = [r for r in all_records if r.get("tier") == name][-window:]
+        records = [r for r in all_records if r.get("tier") == name and r.get("outcome") != "paused"][-window:]
         if not records:
             summary[name] = None
         else: