@allansantos-dev/smart-tool 0.9.40 → 0.9.42

package/model_health.py

@@ -3,17 +3,19 @@ rerank). Never stores request content: the model, the role, the tool that was ru
 text and the time.
 
 Three consumers read it: the tool answer (a warning line when a call of that very tool failed), the setup screen
-(rate table and recent failures) and the tray (one toast per model per day when the rate passes the limit).
+(rate table, failures by type and calls per purpose) and the tray (one toast per model per day when the rate passes the limit).
 """
 import contextlib
 import contextvars
 import io
 import os
+import re
 import sqlite3
 import time
 
 import config
 import project_store
+import usage_meter
 
 WINDOW_S = 24 * 3600
 ALERT_MIN_CALLS = 5
@@ -23,10 +25,17 @@ MUTE_S = 7 * 24 * 3600
 KEEP_S = 7 * 24 * 3600
 KEEP_FAILURES = 500
 ERROR_CHARS = 300
+SAMPLE_CHARS = 160
+GROUP_ROWS = 20
+PURPOSES = ("index_embedding", "query_embedding", "rerank", "embedding_probe", "scope")
 _PATHS = ("/v1/chat/completions", "/v1/embeddings", "/v1/rerank")
 _TOOL = contextvars.ContextVar("model_health_tool", default=None)
 _COLLECTED = contextvars.ContextVar("model_health_collected", default=None)
 _SCHEMA = set()
+_STATUS = re.compile(r"HTTP(?: Error)? (\d{3})")
+_TIMEOUT = re.compile(r"timed out|timeout|WinError 10060|Errno 110\b", re.I)
+_CONNECTION = re.compile(r"URLError|ConnectionError|ConnectionRefused|ConnectionReset|refused|reset|getaddrinfo|"
+                         r"WinError 1006[01]|WinError 10054|Name or service|unreachable", re.I)
 
 
 @contextlib.contextmanager
@@ -56,6 +65,22 @@ def _tables(db):
     _SCHEMA.add(project_store.DB_PATH)
 
 
+def classify(error):
+    """Failure type of a stored error text: '429', '503', '5xx' (any other 5xx), 'auth' (401/403), 'timeout',
+    'connection' or 'other'. An HTTP status in the text wins over words in the response body."""
+    status = _STATUS.search(error or "")
+    if status:
+        code = int(status.group(1))
+        if code in (429, 503):
+            return str(code)
+        if code in (401, 403):
+            return "auth"
+        return "5xx" if code >= 500 else "other"
+    if _TIMEOUT.search(error or ""):
+        return "timeout"
+    return "connection" if _CONNECTION.search(error or "") else "other"
+
+
 def role_of(model, cfg=None):
     """Name of the config field that selects `model` ('other' when none does)."""
     cfg = cfg if cfg is not None else config.load_config()
@@ -111,8 +136,24 @@ def warning_text(failures, setup_url):
             f"{first['error']}. Pick another model: {setup_url}/setup#models")
 
 
+def _groups(rows):
+    """Failure rows (latest first) as one entry per (type, role): count, first and last time, the latest message cut to
+    SAMPLE_CHARS and the latest GROUP_ROWS raw rows. Largest group first."""
+    groups = {}
+    for row in rows:
+        group = groups.setdefault((classify(row["error"]), row["role"]), {
+            "type": classify(row["error"]), "role": row["role"], "count": 0, "first_at": row["at"], "last_at": row["at"],
+            "sample": row["error"][:SAMPLE_CHARS], "rows": []})
+        group["count"] += 1
+        group["first_at"] = row["at"]
+        if len(group["rows"]) < GROUP_ROWS:
+            group["rows"].append(row)
+    return sorted(groups.values(), key=lambda group: (-group["count"], -group["last_at"]))
+
+
 def summary(now=None):
-    """Per model and role over the last 24 h, plus the 20 latest failures."""
+    """Per model and role over the last 24 h, every kept failure of the window grouped by type and role, and the calls
+    per purpose of the usage meter (its days are UTC, so that part covers the window's days, not its exact hours)."""
     now = now or time.time()
     with project_store._LOCK, _db() as db:
         _tables(db)
@@ -121,7 +162,7 @@ def summary(now=None):
             "SELECT model,role,SUM(calls) AS calls,SUM(errors) AS errors FROM model_calls WHERE hour>=? "
             "GROUP BY model,role ORDER BY SUM(errors) DESC,SUM(calls) DESC", (int((now - WINDOW_S) // 3600),))]
         failures = [dict(row) for row in db.execute(
-            "SELECT at,model,role,tool,path,error FROM model_failures WHERE at>=? ORDER BY at DESC LIMIT 20", (now - WINDOW_S,))]
+            "SELECT at,model,role,tool,path,error FROM model_failures WHERE at>=? ORDER BY at DESC", (now - WINDOW_S,))]
         for row in models:
             last = db.execute("SELECT error FROM model_failures WHERE model=? AND role=? ORDER BY at DESC LIMIT 1",
                               (row["model"], row["role"])).fetchone()
@@ -129,7 +170,8 @@ def summary(now=None):
     for row in models:
         row["rate"] = round(row["errors"] / row["calls"], 3) if row["calls"] else 0.0
         row["failing"] = row["calls"] >= ALERT_MIN_CALLS and row["rate"] >= ALERT_RATE
-    return {"window_hours": WINDOW_S // 3600, "models": models, "failures": failures}
+    return {"window_hours": WINDOW_S // 3600, "models": models, "groups": _groups(failures),
+            "calls_by_purpose": usage_meter.calls_by_purpose(now - WINDOW_S, PURPOSES)}
 
 
 def alerts_due(now=None):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@allansantos-dev/smart-tool",
-  "version": "0.9.40",
+  "version": "0.9.42",
   "description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
   "license": "Apache-2.0",
   "author": "Allan Santos",

package/project_monitor.py

@@ -20,6 +20,8 @@ IDLE_TICK_S = 5
 # A folder that changes all the time (a tool's state folder) is indexed at most this often by the monitor; the changes
 # keep accumulating and one job covers them, and a search meanwhile reads the stale files straight from disk.
 MIN_JOB_GAP_S = 60
+# While the changes do not stop, each job lengthens the wait for the next one; a wait that passes unused resets it.
+JOB_GAP_STEPS_S = (MIN_JOB_GAP_S, 120, 240, 480, 900)
 FULL_CHECK_S = 86400
 MAX_EVENT_PATHS = 1000
 
@@ -27,7 +29,8 @@ MAX_EVENT_PATHS = 1000
 def relevant_path(path):
     parts = path.replace("\\", "/").strip("/").split("/")
     return bool(parts and all(part not in ("", ".", "..") and not part.startswith(".")
-                              and not index_scope.always_excluded(part) for part in parts))
+                              and not index_scope.always_excluded(part) for part in parts)
+                and not index_scope.is_transient(parts[-1]))
 
 
 def snapshot(root, scope):
@@ -58,6 +61,7 @@ class ProjectMonitor:
         self._git_checked = {}
         self._pending = {}
         self._next_job = {}
+        self._gap_level = {}
         self._filters = {}
         self._views = {}
         self._lock = threading.RLock()
@@ -155,6 +159,17 @@ class ProjectMonitor:
             project_store.update(key, watch_backend="windows_events", watch_error="")
             watcher.start()
 
+    def _job_gap(self, key, now):
+        """Wait before the next job of a project, called when a job is enqueued at `now`: one step longer than the last
+        one when the previous wait was used up at once (at most twice its length went by), the first step otherwise."""
+        level, at = self._gap_level.get(key, (0, None))
+        if at is None or now - at >= 2 * JOB_GAP_STEPS_S[level]:
+            level = 0
+        else:
+            level = min(level + 1, len(JOB_GAP_STEPS_S) - 1)
+        self._gap_level[key] = (level, now)
+        return JOB_GAP_STEPS_S[level]
+
     def tick(self):
         now = time.monotonic()
         projects = project_store.all_projects()
@@ -223,7 +238,7 @@ class ProjectMonitor:
             except Exception as exc:
                 project_store.update(key, status="error", last_error=str(exc)[:300], next_retry_at=time.time() + 60)
                 continue
-            self._next_job[key] = now + MIN_JOB_GAP_S
+            self._next_job[key] = now + self._job_gap(key, now)
             with self._lock:
                 # Um evento recebido durante o submit continua pendente para a próxima geração.
                 if self._pending.get(key) is pending and pending.get("revision", 0) == revision:

package/scout.py

@@ -102,7 +102,7 @@ def is_intent_question(question):
 
 def disk_lines(root, path, start, end):
     """`[(line_number, text)]` of lines `start..end` of the file as it is on disk now, or None when the file cannot be
-    read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf)."""
+    read by the index rules (secret, dotfile, too large, binary, gone) or holds no line text (docx, pdf, pptx, xlsx)."""
     if document_text.location_kind(path) != "source_line":
         return None
     data, encoding = indexer.read_indexable(root, path)

package/setup_ui.py

@@ -255,6 +255,10 @@ def handle_integration_get():
         duplicate_mode, duplicate_error = config.duplicate_mode(), ""
     except ValueError as exc:
         duplicate_mode, duplicate_error = "", str(exc)
+    try:
+        dependency_mode, dependency_error = config.dependency_mode(), ""
+    except ValueError as exc:
+        dependency_mode, dependency_error = "", str(exc)
     try:
         proposals_mode, proposals_error = config.pattern_proposals(), ""
     except ValueError as exc:
@@ -267,6 +271,8 @@ def handle_integration_get():
             "hook_modes": list(config.HOOK_MODES), "doc_mode": doc_mode, "doc_mode_error": doc_error,
             "doc_modes": list(config.DOC_MODES), "duplicate_mode": duplicate_mode,
             "duplicate_mode_error": duplicate_error, "duplicate_modes": list(config.DUPLICATE_MODES),
+            "dependency_mode": dependency_mode, "dependency_mode_error": dependency_error,
+            "dependency_modes": list(config.DEPENDENCY_MODES),
             "pattern_proposals": proposals_mode, "pattern_proposals_error": proposals_error,
             "block_patterns": {"proposals": stored["proposals"], "accepted": stored["accepted"],
                                "rejected": len(stored["rejected"]), "error": patterns_error},
@@ -285,10 +291,11 @@ def handle_integration_save(body):
         except (OSError, ValueError) as exc:
             raise SetupError(str(exc))
         return handle_integration_get()
-    if isinstance(body, dict) and body.get("action") in ("hook_mode", "doc_mode", "duplicate_mode", "pattern_proposals"):
+    if isinstance(body, dict) and body.get("action") in ("hook_mode", "doc_mode", "duplicate_mode", "dependency_mode", "pattern_proposals"):
         action = body["action"]
         allowed = {"hook_mode": config.HOOK_MODES, "doc_mode": config.DOC_MODES,
-                   "duplicate_mode": config.DUPLICATE_MODES, "pattern_proposals": config.PATTERN_PROPOSAL_MODES}[action]
+                   "duplicate_mode": config.DUPLICATE_MODES, "dependency_mode": config.DEPENDENCY_MODES,
+                   "pattern_proposals": config.PATTERN_PROPOSAL_MODES}[action]
         if body.get("mode") not in allowed:
             raise SetupError(f"Invalid mode: use {', '.join(allowed)}.")
         try:
@@ -318,21 +325,25 @@ def handle_integration_save(body):
 def handle_audit(body):
     """Setup audit panel: `index` lists the registered projects with their last audit summary; `report` returns the
     last stored report of one of them; `run` audits it now (MCP servers contacted unless probe is false). Only
-    registered project folders are accepted. The opt-in session-start report in Claude Code: `hook_state`,
+    registered project folders are accepted. The opt-in session-start report in Claude Code or Codex (`client`, default
+    claude; Codex runs it only after /hooks approval, state `waiting_approval`): `hook_state`,
     `hook_preview` (the exact entry and a token), `hook_install` (with that token, after a backup) and `hook_remove`
     (only Smart Tool's entry, after a backup)."""
     if not isinstance(body, dict):
         raise SetupError("Invalid request.")
     roots = sorted({p["root"] for p in project_store.all_projects() if p.get("root")}, key=str.casefold)
     try:
+        client = body.get("client", "claude")
         if body.get("action") == "hook_state":
-            return {"state": client_hooks.audit_state(), "file": client_hooks.CLIENTS["claude"]["files"][0]}
+            state = client_hooks.audit_state(client)
+            detail = {"detail": "installed, waiting for your approval in /hooks"} if state == client_hooks.AUDIT_WAITING else {}
+            return {"state": state, "file": client_hooks.CLIENTS[client]["files"][0], **detail}
         if body.get("action") == "hook_preview":
-            return client_hooks.audit_preview()
+            return client_hooks.audit_preview(client)
         if body.get("action") == "hook_install":
-            return client_hooks.audit_install(body.get("token"), True)
+            return client_hooks.audit_install(body.get("token"), True, client)
         if body.get("action") == "hook_remove":
-            return client_hooks.audit_remove()
+            return client_hooks.audit_remove(client)
         if body.get("action") == "index":
             stored = setup_audit.index()
             found = {root: stored[setup_audit.index_key(root)] for root in roots if setup_audit.index_key(root) in stored}
@@ -740,7 +751,7 @@ _PAGE_TEMPLATE = """<!doctype html>
 
       <div class="card" id="health-card">
         <div class="card-head"><h3>Model failures (last 24 h)</h3><span class="pill" id="health-pill" data-tone="busy">Checking</span></div>
-        <p class="lede">Every failed call to a gateway model, by role and tool. A model that fails half of its calls (5 or more) also raises a Windows notification.</p>
+        <p class="lede">Failed calls to a gateway model, grouped by type (429, 503, timeout, ...) and role, with the latest lines of each group. A model that fails half of its calls (5 or more) also raises a Windows notification.</p>
         <div id="health-table"></div>
       </div>
 
@@ -802,8 +813,9 @@ _PAGE_TEMPLATE = """<!doctype html>
       <header class="page-head"><h2 id="h-audit" tabindex="-1">Setup audit</h2>
         <p>What Claude Code and Codex load for a project, and where they collide: hooks deciding on the same tool at once, hooks or MCP servers that do not work, tools offered twice, skills and instructions one agent never sees. It runs on its own for the registered projects and never changes anything.</p></header>
       <div class="card">
-        <div class="card-head"><h3>Report at session start (Claude Code)</h3><span class="pill" id="audit-hook-pill" data-tone="busy">Checking</span></div>
-        <p class="lede">Off by default. When on, every new Claude Code session starts with one short line about the stored audit of its folder (errors and warnings only, nothing runs). It adds a SessionStart entry to <span class="mono" id="audit-hook-file"></span>, written after a backup, and turning it off removes only that entry. Without it, the agent gets the same line once, on its first search.</p>
+        <div class="card-head"><h3>Report at session start</h3><span class="pill" id="audit-hook-pill" data-tone="busy">Checking</span></div>
+        <div class="field"><label for="sel-audit-hook-client">Client</label><select id="sel-audit-hook-client"><option value="claude">Claude Code</option><option value="codex">Codex</option></select></div>
+        <p class="lede">Off by default. When on, every new session starts with one short line about the stored audit of its folder (errors and warnings only, nothing runs). It adds a SessionStart entry to <span class="mono" id="audit-hook-file"></span>, written after a backup, and turning it off removes only that entry. Codex runs a new hook only after you approve it in /hooks. Without it, the agent gets the same line once, on its first search.</p>
         <div class="actions"><button type="button" class="primary" id="btn-audit-hook-on" hidden>Turn on</button><button type="button" id="btn-audit-hook-off" hidden>Turn off</button></div>
         <div id="audit-hook-confirm" hidden>
           <p class="hint" id="audit-hook-where"></p>
@@ -1153,15 +1165,29 @@ for (const [id, action] of [["btn-upgrade-apply", "apply"], ["btn-upgrade-declin
     }
   });
 }
+const FAILURE_TYPES = { "429": "429 Too Many Requests", "503": "503 Service Unavailable", "5xx": "Other 5xx", timeout: "Timeout",
+  connection: "Connection", auth: "Authentication (401/403)", other: "Other" };
+const PURPOSE_LABELS = { index_embedding: "Indexing", query_embedding: "Query", rerank: "Rerank", embedding_probe: "Model check", scope: "Scope" };
+function healthHtml(data) {
+  const when = (at) => esc(new Date(at * 1000).toLocaleString());
+  const models = data.models.map((row) => `<tr><td>${esc(row.role)}</td><td>${esc(row.model)}</td><td>${row.errors} / ${row.calls}</td><td>${Math.round(row.rate * 100)}%</td></tr>`).join("");
+  const purposes = data.calls_by_purpose.map((row) => `<tr><td>${esc(PURPOSE_LABELS[row.purpose] || row.purpose)}</td><td>${row.attempts}</td><td>${row.errors}</td></tr>`).join("");
+  const groups = data.groups.map((group) => {
+    const rows = group.rows.map((row) => `<tr><td>${when(row.at)}</td><td>${esc(row.model)}</td><td>${esc(row.tool || "-")}</td><td>${esc(row.error)}</td></tr>`).join("");
+    return `<details class="more health-group" data-type="${esc(group.type)}"><summary>${esc(FAILURE_TYPES[group.type] || group.type)} · ${esc(group.role)} · ${plural(group.count, "failure")} · ${when(group.first_at)} to ${when(group.last_at)}</summary>`
+      + `<p class="mono">${esc(group.sample)}</p><table><tr><th>When</th><th>Model</th><th>Tool</th><th>Error</th></tr>${rows}</table>`
+      + (group.count > group.rows.length ? `<p>Showing the latest ${group.rows.length} of ${group.count}.</p>` : "") + "</details>";
+  }).join("");
+  return (models ? `<table><tr><th>Role</th><th>Model</th><th>Failed / calls</th><th>Rate</th></tr>${models}</table>` : "<p>No model call in the last 24 hours.</p>")
+    + (purposes ? `<h4>Calls by purpose</h4><table><tr><th>Purpose</th><th>Calls</th><th>Failed</th></tr>${purposes}</table>` : "")
+    + (groups ? `<h4>Failures by type</h4>${groups}` : "");
+}
 async function loadHealth() {
   try {
     const data = await getJSON("/setup/models/health");
     const failing = data.models.filter((row) => row.failing).length;
     setPill("health-pill", failing ? "bad" : "ok", failing ? plural(failing, "model") + " failing" : "No problem");
-    const models = data.models.map((row) => `<tr><td>${esc(row.role)}</td><td>${esc(row.model)}</td><td>${row.errors} / ${row.calls}</td><td>${Math.round(row.rate * 100)}%</td></tr>`).join("");
-    const failures = data.failures.map((row) => `<tr><td>${esc(new Date(row.at * 1000).toLocaleString())}</td><td>${esc(row.role)}</td><td>${esc(row.model)}</td><td>${esc(row.tool || "-")}</td><td>${esc(row.error)}</td></tr>`).join("");
-    $("health-table").innerHTML = (models ? `<table><tr><th>Role</th><th>Model</th><th>Failed / calls</th><th>Rate</th></tr>${models}</table>` : "<p>No model call in the last 24 hours.</p>")
-      + (failures ? `<table><tr><th>When</th><th>Role</th><th>Model</th><th>Tool</th><th>Error</th></tr>${failures}</table>` : "");
+    $("health-table").innerHTML = healthHtml(data);
   } catch (error) {
     setPill("health-pill", "bad", "Unavailable");
     $("health-table").textContent = "Could not read the failure log: " + error.message;
@@ -1270,6 +1296,10 @@ const RULES = [
     intro: "Compares the functions an edit writes with the indexed code: identical bodies, and near-identical ones with only local names changed. It never blocks.",
     options: [["warn", "Warn", "The edit runs; the agent is told when a function copies one already in the project.", true],
               ["off", "Off", "No duplicate checks."]] },
+  { key: "dependency_mode", title: "Vulnerable dependencies on edit", applies: "Applies on the next edit.",
+    intro: "Looks up the dependencies an edit of a dependency manifest adds or changes in the OSV advisory database. It never blocks.",
+    options: [["warn", "Warn", "The edit runs; the agent is told which added or changed dependencies have known advisories.", true],
+              ["off", "Off", "No dependency checks and no network call."]] },
   { key: "pattern_proposals", title: "Block patterns proposed by a model", applies: "Applies to the next review.",
     intro: "Afterwards, never delaying a call, the scope model (else the router model) reviews searches the fixed rule let through and may propose block patterns. Nothing blocks until you accept it on the Redirects page.",
     options: [["on", "On", "Review in the background and list proposals for you.", true],
@@ -1861,14 +1891,17 @@ async function loadAudit() {
   }
   renderOverview();
 }
+function auditHookClient() { return $("sel-audit-hook-client").value; }
+$("sel-audit-hook-client").addEventListener("change", () => { setMsg("audit-hook-msg", "", ""); loadAuditHook(); });
 async function loadAuditHook() {
   try {
-    const hook = await post("/setup/audit", { action: "hook_state" });
+    const hook = await post("/setup/audit", { action: "hook_state", client: auditHookClient() });
     $("audit-hook-file").textContent = hook.file;
-    const on = hook.state === "current";
+    const on = hook.state === "current" || hook.state === "waiting_approval";
     const pillEl = $("audit-hook-pill");
-    pillEl.textContent = on ? "On" : hook.state === "outdated" ? "Outdated" : "Off";
-    pillEl.dataset.tone = on ? "ok" : hook.state === "outdated" ? "warn" : "idle";
+    pillEl.textContent = hook.state === "waiting_approval" ? "Waiting for approval in /hooks" : on ? "On" : hook.state === "outdated" ? "Outdated" : "Off";
+    pillEl.dataset.tone = hook.state === "waiting_approval" || hook.state === "outdated" ? "warn" : on ? "ok" : "idle";
+    if (hook.detail) setMsg("audit-hook-msg", "", hook.detail + ".");
     $("btn-audit-hook-on").hidden = on;
     $("btn-audit-hook-on").textContent = hook.state === "outdated" ? "Update" : "Turn on";
     $("btn-audit-hook-off").hidden = hook.state === "missing";
@@ -1879,7 +1912,7 @@ async function loadAuditHook() {
 $("btn-audit-hook-on").addEventListener("click", async () => {
   setMsg("audit-hook-msg", "", "Preparing the change…");
   try {
-    const plan = await post("/setup/audit", { action: "hook_preview" });
+    const plan = await post("/setup/audit", { action: "hook_preview", client: auditHookClient() });
     setMsg("audit-hook-msg", "", "");
     $("audit-hook-where").textContent = "Adds this entry under " + plan.event + " in " + plan.file + ".";
     $("audit-hook-preview").textContent = JSON.stringify(plan.change, null, 2);
@@ -1887,10 +1920,10 @@ $("btn-audit-hook-on").addEventListener("click", async () => {
     $("btn-audit-hook-write").onclick = async () => {
       $("btn-audit-hook-write").disabled = true;
       try {
-        const result = await post("/setup/audit", { action: "hook_install", token: plan.token });
+        const result = await post("/setup/audit", { action: "hook_install", token: plan.token, client: auditHookClient() });
         $("audit-hook-confirm").hidden = true;
         await loadAuditHook();
-        setMsg("audit-hook-msg", "ok", "Turned on. Backup: " + (result.backup || "none, the file was new") + ".");
+        setMsg("audit-hook-msg", "ok", "Turned on. Backup: " + (result.backup || "none, the file was new") + "." + (result.after_install ? " " + result.after_install : ""));
       } catch (error) {
         setMsg("audit-hook-msg", "error", error.message);
       } finally {
@@ -1904,7 +1937,7 @@ $("btn-audit-hook-on").addEventListener("click", async () => {
 $("btn-audit-hook-cancel").addEventListener("click", () => { $("audit-hook-confirm").hidden = true; });
 $("btn-audit-hook-off").addEventListener("click", async () => {
   try {
-    const result = await post("/setup/audit", { action: "hook_remove" });
+    const result = await post("/setup/audit", { action: "hook_remove", client: auditHookClient() });
     await loadAuditHook();
     setMsg("audit-hook-msg", "ok", "Turned off. Backup: " + (result.backup || "none") + ".");
   } catch (error) {

package/smart_tool_daemon.py

@@ -676,69 +676,97 @@ def _embed_retry_delay(exc, attempt):
     return min(8, 2 ** (attempt - 1)) + random.uniform(0, .25)
 
 
+def _wait_to_retry(exc, attempt, deadline=None, cancel_check=None):
+    """Sleeps the wait the gateway asked for (`Retry-After`, else exponential with jitter) before the next attempt;
+    raises TimeoutError when that wait does not fit in `deadline`."""
+    delay = _embed_retry_delay(exc, attempt)
+    if deadline is not None and delay >= _research_time_left(deadline):
+        raise TimeoutError("Not enough time left to honor the gateway backoff.")
+    if cancel_check:
+        until = time.monotonic() + delay
+        while time.monotonic() < until:
+            cancel_check()
+            time.sleep(min(.2, max(0, until - time.monotonic())))
+    else:
+        time.sleep(delay)
+
+
+def _embed_request(model, batch, token, deadline=None, on_retry=None, cancel_check=None):
+    """Vectors of one embeddings request in input order, repeating a transient failure up to EMBED_MAX_ATTEMPTS times."""
+    for attempt in range(1, EMBED_MAX_ATTEMPTS + 1):
+        if cancel_check:
+            cancel_check()
+        timeout = (min(EMBED_REQUEST_TIMEOUT_S, _research_time_left(deadline))
+                   if deadline is not None else EMBED_REQUEST_TIMEOUT_S)
+        try:
+            response = model_client.fetch(
+                "/v1/embeddings", token, method="POST", body={"model": model, "input": batch},
+                timeout=timeout,
+            )
+        except Exception as exc:
+            if attempt >= EMBED_MAX_ATTEMPTS or not _transient_embed_error(exc):
+                raise
+            if on_retry:
+                on_retry(attempt + 1, len(batch))
+            _wait_to_retry(exc, attempt, deadline, cancel_check)
+            continue
+        if cancel_check:
+            cancel_check()
+        rows = response.get("data", []) if isinstance(response, dict) else []
+        if not isinstance(rows, list) or len(rows) != len(batch):
+            raise RuntimeError("The gateway returned a different number of embeddings than the input.")
+        ordered = [None] * len(batch)
+        for row in rows:
+            if not isinstance(row, dict):
+                raise indexer.IndexVectorError("Invalid embedding row.")
+            idx = row.get("index", 0 if len(batch) == 1 else None)
+            if type(idx) is not int or not 0 <= idx < len(batch) or ordered[idx] is not None:
+                raise indexer.IndexVectorError("Embedding indexes missing, duplicated or outside the batch.")
+            ordered[idx] = row.get("embedding")
+        return ordered
+
+
 def _embed(model, texts, token, deadline=None, on_retry=None, cancel_check=None):
+    """One vector per text. A request the gateway refuses for its size (HTTP 400 or 413) is split in two halves, each
+    sent once; a half that is refused again raises."""
     if local_embedder.is_local(model):
         return local_embedder.embed(model, texts, deadline=deadline, cancel_check=cancel_check, on_retry=on_retry)
     vectors = []
     dimensions = None
     for batch in _embed_batches(texts):
-        max_attempts = EMBED_MAX_ATTEMPTS if deadline is None or on_retry is not None else 1
-        for attempt in range(1, max_attempts + 1):
-            if cancel_check:
-                cancel_check()
-            timeout = (min(EMBED_REQUEST_TIMEOUT_S, _research_time_left(deadline))
-                       if deadline is not None else EMBED_REQUEST_TIMEOUT_S)
-            try:
-                response = model_client.fetch(
-                    "/v1/embeddings", token, method="POST", body={"model": model, "input": batch},
-                    timeout=timeout,
-                )
-            except Exception as exc:
-                if attempt >= max_attempts or not _transient_embed_error(exc):
-                    raise
-                if on_retry:
-                    on_retry(attempt + 1, len(batch))
-                delay = _embed_retry_delay(exc, attempt)
-                if deadline is not None and delay >= _research_time_left(deadline):
-                    raise TimeoutError("Not enough time left to honor the gateway backoff.")
-                if cancel_check:
-                    until = time.monotonic() + delay
-                    while time.monotonic() < until:
-                        cancel_check()
-                        time.sleep(min(.2, max(0, until - time.monotonic())))
-                else:
-                    time.sleep(delay)
-                continue
-            if cancel_check:
-                cancel_check()
-            rows = response.get("data", []) if isinstance(response, dict) else []
-            if not isinstance(rows, list) or len(rows) != len(batch):
-                raise RuntimeError("The gateway returned a different number of embeddings than the input.")
-            ordered = [None] * len(batch)
-            for row in rows:
-                if not isinstance(row, dict):
-                    raise indexer.IndexVectorError("Invalid embedding row.")
-                idx = row.get("index", 0 if len(batch) == 1 else None)
-                if type(idx) is not int or not 0 <= idx < len(batch) or ordered[idx] is not None:
-                    raise indexer.IndexVectorError("Embedding indexes missing, duplicated or outside the batch.")
-                vector = row.get("embedding")
-                dimensions = indexer.validate_vector(vector, dimensions)
-                ordered[idx] = vector
-            vectors.extend(ordered)
-            break
+        try:
+            ordered = _embed_request(model, batch, token, deadline, on_retry, cancel_check)
+        except urllib.error.HTTPError as exc:
+            if exc.code not in (400, 413) or len(batch) < 2:
+                raise
+            half = len(batch) // 2
+            ordered = (_embed_request(model, batch[:half], token, deadline, on_retry, cancel_check)
+                       + _embed_request(model, batch[half:], token, deadline, on_retry, cancel_check))
+        for vector in ordered:
+            dimensions = indexer.validate_vector(vector, dimensions)
+        vectors.extend(ordered)
     return vectors
 
 
 def _rerank(model, query, documents, top_n, token, deadline=None):
+    """Rows `{index, relevance_score}` of the best `top_n` documents. Only an HTTP answer that is worth repeating
+    (429, 5xx) is asked again, after the wait of `_wait_to_retry`; a timeout fails at once so a search does not stall."""
     if not documents:
         return []
     top_n = min(top_n, len(documents))
-    timeout = min(15, _research_time_left(deadline)) if deadline is not None else 15
-    response = model_client.fetch(
-        "/v1/rerank", token, method="POST",
-        body={"model": model, "query": query, "documents": documents, "top_n": top_n},
-        timeout=timeout,
-    )
+    for attempt in range(1, EMBED_MAX_ATTEMPTS + 1):
+        timeout = min(15, _research_time_left(deadline)) if deadline is not None else 15
+        try:
+            response = model_client.fetch(
+                "/v1/rerank", token, method="POST",
+                body={"model": model, "query": query, "documents": documents, "top_n": top_n},
+                timeout=timeout,
+            )
+            break
+        except urllib.error.HTTPError as exc:
+            if attempt >= EMBED_MAX_ATTEMPTS or not _transient_embed_error(exc):
+                raise
+            _wait_to_retry(exc, attempt, deadline)
     rows = response.get("results", []) if isinstance(response, dict) else []
     if not isinstance(rows, list) or not rows:
         raise ValueError("The reranker returned no valid results.")
@@ -1054,7 +1082,9 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
         return result
     model = model or local_embedder.resolve(cfg.get("embedding_model"), indexer.info(root).get("model_id"))
     lexical = model == indexer.LEXICAL_MODEL
-    if not lexical and query_vector is None:
+    current = indexer.info(root)
+    known_dimensions = current.get("dimensions") if current.get("model_id") == model else None
+    if not lexical and query_vector is None and not known_dimensions:
         _update_job(job_id, phase="model_probe")
         query_vector = _embed(model, ["Smart Tool: indexing dimension check."], token,
                               deadline=_job_deadline(job_id), cancel_check=checkpoint,
@@ -1062,7 +1092,6 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
                                   retry_attempt=attempt, batch_items=items))[0]
     checkpoint()
     _update_job(job_id, phase="indexing")
-    current = indexer.info(root)
     full_check = bool(arguments.get("full_check") or project.get("needs_full_check") or
                       time.time() - current.get("full_scan_at", 0) >= project_monitor.FULL_CHECK_S)
     stats = indexer.reindex(
@@ -1072,7 +1101,7 @@ def _prepare_project_index(arguments, job_id, cfg, token, query_vector=None, pre
                                  job_id, phase="indexing_retry", retry_attempt=attempt, batch_items=items)),
         on_progress=lambda processed, total: _update_job(
             job_id, phase="indexing", processed_files=processed, total_files=total),
-        model_id=model, expected_dimensions=None if lexical else len(query_vector),
+        model_id=model, expected_dimensions=None if lexical else known_dimensions or len(query_vector),
         force_rebuild=bool(arguments.get("force_rebuild")), full_check=full_check,
         changed_paths=set(arguments.get("changed_paths") or []) | set(project.get('dirty_paths', [])), cancel_check=checkpoint,
     )
@@ -1110,6 +1139,33 @@ LEXICAL_WARNING = ("No embedding model: index and search are lexical only (exact
                    "Configure an embedding_model for semantic search.")
 
 
+def _rerank_blocks(model, groups, queries, found, vectors, token, deadline=None):
+    """`({group: rows}, {group: error})` of the blocks that have candidates and a query vector. One rerank request goes
+    out per distinct query, not per block: the candidates of the blocks that share a query travel together and come back
+    split by block (`index` is the position inside the block's candidates). A reranker scores each document against the
+    query on its own, so a block gets the order and scores a request of its own would give."""
+    ranked, failed = {}, {}
+    by_query = {}
+    for group in groups:
+        if found[group] and vectors.get(queries[group]) is not None:
+            by_query.setdefault(queries[group], []).append(group)
+    for query, members in by_query.items():
+        owners = [(group, position) for group in members for position in range(len(found[group]))]
+        documents = [found[group][position][4] for group, position in owners]
+        try:
+            rows = _rerank(model, query, documents, len(documents), token, deadline=deadline)
+        except Exception as exc:
+            failed.update({group: _project_error(exc) for group in members})
+            continue
+        ranked.update({group: [] for group in members})
+        for row in rows:
+            group, position = owners[row["index"]]
+            ranked[group].append({"index": position, "relevance_score": row["relevance_score"]})
+        failed.update({group: "ValueError: The reranker returned no result for this block."
+                       for group in members if not ranked[group]})
+    return ranked, failed
+
+
 def _perform_smart_search(arguments, job_id):
     return contextvars.copy_context().run(_perform_smart_search_isolated, arguments, job_id)
 
@@ -1220,22 +1276,22 @@ def _perform_smart_search_isolated(arguments, job_id):
                                    include=lambda path, group=group: index_profile.kind(path, profile) == group)
              for group in groups}
 
+    reranked, rerank_failed = {}, {}
+    if cfg.get("rerank_model"):
+        _update_job(job_id, phase="reranking")
+        reranked, rerank_failed = _rerank_blocks(cfg["rerank_model"], groups, queries, found, vectors, token,
+                                                 deadline=_job_deadline(job_id))
+
     def search_block(group):
         top_k = limits[group]
         query, vector, candidates = queries[group], vectors.get(queries[group]), found[group]
         ordered = [(score, "hybrid" if vector is not None else "lexical", row) for score, *row in candidates[:top_k]]
         problem = None
-        if cfg.get("rerank_model") and candidates and vector is not None:
-            _update_job(job_id, phase="reranking")
-            try:
-                reranked = _rerank(cfg["rerank_model"], query, [c[4] for c in candidates], top_k, token,
-                                   deadline=_job_deadline(job_id))
-                if not reranked or any(type(item.get("index")) is not int or not 0 <= item["index"] < len(candidates)
-                                       for item in reranked):
-                    raise ValueError("Invalid index in the reranker response.")
-                ordered = [(float(item["relevance_score"]), "rerank", candidates[item["index"]][1:]) for item in reranked]
-            except Exception as exc:
-                problem = f"Rerank unavailable for {group}; hybrid lexical/vector order kept. " + _project_error(exc)
+        if group in rerank_failed:
+            problem = f"Rerank unavailable for {group}; hybrid lexical/vector order kept. " + rerank_failed[group]
+        elif group in reranked:
+            ordered = [(float(item["relevance_score"]), "rerank", candidates[item["index"]][1:])
+                       for item in reranked[group][:top_k]]
         exact = indexer.overlay_exact_keys(overlay, query)
         if exact:
             pinned = [(c[0], "overlay", c[1:]) for c in candidates if tuple(c[1:4]) in exact][:2]

package/usage_meter.py

@@ -97,6 +97,19 @@ def _record_job(job_id,payload):
         db.execute('DELETE FROM usage_index_jobs WHERE at<?',(time.time()-366*86400,))
 
 
+def calls_by_purpose(since,purposes):
+    """Attempts and errors of every project per purpose in `purposes`, from the UTC day of the epoch time `since` on
+    (the table keeps days, not hours). Purposes with no call are left out."""
+    day=datetime.datetime.fromtimestamp(since,datetime.timezone.utc).date().isoformat()
+    marks=','.join('?' for _ in purposes)
+    with project_store._LOCK,project_store._db() as db:
+        _tables(db)
+        rows=db.execute('SELECT purpose,SUM(attempts),SUM(errors) FROM usage_daily WHERE day>=? AND purpose IN ('+marks+') GROUP BY purpose',
+                        (day,*purposes)).fetchall()
+    found={purpose:{'purpose':purpose,'attempts':attempts,'errors':errors} for purpose,attempts,errors in rows}
+    return [found[purpose] for purpose in purposes if purpose in found]
+
+
 def summary(project_id,days=30,view_id=None):
     if type(days) is not int or not 1<=days<=366:raise ValueError('Choose a period from 1 to 366 days.')
     with project_store._LOCK,project_store._db() as db: