@allansantos-dev/smart-tool 0.0.0-stage → 0.9.1

package/research_cache.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""Pesquisas reutilizáveis com conteúdo protegido por DPAPI no perfil Windows.
+
+O SQLite guarda apenas hash da consulta, horários e um blob cifrado. Consultas,
+trechos, achados e URLs ficam dentro do blob. Falha de leitura/gravação desliga o
+reúso nesta chamada; a pesquisa web continua normalmente.
+"""
+import array
+import base64
+import hashlib
+import json
+import os
+import paths
+import sqlite3
+import threading
+import time
+from contextlib import contextmanager
+
+import secret_store
+
+
+CACHE_PATH = os.path.join(paths.DATA_DIR, "research-cache.sqlite3")
+MAX_ENTRIES = 500
+# Namespaces with their own quota stay out of MAX_ENTRIES, so web_fetch pages never evict web_search answers.
+OWN_QUOTA = {"web_fetch": 300}
+SCHEMA = 2
+MAX_AGE_S = 30 * 86400
+_LOCK = threading.RLock()
+
+
+def _key(depth, query):
+    normalized = " ".join(str(query or "").casefold().split())
+    return hashlib.sha256(f"{depth}\0{normalized}".encode("utf-8")).hexdigest()
+
+
+def _connect():
+    os.makedirs(os.path.dirname(CACHE_PATH), exist_ok=True)
+    connection = sqlite3.connect(CACHE_PATH, timeout=5)
+    connection.execute("""
+        CREATE TABLE IF NOT EXISTS entries (
+            id TEXT PRIMARY KEY,
+            depth TEXT NOT NULL,
+            payload TEXT NOT NULL,
+            saved_at REAL NOT NULL,
+            validated_at REAL NOT NULL
+        )
+    """)
+    return connection
+
+
+@contextmanager
+def _connection():
+    db = _connect()
+    try:
+        with db:
+            yield db
+    finally:
+        db.close()
+
+
+def candidates(depth, query, limit=200):
+    """Entrada exata primeiro, depois recentes para confirmação semântica."""
+    if not secret_store.available():
+        return []
+    key = _key(depth, query)
+    try:
+        with _LOCK, _connection() as db:
+            db.execute("DELETE FROM entries WHERE saved_at < ?", (time.time() - MAX_AGE_S,))
+            rows = db.execute(
+                "SELECT id, payload, saved_at, validated_at FROM entries "
+                "WHERE depth = ? ORDER BY CASE WHEN id = ? THEN 0 ELSE 1 END, "
+                "validated_at DESC LIMIT ?", (depth, key, limit),
+            ).fetchall()
+    except (OSError, sqlite3.Error):
+        return []
+    result = []
+    for entry_id, protected, saved_at, validated_at in rows:
+        try:
+            if not secret_store.is_protected(protected):
+                continue
+            payload = json.loads(secret_store.unprotect(protected))
+            if not isinstance(payload, dict) or payload.get("schema") != SCHEMA:
+                continue
+            vector = payload.pop("topic_vector", None)
+            if isinstance(vector, dict):
+                values = array.array("f")
+                values.frombytes(base64.b64decode(vector["b64"]))
+                payload["topic_vector"] = {"model": vector["model"], "values": values}
+            result.append({"id": entry_id, "saved_at": saved_at,
+                           "validated_at": validated_at, **payload})
+        except (ValueError, TypeError, KeyError, secret_store.SecretProtectionError):
+            continue
+    return result
+
+
+def _packed_vector(model, vector):
+    if not model or not vector:
+        return None
+    return {"model": model, "b64": base64.b64encode(array.array("f", vector).tobytes()).decode("ascii")}
+
+
+def put(depth, query, result, classification, topic_model=None, topic_vector=None, origin=None):
+    """Grava a pesquisa e devolve o horário da gravação, ou None se não gravou."""
+    if not secret_store.available():
+        return None
+    now = time.time()
+    saved_at, validated_at = (origin["saved_at"], origin["validated_at"]) if origin else (now, now)
+    payload = {"schema": SCHEMA, "query": query, "depth": depth,
+               "result": result, "classification": classification}
+    if origin:
+        payload["origin_query"] = origin.get("origin_query") or origin["query"]
+    packed = _packed_vector(topic_model, topic_vector)
+    if packed:
+        payload["topic_vector"] = packed
+    try:
+        protected = secret_store.protect(json.dumps(payload, ensure_ascii=False))
+        with _LOCK, _connection() as db:
+            db.execute(
+                "INSERT OR REPLACE INTO entries(id, depth, payload, saved_at, validated_at) "
+                "VALUES (?, ?, ?, ?, ?)", (_key(depth, query), depth, protected, saved_at, validated_at),
+            )
+            own = sorted(OWN_QUOTA)
+            marks = ",".join("?" * len(own))
+            if depth in OWN_QUOTA:
+                db.execute("DELETE FROM entries WHERE depth = ? AND id NOT IN "
+                           "(SELECT id FROM entries WHERE depth = ? ORDER BY saved_at DESC LIMIT ?)",
+                           (depth, depth, OWN_QUOTA[depth]))
+            else:
+                db.execute(f"DELETE FROM entries WHERE depth NOT IN ({marks}) AND id NOT IN "
+                           f"(SELECT id FROM entries WHERE depth NOT IN ({marks}) ORDER BY saved_at DESC LIMIT ?)",
+                           (*own, *own, MAX_ENTRIES))
+        return saved_at
+    except (OSError, ValueError, TypeError, sqlite3.Error, secret_store.SecretProtectionError):
+        return None
+
+
+def complete(depth, query, saved_at, classification, topic_model=None, topic_vector=None):
+    """Completa a entrada gravada em saved_at; uma regravação posterior não é sobrescrita."""
+    try:
+        with _LOCK, _connection() as db:
+            row = db.execute("SELECT payload FROM entries WHERE id = ? AND saved_at = ?",
+                             (_key(depth, query), saved_at)).fetchone()
+            if row is None:
+                return False
+            payload = json.loads(secret_store.unprotect(row[0]))
+            payload["classification"] = classification
+            packed = _packed_vector(topic_model, topic_vector)
+            if packed:
+                payload["topic_vector"] = packed
+            db.execute("UPDATE entries SET payload = ? WHERE id = ? AND saved_at = ?",
+                       (secret_store.protect(json.dumps(payload, ensure_ascii=False)),
+                        _key(depth, query), saved_at))
+        return True
+    except (OSError, ValueError, TypeError, sqlite3.Error, secret_store.SecretProtectionError):
+        return False
+
+
+def mark_validated(entry_id):
+    try:
+        with _LOCK, _connection() as db:
+            db.execute("UPDATE entries SET validated_at = ? WHERE id = ?", (time.time(), entry_id))
+    except (OSError, sqlite3.Error):
+        pass
+
+
+def discard(entry_id):
+    try:
+        with _LOCK, _connection() as db:
+            db.execute("DELETE FROM entries WHERE id = ?", (entry_id,))
+    except (OSError, sqlite3.Error):
+        pass

package/research_metrics.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Trilha de calibração da memória (cosseno + rerank) e do teto de pesquisa:
+`site_memory_similarity_threshold` pré-filtra tópicos, o reranker confirma o
+reaproveitamento, e `research_max_iterations` limita o retorno por rodada. Write-only, como
+`router-metrics.jsonl` — nada no daemon lê isto de volta; quem lê é
+`.token-guard/analyze_research.py`.
+
+Grava só o que a calibração usa: textos de consulta/tópico truncados e números. Nunca
+snippets nem páginas, que são conteúdo de terceiros e não ajudam a decidir o valor.
+"""
+import json
+import os
+import paths
+import threading
+import time
+
+import atomic_io
+
+METRICS_PATH = os.path.join(paths.DATA_DIR, "research-metrics.jsonl")
+METRICS_MAX_BYTES = 2 * 1024 * 1024
+MAX_TEXT_CHARS = 200
+MAX_PAIRS = 10
+
+# O daemon é multi-thread (fan-out síncrono e jobs de pesquisa em pools próprios): sem
+# lock, dois appends grandes podem se intercalar na mesma linha.
+_LOCK = threading.Lock()
+
+
+def _short(text):
+    text = str(text or "")
+    if len(text) > MAX_TEXT_CHARS:
+        return text[:MAX_TEXT_CHARS] + "..."
+    return text
+
+
+def _append(record):
+    record = {"ts": time.time(), **record}
+    try:
+        line = json.dumps(record, ensure_ascii=False) + "\n"
+        with _LOCK:
+            os.makedirs(os.path.dirname(METRICS_PATH), exist_ok=True)
+            atomic_io.rotate(METRICS_PATH, METRICS_MAX_BYTES)
+            with open(METRICS_PATH, "a", encoding="utf-8") as f:
+                f.write(line)
+    except (OSError, TypeError, ValueError):
+        # Métrica é para calibrar depois: disco cheio ou arquivo travado não pode derrubar
+        # uma pesquisa.
+        pass
+
+
+def log_site_memory(query, threshold, scored_topics, recalled, rerank_scores=None, rerank_threshold=None):
+    """Uma correlação da memória do dia. `scored_topics`: `[(tópico, similaridade)]` de
+    todos os tópicos comparados — só os `MAX_PAIRS` mais altos vão pro disco, que são os
+    que decidem onde o limiar corta."""
+    try:
+        ranked = sorted(scored_topics, key=lambda pair: pair[1], reverse=True)
+        record = {
+            "kind": "site_memory",
+            "query": _short(query),
+            "threshold": threshold,
+            "topics": len(ranked),
+            "matched": sum(1 for _, score in ranked if score >= threshold),
+            "recalled": recalled,
+            "pairs": [
+                {
+                    "topic": _short(topic), "score": round(float(score), 4),
+                    **({"rerank_score": round(float(rerank_scores[topic]), 4)}
+                       if rerank_scores is not None and topic in rerank_scores else {}),
+                }
+                for topic, score in ranked[:MAX_PAIRS]
+            ],
+        }
+        if rerank_scores is not None:
+            record["rerank_threshold"] = rerank_threshold
+            record["rerank_matched"] = sum(
+                score >= rerank_threshold for score in rerank_scores.values()
+            )
+    except Exception:
+        return
+    _append(record)
+
+
+def log_research_job(topic, max_iterations, rounds, stopped_by, sources, elapsed_s):
+    """Um job `depth='research'` concluído. `rounds` já traz, por rodada, quanto ela
+    acrescentou (`new_urls`, `findings`) — é a curva de retorno marginal que decide o
+    teto."""
+    try:
+        record = {
+            "kind": "research",
+            "topic": _short(topic),
+            "max_iterations": max_iterations,
+            "stopped_by": stopped_by,
+            "sources": sources,
+            "elapsed_s": round(elapsed_s, 1),
+            "rounds": [
+                {
+                    "query": _short(r.get("query")),
+                    "results": r.get("results_count", 0),
+                    "new_urls": r.get("new_urls", 0),
+                    "findings": r.get("findings", 0),
+                    "gaps": r.get("gaps", 0),
+                    "continue": bool(r.get("continue")),
+                    "reason": _short(r.get("reason")),
+                }
+                for r in rounds
+            ],
+        }
+    except Exception:
+        return
+    _append(record)