@allansantos-dev/smart-tool 0.0.0-stage → 0.9.1

package/integration_metrics.py

@@ -0,0 +1,82 @@
+import collections
+import json
+import os
+import paths
+import threading
+import time
+
+import atomic_io
+import router
+
+PATH = os.path.join(paths.DATA_DIR, "integration-metrics.jsonl")
+MAX_BYTES = 20 * 1024 * 1024
+_LOCK = threading.Lock()
+
+
+def record(event, **fields):
+    line = json.dumps({"ts": time.time(), "event": event, **fields}, ensure_ascii=False)
+    with _LOCK:
+        os.makedirs(os.path.dirname(PATH), exist_ok=True)
+        atomic_io.rotate(PATH, MAX_BYTES)
+        with open(PATH, "a", encoding="utf-8") as stream:
+            stream.write(line + "\n")
+
+
+def _read(path, since):
+    rows = []
+    for candidate in (path + ".1", path):
+        try:
+            with open(candidate, encoding="utf-8") as stream:
+                for line in stream:
+                    try:
+                        row = json.loads(line)
+                    except ValueError:
+                        continue
+                    if isinstance(row, dict) and isinstance(row.get("ts"), (int, float)) and row["ts"] >= since:
+                        rows.append(row)
+        except FileNotFoundError:
+            continue
+    return rows
+
+
+def summary(days=7):
+    since = time.time() - days * 86400
+    events = _read(PATH, since)
+    by_client = collections.defaultdict(collections.Counter)
+    reasons, rejects = collections.Counter(), collections.Counter()
+    for row in events:
+        by_client[row.get("client") or "desconhecido"][row.get("event", "?")] += 1
+        if row.get("event") == "search" and row.get("single_language_reason"):
+            reasons[row["single_language_reason"][:120]] += 1
+        if row.get("event") == "reject":
+            rejects[row.get("kind", "outro")] += 1
+    web = [row for row in events if row.get("event") == "web_search"]
+    origin = lambda row: "error" if row.get("error") else row.get("cache", "?")
+    by_cache = collections.Counter(origin(row) for row in web)
+    elapsed = collections.defaultdict(list)
+    for row in web:
+        if isinstance(row.get("elapsed_s"), (int, float)):
+            elapsed[origin(row)].append(row["elapsed_s"])
+    reused = sum(count for status, count in by_cache.items() if status not in ("fresh", "error"))
+    web_cache = {
+        "calls": len(web), "reused": reused, "reuse_rate": round(reused / len(web), 3) if web else None,
+        "by_cache": dict(by_cache),
+        "probes": sum(1 for row in web if row.get("probe")),
+        "probe_verdicts": dict(collections.Counter(row["verdict"] for row in web if row.get("verdict"))),
+        "errors": dict(collections.Counter(row["error"] for row in web if row.get("error"))),
+        "reused_volatility": dict(collections.Counter(row.get("volatility") or "?" for row in web
+                                                      if origin(row) not in ("fresh", "error"))),
+        "bilingual": sum(1 for row in web if row.get("bilingual")),
+        "degraded": sum(1 for row in web if row.get("degraded")),
+        "with_warnings": sum(1 for row in web if row.get("warnings")),
+        "avg_elapsed_s": {status: round(sum(values) / len(values), 2) for status, values in elapsed.items()},
+    }
+    routed = _read(router.METRICS_PATH, since)
+    decisions = collections.defaultdict(collections.Counter)
+    for row in routed:
+        decisions[row.get("client") or "no client"][row.get("decision", "?")] += 1
+    return {"days": days, "events_by_client": {k: dict(v) for k, v in by_client.items()},
+            "rejects_by_kind": dict(rejects), "single_language_reasons": dict(reasons.most_common(20)),
+            "hook_decisions_by_client": {k: dict(v) for k, v in decisions.items()},
+            "web_search_cache": web_cache,
+            "files": {"integration": PATH, "router": router.METRICS_PATH}}

package/local_embedder.py

@@ -0,0 +1,172 @@
+"""Embedder local do mcp-memory (sidecar granite em ~/.mcp-memory/embedding.properties) como reserva sem modelo no gateway.
+
+Medido em 2026-10-05 no gabarito do rerank (42 alvos de código, sem rerank, docs/spikes/local_embedder_spike.py):
+granite-embedding-107m-multilingual local 42/35/41 (no pool / 1º / top 5), MRR@5 0,892; gateway
+text-embedding-3-small do gateway 42/35/40, 0,885; só lexical 38/29/36, 0,767.
+"""
+import http.client
+import json
+import os
+import threading
+import time
+import urllib.parse
+
+import config
+import indexer
+
+PREFIX = "local:"
+PROPERTIES = os.path.join(os.path.expanduser("~"), ".mcp-memory", "embedding.properties")
+BATCH = 16
+INFO_TTL_S = 60
+MAX_ATTEMPTS = 3
+REQUEST_TIMEOUT_S = 60
+_lock = threading.Lock()
+_info = {"at": 0.0, "value": None}
+
+
+def url():
+    """Sidecar base URL; None when absent or `tei.url=off`. localhost becomes 127.0.0.1: on Windows ::1 is tried
+    first and a sidecar bound to IPv4 only costs ~2 s per connection (measured)."""
+    try:
+        with open(PROPERTIES, encoding="utf-8") as stream:
+            for line in stream:
+                key, _, value = line.strip().partition("=")
+                value = value.strip()
+                if key.strip() == "tei.url":
+                    if not value or value.lower() == "off":
+                        return None
+                    parsed = urllib.parse.urlsplit(value.rstrip("/"))
+                    if parsed.hostname == "localhost":
+                        parsed = parsed._replace(netloc=parsed.netloc.replace("localhost", "127.0.0.1", 1))
+                    return urllib.parse.urlunsplit(parsed)
+    except OSError:
+        return None
+    return None
+
+
+def configured():
+    return url() is not None
+
+
+def _connection(base, timeout):
+    parsed = urllib.parse.urlsplit(base)
+    cls = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection
+    return cls(parsed.hostname, parsed.port, timeout=timeout)
+
+
+def _request(conn, method, path, body=None):
+    headers = {"Content-Type": "application/json"} if body is not None else {}
+    conn.request(method, path, body=json.dumps(body).encode("utf-8") if body is not None else None, headers=headers)
+    response = conn.getresponse()
+    data = response.read()
+    if response.status >= 400:
+        raise ConnectionError(f"local embedder answered HTTP {response.status}")
+    return json.loads(data)
+
+
+def _probe(base, timeout):
+    conn = _connection(base, timeout)
+    try:
+        data = _request(conn, "GET", "/info")
+    finally:
+        conn.close()
+    if not data.get("model_id") or int(data.get("dimension") or 0) <= 0:
+        raise ValueError("local embedder has no model_id/dimension in /info")
+    return {"model": PREFIX + data["model_id"], "dimensions": int(data["dimension"]), "url": base,
+            "sha256": data.get("model_sha256")}
+
+
+def info(timeout=1.5):
+    """Sidecar /info cached for INFO_TTL_S; None when not configured or not answering. The probe runs under the lock
+    so a slow failing probe never overwrites a newer good answer."""
+    with _lock:
+        if time.monotonic() - _info["at"] < INFO_TTL_S:
+            return _info["value"]
+        base, value = url(), None
+        if base:
+            try:
+                value = _probe(base, timeout)
+            except (OSError, ValueError, http.client.HTTPException):
+                value = None
+        _info.update(at=time.monotonic(), value=value)
+        return value
+
+
+def is_local(model):
+    return bool(model) and model.startswith(PREFIX)
+
+
+def resolve(configured_model, current=None, mode=None):
+    """Embedding model for the index, following config `local_models` (fallback | off | prefer).
+
+    An index already built with the local model is kept while the sidecar is briefly down, so a restart of the
+    sidecar never triggers a rebuild to lexical and back."""
+    mode = mode or config.local_models_mode()
+    if mode == "off":
+        return configured_model or indexer.LEXICAL_MODEL
+    if configured_model and mode == "fallback":
+        return configured_model
+    available = info()
+    if available:
+        return available["model"]
+    if is_local(current):
+        return current
+    return configured_model or indexer.LEXICAL_MODEL
+
+
+def _left(deadline):
+    if deadline is None:
+        return REQUEST_TIMEOUT_S
+    left = deadline - time.monotonic()
+    if left <= 0:
+        raise TimeoutError("Deadline exceeded before local embeddings finished.")
+    return min(REQUEST_TIMEOUT_S, left)
+
+
+def embed(model, texts, deadline=None, cancel_check=None, on_retry=None):
+    base = url()
+    if not base:
+        raise RuntimeError(f"mcp-memory local embedder is not configured ({PROPERTIES}).")
+    # Fresh /info before and after: /embed does not echo the model, and a cached answer could let vectors of a
+    # different model reach the persistent vector cache under this model's contract.
+    try:
+        before = _probe(base, _left(deadline))
+    except (OSError, ValueError, http.client.HTTPException) as exc:
+        raise RuntimeError(f"mcp-memory local embedder is not responding ({base}): {exc}") from None
+    if before["model"] != model:
+        raise indexer.IndexCompatibilityError(
+            f"The local embedder is now {before['model']}; the index was built with {model} and must be rebuilt.")
+    vectors = []
+    conn = _connection(base, _left(deadline))
+    try:
+        for start in range(0, len(texts), BATCH):
+            part = texts[start:start + BATCH]
+            for attempt in range(1, MAX_ATTEMPTS + 1):
+                if cancel_check:
+                    cancel_check()
+                conn.timeout = _left(deadline)
+                try:
+                    got = _request(conn, "POST", "/embed", {"inputs": part})
+                    break
+                except (OSError, http.client.HTTPException) as exc:
+                    conn.close()
+                    conn = _connection(base, _left(deadline))
+                    if attempt == MAX_ATTEMPTS:
+                        raise RuntimeError(f"Local embedder failed after {MAX_ATTEMPTS} attempts: {exc}") from None
+                    if on_retry:
+                        on_retry(attempt + 1, len(part))
+                    delay = attempt
+                    if deadline is not None and time.monotonic() + delay >= deadline:
+                        raise TimeoutError("Not enough time left to retry the local embedder.") from None
+                    time.sleep(delay)
+            if not isinstance(got, list) or len(got) != len(part):
+                raise indexer.IndexVectorError("Local embedder returned a vector count different from the input.")
+            for vector in got:
+                indexer.validate_vector(vector, before["dimensions"])
+            vectors += got
+    finally:
+        conn.close()
+    after = _probe(base, _left(deadline))
+    if (after["model"], after.get("sha256")) != (before["model"], before.get("sha256")):
+        raise indexer.IndexCompatibilityError("The local embedder changed models during the call; try again.")
+    return vectors

package/model_adapters/openai_compatible.py

@@ -0,0 +1,57 @@
+"""Built-in model adapter for any OpenAI-compatible API: OpenAI, OpenRouter, a LiteLLM proxy, Ollama, vLLM...
+
+The base URL ends where the OpenAI SDK's `base_url` ends (usually `/v1`); the API key, when given, goes as a Bearer
+token. A model adapter is a module with NAME, LABEL, FIELDS, request() and verify(); see gateway.py for the contract.
+"""
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+
+NAME = "openai_compatible"
+LABEL = "OpenAI-compatible"
+FIELDS = (
+    {"name": "base_url", "label": "Base URL", "required": True, "placeholder": "https://api.openai.com/v1",
+     "hint": "Where the API starts, like the OpenAI SDK base_url: https://openrouter.ai/api/v1, "
+             "http://localhost:11434/v1."},
+    {"name": "api_key", "label": "API key", "secret": True,
+     "hint": "Optional (Ollama and local proxies do not need one)."},
+)
+_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+
+
+def _base(options):
+    url = str(options.get("base_url") or "").strip().rstrip("/")
+    parsed = urllib.parse.urlparse(url)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.query or parsed.fragment:
+        raise ValueError("Enter the OpenAI-compatible base URL, for example https://api.openai.com/v1.")
+    return url
+
+
+def request(options, route, body=None, timeout=15):
+    """POST when body is given, GET otherwise. Returns (json, response headers); HTTP errors propagate."""
+    data = json.dumps(body).encode("utf-8") if body is not None else None
+    headers = {"Content-Type": "application/json"} if data is not None else {}
+    if options.get("api_key"):
+        headers["Authorization"] = "Bearer " + options["api_key"]
+    req = urllib.request.Request(f"{_base(options)}/{route}", data=data, headers=headers,
+                                 method="POST" if data is not None else "GET")
+    with _OPENER.open(req, timeout=timeout) as response:
+        return json.load(response), response.headers
+
+
+def verify(options):
+    url = _base(options)
+    try:
+        data, _headers = request(options, "models", timeout=10)
+    except urllib.error.HTTPError as exc:
+        if exc.code in (401, 403):
+            raise RuntimeError(f"{url} rejected the API key (HTTP {exc.code}).") from None
+        if exc.code == 404:
+            raise RuntimeError(f"{url}/models does not exist (HTTP 404): the base URL must end where the API "
+                               "starts, usually at /v1.") from None
+        raise RuntimeError(f"{url} answered HTTP {exc.code} on /models.") from None
+    except (OSError, ValueError) as exc:
+        raise RuntimeError(f"{url} did not respond on /models: {getattr(exc, 'reason', exc)}") from None
+    if not isinstance(data, dict) or not isinstance(data.get("data"), list):
+        raise RuntimeError(f"{url}/models did not return the list in OpenAI format ({{\"data\": [...]}}).")

package/model_catalog.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Catálogo de modelos do gateway para `configure.py` e a tela de setup do Smart Tool.
+
+`/v1/model/info` (proxy LiteLLM) traz o `mode` de cada modelo; um gateway OpenAI padrão só tem `/v1/models`, e aí o
+modo sai do nome (embed, rerank; o resto é chat). A lista é sugestão: a tela aceita qualquer id.
+"""
+import re
+import threading
+import urllib.error
+
+import gateway
+import model_client
+
+_LIGHTWEIGHT_KEYWORDS = ("mini", "nano", "flash", "lite", "haiku", "small")
+_LIGHTWEIGHT_RE = re.compile(
+    r"\b(?:" + "|".join(_LIGHTWEIGHT_KEYWORDS) + r")\b"
+)
+
+_cache_lock = threading.Lock()
+_cached_rows = None
+_cached_selection_id = None
+
+
+def is_lightweight(model_id):
+    """Heurística por palavra completa (o catálogo não tem campo de tier) — cobre os provedores sem hardcode.
+    \\b evita falso positivo tipo "gemini" casando a substring "mini"."""
+    model_id = (model_id or "").lower()
+    return bool(_LIGHTWEIGHT_RE.search(model_id))
+
+
+def _fetch_rows():
+    """`/v1/model/info` devolve chat+embedding+rerank misturados no mesmo payload --
+    cacheado pra vida do processo. Lock só em volta do preenchimento: leitura depois de
+    populado é sempre segura, sem necessidade de guarda."""
+    global _cached_rows, _cached_selection_id
+    selected = gateway.fingerprint()
+    if _cached_rows is not None and selected == _cached_selection_id:
+        return _cached_rows
+    with _cache_lock:
+        if _cached_rows is None or selected != _cached_selection_id:
+            token = model_client.get_token()
+            try:
+                rows = model_client.fetch("/v1/model/info", token).get("data", [])
+            except urllib.error.HTTPError as exc:
+                exc.close()
+                if exc.code not in (400, 404, 405):
+                    raise
+                rows = [{"model_name": row.get("id"), "model_info": {"mode": _mode_from_name(row.get("id") or "")}}
+                        for row in model_client.fetch("/v1/models", token).get("data", [])]
+            _cached_rows = rows
+            _cached_selection_id = selected
+    return _cached_rows
+
+
+def _mode_from_name(model_id):
+    name = model_id.lower()
+    return "rerank" if "rerank" in name else "embedding" if "embed" in name else "chat"
+
+
+def list_models(mode="chat"):
+    """Ids de modelo disponíveis para a key atual num dado `mode`
+    (`chat`, `embedding` ou `rerank`), deduplicados e ordenados por nome."""
+    rows = _fetch_rows()
+
+    names = set()
+    for row in rows:
+        info = row.get("model_info") or {}
+        if info.get("mode") != mode:
+            continue
+        name = row.get("model_name")
+        if name:
+            names.add(name)
+
+    return sorted(names)

package/model_client.py

@@ -0,0 +1,38 @@
+"""Smart Tool model client: talks to the configured model gateway through its adapter."""
+import contextvars
+import time
+import usage_meter
+
+import gateway
+
+OFFLINE = contextvars.ContextVar("model_offline", default=None)
+
+
+def get_token():
+    """Kept for callers that pass a token along: credentials now live in the model adapter. Fails when no adapter."""
+    if OFFLINE.get():
+        raise RuntimeError(OFFLINE.get())
+    if gateway.selected()[0] is None:
+        raise RuntimeError("No model gateway configured: choose an adapter in the setup screen.")
+    return ""
+
+
+def fetch(path, token, method="GET", body=None, timeout=15):
+    if OFFLINE.get():
+        raise RuntimeError(OFFLINE.get())
+    started=time.monotonic();result=None;headers=None;failure=None
+    try:
+        result,headers=_fetch(path,token,method,body,timeout)
+        return result
+    except Exception as exc:
+        failure=exc
+        raise
+    finally:
+        usage_meter.record(path,body,result,headers,failure,time.monotonic()-started)
+
+
+def _fetch(path, _token, method="GET", body=None, timeout=15):
+    # Callers name routes as /v1/...; the adapter knows where its API starts.
+    if not path.startswith("/v1/"):
+        raise ValueError("Only /v1/ routes are accepted by this client.")
+    return gateway.request(path[len("/v1/"):], body=body, timeout=timeout)

package/model_defaults.py

@@ -0,0 +1,70 @@
+"""Preenche modelos recomendados somente após consultar o catálogo do gateway configurado."""
+
+import re
+
+import config
+import model_catalog
+
+
+# Matched as the last path/dot segment of the catalog id, so "gpt-4o-mini" also finds "openai/gpt-4o-mini" (OpenRouter)
+# and "openai.gpt-4o-mini" (LiteLLM aliases). Router: small, no reasoning (it runs before tool calls).
+RECOMMENDED_MODELS = {
+    "router_model": ("chat", "gpt-4o-mini"),
+    "scope_model": ("chat", "gpt-5-mini"),
+    "research_model": ("chat", "gpt-oss-20b"),
+    "embedding_model": ("embedding", "text-embedding-3-small"),
+    "rerank_model": ("rerank", "rerank-3-5"),
+}
+
+
+def _matching(catalog, name):
+    return next((model_id for model_id in sorted(catalog) if re.split(r"[/.:]", model_id)[-1] == name
+                 or model_id.endswith(("/" + name, "." + name)) or model_id == name), None)
+
+
+def ensure_recommended_models(key_ready):
+    """Preenche só campos vazios e só com IDs presentes no catálogo permitido.
+
+    Retorna os campos ainda vazios e uma explicação curta para a tela de setup/tray.
+    Um arquivo corrompido ou uma seleção manual nunca é substituído.
+    """
+    corrupted = config.corrupted_reason()
+    if corrupted:
+        return {"status": "config_corrupt", "missing": list(RECOMMENDED_MODELS),
+                "message": corrupted}
+
+    cfg = config.load_config()
+    missing = [field for field in RECOMMENDED_MODELS if not cfg.get(field)]
+    if not key_ready:
+        return {"status": "needs_key", "missing": missing,
+                "message": "Model gateway unavailable; check the base URL and the API key."}
+    if not missing:
+        return {"status": "ready", "missing": [], "message": "Models configured."}
+
+    try:
+        available = {mode: set(model_catalog.list_models(mode)) for mode in
+                     {RECOMMENDED_MODELS[field][0] for field in missing}}
+    except Exception:
+        return {"status": "catalog_unavailable", "missing": missing,
+                "message": "Catalog unavailable. Check the base URL, the API key and the connection to the gateway."}
+
+    choices = {field: found for field in missing
+               for mode, name in [RECOMMENDED_MODELS[field]]
+               for found in [_matching(available[mode], name)] if found}
+    if choices:
+        def fill_empty(current):
+            for field, model_id in choices.items():
+                if not current.get(field):
+                    current[field] = model_id
+
+        try:
+            cfg = config.update_config(fill_empty)
+        except (config.ConfigCorruptedError, OSError):
+            return {"status": "config_corrupt", "missing": missing,
+                    "message": "Could not save the configuration. Open Smart Tool to check the file."}
+
+    missing = [field for field in RECOMMENDED_MODELS if not cfg.get(field)]
+    if missing:
+        return {"status": "needs_models", "missing": missing,
+                "message": "Some recommended models are not in this gateway's catalog. Choose them in the settings."}
+    return {"status": "ready", "missing": [], "message": "Recommended models configured."}

package/package.json

@@ -1,6 +1,35 @@
 {
   "name": "@allansantos-dev/smart-tool",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.9.1",
+  "description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
+  "license": "Apache-2.0",
+  "author": "Allan Santos",
+  "homepage": "https://allansantos-dv.github.io/smart-tool/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/AllanSantos-DV/smart-tool-dist.git"
+  },
+  "keywords": [
+    "mcp",
+    "claude-code",
+    "codex",
+    "semantic-search",
+    "web-search",
+    "coding-agent"
+  ],
+  "bin": {
+    "smart-tool": "bin/smart-tool.mjs"
+  },
+  "os": [
+    "win32"
+  ],
+  "cpu": [
+    "x64"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/paths.py

@@ -0,0 +1,6 @@
+"""Where Smart Tool keeps its state: ~/.smart-tool (config.json and model adapters) and ~/.smart-tool/data (indexes,
+daemon registry, caches, metrics). Pure stdlib so the installer can import it too."""
+import os
+
+HOME_DIR = os.path.join(os.path.expanduser("~"), ".smart-tool")
+DATA_DIR = os.path.join(HOME_DIR, "data")

package/project_identity.py

@@ -0,0 +1,42 @@
+"""Identidade única de raízes e contenção de caminhos, inclusive junctions Windows."""
+import hashlib
+import os
+
+
+def display_root(root):
+    if not isinstance(root, (str, os.PathLike)) or not str(root).strip():
+        raise ValueError("Provide the project folder.")
+    return os.path.realpath(os.path.abspath(os.path.expanduser(os.fspath(root))))
+
+
+def canonical_root(root):
+    return os.path.normcase(display_root(root))
+
+
+def project_id(root):
+    return hashlib.sha256(canonical_root(root).encode("utf-8")).hexdigest()[:16]
+
+
+def legacy_ids(root):
+    """Chaves antigas conhecidas; a migração copia os dados e preserva a origem."""
+    variants = [os.path.abspath(os.fspath(root)), display_root(root),
+                os.path.normcase(os.path.abspath(os.fspath(root)))]
+    return list(dict.fromkeys(hashlib.sha256(path.encode("utf-8")).hexdigest()[:16]
+                              for path in variants))
+
+
+def within_root(root, candidate):
+    try:
+        base = canonical_root(root)
+        resolved = canonical_root(candidate)
+        return os.path.commonpath([base, resolved]) == base
+    except (OSError, ValueError, TypeError):
+        return False
+
+
+def state_path(path, state_dir):
+    """Valida o alvo antes de remover/substituir arquivos de estado."""
+    absolute = os.path.abspath(path)
+    if not within_root(state_dir, absolute) or canonical_root(absolute) == canonical_root(state_dir):
+        raise ValueError("Operation refused: path is outside the state directory.")
+    return absolute