@tiny-fish/hermes 0.1.1

package/tinyfish_hermes/credit_policy.py

@@ -0,0 +1,128 @@
+"""Credit policy gating for TinyFish features that consume credits."""
+
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import urlparse
+
+from .config import (
+    CreditFeature,
+    CreditPolicy,
+    browser_cloud_provider,
+    credit_policy,
+    load_config,
+    normalize_feature,
+)
+
+FEATURE_LABELS: dict[CreditFeature, str] = {
+    "browser": "TinyFish Browser",
+}
+
+
+def target_domain(target: str | None) -> str:
+    if not target:
+        return ""
+    parsed = urlparse(target)
+    if parsed.netloc:
+        return parsed.netloc
+    return target[:120]
+
+
+def policy_message(
+    feature: CreditFeature | str, policy: CreditPolicy | None = None
+) -> str:
+    normalized = normalize_feature(str(feature))
+    resolved = policy or credit_policy(normalized)
+    return (
+        f"{FEATURE_LABELS[normalized]} sessions consume TinyFish credits. "
+        f"Current policy is '{resolved}'."
+    )
+
+
+def block_message(feature: CreditFeature | str) -> str:
+    normalized = normalize_feature(str(feature))
+    return (
+        f"BLOCKED: {policy_message(normalized, 'deny')} "
+        f"Set `tinyfish.credit_policy.{normalized}` to `request` (per-session "
+        "approval) or `allow` in Hermes config; `hermes tinyfish browser` "
+        "manages this setting."
+    )
+
+
+def approval_reason(
+    feature: CreditFeature | str, operation: str, target: str | None = None
+) -> str:
+    normalized = normalize_feature(str(feature))
+    domain = target_domain(target)
+    target_text = f" Target: {domain}." if domain else ""
+    return (
+        f"{FEATURE_LABELS[normalized]} operation '{operation}' consumes "
+        f"TinyFish credits.{target_text}"
+    )
+
+
+def request_credit_approval(
+    feature: CreditFeature | str, operation: str, target: str | None = None
+) -> tuple[bool, str]:
+    normalized = normalize_feature(str(feature))
+    policy = credit_policy(normalized)
+    if policy == "deny":
+        return False, block_message(normalized)
+    if policy == "allow":
+        return True, ""
+
+    reason = approval_reason(normalized, operation, target)
+    try:
+        from tools.approval import request_tool_approval
+
+        # Domain-grained key: one [a]lways covers the browse; new domains still prompt.
+        result = request_tool_approval(
+            f"tinyfish_{normalized}",
+            reason,
+            rule_key=f"tinyfish:{normalized}:{target_domain(target)}",
+        )
+    except Exception as exc:
+        return False, (
+            f"BLOCKED: {FEATURE_LABELS[normalized]} requires approval, but Hermes' "
+            f"approval gate is unavailable ({exc})."
+        )
+    if result.get("approved"):
+        return True, ""
+    return False, str(
+        result.get("message")
+        or f"BLOCKED: approval required for {FEATURE_LABELS[normalized]}"
+    )
+
+
+def _directive_for_feature(
+    feature: CreditFeature,
+    operation: str,
+    target: str | None,
+    config: dict[str, Any] | None = None,
+) -> dict[str, str] | None:
+    policy = credit_policy(feature, config)
+    if policy == "deny":
+        return {"action": "block", "message": block_message(feature)}
+    if policy == "request":
+        return {
+            "action": "approve",
+            "message": approval_reason(feature, operation, target),
+            "rule_key": f"tinyfish:{feature}:{target_domain(target)}",
+        }
+    return None
+
+
+def pre_tool_call_policy(
+    tool_name: str, args: dict[str, Any] | None = None, **_: Any
+) -> dict[str, str] | None:
+    """Hermes plugin hook for policy-gating TinyFish credit-consuming tools."""
+
+    params = args or {}
+
+    if not tool_name.startswith("browser_"):
+        return None
+    config = load_config()
+    if browser_cloud_provider(config) != "tinyfish":
+        return None
+    target = str(params.get("url") or params.get("target") or "")
+    return _directive_for_feature("browser", tool_name, target, config)

package/tinyfish_hermes/normalize.py

@@ -0,0 +1,110 @@
+"""Normalize TinyFish REST payloads into Hermes web-provider shapes."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def _document_text(value: Any) -> str:
+    # With format=json the Fetch API returns a document tree, not a string.
+    if isinstance(value, str):
+        return value
+    if value is None:
+        return ""
+    try:
+        return json.dumps(value, separators=(",", ":"))
+    except (TypeError, ValueError):
+        return str(value)
+
+
+def normalize_search_response(payload: Any, limit: int = 5) -> dict[str, Any]:
+    """Return Hermes' standard web-search response envelope."""
+
+    results = payload.get("results") if isinstance(payload, dict) else None
+    if not isinstance(results, list):
+        results = []
+
+    count = max(1, int(limit or 5))
+    web_results: list[dict[str, Any]] = []
+    for idx, item in enumerate(results[:count]):
+        if not isinstance(item, dict):
+            continue
+        url = str(item.get("url") or "")
+        web_results.append(
+            {
+                "title": str(item.get("title") or item.get("site_name") or url),
+                "url": url,
+                "description": str(item.get("snippet") or ""),
+                "position": int(item.get("position") or idx + 1),
+            }
+        )
+    return {"success": True, "data": {"web": web_results}}
+
+
+def normalize_fetch_documents(
+    payload: Any, fallback_urls: list[str] | None = None
+) -> list[dict[str, Any]]:
+    """Return Hermes' standard extract document list."""
+
+    urls = list(fallback_urls or [])
+    results = payload.get("results") if isinstance(payload, dict) else None
+    errors = payload.get("errors") if isinstance(payload, dict) else None
+    if not isinstance(results, list):
+        results = []
+    if not isinstance(errors, list):
+        errors = []
+
+    documents: list[dict[str, Any]] = []
+    for idx, item in enumerate(results):
+        if not isinstance(item, dict):
+            continue
+        url = str(
+            item.get("url")
+            or item.get("final_url")
+            or (urls[idx] if idx < len(urls) else "")
+        )
+        raw = _document_text(item.get("text"))
+        documents.append(
+            {
+                "url": url,
+                "title": str(item.get("title") or ""),
+                "content": raw,
+                "raw_content": raw,
+                "metadata": {
+                    "sourceURL": url,
+                    "finalURL": str(item.get("final_url") or url),
+                    "description": str(item.get("description") or ""),
+                    "language": str(item.get("language") or ""),
+                },
+            }
+        )
+
+    for idx, item in enumerate(errors):
+        if not isinstance(item, dict):
+            continue
+        url = str(item.get("url") or (urls[idx] if idx < len(urls) else ""))
+        documents.append(
+            {
+                "url": url,
+                "title": "",
+                "content": "",
+                "raw_content": "",
+                "error": str(item.get("error") or "fetch failed"),
+                "metadata": {"sourceURL": url},
+            }
+        )
+
+    if not documents and urls:
+        return [
+            {
+                "url": url,
+                "title": "",
+                "content": "",
+                "raw_content": "",
+                "error": "TinyFish returned no content",
+                "metadata": {"sourceURL": url},
+            }
+            for url in urls
+        ]
+    return documents

package/tinyfish_hermes/provider.py

@@ -0,0 +1,163 @@
+"""Hermes WebSearchProvider implementation for TinyFish, REST-only."""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+try:
+    from agent.web_search_provider import WebSearchProvider as _HermesWebSearchProvider
+except Exception:  # pragma: no cover - lets the package import outside Hermes
+
+    class _HermesWebSearchProvider:  # type: ignore[no-redef]
+        pass
+
+
+from . import rest_client
+from .config import default_fetch_format, fetch_options, search_options
+from .normalize import normalize_fetch_documents, normalize_search_response
+
+logger = logging.getLogger(__name__)
+
+API_KEY_ENV_VARS = ("TINYFISH_API_KEY", "MCP_TINYFISH_API_KEY")
+API_KEY_URL = "https://agent.tinyfish.ai/api-keys"
+MISSING_KEY_ERROR = (
+    "TinyFish API key not found. Set TINYFISH_API_KEY or MCP_TINYFISH_API_KEY; "
+    f"create a key at {API_KEY_URL}."
+)
+
+
+def _provider_env(name: str) -> str:
+    """Read Hermes config-aware environment values when available."""
+
+    try:
+        from agent.web_search_provider import get_provider_env
+
+        return str(get_provider_env(name) or "").strip()
+    except Exception:
+        return str(os.getenv(name, "") or "").strip()
+
+
+def _api_key() -> str:
+    for name in API_KEY_ENV_VARS:
+        value = _provider_env(name)
+        if value:
+            return value
+    return ""
+
+
+def _is_interrupted() -> bool:
+    try:
+        from tools.interrupt import is_interrupted
+
+        return bool(is_interrupted())
+    except Exception:
+        return False
+
+
+def _safe_rest_failure(operation: str, exc: Exception) -> str:
+    """Keep controlled REST diagnostics while suppressing arbitrary exceptions."""
+
+    if isinstance(exc, rest_client.TinyFishRestError):
+        return f"TinyFish REST {operation} failed: {exc}"
+    return f"TinyFish REST {operation} failed ({type(exc).__name__})."
+
+
+def _error_document(url: str, error: str) -> dict[str, Any]:
+    return {
+        "url": url,
+        "title": "",
+        "content": "",
+        "raw_content": "",
+        "error": error,
+        "metadata": {"sourceURL": url},
+    }
+
+
+class TinyFishWebSearchProvider(_HermesWebSearchProvider):  # type: ignore[misc]
+    """TinyFish Search and Fetch provider using API-key REST calls."""
+
+    @property
+    def name(self) -> str:
+        return "tinyfish"
+
+    @property
+    def display_name(self) -> str:
+        return "TinyFish"
+
+    def is_available(self) -> bool:
+        return bool(_api_key())
+
+    def supports_search(self) -> bool:
+        return True
+
+    def supports_extract(self) -> bool:
+        return True
+
+    def supports_crawl(self) -> bool:
+        return False
+
+    def get_setup_schema(self) -> dict[str, Any]:
+        return {
+            "name": "TinyFish",
+            "badge": "free search/fetch",
+            "tag": "TinyFish Search and Fetch via API key.",
+            "env_vars": [
+                {
+                    "key": "TINYFISH_API_KEY",
+                    "prompt": "TinyFish API key",
+                    "url": API_KEY_URL,
+                }
+            ],
+        }
+
+    def search(self, query: str, limit: int = 5) -> dict[str, Any]:
+        if _is_interrupted():
+            return {"success": False, "error": "Interrupted"}
+        api_key = _api_key()
+        if not api_key:
+            return {"success": False, "error": MISSING_KEY_ERROR}
+        try:
+            raw = rest_client.search(query, api_key=api_key, **search_options())
+            return normalize_search_response(raw, limit=limit)
+        except Exception as exc:  # noqa: BLE001 - controlled failure envelope
+            logger.warning("TinyFish REST search failed (%s)", type(exc).__name__)
+            return {"success": False, "error": _safe_rest_failure("search", exc)}
+
+    def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
+        if _is_interrupted():
+            return [_error_document(url, "Interrupted") for url in urls]
+        api_key = _api_key()
+        if not api_key:
+            return [_error_document(url, MISSING_KEY_ERROR) for url in urls]
+        output_format = str(
+            kwargs.get("format")
+            or kwargs.get("output_format")
+            or default_fetch_format()
+        )
+        options = fetch_options()
+        documents: list[dict[str, Any]] = []
+        # The Fetch API caps a request at 10 URLs; oversized batches 400.
+        for start in range(0, len(urls), rest_client.FETCH_MAX_URLS):
+            if start and _is_interrupted():
+                documents.extend(
+                    _error_document(url, "Interrupted") for url in urls[start:]
+                )
+                break
+            chunk = urls[start : start + rest_client.FETCH_MAX_URLS]
+            try:
+                raw = rest_client.fetch(
+                    chunk,
+                    api_key=api_key,
+                    output_format=output_format,
+                    **options,
+                )
+                documents.extend(normalize_fetch_documents(raw, fallback_urls=chunk))
+            except Exception as exc:  # noqa: BLE001 - controlled failure envelope
+                logger.warning("TinyFish REST fetch failed (%s)", type(exc).__name__)
+                documents.extend(
+                    _error_document(url, _safe_rest_failure("fetch", exc))
+                    for url in chunk
+                )
+        return documents

package/tinyfish_hermes/rest_client.py

@@ -0,0 +1,276 @@
+"""TinyFish Search and Fetch REST client."""
+
+from __future__ import annotations
+
+import math
+import time
+from typing import Any, NoReturn, cast
+
+import httpx
+
+from . import __version__
+
+SEARCH_URL = "https://api.search.tinyfish.ai"
+FETCH_URL = "https://api.fetch.tinyfish.ai"
+FETCH_MAX_URLS = 10
+BROWSER_URL = "https://api.browser.tinyfish.ai"
+WALLET_URL = "https://agent.tinyfish.ai/v1/wallet"
+SEARCH_USAGE_URL = f"{SEARCH_URL}/usage"
+FETCH_USAGE_URL = f"{FETCH_URL}/usage"
+
+_BROWSER_CLOSE_MAX_ATTEMPTS = 3
+_BROWSER_CLOSE_RETRYABLE_STATUSES = frozenset({409, 429, 500, 503, 504})
+_BROWSER_CLOSE_RETRY_AFTER_CAP_SECONDS = 5.0
+_BROWSER_CLOSE_RETRY_BASE_SECONDS = 0.25
+
+
+class TinyFishRestError(RuntimeError):
+    """Raised for TinyFish REST transport or HTTP failures."""
+
+
+class TinyFishWalletNotFound(TinyFishRestError):
+    """Raised when an account uses legacy billing or has no wallet yet."""
+
+
+def _headers(api_key: str) -> dict[str, str]:
+    # Without these, telemetry cannot tell the plugin from raw curl.
+    return {
+        "X-API-Key": api_key,
+        "Accept": "application/json",
+        "X-TF-Client-Name": "hermes",
+        "X-TF-Client-Version": __version__,
+    }
+
+
+def _json_headers(api_key: str) -> dict[str, str]:
+    headers = _headers(api_key)
+    headers["Content-Type"] = "application/json"
+    return headers
+
+
+def _raise_http_error(service: str, exc: httpx.HTTPStatusError) -> NoReturn:
+    status = exc.response.status_code
+    if status == 402:
+        raise TinyFishRestError(
+            f"{service} returned HTTP 402. TinyFish credits or billing may be required."
+        ) from exc
+    raise TinyFishRestError(f"{service} returned HTTP {status}") from exc
+
+
+def search(
+    query: str,
+    *,
+    api_key: str,
+    timeout: float = 30.0,
+    location: str | None = None,
+    language: str | None = None,
+    recency_minutes: int | None = None,
+    after_date: str | None = None,
+    before_date: str | None = None,
+    domain_type: str | None = None,
+    page: int | None = None,
+    purpose: str | None = None,
+) -> dict[str, Any]:
+    """Run a TinyFish Search API query."""
+
+    params: dict[str, Any] = {"query": query}
+    for key, value in {
+        "location": location,
+        "language": language,
+        "recency_minutes": recency_minutes,
+        "after_date": after_date,
+        "before_date": before_date,
+        "domain_type": domain_type,
+        "page": page,
+        "purpose": purpose,
+    }.items():
+        if value is not None and value != "":
+            params[key] = value
+
+    try:
+        response = httpx.get(
+            SEARCH_URL,
+            params=params,
+            headers=_headers(api_key),
+            timeout=timeout,
+        )
+        response.raise_for_status()
+        return cast(dict[str, Any], response.json())
+    except httpx.HTTPStatusError as exc:
+        _raise_http_error("TinyFish Search", exc)
+    except httpx.RequestError as exc:
+        raise TinyFishRestError(f"Could not reach TinyFish Search: {exc}") from exc
+    except ValueError as exc:
+        raise TinyFishRestError("TinyFish Search returned invalid JSON") from exc
+
+
+def fetch(
+    urls: list[str],
+    *,
+    api_key: str,
+    output_format: str = "markdown",
+    links: bool | None = None,
+    image_links: bool | None = None,
+    ttl: int | None = None,
+    per_url_timeout_ms: int | None = None,
+    # Batches can run ~120s server-side; docs recommend a client timeout >= 150s.
+    timeout: float = 150.0,
+) -> dict[str, Any]:
+    """Run the TinyFish Fetch API for one or more URLs."""
+
+    body: dict[str, Any] = {"urls": urls, "format": output_format}
+    for key, value in {
+        "links": links,
+        "image_links": image_links,
+        "ttl": ttl,
+        "per_url_timeout_ms": per_url_timeout_ms,
+    }.items():
+        if value is not None:
+            body[key] = value
+
+    try:
+        response = httpx.post(
+            FETCH_URL,
+            json=body,
+            headers=_json_headers(api_key),
+            timeout=timeout,
+        )
+        response.raise_for_status()
+        return cast(dict[str, Any], response.json())
+    except httpx.HTTPStatusError as exc:
+        _raise_http_error("TinyFish Fetch", exc)
+    except httpx.RequestError as exc:
+        raise TinyFishRestError(f"Could not reach TinyFish Fetch: {exc}") from exc
+    except ValueError as exc:
+        raise TinyFishRestError("TinyFish Fetch returned invalid JSON") from exc
+
+
+def create_browser_session(
+    *,
+    api_key: str,
+    url: str | None = None,
+    timeout_seconds: int | None = None,
+    timeout: float = 90.0,
+) -> dict[str, Any]:
+    """Create a TinyFish Browser session."""
+
+    body: dict[str, Any] = {}
+    if url:
+        body["url"] = url
+    if timeout_seconds is not None:
+        body["timeout_seconds"] = timeout_seconds
+    try:
+        response = httpx.post(
+            BROWSER_URL,
+            json=body,
+            headers=_json_headers(api_key),
+            timeout=timeout,
+        )
+        response.raise_for_status()
+        return cast(dict[str, Any], response.json())
+    except httpx.HTTPStatusError as exc:
+        _raise_http_error("TinyFish Browser", exc)
+    except httpx.RequestError as exc:
+        raise TinyFishRestError(f"Could not reach TinyFish Browser: {exc}") from exc
+    except ValueError as exc:
+        raise TinyFishRestError("TinyFish Browser returned invalid JSON") from exc
+
+
+def _browser_close_retry_delay(response: httpx.Response | None, attempt: int) -> float:
+    if response is not None:
+        retry_after = str(response.headers.get("Retry-After", "") or "").strip()
+        if retry_after:
+            try:
+                seconds = float(retry_after)
+            except ValueError:
+                seconds = -1.0
+            if math.isfinite(seconds) and seconds >= 0:
+                return min(seconds, _BROWSER_CLOSE_RETRY_AFTER_CAP_SECONDS)
+    return _BROWSER_CLOSE_RETRY_BASE_SECONDS * (2.0**attempt)
+
+
+def close_browser_session(
+    session_id: str, *, api_key: str, timeout: float = 15.0
+) -> bool:
+    """Terminate a Browser session with bounded transient-failure retries."""
+
+    # A 404 is not success: it can also mean the session belongs to another key.
+    url = f"{BROWSER_URL}/{session_id}"
+    for attempt in range(_BROWSER_CLOSE_MAX_ATTEMPTS):
+        response: httpx.Response | None = None
+        try:
+            response = httpx.delete(url, headers=_headers(api_key), timeout=timeout)
+        except httpx.RequestError:
+            retry = True
+        else:
+            if 200 <= response.status_code < 300:
+                return True
+            retry = response.status_code in _BROWSER_CLOSE_RETRYABLE_STATUSES
+
+        if not retry or attempt + 1 >= _BROWSER_CLOSE_MAX_ATTEMPTS:
+            return False
+        delay = _browser_close_retry_delay(response, attempt)
+        if delay > 0:
+            time.sleep(delay)
+    return False
+
+
+def _read_usage(
+    url: str,
+    *,
+    service: str,
+    api_key: str,
+    timeout: float,
+) -> dict[str, Any]:
+    try:
+        response = httpx.get(url, headers=_headers(api_key), timeout=timeout)
+        response.raise_for_status()
+        return cast(dict[str, Any], response.json())
+    except httpx.HTTPStatusError as exc:
+        _raise_http_error(service, exc)
+    except httpx.RequestError as exc:
+        raise TinyFishRestError(f"Could not reach {service}: {exc}") from exc
+    except ValueError as exc:
+        raise TinyFishRestError(f"{service} returned invalid JSON") from exc
+
+
+def search_usage(*, api_key: str, timeout: float = 30.0) -> dict[str, Any]:
+    """Return TinyFish Search operation history."""
+
+    return _read_usage(
+        SEARCH_USAGE_URL,
+        service="TinyFish Search usage",
+        api_key=api_key,
+        timeout=timeout,
+    )
+
+
+def fetch_usage(*, api_key: str, timeout: float = 30.0) -> dict[str, Any]:
+    """Return TinyFish Fetch operation history."""
+
+    return _read_usage(
+        FETCH_USAGE_URL,
+        service="TinyFish Fetch usage",
+        api_key=api_key,
+        timeout=timeout,
+    )
+
+
+def wallet(*, api_key: str, timeout: float = 30.0) -> dict[str, Any]:
+    """Return the caller's TinyFish wallet balance and billing rates."""
+
+    try:
+        response = httpx.get(WALLET_URL, headers=_headers(api_key), timeout=timeout)
+        response.raise_for_status()
+        return cast(dict[str, Any], response.json())
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 404:
+            raise TinyFishWalletNotFound(
+                "TinyFish has no wallet for this account "
+                "(legacy billing or no Metronome customer yet)."
+            ) from exc
+        _raise_http_error("TinyFish Wallet", exc)
+    except httpx.RequestError as exc:
+        raise TinyFishRestError(f"Could not reach TinyFish Wallet: {exc}") from exc
+    except ValueError as exc:
+        raise TinyFishRestError("TinyFish Wallet returned invalid JSON") from exc