@shirlytaylor73/smart-search 0.2.0-beta.1

package/src/smart_search/logger.py

@@ -0,0 +1,43 @@
+import logging
+import os
+from datetime import datetime
+from .config import config
+
+logger = logging.getLogger("smart_search")
+logger.setLevel(getattr(logging, config.log_level))
+logger.addHandler(logging.NullHandler())
+
+
+def _file_logging_enabled() -> bool:
+    return config.debug_enabled or config.log_to_file_enabled
+
+
+def _configure_file_logging() -> None:
+    if not _file_logging_enabled():
+        return
+    if any(isinstance(handler, logging.FileHandler) for handler in logger.handlers):
+        return
+
+    log_dir = config.log_dir
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / f"smart_search_{datetime.now().strftime('%Y%m%d')}.log"
+
+    file_handler = logging.FileHandler(log_file, encoding="utf-8")
+    file_handler.setLevel(getattr(logging, config.log_level))
+
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+
+_configure_file_logging()
+
+async def log_info(ctx, message: str, is_debug: bool = False):
+    if is_debug:
+        logger.info(message)
+
+    if ctx:
+        await ctx.info(message)

package/src/smart_search/providers/__init__.py

@@ -0,0 +1,20 @@
+from .base import BaseSearchProvider, SearchResult
+from .context7 import Context7Provider
+from .openai_compatible import OpenAICompatibleSearchProvider
+from .xai_responses import XAIResponsesSearchProvider
+from .exa import ExaSearchProvider
+from .jina import JinaReaderProvider
+from .zhipu import ZhipuWebSearchProvider
+from .zhipu_mcp import ZhipuMCPProvider
+
+__all__ = [
+    "BaseSearchProvider",
+    "SearchResult",
+    "Context7Provider",
+    "OpenAICompatibleSearchProvider",
+    "XAIResponsesSearchProvider",
+    "ExaSearchProvider",
+    "JinaReaderProvider",
+    "ZhipuWebSearchProvider",
+    "ZhipuMCPProvider",
+]

package/src/smart_search/providers/base.py

@@ -0,0 +1,41 @@
+from abc import ABC, abstractmethod
+from typing import Dict, List
+
+
+class SearchResult:
+    def __init__(
+        self,
+        title: str,
+        url: str,
+        snippet: str,
+        source: str = "",
+        published_date: str = "",
+    ):
+        self.title = title
+        self.url = url
+        self.snippet = snippet
+        self.source = source
+        self.published_date = published_date
+
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "title": self.title,
+            "url": self.url,
+            "snippet": self.snippet,
+            "source": self.source,
+            "published_date": self.published_date,
+        }
+
+
+class BaseSearchProvider(ABC):
+    def __init__(self, api_url: str, api_key: str):
+        self.api_url = api_url
+        self.api_key = api_key
+
+    @abstractmethod
+    async def search(self, query: str, max_results: int = 5) -> List[SearchResult]:
+        pass
+
+    @abstractmethod
+    def get_provider_name(self) -> str:
+        pass

package/src/smart_search/providers/context7.py

@@ -0,0 +1,141 @@
+import json
+import time
+from typing import Any
+from urllib.parse import quote
+
+import httpx
+from tenacity import AsyncRetrying, retry_if_exception, stop_after_attempt, wait_random_exponential
+
+from .base import BaseSearchProvider
+from ..config import config
+from ..logger import log_info
+
+
+RETRYABLE_STATUS_CODES = {408, 429, 500, 502, 503, 504}
+
+
+def _is_retryable_exception(exc) -> bool:
+    if isinstance(exc, (httpx.TimeoutException, httpx.NetworkError, httpx.ConnectError)):
+        return True
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code in RETRYABLE_STATUS_CODES
+    return False
+
+
+def _normalize_library(item: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "id": item.get("id") or "",
+        "title": item.get("title") or "",
+        "description": item.get("description") or "",
+        "trust_score": item.get("trustScore"),
+        "benchmark_score": item.get("benchmarkScore"),
+        "total_snippets": item.get("totalSnippets"),
+        "stars": item.get("stars"),
+        "provider": "context7",
+    }
+
+
+class Context7Provider(BaseSearchProvider):
+    def __init__(self, api_url: str, api_key: str, timeout: float = 30.0):
+        super().__init__(api_url.rstrip("/"), api_key)
+        self.timeout = timeout
+
+    def get_provider_name(self) -> str:
+        return "Context7"
+
+    async def search(self, query: str, max_results: int = 5) -> str:
+        return await self.library(query)
+
+    def _headers(self) -> dict[str, str]:
+        headers = {
+            "Accept": "application/json, text/plain",
+            "X-Context7-Source": "smart-search",
+        }
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    async def library(self, name: str, query: str = "", ctx=None) -> str:
+        request_query = f"{name} {query}".strip()
+        endpoint = f"{self.api_url}/api/v2/search?query={quote(request_query)}"
+        await log_info(ctx, f"Context7 library: {request_query}", config.debug_enabled)
+        start_time = time.time()
+        try:
+            data = await self._get_with_retry(endpoint)
+            elapsed_ms = round((time.time() - start_time) * 1000, 2)
+            raw_results = data if isinstance(data, list) else data.get("results", [])
+            results = [_normalize_library(item) for item in raw_results or []]
+            output = {
+                "ok": True,
+                "query": request_query,
+                "provider": "context7",
+                "results": results,
+                "total": len(results),
+                "elapsed_ms": elapsed_ms,
+            }
+        except Exception as e:
+            elapsed_ms = round((time.time() - start_time) * 1000, 2)
+            output = {
+                "ok": False,
+                "query": request_query,
+                "provider": "context7",
+                "error": str(e),
+                "elapsed_ms": elapsed_ms,
+            }
+        return json.dumps(output, ensure_ascii=False, indent=2)
+
+    async def docs(self, library_id: str, query: str, ctx=None) -> str:
+        endpoint = f"{self.api_url}/api/v2/context?libraryId={quote(library_id, safe='')}&query={quote(query)}"
+        await log_info(ctx, f"Context7 docs: {library_id} {query}", config.debug_enabled)
+        start_time = time.time()
+        try:
+            data = await self._get_with_retry(endpoint)
+            elapsed_ms = round((time.time() - start_time) * 1000, 2)
+            snippets = data.get("codeSnippets", []) if isinstance(data, dict) else []
+            info = data.get("infoSnippets", []) if isinstance(data, dict) else []
+            content = json.dumps(data, ensure_ascii=False) if isinstance(data, dict) else str(data)
+            output = {
+                "ok": True,
+                "library_id": library_id,
+                "query": query,
+                "provider": "context7",
+                "code_snippets": snippets,
+                "info_snippets": info,
+                "results": snippets + info,
+                "total": len(snippets) + len(info),
+                "content": content,
+                "elapsed_ms": elapsed_ms,
+            }
+        except Exception as e:
+            elapsed_ms = round((time.time() - start_time) * 1000, 2)
+            output = {
+                "ok": False,
+                "library_id": library_id,
+                "query": query,
+                "provider": "context7",
+                "error": str(e),
+                "elapsed_ms": elapsed_ms,
+            }
+        return json.dumps(output, ensure_ascii=False, indent=2)
+
+    async def _get_with_retry(self, endpoint: str) -> Any:
+        timeout = httpx.Timeout(connect=6.0, read=self.timeout, write=10.0, pool=None)
+        async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+            async for attempt in AsyncRetrying(
+                stop=stop_after_attempt(config.retry_max_attempts + 1),
+                wait=wait_random_exponential(multiplier=config.retry_multiplier, max=config.retry_max_wait),
+                retry=retry_if_exception(_is_retryable_exception),
+                reraise=True,
+            ):
+                with attempt:
+                    response = await client.get(endpoint, headers=self._headers())
+                    response.raise_for_status()
+                    content_type = response.headers.get("content-type", "")
+                    if "application/json" in content_type:
+                        return response.json()
+                    text = response.text
+                    try:
+                        return json.loads(text)
+                    except json.JSONDecodeError:
+                        return {"content": text, "results": []}
+        return {}

package/src/smart_search/providers/exa.py

@@ -0,0 +1,206 @@
+import json
+import time
+from typing import Any
+
+import httpx
+from tenacity import AsyncRetrying, retry_if_exception, stop_after_attempt, wait_random_exponential
+
+from .base import BaseSearchProvider
+from ..config import config
+from ..logger import log_info
+
+
+RETRYABLE_STATUS_CODES = {408, 429, 500, 502, 503, 504}
+
+
+def _is_retryable_exception(exc) -> bool:
+    if isinstance(exc, (httpx.TimeoutException, httpx.NetworkError, httpx.ConnectError)):
+        return True
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code in RETRYABLE_STATUS_CODES
+    return False
+
+
+def _normalize_result(item: dict[str, Any], *, include_text: bool, include_highlights: bool) -> dict[str, Any]:
+    out = {
+        "id": item.get("id"),
+        "title": item.get("title") or "",
+        "url": item.get("url") or item.get("id") or "",
+        "publishedDate": item.get("publishedDate"),
+        "author": item.get("author") or "",
+        "score": item.get("score"),
+    }
+    if include_text and "text" in item:
+        out["text"] = item.get("text") or ""
+    if include_highlights and "highlights" in item:
+        out["highlights"] = item.get("highlights") or []
+    if "image" in item:
+        out["image"] = item.get("image")
+    if "favicon" in item:
+        out["favicon"] = item.get("favicon")
+    return out
+
+
+def _error_payload(exc: Exception) -> dict[str, Any]:
+    if isinstance(exc, httpx.HTTPStatusError):
+        status_code = exc.response.status_code
+        body = exc.response.text.strip()
+        detail = f" - {body[:500]}" if body else ""
+        if status_code == 429:
+            error_type = "rate_limited"
+        elif status_code in {400, 422}:
+            error_type = "parameter_error"
+        elif status_code in {401, 403}:
+            error_type = "auth_error"
+        else:
+            error_type = "network_error"
+        return {"error_type": error_type, "error": f"HTTP {status_code}: {exc.response.reason_phrase}{detail}"}
+    if isinstance(exc, httpx.TimeoutException):
+        return {"error_type": "timeout", "error": "request timed out"}
+    if isinstance(exc, httpx.RequestError):
+        return {"error_type": "network_error", "error": str(exc)}
+    return {"error_type": "runtime_error", "error": str(exc)}
+
+
+class ExaSearchProvider(BaseSearchProvider):
+    def __init__(self, api_url: str, api_key: str, timeout: float = 30.0):
+        super().__init__(api_url, api_key)
+        self.timeout = timeout
+
+    def get_provider_name(self) -> str:
+        return "Exa"
+
+    async def search(
+        self,
+        query: str,
+        num_results: int = 5,
+        search_type: str = "neural",
+        include_text: bool = False,
+        include_highlights: bool = False,
+        start_published_date: str | None = None,
+        include_domains: list[str] | None = None,
+        exclude_domains: list[str] | None = None,
+        category: str | None = None,
+        ctx=None,
+    ) -> str:
+        endpoint = f"{self.api_url.rstrip('/')}/search"
+        headers = {
+            "accept": "application/json",
+            "content-type": "application/json",
+            "x-api-key": self.api_key,
+        }
+        payload: dict[str, Any] = {
+            "query": query,
+            "numResults": num_results,
+            "type": search_type,
+            "useAutoprompt": True,
+        }
+        if include_text or include_highlights:
+            payload["contents"] = {
+                "text": include_text,
+                "highlights": include_highlights,
+            }
+        if start_published_date:
+            payload["startPublishedDate"] = start_published_date
+        if include_domains:
+            payload["includeDomains"] = include_domains
+        if exclude_domains:
+            payload["excludeDomains"] = exclude_domains
+        if category:
+            payload["category"] = category
+
+        await log_info(ctx, f"Exa search: {query}", config.debug_enabled)
+
+        start_time = time.time()
+        try:
+            data = await self._request_with_retry(endpoint, headers, payload, ctx)
+            elapsed_ms = round((time.time() - start_time) * 1000, 2)
+
+            results = [
+                _normalize_result(item, include_text=include_text, include_highlights=include_highlights)
+                for item in data.get("results", [])
+            ]
+
+            output = {
+                "ok": True,
+                "query": query,
+                "search_type": search_type,
+                "results": results,
+                "total": len(results),
+                "elapsed_ms": elapsed_ms,
+            }
+        except Exception as e:
+            elapsed_ms = round((time.time() - start_time) * 1000, 2)
+            error = _error_payload(e)
+            output = {
+                "ok": False,
+                "query": query,
+                "error_type": error["error_type"],
+                "error": error["error"],
+                "elapsed_ms": elapsed_ms,
+            }
+
+        await log_info(ctx, "Exa search finished!", config.debug_enabled)
+        return json.dumps(output, ensure_ascii=False, indent=2)
+
+    async def find_similar(self, url: str, num_results: int = 5, ctx=None) -> str:
+        endpoint = f"{self.api_url.rstrip('/')}/findSimilar"
+        headers = {
+            "accept": "application/json",
+            "content-type": "application/json",
+            "x-api-key": self.api_key,
+        }
+        payload = {
+            "url": url,
+            "numResults": num_results,
+        }
+
+        await log_info(ctx, f"Exa find_similar: {url}", config.debug_enabled)
+
+        start_time = time.time()
+        try:
+            data = await self._request_with_retry(endpoint, headers, payload, ctx)
+            elapsed_ms = round((time.time() - start_time) * 1000, 2)
+
+            results = [
+                _normalize_result(item, include_text=False, include_highlights=False)
+                for item in data.get("results", [])
+            ]
+
+            output = {
+                "ok": True,
+                "url": url,
+                "results": results,
+                "total": len(results),
+                "elapsed_ms": elapsed_ms,
+            }
+        except Exception as e:
+            elapsed_ms = round((time.time() - start_time) * 1000, 2)
+            error = _error_payload(e)
+            output = {
+                "ok": False,
+                "url": url,
+                "error_type": error["error_type"],
+                "error": error["error"],
+                "elapsed_ms": elapsed_ms,
+            }
+
+        await log_info(ctx, "Exa find_similar finished!", config.debug_enabled)
+        return json.dumps(output, ensure_ascii=False, indent=2)
+
+    async def _request_with_retry(
+        self, endpoint: str, headers: dict, payload: dict, ctx=None
+    ) -> dict[str, Any]:
+        timeout = httpx.Timeout(connect=6.0, read=self.timeout, write=10.0, pool=None)
+
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            async for attempt in AsyncRetrying(
+                stop=stop_after_attempt(config.retry_max_attempts + 1),
+                wait=wait_random_exponential(multiplier=config.retry_multiplier, max=config.retry_max_wait),
+                retry=retry_if_exception(_is_retryable_exception),
+                reraise=True,
+            ):
+                with attempt:
+                    response = await client.post(endpoint, headers=headers, json=payload)
+                    response.raise_for_status()
+                    return response.json()

package/src/smart_search/providers/jina.py

@@ -0,0 +1,136 @@
+import json
+import time
+from typing import Any
+
+import httpx
+
+
+CHALLENGE_MARKERS = (
+    "title: just a moment",
+    "checking if the site connection is secure",
+    "attention required! | cloudflare",
+    "enable javascript and cookies to continue",
+)
+
+
+def _elapsed_ms(start: float) -> float:
+    return round((time.time() - start) * 1000, 2)
+
+
+def _error_payload(exc: Exception) -> dict[str, str]:
+    if isinstance(exc, httpx.HTTPStatusError):
+        status_code = exc.response.status_code
+        if status_code in {401, 403}:
+            error_type = "auth_error"
+        elif status_code == 422:
+            error_type = "parameter_error"
+        elif status_code == 429:
+            error_type = "rate_limited"
+        else:
+            error_type = "network_error"
+        body = (exc.response.text or exc.response.reason_phrase or "")[:300]
+        return {"error_type": error_type, "error": f"HTTP {status_code}: {body}"}
+    if isinstance(exc, httpx.TimeoutException):
+        return {"error_type": "timeout", "error": "request timed out"}
+    if isinstance(exc, httpx.RequestError):
+        return {"error_type": "network_error", "error": str(exc)}
+    return {"error_type": "runtime_error", "error": str(exc)}
+
+
+def _mask_secret(text: str, secret: str) -> str:
+    return text.replace(secret, "***") if secret else text
+
+
+def _quality_error(content: str) -> str:
+    lower = content.strip().lower()
+    if not lower:
+        return "empty response"
+    for marker in CHALLENGE_MARKERS:
+        if marker in lower:
+            return f"low-quality challenge page detected: {marker}"
+    return ""
+
+
+class JinaReaderProvider:
+    def __init__(
+        self,
+        reader_api_url: str,
+        api_key: str | None = None,
+        respond_with: str = "",
+        timeout: float = 30.0,
+    ):
+        self.reader_api_url = reader_api_url.rstrip("/")
+        self.api_key = api_key or ""
+        self.respond_with = respond_with.strip()
+        self.timeout = timeout
+
+    async def fetch(self, url: str) -> str:
+        start = time.time()
+        if self.respond_with and not self.api_key:
+            return json.dumps(
+                {
+                    "ok": False,
+                    "provider": "jina",
+                    "url": url,
+                    "error_type": "config_error",
+                    "error": "JINA_RESPOND_WITH requires JINA_API_KEY.",
+                    "elapsed_ms": _elapsed_ms(start),
+                },
+                ensure_ascii=False,
+                indent=2,
+            )
+
+        headers = {"X-Return-Format": "markdown", "Accept": "text/plain, text/markdown, */*"}
+        if self.respond_with:
+            headers["X-Respond-With"] = self.respond_with
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        endpoint = f"{self.reader_api_url}/{url}"
+        try:
+            timeout = httpx.Timeout(connect=6.0, read=self.timeout, write=10.0, pool=None)
+            async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+                response = await client.get(endpoint, headers=headers)
+                response.raise_for_status()
+            content = response.text.strip()
+            quality_error = _quality_error(content)
+            if quality_error:
+                return json.dumps(
+                    {
+                        "ok": False,
+                        "provider": "jina",
+                        "url": url,
+                        "error_type": "quality_error",
+                        "error": quality_error,
+                        "content": content,
+                        "elapsed_ms": _elapsed_ms(start),
+                    },
+                    ensure_ascii=False,
+                    indent=2,
+                )
+            return json.dumps(
+                {
+                    "ok": True,
+                    "provider": "jina",
+                    "url": url,
+                    "content": content,
+                    "elapsed_ms": _elapsed_ms(start),
+                },
+                ensure_ascii=False,
+                indent=2,
+            )
+        except Exception as e:
+            error = _error_payload(e)
+            message = _mask_secret(error["error"], self.api_key)
+            return json.dumps(
+                {
+                    "ok": False,
+                    "provider": "jina",
+                    "url": url,
+                    "error_type": error["error_type"],
+                    "error": message,
+                    "elapsed_ms": _elapsed_ms(start),
+                },
+                ensure_ascii=False,
+                indent=2,
+            )