@drico2008/fincli 0.1.3 → 0.2.2

package/fincli/app/services/web_research.py

@@ -0,0 +1,267 @@
+"""Lightweight web research service for AI assistance."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from html import unescape
+from html.parser import HTMLParser
+import re
+from urllib.parse import parse_qs, quote_plus, unquote, urlparse
+from xml.etree import ElementTree
+
+import httpx
+
+from fincli.app.utils.errors import ProviderError
+
+
+@dataclass(frozen=True, slots=True)
+class WebSearchResult:
+    title: str
+    url: str
+    snippet: str = ""
+    content: str = ""
+
+
+class WebResearchService:
+    """Search and fetch public web pages without browser automation."""
+
+    def __init__(self, client: httpx.AsyncClient | None = None, timeout_seconds: float = 6.0) -> None:
+        self._client = client
+        self.timeout_seconds = timeout_seconds
+
+    async def research(self, query: str, limit: int = 3) -> list[WebSearchResult]:
+        normalized = query.strip()
+        if not normalized:
+            return []
+        search_results = await self.search(normalized, limit=limit)
+        enriched: list[WebSearchResult] = []
+        for result in search_results[:limit]:
+            content = await self.fetch_text(result.url)
+            enriched.append(
+                WebSearchResult(
+                    title=result.title,
+                    url=result.url,
+                    snippet=result.snippet,
+                    content=content,
+                )
+            )
+        return enriched
+
+    async def search(self, query: str, limit: int = 5) -> list[WebSearchResult]:
+        errors: list[str] = []
+        for searcher in (self._search_duckduckgo, self._search_google_news):
+            try:
+                results = await searcher(query, limit)
+            except ProviderError as exc:
+                errors.append(str(exc))
+                continue
+            if results:
+                return results
+        detail = "\n".join(f"- {error}" for error in errors) if errors else "Tidak ada hasil publik."
+        raise ProviderError(
+            "Semua web search provider gagal atau kosong.",
+            f"{detail}\nCoba ulangi, sederhanakan query, atau cek koneksi/DNS.",
+        )
+
+    async def _search_duckduckgo(self, query: str, limit: int) -> list[WebSearchResult]:
+        html = await self._get_text(f"https://duckduckgo.com/html/?q={quote_plus(query)}")
+        parser = _DuckDuckGoParser()
+        parser.feed(html)
+        results: list[WebSearchResult] = []
+        seen: set[str] = set()
+        for item in parser.results:
+            target = _clean_duckduckgo_url(item.url)
+            if not target or target in seen:
+                continue
+            seen.add(target)
+            results.append(WebSearchResult(title=_clean_text(item.title), url=target, snippet=_clean_text(item.snippet)))
+            if len(results) >= limit:
+                break
+        return results
+
+    async def _search_google_news(self, query: str, limit: int) -> list[WebSearchResult]:
+        rss = await self._get_text(f"https://news.google.com/rss/search?q={quote_plus(query)}&hl=id&gl=ID&ceid=ID:id")
+        try:
+            root = ElementTree.fromstring(rss)
+        except ElementTree.ParseError as exc:
+            raise ProviderError("Google News RSS tidak valid.") from exc
+
+        results: list[WebSearchResult] = []
+        seen: set[str] = set()
+        for item in root.findall(".//item"):
+            title = _clean_text(item.findtext("title") or "")
+            url = _clean_text(item.findtext("link") or "")
+            snippet = _clean_text(_html_to_text(item.findtext("description") or ""))
+            if not title or not url or url in seen:
+                continue
+            seen.add(url)
+            results.append(WebSearchResult(title=title, url=url, snippet=snippet))
+            if len(results) >= limit:
+                break
+        return results
+
+    async def fetch_text(self, url: str, max_chars: int = 2400) -> str:
+        if not url.startswith(("http://", "https://")):
+            return ""
+        try:
+            html = await self._get_text(url)
+        except ProviderError:
+            return ""
+        text = _html_to_text(html)
+        return text[:max_chars]
+
+    async def _get_text(self, url: str) -> str:
+        headers = {
+            "User-Agent": "FinCLI/0.1 web research (+https://www.npmjs.com/package/@drico2008/fincli)",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.7",
+        }
+        close_client = self._client is None
+        client = self._client or httpx.AsyncClient(timeout=self.timeout_seconds, follow_redirects=True, headers=headers)
+        try:
+            response = await client.get(url, headers=headers)
+            response.raise_for_status()
+            return response.text
+        except httpx.TimeoutException as exc:
+            raise ProviderError("Web research timeout.", f"URL: {url}") from exc
+        except httpx.HTTPStatusError as exc:
+            raise ProviderError(f"Web research gagal: HTTP {exc.response.status_code}.", f"URL: {url}") from exc
+        except httpx.RequestError as exc:
+            raise ProviderError(f"Web research gagal terhubung: {exc}.", f"URL: {url}") from exc
+        finally:
+            if close_client:
+                await client.aclose()
+
+
+def should_use_web_research(prompt: str) -> bool:
+    """Detect prompts that benefit from current public web context."""
+    normalized = prompt.lower()
+    keywords = (
+        "terkini",
+        "terbaru",
+        "hari ini",
+        "sekarang",
+        "saat ini",
+        "update",
+        "berita",
+        "news",
+        "web",
+        "search",
+        "cari",
+        "penyebab",
+        "mengapa",
+        "kenapa",
+        "rupiah",
+        "inflasi",
+        "suku bunga",
+        "bank indonesia",
+        "fed",
+        "dollar",
+        "dolar",
+        "yield",
+        "minyak",
+        "emas",
+    )
+    return any(keyword in normalized for keyword in keywords)
+
+
+def build_web_research_context(results: list[WebSearchResult]) -> str:
+    if not results:
+        return "Web Research: no public web context returned."
+    sections = ["Web Research Context:"]
+    for index, result in enumerate(results, start=1):
+        sections.extend(
+            [
+                f"{index}. {result.title}",
+                f"URL: {result.url}",
+                f"Snippet: {result.snippet or 'N/A'}",
+                f"Extract: {result.content or 'N/A'}",
+            ]
+        )
+    return "\n".join(sections)
+
+
+class _DuckResult:
+    def __init__(self) -> None:
+        self.title = ""
+        self.url = ""
+        self.snippet = ""
+
+
+class _DuckDuckGoParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.results: list[_DuckResult] = []
+        self._current: _DuckResult | None = None
+        self._capture: str | None = None
+        self._buffer: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        attr = dict(attrs)
+        classes = attr.get("class", "")
+        if tag == "a" and "result__a" in classes:
+            self._current = _DuckResult()
+            self._current.url = attr.get("href", "") or ""
+            self._capture = "title"
+            self._buffer = []
+        elif self._current is not None and tag in {"a", "div"} and "result__snippet" in classes:
+            self._capture = "snippet"
+            self._buffer = []
+
+    def handle_data(self, data: str) -> None:
+        if self._capture:
+            self._buffer.append(data)
+
+    def handle_endtag(self, tag: str) -> None:
+        if self._current is None or self._capture is None:
+            return
+        if self._capture == "title" and tag == "a":
+            self._current.title = _clean_text(" ".join(self._buffer))
+            self._capture = None
+            self._buffer = []
+        elif self._capture == "snippet" and tag in {"a", "div"}:
+            self._current.snippet = _clean_text(" ".join(self._buffer))
+            self.results.append(self._current)
+            self._current = None
+            self._capture = None
+            self._buffer = []
+
+
+class _TextExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.parts: list[str] = []
+        self._skip_depth = 0
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag in {"script", "style", "noscript", "svg"}:
+            self._skip_depth += 1
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in {"script", "style", "noscript", "svg"} and self._skip_depth:
+            self._skip_depth -= 1
+
+    def handle_data(self, data: str) -> None:
+        if not self._skip_depth:
+            cleaned = _clean_text(data)
+            if cleaned:
+                self.parts.append(cleaned)
+
+
+def _html_to_text(html: str) -> str:
+    extractor = _TextExtractor()
+    extractor.feed(html)
+    return _clean_text(" ".join(extractor.parts))
+
+
+def _clean_text(value: str) -> str:
+    text = unescape(value)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+def _clean_duckduckgo_url(url: str) -> str:
+    parsed = urlparse(url)
+    if parsed.netloc.endswith("duckduckgo.com") and parsed.path.startswith("/l/"):
+        target = parse_qs(parsed.query).get("uddg", [""])[0]
+        return unquote(target)
+    return url

package/fincli/app/storage/cache.py

@@ -1,4 +1,4 @@
-"""Lightweight in-memory cache scaffold for provider responses."""
+"""Lightweight in-memory TTL cache for provider responses."""
 
 from __future__ import annotations
 
@@ -16,7 +16,7 @@ class CacheEntry(Generic[T]):
 
 
 class TTLCache(Generic[T]):
-    """Simple TTL cache for Phase 1. Provider modules can replace this later."""
+    """Simple runtime TTL cache used by router and provider workflows."""
 
     def __init__(self, default_ttl: int = 300) -> None:
         self.default_ttl = default_ttl

package/fincli/app/storage/config.py

@@ -1,112 +1,146 @@
-"""Configuration loading and persistence.
-
-Secrets are read from environment variables. Non-secret preferences are stored
-in a local JSON config file under ~/.fincli by default.
-"""
-
-from __future__ import annotations
-
-from dataclasses import asdict, dataclass, field
-import json
-import os
-from pathlib import Path
-from typing import Any
-
+"""Configuration loading and persistence.
+
+Secrets are read from environment variables. Non-secret preferences are stored
+in a local JSON config file under ~/.fincli by default.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+import json
+import os
+from pathlib import Path
+from typing import Any
+
 try:
-    from dotenv import load_dotenv
+    from dotenv import dotenv_values, load_dotenv
 except ImportError:  # pragma: no cover - dependency exists in normal install
     load_dotenv = None  # type: ignore[assignment]
-
-from fincli.app.utils.errors import ConfigError
-from fincli.app.utils.formatting import mask_secret
-from fincli.app.storage.config_paths import APP_DIR, CONFIG_FILE
-from fincli.app.storage.secrets import load_local_secrets
-
-
-@dataclass(slots=True)
-class FinCLISettings:
-    ai_provider: str = "openrouter"
-    ai_model: str = "openai/gpt-4o-mini"
+    dotenv_values = None  # type: ignore[assignment]
+
+from fincli.app.utils.errors import ConfigError
+from fincli.app.utils.formatting import mask_secret
+from fincli.app.storage.config_paths import APP_DIR, CONFIG_FILE
+from fincli.app.storage.secrets import load_local_secrets
+
+
+@dataclass(slots=True)
+class FinCLISettings:
+    ai_provider: str = "openrouter"
+    ai_model: str = "openai/gpt-4o-mini"
     market_provider: str = "yfinance"
     news_provider: str = "yfinance"
     market_provider_priority: list[str] = field(default_factory=lambda: ["yfinance"])
+    news_provider_priority: list[str] = field(
+        default_factory=lambda: ["yfinance", "google_news_rss", "yahoo_finance_rss", "marketaux", "newsapi", "gnews"]
+    )
     timezone: str = "Asia/Jakarta"
-    default_currency: str = "USD"
-    cache_ttl_seconds: int = 300
-    theme: str = "fincli-dark"
-
-    def safe_dict(self) -> dict[str, Any]:
-        """Return display-safe config, including masked secret status."""
-        data = asdict(self)
-        data["api_keys"] = {
-            "openrouter": mask_secret(os.getenv("OPENROUTER_API_KEY")),
-            "gemini": mask_secret(os.getenv("GEMINI_API_KEY")),
-            "anthropic": mask_secret(os.getenv("ANTHROPIC_API_KEY")),
-            "openai": mask_secret(os.getenv("OPENAI_API_KEY")),
-            "together": mask_secret(os.getenv("TOGETHER_API_KEY")),
-            "huggingface": mask_secret(os.getenv("HUGGINGFACE_API_KEY")),
-            "groq": mask_secret(os.getenv("GROQ_API_KEY")),
-            "market_data": mask_secret(os.getenv("MARKET_DATA_API_KEY")),
+    default_currency: str = "USD"
+    cache_ttl_seconds: int = 300
+    theme: str = "fincli-dark"
+
+    def safe_dict(self) -> dict[str, Any]:
+        """Return display-safe config, including masked secret status."""
+        data = asdict(self)
+        data["api_keys"] = {
+            "openrouter": mask_secret(os.getenv("OPENROUTER_API_KEY")),
+            "gemini": mask_secret(os.getenv("GEMINI_API_KEY")),
+            "anthropic": mask_secret(os.getenv("ANTHROPIC_API_KEY")),
+            "openai": mask_secret(os.getenv("OPENAI_API_KEY")),
+            "together": mask_secret(os.getenv("TOGETHER_API_KEY")),
+            "huggingface": mask_secret(os.getenv("HUGGINGFACE_API_KEY")),
+            "groq": mask_secret(os.getenv("GROQ_API_KEY")),
+            "market_data": mask_secret(os.getenv("MARKET_DATA_API_KEY")),
             "news_data": mask_secret(os.getenv("NEWS_DATA_API_KEY")),
             "finnhub": mask_secret(os.getenv("FINNHUB_API_KEY")),
             "twelvedata": mask_secret(os.getenv("TWELVE_DATA_API_KEY")),
+            "alphavantage": mask_secret(os.getenv("ALPHA_VANTAGE_API_KEY")),
+            "marketaux": mask_secret(os.getenv("MARKETAUX_API_KEY")),
+            "newsapi": mask_secret(os.getenv("NEWSAPI_API_KEY")),
+            "gnews": mask_secret(os.getenv("GNEWS_API_KEY")),
+            "stocknewsapi": mask_secret(os.getenv("STOCKNEWSAPI_API_KEY")),
+            "apitube": mask_secret(os.getenv("APITUBE_API_KEY")),
+            "benzinga": mask_secret(os.getenv("BENZINGA_API_KEY")),
+            "polygon": mask_secret(os.getenv("POLYGON_API_KEY")),
+            "tiingo": mask_secret(os.getenv("TIINGO_API_KEY")),
+            "fmp": mask_secret(os.getenv("FMP_API_KEY")),
+            "eodhd": mask_secret(os.getenv("EODHD_API_KEY")),
+            "custom_news": mask_secret(os.getenv("CUSTOM_NEWS_API_KEY") or os.getenv("NEWS_DATA_API_KEY")),
         }
         return data
-
-
-class ConfigManager:
-    """Load, update, and persist non-secret FinCLI settings."""
-
-    def __init__(self, config_file: Path = CONFIG_FILE) -> None:
-        self.config_file = config_file
-        self.settings = self.load()
-
+
+
+class ConfigManager:
+    """Load, update, and persist non-secret FinCLI settings."""
+
+    def __init__(self, config_file: Path = CONFIG_FILE) -> None:
+        self.config_file = config_file
+        self.settings = self.load()
+
     def load(self) -> FinCLISettings:
+        dotenv_loaded_keys: set[str] = set()
         if load_dotenv is not None:
+            before_env = dict(os.environ)
             load_dotenv()
-        load_local_secrets()
-
-        if not self.config_file.exists():
-            return FinCLISettings()
-
-        try:
-            raw = json.loads(self.config_file.read_text(encoding="utf-8"))
-            allowed = FinCLISettings.__dataclass_fields__.keys()
-            filtered = {key: value for key, value in raw.items() if key in allowed}
-            return FinCLISettings(**filtered)
-        except Exception as exc:  # noqa: BLE001
-            raise ConfigError(
-                "Config lokal gagal dibaca.",
-                "Periksa ~/.fincli/config.json atau hapus file tersebut untuk memakai default.",
-            ) from exc
-
-    def save(self) -> None:
-        try:
-            self.config_file.parent.mkdir(parents=True, exist_ok=True)
-            self.config_file.write_text(
-                json.dumps(asdict(self.settings), indent=2),
-                encoding="utf-8",
-            )
-        except Exception as exc:  # noqa: BLE001
-            raise ConfigError("Config lokal gagal disimpan.") from exc
-
-    def set_ai_model(self, provider: str, model: str) -> None:
-        self.settings.ai_provider = provider.strip().lower()
-        self.settings.ai_model = model.strip()
-        self.save()
-
-    def set_market_provider(self, provider: str) -> None:
-        self.settings.market_provider = provider.strip().lower()
-        self.save()
-
+            if dotenv_values is not None:
+                dotenv_loaded_keys = {
+                    key
+                    for key, value in dotenv_values().items()
+                    if key and value is not None and (key not in before_env or before_env.get(key, "") == "")
+                }
+        # API keys saved from FinCLI commands should override stale project .env values,
+        # while explicit OS/process environment variables remain respected.
+        load_local_secrets(override_keys=dotenv_loaded_keys)
+
+        if not self.config_file.exists():
+            return FinCLISettings()
+
+        try:
+            raw = json.loads(self.config_file.read_text(encoding="utf-8"))
+            allowed = FinCLISettings.__dataclass_fields__.keys()
+            filtered = {key: value for key, value in raw.items() if key in allowed}
+            return FinCLISettings(**filtered)
+        except Exception as exc:  # noqa: BLE001
+            raise ConfigError(
+                "Config lokal gagal dibaca.",
+                "Periksa ~/.fincli/config.json atau hapus file tersebut untuk memakai default.",
+            ) from exc
+
+    def save(self) -> None:
+        try:
+            self.config_file.parent.mkdir(parents=True, exist_ok=True)
+            self.config_file.write_text(
+                json.dumps(asdict(self.settings), indent=2),
+                encoding="utf-8",
+            )
+        except Exception as exc:  # noqa: BLE001
+            raise ConfigError("Config lokal gagal disimpan.") from exc
+
+    def set_ai_model(self, provider: str, model: str) -> None:
+        self.settings.ai_provider = provider.strip().lower()
+        self.settings.ai_model = model.strip()
+        self.save()
+
+    def set_market_provider(self, provider: str) -> None:
+        self.settings.market_provider = provider.strip().lower()
+        self.save()
+
     def set_news_provider(self, provider: str) -> None:
         self.settings.news_provider = provider.strip().lower()
         self.save()
 
-    def set_market_provider_priority(self, providers: list[str]) -> None:
+    def set_news_provider_priority(self, providers: list[str]) -> None:
         normalized = [provider.strip().lower() for provider in providers if provider.strip()]
         if not normalized:
-            normalized = ["yfinance"]
+            normalized = ["yfinance", "google_news_rss", "yahoo_finance_rss"]
+        self.settings.news_provider_priority = normalized
+        self.settings.news_provider = normalized[0]
+        self.save()
+
+    def set_market_provider_priority(self, providers: list[str]) -> None:
+        normalized = [provider.strip().lower() for provider in providers if provider.strip()]
+        if not normalized:
+            normalized = ["yfinance"]
         self.settings.market_provider_priority = normalized
         self.settings.market_provider = normalized[0]
         self.settings.news_provider = normalized[0]