opencandle 0.2.0 → 0.4.0

package/src/providers/web-search.ts

@@ -0,0 +1,293 @@
+import { search, searchNews, SafeSearchType, SearchTimeType } from "duck-duck-scrape";
+import type { SearchResult, SearchResults } from "duck-duck-scrape";
+import type { NewsResult } from "duck-duck-scrape";
+import { httpGet, HttpError } from "../infra/http-client.js";
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import { getConfig } from "../config.js";
+import { withFallback } from "./with-fallback.js";
+import { exaSearch } from "./exa-search.js";
+import { ProviderCredentialError } from "./provider-credential-error.js";
+import type { ProviderResult } from "../runtime/evidence.js";
+import type { WebSearchResult, WebSearchEnvelope } from "../types/sentiment.js";
+
+export interface WebSearchOpts {
+  category: "news" | "general";
+  freshness: "hours" | "day" | "week" | "month";
+  limit: number;
+  /** Override provider: skip cascade, use this provider only. */
+  provider?: "exa" | "brave" | "ddg";
+}
+
+const BARE_TICKER = /^[A-Z]{1,5}$/;
+const CASHTAG = /^\$[A-Z]{1,5}$/;
+
+/**
+ * Normalize queries for financial context.
+ * Only applied when category is "news" — general queries pass through unchanged.
+ */
+export function normalizeFinancialQuery(query: string, category: "news" | "general"): string {
+  if (category !== "news") return query;
+  if (CASHTAG.test(query)) return `${query.slice(1)} stock news`;
+  if (BARE_TICKER.test(query)) return `${query} stock news`;
+  return query;
+}
+
+function stripHtmlTags(text: string): string {
+  return text.replace(/<[^>]*>/g, "");
+}
+
+function mapFreshness(freshness: WebSearchOpts["freshness"]): SearchTimeType {
+  switch (freshness) {
+    case "hours":
+      return SearchTimeType.DAY; // closest available
+    case "day":
+      return SearchTimeType.DAY;
+    case "week":
+      return SearchTimeType.WEEK;
+    case "month":
+      return SearchTimeType.MONTH;
+  }
+}
+
+function extractDomain(url: string): string {
+  try {
+    const hostname = new URL(url).hostname;
+    return hostname.replace(/^www\./, "");
+  } catch {
+    return url;
+  }
+}
+
+function mapGeneralResult(r: SearchResult): WebSearchResult {
+  return {
+    title: r.title,
+    url: r.url,
+    snippet: stripHtmlTags(r.description),
+    source: r.hostname || extractDomain(r.url),
+    published: null,
+    category: "general",
+  };
+}
+
+function mapNewsResult(r: NewsResult): WebSearchResult {
+  return {
+    title: r.title,
+    url: r.url,
+    snippet: r.excerpt,
+    source: r.syndicate || extractDomain(r.url),
+    published: r.date ? new Date(r.date * 1000).toISOString() : null,
+    category: "news",
+  };
+}
+
+function ddgCacheKey(query: string, opts: WebSearchOpts): string {
+  return `web:ddg:${query}:${opts.category}:${opts.freshness}:${opts.limit}`;
+}
+
+export async function ddgSearch(
+  query: string,
+  opts: WebSearchOpts,
+): Promise<WebSearchEnvelope> {
+  const key = ddgCacheKey(query, opts);
+  const cached = cache.get<WebSearchEnvelope>(key);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("ddg");
+
+    let results: WebSearchResult[];
+
+    if (opts.category === "news") {
+      const response = await searchNews(
+        query,
+        { time: mapFreshness(opts.freshness), safeSearch: SafeSearchType.STRICT },
+        undefined,
+      );
+      results = (response.results || []).slice(0, opts.limit).map(mapNewsResult);
+    } else {
+      const response = await search(
+        query,
+        { time: mapFreshness(opts.freshness), safeSearch: SafeSearchType.STRICT },
+        undefined,
+      );
+      results = (response.results || []).slice(0, opts.limit).map(mapGeneralResult);
+    }
+
+    const envelope: WebSearchEnvelope = {
+      query,
+      results,
+      resultCount: results.length,
+      fetchedAt: new Date().toISOString(),
+      provider: "ddg",
+    };
+
+    cache.set(key, envelope, TTL.WEB_SEARCH);
+    return envelope;
+  } catch (error) {
+    const stale = cache.getStale<WebSearchEnvelope>(key, STALE_LIMIT.WEB_SEARCH);
+    if (stale) return stale.value;
+    throw error;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Brave Search
+// ---------------------------------------------------------------------------
+
+const BRAVE_BASE = "https://api.search.brave.com/res/v1";
+
+function mapBraveFreshness(freshness: WebSearchOpts["freshness"]): string {
+  switch (freshness) {
+    case "hours": return "ph";
+    case "day": return "pd";
+    case "week": return "pw";
+    case "month": return "pm";
+  }
+}
+
+interface BraveNewsResult {
+  title: string;
+  url: string;
+  description: string;
+  age?: string;
+  source?: string;
+  meta_url?: { hostname: string };
+}
+
+interface BraveWebResult {
+  title: string;
+  url: string;
+  description: string;
+  age?: string | null;
+  meta_url?: { hostname: string };
+}
+
+function mapBraveNewsResult(r: BraveNewsResult): WebSearchResult {
+  return {
+    title: r.title,
+    url: r.url,
+    snippet: r.description,
+    source: r.source || r.meta_url?.hostname || extractDomain(r.url),
+    published: null, // Brave news returns relative "age" not absolute timestamps
+    category: "news",
+  };
+}
+
+function mapBraveWebResult(r: BraveWebResult): WebSearchResult {
+  return {
+    title: r.title,
+    url: r.url,
+    snippet: r.description,
+    source: r.meta_url?.hostname || extractDomain(r.url),
+    published: null,
+    category: "general",
+  };
+}
+
+function braveCacheKey(query: string, opts: WebSearchOpts): string {
+  return `web:brave:${query}:${opts.category}:${opts.freshness}:${opts.limit}`;
+}
+
+export async function braveSearch(
+  query: string,
+  opts: WebSearchOpts,
+  apiKey: string,
+): Promise<WebSearchEnvelope> {
+  const key = braveCacheKey(query, opts);
+  const cached = cache.get<WebSearchEnvelope>(key);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("brave_search");
+
+    const endpoint = opts.category === "news" ? "news/search" : "web/search";
+    const freshness = mapBraveFreshness(opts.freshness);
+    const url = `${BRAVE_BASE}/${endpoint}?q=${encodeURIComponent(query)}&count=${opts.limit}&freshness=${freshness}`;
+
+    const data = await httpGet<Record<string, unknown>>(url, {
+      headers: {
+        "X-Subscription-Token": apiKey,
+        Accept: "application/json",
+      },
+    });
+
+    let results: WebSearchResult[];
+    if (opts.category === "news") {
+      const newsResults = ((data as any).results || []) as BraveNewsResult[];
+      results = newsResults.slice(0, opts.limit).map(mapBraveNewsResult);
+    } else {
+      const webResults = ((data as any).web?.results || []) as BraveWebResult[];
+      results = webResults.slice(0, opts.limit).map(mapBraveWebResult);
+    }
+
+    const envelope: WebSearchEnvelope = {
+      query,
+      results,
+      resultCount: results.length,
+      fetchedAt: new Date().toISOString(),
+      provider: "brave",
+    };
+
+    cache.set(key, envelope, TTL.WEB_SEARCH);
+    return envelope;
+  } catch (error: any) {
+    const status = error instanceof HttpError ? error.status : error?.statusCode;
+    if (status === 401 || status === 403) {
+      throw new ProviderCredentialError("brave", "stale", status);
+    }
+
+    const stale = cache.getStale<WebSearchEnvelope>(key, STALE_LIMIT.WEB_SEARCH);
+    if (stale) return stale.value;
+    throw error;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Cascade orchestrator
+// ---------------------------------------------------------------------------
+
+const DEFAULT_OPTS: WebSearchOpts = {
+  category: "news",
+  freshness: "day",
+  limit: 10,
+};
+
+export async function searchWeb(
+  query: string,
+  opts?: Partial<WebSearchOpts>,
+): Promise<ProviderResult<WebSearchEnvelope>> {
+  const resolved: WebSearchOpts = { ...DEFAULT_OPTS, ...opts };
+  const normalized = normalizeFinancialQuery(query, resolved.category);
+  const config = getConfig();
+
+  const entries: Array<{ provider: string; fn: () => Promise<WebSearchEnvelope> }> = [];
+
+  // Provider override: skip cascade, use only the specified provider
+  if (resolved.provider) {
+    switch (resolved.provider) {
+      case "exa":
+        entries.push({ provider: "exa", fn: () => exaSearch(normalized, resolved) });
+        break;
+      case "brave":
+        if (!config.braveApiKey) {
+          return { status: "unavailable", reason: "BRAVE_API_KEY not configured", provider: "brave_search" };
+        }
+        entries.push({ provider: "brave_search", fn: () => braveSearch(normalized, resolved, config.braveApiKey!) });
+        break;
+      case "ddg":
+        entries.push({ provider: "ddg", fn: () => ddgSearch(normalized, resolved) });
+        break;
+    }
+    return withFallback<WebSearchEnvelope>(entries);
+  }
+
+  // Default cascade: Exa → Brave → DDG
+  entries.push({ provider: "exa", fn: () => exaSearch(normalized, resolved) });
+  if (config.braveApiKey) {
+    entries.push({ provider: "brave_search", fn: () => braveSearch(normalized, resolved, config.braveApiKey!) });
+  }
+  entries.push({ provider: "ddg", fn: () => ddgSearch(normalized, resolved) });
+
+  return withFallback<WebSearchEnvelope>(entries);
+}

package/src/providers/with-fallback.ts

@@ -0,0 +1,41 @@
+import type { ProviderResult } from "../runtime/evidence.js";
+import { wrapProvider } from "./wrap-provider.js";
+import { getProviderTracker } from "../runtime/run-context.js";
+
+export interface FallbackEntry<T> {
+  provider: string;
+  fn: () => Promise<T>;
+}
+
+/**
+ * Try providers in order, stopping at the first success.
+ * Skips circuit-open providers. Each provider call goes through wrapProvider
+ * (circuit check + failure recording + stale flag propagation).
+ *
+ * Stale cache fallback is NOT managed here — it happens inside each
+ * provider function (Level B). By the time a provider throws to wrapProvider,
+ * it already tried its own stale cache.
+ */
+export async function withFallback<T>(
+  entries: FallbackEntry<T>[],
+): Promise<ProviderResult<T>> {
+  const tracker = getProviderTracker();
+  const attempted: string[] = [];
+
+  for (const entry of entries) {
+    if (tracker?.isCircuitOpen(entry.provider)) continue;
+    attempted.push(entry.provider);
+
+    const result = await wrapProvider(entry.provider, entry.fn);
+    if (result.status === "ok") return result;
+    // wrapProvider already called recordFailure on the tracker
+  }
+
+  return {
+    status: "unavailable",
+    reason: attempted.length > 0
+      ? `all providers failed: ${attempted.join(", ")}`
+      : `all providers circuit-open: ${entries.map((e) => e.provider).join(", ")}`,
+    provider: entries[0]?.provider ?? "unknown",
+  };
+}

package/src/providers/wrap-provider.ts

@@ -0,0 +1,64 @@
+import type { ProviderResult } from "../runtime/evidence.js";
+import { getProviderTracker } from "../runtime/run-context.js";
+import { cache } from "../infra/cache.js";
+import { ProviderCredentialError } from "./provider-credential-error.js";
+
+/**
+ * Wrap a provider function call so that thrown exceptions are caught
+ * and returned as a structured `ProviderResultUnavailable`.
+ *
+ * `ProviderCredentialError` is the one exception: it is re-thrown
+ * unchanged so the tool layer can catch it and emit a tagged
+ * `[OPENCANDLE_CREDENTIAL_REQUIRED ...]` tool-result content block
+ * (see `src/onboarding/tool-tags.ts`). Swallowing credential errors
+ * into `unavailable` would hide the just-in-time setup offer from
+ * the `tool_result` interception handler.
+ *
+ * When a run context is active, checks circuit breaker state before
+ * calling and records failures after.
+ *
+ * After a successful provider call, checks if the cache's stale flag
+ * was set (meaning the provider fell back to stale cached data internally)
+ * and propagates that metadata on the result.
+ */
+export async function wrapProvider<T>(
+  providerId: string,
+  fn: () => Promise<T>,
+): Promise<ProviderResult<T>> {
+  const tracker = getProviderTracker();
+
+  if (tracker?.isCircuitOpen(providerId)) {
+    return {
+      status: "unavailable",
+      reason: "provider_circuit_open",
+      provider: providerId,
+    };
+  }
+
+  try {
+    const data = await fn();
+    const { stale, cachedAt } = cache.consumeStaleFlag();
+    return {
+      status: "ok",
+      data,
+      timestamp: stale ? new Date(cachedAt).toISOString() : new Date().toISOString(),
+      stale: stale || undefined,
+    };
+  } catch (error) {
+    // Credential errors are re-thrown so the tool-layer `withCredentialCheck`
+    // helper can convert them into LLM-visible tagged content. Do NOT record
+    // these as tracker failures — they are a user-config problem, not a
+    // provider reliability problem.
+    if (error instanceof ProviderCredentialError) {
+      throw error;
+    }
+    tracker?.recordFailure(providerId);
+    const reason =
+      error instanceof Error ? error.message : "unknown_error";
+    return {
+      status: "unavailable",
+      reason,
+      provider: providerId,
+    };
+  }
+}