opencandle 0.3.0 → 0.4.0

package/src/providers/exa-search.ts

@@ -0,0 +1,373 @@
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import { getConfig } from "../config.js";
+import { ProviderCredentialError } from "./provider-credential-error.js";
+import type { WebSearchResult, WebSearchEnvelope } from "../types/sentiment.js";
+import type { WebSearchOpts } from "./web-search.js";
+
+const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
+const EXA_API_URL = "https://api.exa.ai/search";
+const TIMEOUT_MS = 5_000;
+const SNIPPET_MAX_CHARS = 300;
+const CONTEXT_MAX_CHARS = 1_000;
+
+let requestIdCounter = 0;
+
+// ---------------------------------------------------------------------------
+// Freshness helpers
+// ---------------------------------------------------------------------------
+
+function freshnessToMs(freshness: WebSearchOpts["freshness"]): number {
+  switch (freshness) {
+    case "hours": return 60 * 60 * 1000;
+    case "day": return 24 * 60 * 60 * 1000;
+    case "week": return 7 * 24 * 60 * 60 * 1000;
+    case "month": return 30 * 24 * 60 * 60 * 1000;
+  }
+}
+
+function enrichQueryForMcp(query: string, freshness: WebSearchOpts["freshness"]): string {
+  switch (freshness) {
+    case "hours": return `${query} past hour`;
+    case "day": return `${query} past 24 hours`;
+    case "week": return `${query} past week`;
+    case "month": return `${query} past month`;
+  }
+}
+
+function startPublishedDate(freshness: WebSearchOpts["freshness"]): string {
+  return new Date(Date.now() - freshnessToMs(freshness)).toISOString();
+}
+
+// ---------------------------------------------------------------------------
+// MCP response parsing
+// ---------------------------------------------------------------------------
+
+interface McpRpcResponse {
+  result?: {
+    content?: Array<{ type?: string; text?: string; _meta?: Record<string, unknown> }>;
+  };
+  error?: {
+    code?: number;
+    message?: string;
+  };
+}
+
+function extractJsonRpcPayload(body: string, contentType: string): McpRpcResponse {
+  // Path 1: plain JSON response
+  if (contentType.includes("application/json")) {
+    return JSON.parse(body) as McpRpcResponse;
+  }
+
+  // Path 2: SSE — scan all data: lines for valid JSON-RPC
+  const dataLines = body.split("\n").filter((line) => line.startsWith("data:"));
+  for (const line of dataLines) {
+    const payload = line.slice(5).trim();
+    if (!payload) continue;
+    try {
+      const candidate = JSON.parse(payload) as McpRpcResponse;
+      if (candidate?.result || candidate?.error) return candidate;
+    } catch {
+      // not valid JSON, try next line
+    }
+  }
+
+  // Path 3: fallback — try parsing entire body
+  try {
+    const candidate = JSON.parse(body) as McpRpcResponse;
+    if (candidate?.result || candidate?.error) return candidate;
+  } catch {
+    // not valid JSON
+  }
+
+  throw new Error("Exa MCP returned empty content");
+}
+
+interface ParsedResult {
+  title: string;
+  url: string;
+  published: string | null;
+  snippet: string;
+}
+
+function parseMcpResultBlocks(text: string): ParsedResult[] {
+  const blocks = text.split(/\n---\n/).filter((b) => b.trim().length > 0);
+  const results: ParsedResult[] = [];
+
+  for (const block of blocks) {
+    const title = block.match(/^Title: (.+)/m)?.[1]?.trim() ?? "";
+    const url = block.match(/^URL: (.+)/m)?.[1]?.trim() ?? "";
+    if (!url) continue;
+
+    const published = block.match(/^Published: (.+)/m)?.[1]?.trim() ?? null;
+
+    // Extract highlights/text content after the header fields
+    let content = "";
+    const hlMatch = block.match(/^Highlights:\n?/m);
+    if (hlMatch?.index != null) {
+      content = block.slice(hlMatch.index + hlMatch[0].length).trim();
+    }
+    const snippet = content.slice(0, SNIPPET_MAX_CHARS);
+
+    results.push({ title, url, published, snippet });
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// Direct API response mapping
+// ---------------------------------------------------------------------------
+
+interface ExaApiResult {
+  title?: string;
+  url?: string;
+  publishedDate?: string;
+  text?: string;
+  highlights?: string[];
+}
+
+interface ExaApiResponse {
+  results?: ExaApiResult[];
+}
+
+function mapApiResults(results: ExaApiResult[]): ParsedResult[] {
+  return results
+    .filter((r) => r.url)
+    .map((r) => ({
+      title: r.title ?? "",
+      url: r.url!,
+      published: r.publishedDate ?? null,
+      snippet: (r.highlights?.join(" ") || r.text || "").slice(0, SNIPPET_MAX_CHARS),
+    }));
+}
+
+// ---------------------------------------------------------------------------
+// Freshness post-filter
+// ---------------------------------------------------------------------------
+
+function filterByFreshness(
+  results: ParsedResult[],
+  freshness: WebSearchOpts["freshness"],
+): ParsedResult[] {
+  const cutoff = Date.now() - freshnessToMs(freshness);
+  return results.filter((r) => {
+    if (!r.published) return true; // benefit of the doubt
+    const pubTime = new Date(r.published).getTime();
+    return !isNaN(pubTime) && pubTime >= cutoff;
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Common helpers
+// ---------------------------------------------------------------------------
+
+function extractDomain(url: string): string {
+  try {
+    return new URL(url).hostname.replace(/^www\./, "");
+  } catch {
+    return url;
+  }
+}
+
+function toWebSearchResults(
+  parsed: ParsedResult[],
+  category: "news" | "general",
+): WebSearchResult[] {
+  return parsed.map((r) => ({
+    title: r.title,
+    url: r.url,
+    snippet: r.snippet,
+    source: extractDomain(r.url),
+    published: r.published,
+    category,
+  }));
+}
+
+function exaCacheKey(query: string, opts: WebSearchOpts): string {
+  return `web:exa:${query}:${opts.category}:${opts.freshness}:${opts.limit}`;
+}
+
+// ---------------------------------------------------------------------------
+// MCP path
+// ---------------------------------------------------------------------------
+
+async function exaMcpSearch(
+  query: string,
+  opts: WebSearchOpts,
+): Promise<WebSearchEnvelope> {
+  const enrichedQuery = enrichQueryForMcp(query, opts.freshness);
+
+  const response = await fetch(EXA_MCP_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Accept: "application/json, text/event-stream",
+    },
+    body: JSON.stringify({
+      jsonrpc: "2.0",
+      id: ++requestIdCounter,
+      method: "tools/call",
+      params: {
+        name: "web_search_exa",
+        arguments: {
+          query: enrichedQuery,
+          numResults: opts.limit,
+          livecrawl: "fallback",
+          type: "auto",
+          contextMaxCharacters: CONTEXT_MAX_CHARS,
+        },
+      },
+    }),
+    signal: AbortSignal.timeout(TIMEOUT_MS),
+  });
+
+  // Anti-abuse handling
+  const contentType = response.headers.get("content-type") ?? "";
+  if (!response.ok) {
+    if (response.status === 429) {
+      const retryAfter = response.headers.get("retry-after");
+      throw new Error(
+        `Exa MCP rate limited (429)${retryAfter ? ` — retry after ${retryAfter}s` : ""}`,
+      );
+    }
+    if (response.status === 403) {
+      throw new Error("Exa MCP blocked (403) — possible IP-based blocking");
+    }
+    throw new Error(`Exa MCP HTTP ${response.status} ${response.statusText}`);
+  }
+
+  if (contentType.includes("text/html")) {
+    throw new Error("Exa MCP returned HTML instead of JSON-RPC (possible challenge page)");
+  }
+
+  const body = await response.text();
+  const payload = extractJsonRpcPayload(body, contentType);
+
+  if (payload.error) {
+    throw new Error(payload.error.message ?? `Exa MCP error: code ${payload.error.code}`);
+  }
+
+  const text = payload.result?.content?.find(
+    (item) => item.type === "text" && typeof item.text === "string",
+  )?.text;
+
+  if (!text || text.trim().length === 0) {
+    // Legitimate zero results
+    return {
+      query,
+      results: [],
+      resultCount: 0,
+      fetchedAt: new Date().toISOString(),
+      provider: "exa",
+    };
+  }
+
+  const parsed = parseMcpResultBlocks(text);
+  const filtered = filterByFreshness(parsed, opts.freshness);
+  const results = toWebSearchResults(filtered, opts.category);
+
+  return {
+    query,
+    results: results.slice(0, opts.limit),
+    resultCount: Math.min(results.length, opts.limit),
+    fetchedAt: new Date().toISOString(),
+    provider: "exa",
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Direct API path
+// ---------------------------------------------------------------------------
+
+async function exaApiSearch(
+  query: string,
+  opts: WebSearchOpts,
+  apiKey: string,
+): Promise<WebSearchEnvelope> {
+  const response = await fetch(EXA_API_URL, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      query,
+      type: "auto",
+      numResults: opts.limit,
+      startPublishedDate: startPublishedDate(opts.freshness),
+      contents: {
+        text: { maxCharacters: CONTEXT_MAX_CHARS },
+        highlights: true,
+      },
+    }),
+    signal: AbortSignal.timeout(TIMEOUT_MS),
+  });
+
+  if (!response.ok) {
+    if (response.status === 401 || response.status === 403) {
+      throw new ProviderCredentialError("exa", "stale", response.status);
+    }
+    const body = await response.text().catch(() => "");
+    throw new Error(`Exa API HTTP ${response.status}: ${body.slice(0, 300)}`);
+  }
+
+  const data = (await response.json()) as ExaApiResponse;
+  const parsed = mapApiResults(data.results ?? []);
+  const filtered = filterByFreshness(parsed, opts.freshness);
+  const results = toWebSearchResults(filtered, opts.category);
+
+  return {
+    query,
+    results: results.slice(0, opts.limit),
+    resultCount: Math.min(results.length, opts.limit),
+    fetchedAt: new Date().toISOString(),
+    provider: "exa",
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Public entry point
+// ---------------------------------------------------------------------------
+
+export async function exaSearch(
+  query: string,
+  opts: WebSearchOpts,
+): Promise<WebSearchEnvelope> {
+  const key = exaCacheKey(query, opts);
+  const cached = cache.get<WebSearchEnvelope>(key);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("exa");
+
+    const config = getConfig();
+    const envelope = config.exaApiKey
+      ? await exaApiSearch(query, opts, config.exaApiKey)
+      : await exaMcpSearch(query, opts);
+
+    cache.set(key, envelope, TTL.WEB_SEARCH);
+    return envelope;
+  } catch (error) {
+    // Re-throw abort errors immediately
+    if (error instanceof Error && error.name === "AbortError") throw error;
+    if (error instanceof DOMException && error.name === "TimeoutError") throw error;
+    // Re-throw credential errors so the tool layer can convert them to the
+    // tagged content block. Stale cache must not mask a real auth failure.
+    if (error instanceof ProviderCredentialError) throw error;
+
+    const stale = cache.getStale<WebSearchEnvelope>(key, STALE_LIMIT.WEB_SEARCH);
+    if (stale) return stale.value;
+    throw error;
+  }
+}
+
+// Exported for testing
+export {
+  parseMcpResultBlocks,
+  extractJsonRpcPayload,
+  enrichQueryForMcp,
+  filterByFreshness,
+  mapApiResults,
+};
+export type { ParsedResult, McpRpcResponse, ExaApiResponse };

package/src/providers/fear-greed.ts

@@ -0,0 +1,45 @@
+import { httpGet } from "../infra/http-client.js";
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import type { FearGreedData } from "../types/sentiment.js";
+
+// alternative.me provides a free crypto Fear & Greed index
+// CNN endpoint (production.dataviz.cnn.io) blocks automated requests (HTTP 418)
+const ENDPOINT = "https://api.alternative.me/fng/?limit=3";
+
+interface AlternativeMeFngResponse {
+  data: Array<{
+    value: string;
+    value_classification: string;
+    timestamp: string;
+  }>;
+}
+
+export async function getFearGreedIndex(): Promise<FearGreedData> {
+  const cacheKey = "feargreed:index";
+  const cached = cache.get<FearGreedData>(cacheKey);
+  if (cached) return cached;
+
+  try {
+    const data = await httpGet<AlternativeMeFngResponse>(ENDPOINT);
+
+    const entries = data.data;
+    const current = entries[0];
+    const value = parseInt(current.value, 10);
+
+    const result: FearGreedData = {
+      value,
+      label: current.value_classification,
+      timestamp: Date.now(),
+      previousClose: entries[1] ? parseInt(entries[1].value, 10) : value,
+      weekAgo: null,
+      monthAgo: null,
+    };
+
+    cache.set(cacheKey, result, TTL.SENTIMENT);
+    return result;
+  } catch (error) {
+    const stale = cache.getStale<FearGreedData>(cacheKey, STALE_LIMIT.SENTIMENT);
+    if (stale) return stale.value;
+    throw error;
+  }
+}

package/src/providers/finnhub.ts

@@ -0,0 +1,124 @@
+import { httpGet, HttpError } from "../infra/http-client.js";
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import { ProviderCredentialError } from "./provider-credential-error.js";
+
+const FINNHUB_BASE = "https://finnhub.io/api/v1";
+
+export interface FinnhubArticle {
+  headline: string;
+  summary: string;
+  source: string;
+  datetime: number;
+  url: string;
+  related: string;
+  id: number;
+  category: string;
+  image: string;
+}
+
+// Ticker → company name mapping for relevance filtering
+const TICKER_NAMES: Record<string, string> = {
+  AAPL: "apple",
+  MSFT: "microsoft",
+  GOOGL: "google",
+  GOOG: "google",
+  AMZN: "amazon",
+  META: "meta",
+  TSLA: "tesla",
+  NVDA: "nvidia",
+  TSM: "tsmc",
+  BABA: "alibaba",
+  NFLX: "netflix",
+  AMD: "amd",
+  INTC: "intel",
+  CRM: "salesforce",
+  ORCL: "oracle",
+};
+
+export function finnhubDateRange(freshness: "hours" | "day" | "week" | "month"): { from: string; to: string } {
+  const now = new Date();
+  const to = formatDate(now);
+
+  switch (freshness) {
+    case "hours":
+      return { from: to, to };
+    case "day": {
+      const d = new Date(now);
+      d.setDate(d.getDate() - 1);
+      return { from: formatDate(d), to };
+    }
+    case "week": {
+      const d = new Date(now);
+      d.setDate(d.getDate() - 7);
+      return { from: formatDate(d), to };
+    }
+    case "month": {
+      const d = new Date(now);
+      d.setDate(d.getDate() - 30);
+      return { from: formatDate(d), to };
+    }
+  }
+}
+
+function formatDate(d: Date): string {
+  return d.toISOString().slice(0, 10);
+}
+
+function cacheKey(symbol: string, from: string, to: string): string {
+  return `finnhub:news:${symbol}:${from}:${to}`;
+}
+
+export function filterByRelevance(
+  articles: FinnhubArticle[],
+  symbol: string,
+  limit = 20,
+): FinnhubArticle[] {
+  const symLower = symbol.toLowerCase();
+  const companyName = TICKER_NAMES[symbol.toUpperCase()];
+
+  const filtered = articles.filter((a) => {
+    const hl = a.headline.toLowerCase();
+    const sm = a.summary.toLowerCase();
+    if (hl.includes(symLower) || sm.includes(symLower)) return true;
+    if (companyName && (hl.includes(companyName) || sm.includes(companyName))) return true;
+    return false;
+  });
+
+  // Most recent first, capped
+  return filtered
+    .sort((a, b) => b.datetime - a.datetime)
+    .slice(0, limit);
+}
+
+export async function getCompanyNews(
+  symbol: string,
+  from: string,
+  to: string,
+  apiKey: string,
+): Promise<FinnhubArticle[]> {
+  const key = cacheKey(symbol, from, to);
+  const cached = cache.get<FinnhubArticle[]>(key);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("finnhub");
+
+    const url = `${FINNHUB_BASE}/company-news?symbol=${encodeURIComponent(symbol)}&from=${from}&to=${to}&token=${apiKey}`;
+    const data = await httpGet<FinnhubArticle[]>(url);
+
+    const articles = Array.isArray(data) ? data : [];
+    const filtered = filterByRelevance(articles, symbol);
+
+    cache.set(key, filtered, TTL.FINNHUB_NEWS);
+    return filtered;
+  } catch (error) {
+    if (error instanceof HttpError && (error.status === 401 || error.status === 403)) {
+      throw new ProviderCredentialError("finnhub", "stale", error.status);
+    }
+
+    const stale = cache.getStale<FinnhubArticle[]>(key, STALE_LIMIT.FINNHUB_NEWS);
+    if (stale) return stale.value;
+    throw error;
+  }
+}

package/src/providers/fred.ts

@@ -0,0 +1,83 @@
+import { httpGet, HttpError } from "../infra/http-client.js";
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import { ProviderCredentialError } from "./provider-credential-error.js";
+import type { FredSeries, FredObservation } from "../types/macro.js";
+
+const BASE_URL = "https://api.stlouisfed.org/fred";
+
+interface FredSeriesResponse {
+  seriess: Array<{
+    id: string;
+    title: string;
+    units: string;
+    frequency: string;
+    last_updated: string;
+  }>;
+}
+
+interface FredObservationsResponse {
+  observations: Array<{
+    date: string;
+    value: string;
+  }>;
+}
+
+export async function getSeries(
+  seriesId: string,
+  apiKey: string,
+  limit: number = 60,
+): Promise<FredSeries> {
+  const cacheKey = `fred:series:${seriesId}:${limit}`;
+  const cached = cache.get<FredSeries>(cacheKey);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("fred");
+
+    // Fetch series metadata and observations in parallel
+    const metaUrl = `${BASE_URL}/series?series_id=${seriesId}&api_key=${apiKey}&file_type=json`;
+    const obsUrl = `${BASE_URL}/series/observations?series_id=${seriesId}&api_key=${apiKey}&file_type=json&sort_order=desc&limit=${limit}`;
+
+    const [metaData, obsData] = await Promise.all([
+      httpGet<FredSeriesResponse>(metaUrl),
+      httpGet<FredObservationsResponse>(obsUrl),
+    ]);
+
+    const meta = metaData.seriess[0];
+    const observations: FredObservation[] = obsData.observations
+      .filter((o) => o.value !== ".")
+      .map((o) => ({
+        date: o.date,
+        value: parseFloat(o.value),
+      }))
+      .reverse(); // chronological order
+
+    const result: FredSeries = {
+      id: meta.id,
+      title: meta.title,
+      observations,
+      units: meta.units,
+      frequency: meta.frequency,
+      lastUpdated: meta.last_updated,
+    };
+
+    cache.set(cacheKey, result, TTL.MACRO);
+    return result;
+  } catch (error) {
+    // FRED historically returns 400 with a body like "Bad Request. The value
+    // for variable api_key is not registered..." when the key is invalid. A
+    // separate 400 can also indicate a bad series_id, which is NOT a credential
+    // problem. We only reclassify as credential-related on 401/403 here to
+    // avoid surfacing a `/connect` prompt when the user's real mistake is a
+    // typo in a series id. Bad-FRED-key users will still see the raw error
+    // message in v1; a body-string check can be added later if this becomes a
+    // reported friction.
+    if (error instanceof HttpError && (error.status === 401 || error.status === 403)) {
+      throw new ProviderCredentialError("fred", "stale", error.status);
+    }
+    const stale = cache.getStale<FredSeries>(cacheKey, STALE_LIMIT.MACRO);
+    if (stale) return stale.value;
+    throw error;
+  }
+}

package/src/providers/index.ts

@@ -0,0 +1,9 @@
+export { getQuote, getHistory, getOptionsChain, clearCrumbCache, getYahooCrumb, computeTimeToExpiry } from "./yahoo-finance.js";
+export { getOverview, getEarnings, getFinancials } from "./alpha-vantage.js";
+export { getSeries } from "./fred.js";
+export { getCryptoPrice, getCryptoHistory } from "./coingecko.js";
+export { getSubredditPosts, scoreSentiment } from "./reddit.js";
+export { searchFilings, type SECFiling } from "./sec-edgar.js";
+export { getFearGreedIndex } from "./fear-greed.js";
+export { searchWeb, ddgSearch, braveSearch, normalizeFinancialQuery } from "./web-search.js";
+export type { WebSearchOpts } from "./web-search.js";

package/src/providers/provider-credential-error.ts

@@ -0,0 +1,23 @@
+// Typed error thrown by providers when they detect a missing or stale credential.
+//
+// Providers remain pure fetchers that throw on failure — this class is the
+// structured hand-off to the tool layer, where `withCredentialCheck` catches
+// the error and emits a tagged tool-result content string.
+//
+// Keep this module dependency-free (aside from the ProviderId type) so it can
+// be imported from anywhere without pulling in config loading or registry
+// side effects.
+
+import type { ProviderId } from "../onboarding/providers.js";
+
+export class ProviderCredentialError extends Error {
+  readonly name = "ProviderCredentialError";
+
+  constructor(
+    readonly provider: ProviderId,
+    readonly reason: "missing" | "stale",
+    readonly httpStatus?: number,
+  ) {
+    super(`credential_required:${provider}:${reason}`);
+  }
+}