opencandle 0.4.0 → 0.6.0

package/src/providers/tradingview.ts

@@ -0,0 +1,399 @@
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
+import { httpPost } from "../infra/http-client.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+
+const BASE_URL = "https://scanner.tradingview.com";
+const DATA_CAVEAT =
+  "TradingView scanner data may be delayed about 15 minutes and comes from an unofficial endpoint.";
+const BROWSER_UA =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+
+const QUOTE_COLUMNS = [
+  "name",
+  "close",
+  "change",
+  "change_abs",
+  "volume",
+  "exchange",
+  "market",
+  "description",
+  "type",
+  "typespecs",
+] as const;
+
+export const DEFAULT_COLUMNS = [
+  "name",
+  "close",
+  "change",
+  "volume",
+  "market_cap_basic",
+  "price_earnings_ttm",
+] as const;
+
+export type ScreenFilterOp =
+  | "greater"
+  | "egreater"
+  | "less"
+  | "eless"
+  | "equal"
+  | "nequal"
+  | "in_range"
+  | "not_in_range"
+  | "crosses"
+  | "crosses_above"
+  | "crosses_below"
+  | "above%"
+  | "below%"
+  | "match"
+  | "nmatch"
+  | "has"
+  | "has_none_of"
+  | "empty"
+  | "nempty";
+
+export interface ScreenFilterClause {
+  field: string;
+  op: ScreenFilterOp;
+  value?: unknown;
+}
+
+export interface ScreenSort {
+  field: string;
+  direction?: "asc" | "desc";
+}
+
+export interface ScreenStocksOpts {
+  market?: string;
+  columns?: string[];
+  filter?: ScreenFilterClause[];
+  sort?: ScreenSort;
+  limit?: number;
+}
+
+export interface ScreenerRow {
+  tvSymbol: string;
+  symbol: string;
+  values: Record<string, unknown | null>;
+  sourceProvider: "tradingview";
+  dataCaveat: string;
+}
+
+export interface TradingViewQuote {
+  requestedSymbol: string;
+  tvSymbol: string;
+  symbol: string;
+  price: number;
+  change: number;
+  changePercent: number;
+  volume: number;
+  exchange?: string;
+  market?: string;
+  name?: string;
+  type?: string;
+  typespecs?: unknown;
+  sourceProvider: "tradingview";
+  dataCaveat: string;
+}
+
+interface TradingViewResponse {
+  fields?: string[];
+  symbols?: Array<{
+    s: string;
+    f: unknown[];
+  }>;
+}
+
+interface ScannerBody {
+  markets?: string[];
+  symbols?: { tickers?: string[]; query?: { types: string[] } };
+  columns: readonly string[];
+  filter?: Array<{ left: string; operation: string; right?: unknown }>;
+  sort?: { sortBy: string; sortOrder: "asc" | "desc" };
+  range: [number, number];
+  options: { lang: "en" };
+}
+
+interface DecodedRow {
+  tvSymbol: string;
+  symbol: string;
+  values: Record<string, unknown | null>;
+}
+
+export function buildTvSymbol(exchange: string, ticker: string): string {
+  const trimmedTicker = ticker.trim().toUpperCase();
+  if (trimmedTicker.includes(":")) return trimmedTicker;
+  const trimmedExchange = exchange.trim().toUpperCase();
+  return trimmedExchange ? `${trimmedExchange}:${trimmedTicker}` : trimmedTicker;
+}
+
+export async function screenStocks(opts: ScreenStocksOpts = {}): Promise<ScreenerRow[]> {
+  const market = opts.market?.trim().toLowerCase() || "america";
+  const columns = opts.columns?.length ? opts.columns : [...DEFAULT_COLUMNS];
+  const body = buildScannerBody({
+    market,
+    columns,
+    filter: opts.filter,
+    sort: opts.sort,
+    limit: opts.limit,
+  });
+  const rows = decodeScannerRows(await scannerFetch(market, body), columns);
+  return rows.map((row) => ({
+    ...row,
+    sourceProvider: "tradingview" as const,
+    dataCaveat: DATA_CAVEAT,
+  }));
+}
+
+export async function getQuotes(symbols: string[]): Promise<TradingViewQuote[]> {
+  const requested = symbols.map(normalizeRequestedSymbol).filter(Boolean);
+  const qualified = requested.filter(isTradingViewQualified);
+  const bare = requested.filter(
+    (symbol) => !isTradingViewQualified(symbol) && !shouldSkipTradingView(symbol),
+  );
+
+  const resolved = new Map<string, TradingViewQuote>();
+
+  if (qualified.length > 0) {
+    const body = buildQualifiedQuoteBody(qualified);
+    const rows = decodeScannerRows(await scannerFetch("global", body), [...QUOTE_COLUMNS]);
+    const byTvSymbol = new Map(rows.map((row) => [row.tvSymbol.toUpperCase(), row]));
+    for (const symbol of qualified) {
+      const row = byTvSymbol.get(symbol);
+      const quote = row ? rowToQuote(symbol, row) : undefined;
+      if (quote) resolved.set(symbol, quote);
+    }
+  }
+
+  if (bare.length > 0) {
+    const body = buildBareQuoteBody(bare);
+    const rows = decodeScannerRows(await scannerFetch("america", body), [...QUOTE_COLUMNS]);
+    const byName = groupRowsByName(rows);
+    for (const symbol of bare) {
+      const row = pickPrimaryListing(byName.get(symbol) ?? []);
+      const quote = row ? rowToQuote(symbol, row) : undefined;
+      if (quote) resolved.set(symbol, quote);
+    }
+  }
+
+  return requested.flatMap((symbol) => {
+    const quote = resolved.get(symbol);
+    return quote ? [quote] : [];
+  });
+}
+
+function buildScannerBody(
+  opts: Required<Pick<ScreenStocksOpts, "market" | "columns">> &
+    Pick<ScreenStocksOpts, "filter" | "sort" | "limit">,
+): ScannerBody {
+  const limit = clampLimit(opts.limit);
+  return {
+    markets: [opts.market],
+    columns: opts.columns,
+    ...(opts.filter?.length && { filter: opts.filter.map(mapFilterClause) }),
+    ...(opts.sort && {
+      sort: {
+        sortBy: opts.sort.field,
+        sortOrder: opts.sort.direction ?? "desc",
+      },
+    }),
+    range: [0, limit],
+    options: { lang: "en" },
+  };
+}
+
+function buildQualifiedQuoteBody(tickers: string[]): ScannerBody {
+  return {
+    symbols: { tickers },
+    columns: QUOTE_COLUMNS,
+    range: [0, clampLimit(tickers.length)],
+    options: { lang: "en" },
+  };
+}
+
+function buildBareQuoteBody(symbols: string[]): ScannerBody {
+  return {
+    markets: ["america"],
+    symbols: { query: { types: [] } },
+    columns: QUOTE_COLUMNS,
+    filter: [
+      { left: "name", operation: "in_range", right: symbols },
+      { left: "is_primary", operation: "equal", right: true },
+      { left: "type", operation: "in_range", right: ["stock", "fund", "dr"] },
+    ],
+    range: [0, 500],
+    options: { lang: "en" },
+  };
+}
+
+function mapFilterClause(clause: ScreenFilterClause): {
+  left: string;
+  operation: string;
+  right?: unknown;
+} {
+  return {
+    left: clause.field,
+    operation: clause.op,
+    ...(clause.value !== undefined && { right: clause.value }),
+  };
+}
+
+async function scannerFetch(market: string, body: ScannerBody): Promise<TradingViewResponse> {
+  const endpoint = `${BASE_URL}/${market}/scan2?label-product=screener-stock`;
+  const cacheKey = `tradingview:scan2:${market}:${stableStringify(body)}`;
+  const cached = cache.get<TradingViewResponse>(cacheKey);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("tradingview");
+    const response = await httpPost<TradingViewResponse>(endpoint, body, {
+      headers: {
+        Origin: "https://www.tradingview.com",
+        Referer: "https://www.tradingview.com/",
+        "User-Agent": BROWSER_UA,
+      },
+    });
+    cache.set(cacheKey, response, TTL.SCREENER);
+    return response;
+  } catch (error) {
+    const stale = cache.getStale<TradingViewResponse>(cacheKey, STALE_LIMIT.SCREENER);
+    if (stale) return stale.value;
+    throw error;
+  }
+}
+
+function decodeScannerRows(
+  response: TradingViewResponse,
+  requestedColumns: readonly string[],
+): DecodedRow[] {
+  const fields = response.fields ?? [];
+  return (response.symbols ?? []).map((row) => {
+    const values: Record<string, unknown | null> = {};
+    for (const column of requestedColumns) {
+      const index = fields.indexOf(column);
+      values[column] = index >= 0 ? (row.f[index] ?? null) : null;
+    }
+    return {
+      tvSymbol: row.s,
+      symbol: stringValue(values.name) ?? symbolFromTvSymbol(row.s),
+      values,
+    };
+  });
+}
+
+function rowToQuote(requestedSymbol: string, row: DecodedRow): TradingViewQuote | undefined {
+  const price = numberValue(row.values.close);
+  if (price === undefined) return undefined;
+  const changePercent = numberValue(row.values.change) ?? 0;
+  const changeAbs = numberValue(row.values.change_abs) ?? 0;
+  return {
+    requestedSymbol,
+    tvSymbol: row.tvSymbol,
+    symbol: row.symbol,
+    price,
+    change: changeAbs,
+    changePercent,
+    volume: numberValue(row.values.volume) ?? 0,
+    exchange: stringValue(row.values.exchange),
+    market: stringValue(row.values.market),
+    name: stringValue(row.values.description),
+    type: stringValue(row.values.type),
+    typespecs: row.values.typespecs ?? undefined,
+    sourceProvider: "tradingview",
+    dataCaveat: DATA_CAVEAT,
+  };
+}
+
+function groupRowsByName(rows: DecodedRow[]): Map<string, DecodedRow[]> {
+  const grouped = new Map<string, DecodedRow[]>();
+  for (const row of rows) {
+    const name = row.symbol.toUpperCase();
+    grouped.set(name, [...(grouped.get(name) ?? []), row]);
+  }
+  return grouped;
+}
+
+function pickPrimaryListing(rows: DecodedRow[]): DecodedRow | undefined {
+  if (rows.length === 0) return undefined;
+  return [...rows].sort(compareListings)[0];
+}
+
+function compareListings(a: DecodedRow, b: DecodedRow): number {
+  return (
+    scoreMarket(a) - scoreMarket(b) ||
+    scoreType(a) - scoreType(b) ||
+    scoreExchange(a) - scoreExchange(b) ||
+    a.tvSymbol.localeCompare(b.tvSymbol)
+  );
+}
+
+function scoreMarket(row: DecodedRow): number {
+  return stringValue(row.values.market)?.toLowerCase() === "america" ? 0 : 1;
+}
+
+function scoreType(row: DecodedRow): number {
+  const type = stringValue(row.values.type)?.toLowerCase();
+  if (type === "stock") return 0;
+  if (type === "fund") return 1;
+  if (type === "dr") return 2;
+  return 3;
+}
+
+function scoreExchange(row: DecodedRow): number {
+  const exchange = stringValue(row.values.exchange)?.toUpperCase();
+  if (exchange === "NASDAQ") return 0;
+  if (exchange === "NYSE") return 1;
+  if (exchange === "AMEX") return 2;
+  return 3;
+}
+
+function clampLimit(limit: number | undefined): number {
+  if (limit == null) return 50;
+  if (limit < 1) return 50;
+  return Math.min(Math.floor(limit), 500);
+}
+
+function normalizeRequestedSymbol(symbol: string): string {
+  return symbol.trim().toUpperCase();
+}
+
+export function isTradingViewQualified(symbol: string): boolean {
+  return /^[A-Z0-9_]+:[A-Z0-9._-]+$/.test(symbol);
+}
+
+export function shouldSkipTradingView(symbol: string): boolean {
+  return /(?:-USD|\.(?:TO|DE|T|L|HK))$/i.test(symbol);
+}
+
+export function canUseTradingViewQuote(symbol: string): boolean {
+  const normalized = normalizeRequestedSymbol(symbol);
+  return (
+    normalized.length > 0 &&
+    (isTradingViewQualified(normalized) || !shouldSkipTradingView(normalized))
+  );
+}
+
+function symbolFromTvSymbol(tvSymbol: string): string {
+  return tvSymbol.includes(":") ? tvSymbol.split(":").at(-1)! : tvSymbol;
+}
+
+function stringValue(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function numberValue(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function stableStringify(value: unknown): string {
+  return JSON.stringify(sortObjectKeys(value));
+}
+
+function sortObjectKeys(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(sortObjectKeys);
+  if (!value || typeof value !== "object") return value;
+  return Object.fromEntries(
+    Object.entries(value)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([key, nested]) => [key, sortObjectKeys(nested)]),
+  );
+}

package/src/providers/twitter.ts

@@ -1,10 +1,10 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import Database from "better-sqlite3";
 import { Scraper, SearchMode } from "@the-convocation/twitter-scraper";
-import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
-import { rateLimiter } from "../infra/rate-limiter.js";
+import Database from "better-sqlite3";
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
 import { getBrowserProfileDir } from "../infra/opencandle-paths.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
 import type { TwitterSentimentResult, TwitterTweet } from "../types/sentiment.js";
 
 // ── Cookie extraction ────────────────────────────────────
@@ -35,7 +35,7 @@ export function readTwitterCookies(profileDir: string): FirefoxCookie[] {
 
 // ── Sentiment scoring ────────────────────────────────────
 
-import { BULLISH_TERMS, BEARISH_TERMS } from "../sentiment/keywords.js";
+import { BEARISH_TERMS, BULLISH_TERMS } from "../sentiment/keywords.js";
 
 export function scoreTwitterSentiment(
   tweets: Array<{ text: string; likes: number; retweets: number }>,
@@ -68,7 +68,7 @@ export function scoreTwitterSentiment(
 // ── Query normalization ──────────────────────────────────
 
 export function normalizeQuery(query: string): string {
-  if (/^[A-Z]{1,5}$/.test(query)) return "$" + query;
+  if (/^[A-Z]{1,5}$/.test(query)) return `$${query}`;
   return query;
 }
 
@@ -135,9 +135,7 @@ export async function getTwitterSentiment(
     const tickerRegex = /\$([A-Z]{1,5})\b/g;
     const mentionCounts = new Map<string, number>();
     // Exclude the searched ticker itself from co-mentions
-    const searchedTicker = normalizedQuery.startsWith("$")
-      ? normalizedQuery.slice(1)
-      : null;
+    const searchedTicker = normalizedQuery.startsWith("$") ? normalizedQuery.slice(1) : null;
     for (const t of tweets) {
       for (const match of t.text.matchAll(tickerRegex)) {
         const ticker = match[1];

package/src/providers/web-search.ts

@@ -1,15 +1,14 @@
-import { search, searchNews, SafeSearchType, SearchTimeType } from "duck-duck-scrape";
-import type { SearchResult, SearchResults } from "duck-duck-scrape";
-import type { NewsResult } from "duck-duck-scrape";
-import { httpGet, HttpError } from "../infra/http-client.js";
-import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
-import { rateLimiter } from "../infra/rate-limiter.js";
+import type { NewsResult, SearchResult } from "duck-duck-scrape";
+import { SafeSearchType, SearchTimeType, search, searchNews } from "duck-duck-scrape";
 import { getConfig } from "../config.js";
-import { withFallback } from "./with-fallback.js";
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
+import { HttpError, httpGet } from "../infra/http-client.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import type { ProviderResult } from "../runtime/evidence.js";
+import type { WebSearchEnvelope, WebSearchResult } from "../types/sentiment.js";
 import { exaSearch } from "./exa-search.js";
 import { ProviderCredentialError } from "./provider-credential-error.js";
-import type { ProviderResult } from "../runtime/evidence.js";
-import type { WebSearchResult, WebSearchEnvelope } from "../types/sentiment.js";
+import { withFallback } from "./with-fallback.js";
 
 export interface WebSearchOpts {
   category: "news" | "general";
@@ -85,10 +84,7 @@ function ddgCacheKey(query: string, opts: WebSearchOpts): string {
   return `web:ddg:${query}:${opts.category}:${opts.freshness}:${opts.limit}`;
 }
 
-export async function ddgSearch(
-  query: string,
-  opts: WebSearchOpts,
-): Promise<WebSearchEnvelope> {
+export async function ddgSearch(query: string, opts: WebSearchOpts): Promise<WebSearchEnvelope> {
   const key = ddgCacheKey(query, opts);
   const cached = cache.get<WebSearchEnvelope>(key);
   if (cached) return cached;
@@ -139,10 +135,14 @@ const BRAVE_BASE = "https://api.search.brave.com/res/v1";
 
 function mapBraveFreshness(freshness: WebSearchOpts["freshness"]): string {
   switch (freshness) {
-    case "hours": return "ph";
-    case "day": return "pd";
-    case "week": return "pw";
-    case "month": return "pm";
+    case "hours":
+      return "ph";
+    case "day":
+      return "pd";
+    case "week":
+      return "pw";
+    case "month":
+      return "pm";
   }
 }
 
@@ -271,9 +271,16 @@ export async function searchWeb(
         break;
       case "brave":
         if (!config.braveApiKey) {
-          return { status: "unavailable", reason: "BRAVE_API_KEY not configured", provider: "brave_search" };
+          return {
+            status: "unavailable",
+            reason: "BRAVE_API_KEY not configured",
+            provider: "brave_search",
+          };
         }
-        entries.push({ provider: "brave_search", fn: () => braveSearch(normalized, resolved, config.braveApiKey!) });
+        entries.push({
+          provider: "brave_search",
+          fn: () => braveSearch(normalized, resolved, config.braveApiKey!),
+        });
         break;
       case "ddg":
         entries.push({ provider: "ddg", fn: () => ddgSearch(normalized, resolved) });
@@ -285,7 +292,10 @@ export async function searchWeb(
   // Default cascade: Exa → Brave → DDG
   entries.push({ provider: "exa", fn: () => exaSearch(normalized, resolved) });
   if (config.braveApiKey) {
-    entries.push({ provider: "brave_search", fn: () => braveSearch(normalized, resolved, config.braveApiKey!) });
+    entries.push({
+      provider: "brave_search",
+      fn: () => braveSearch(normalized, resolved, config.braveApiKey!),
+    });
   }
   entries.push({ provider: "ddg", fn: () => ddgSearch(normalized, resolved) });
 

package/src/providers/with-fallback.ts

@@ -1,6 +1,6 @@
 import type { ProviderResult } from "../runtime/evidence.js";
-import { wrapProvider } from "./wrap-provider.js";
 import { getProviderTracker } from "../runtime/run-context.js";
+import { wrapProvider } from "./wrap-provider.js";
 
 export interface FallbackEntry<T> {
   provider: string;
@@ -16,11 +16,10 @@ export interface FallbackEntry<T> {
  * provider function (Level B). By the time a provider throws to wrapProvider,
  * it already tried its own stale cache.
  */
-export async function withFallback<T>(
-  entries: FallbackEntry<T>[],
-): Promise<ProviderResult<T>> {
+export async function withFallback<T>(entries: FallbackEntry<T>[]): Promise<ProviderResult<T>> {
   const tracker = getProviderTracker();
   const attempted: string[] = [];
+  const failures: string[] = [];
 
   for (const entry of entries) {
     if (tracker?.isCircuitOpen(entry.provider)) continue;
@@ -28,14 +27,16 @@ export async function withFallback<T>(
 
     const result = await wrapProvider(entry.provider, entry.fn);
     if (result.status === "ok") return result;
+    failures.push(`${entry.provider}: ${result.reason}`);
     // wrapProvider already called recordFailure on the tracker
   }
 
   return {
     status: "unavailable",
-    reason: attempted.length > 0
-      ? `all providers failed: ${attempted.join(", ")}`
-      : `all providers circuit-open: ${entries.map((e) => e.provider).join(", ")}`,
+    reason:
+      attempted.length > 0
+        ? `all providers failed: ${failures.join("; ")}`
+        : `all providers circuit-open: ${entries.map((e) => e.provider).join(", ")}`,
     provider: entries[0]?.provider ?? "unknown",
   };
 }

package/src/providers/wrap-provider.ts

@@ -1,6 +1,7 @@
+import { runWithStaleMetadata } from "../infra/cache.js";
 import type { ProviderResult } from "../runtime/evidence.js";
 import { getProviderTracker } from "../runtime/run-context.js";
-import { cache } from "../infra/cache.js";
+import { InvalidSymbolError } from "./errors.js";
 import { ProviderCredentialError } from "./provider-credential-error.js";
 
 /**
@@ -17,9 +18,8 @@ import { ProviderCredentialError } from "./provider-credential-error.js";
  * When a run context is active, checks circuit breaker state before
  * calling and records failures after.
  *
- * After a successful provider call, checks if the cache's stale flag
- * was set (meaning the provider fell back to stale cached data internally)
- * and propagates that metadata on the result.
+ * After a successful provider call, propagates stale cache metadata recorded
+ * in the current provider call's async scope.
  */
 export async function wrapProvider<T>(
   providerId: string,
@@ -36,13 +36,12 @@ export async function wrapProvider<T>(
   }
 
   try {
-    const data = await fn();
-    const { stale, cachedAt } = cache.consumeStaleFlag();
+    const { value: data, stale } = await runWithStaleMetadata(fn);
     return {
       status: "ok",
       data,
-      timestamp: stale ? new Date(cachedAt).toISOString() : new Date().toISOString(),
-      stale: stale || undefined,
+      timestamp: stale ? new Date(stale.cachedAt).toISOString() : new Date().toISOString(),
+      stale: stale ? true : undefined,
     };
   } catch (error) {
     // Credential errors are re-thrown so the tool-layer `withCredentialCheck`
@@ -52,9 +51,15 @@ export async function wrapProvider<T>(
     if (error instanceof ProviderCredentialError) {
       throw error;
     }
+    if (error instanceof InvalidSymbolError) {
+      return {
+        status: "unavailable",
+        reason: error.message,
+        provider: providerId,
+      };
+    }
     tracker?.recordFailure(providerId);
-    const reason =
-      error instanceof Error ? error.message : "unknown_error";
+    const reason = error instanceof Error ? error.message : "unknown_error";
     return {
       status: "unavailable",
       reason,