opencandle 0.5.0 → 0.7.0

package/src/providers/tradingview.ts

@@ -0,0 +1,399 @@
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
+import { httpPost } from "../infra/http-client.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+
+const BASE_URL = "https://scanner.tradingview.com";
+const DATA_CAVEAT =
+  "TradingView scanner data may be delayed about 15 minutes and comes from an unofficial endpoint.";
+const BROWSER_UA =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+
+const QUOTE_COLUMNS = [
+  "name",
+  "close",
+  "change",
+  "change_abs",
+  "volume",
+  "exchange",
+  "market",
+  "description",
+  "type",
+  "typespecs",
+] as const;
+
+export const DEFAULT_COLUMNS = [
+  "name",
+  "close",
+  "change",
+  "volume",
+  "market_cap_basic",
+  "price_earnings_ttm",
+] as const;
+
+export type ScreenFilterOp =
+  | "greater"
+  | "egreater"
+  | "less"
+  | "eless"
+  | "equal"
+  | "nequal"
+  | "in_range"
+  | "not_in_range"
+  | "crosses"
+  | "crosses_above"
+  | "crosses_below"
+  | "above%"
+  | "below%"
+  | "match"
+  | "nmatch"
+  | "has"
+  | "has_none_of"
+  | "empty"
+  | "nempty";
+
+export interface ScreenFilterClause {
+  field: string;
+  op: ScreenFilterOp;
+  value?: unknown;
+}
+
+export interface ScreenSort {
+  field: string;
+  direction?: "asc" | "desc";
+}
+
+export interface ScreenStocksOpts {
+  market?: string;
+  columns?: string[];
+  filter?: ScreenFilterClause[];
+  sort?: ScreenSort;
+  limit?: number;
+}
+
+export interface ScreenerRow {
+  tvSymbol: string;
+  symbol: string;
+  values: Record<string, unknown | null>;
+  sourceProvider: "tradingview";
+  dataCaveat: string;
+}
+
+export interface TradingViewQuote {
+  requestedSymbol: string;
+  tvSymbol: string;
+  symbol: string;
+  price: number;
+  change: number;
+  changePercent: number;
+  volume: number;
+  exchange?: string;
+  market?: string;
+  name?: string;
+  type?: string;
+  typespecs?: unknown;
+  sourceProvider: "tradingview";
+  dataCaveat: string;
+}
+
+interface TradingViewResponse {
+  fields?: string[];
+  symbols?: Array<{
+    s: string;
+    f: unknown[];
+  }>;
+}
+
+interface ScannerBody {
+  markets?: string[];
+  symbols?: { tickers?: string[]; query?: { types: string[] } };
+  columns: readonly string[];
+  filter?: Array<{ left: string; operation: string; right?: unknown }>;
+  sort?: { sortBy: string; sortOrder: "asc" | "desc" };
+  range: [number, number];
+  options: { lang: "en" };
+}
+
+interface DecodedRow {
+  tvSymbol: string;
+  symbol: string;
+  values: Record<string, unknown | null>;
+}
+
+export function buildTvSymbol(exchange: string, ticker: string): string {
+  const trimmedTicker = ticker.trim().toUpperCase();
+  if (trimmedTicker.includes(":")) return trimmedTicker;
+  const trimmedExchange = exchange.trim().toUpperCase();
+  return trimmedExchange ? `${trimmedExchange}:${trimmedTicker}` : trimmedTicker;
+}
+
+export async function screenStocks(opts: ScreenStocksOpts = {}): Promise<ScreenerRow[]> {
+  const market = opts.market?.trim().toLowerCase() || "america";
+  const columns = opts.columns?.length ? opts.columns : [...DEFAULT_COLUMNS];
+  const body = buildScannerBody({
+    market,
+    columns,
+    filter: opts.filter,
+    sort: opts.sort,
+    limit: opts.limit,
+  });
+  const rows = decodeScannerRows(await scannerFetch(market, body), columns);
+  return rows.map((row) => ({
+    ...row,
+    sourceProvider: "tradingview" as const,
+    dataCaveat: DATA_CAVEAT,
+  }));
+}
+
+export async function getQuotes(symbols: string[]): Promise<TradingViewQuote[]> {
+  const requested = symbols.map(normalizeRequestedSymbol).filter(Boolean);
+  const qualified = requested.filter(isTradingViewQualified);
+  const bare = requested.filter(
+    (symbol) => !isTradingViewQualified(symbol) && !shouldSkipTradingView(symbol),
+  );
+
+  const resolved = new Map<string, TradingViewQuote>();
+
+  if (qualified.length > 0) {
+    const body = buildQualifiedQuoteBody(qualified);
+    const rows = decodeScannerRows(await scannerFetch("global", body), [...QUOTE_COLUMNS]);
+    const byTvSymbol = new Map(rows.map((row) => [row.tvSymbol.toUpperCase(), row]));
+    for (const symbol of qualified) {
+      const row = byTvSymbol.get(symbol);
+      const quote = row ? rowToQuote(symbol, row) : undefined;
+      if (quote) resolved.set(symbol, quote);
+    }
+  }
+
+  if (bare.length > 0) {
+    const body = buildBareQuoteBody(bare);
+    const rows = decodeScannerRows(await scannerFetch("america", body), [...QUOTE_COLUMNS]);
+    const byName = groupRowsByName(rows);
+    for (const symbol of bare) {
+      const row = pickPrimaryListing(byName.get(symbol) ?? []);
+      const quote = row ? rowToQuote(symbol, row) : undefined;
+      if (quote) resolved.set(symbol, quote);
+    }
+  }
+
+  return requested.flatMap((symbol) => {
+    const quote = resolved.get(symbol);
+    return quote ? [quote] : [];
+  });
+}
+
+function buildScannerBody(
+  opts: Required<Pick<ScreenStocksOpts, "market" | "columns">> &
+    Pick<ScreenStocksOpts, "filter" | "sort" | "limit">,
+): ScannerBody {
+  const limit = clampLimit(opts.limit);
+  return {
+    markets: [opts.market],
+    columns: opts.columns,
+    ...(opts.filter?.length && { filter: opts.filter.map(mapFilterClause) }),
+    ...(opts.sort && {
+      sort: {
+        sortBy: opts.sort.field,
+        sortOrder: opts.sort.direction ?? "desc",
+      },
+    }),
+    range: [0, limit],
+    options: { lang: "en" },
+  };
+}
+
+function buildQualifiedQuoteBody(tickers: string[]): ScannerBody {
+  return {
+    symbols: { tickers },
+    columns: QUOTE_COLUMNS,
+    range: [0, clampLimit(tickers.length)],
+    options: { lang: "en" },
+  };
+}
+
+function buildBareQuoteBody(symbols: string[]): ScannerBody {
+  return {
+    markets: ["america"],
+    symbols: { query: { types: [] } },
+    columns: QUOTE_COLUMNS,
+    filter: [
+      { left: "name", operation: "in_range", right: symbols },
+      { left: "is_primary", operation: "equal", right: true },
+      { left: "type", operation: "in_range", right: ["stock", "fund", "dr"] },
+    ],
+    range: [0, 500],
+    options: { lang: "en" },
+  };
+}
+
+function mapFilterClause(clause: ScreenFilterClause): {
+  left: string;
+  operation: string;
+  right?: unknown;
+} {
+  return {
+    left: clause.field,
+    operation: clause.op,
+    ...(clause.value !== undefined && { right: clause.value }),
+  };
+}
+
+async function scannerFetch(market: string, body: ScannerBody): Promise<TradingViewResponse> {
+  const endpoint = `${BASE_URL}/${market}/scan2?label-product=screener-stock`;
+  const cacheKey = `tradingview:scan2:${market}:${stableStringify(body)}`;
+  const cached = cache.get<TradingViewResponse>(cacheKey);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("tradingview");
+    const response = await httpPost<TradingViewResponse>(endpoint, body, {
+      headers: {
+        Origin: "https://www.tradingview.com",
+        Referer: "https://www.tradingview.com/",
+        "User-Agent": BROWSER_UA,
+      },
+    });
+    cache.set(cacheKey, response, TTL.SCREENER);
+    return response;
+  } catch (error) {
+    const stale = cache.getStale<TradingViewResponse>(cacheKey, STALE_LIMIT.SCREENER);
+    if (stale) return stale.value;
+    throw error;
+  }
+}
+
+function decodeScannerRows(
+  response: TradingViewResponse,
+  requestedColumns: readonly string[],
+): DecodedRow[] {
+  const fields = response.fields ?? [];
+  return (response.symbols ?? []).map((row) => {
+    const values: Record<string, unknown | null> = {};
+    for (const column of requestedColumns) {
+      const index = fields.indexOf(column);
+      values[column] = index >= 0 ? (row.f[index] ?? null) : null;
+    }
+    return {
+      tvSymbol: row.s,
+      symbol: stringValue(values.name) ?? symbolFromTvSymbol(row.s),
+      values,
+    };
+  });
+}
+
+function rowToQuote(requestedSymbol: string, row: DecodedRow): TradingViewQuote | undefined {
+  const price = numberValue(row.values.close);
+  if (price === undefined) return undefined;
+  const changePercent = numberValue(row.values.change) ?? 0;
+  const changeAbs = numberValue(row.values.change_abs) ?? 0;
+  return {
+    requestedSymbol,
+    tvSymbol: row.tvSymbol,
+    symbol: row.symbol,
+    price,
+    change: changeAbs,
+    changePercent,
+    volume: numberValue(row.values.volume) ?? 0,
+    exchange: stringValue(row.values.exchange),
+    market: stringValue(row.values.market),
+    name: stringValue(row.values.description),
+    type: stringValue(row.values.type),
+    typespecs: row.values.typespecs ?? undefined,
+    sourceProvider: "tradingview",
+    dataCaveat: DATA_CAVEAT,
+  };
+}
+
+function groupRowsByName(rows: DecodedRow[]): Map<string, DecodedRow[]> {
+  const grouped = new Map<string, DecodedRow[]>();
+  for (const row of rows) {
+    const name = row.symbol.toUpperCase();
+    grouped.set(name, [...(grouped.get(name) ?? []), row]);
+  }
+  return grouped;
+}
+
+function pickPrimaryListing(rows: DecodedRow[]): DecodedRow | undefined {
+  if (rows.length === 0) return undefined;
+  return [...rows].sort(compareListings)[0];
+}
+
+function compareListings(a: DecodedRow, b: DecodedRow): number {
+  return (
+    scoreMarket(a) - scoreMarket(b) ||
+    scoreType(a) - scoreType(b) ||
+    scoreExchange(a) - scoreExchange(b) ||
+    a.tvSymbol.localeCompare(b.tvSymbol)
+  );
+}
+
+function scoreMarket(row: DecodedRow): number {
+  return stringValue(row.values.market)?.toLowerCase() === "america" ? 0 : 1;
+}
+
+function scoreType(row: DecodedRow): number {
+  const type = stringValue(row.values.type)?.toLowerCase();
+  if (type === "stock") return 0;
+  if (type === "fund") return 1;
+  if (type === "dr") return 2;
+  return 3;
+}
+
+function scoreExchange(row: DecodedRow): number {
+  const exchange = stringValue(row.values.exchange)?.toUpperCase();
+  if (exchange === "NASDAQ") return 0;
+  if (exchange === "NYSE") return 1;
+  if (exchange === "AMEX") return 2;
+  return 3;
+}
+
+function clampLimit(limit: number | undefined): number {
+  if (limit == null) return 50;
+  if (limit < 1) return 50;
+  return Math.min(Math.floor(limit), 500);
+}
+
+function normalizeRequestedSymbol(symbol: string): string {
+  return symbol.trim().toUpperCase();
+}
+
+export function isTradingViewQualified(symbol: string): boolean {
+  return /^[A-Z0-9_]+:[A-Z0-9._-]+$/.test(symbol);
+}
+
+export function shouldSkipTradingView(symbol: string): boolean {
+  return /(?:-USD|\.(?:TO|DE|T|L|HK))$/i.test(symbol);
+}
+
+export function canUseTradingViewQuote(symbol: string): boolean {
+  const normalized = normalizeRequestedSymbol(symbol);
+  return (
+    normalized.length > 0 &&
+    (isTradingViewQualified(normalized) || !shouldSkipTradingView(normalized))
+  );
+}
+
+function symbolFromTvSymbol(tvSymbol: string): string {
+  return tvSymbol.includes(":") ? tvSymbol.split(":").at(-1)! : tvSymbol;
+}
+
+function stringValue(value: unknown): string | undefined {
+  return typeof value === "string" ? value : undefined;
+}
+
+function numberValue(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function stableStringify(value: unknown): string {
+  return JSON.stringify(sortObjectKeys(value));
+}
+
+function sortObjectKeys(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(sortObjectKeys);
+  if (!value || typeof value !== "object") return value;
+  return Object.fromEntries(
+    Object.entries(value)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([key, nested]) => [key, sortObjectKeys(nested)]),
+  );
+}

package/src/providers/twitter-cli.ts

@@ -0,0 +1,233 @@
+import { spawn } from "node:child_process";
+import type { TwitterTweet } from "../types/sentiment.js";
+import { ExternalToolError, ExternalToolNotInstalled } from "./external-tool-error.js";
+
+const TWITTER_CLI_BINARY = "twitter";
+const TWITTER_CLI_TOOL_NAME = "twitter-cli";
+const TWITTER_CLI_INSTALL_CMD = "uv tool install twitter-cli";
+const COMMAND_TIMEOUT_MS = 20_000;
+const MAX_OUTPUT_CHARS = 2_000_000;
+
+export interface RawTweet {
+  readonly id?: string;
+  readonly text?: string;
+  readonly author?: {
+    readonly username?: string;
+    readonly screenName?: string;
+    readonly name?: string;
+  };
+  readonly username?: string;
+  readonly url?: string;
+  readonly permanentUrl?: string;
+  readonly createdAt?: string | number;
+  readonly created_at?: string | number;
+  readonly likeCount?: number;
+  readonly likes?: number;
+  readonly retweetCount?: number;
+  readonly retweets?: number;
+  readonly replyCount?: number;
+  readonly replies?: number;
+  readonly viewCount?: number | null;
+  readonly views?: number | null;
+  readonly metrics?: {
+    readonly likes?: number;
+    readonly retweets?: number;
+    readonly replies?: number;
+    readonly views?: number | null;
+  };
+}
+
+interface TwitterCliEnvelope<T> {
+  readonly ok: boolean;
+  readonly schema_version: string;
+  readonly data: T;
+  readonly error?: {
+    readonly code?: string;
+    readonly message?: string;
+  };
+}
+
+interface CommandResult {
+  readonly code: number | null;
+  readonly stdout: string;
+  readonly stderr: string;
+}
+
+type TwitterCliCommandRunner = (command: string, args: readonly string[]) => Promise<CommandResult>;
+
+let commandRunner: TwitterCliCommandRunner = runCommand;
+
+export function setTwitterCliCommandRunnerForTests(runner: TwitterCliCommandRunner): void {
+  commandRunner = runner;
+}
+
+export function resetTwitterCliCommandRunnerForTests(): void {
+  commandRunner = runCommand;
+}
+
+export async function searchTweets(query: string, max = 20): Promise<TwitterTweet[]> {
+  const envelope = await runTwitterCli<TwitterCliEnvelope<RawTweet[]>>([
+    "search",
+    query,
+    "--max",
+    String(max),
+    "-t",
+    "Latest",
+    "--json",
+  ]);
+
+  if (!envelope.ok) {
+    throw new ExternalToolError(
+      TWITTER_CLI_TOOL_NAME,
+      redactSensitiveOutput(envelope.error?.message ?? "twitter-cli returned an error"),
+      envelope.error?.code,
+    );
+  }
+  if (!Array.isArray(envelope.data)) {
+    throw new ExternalToolError(TWITTER_CLI_TOOL_NAME, "twitter-cli returned invalid tweet data");
+  }
+  return envelope.data.map(adaptRawTweet);
+}
+
+async function runTwitterCli<T>(args: readonly string[]): Promise<T> {
+  let result: CommandResult;
+  try {
+    result = await commandRunner(TWITTER_CLI_BINARY, args);
+  } catch (err) {
+    const nodeError = err as NodeJS.ErrnoException;
+    if (nodeError.code === "ENOENT") {
+      throw new ExternalToolNotInstalled(TWITTER_CLI_TOOL_NAME, TWITTER_CLI_INSTALL_CMD);
+    }
+    throw new ExternalToolError(
+      TWITTER_CLI_TOOL_NAME,
+      redactSensitiveOutput(err instanceof Error ? err.message : String(err)),
+    );
+  }
+
+  if (result.code !== 0) {
+    const envelopeError = parseCliErrorEnvelope(result.stdout);
+    if (envelopeError) {
+      throw new ExternalToolError(
+        TWITTER_CLI_TOOL_NAME,
+        redactSensitiveOutput(envelopeError.message),
+        envelopeError.code,
+      );
+    }
+    throw new ExternalToolError(
+      TWITTER_CLI_TOOL_NAME,
+      redactSensitiveOutput(result.stderr.trim() || `twitter-cli exited with code ${result.code}`),
+    );
+  }
+
+  try {
+    return JSON.parse(result.stdout) as T;
+  } catch {
+    throw new ExternalToolError(
+      TWITTER_CLI_TOOL_NAME,
+      `twitter-cli returned non-JSON output: ${redactSensitiveOutput(result.stdout.slice(0, 200))}`,
+    );
+  }
+}
+
+function parseCliErrorEnvelope(stdout: string): { code?: string; message: string } | null {
+  try {
+    const parsed = JSON.parse(stdout) as {
+      ok?: unknown;
+      error?: { code?: unknown; message?: unknown };
+    };
+    if (parsed.ok !== false || typeof parsed.error?.message !== "string") return null;
+    return {
+      code: typeof parsed.error.code === "string" ? parsed.error.code : undefined,
+      message: parsed.error.message,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function adaptRawTweet(raw: RawTweet): TwitterTweet {
+  return {
+    id: stringValue(raw.id),
+    text: stringValue(raw.text).slice(0, 280),
+    author:
+      stringValue(raw.author?.username) ||
+      stringValue(raw.author?.screenName) ||
+      stringValue(raw.username) ||
+      "unknown",
+    likes: numberValue(raw.metrics?.likes ?? raw.likeCount ?? raw.likes),
+    retweets: numberValue(raw.metrics?.retweets ?? raw.retweetCount ?? raw.retweets),
+    replies: numberValue(raw.metrics?.replies ?? raw.replyCount ?? raw.replies),
+    views: nullableNumberValue(raw.metrics?.views ?? raw.viewCount ?? raw.views),
+    url: stringValue(raw.url ?? raw.permanentUrl),
+    created: normalizeCreatedAt(raw.createdAt ?? raw.created_at),
+  };
+}
+
+function normalizeCreatedAt(value: string | number | undefined): string {
+  if (typeof value === "number") {
+    const millis = value > 1_000_000_000_000 ? value : value * 1000;
+    return new Date(millis).toISOString();
+  }
+  if (typeof value === "string" && value.length > 0) {
+    const millis = Date.parse(value);
+    if (!Number.isNaN(millis)) return new Date(millis).toISOString();
+  }
+  return new Date(0).toISOString();
+}
+
+function stringValue(value: unknown): string {
+  return typeof value === "string" ? value : "";
+}
+
+function numberValue(value: unknown): number {
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+function nullableNumberValue(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+export function redactSensitiveOutput(input: string): string {
+  const xCookieNames =
+    "auth_token|ct0|twid|kdt|guest_id|guest_id_ads|guest_id_marketing|personalization_id";
+  return input
+    .slice(0, MAX_OUTPUT_CHARS)
+    .replace(/\b(cookie|set-cookie)\s*:\s*[^\r\n]+/gi, "$1: [redacted]")
+    .replace(new RegExp(`\\b(${xCookieNames})\\b\\s*[:=]\\s*[^;\\s,)]+`, "gi"), "$1=[redacted]")
+    .replace(new RegExp(`\\b(${xCookieNames})=([^;\\s,)]+)`, "gi"), "$1=[redacted]");
+}
+
+function runCommand(command: string, args: readonly string[]): Promise<CommandResult> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+
+    const timeout = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      child.kill("SIGTERM");
+      reject(new Error(`${command} timed out after ${COMMAND_TIMEOUT_MS}ms`));
+    }, COMMAND_TIMEOUT_MS);
+
+    child.stdout.on("data", (chunk: Buffer) => {
+      stdout = (stdout + chunk.toString("utf8")).slice(0, MAX_OUTPUT_CHARS);
+    });
+    child.stderr.on("data", (chunk: Buffer) => {
+      stderr = (stderr + chunk.toString("utf8")).slice(0, MAX_OUTPUT_CHARS);
+    });
+    child.on("error", (err) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      reject(err);
+    });
+    child.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      resolve({ code, stdout, stderr });
+    });
+  });
+}