opencandle 0.3.0 → 0.5.0

package/src/providers/reddit.ts

@@ -0,0 +1,151 @@
+import { httpGet } from "../infra/http-client.js";
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import type { RedditSentimentResult } from "../types/sentiment.js";
+import { BULLISH_TERMS, BEARISH_TERMS } from "../sentiment/keywords.js";
+
+interface RedditListingResponse {
+  data: {
+    children: Array<{
+      data: {
+        id: string;
+        title: string;
+        selftext: string;
+        author: string;
+        score: number;
+        num_comments: number;
+        permalink: string;
+        created_utc: number;
+      };
+    }>;
+  };
+}
+
+const REDDIT_HEADERS = { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" };
+
+export async function getSubredditPosts(
+  subreddit: string,
+  limit: number = 25,
+): Promise<RedditSentimentResult> {
+  const cacheKey = `reddit:${subreddit}:${limit}`;
+  const cached = cache.get<RedditSentimentResult>(cacheKey);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("reddit");
+    const url = `https://www.reddit.com/r/${encodeURIComponent(subreddit)}/hot.json?limit=${limit}`;
+    const data = await httpGet<RedditListingResponse>(url, {
+      headers: REDDIT_HEADERS,
+    });
+
+    const posts = data.data.children.map((child) => ({
+      id: child.data.id,
+      title: child.data.title,
+      selftext: child.data.selftext ?? "",
+      author: child.data.author ?? "unknown",
+      score: child.data.score,
+      comments: child.data.num_comments,
+      url: `https://reddit.com${child.data.permalink}`,
+      created: new Date(child.data.created_utc * 1000).toISOString(),
+    }));
+
+    // Extract ticker-like mentions ($AAPL, $TSLA, etc.)
+    const tickerRegex = /\$([A-Z]{1,5})\b/g;
+    const mentionCounts = new Map<string, number>();
+    for (const post of posts) {
+      for (const match of post.title.matchAll(tickerRegex)) {
+        const ticker = match[1];
+        mentionCounts.set(ticker, (mentionCounts.get(ticker) ?? 0) + 1);
+      }
+    }
+    const topMentions = [...mentionCounts.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10)
+      .map(([ticker]) => ticker);
+
+    const sentiment = scoreSentiment(posts);
+
+    const result: RedditSentimentResult = {
+      subreddit,
+      postCount: posts.length,
+      posts,
+      topMentions,
+      sentimentScore: sentiment.score,
+      bullishCount: sentiment.bullish,
+      bearishCount: sentiment.bearish,
+      fetchedAt: new Date().toISOString(),
+    };
+
+    cache.set(cacheKey, result, TTL.SENTIMENT);
+    return result;
+  } catch (error) {
+    const stale = cache.getStale<RedditSentimentResult>(cacheKey, STALE_LIMIT.SENTIMENT);
+    if (stale) return stale.value;
+    throw error;
+  }
+}
+
+// ── Comment fetching ────────────────────────────────────
+
+export interface RedditComment {
+  id: string;
+  body: string;
+  author: string;
+  score: number;
+  permalink: string;
+}
+
+const COMMENT_TTL = 30 * 60 * 1000; // 30 minutes
+
+export async function getPostComments(
+  subreddit: string,
+  postId: string,
+  limit: number = 5,
+): Promise<RedditComment[]> {
+  const cacheKey = `reddit:comments:${subreddit}:${postId}:${limit}`;
+  const cached = cache.get<RedditComment[]>(cacheKey);
+  if (cached) return cached;
+
+  await rateLimiter.acquire("reddit_comments");
+  const url = `https://www.reddit.com/r/${encodeURIComponent(subreddit)}/comments/${postId}.json`;
+  const data = await httpGet<Array<{ data: { children: Array<{ kind: string; data: { id: string; body?: string; author?: string; score?: number; permalink?: string } }> } }>>(url, {
+    headers: REDDIT_HEADERS,
+  });
+
+  // Comments are in the second listing element
+  const commentListing = data[1]?.data?.children ?? [];
+  const comments: RedditComment[] = commentListing
+    .filter((c) => c.kind === "t1" && c.data.body)
+    .sort((a, b) => (b.data.score ?? 0) - (a.data.score ?? 0))
+    .slice(0, limit)
+    .map((c) => ({
+      id: c.data.id,
+      body: c.data.body!,
+      author: c.data.author ?? "unknown",
+      score: c.data.score ?? 0,
+      permalink: `https://reddit.com${c.data.permalink ?? ""}`,
+    }));
+
+  cache.set(cacheKey, comments, COMMENT_TTL);
+  return comments;
+}
+
+// ── Sentiment scoring ───────────────────────────────────
+
+export function scoreSentiment(
+  posts: Array<{ title: string }>,
+): { score: number; bullish: number; bearish: number } {
+  let bullish = 0;
+  let bearish = 0;
+  for (const post of posts) {
+    const lower = post.title.toLowerCase();
+    bullish += BULLISH_TERMS.filter((t) => lower.includes(t)).length;
+    bearish += BEARISH_TERMS.filter((t) => lower.includes(t)).length;
+  }
+  const total = bullish + bearish;
+  return {
+    score: total === 0 ? 0 : (bullish - bearish) / total,
+    bullish,
+    bearish,
+  };
+}

package/src/providers/sec-edgar.ts

@@ -0,0 +1,312 @@
+import { httpGet } from "../infra/http-client.js";
+import { cache, TTL } from "../infra/cache.js";
+
+const EFTS_BASE = "https://efts.sec.gov/LATEST/search-index";
+const COMPANY_TICKERS_URL = "https://www.sec.gov/files/company_tickers.json";
+const SUBMISSIONS_BASE = "https://data.sec.gov/submissions";
+
+export interface SECFiling {
+  formType: string;
+  filedDate: string;
+  periodOfReport: string;
+  entityName: string;
+  accessionNumber: string;
+  url: string;
+  primaryDocumentUrl?: string;
+  items?: string[];
+  evidenceSnippets?: string[];
+}
+
+export interface SearchFilingsOptions {
+  includeSnippets?: boolean;
+  snippetLimitPerFiling?: number;
+}
+
+interface EFTSResponse {
+  hits: {
+    hits: Array<{
+      _id: string;
+      _source: {
+        file_date: string;
+        form: string;
+        adsh: string;
+        display_names: string[];
+        period_ending: string;
+        ciks: string[];
+        items?: string[];
+      };
+    }>;
+  };
+}
+
+interface CompanyTickerEntry {
+  cik_str: number;
+  ticker: string;
+  title: string;
+}
+
+interface SubmissionsResponse {
+  name: string;
+  tickers?: string[];
+  filings: {
+    recent: {
+      accessionNumber: string[];
+      filingDate: string[];
+      reportDate: string[];
+      form: string[];
+      primaryDocument: string[];
+      items?: string[];
+    };
+  };
+}
+
+export async function searchFilings(
+  ticker: string,
+  formTypes: string[] = ["10-K", "10-Q", "8-K"],
+  limit: number = 10,
+  options: SearchFilingsOptions = {},
+): Promise<SECFiling[]> {
+  const cacheKey = `sec:${ticker}:${formTypes.join(",")}:${limit}:${options.includeSnippets ? "snippets" : "metadata"}`;
+  const cached = cache.get<SECFiling[]>(cacheKey);
+  if (cached) return cached;
+
+  if (options.includeSnippets) {
+    const submissions = await searchFilingsFromCompanySubmissions(ticker, formTypes, limit).catch(() => []);
+    if (submissions.length > 0) {
+      await enrichWithEvidenceSnippets(submissions, options.snippetLimitPerFiling ?? 3);
+      cache.set(cacheKey, submissions, TTL.FUNDAMENTALS);
+      return submissions;
+    }
+  }
+
+  const params = new URLSearchParams({
+    q: ticker,
+    forms: formTypes.join(","),
+    dateRange: "custom",
+    startdt: getDateYearsAgo(3),
+    enddt: new Date().toISOString().split("T")[0],
+    from: "0",
+    size: String(limit),
+  });
+
+  const url = `${EFTS_BASE}?${params}`;
+  const data = await httpGet<EFTSResponse>(url, {
+    headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+  });
+
+  // Deduplicate by accession number (EDGAR returns multiple hits per filing)
+  const seen = new Set<string>();
+  const filings: SECFiling[] = [];
+
+  for (const hit of data.hits?.hits ?? []) {
+    const src = hit._source;
+    const accession = src.adsh;
+    if (!accession || seen.has(accession)) continue;
+    if (!matchesTicker(src.display_names, ticker)) continue;
+    seen.add(accession);
+
+    const cik = src.ciks?.[0] ?? "";
+    const displayName = src.display_names?.[0] ?? "";
+    // Extract entity name from display format: "APPLE INC  (AAPL)  (CIK 0000320193)"
+    const entityName = displayName.split("(")[0]?.trim() ?? displayName;
+    const archiveDir = buildEdgarArchiveDir(cik, accession);
+    const primaryDocumentUrl = buildPrimaryDocumentUrl(archiveDir, hit._id);
+
+    filings.push({
+      formType: src.form ?? "",
+      filedDate: src.file_date ?? "",
+      periodOfReport: src.period_ending ?? "",
+      entityName,
+      accessionNumber: accession,
+      url: `${archiveDir}/${accession}-index.htm`,
+      primaryDocumentUrl,
+      items: src.items ?? [],
+    });
+
+    if (filings.length >= limit) break;
+  }
+
+  if (options.includeSnippets) {
+    await enrichWithEvidenceSnippets(filings, options.snippetLimitPerFiling ?? 3);
+  }
+
+  cache.set(cacheKey, filings, TTL.FUNDAMENTALS);
+  return filings;
+}
+
+async function searchFilingsFromCompanySubmissions(
+  ticker: string,
+  formTypes: string[],
+  limit: number,
+): Promise<SECFiling[]> {
+  const company = await resolveCompanyTicker(ticker);
+  if (!company) return [];
+
+  const cik = String(company.cik_str).padStart(10, "0");
+  const data = await httpGet<SubmissionsResponse>(`${SUBMISSIONS_BASE}/CIK${cik}.json`, {
+    headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+  });
+
+  const filings: SECFiling[] = [];
+  const recent = data.filings.recent;
+  for (let i = 0; i < recent.form.length && filings.length < limit; i++) {
+    const formType = recent.form[i] ?? "";
+    if (!formTypes.includes(formType)) continue;
+    const accession = recent.accessionNumber[i];
+    if (!accession) continue;
+    const archiveDir = buildEdgarArchiveDir(cik, accession);
+    const primaryDocument = recent.primaryDocument[i];
+    filings.push({
+      formType,
+      filedDate: recent.filingDate[i] ?? "",
+      periodOfReport: recent.reportDate[i] ?? "",
+      entityName: data.name || company.title,
+      accessionNumber: accession,
+      url: `${archiveDir}/${accession}-index.htm`,
+      primaryDocumentUrl: primaryDocument ? `${archiveDir}/${primaryDocument}` : undefined,
+      items: splitFilingItems(recent.items?.[i]),
+    });
+  }
+  return filings;
+}
+
+async function resolveCompanyTicker(ticker: string): Promise<CompanyTickerEntry | undefined> {
+  const cacheKey = "sec:company-tickers";
+  let companies = cache.get<CompanyTickerEntry[]>(cacheKey);
+  if (!companies) {
+    const data = await httpGet<Record<string, CompanyTickerEntry>>(COMPANY_TICKERS_URL, {
+      headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+    });
+    companies = Object.values(data);
+    cache.set(cacheKey, companies, TTL.FUNDAMENTALS);
+  }
+  const normalized = ticker.toUpperCase();
+  return companies.find((company) => company.ticker.toUpperCase() === normalized);
+}
+
+function splitFilingItems(raw: string | undefined): string[] {
+  return raw
+    ? raw.split(",").map((item) => item.trim()).filter(Boolean)
+    : [];
+}
+
+function matchesTicker(displayNames: string[] | undefined, ticker: string): boolean {
+  const normalized = ticker.toUpperCase();
+  return (displayNames ?? []).some((name) => {
+    const tickerGroups = name.match(/\(([^)]*)\)/g) ?? [];
+    return tickerGroups.some((group) =>
+      group
+        .slice(1, -1)
+        .split(",")
+        .map((part) => part.trim().toUpperCase())
+        .includes(normalized),
+    );
+  });
+}
+
+function buildEdgarArchiveDir(cik: string, accession: string): string {
+  const cikNum = cik.replace(/^0+/, "");
+  const accessionNoDash = accession.replace(/-/g, "");
+  return `https://www.sec.gov/Archives/edgar/data/${cikNum}/${accessionNoDash}`;
+}
+
+function buildPrimaryDocumentUrl(archiveDir: string, hitId: string): string | undefined {
+  const fileName = hitId.split(":")[1];
+  if (!fileName || !/\.(?:htm|html|txt)$/i.test(fileName)) return undefined;
+  return `${archiveDir}/${fileName}`;
+}
+
+function getDateYearsAgo(years: number): string {
+  const d = new Date();
+  d.setFullYear(d.getFullYear() - years);
+  return d.toISOString().split("T")[0];
+}
+
+async function enrichWithEvidenceSnippets(filings: SECFiling[], limitPerFiling: number): Promise<void> {
+  await Promise.all(
+    filings.map(async (filing) => {
+      if (!filing.primaryDocumentUrl) return;
+      try {
+        const raw = await fetchText(filing.primaryDocumentUrl);
+        filing.evidenceSnippets = extractEvidenceSnippets(raw, limitPerFiling);
+      } catch {
+        filing.evidenceSnippets = [];
+      }
+    }),
+  );
+}
+
+async function fetchText(url: string): Promise<string> {
+  const response = await fetch(url, {
+    headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+  });
+  if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);
+  return response.text();
+}
+
+const EVIDENCE_KEYWORDS = [
+  "risk factor",
+  "legal proceedings",
+  "litigation",
+  "regulatory",
+  "regulation",
+  "revenue",
+  "concentration",
+  "management's discussion",
+  "management discussion",
+  "liquidity",
+  "outlook",
+  "guidance",
+  "material weakness",
+  "going concern",
+];
+
+function extractEvidenceSnippets(raw: string, limit: number): string[] {
+  const text = stripFilingMarkup(raw);
+  const snippets: string[] = [];
+  const lower = text.toLowerCase();
+
+  for (const keyword of EVIDENCE_KEYWORDS) {
+    let searchFrom = 0;
+    while (searchFrom < lower.length) {
+      const index = lower.indexOf(keyword, searchFrom);
+      if (index === -1) break;
+      searchFrom = index + keyword.length;
+
+      const start = Math.max(0, index - 220);
+      const end = Math.min(text.length, index + keyword.length + 420);
+      const snippet = text.slice(start, end).replace(/\s+/g, " ").trim();
+      if (
+        snippet &&
+        !isLikelyTableOfContentsSnippet(snippet) &&
+        !snippets.some((existing) => existing.includes(snippet.slice(0, 80)))
+      ) {
+        snippets.push(snippet);
+        break;
+      }
+    }
+    if (snippets.length >= limit) break;
+  }
+
+  return snippets;
+}
+
+function stripFilingMarkup(raw: string): string {
+  return raw
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&#160;/g, " ")
+    .replace(/&#8217;/g, "'")
+    .replace(/&#8220;|&#8221;/g, "\"")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function isLikelyTableOfContentsSnippet(snippet: string): boolean {
+  const lower = snippet.toLowerCase();
+  const itemHeadingCount = lower.match(/\bitem\s+\d+[a-z]?\b/g)?.length ?? 0;
+  return lower.includes("table of contents") && itemHeadingCount >= 2;
+}

package/src/providers/twitter.ts

@@ -0,0 +1,173 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import Database from "better-sqlite3";
+import { Scraper, SearchMode } from "@the-convocation/twitter-scraper";
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import { getBrowserProfileDir } from "../infra/opencandle-paths.js";
+import type { TwitterSentimentResult, TwitterTweet } from "../types/sentiment.js";
+
+// ── Cookie extraction ────────────────────────────────────
+
+interface FirefoxCookie {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+}
+
+export function readTwitterCookies(profileDir: string): FirefoxCookie[] {
+  const dbPath = join(profileDir, "cookies.sqlite");
+  if (!existsSync(dbPath)) return [];
+
+  const db = new Database(dbPath, { readonly: true });
+  try {
+    const rows = db
+      .prepare(
+        `SELECT name, value, host AS domain, path FROM moz_cookies WHERE host LIKE ? OR host LIKE ?`,
+      )
+      .all("%x.com%", "%twitter.com%") as FirefoxCookie[];
+    return rows;
+  } finally {
+    db.close();
+  }
+}
+
+// ── Sentiment scoring ────────────────────────────────────
+
+import { BULLISH_TERMS, BEARISH_TERMS } from "../sentiment/keywords.js";
+
+export function scoreTwitterSentiment(
+  tweets: Array<{ text: string; likes: number; retweets: number }>,
+): { score: number; bullish: number; bearish: number } {
+  let bullishWeight = 0;
+  let bearishWeight = 0;
+  let bullishCount = 0;
+  let bearishCount = 0;
+
+  for (const tweet of tweets) {
+    const lower = tweet.text.toLowerCase();
+    const engagement = 1 + (tweet.likes ?? 0) + (tweet.retweets ?? 0);
+    const tweetBullish = BULLISH_TERMS.filter((t) => lower.includes(t)).length;
+    const tweetBearish = BEARISH_TERMS.filter((t) => lower.includes(t)).length;
+
+    bullishCount += tweetBullish;
+    bearishCount += tweetBearish;
+    bullishWeight += tweetBullish * engagement;
+    bearishWeight += tweetBearish * engagement;
+  }
+
+  const totalWeight = bullishWeight + bearishWeight;
+  return {
+    score: totalWeight === 0 ? 0 : (bullishWeight - bearishWeight) / totalWeight,
+    bullish: bullishCount,
+    bearish: bearishCount,
+  };
+}
+
+// ── Query normalization ──────────────────────────────────
+
+export function normalizeQuery(query: string): string {
+  if (/^[A-Z]{1,5}$/.test(query)) return "$" + query;
+  return query;
+}
+
+// ── Main provider function ───────────────────────────────
+
+export async function getTwitterSentiment(
+  query: string,
+  limit: number = 50,
+  hours: number = 24,
+): Promise<TwitterSentimentResult> {
+  const normalizedQuery = normalizeQuery(query);
+  const cacheKey = `twitter:${normalizedQuery}:${limit}:${hours}`;
+  const cached = cache.get<TwitterSentimentResult>(cacheKey);
+  if (cached) return cached;
+
+  await rateLimiter.acquire("twitter");
+
+  try {
+    const profileDir = getBrowserProfileDir();
+    const cookies = readTwitterCookies(profileDir);
+
+    const authToken = cookies.find((c) => c.name === "auth_token");
+    const ct0 = cookies.find((c) => c.name === "ct0");
+
+    if (!authToken || !ct0) {
+      throw new Error("No Twitter session found.");
+    }
+
+    const scraper = new Scraper();
+    const cookieStrings = cookies.map(
+      (c) => `${c.name}=${c.value}; Domain=${c.domain}; Path=${c.path}`,
+    );
+    await scraper.setCookies(cookieStrings);
+
+    const loggedIn = await scraper.isLoggedIn();
+    if (!loggedIn) {
+      throw new Error("Twitter session expired.");
+    }
+
+    const cutoff = new Date(Date.now() - hours * 3_600_000);
+    const tweets: TwitterTweet[] = [];
+    const results = scraper.searchTweets(normalizedQuery, limit, SearchMode.Latest);
+
+    for await (const tweet of results) {
+      const created = tweet.timeParsed ?? new Date(0);
+      if (created < cutoff) continue;
+
+      tweets.push({
+        id: tweet.id ?? "",
+        text: tweet.text?.slice(0, 280) ?? "",
+        author: tweet.username ?? "unknown",
+        likes: tweet.likes ?? 0,
+        retweets: tweet.retweets ?? 0,
+        replies: tweet.replies ?? 0,
+        views: tweet.views ?? null,
+        url: tweet.permanentUrl ?? "",
+        created: created.toISOString(),
+      });
+
+      if (tweets.length >= limit) break;
+    }
+
+    // Extract co-mentioned cashtags
+    const tickerRegex = /\$([A-Z]{1,5})\b/g;
+    const mentionCounts = new Map<string, number>();
+    // Exclude the searched ticker itself from co-mentions
+    const searchedTicker = normalizedQuery.startsWith("$")
+      ? normalizedQuery.slice(1)
+      : null;
+    for (const t of tweets) {
+      for (const match of t.text.matchAll(tickerRegex)) {
+        const ticker = match[1];
+        if (ticker === searchedTicker) continue;
+        mentionCounts.set(ticker, (mentionCounts.get(ticker) ?? 0) + 1);
+      }
+    }
+    const topMentions = [...mentionCounts.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10)
+      .map(([ticker]) => ticker);
+
+    const sentiment = scoreTwitterSentiment(tweets);
+
+    const result: TwitterSentimentResult = {
+      query: normalizedQuery,
+      tweetCount: tweets.length,
+      tweets,
+      sentimentScore: sentiment.score,
+      bullishCount: sentiment.bullish,
+      bearishCount: sentiment.bearish,
+      topMentions,
+      fetchedAt: new Date().toISOString(),
+    };
+
+    cache.set(cacheKey, result, TTL.SENTIMENT);
+    return result;
+  } catch (error) {
+    const stale = cache.getStale<TwitterSentimentResult>(cacheKey, STALE_LIMIT.SENTIMENT);
+    if (stale) return stale.value;
+    throw error;
+  }
+}