opencandle 0.3.0 → 0.4.0

package/src/providers/reddit.ts

@@ -0,0 +1,151 @@
+import { httpGet } from "../infra/http-client.js";
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import type { RedditSentimentResult } from "../types/sentiment.js";
+import { BULLISH_TERMS, BEARISH_TERMS } from "../sentiment/keywords.js";
+
+interface RedditListingResponse {
+  data: {
+    children: Array<{
+      data: {
+        id: string;
+        title: string;
+        selftext: string;
+        author: string;
+        score: number;
+        num_comments: number;
+        permalink: string;
+        created_utc: number;
+      };
+    }>;
+  };
+}
+
+const REDDIT_HEADERS = { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" };
+
+export async function getSubredditPosts(
+  subreddit: string,
+  limit: number = 25,
+): Promise<RedditSentimentResult> {
+  const cacheKey = `reddit:${subreddit}:${limit}`;
+  const cached = cache.get<RedditSentimentResult>(cacheKey);
+  if (cached) return cached;
+
+  try {
+    await rateLimiter.acquire("reddit");
+    const url = `https://www.reddit.com/r/${encodeURIComponent(subreddit)}/hot.json?limit=${limit}`;
+    const data = await httpGet<RedditListingResponse>(url, {
+      headers: REDDIT_HEADERS,
+    });
+
+    const posts = data.data.children.map((child) => ({
+      id: child.data.id,
+      title: child.data.title,
+      selftext: child.data.selftext ?? "",
+      author: child.data.author ?? "unknown",
+      score: child.data.score,
+      comments: child.data.num_comments,
+      url: `https://reddit.com${child.data.permalink}`,
+      created: new Date(child.data.created_utc * 1000).toISOString(),
+    }));
+
+    // Extract ticker-like mentions ($AAPL, $TSLA, etc.)
+    const tickerRegex = /\$([A-Z]{1,5})\b/g;
+    const mentionCounts = new Map<string, number>();
+    for (const post of posts) {
+      for (const match of post.title.matchAll(tickerRegex)) {
+        const ticker = match[1];
+        mentionCounts.set(ticker, (mentionCounts.get(ticker) ?? 0) + 1);
+      }
+    }
+    const topMentions = [...mentionCounts.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10)
+      .map(([ticker]) => ticker);
+
+    const sentiment = scoreSentiment(posts);
+
+    const result: RedditSentimentResult = {
+      subreddit,
+      postCount: posts.length,
+      posts,
+      topMentions,
+      sentimentScore: sentiment.score,
+      bullishCount: sentiment.bullish,
+      bearishCount: sentiment.bearish,
+      fetchedAt: new Date().toISOString(),
+    };
+
+    cache.set(cacheKey, result, TTL.SENTIMENT);
+    return result;
+  } catch (error) {
+    const stale = cache.getStale<RedditSentimentResult>(cacheKey, STALE_LIMIT.SENTIMENT);
+    if (stale) return stale.value;
+    throw error;
+  }
+}
+
+// ── Comment fetching ────────────────────────────────────
+
+export interface RedditComment {
+  id: string;
+  body: string;
+  author: string;
+  score: number;
+  permalink: string;
+}
+
+const COMMENT_TTL = 30 * 60 * 1000; // 30 minutes
+
+export async function getPostComments(
+  subreddit: string,
+  postId: string,
+  limit: number = 5,
+): Promise<RedditComment[]> {
+  const cacheKey = `reddit:comments:${subreddit}:${postId}:${limit}`;
+  const cached = cache.get<RedditComment[]>(cacheKey);
+  if (cached) return cached;
+
+  await rateLimiter.acquire("reddit_comments");
+  const url = `https://www.reddit.com/r/${encodeURIComponent(subreddit)}/comments/${postId}.json`;
+  const data = await httpGet<Array<{ data: { children: Array<{ kind: string; data: { id: string; body?: string; author?: string; score?: number; permalink?: string } }> } }>>(url, {
+    headers: REDDIT_HEADERS,
+  });
+
+  // Comments are in the second listing element
+  const commentListing = data[1]?.data?.children ?? [];
+  const comments: RedditComment[] = commentListing
+    .filter((c) => c.kind === "t1" && c.data.body)
+    .sort((a, b) => (b.data.score ?? 0) - (a.data.score ?? 0))
+    .slice(0, limit)
+    .map((c) => ({
+      id: c.data.id,
+      body: c.data.body!,
+      author: c.data.author ?? "unknown",
+      score: c.data.score ?? 0,
+      permalink: `https://reddit.com${c.data.permalink ?? ""}`,
+    }));
+
+  cache.set(cacheKey, comments, COMMENT_TTL);
+  return comments;
+}
+
+// ── Sentiment scoring ───────────────────────────────────
+
+export function scoreSentiment(
+  posts: Array<{ title: string }>,
+): { score: number; bullish: number; bearish: number } {
+  let bullish = 0;
+  let bearish = 0;
+  for (const post of posts) {
+    const lower = post.title.toLowerCase();
+    bullish += BULLISH_TERMS.filter((t) => lower.includes(t)).length;
+    bearish += BEARISH_TERMS.filter((t) => lower.includes(t)).length;
+  }
+  const total = bullish + bearish;
+  return {
+    score: total === 0 ? 0 : (bullish - bearish) / total,
+    bullish,
+    bearish,
+  };
+}

package/src/providers/sec-edgar.ts

@@ -0,0 +1,96 @@
+import { httpGet } from "../infra/http-client.js";
+import { cache, TTL } from "../infra/cache.js";
+
+const EFTS_BASE = "https://efts.sec.gov/LATEST/search-index";
+
+export interface SECFiling {
+  formType: string;
+  filedDate: string;
+  periodOfReport: string;
+  entityName: string;
+  accessionNumber: string;
+  url: string;
+}
+
+interface EFTSResponse {
+  hits: {
+    hits: Array<{
+      _id: string;
+      _source: {
+        file_date: string;
+        form: string;
+        adsh: string;
+        display_names: string[];
+        period_ending: string;
+        ciks: string[];
+      };
+    }>;
+  };
+}
+
+export async function searchFilings(
+  ticker: string,
+  formTypes: string[] = ["10-K", "10-Q", "8-K"],
+  limit: number = 10,
+): Promise<SECFiling[]> {
+  const cacheKey = `sec:${ticker}:${formTypes.join(",")}:${limit}`;
+  const cached = cache.get<SECFiling[]>(cacheKey);
+  if (cached) return cached;
+
+  const params = new URLSearchParams({
+    q: ticker,
+    forms: formTypes.join(","),
+    dateRange: "custom",
+    startdt: getDateYearsAgo(3),
+    enddt: new Date().toISOString().split("T")[0],
+    from: "0",
+    size: String(limit),
+  });
+
+  const url = `${EFTS_BASE}?${params}`;
+  const data = await httpGet<EFTSResponse>(url, {
+    headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+  });
+
+  // Deduplicate by accession number (EDGAR returns multiple hits per filing)
+  const seen = new Set<string>();
+  const filings: SECFiling[] = [];
+
+  for (const hit of data.hits?.hits ?? []) {
+    const src = hit._source;
+    const accession = src.adsh;
+    if (!accession || seen.has(accession)) continue;
+    seen.add(accession);
+
+    const cik = src.ciks?.[0] ?? "";
+    const displayName = src.display_names?.[0] ?? "";
+    // Extract entity name from display format: "APPLE INC  (AAPL)  (CIK 0000320193)"
+    const entityName = displayName.split("(")[0]?.trim() ?? displayName;
+
+    filings.push({
+      formType: src.form ?? "",
+      filedDate: src.file_date ?? "",
+      periodOfReport: src.period_ending ?? "",
+      entityName,
+      accessionNumber: accession,
+      url: buildEdgarUrl(cik, accession),
+    });
+
+    if (filings.length >= limit) break;
+  }
+
+  cache.set(cacheKey, filings, TTL.FUNDAMENTALS);
+  return filings;
+}
+
+function buildEdgarUrl(cik: string, accession: string): string {
+  const cikNum = cik.replace(/^0+/, "");
+  const accessionNoDash = accession.replace(/-/g, "");
+  return `https://www.sec.gov/Archives/edgar/data/${cikNum}/${accessionNoDash}/${accession}-index.htm`;
+}
+
+function getDateYearsAgo(years: number): string {
+  const d = new Date();
+  d.setFullYear(d.getFullYear() - years);
+  return d.toISOString().split("T")[0];
+}

package/src/providers/twitter.ts

@@ -0,0 +1,173 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import Database from "better-sqlite3";
+import { Scraper, SearchMode } from "@the-convocation/twitter-scraper";
+import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import { getBrowserProfileDir } from "../infra/opencandle-paths.js";
+import type { TwitterSentimentResult, TwitterTweet } from "../types/sentiment.js";
+
+// ── Cookie extraction ────────────────────────────────────
+
+interface FirefoxCookie {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+}
+
+export function readTwitterCookies(profileDir: string): FirefoxCookie[] {
+  const dbPath = join(profileDir, "cookies.sqlite");
+  if (!existsSync(dbPath)) return [];
+
+  const db = new Database(dbPath, { readonly: true });
+  try {
+    const rows = db
+      .prepare(
+        `SELECT name, value, host AS domain, path FROM moz_cookies WHERE host LIKE ? OR host LIKE ?`,
+      )
+      .all("%x.com%", "%twitter.com%") as FirefoxCookie[];
+    return rows;
+  } finally {
+    db.close();
+  }
+}
+
+// ── Sentiment scoring ────────────────────────────────────
+
+import { BULLISH_TERMS, BEARISH_TERMS } from "../sentiment/keywords.js";
+
+export function scoreTwitterSentiment(
+  tweets: Array<{ text: string; likes: number; retweets: number }>,
+): { score: number; bullish: number; bearish: number } {
+  let bullishWeight = 0;
+  let bearishWeight = 0;
+  let bullishCount = 0;
+  let bearishCount = 0;
+
+  for (const tweet of tweets) {
+    const lower = tweet.text.toLowerCase();
+    const engagement = 1 + (tweet.likes ?? 0) + (tweet.retweets ?? 0);
+    const tweetBullish = BULLISH_TERMS.filter((t) => lower.includes(t)).length;
+    const tweetBearish = BEARISH_TERMS.filter((t) => lower.includes(t)).length;
+
+    bullishCount += tweetBullish;
+    bearishCount += tweetBearish;
+    bullishWeight += tweetBullish * engagement;
+    bearishWeight += tweetBearish * engagement;
+  }
+
+  const totalWeight = bullishWeight + bearishWeight;
+  return {
+    score: totalWeight === 0 ? 0 : (bullishWeight - bearishWeight) / totalWeight,
+    bullish: bullishCount,
+    bearish: bearishCount,
+  };
+}
+
+// ── Query normalization ──────────────────────────────────
+
+export function normalizeQuery(query: string): string {
+  if (/^[A-Z]{1,5}$/.test(query)) return "$" + query;
+  return query;
+}
+
+// ── Main provider function ───────────────────────────────
+
+export async function getTwitterSentiment(
+  query: string,
+  limit: number = 50,
+  hours: number = 24,
+): Promise<TwitterSentimentResult> {
+  const normalizedQuery = normalizeQuery(query);
+  const cacheKey = `twitter:${normalizedQuery}:${limit}:${hours}`;
+  const cached = cache.get<TwitterSentimentResult>(cacheKey);
+  if (cached) return cached;
+
+  await rateLimiter.acquire("twitter");
+
+  try {
+    const profileDir = getBrowserProfileDir();
+    const cookies = readTwitterCookies(profileDir);
+
+    const authToken = cookies.find((c) => c.name === "auth_token");
+    const ct0 = cookies.find((c) => c.name === "ct0");
+
+    if (!authToken || !ct0) {
+      throw new Error("No Twitter session found.");
+    }
+
+    const scraper = new Scraper();
+    const cookieStrings = cookies.map(
+      (c) => `${c.name}=${c.value}; Domain=${c.domain}; Path=${c.path}`,
+    );
+    await scraper.setCookies(cookieStrings);
+
+    const loggedIn = await scraper.isLoggedIn();
+    if (!loggedIn) {
+      throw new Error("Twitter session expired.");
+    }
+
+    const cutoff = new Date(Date.now() - hours * 3_600_000);
+    const tweets: TwitterTweet[] = [];
+    const results = scraper.searchTweets(normalizedQuery, limit, SearchMode.Latest);
+
+    for await (const tweet of results) {
+      const created = tweet.timeParsed ?? new Date(0);
+      if (created < cutoff) continue;
+
+      tweets.push({
+        id: tweet.id ?? "",
+        text: tweet.text?.slice(0, 280) ?? "",
+        author: tweet.username ?? "unknown",
+        likes: tweet.likes ?? 0,
+        retweets: tweet.retweets ?? 0,
+        replies: tweet.replies ?? 0,
+        views: tweet.views ?? null,
+        url: tweet.permanentUrl ?? "",
+        created: created.toISOString(),
+      });
+
+      if (tweets.length >= limit) break;
+    }
+
+    // Extract co-mentioned cashtags
+    const tickerRegex = /\$([A-Z]{1,5})\b/g;
+    const mentionCounts = new Map<string, number>();
+    // Exclude the searched ticker itself from co-mentions
+    const searchedTicker = normalizedQuery.startsWith("$")
+      ? normalizedQuery.slice(1)
+      : null;
+    for (const t of tweets) {
+      for (const match of t.text.matchAll(tickerRegex)) {
+        const ticker = match[1];
+        if (ticker === searchedTicker) continue;
+        mentionCounts.set(ticker, (mentionCounts.get(ticker) ?? 0) + 1);
+      }
+    }
+    const topMentions = [...mentionCounts.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10)
+      .map(([ticker]) => ticker);
+
+    const sentiment = scoreTwitterSentiment(tweets);
+
+    const result: TwitterSentimentResult = {
+      query: normalizedQuery,
+      tweetCount: tweets.length,
+      tweets,
+      sentimentScore: sentiment.score,
+      bullishCount: sentiment.bullish,
+      bearishCount: sentiment.bearish,
+      topMentions,
+      fetchedAt: new Date().toISOString(),
+    };
+
+    cache.set(cacheKey, result, TTL.SENTIMENT);
+    return result;
+  } catch (error) {
+    const stale = cache.getStale<TwitterSentimentResult>(cacheKey, STALE_LIMIT.SENTIMENT);
+    if (stale) return stale.value;
+    throw error;
+  }
+}