opencandle 0.5.0 → 0.7.0

package/src/providers/reddit-cli.ts

@@ -0,0 +1,317 @@
+import { spawn } from "node:child_process";
+import { ExternalToolError, ExternalToolNotInstalled } from "./external-tool-error.js";
+import type { RedditComment } from "./reddit.js";
+
+const RDT_BINARY = "rdt";
+const RDT_INSTALL_CMD = "uv tool install rdt-cli";
+const COMMAND_TIMEOUT_MS = 20_000;
+const MAX_OUTPUT_CHARS = 2_000_000;
+
+interface RdtEnvelope<T> {
+  readonly ok: boolean;
+  readonly schema_version: string;
+  readonly data?: T;
+  readonly error?: {
+    readonly code?: string;
+    readonly message?: string;
+  };
+}
+
+export interface RdtPost {
+  readonly id: string;
+  readonly title: string;
+  readonly selftext: string;
+  readonly author: string;
+  readonly score: number;
+  readonly comments: number;
+  readonly subreddit?: string;
+  readonly url: string;
+  readonly permalink?: string;
+  readonly created: string;
+}
+
+interface RawRdtPost {
+  readonly id?: unknown;
+  readonly title?: unknown;
+  readonly selftext?: unknown;
+  readonly author?: unknown;
+  readonly score?: unknown;
+  readonly num_comments?: unknown;
+  readonly subreddit?: unknown;
+  readonly url?: unknown;
+  readonly permalink?: unknown;
+  readonly created_utc?: unknown;
+}
+
+interface RawRdtComment {
+  readonly id?: unknown;
+  readonly body?: unknown;
+  readonly author?: unknown;
+  readonly score?: unknown;
+  readonly permalink?: unknown;
+}
+
+interface RdtListingChild<T> {
+  readonly kind?: string;
+  readonly data?: T;
+}
+
+interface RdtListing<T> {
+  readonly data?: {
+    readonly children?: Array<RdtListingChild<T>>;
+  };
+}
+
+interface CommandResult {
+  readonly code: number | null;
+  readonly stdout: string;
+  readonly stderr: string;
+}
+
+type RdtCommandRunner = (command: string, args: readonly string[]) => Promise<CommandResult>;
+
+let commandRunner: RdtCommandRunner = runCommand;
+
+export function setRdtCommandRunnerForTests(runner: RdtCommandRunner): void {
+  commandRunner = runner;
+}
+
+export function resetRdtCommandRunnerForTests(): void {
+  commandRunner = runCommand;
+}
+
+export async function searchRedditPosts(
+  query: string,
+  opts: { subreddit?: string; limit: number },
+): Promise<RdtPost[]> {
+  const args = ["search", query];
+  if (opts.subreddit) {
+    args.push("--subreddit", opts.subreddit);
+  }
+  args.push("--json", "--compact", "-n", String(opts.limit));
+  const data = await runRdt<RawRdtPost[]>(args);
+  if (!Array.isArray(data)) {
+    throw new ExternalToolError(RDT_BINARY, "rdt-cli returned invalid search data");
+  }
+  return data.map(adaptPost).filter((post) => post.id.length > 0);
+}
+
+export async function listSubredditPosts(
+  subreddit: string,
+  opts: { limit: number },
+): Promise<RdtPost[]> {
+  const data = await runRdt<RawRdtPost[]>([
+    "sub",
+    subreddit,
+    "--json",
+    "--compact",
+    "-n",
+    String(opts.limit),
+  ]);
+  if (!Array.isArray(data)) {
+    throw new ExternalToolError(RDT_BINARY, "rdt-cli returned invalid subreddit data");
+  }
+  return data.map((raw) => adaptPost({ ...raw, subreddit })).filter((post) => post.id.length > 0);
+}
+
+export async function readRedditPost(
+  postId: string,
+  opts: { limit: number },
+): Promise<{ post: RdtPost; comments: RedditComment[] }> {
+  const data = await runRdt<Array<RdtListing<RawRdtPost | RawRdtComment>>>([
+    "read",
+    postId,
+    "--json",
+    "-n",
+    String(opts.limit),
+  ]);
+  if (!Array.isArray(data)) {
+    throw new ExternalToolError(RDT_BINARY, "rdt-cli returned invalid read data");
+  }
+
+  const rawPost = data[0]?.data?.children?.find((child) => child.kind === "t3")?.data;
+  if (!rawPost) {
+    throw new ExternalToolError(RDT_BINARY, "rdt-cli read output did not include a post");
+  }
+
+  const post = adaptPost(rawPost as RawRdtPost);
+  const commentListing = data[1]?.data?.children ?? [];
+  const comments = commentListing
+    .filter((child): child is RdtListingChild<RawRdtComment> => child.kind === "t1")
+    .map((child) => adaptComment(child.data, post.permalink))
+    .filter((comment): comment is RedditComment => comment !== null)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, opts.limit);
+
+  return { post, comments };
+}
+
+async function runRdt<T>(args: readonly string[]): Promise<T> {
+  let result: CommandResult;
+  try {
+    result = await commandRunner(RDT_BINARY, args);
+  } catch (err) {
+    const nodeError = err as NodeJS.ErrnoException;
+    if (nodeError.code === "ENOENT") {
+      throw new ExternalToolNotInstalled(RDT_BINARY, RDT_INSTALL_CMD);
+    }
+    throw new ExternalToolError(
+      RDT_BINARY,
+      redactSensitiveOutput(err instanceof Error ? err.message : String(err)),
+    );
+  }
+
+  if (result.code !== 0) {
+    const envelopeError = parseCliErrorEnvelope(result.stdout);
+    if (envelopeError) {
+      throw new ExternalToolError(
+        RDT_BINARY,
+        redactSensitiveOutput(envelopeError.message),
+        envelopeError.code,
+      );
+    }
+    throw new ExternalToolError(
+      RDT_BINARY,
+      redactSensitiveOutput(result.stderr.trim() || `rdt-cli exited with code ${result.code}`),
+    );
+  }
+
+  let envelope: RdtEnvelope<T>;
+  try {
+    envelope = JSON.parse(result.stdout) as RdtEnvelope<T>;
+  } catch {
+    throw new ExternalToolError(
+      RDT_BINARY,
+      `rdt-cli returned non-JSON output: ${redactSensitiveOutput(result.stdout.slice(0, 200))}`,
+    );
+  }
+
+  if (!envelope.ok) {
+    throw new ExternalToolError(
+      RDT_BINARY,
+      redactSensitiveOutput(envelope.error?.message ?? "rdt-cli returned an error"),
+      envelope.error?.code,
+    );
+  }
+  if (envelope.data === undefined) {
+    throw new ExternalToolError(RDT_BINARY, "rdt-cli returned no data");
+  }
+  return envelope.data;
+}
+
+function parseCliErrorEnvelope(stdout: string): { code?: string; message: string } | null {
+  try {
+    const parsed = JSON.parse(stdout) as {
+      ok?: unknown;
+      error?: { code?: unknown; message?: unknown };
+    };
+    if (parsed.ok !== false || typeof parsed.error?.message !== "string") return null;
+    return {
+      code: typeof parsed.error.code === "string" ? parsed.error.code : undefined,
+      message: parsed.error.message,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function adaptPost(raw: RawRdtPost): RdtPost {
+  const permalink = stringValue(raw.permalink);
+  const url = stringValue(raw.url) || (permalink ? `https://reddit.com${permalink}` : "");
+  return {
+    id: stringValue(raw.id),
+    title: stringValue(raw.title),
+    selftext: stringValue(raw.selftext),
+    author: stringValue(raw.author) || "unknown",
+    score: numberValue(raw.score),
+    comments: numberValue(raw.num_comments),
+    subreddit: stringValue(raw.subreddit) || undefined,
+    url,
+    permalink: permalink || undefined,
+    created: normalizeCreatedAt(raw.created_utc),
+  };
+}
+
+function adaptComment(
+  raw: RawRdtComment | undefined,
+  postPermalink?: string,
+): RedditComment | null {
+  if (!raw || typeof raw.body !== "string" || raw.body.length === 0) return null;
+  const permalink = stringValue(raw.permalink) || postPermalink || "";
+  return {
+    id: stringValue(raw.id),
+    body: raw.body,
+    author: stringValue(raw.author) || "unknown",
+    score: numberValue(raw.score),
+    permalink: permalink ? `https://reddit.com${permalink}` : "",
+  };
+}
+
+function normalizeCreatedAt(value: unknown): string {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    const millis = value > 1_000_000_000_000 ? value : value * 1000;
+    return new Date(millis).toISOString();
+  }
+  if (typeof value === "string" && value.length > 0) {
+    const millis = Date.parse(value);
+    if (!Number.isNaN(millis)) return new Date(millis).toISOString();
+  }
+  return new Date(0).toISOString();
+}
+
+function stringValue(value: unknown): string {
+  return typeof value === "string" ? value : "";
+}
+
+function numberValue(value: unknown): number {
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+export function redactSensitiveOutput(input: string): string {
+  return input
+    .slice(0, MAX_OUTPUT_CHARS)
+    .replace(
+      /\b([a-z0-9_]*(?:cookie|session|token)[a-z0-9_]*)\b\s*[:=]\s*[^;\s,)]+/gi,
+      "$1=[redacted]",
+    )
+    .replace(/\b([a-z0-9_]*(?:cookie|session|token)[a-z0-9_]*)=([^;\s,)]+)/gi, "$1=[redacted]")
+    .replace(
+      /(?:~|\/Users\/[^/\s]+|\/home\/[^/\s]+)?\/\.config\/rdt-cli\/credential\.json/g,
+      "[redacted-credential-path]",
+    );
+}
+
+function runCommand(command: string, args: readonly string[]): Promise<CommandResult> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+
+    const timeout = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      child.kill("SIGTERM");
+      reject(new Error(`${command} timed out after ${COMMAND_TIMEOUT_MS}ms`));
+    }, COMMAND_TIMEOUT_MS);
+
+    child.stdout.on("data", (chunk: Buffer) => {
+      stdout = (stdout + chunk.toString("utf8")).slice(0, MAX_OUTPUT_CHARS);
+    });
+    child.stderr.on("data", (chunk: Buffer) => {
+      stderr = (stderr + chunk.toString("utf8")).slice(0, MAX_OUTPUT_CHARS);
+    });
+    child.on("error", (err) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      reject(err);
+    });
+    child.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      resolve({ code, stdout, stderr });
+    });
+  });
+}

package/src/providers/reddit.ts

@@ -1,53 +1,23 @@
-import { httpGet } from "../infra/http-client.js";
-import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
 import { rateLimiter } from "../infra/rate-limiter.js";
+import { BEARISH_TERMS, BULLISH_TERMS } from "../sentiment/keywords.js";
 import type { RedditSentimentResult } from "../types/sentiment.js";
-import { BULLISH_TERMS, BEARISH_TERMS } from "../sentiment/keywords.js";
-
-interface RedditListingResponse {
-  data: {
-    children: Array<{
-      data: {
-        id: string;
-        title: string;
-        selftext: string;
-        author: string;
-        score: number;
-        num_comments: number;
-        permalink: string;
-        created_utc: number;
-      };
-    }>;
-  };
-}
-
-const REDDIT_HEADERS = { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" };
+import { listSubredditPosts, readRedditPost, searchRedditPosts } from "./reddit-cli.js";
 
 export async function getSubredditPosts(
   subreddit: string,
   limit: number = 25,
+  query?: string,
 ): Promise<RedditSentimentResult> {
-  const cacheKey = `reddit:${subreddit}:${limit}`;
+  const cacheKey = `reddit:${subreddit}:${query ?? "hot"}:${limit}`;
   const cached = cache.get<RedditSentimentResult>(cacheKey);
   if (cached) return cached;
 
   try {
     await rateLimiter.acquire("reddit");
-    const url = `https://www.reddit.com/r/${encodeURIComponent(subreddit)}/hot.json?limit=${limit}`;
-    const data = await httpGet<RedditListingResponse>(url, {
-      headers: REDDIT_HEADERS,
-    });
-
-    const posts = data.data.children.map((child) => ({
-      id: child.data.id,
-      title: child.data.title,
-      selftext: child.data.selftext ?? "",
-      author: child.data.author ?? "unknown",
-      score: child.data.score,
-      comments: child.data.num_comments,
-      url: `https://reddit.com${child.data.permalink}`,
-      created: new Date(child.data.created_utc * 1000).toISOString(),
-    }));
+    const posts = query
+      ? await searchRedditPosts(query, { subreddit, limit })
+      : await listSubredditPosts(subreddit, { limit });
 
     // Extract ticker-like mentions ($AAPL, $TSLA, etc.)
     const tickerRegex = /\$([A-Z]{1,5})\b/g;
@@ -107,24 +77,7 @@ export async function getPostComments(
   if (cached) return cached;
 
   await rateLimiter.acquire("reddit_comments");
-  const url = `https://www.reddit.com/r/${encodeURIComponent(subreddit)}/comments/${postId}.json`;
-  const data = await httpGet<Array<{ data: { children: Array<{ kind: string; data: { id: string; body?: string; author?: string; score?: number; permalink?: string } }> } }>>(url, {
-    headers: REDDIT_HEADERS,
-  });
-
-  // Comments are in the second listing element
-  const commentListing = data[1]?.data?.children ?? [];
-  const comments: RedditComment[] = commentListing
-    .filter((c) => c.kind === "t1" && c.data.body)
-    .sort((a, b) => (b.data.score ?? 0) - (a.data.score ?? 0))
-    .slice(0, limit)
-    .map((c) => ({
-      id: c.data.id,
-      body: c.data.body!,
-      author: c.data.author ?? "unknown",
-      score: c.data.score ?? 0,
-      permalink: `https://reddit.com${c.data.permalink ?? ""}`,
-    }));
+  const { comments } = await readRedditPost(postId, { limit });
 
   cache.set(cacheKey, comments, COMMENT_TTL);
   return comments;
@@ -132,9 +85,11 @@ export async function getPostComments(
 
 // ── Sentiment scoring ───────────────────────────────────
 
-export function scoreSentiment(
-  posts: Array<{ title: string }>,
-): { score: number; bullish: number; bearish: number } {
+export function scoreSentiment(posts: Array<{ title: string }>): {
+  score: number;
+  bullish: number;
+  bearish: number;
+} {
   let bullish = 0;
   let bearish = 0;
   for (const post of posts) {

package/src/providers/sec-edgar.ts

@@ -1,9 +1,11 @@
-import { httpGet } from "../infra/http-client.js";
 import { cache, TTL } from "../infra/cache.js";
+import { httpGet } from "../infra/http-client.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
 
 const EFTS_BASE = "https://efts.sec.gov/LATEST/search-index";
 const COMPANY_TICKERS_URL = "https://www.sec.gov/files/company_tickers.json";
 const SUBMISSIONS_BASE = "https://data.sec.gov/submissions";
+const SEC_DOCUMENT_FETCH_TIMEOUT_MS = 10_000;
 
 export interface SECFiling {
   formType: string;
@@ -15,6 +17,7 @@ export interface SECFiling {
   primaryDocumentUrl?: string;
   items?: string[];
   evidenceSnippets?: string[];
+  evidenceWarning?: string;
 }
 
 export interface SearchFilingsOptions {
@@ -71,7 +74,9 @@ export async function searchFilings(
   if (cached) return cached;
 
   if (options.includeSnippets) {
-    const submissions = await searchFilingsFromCompanySubmissions(ticker, formTypes, limit).catch(() => []);
+    const submissions = await searchFilingsFromCompanySubmissions(ticker, formTypes, limit).catch(
+      () => [],
+    );
     if (submissions.length > 0) {
       await enrichWithEvidenceSnippets(submissions, options.snippetLimitPerFiling ?? 3);
       cache.set(cacheKey, submissions, TTL.FUNDAMENTALS);
@@ -186,7 +191,10 @@ async function resolveCompanyTicker(ticker: string): Promise<CompanyTickerEntry
 
 function splitFilingItems(raw: string | undefined): string[] {
   return raw
-    ? raw.split(",").map((item) => item.trim()).filter(Boolean)
+    ? raw
+        .split(",")
+        .map((item) => item.trim())
+        .filter(Boolean)
     : [];
 }
 
@@ -222,23 +230,29 @@ function getDateYearsAgo(years: number): string {
   return d.toISOString().split("T")[0];
 }
 
-async function enrichWithEvidenceSnippets(filings: SECFiling[], limitPerFiling: number): Promise<void> {
+async function enrichWithEvidenceSnippets(
+  filings: SECFiling[],
+  limitPerFiling: number,
+): Promise<void> {
   await Promise.all(
     filings.map(async (filing) => {
       if (!filing.primaryDocumentUrl) return;
       try {
         const raw = await fetchText(filing.primaryDocumentUrl);
         filing.evidenceSnippets = extractEvidenceSnippets(raw, limitPerFiling);
-      } catch {
+      } catch (error) {
         filing.evidenceSnippets = [];
+        filing.evidenceWarning = `Evidence fetch failed for primary document: ${error instanceof Error ? error.message : String(error)}`;
       }
     }),
   );
 }
 
 async function fetchText(url: string): Promise<string> {
+  await rateLimiter.acquire("sec_edgar");
   const response = await fetch(url, {
     headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+    signal: AbortSignal.timeout(SEC_DOCUMENT_FETCH_TIMEOUT_MS),
   });
   if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);
   return response.text();
@@ -300,7 +314,7 @@ function stripFilingMarkup(raw: string): string {
     .replace(/&amp;/gi, "&")
     .replace(/&#160;/g, " ")
     .replace(/&#8217;/g, "'")
-    .replace(/&#8220;|&#8221;/g, "\"")
+    .replace(/&#8220;|&#8221;/g, '"')
     .replace(/\s+/g, " ")
     .trim();
 }