opencandle 0.4.0 → 0.6.0

package/src/providers/exa-search.ts

@@ -1,8 +1,8 @@
-import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
-import { rateLimiter } from "../infra/rate-limiter.js";
 import { getConfig } from "../config.js";
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
+import type { WebSearchEnvelope, WebSearchResult } from "../types/sentiment.js";
 import { ProviderCredentialError } from "./provider-credential-error.js";
-import type { WebSearchResult, WebSearchEnvelope } from "../types/sentiment.js";
 import type { WebSearchOpts } from "./web-search.js";
 
 const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
@@ -19,19 +19,27 @@ let requestIdCounter = 0;
 
 function freshnessToMs(freshness: WebSearchOpts["freshness"]): number {
   switch (freshness) {
-    case "hours": return 60 * 60 * 1000;
-    case "day": return 24 * 60 * 60 * 1000;
-    case "week": return 7 * 24 * 60 * 60 * 1000;
-    case "month": return 30 * 24 * 60 * 60 * 1000;
+    case "hours":
+      return 60 * 60 * 1000;
+    case "day":
+      return 24 * 60 * 60 * 1000;
+    case "week":
+      return 7 * 24 * 60 * 60 * 1000;
+    case "month":
+      return 30 * 24 * 60 * 60 * 1000;
   }
 }
 
 function enrichQueryForMcp(query: string, freshness: WebSearchOpts["freshness"]): string {
   switch (freshness) {
-    case "hours": return `${query} past hour`;
-    case "day": return `${query} past 24 hours`;
-    case "week": return `${query} past week`;
-    case "month": return `${query} past month`;
+    case "hours":
+      return `${query} past hour`;
+    case "day":
+      return `${query} past 24 hours`;
+    case "week":
+      return `${query} past week`;
+    case "month":
+      return `${query} past month`;
   }
 }
 
@@ -192,10 +200,7 @@ function exaCacheKey(query: string, opts: WebSearchOpts): string {
 // MCP path
 // ---------------------------------------------------------------------------
 
-async function exaMcpSearch(
-  query: string,
-  opts: WebSearchOpts,
-): Promise<WebSearchEnvelope> {
+async function exaMcpSearch(query: string, opts: WebSearchOpts): Promise<WebSearchEnvelope> {
   const enrichedQuery = enrichQueryForMcp(query, opts.freshness);
 
   const response = await fetch(EXA_MCP_URL, {
@@ -330,10 +335,7 @@ async function exaApiSearch(
 // Public entry point
 // ---------------------------------------------------------------------------
 
-export async function exaSearch(
-  query: string,
-  opts: WebSearchOpts,
-): Promise<WebSearchEnvelope> {
+export async function exaSearch(query: string, opts: WebSearchOpts): Promise<WebSearchEnvelope> {
   const key = exaCacheKey(query, opts);
   const cached = cache.get<WebSearchEnvelope>(key);
   if (cached) return cached;
@@ -362,12 +364,12 @@ export async function exaSearch(
   }
 }
 
+export type { ExaApiResponse, McpRpcResponse, ParsedResult };
 // Exported for testing
 export {
-  parseMcpResultBlocks,
-  extractJsonRpcPayload,
   enrichQueryForMcp,
+  extractJsonRpcPayload,
   filterByFreshness,
   mapApiResults,
+  parseMcpResultBlocks,
 };
-export type { ParsedResult, McpRpcResponse, ExaApiResponse };

package/src/providers/fear-greed.ts

@@ -1,5 +1,5 @@
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
 import { httpGet } from "../infra/http-client.js";
-import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
 import type { FearGreedData } from "../types/sentiment.js";
 
 // alternative.me provides a free crypto Fear & Greed index

package/src/providers/finnhub.ts

@@ -1,5 +1,5 @@
-import { httpGet, HttpError } from "../infra/http-client.js";
-import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
+import { HttpError, httpGet } from "../infra/http-client.js";
 import { rateLimiter } from "../infra/rate-limiter.js";
 import { ProviderCredentialError } from "./provider-credential-error.js";
 
@@ -36,7 +36,10 @@ const TICKER_NAMES: Record<string, string> = {
   ORCL: "oracle",
 };
 
-export function finnhubDateRange(freshness: "hours" | "day" | "week" | "month"): { from: string; to: string } {
+export function finnhubDateRange(freshness: "hours" | "day" | "week" | "month"): {
+  from: string;
+  to: string;
+} {
   const now = new Date();
   const to = formatDate(now);
 
@@ -86,9 +89,7 @@ export function filterByRelevance(
   });
 
   // Most recent first, capped
-  return filtered
-    .sort((a, b) => b.datetime - a.datetime)
-    .slice(0, limit);
+  return filtered.sort((a, b) => b.datetime - a.datetime).slice(0, limit);
 }
 
 export async function getCompanyNews(

package/src/providers/fred.ts

@@ -1,8 +1,8 @@
-import { httpGet, HttpError } from "../infra/http-client.js";
-import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
+import { HttpError, httpGet } from "../infra/http-client.js";
 import { rateLimiter } from "../infra/rate-limiter.js";
+import type { FredObservation, FredSeries } from "../types/macro.js";
 import { ProviderCredentialError } from "./provider-credential-error.js";
-import type { FredSeries, FredObservation } from "../types/macro.js";
 
 const BASE_URL = "https://api.stlouisfed.org/fred";
 

package/src/providers/index.ts

@@ -1,9 +1,17 @@
-export { getQuote, getHistory, getOptionsChain, clearCrumbCache, getYahooCrumb, computeTimeToExpiry } from "./yahoo-finance.js";
-export { getOverview, getEarnings, getFinancials } from "./alpha-vantage.js";
+export { getEarnings, getFinancials, getOverview } from "./alpha-vantage.js";
+export { getCryptoHistory, getCryptoPrice } from "./coingecko.js";
+export { getFearGreedIndex } from "./fear-greed.js";
 export { getSeries } from "./fred.js";
-export { getCryptoPrice, getCryptoHistory } from "./coingecko.js";
 export { getSubredditPosts, scoreSentiment } from "./reddit.js";
-export { searchFilings, type SECFiling } from "./sec-edgar.js";
-export { getFearGreedIndex } from "./fear-greed.js";
-export { searchWeb, ddgSearch, braveSearch, normalizeFinancialQuery } from "./web-search.js";
+export { type SECFiling, searchFilings } from "./sec-edgar.js";
+export { getQuotes, screenStocks } from "./tradingview.js";
 export type { WebSearchOpts } from "./web-search.js";
+export { braveSearch, ddgSearch, normalizeFinancialQuery, searchWeb } from "./web-search.js";
+export {
+  clearCrumbCache,
+  computeTimeToExpiry,
+  getHistory,
+  getOptionsChain,
+  getQuote,
+  getYahooCrumb,
+} from "./yahoo-finance.js";

package/src/providers/reddit.ts

@@ -1,8 +1,8 @@
+import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
 import { httpGet } from "../infra/http-client.js";
-import { cache, TTL, STALE_LIMIT } from "../infra/cache.js";
 import { rateLimiter } from "../infra/rate-limiter.js";
+import { BEARISH_TERMS, BULLISH_TERMS } from "../sentiment/keywords.js";
 import type { RedditSentimentResult } from "../types/sentiment.js";
-import { BULLISH_TERMS, BEARISH_TERMS } from "../sentiment/keywords.js";
 
 interface RedditListingResponse {
   data: {
@@ -108,7 +108,16 @@ export async function getPostComments(
 
   await rateLimiter.acquire("reddit_comments");
   const url = `https://www.reddit.com/r/${encodeURIComponent(subreddit)}/comments/${postId}.json`;
-  const data = await httpGet<Array<{ data: { children: Array<{ kind: string; data: { id: string; body?: string; author?: string; score?: number; permalink?: string } }> } }>>(url, {
+  const data = await httpGet<
+    Array<{
+      data: {
+        children: Array<{
+          kind: string;
+          data: { id: string; body?: string; author?: string; score?: number; permalink?: string };
+        }>;
+      };
+    }>
+  >(url, {
     headers: REDDIT_HEADERS,
   });
 
@@ -132,9 +141,11 @@ export async function getPostComments(
 
 // ── Sentiment scoring ───────────────────────────────────
 
-export function scoreSentiment(
-  posts: Array<{ title: string }>,
-): { score: number; bullish: number; bearish: number } {
+export function scoreSentiment(posts: Array<{ title: string }>): {
+  score: number;
+  bullish: number;
+  bearish: number;
+} {
   let bullish = 0;
   let bearish = 0;
   for (const post of posts) {

package/src/providers/sec-edgar.ts

@@ -1,7 +1,11 @@
-import { httpGet } from "../infra/http-client.js";
 import { cache, TTL } from "../infra/cache.js";
+import { httpGet } from "../infra/http-client.js";
+import { rateLimiter } from "../infra/rate-limiter.js";
 
 const EFTS_BASE = "https://efts.sec.gov/LATEST/search-index";
+const COMPANY_TICKERS_URL = "https://www.sec.gov/files/company_tickers.json";
+const SUBMISSIONS_BASE = "https://data.sec.gov/submissions";
+const SEC_DOCUMENT_FETCH_TIMEOUT_MS = 10_000;
 
 export interface SECFiling {
   formType: string;
@@ -10,6 +14,15 @@ export interface SECFiling {
   entityName: string;
   accessionNumber: string;
   url: string;
+  primaryDocumentUrl?: string;
+  items?: string[];
+  evidenceSnippets?: string[];
+  evidenceWarning?: string;
+}
+
+export interface SearchFilingsOptions {
+  includeSnippets?: boolean;
+  snippetLimitPerFiling?: number;
 }
 
 interface EFTSResponse {
@@ -23,20 +36,54 @@ interface EFTSResponse {
         display_names: string[];
         period_ending: string;
         ciks: string[];
+        items?: string[];
       };
     }>;
   };
 }
 
+interface CompanyTickerEntry {
+  cik_str: number;
+  ticker: string;
+  title: string;
+}
+
+interface SubmissionsResponse {
+  name: string;
+  tickers?: string[];
+  filings: {
+    recent: {
+      accessionNumber: string[];
+      filingDate: string[];
+      reportDate: string[];
+      form: string[];
+      primaryDocument: string[];
+      items?: string[];
+    };
+  };
+}
+
 export async function searchFilings(
   ticker: string,
   formTypes: string[] = ["10-K", "10-Q", "8-K"],
   limit: number = 10,
+  options: SearchFilingsOptions = {},
 ): Promise<SECFiling[]> {
-  const cacheKey = `sec:${ticker}:${formTypes.join(",")}:${limit}`;
+  const cacheKey = `sec:${ticker}:${formTypes.join(",")}:${limit}:${options.includeSnippets ? "snippets" : "metadata"}`;
   const cached = cache.get<SECFiling[]>(cacheKey);
   if (cached) return cached;
 
+  if (options.includeSnippets) {
+    const submissions = await searchFilingsFromCompanySubmissions(ticker, formTypes, limit).catch(
+      () => [],
+    );
+    if (submissions.length > 0) {
+      await enrichWithEvidenceSnippets(submissions, options.snippetLimitPerFiling ?? 3);
+      cache.set(cacheKey, submissions, TTL.FUNDAMENTALS);
+      return submissions;
+    }
+  }
+
   const params = new URLSearchParams({
     q: ticker,
     forms: formTypes.join(","),
@@ -60,12 +107,15 @@ export async function searchFilings(
     const src = hit._source;
     const accession = src.adsh;
     if (!accession || seen.has(accession)) continue;
+    if (!matchesTicker(src.display_names, ticker)) continue;
     seen.add(accession);
 
     const cik = src.ciks?.[0] ?? "";
     const displayName = src.display_names?.[0] ?? "";
     // Extract entity name from display format: "APPLE INC  (AAPL)  (CIK 0000320193)"
     const entityName = displayName.split("(")[0]?.trim() ?? displayName;
+    const archiveDir = buildEdgarArchiveDir(cik, accession);
+    const primaryDocumentUrl = buildPrimaryDocumentUrl(archiveDir, hit._id);
 
     filings.push({
       formType: src.form ?? "",
@@ -73,20 +123,105 @@ export async function searchFilings(
       periodOfReport: src.period_ending ?? "",
       entityName,
       accessionNumber: accession,
-      url: buildEdgarUrl(cik, accession),
+      url: `${archiveDir}/${accession}-index.htm`,
+      primaryDocumentUrl,
+      items: src.items ?? [],
     });
 
     if (filings.length >= limit) break;
   }
 
+  if (options.includeSnippets) {
+    await enrichWithEvidenceSnippets(filings, options.snippetLimitPerFiling ?? 3);
+  }
+
   cache.set(cacheKey, filings, TTL.FUNDAMENTALS);
   return filings;
 }
 
-function buildEdgarUrl(cik: string, accession: string): string {
+async function searchFilingsFromCompanySubmissions(
+  ticker: string,
+  formTypes: string[],
+  limit: number,
+): Promise<SECFiling[]> {
+  const company = await resolveCompanyTicker(ticker);
+  if (!company) return [];
+
+  const cik = String(company.cik_str).padStart(10, "0");
+  const data = await httpGet<SubmissionsResponse>(`${SUBMISSIONS_BASE}/CIK${cik}.json`, {
+    headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+  });
+
+  const filings: SECFiling[] = [];
+  const recent = data.filings.recent;
+  for (let i = 0; i < recent.form.length && filings.length < limit; i++) {
+    const formType = recent.form[i] ?? "";
+    if (!formTypes.includes(formType)) continue;
+    const accession = recent.accessionNumber[i];
+    if (!accession) continue;
+    const archiveDir = buildEdgarArchiveDir(cik, accession);
+    const primaryDocument = recent.primaryDocument[i];
+    filings.push({
+      formType,
+      filedDate: recent.filingDate[i] ?? "",
+      periodOfReport: recent.reportDate[i] ?? "",
+      entityName: data.name || company.title,
+      accessionNumber: accession,
+      url: `${archiveDir}/${accession}-index.htm`,
+      primaryDocumentUrl: primaryDocument ? `${archiveDir}/${primaryDocument}` : undefined,
+      items: splitFilingItems(recent.items?.[i]),
+    });
+  }
+  return filings;
+}
+
+async function resolveCompanyTicker(ticker: string): Promise<CompanyTickerEntry | undefined> {
+  const cacheKey = "sec:company-tickers";
+  let companies = cache.get<CompanyTickerEntry[]>(cacheKey);
+  if (!companies) {
+    const data = await httpGet<Record<string, CompanyTickerEntry>>(COMPANY_TICKERS_URL, {
+      headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+    });
+    companies = Object.values(data);
+    cache.set(cacheKey, companies, TTL.FUNDAMENTALS);
+  }
+  const normalized = ticker.toUpperCase();
+  return companies.find((company) => company.ticker.toUpperCase() === normalized);
+}
+
+function splitFilingItems(raw: string | undefined): string[] {
+  return raw
+    ? raw
+        .split(",")
+        .map((item) => item.trim())
+        .filter(Boolean)
+    : [];
+}
+
+function matchesTicker(displayNames: string[] | undefined, ticker: string): boolean {
+  const normalized = ticker.toUpperCase();
+  return (displayNames ?? []).some((name) => {
+    const tickerGroups = name.match(/\(([^)]*)\)/g) ?? [];
+    return tickerGroups.some((group) =>
+      group
+        .slice(1, -1)
+        .split(",")
+        .map((part) => part.trim().toUpperCase())
+        .includes(normalized),
+    );
+  });
+}
+
+function buildEdgarArchiveDir(cik: string, accession: string): string {
   const cikNum = cik.replace(/^0+/, "");
   const accessionNoDash = accession.replace(/-/g, "");
-  return `https://www.sec.gov/Archives/edgar/data/${cikNum}/${accessionNoDash}/${accession}-index.htm`;
+  return `https://www.sec.gov/Archives/edgar/data/${cikNum}/${accessionNoDash}`;
+}
+
+function buildPrimaryDocumentUrl(archiveDir: string, hitId: string): string | undefined {
+  const fileName = hitId.split(":")[1];
+  if (!fileName || !/\.(?:htm|html|txt)$/i.test(fileName)) return undefined;
+  return `${archiveDir}/${fileName}`;
 }
 
 function getDateYearsAgo(years: number): string {
@@ -94,3 +229,98 @@ function getDateYearsAgo(years: number): string {
   d.setFullYear(d.getFullYear() - years);
   return d.toISOString().split("T")[0];
 }
+
+async function enrichWithEvidenceSnippets(
+  filings: SECFiling[],
+  limitPerFiling: number,
+): Promise<void> {
+  await Promise.all(
+    filings.map(async (filing) => {
+      if (!filing.primaryDocumentUrl) return;
+      try {
+        const raw = await fetchText(filing.primaryDocumentUrl);
+        filing.evidenceSnippets = extractEvidenceSnippets(raw, limitPerFiling);
+      } catch (error) {
+        filing.evidenceSnippets = [];
+        filing.evidenceWarning = `Evidence fetch failed for primary document: ${error instanceof Error ? error.message : String(error)}`;
+      }
+    }),
+  );
+}
+
+async function fetchText(url: string): Promise<string> {
+  await rateLimiter.acquire("sec_edgar");
+  const response = await fetch(url, {
+    headers: { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" },
+    signal: AbortSignal.timeout(SEC_DOCUMENT_FETCH_TIMEOUT_MS),
+  });
+  if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);
+  return response.text();
+}
+
+const EVIDENCE_KEYWORDS = [
+  "risk factor",
+  "legal proceedings",
+  "litigation",
+  "regulatory",
+  "regulation",
+  "revenue",
+  "concentration",
+  "management's discussion",
+  "management discussion",
+  "liquidity",
+  "outlook",
+  "guidance",
+  "material weakness",
+  "going concern",
+];
+
+function extractEvidenceSnippets(raw: string, limit: number): string[] {
+  const text = stripFilingMarkup(raw);
+  const snippets: string[] = [];
+  const lower = text.toLowerCase();
+
+  for (const keyword of EVIDENCE_KEYWORDS) {
+    let searchFrom = 0;
+    while (searchFrom < lower.length) {
+      const index = lower.indexOf(keyword, searchFrom);
+      if (index === -1) break;
+      searchFrom = index + keyword.length;
+
+      const start = Math.max(0, index - 220);
+      const end = Math.min(text.length, index + keyword.length + 420);
+      const snippet = text.slice(start, end).replace(/\s+/g, " ").trim();
+      if (
+        snippet &&
+        !isLikelyTableOfContentsSnippet(snippet) &&
+        !snippets.some((existing) => existing.includes(snippet.slice(0, 80)))
+      ) {
+        snippets.push(snippet);
+        break;
+      }
+    }
+    if (snippets.length >= limit) break;
+  }
+
+  return snippets;
+}
+
+function stripFilingMarkup(raw: string): string {
+  return raw
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&#160;/g, " ")
+    .replace(/&#8217;/g, "'")
+    .replace(/&#8220;|&#8221;/g, '"')
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function isLikelyTableOfContentsSnippet(snippet: string): boolean {
+  const lower = snippet.toLowerCase();
+  const itemHeadingCount = lower.match(/\bitem\s+\d+[a-z]?\b/g)?.length ?? 0;
+  return lower.includes("table of contents") && itemHeadingCount >= 2;
+}