@fin.cx/opendata 3.0.0 → 3.1.0

package/ts/stocks/providers/provider.secedgar.ts

@@ -0,0 +1,429 @@
+import * as plugins from '../../plugins.js';
+import type {
+  IFundamentalsProvider,
+  IStockFundamentals,
+  IFundamentalsRequest,
+  IFundamentalsCurrentRequest,
+  IFundamentalsBatchRequest
+} from '../interfaces/fundamentals.js';
+
+/**
+ * Configuration for SEC EDGAR provider
+ */
+export interface ISecEdgarConfig {
+  userAgent: string; // Required: Format "Company Name Email" (e.g., "fin.cx info@fin.cx")
+  cikCacheTTL?: number; // Default: 30 days (CIK codes rarely change)
+  fundamentalsCacheTTL?: number; // Default: 90 days (quarterly filings)
+  timeout?: number; // Request timeout in ms
+}
+
+/**
+ * Rate limiter for SEC EDGAR API
+ * SEC requires: 10 requests per second maximum
+ */
+class RateLimiter {
+  private requestTimes: number[] = [];
+  private readonly maxRequestsPerSecond = 10;
+
+  public async waitForSlot(): Promise<void> {
+    const now = Date.now();
+    const oneSecondAgo = now - 1000;
+
+    // Remove requests older than 1 second
+    this.requestTimes = this.requestTimes.filter(time => time > oneSecondAgo);
+
+    // If we've hit the limit, wait
+    if (this.requestTimes.length >= this.maxRequestsPerSecond) {
+      const oldestRequest = this.requestTimes[0];
+      const waitTime = 1000 - (now - oldestRequest) + 10; // +10ms buffer
+      await plugins.smartdelay.delayFor(waitTime);
+      return this.waitForSlot(); // Recursively check again
+    }
+
+    // Record this request
+    this.requestTimes.push(now);
+  }
+}
+
+/**
+ * SEC EDGAR Fundamental Data Provider
+ *
+ * Features:
+ * - Free public access (no API key required)
+ * - All US public companies
+ * - Financial data from 10-K/10-Q filings
+ * - US GAAP standardized metrics
+ * - Historical data back to ~2009
+ * - < 1 minute filing delay
+ *
+ * Documentation: https://www.sec.gov/edgar/sec-api-documentation
+ *
+ * Rate Limits:
+ * - 10 requests per second (enforced by SEC)
+ * - Requires User-Agent header in format: "Company Name Email"
+ *
+ * Data Sources:
+ * - Company Facts API: /api/xbrl/companyfacts/CIK##########.json
+ * - Ticker Lookup: /files/company_tickers.json
+ */
+export class SecEdgarProvider implements IFundamentalsProvider {
+  public name = 'SEC EDGAR';
+  public priority = 100; // High priority - free, authoritative, comprehensive
+  public readonly requiresAuth = false; // No API key needed!
+  public readonly rateLimit = {
+    requestsPerMinute: 600, // 10 requests/second = 600/minute
+    requestsPerDay: undefined // No daily limit
+  };
+
+  private logger = console;
+  private baseUrl = 'https://data.sec.gov/api/xbrl';
+  private tickersUrl = 'https://www.sec.gov/files/company_tickers.json';
+  private userAgent: string;
+  private config: Required<ISecEdgarConfig>;
+
+  // Caching
+  private cikCache = new Map<string, { cik: string; timestamp: Date }>();
+  private tickerListCache: { data: any; timestamp: Date } | null = null;
+
+  // Rate limiting
+  private rateLimiter = new RateLimiter();
+
+  constructor(config: ISecEdgarConfig) {
+    // Validate User-Agent
+    if (!config.userAgent) {
+      throw new Error('User-Agent is required for SEC EDGAR provider (format: "Company Name Email")');
+    }
+
+    // Validate User-Agent format (must contain space and @ symbol)
+    if (!config.userAgent.includes(' ') || !config.userAgent.includes('@')) {
+      throw new Error(
+        'Invalid User-Agent format. Required: "Company Name Email" (e.g., "fin.cx info@fin.cx")'
+      );
+    }
+
+    this.userAgent = config.userAgent;
+    this.config = {
+      userAgent: config.userAgent,
+      cikCacheTTL: config.cikCacheTTL || 30 * 24 * 60 * 60 * 1000, // 30 days
+      fundamentalsCacheTTL: config.fundamentalsCacheTTL || 90 * 24 * 60 * 60 * 1000, // 90 days
+      timeout: config.timeout || 30000
+    };
+  }
+
+  /**
+   * Unified data fetching method
+   */
+  public async fetchData(
+    request: IFundamentalsRequest
+  ): Promise<IStockFundamentals | IStockFundamentals[]> {
+    switch (request.type) {
+      case 'fundamentals-current':
+        return this.fetchFundamentals(request);
+      case 'fundamentals-batch':
+        return this.fetchBatchFundamentals(request);
+      default:
+        throw new Error(`Unsupported request type: ${(request as any).type}`);
+    }
+  }
+
+  /**
+   * Fetch fundamental data for a single ticker
+   */
+  private async fetchFundamentals(request: IFundamentalsCurrentRequest): Promise<IStockFundamentals> {
+    try {
+      // 1. Get CIK for ticker (with caching)
+      const cik = await this.getCIK(request.ticker);
+
+      // 2. Fetch company facts from SEC (with rate limiting)
+      const companyFacts = await this.fetchCompanyFacts(cik);
+
+      // 3. Parse facts into structured fundamental data
+      return this.parseCompanyFacts(request.ticker, cik, companyFacts);
+    } catch (error) {
+      this.logger.error(`Failed to fetch fundamentals for ${request.ticker}:`, error);
+      throw new Error(`SEC EDGAR: Failed to fetch fundamentals for ${request.ticker}: ${error.message}`);
+    }
+  }
+
+  /**
+   * Fetch fundamental data for multiple tickers
+   */
+  private async fetchBatchFundamentals(
+    request: IFundamentalsBatchRequest
+  ): Promise<IStockFundamentals[]> {
+    const results: IStockFundamentals[] = [];
+    const errors: string[] = [];
+
+    for (const ticker of request.tickers) {
+      try {
+        const fundamentals = await this.fetchFundamentals({
+          type: 'fundamentals-current',
+          ticker
+        });
+        results.push(fundamentals);
+      } catch (error) {
+        this.logger.warn(`Failed to fetch fundamentals for ${ticker}:`, error);
+        errors.push(`${ticker}: ${error.message}`);
+        // Continue with other tickers
+      }
+    }
+
+    if (results.length === 0) {
+      throw new Error(`Failed to fetch fundamentals for all tickers. Errors: ${errors.join(', ')}`);
+    }
+
+    return results;
+  }
+
+  /**
+   * Get CIK (Central Index Key) for a ticker symbol
+   * Uses SEC's public ticker-to-CIK mapping file
+   */
+  private async getCIK(ticker: string): Promise<string> {
+    const tickerUpper = ticker.toUpperCase();
+
+    // Check cache first
+    const cached = this.cikCache.get(tickerUpper);
+    if (cached) {
+      const age = Date.now() - cached.timestamp.getTime();
+      if (age < this.config.cikCacheTTL) {
+        return cached.cik;
+      }
+      // Cache expired, remove it
+      this.cikCache.delete(tickerUpper);
+    }
+
+    // Fetch ticker list (with caching at list level)
+    const tickers = await this.fetchTickerList();
+
+    // Find ticker in list (case-insensitive)
+    const entry = Object.values(tickers).find((t: any) => t.ticker === tickerUpper);
+
+    if (!entry) {
+      throw new Error(`CIK not found for ticker ${ticker}`);
+    }
+
+    const cik = String((entry as any).cik_str);
+
+    // Cache the result
+    this.cikCache.set(tickerUpper, {
+      cik,
+      timestamp: new Date()
+    });
+
+    return cik;
+  }
+
+  /**
+   * Fetch the SEC ticker-to-CIK mapping list
+   * Cached for 24 hours (list updates daily)
+   */
+  private async fetchTickerList(): Promise<any> {
+    // Check cache
+    if (this.tickerListCache) {
+      const age = Date.now() - this.tickerListCache.timestamp.getTime();
+      if (age < 24 * 60 * 60 * 1000) { // 24 hours
+        return this.tickerListCache.data;
+      }
+    }
+
+    // Wait for rate limit slot
+    await this.rateLimiter.waitForSlot();
+
+    // Fetch from SEC
+    const response = await plugins.smartrequest.SmartRequest.create()
+      .url(this.tickersUrl)
+      .headers({
+        'User-Agent': this.userAgent,
+        'Accept': 'application/json'
+      })
+      .timeout(this.config.timeout)
+      .get();
+
+    const data = await response.json();
+
+    // Cache the list
+    this.tickerListCache = {
+      data,
+      timestamp: new Date()
+    };
+
+    return data;
+  }
+
+  /**
+   * Fetch company facts from SEC EDGAR
+   */
+  private async fetchCompanyFacts(cik: string): Promise<any> {
+    // Pad CIK to 10 digits
+    const paddedCIK = cik.padStart(10, '0');
+    const url = `${this.baseUrl}/companyfacts/CIK${paddedCIK}.json`;
+
+    // Wait for rate limit slot
+    await this.rateLimiter.waitForSlot();
+
+    // Fetch from SEC
+    const response = await plugins.smartrequest.SmartRequest.create()
+      .url(url)
+      .headers({
+        'User-Agent': this.userAgent,
+        'Accept': 'application/json',
+        'Accept-Encoding': 'gzip, deflate',
+        'Host': 'data.sec.gov'
+      })
+      .timeout(this.config.timeout)
+      .get();
+
+    const data = await response.json();
+
+    // Validate response
+    if (!data || !data.facts) {
+      throw new Error('Invalid response from SEC EDGAR API');
+    }
+
+    return data;
+  }
+
+  /**
+   * Parse SEC company facts into structured fundamental data
+   */
+  private parseCompanyFacts(ticker: string, cik: string, data: any): IStockFundamentals {
+    const usGaap = data.facts?.['us-gaap'];
+
+    if (!usGaap) {
+      throw new Error('No US GAAP data available');
+    }
+
+    // Extract latest values for key metrics
+    const fundamentals: IStockFundamentals = {
+      ticker: ticker.toUpperCase(),
+      cik: cik,
+      companyName: data.entityName,
+      provider: this.name,
+      timestamp: new Date(),
+      fetchedAt: new Date(),
+
+      // Per-share metrics
+      earningsPerShareBasic: this.getLatestValue(usGaap, 'EarningsPerShareBasic'),
+      earningsPerShareDiluted: this.getLatestValue(usGaap, 'EarningsPerShareDiluted'),
+      sharesOutstanding: this.getLatestValue(usGaap, 'CommonStockSharesOutstanding'),
+      weightedAverageSharesOutstanding: this.getLatestValue(
+        usGaap,
+        'WeightedAverageNumberOfSharesOutstandingBasic'
+      ),
+
+      // Income statement
+      revenue: this.getLatestValue(usGaap, 'Revenues') ||
+               this.getLatestValue(usGaap, 'RevenueFromContractWithCustomerExcludingAssessedTax'),
+      netIncome: this.getLatestValue(usGaap, 'NetIncomeLoss'),
+      operatingIncome: this.getLatestValue(usGaap, 'OperatingIncomeLoss'),
+      grossProfit: this.getLatestValue(usGaap, 'GrossProfit'),
+      costOfRevenue: this.getLatestValue(usGaap, 'CostOfRevenue'),
+
+      // Balance sheet
+      assets: this.getLatestValue(usGaap, 'Assets'),
+      liabilities: this.getLatestValue(usGaap, 'Liabilities'),
+      stockholdersEquity: this.getLatestValue(usGaap, 'StockholdersEquity'),
+      cash: this.getLatestValue(usGaap, 'CashAndCashEquivalentsAtCarryingValue'),
+      propertyPlantEquipment: this.getLatestValue(usGaap, 'PropertyPlantAndEquipmentNet'),
+
+      // Metadata (from latest available data point)
+      fiscalYear: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fy,
+      fiscalQuarter: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fp,
+      filingDate: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.filed
+        ? new Date(this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')!.filed)
+        : undefined,
+      form: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.form
+    };
+
+    return fundamentals;
+  }
+
+  /**
+   * Get the latest value for a US GAAP metric
+   */
+  private getLatestValue(usGaap: any, metricName: string): number | undefined {
+    const metric = usGaap[metricName];
+    if (!metric?.units) {
+      return undefined;
+    }
+
+    // Get the first unit type (USD, shares, etc.)
+    const unitType = Object.keys(metric.units)[0];
+    const values = metric.units[unitType];
+
+    if (!values || !Array.isArray(values) || values.length === 0) {
+      return undefined;
+    }
+
+    // Get the latest value (last in array)
+    const latest = values[values.length - 1];
+    return latest?.val;
+  }
+
+  /**
+   * Get metadata from the latest data point
+   */
+  private getLatestMetadata(usGaap: any, metricName: string): any | undefined {
+    const metric = usGaap[metricName];
+    if (!metric?.units) {
+      return undefined;
+    }
+
+    const unitType = Object.keys(metric.units)[0];
+    const values = metric.units[unitType];
+
+    if (!values || !Array.isArray(values) || values.length === 0) {
+      return undefined;
+    }
+
+    return values[values.length - 1];
+  }
+
+  /**
+   * Check if SEC EDGAR API is available
+   */
+  public async isAvailable(): Promise<boolean> {
+    try {
+      // Test with Apple's well-known CIK
+      const url = `${this.baseUrl}/companyfacts/CIK0000320193.json`;
+
+      const response = await plugins.smartrequest.SmartRequest.create()
+        .url(url)
+        .headers({
+          'User-Agent': this.userAgent,
+          'Accept': 'application/json'
+        })
+        .timeout(5000)
+        .get();
+
+      const data = await response.json();
+      return data && data.facts !== undefined;
+    } catch (error) {
+      this.logger.warn('SEC EDGAR provider is not available:', error);
+      return false;
+    }
+  }
+
+  /**
+   * Get cache statistics
+   */
+  public getCacheStats(): {
+    cikCacheSize: number;
+    hasTickerList: boolean;
+  } {
+    return {
+      cikCacheSize: this.cikCache.size,
+      hasTickerList: this.tickerListCache !== null
+    };
+  }
+
+  /**
+   * Clear all caches
+   */
+  public clearCache(): void {
+    this.cikCache.clear();
+    this.tickerListCache = null;
+    this.logger.log('SEC EDGAR cache cleared');
+  }
+}