ruby-edgar 0.1.0

data/lib/edgar/filing_facts.rb

@@ -0,0 +1,312 @@
+# frozen_string_literal: true
+
+module Edgar
+  # Financial facts extracted from a filing's XBRL instance document,
+  # providing the same duck-typed interface as EntityFacts but scoped
+  # to the periods relevant to the filing (annual for 10-K, quarterly for 10-Q).
+  class FilingFacts
+    attr_reader :form_type, :contexts, :facts_map
+
+    def initialize(xbrl, form_type: nil)
+      @xbrl = xbrl
+      @form_type = form_type
+      @contexts = build_context_index
+      @facts_map = build_facts_map
+    end
+
+    def available?
+      @facts_map.any?
+    end
+
+    def concept(name)
+      values = @facts_map[name]
+      return nil unless values&.any?
+
+      { "label" => name, "units" => { "USD" => values } }
+    end
+
+    def concept_values(name, limit: nil, annual: nil)
+      c = concept(name)
+      return [] unless c
+
+      units = c["units"]
+      return [] unless units
+
+      values = units.values.flatten
+
+      if annual
+        has_fp = values.first&.key?("fp")
+        if has_fp
+          fy_values = values.select { |v| v["fp"] == "FY" }
+          values = fy_values unless fy_values.empty?
+        end
+      end
+
+      # Sort by filed date desc, then end date desc (matching Python edgartools'
+      # max by (filing_date, period_end) semantics)
+      values = values.sort_by { |v| [v["end"] || "", v["end"] || ""] }.reverse
+
+      values = values.first(limit) if limit
+      values
+    end
+
+    def concepts
+      @facts_map.keys
+    end
+
+    def us_gaap
+      @facts_map
+    end
+
+    def ifrs_full
+      {}
+    end
+
+    def dei
+      {}
+    end
+
+    # Standardized financial accessors
+    # When +limit:+ is given, returns an array of raw fact hashes (legacy).
+    # Otherwise returns a single numeric value (matching Python edgartools).
+
+    def get_revenue(limit: nil, annual: true, period: nil)
+      # Matches Python edgartools EntityFacts concept variants
+      revenue_concepts = %w[RevenueFromContractWithCustomerExcludingAssessedTax
+                            SalesRevenueNet
+                            Revenues Revenue
+                            TotalRevenues NetSales]
+      if limit
+        revenue_concepts.each do |name|
+          vals = concept_values(name, limit: limit, annual: annual)
+          return vals unless vals.empty?
+        end
+      else
+        revenue_concepts.each do |name|
+          val = _select_value(name, annual: annual, period: period)
+          return val if val
+        end
+      end
+      nil
+    end
+
+    def get_net_income(limit: nil, annual: true, period: nil)
+      # Matches Python edgartools EntityFacts concept variants
+      net_income_concepts = %w[NetIncomeLoss ProfitLoss NetIncome
+                               NetEarnings NetIncomeLossAttributableToParent]
+      if limit
+        net_income_concepts.each do |name|
+          v = concept_values(name, limit: limit, annual: annual)
+          return v unless v.empty?
+        end
+      else
+        net_income_concepts.each do |name|
+          val = _select_value(name, annual: annual, period: period)
+          return val if val
+        end
+      end
+      nil
+    end
+
+    def get_operating_income(limit: nil, annual: true, period: nil)
+      # Matches Python edgartools EntityFacts concept variants
+      op_income_concepts = %w[OperatingIncomeLoss OperatingIncome
+                              IncomeLossFromOperations OperatingProfit]
+      if limit
+        op_income_concepts.each do |name|
+          v = concept_values(name, limit: limit, annual: annual)
+          return v unless v.empty?
+        end
+      else
+        op_income_concepts.each do |name|
+          val = _select_value(name, annual: annual, period: period)
+          return val if val
+        end
+      end
+      nil
+    end
+
+    def get_total_assets(limit: nil, annual: true, period: nil)
+      # Matches Python edgartools EntityFacts concept variants
+      asset_concepts = %w[Assets TotalAssets AssetsCurrent]
+      if limit
+        asset_concepts.each do |name|
+          v = concept_values(name, limit: limit, annual: annual)
+          return v unless v.empty?
+        end
+      else
+        asset_concepts.each do |name|
+          val = _select_value(name, annual: annual, period: period)
+          return val if val
+        end
+      end
+      nil
+    end
+
+    def get_total_liabilities(limit: nil, annual: true, period: nil)
+      # Matches Python edgartools EntityFacts concept variants
+      liability_concepts = %w[Liabilities TotalLiabilities LiabilitiesAndStockholdersEquity]
+      if limit
+        liability_concepts.each do |name|
+          v = concept_values(name, limit: limit, annual: annual)
+          return v unless v.empty?
+        end
+      else
+        liability_concepts.each do |name|
+          val = _select_value(name, annual: annual, period: period)
+          return val if val
+        end
+      end
+      nil
+    end
+
+    def get_shareholders_equity(limit: nil, annual: true, period: nil)
+      # Matches Python edgartools EntityFacts concept variants
+      equity_concepts = %w[StockholdersEquity ShareholdersEquity
+                           TotalEquity PartnersCapital MembersEquity]
+      if limit
+        equity_concepts.each do |name|
+          v = concept_values(name, limit: limit, annual: annual)
+          return v unless v.empty?
+        end
+      else
+        equity_concepts.each do |name|
+          val = _select_value(name, annual: annual, period: period)
+          return val if val
+        end
+      end
+      nil
+    end
+    alias get_stockholders_equity get_shareholders_equity
+
+    def get_gross_profit(limit: nil, annual: true, period: nil)
+      # Matches Python edgartools EntityFacts concept variants
+      gross_profit_concepts = %w[GrossProfit GrossMargin]
+      if limit
+        gross_profit_concepts.each do |name|
+          v = concept_values(name, limit: limit, annual: annual)
+          return v unless v.empty?
+        end
+      else
+        gross_profit_concepts.each do |name|
+          val = _select_value(name, annual: annual, period: period)
+          return val if val
+        end
+      end
+      nil
+    end
+
+    # Select the best single value for a concept, matching Python edgartools'
+    # _get_standardized_concept_value semantics.
+    # When +annual+ is true (default), prefers FY (fiscal-year) facts.
+    # When +period+ is given (e.g. "2025-FY"), filters to that specific period.
+    # When multiple facts exist, selects by (end desc) matching Python's
+    # max by (period_end). FilingFacts doesn't have filed dates, so falls
+    # back to most recent end date.
+    def _select_value(concept_name, annual: true, period: nil)
+      c = concept(concept_name)
+      return nil unless c
+
+      values = c["units"]&.values&.flatten || []
+      return nil if values.empty?
+
+      if annual && period.nil?
+        has_fp = values.first&.key?("fp")
+        if has_fp
+          fy_values = values.select { |v| v["fp"] == "FY" }
+          values = fy_values unless fy_values.empty?
+        end
+      end
+
+      if period
+        fy_str, fp_str = period.split("-")
+        values = values.select { |v| v["fy"]&.to_s == fy_str && v["fp"] == fp_str }
+      end
+
+      return nil if values.empty?
+
+      # Select by end date desc (FilingFacts lacks filed dates, so use end date)
+      best = values.max_by { |v| v["end"] || v["start"] || "" }
+
+      best["val"]
+    end
+
+    private
+
+    def build_context_index
+      @xbrl.context_refs.to_h { |ctx| [ctx[:id], ctx] }
+    end
+
+    def build_facts_map
+      map = {}
+      @xbrl.financial_concepts.each do |fact|
+        name = fact[:name]
+        ctx = @contexts[fact[:context_ref]]
+        next unless ctx
+
+        next unless period_matches_form?(ctx)
+
+        # Skip segmented (dimension-level) facts — prefer entity-level
+        # (no segment dimensions) to avoid picking product/geographic
+        # segment values over total-entity values.
+        next if ctx[:segmented]
+
+        end_date = ctx[:end_date] || ctx[:instant]
+        start_date = ctx[:start_date]
+
+        fp = nil
+        fy = end_date ? Date.parse(end_date).year : nil
+        if start_date && end_date
+          duration = days_between(start_date, end_date)
+          fp = "FY" if duration > 300
+        end
+
+        value = {
+          "val" => fact[:value].to_f,
+          "end" => end_date,
+          "start" => start_date,
+          "fp" => fp,
+          "fy" => fy,
+          "unit" => extract_unit(fact[:unit_ref])
+        }
+
+        map[name] ||= []
+        map[name] << value
+      end
+      map
+    end
+
+    def period_matches_form?(context)
+      return true unless @form_type
+
+      if @form_type.start_with?("10-K")
+        if context[:start_date] && context[:end_date]
+          duration = days_between(context[:start_date], context[:end_date])
+          return false if duration < 300 && duration.positive?
+        end
+      elsif @form_type.start_with?("10-Q")
+        if context[:start_date] && context[:end_date]
+          duration = days_between(context[:start_date], context[:end_date])
+          return false if duration > 200
+        end
+      end
+      true
+    end
+
+    def days_between(start_str, end_str)
+      (Date.parse(end_str) - Date.parse(start_str)).to_i
+    rescue ArgumentError
+      0
+    end
+
+    def extract_unit(unit_ref)
+      return nil unless unit_ref
+
+      unit = @xbrl.units.find { |u| u[:id] == unit_ref }
+      return nil unless unit
+
+      measure = unit[:measure]
+      measure&.sub("iso4217:", "")
+    end
+  end
+end

data/lib/edgar/filing_header.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Edgar
+  # Parses the <SEC-HEADER> section of an SEC filing text.
+  class FilingHeader
+    attr_reader :raw_text, :fields
+
+    def initialize(sgml_text)
+      @raw_text = ""
+      @fields = {}
+
+      parse_header(sgml_text.to_s)
+    end
+
+    def filers
+      @fields["FILER"] || []
+    end
+
+    def period_of_report
+      raw = @fields["CONFORMED PERIOD OF REPORT"]&.first
+      return nil unless raw
+
+      # Format YYYYMMDD as YYYY-MM-DD (ISO 8601)
+      if raw.match?(/\A\d{8}\z/)
+        "#{raw[0..3]}-#{raw[4..5]}-#{raw[6..7]}"
+      else
+        raw
+      end
+    end
+
+    def filer_cik
+      filers.first["COMPANY DATA CENTRAL INDEX KEY"] || filers.first["CENTRAL INDEX KEY"] if filers.any?
+    end
+
+    def filer_company
+      filers.first["COMPANY DATA COMPANY CONFORMED NAME"] || filers.first["COMPANY CONFORMED NAME"] if filers.any?
+    end
+
+    def form_type
+      @fields["FORM TYPE"]&.first || @fields["CONFORMED SUBMISSION TYPE"]&.first
+    end
+
+    def accession_number
+      @fields["ACCESSION NUMBER"]&.first
+    end
+
+    def to_s
+      "#<FilingHeader #{@fields.keys.length} keys>"
+    end
+
+    private
+
+    def parse_header(text)
+      header_match = text.match(%r{<SEC-HEADER>(.*?)</SEC-HEADER>}m)
+      return unless header_match
+
+      @raw_text = header_match[1]
+
+      filer_data = nil
+      filer_section = nil
+
+      @raw_text.each_line do |line|
+        strip = line.strip
+        next if strip.empty?
+
+        indent = line[/\A[ \t]*/].length
+        tab_count = line[/\A\t*/].length
+
+        if indent.zero?
+          filer_data, filer_section = finish_filer(filer_data, filer_section)
+          filer_data, filer_section = parse_top_level(line, strip, filer_data)
+        elsif tab_count >= 1 || indent >= 1
+          filer_data, filer_section = parse_indented(strip, filer_data, filer_section)
+        end
+      end
+
+      return unless filer_data&.any?
+
+      @fields["FILER"] ||= []
+      @fields["FILER"] << filer_data
+    end
+
+    def finish_filer(filer_data, _filer_section)
+      return [nil, nil] unless filer_data
+
+      @fields["FILER"] ||= []
+      @fields["FILER"] << filer_data
+      [nil, nil]
+    end
+
+    def parse_top_level(_line, strip, filer_data)
+      if ["FILER:", "FILER"].include?(strip)
+        filer_data = {}
+        filer_section = nil
+        return [filer_data, filer_section] if strip.end_with?(":")
+      end
+
+      return [filer_data, nil] unless strip.include?(":")
+
+      k, v = strip.split(":", 2).map(&:strip)
+      k = k.strip.gsub(/\s+/, " ") if k
+      if k && !k.empty?
+        @fields[k] ||= []
+        @fields[k] << v if v && !v.empty?
+      end
+      [filer_data, nil]
+    end
+
+    def parse_indented(strip, filer_data, filer_section)
+      if filer_data
+        if strip.end_with?(":") && !strip.start_with?("-")
+          filer_section = strip.sub(/:+\z/, "").strip
+        elsif strip.include?(":")
+          k, v = strip.split(":", 2).map(&:strip)
+          key = filer_section ? "#{filer_section} #{k}" : k
+          filer_data[key] = v if k && v && !k.empty?
+        end
+      elsif strip.include?(":") && strip.end_with?(":")
+        filer_section = strip.sub(/:+\z/, "").strip
+      end
+      [filer_data, filer_section]
+    end
+  end
+end

data/lib/edgar/filing_homepage.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Edgar
+  # Parses the SEC filing HTML index page listing document attachments.
+  class FilingHomepage
+    attr_reader :url, :soup, :attachments
+
+    def initialize(url, soup = nil)
+      @url = url
+      @soup = soup || fetch_soup
+      @attachments = parse_attachments
+    end
+
+    def self.empty
+      new("", Nokogiri::HTML(""))
+    end
+
+    def documents
+      @attachments.select { |a| a.html? || a.text? }
+    end
+
+    def datafiles
+      @attachments.select(&:xml?)
+    end
+
+    def primary_documents
+      @attachments.primary_documents
+    end
+
+    def primary_html_document
+      @attachments.primary_html_document
+    end
+
+    def primary_xml_document
+      @attachments.primary_xml_document
+    end
+
+    def xbrl_document
+      @attachments.xbrl_document
+    end
+
+    def period_of_report
+      text = @soup.text
+      match = text.match(/Period of Report\s*:\s*(\S+)/i)
+      match ? match[1] : nil
+    end
+
+    def filers
+      @soup.css("div.filer, .companyInfo, .mailer").map do |el|
+        { name: el.text.strip }
+      end
+    end
+
+    def to_s
+      "#<FilingHomepage #{@url}>"
+    end
+
+    private
+
+    def fetch_soup
+      return Nokogiri::HTML("") if @url.empty?
+
+      html = Edgar.http_client.download_text(@url)
+      Nokogiri::HTML(html)
+    rescue StandardError => e
+      warn "Failed to fetch homepage: #{e.message}"
+      Nokogiri::HTML("")
+    end
+
+    def parse_attachments
+      result = []
+      rows = @soup.css("table.tableFile tr")
+
+      rows.each do |row|
+        cells = row.css("td")
+        next if cells.length < 4 || cells[0]["class"] == "header"
+
+        seq_text = cells[0]&.text&.strip
+        next if seq_text.nil? || seq_text.empty?
+        next if seq_text == "Seq"
+
+        link = row.at_css("a")
+        next unless link
+
+        href = link["href"].to_s.strip
+        next if href.empty?
+
+        full_url = if href.start_with?("/")
+                     "https://www.sec.gov#{href}"
+                   elsif href.start_with?("http")
+                     href
+                   else
+                     URI.join(@url, href).to_s
+                   end
+
+        seq = seq_text.to_i
+        description = cells[1]&.text&.strip
+        doc_name = link.text.strip
+        doc_type = cells[3]&.text&.strip
+        size_text = cells[4]&.text&.strip
+        size = size_text&.gsub(/[^0-9.]/, "")
+
+        result << Attachment.new(
+          sequence_number: seq,
+          description: description,
+          document: doc_name.empty? ? href.split("/").last : doc_name,
+          document_type: doc_type,
+          size: size,
+          url: full_url
+        )
+      end
+
+      Attachments.new(result)
+    end
+  end
+end

data/lib/edgar/filing_index.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Edgar
+  # Parses SEC quarterly EDGAR index .gz files into index entries.
+  class FilingIndex
+    IndexEntry = Struct.new(:form, :company, :cik, :filing_date, :accession_number, keyword_init: true)
+
+    attr_reader :year, :quarter, :type
+
+    def initialize(year, quarter, type: "company")
+      @year = year.to_i
+      @quarter = quarter.to_i
+      @type = type
+    end
+
+    def entries
+      @entries ||= parse_index
+    end
+
+    def self.for(year, quarter, type: "company")
+      new(year, quarter, type: type).entries
+    end
+
+    INDEX_LINE_RE = /\A(.+?)\s{2,}(\S{1,20})\s{2,}(\d{1,10})\s{2,}(\d{4}-\d{2}-\d{2})\s{2,}(\S+)\s*\z/
+
+    private
+
+    def parse_index
+      url = Config.full_index_url(@year, @quarter, @type)
+      content = download_and_decompress(url)
+      parse_fixed_width(content)
+    rescue StandardError => e
+      warn "Failed to fetch index for #{@year}/Q#{@quarter} (#{@type}): #{e.message}"
+      []
+    end
+
+    def download_and_decompress(url)
+      Edgar.http_client.download_text(url)
+    end
+
+    def parse_fixed_width(content)
+      text = content.encode("UTF-8", invalid: :replace, undef: :replace)
+      lines = text.lines
+      entries = []
+      started = false
+
+      lines.each do |line|
+        if line.include?("----")
+          started = true
+          next
+        end
+        next unless started
+
+        row = parse_index_row(line)
+        next unless row
+
+        entries << IndexEntry.new(row)
+      end
+
+      entries
+    end
+
+    def parse_index_row(line)
+      line = line.sub(/\n\z/, "")
+      return nil if line.strip.empty?
+
+      m = line.match(INDEX_LINE_RE)
+      return nil unless m
+
+      company = m[1].strip
+      form = m[2].strip
+      cik = m[3].to_i
+      filing_date = m[4].strip
+      fname = m[5].strip
+
+      accession_number = File.basename(fname, ".txt")
+
+      {
+        company: company,
+        form: form,
+        cik: cik,
+        filing_date: filing_date,
+        accession_number: accession_number
+      }
+    end
+  end
+end

data/lib/edgar/filing_sections.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Edgar
+  # Extracts section markers (Item 1, Item 1A, etc.) from filing text.
+  class FilingSections
+    SECTION_MARKERS = {
+      "1" => "Business",
+      "1A" => "Risk Factors",
+      "1B" => "Unresolved Staff Comments",
+      "2" => "Properties",
+      "3" => "Legal Proceedings",
+      "4" => "Mine Safety Disclosures",
+      "5" => "Market for Registrant's Common Equity",
+      "6" => "Selected Financial Data",
+      "7" => "Management's Discussion and Analysis",
+      "7A" => "Quantitative and Qualitative Disclosures About Market Risk",
+      "8" => "Financial Statements and Supplementary Data",
+      "9" => "Changes in and Disagreements with Accountants",
+      "9A" => "Controls and Procedures",
+      "9B" => "Other Information",
+      "10" => "Directors and Executive Officers",
+      "11" => "Executive Compensation",
+      "12" => "Security Ownership of Certain Beneficial Owners",
+      "13" => "Certain Relationships and Related Transactions",
+      "14" => "Principal Accountant Fees and Services",
+      "15" => "Exhibits and Financial Statement Schedules"
+    }.freeze
+
+    def self.extract_sections(sgml_text)
+      markers = {}
+      sgml_text.to_s.each_line do |line|
+        next unless line.strip =~ /^Item\s+(\d+[A-Za-z]?)\.?\s+(.+)/i
+
+        markers[::Regexp.last_match(1)] = ::Regexp.last_match(2).strip
+      end
+      markers
+    end
+
+    def self.find_sections(sgml_text, **_unused)
+      extract_sections(sgml_text)
+    end
+  end
+end