finmodeling 0.1

data/examples/show_report.rb

@@ -0,0 +1,218 @@
+#!/usr/bin/env ruby
+
+require 'finmodeling'
+
+class Arguments
+  def self.show_usage_and_exit
+    puts "usage:"
+    puts "\t#{__FILE__} [options] <stock symbol> <10-k|10-q> <0, for most recent|-1 for prevous|-2, etc>"
+    puts "\t#{__FILE__} [options] <report URL> <10-k|10-q>"
+    puts
+    puts "\tOptions:"
+    puts "\t\t--no-cache: disables caching"
+    puts "\t\t--show-disclosures: prints out all the disclosure calculations in the filing"
+    exit
+  end
+  
+  def self.parse(raw_args)
+    parsed_args = { :stock_symbol => nil, :filing_url => nil, 
+                    :report_type => nil, :report_offset => nil, 
+                    :show_disclosures => false }
+
+    while raw_args.any? && raw_args.first =~ /^--/
+      case raw_args.first.downcase
+        when '--no-cache'
+          FinModeling::Config.disable_caching
+          puts "Caching is #{FinModeling::Config.caching_enabled? ? "enabled" : "disabled"}"
+        when '--show-disclosures'
+          parsed_args[:show_disclosures] = true
+          puts "Showing disclosures"
+        else
+          self.show_usage_and_exit
+      end
+      raw_args = raw_args[1..-1]
+    end
+
+    if raw_args[0] =~ /http/
+      return self.parse_just_a_url(raw_args, parsed_args)
+    else
+      return self.parse_symbol_etc(raw_args, parsed_args)
+    end
+  end
+
+  protected
+
+  def self.parse_just_a_url(raw_args, parsed)
+    self.show_usage_and_exit if raw_args.length != 2
+
+    parsed_args[:filing_url]  = raw_args[0]
+    parsed_args[:report_type] = case raw_args[1].downcase
+      when "10-k"
+        :annual_report
+      when "10-q"
+        :quarterly_report
+      else
+        self.show_usage_and_exit
+    end
+
+    return parsed_args
+  end
+  
+  def self.parse_symbol_etc(raw_args, parsed_args)
+    self.show_usage_and_exit if raw_args.length != 3
+  
+    parsed_args[:stock_symbol]  = raw_args[0]
+    parsed_args[:report_type]   = case raw_args[1].downcase
+      when "10-k"
+        :annual_report
+      when "10-q"
+        :quarterly_report
+      else
+        self.show_usage_and_exit
+    end
+    parsed_args[:report_offset] = raw_args[2].to_i
+
+    return parsed_args
+  end
+end
+
+def get_company_filing_url(stock_symbol, report_type, report_offset)
+  company = FinModeling::Company.find(stock_symbol)
+  raise RuntimeError.new("couldn't find company") if company.nil?
+  puts "company name: #{company.name}"
+
+  filing_url = case report_type
+    when :annual_report
+      raise RuntimeError.new("company has no annual reports") if company.annual_reports.length == 0
+      company.annual_reports[-1+report_offset].link
+    when :quarterly_report
+      raise RuntimeError.new("company has no quarterly reports") if company.annual_reports.length == 0
+      company.quarterly_reports[-1+report_offset].link
+  end
+
+  return filing_url
+end
+
+def get_filing(filing_url, report_type)
+  puts "url:          #{filing_url}\n"
+  return FinModeling::AnnualReportFiling.download(   filing_url) if report_type == :annual_report
+  return FinModeling::QuarterlyReportFiling.download(filing_url) if report_type == :quarterly_report
+end
+
+def print_balance_sheet(filing, report_type)
+  period = filing.balance_sheet.periods.last
+  puts "Balance Sheet (#{period.to_pretty_s})"
+  
+  summaries = []
+  summaries << filing.balance_sheet.assets_calculation.summary(:period => period)
+  summaries << filing.balance_sheet.liabs_and_equity_calculation.summary(:period => period)
+
+  print_summaries(summaries)
+end
+
+def print_reformulated_balance_sheet(filing, report_type)
+  period = filing.balance_sheet.periods.last
+
+  reformed_balance_sheet = filing.balance_sheet.reformulated(period)
+
+  summaries = []
+  summaries << reformed_balance_sheet.net_operating_assets
+  summaries << reformed_balance_sheet.net_financial_assets
+  summaries << reformed_balance_sheet.common_shareholders_equity
+
+  print_summaries(summaries)
+end
+
+def print_income_statement(filing, report_type)
+  period  = filing.income_statement.net_income_calculation.periods.yearly.last    if report_type == :annual_report
+  period  = filing.income_statement.net_income_calculation.periods.quarterly.last if report_type == :quarterly_report
+  puts "Income Statement (#{period.to_pretty_s})"
+
+  summaries = []
+  summaries << filing.income_statement.net_income_calculation.summary(:period => period)
+
+  print_summaries(summaries)
+end
+
+def print_reformulated_income_statement(filing, report_type)
+  period  = filing.income_statement.net_income_calculation.periods.yearly.last    if report_type == :annual_report
+  period  = filing.income_statement.net_income_calculation.periods.quarterly.last if report_type == :quarterly_report
+
+  reformed_inc_stmt  = filing.income_statement.reformulated(period)
+
+  summaries = []
+  summaries << reformed_inc_stmt.gross_revenue
+  summaries << reformed_inc_stmt.income_from_sales_before_tax
+  summaries << reformed_inc_stmt.income_from_sales_after_tax
+  summaries << reformed_inc_stmt.operating_income_after_tax
+  summaries << reformed_inc_stmt.net_financing_income
+  summaries << reformed_inc_stmt.comprehensive_income
+
+  print_summaries(summaries)
+end
+
+def print_cash_flow_statement(filing, report_type)
+  period = filing.cash_flow_statement.periods.yearly.last    if report_type == :annual_report
+  period = filing.cash_flow_statement.periods.quarterly.last if report_type == :quarterly_report
+  puts "Cash Flow Statement (#{period.to_pretty_s})"
+  
+  summaries = []
+  summaries << filing.cash_flow_statement.cash_change_calculation.summary(:period => period)
+
+  print_summaries(summaries)
+end
+
+def print_reformulated_cash_flow_statement(filing, report_type)
+  period = filing.cash_flow_statement.periods.yearly.last    if report_type == :annual_report
+  period = filing.cash_flow_statement.periods.quarterly.last if report_type == :quarterly_report
+  
+  reformed_cash_flow_stmt  = filing.cash_flow_statement.reformulated(period)
+
+  summaries = []
+  summaries << reformed_cash_flow_stmt.cash_from_operations
+  summaries << reformed_cash_flow_stmt.cash_investments_in_operations
+  summaries << reformed_cash_flow_stmt.payments_to_debtholders
+  summaries << reformed_cash_flow_stmt.payments_to_stockholders
+  summaries << reformed_cash_flow_stmt.free_cash_flow
+  summaries << reformed_cash_flow_stmt.financing_flows
+
+  print_summaries(summaries)
+end
+
+def print_disclosures(filing, report_type)
+  puts "Disclosures"
+
+  summaries = []
+  filing.disclosures.each do |disclosure|
+    summaries << disclosure.summary(:period => disclosure.periods.last)
+  end
+
+  print_summaries(summaries)
+end
+
+def print_summaries(summaries)
+  summaries.each do |summary|
+    summary.key_width = 60
+    summary.val_width = 18
+    summary.print
+  end
+end
+
+
+
+args = Arguments.parse(ARGV)
+if args[:filing_url].nil?
+  args[:filing_url] = get_company_filing_url(args[:stock_symbol], args[:report_type], args[:report_offset])
+end
+
+filing = get_filing(        args[:filing_url], args[:report_type])
+
+print_balance_sheet(                   filing, args[:report_type])
+print_reformulated_balance_sheet(      filing, args[:report_type])
+print_income_statement(                filing, args[:report_type])
+print_reformulated_income_statement(   filing, args[:report_type])
+print_cash_flow_statement(             filing, args[:report_type])
+print_reformulated_cash_flow_statement(filing, args[:report_type])
+print_disclosures(                     filing, args[:report_type]) if args[:show_disclosures]
+
+raise RuntimeError.new("filing is not valid") if !filing.is_valid?

data/examples/show_reports.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+require 'finmodeling'
+
+class Arguments
+  def self.show_usage_and_exit
+    puts "usage:"
+    puts "\t#{__FILE__} [options] <stock symbol> <start date, e.g. '2010-01-01'>"
+    puts
+    puts "\tOptions:"
+    puts "\t\t--num-forecasts <num>: how many periods to forecast"
+    puts "\t\t--no-cache: disable caching"
+    puts "\t\t--balance-detail: show details about the balance sheet calculation"
+    puts "\t\t--income-detail: show details about the net income calculation"
+    exit
+  end
+  
+  def self.parse(args)
+    a = { :stock_symbol => nil, :start_date => nil, :num_forecasts => nil }
+
+    while args.any? && args.first =~ /^--/
+      case args.first.downcase
+        when '--no-cache'
+          FinModeling::Config.disable_caching
+          puts "Caching is #{FinModeling::Config.caching_enabled? ? "enabled" : "disabled"}"
+        when '--balance-detail'
+          FinModeling::Config.enable_balance_detail
+          puts "Balance sheet detail is #{FinModeling::Config.balance_detail_enabled? ? "enabled" : "disabled"}"
+        when '--income-detail'
+          FinModeling::Config.enable_income_detail
+          puts "Net income detail is #{FinModeling::Config.income_detail_enabled? ? "enabled" : "disabled"}"
+        when '--num-forecasts'
+          a[:num_forecasts] = args[1].to_i
+          self.show_usage_and_exit unless a[:num_forecasts] >= 1
+          puts "Forecasting #{a[:num_forecasts]} periods"
+          args = args[1..-1]
+        else
+          self.show_usage_and_exit
+      end
+      args = args[1..-1]
+    end
+  
+    self.show_usage_and_exit if args.length != 2
+    a[:stock_symbol]  = args[0]
+    a[:start_date] = Time.parse(args[1])
+  
+    return a
+  end
+end
+
+args = Arguments.parse(ARGV)
+
+company = FinModeling::Company.find(args[:stock_symbol])
+raise RuntimeError.new("couldn't find company") if !company
+puts "company name: #{company.name}"
+
+filings = FinModeling::CompanyFilings.new(company.filings_since_date(args[:start_date]))
+if filings.empty?
+  puts "No filings..."
+  exit
+end
+
+forecasts = filings.forecasts(filings.choose_forecasting_policy, num_quarters=args[:num_forecasts]) if args[:num_forecasts]
+
+bs_analyses = filings.balance_sheet_analyses 
+bs_analyses += forecasts.balance_sheet_analyses(filings) if forecasts
+bs_analyses.totals_row_enabled = false 
+bs_analyses.print
+filings.balance_sheet_analyses.print_extras if filings.balance_sheet_analyses.respond_to?(:print_extras)
+
+is_analyses = filings.income_statement_analyses 
+is_analyses += forecasts.income_statement_analyses(filings) if forecasts
+is_analyses.totals_row_enabled = false 
+is_analyses.print
+filings.income_statement_analyses.print_extras if filings.income_statement_analyses.respond_to?(:print_extras)
+
+filings.cash_flow_statement_analyses.print

data/finmodeling.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/finmodeling/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jim Lindstrom"]
+  gem.email         = ["jim.lindstrom@gmail.com"]
+  gem.description   = %q{A gem for manipulating XBRL financial filings}
+  gem.summary       = %q{A gem for manipulating XBRL financial filings}
+  gem.homepage      = "https://github.com/jimlindstrom/FinModeling"
+
+  gem.add_dependency("fileutils")
+  gem.add_dependency("sec_query")
+  gem.add_dependency("edgar")
+
+  gem.add_dependency("xbrlware-ruby19", "1.1.2.19.2")
+  gem.add_dependency("xbrlware-extras", "1.1.2.19.2")
+
+  gem.add_dependency("sec_query")
+  gem.add_dependency("naive_bayes")
+  gem.add_dependency("statsample")
+
+  gem.add_development_dependency("rspec", "2.5")
+  gem.add_development_dependency("rake")
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "finmodeling"
+  gem.require_paths = ["lib"]
+  gem.version       = FinModeling::VERSION
+end

data/lib/finmodeling/annual_report_filing.rb

@@ -0,0 +1,104 @@
+module FinModeling
+
+  class AnnualReportFiling < CompanyFiling
+
+    CONSTRUCTOR_PATH = File.join(FinModeling::BASE_PATH, "constructors/")
+    SCHEMA_VERSION_ITEM = "@schema_version"
+    CURRENT_SCHEMA_VERSION = 1.1
+    # History:
+    # 1.0: initial version
+    # 1.1: added CFS to quarterly filings
+    #      added disclosures
+    #      renamed fake(.*)report to cached(.*)report
+
+    def self.download(url)
+      uid = url.split("/")[-2..-1].join('-').gsub(/\.[A-zA-z]*$/, '')
+      constructor_file = CONSTRUCTOR_PATH + uid + '.rb'
+      if File.exists?(constructor_file) && FinModeling::Config.caching_enabled?
+        begin
+          eval(File.read(constructor_file))
+          #puts "info: annual report, cache hit. schema version: #{@schema_version}"
+          return @filing if @schema_version == CURRENT_SCHEMA_VERSION
+        rescue
+          #puts "warn: annual report, cache hit. error eval'ing though."
+        end
+      end
+
+      filing = super(url)
+
+      FileUtils.mkdir_p(CONSTRUCTOR_PATH) if !File.exists?(CONSTRUCTOR_PATH)
+      file = File.open(constructor_file, "w")
+      filing.write_constructor(file, "@filing")
+      file.close
+
+      return filing
+    end
+
+    def balance_sheet
+      if @balance_sheet.nil?
+        calculations=@taxonomy.callb.calculation
+        bal_sheet = calculations.find{ |x| (x.clean_downcased_title =~ /statement.*financial.*position/) or
+                                           (x.clean_downcased_title =~ /statement.*financial.*condition/) or
+                                           (x.clean_downcased_title =~ /balance.*sheet/) }
+        if bal_sheet.nil?
+          raise RuntimeError.new("Couldn't find balance sheet in: " + calculations.map{ |x| "\"#{x.clean_downcased_title}\"" }.join("; "))
+        end
+    
+        @balance_sheet = BalanceSheetCalculation.new(bal_sheet)
+      end
+      return @balance_sheet
+    end
+
+    def income_statement
+      if @income_stmt.nil?
+        calculations=@taxonomy.callb.calculation
+        inc_stmt = calculations.find{ |x| (x.clean_downcased_title =~ /statement.*operations/) or
+                                          (x.clean_downcased_title =~ /statement[s]*.*of.*earnings/) or
+                                          (x.clean_downcased_title =~ /statement[s]*.*of.*income/) or
+                                          (x.clean_downcased_title =~ /statement[s]*.*of.*net.*income/) }
+        if inc_stmt.nil?
+          raise RuntimeError.new("Couldn't find income statement in: " + calculations.map{ |x| "\"#{x.clean_downcased_title}\"" }.join("; "))
+        end
+    
+        @income_stmt = IncomeStatementCalculation.new(inc_stmt)
+      end
+      return @income_stmt
+    end
+
+    def cash_flow_statement
+      if @cash_flow_stmt.nil?
+        calculations=@taxonomy.callb.calculation
+        cash_flow_stmt = calculations.find{ |x| (x.clean_downcased_title =~ /statement.*cash.*flows/) or
+                                                (x.clean_downcased_title =~ /^cash flows$/) }
+        if cash_flow_stmt.nil?
+          raise RuntimeError.new("Couldn't find cash flow statement in: " + calculations.map{ |x| "\"#{x.clean_downcased_title}\"" }.join("; "))
+        end
+    
+        @cash_flow_stmt = CashFlowStatementCalculation.new(cash_flow_stmt)
+      end
+      return @cash_flow_stmt
+    end
+
+    def is_valid?
+      return (income_statement.is_valid? and balance_sheet.is_valid? and cash_flow_statement.is_valid?)
+    end
+
+    def write_constructor(file, item_name)
+      balance_sheet.write_constructor(      file, bs_name  = item_name + "_bs")
+      income_statement.write_constructor(   file, is_name  = item_name + "_is")
+      cash_flow_statement.write_constructor(file, cfs_name = item_name + "_cfs")
+
+      names_of_discs = []
+      disclosures.each_with_index do |disclosure, idx|
+        name_of_disc = item_name + "_disc#{idx}"
+        disclosure.write_constructor(file, name_of_disc)
+        names_of_discs << name_of_disc
+      end
+      names_of_discs_str = "[" + names_of_discs.join(',') + "]"
+
+      file.puts "#{SCHEMA_VERSION_ITEM} = #{CURRENT_SCHEMA_VERSION}" 
+
+      file.puts "#{item_name} = FinModeling::CachedAnnualFiling.new(#{bs_name}, #{is_name}, #{cfs_name}, #{names_of_discs_str})"
+    end
+  end
+end

data/lib/finmodeling/array_with_stats.rb

@@ -0,0 +1,22 @@
+module FinModeling
+
+  class ArrayWithStats < Array
+    def mean
+      return nil if empty?
+      self.inject(:+) / self.length
+    end
+
+    def variance
+      x_sqrd = self.map{ |x| x*x }
+      x_sqrd_mean = (ArrayWithStats.new(x_sqrd).mean)
+      x_sqrd_mean - (mean**2)
+    end
+
+    def linear_regression
+      x = Array(0..(self.length-1)).to_scale
+      y = self.to_scale
+      Statsample::Regression.simple(x,y)
+    end
+  end
+
+end

data/lib/finmodeling/assets_calculation.rb

@@ -0,0 +1,36 @@
+module FinModeling
+
+  class AssetsCalculation < CompanyFilingCalculation
+    include CanCacheClassifications
+    include CanCacheSummaries
+    include CanClassifyRows
+
+    BASE_FILENAME = File.join(FinModeling::BASE_PATH, "summaries/ai_")
+
+    ALL_STATES  =          [ :oa, :fa ]
+    NEXT_STATES = { nil => [ :oa, :fa ],
+                    :oa => [ :oa, :fa ],
+                    :fa => [ :oa, :fa ] }
+
+    def summary(args)
+      summary_cache_key = args[:period].to_pretty_s
+      thesummary = lookup_cached_summary(summary_cache_key)
+      return thesummary if !thesummary.nil?
+
+      mapping = Xbrlware::ValueMapping.new
+      mapping.policy[:credit] = :flip
+
+      thesummary = super(:period => args[:period], :mapping => mapping)
+      if !lookup_cached_classifications(BASE_FILENAME, thesummary.rows)
+        lookahead = [4, thesummary.rows.length-1].min
+        classify_rows(ALL_STATES, NEXT_STATES, thesummary.rows, FinModeling::AssetsItem, lookahead)
+        save_cached_classifications(BASE_FILENAME, thesummary.rows)
+      end
+
+      save_cached_summary(summary_cache_key, thesummary)
+
+      return thesummary
+    end
+
+  end
+end

data/lib/finmodeling/assets_item.rb

@@ -0,0 +1,14 @@
+module FinModeling
+  class AssetsItem < String
+    include HasStringClassifier
+
+    BASE_FILENAME = File.join(FinModeling::BASE_PATH, "classifiers/ai_")
+    TYPES         = [ :oa, :fa ]
+
+    has_string_classifier(TYPES, AssetsItem)
+
+    def self.load_vectors_and_train
+      self._load_vectors_and_train(BASE_FILENAME, FinModeling::AssetsItem::TRAINING_VECTORS)
+    end
+  end
+end