finmodeling 0.1

data/lib/finmodeling/company_filings.rb

@@ -0,0 +1,100 @@
+module FinModeling
+  class CompanyFilings < Array
+    def balance_sheet_analyses
+      if !@balance_sheet_analyses
+        re_bs = nil
+        self.each do |filing|
+          prev_re_bs = re_bs
+          re_bs = filing.balance_sheet.reformulated(filing.balance_sheet.periods.last)
+          next_analysis = re_bs.analysis(prev_re_bs)
+    
+          @balance_sheet_analyses = @balance_sheet_analyses + next_analysis if  @balance_sheet_analyses
+          @balance_sheet_analyses =                           next_analysis if !@balance_sheet_analyses
+        end
+        @balance_sheet_analyses = BalanceSheetAnalyses.new(@balance_sheet_analyses)
+      end
+      return @balance_sheet_analyses
+    end
+  
+    def income_statement_analyses
+      if !@income_statement_analyses
+        prev_re_bs, prev_re_is, prev_filing  = [nil, nil, nil]
+        self.each do |filing|
+          re_is = filing.income_statement.latest_quarterly_reformulated(prev_filing ? prev_filing.income_statement : nil)
+          re_bs = filing.balance_sheet.reformulated(filing.balance_sheet.periods.last)
+    
+          next_analysis = FinModeling::ReformulatedIncomeStatement.empty_analysis if !re_is
+          next_analysis = re_is.analysis(re_bs, prev_re_is, prev_re_bs)           if  re_is
+        
+          @income_statement_analyses = @income_statement_analyses + next_analysis if  @income_statement_analyses
+          @income_statement_analyses =                              next_analysis if !@income_statement_analyses
+      
+          prev_re_bs, prev_re_is, prev_filing  = [re_bs, re_is, filing]
+        end
+        @income_statement_analyses = IncomeStatementAnalyses.new(@income_statement_analyses)
+      end
+      return @income_statement_analyses
+    end
+  
+    def cash_flow_statement_analyses
+      if !@cash_flow_statement_analyses
+        prev_filing, prev_re_cfs = [nil, nil]
+        self.each do |filing|
+          re_is = filing.income_statement.latest_quarterly_reformulated(prev_filing ? prev_filing.income_statement : nil)
+          re_cfs = filing.cash_flow_statement.latest_quarterly_reformulated(prev_filing ? prev_filing.cash_flow_statement : nil)
+      
+          next_analysis = FinModeling::ReformulatedCashFlowStatement.empty_analysis if !re_cfs
+          next_analysis = re_cfs.analysis(re_is)                                    if  re_cfs
+        
+          @cash_flow_statement_analyses = @cash_flow_statement_analyses + next_analysis if  @cash_flow_statement_analyses
+          @cash_flow_statement_analyses =                                 next_analysis if !@cash_flow_statement_analyses
+    
+          prev_filing, prev_re_cfs = [filing, re_cfs]
+        end
+        @cash_flow_statement_analyses.totals_row_enabled = false
+      end
+      return @cash_flow_statement_analyses
+    end
+  
+    def choose_forecasting_policy
+      if length < 3
+        return FinModeling::GenericForecastingPolicy.new
+      else
+        isa = income_statement_analyses
+        args = { }
+        args[:revenue_growth] = isa.revenue_growth_row.valid_vals.mean
+        args[:sales_pm      ] = isa.operating_pm_row.valid_vals.mean
+        args[:sales_over_noa] = isa.sales_over_noa_row.valid_vals.mean
+        args[:fi_over_nfa   ] = isa.fi_over_nfa_row.valid_vals.mean
+        return FinModeling::ConstantForecastingPolicy.new(args)
+      end
+    end
+  
+    def forecasts(policy, num_quarters)
+      f = Forecasts.new
+
+      last_re_bs = self.last.balance_sheet.reformulated(self.last.balance_sheet.periods.last)
+
+      last_last_is = (self.length >= 2) ? self[-2].income_statement : nil
+      puts "warning: last_last_is is nil..." if !last_last_is
+      last_re_is = self.last.income_statement.latest_quarterly_reformulated(last_last_is)
+      raise RuntimeError.new("last_re_is is nil!") if !last_re_is
+
+      num_quarters.times do |i|
+        next_bs_period = last_re_bs.period.plus_n_months(3)
+        next_is_period = last_re_is.period.plus_n_months(3)
+  
+        next_re_is = FinModeling::ReformulatedIncomeStatement.forecast_next(next_is_period, policy, last_re_bs, last_re_is)
+        next_re_bs = FinModeling::ReformulatedBalanceSheet   .forecast_next(next_bs_period, policy, last_re_bs, next_re_is)
+
+        f.reformulated_income_statements << next_re_is
+        f.reformulated_balance_sheets    << next_re_bs
+
+        last_last_re_is, last_re_bs, last_re_is = [last_re_is, next_re_bs, next_re_is]
+      end
+
+      return f
+    end
+
+  end
+end

data/lib/finmodeling/config.rb

@@ -0,0 +1,37 @@
+module FinModeling
+  class Config
+    @@caching_enabled = true
+    def self.enable_caching
+      @@caching_enabled = true
+    end
+    def self.disable_caching
+      @@caching_enabled = false
+    end
+    def self.caching_enabled?
+      @@caching_enabled
+    end
+
+    @@balance_detail_enabled = false
+    def self.enable_balance_detail
+      @@balance_detail_enabled = true
+    end
+    def self.disable_balance_detail
+      @@balance_detail_enabled = false
+    end
+    def self.balance_detail_enabled?
+      @@balance_detail_enabled
+    end
+
+    @@income_detail_enabled = false
+    def self.enable_income_detail
+      @@income_detail_enabled = true
+    end
+    def self.disable_income_detail
+      @@income_detail_enabled = false
+    end
+    def self.income_detail_enabled?
+      @@income_detail_enabled
+    end
+  end
+end
+

data/lib/finmodeling/constant_forecasting_policy.rb

@@ -0,0 +1,23 @@
+module FinModeling
+  class ConstantForecastingPolicy 
+    def initialize(args)
+      @vals = args
+    end
+
+    def revenue_growth
+      @vals[:revenue_growth]
+    end
+  
+    def sales_pm
+      @vals[:sales_pm]
+    end
+  
+    def fi_over_nfa
+      @vals[:fi_over_nfa]
+    end
+  
+    def sales_over_noa
+      @vals[:sales_over_noa]
+    end
+  end
+end

data/lib/finmodeling/factory.rb

@@ -0,0 +1,27 @@
+module FinModeling
+  class Factory
+    def self.IncomeStatementCalculation(args = {})
+      entity_details = {}
+      title = ""
+      role = ""
+      href = ""
+      arcs = []
+      contexts = nil
+
+      calculation = Xbrlware::Linkbase::CalculationLinkbase::Calculation.new(entity_details, title, role, href, arcs, contexts)
+      return FinModeling::IncomeStatementCalculation.new(calculation)
+    end
+
+    def self.BalanceSheetCalculation(args = {})
+      entity_details = {}
+      title = ""
+      role = ""
+      href = ""
+      arcs = []
+      contexts = nil
+
+      calculation = Xbrlware::Linkbase::CalculationLinkbase::Calculation.new(entity_details, title, role, href, arcs, contexts)
+      return FinModeling::BalanceSheetCalculation.new(calculation)
+    end
+  end
+end

data/lib/finmodeling/float_helpers.rb

@@ -0,0 +1,17 @@
+class Fixnum
+  def to_nearest_million
+    return (self/1000000.0).round.to_f
+  end
+  def to_nearest_thousand
+    return (self/1000.0).round.to_f
+  end
+end
+
+class Float
+  def to_nearest_million
+    return (self/1000000.0).round.to_f
+  end
+  def to_nearest_thousand
+    return (self/1000.0).round.to_f
+  end
+end

data/lib/finmodeling/forecasts.rb

@@ -0,0 +1,48 @@
+module FinModeling
+  class Forecasts
+    attr_accessor :reformulated_income_statements, :reformulated_balance_sheets
+
+    def initialize
+      @reformulated_income_statements = []
+      @reformulated_balance_sheets    = []
+    end
+
+    def balance_sheet_analyses(filings)
+      if !@balance_sheet_analyses
+        prev_filing = filings.last
+        prev_re_bs = prev_filing.balance_sheet.reformulated(prev_filing.balance_sheet.periods.last)
+        @reformulated_balance_sheets.each do |re_bs|
+          next_analysis = re_bs.analysis(prev_re_bs)
+    
+          @balance_sheet_analyses = @balance_sheet_analyses + next_analysis if  @balance_sheet_analyses
+          @balance_sheet_analyses =                           next_analysis if !@balance_sheet_analyses
+          prev_re_bs = re_bs
+        end
+        @balance_sheet_analyses = BalanceSheetAnalyses.new(@balance_sheet_analyses)
+      end
+      return @balance_sheet_analyses
+    end
+  
+    def income_statement_analyses(filings)
+      if !@income_statement_analyses
+        prev_filing = filings.last
+        prev_re_bs = prev_filing.balance_sheet.reformulated(prev_filing.balance_sheet.periods.last)
+        prev_prev_is = (filings.length > 2) ? filings[-2].income_statement : nil
+        prev_re_is = prev_filing.income_statement.latest_quarterly_reformulated(prev_prev_is)
+      
+        @reformulated_income_statements.zip(@reformulated_balance_sheets).each do |re_is, re_bs|
+          next_analysis = FinModeling::ReformulatedIncomeStatement.empty_analysis if !re_is
+          next_analysis = re_is.analysis(re_bs, prev_re_is, prev_re_bs)           if  re_is
+        
+          @income_statement_analyses = @income_statement_analyses + next_analysis if  @income_statement_analyses
+          @income_statement_analyses =                              next_analysis if !@income_statement_analyses
+      
+          prev_re_bs, prev_re_is  = [re_bs, re_is]
+        end
+        @income_statement_analyses = IncomeStatementAnalyses.new(@income_statement_analyses)
+      end
+      return @income_statement_analyses
+    end
+
+  end
+end

data/lib/finmodeling/generic_forecasting_policy.rb

@@ -0,0 +1,19 @@
+module FinModeling
+  class GenericForecastingPolicy 
+    def revenue_growth
+      0.04
+    end
+  
+    def sales_pm
+      0.20
+    end
+  
+    def fi_over_nfa
+      0.01
+    end
+  
+    def sales_over_noa
+      2.00
+    end
+  end
+end

data/lib/finmodeling/has_string_classifer.rb

@@ -0,0 +1,96 @@
+module FinModeling
+
+  module HasStringClassifier
+    module ClassMethods
+      @klasses     ||= []
+      @classifiers ||= {}
+      @item_klass  ||= nil
+
+      def has_string_classifier(klasses, item_klass)
+        @klasses     = klasses
+        @item_klass  = item_klass
+        @classifiers = Hash[ *klasses.zip(klasses.map{ |x| NaiveBayes.new(:yes, :no) }).flatten ]
+      end
+  
+      def _load_vectors_and_train(base_filename, vectors)
+        FileUtils.mkdir_p(File.dirname(base_filename)) if !File.exists?(File.dirname(base_filename))
+        success = FinModeling::Config.caching_enabled?
+        @klasses.each do |cur_klass|
+          filename = base_filename + cur_klass.to_s + ".db"
+          success = success && File.exists?(filename)
+          if success
+            @classifiers[cur_klass] = NaiveBayes.load(filename)
+          else
+            @classifiers[cur_klass].db_filepath = filename
+          end
+        end
+        return if success
+  
+        vectors.each do |vector|
+          begin
+            item = @item_klass.new(vector[:item_string])
+            item.train(vector[:klass])
+          rescue Exception => e
+            puts "\"#{vector[:item_string]}\" has a bogus klass: \"#{vector[:klass]}\""
+            puts "\t" + e.message
+            puts "\t" + e.backtrace.inspect.gsub(/, /, "\n\t ")
+          end
+        end
+  
+        @klasses.each do |cur_klass|
+          @classifiers[cur_klass].save
+        end
+      end
+
+      def klasses
+        @klasses
+      end
+
+      def classifiers
+        @classifiers
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def train(expected_klass)
+      raise TypeError.new("#{expected_klass} is not in #{self.class.klasses}") if !self.class.klasses.include?(expected_klass)
+
+      self.class.klasses.each do |cur_klass|
+        is_expected_klass = (expected_klass == cur_klass) ? :yes : :no
+        self.class.classifiers[cur_klass].train(is_expected_klass, *tokenize)
+      end
+    end
+
+    def classification_estimates
+      tokens = tokenize
+
+      estimates = {}
+      self.class.klasses.each do |cur_klass|
+        ret = self.class.classifiers[cur_klass].classify(*tokens)
+        result = {:klass=>ret[0], :confidence=>ret[1]}
+        estimates[cur_klass] = (result[:klass] == :yes) ? result[:confidence] : -result[:confidence]
+      end
+
+      return estimates
+    end
+
+    def classify
+      estimates = classification_estimates
+      best_guess_klass = estimates.keys.sort{ |x,y| estimates[x] <=> estimates[y] }.last
+      return best_guess_klass
+    end
+
+    def tokenize
+      words = ["^"] + self.downcase.split(" ") + ["$"]
+
+      tokens = [1, 2, 3].collect do |words_per_token|
+        words.each_cons(words_per_token).to_a.map{|x| x.join(" ") }
+      end
+      return tokens.flatten
+    end
+  end
+
+end

data/lib/finmodeling/income_statement_analyses.rb

@@ -0,0 +1,74 @@
+module FinModeling
+
+  class IncomeStatementAnalyses < CalculationSummary
+    def initialize(calc_summary)
+      @title              = calc_summary.title
+      @rows               = calc_summary.rows
+      @header_row         = calc_summary.header_row
+      @key_width          = calc_summary.key_width
+      @val_width          = calc_summary.val_width
+      @max_decimals       = calc_summary.max_decimals
+      @totals_row_enabled = false
+    end
+
+    def print_extras
+      if operating_pm_row && operating_pm_row.valid_vals.any?
+        lr = operating_pm_row.valid_vals.linear_regression
+        puts "\t\toperating pm: "+
+             "a:#{lr.a.to_s.cap_decimals(4)}, "+
+             "b:#{lr.b.to_s.cap_decimals(4)}, "+
+             "r:#{lr.r.to_s.cap_decimals(4)}, "+
+             "var:#{operating_pm_row.valid_vals.variance.to_s.cap_decimals(4)}"
+      end
+
+      if sales_over_noa_row && sales_over_noa_row.valid_vals.any?
+        lr = sales_over_noa_row.valid_vals.linear_regression
+        puts "\t\tsales / noa: "+
+             "a:#{lr.a.to_s.cap_decimals(4)}, "+
+             "b:#{lr.b.to_s.cap_decimals(4)}, "+
+             "r:#{lr.r.to_s.cap_decimals(4)}, "+
+             "var:#{sales_over_noa_row.valid_vals.variance.to_s.cap_decimals(4)}"
+      end
+
+      if revenue_growth_row && revenue_growth_row.valid_vals.any?
+        lr = revenue_growth_row.valid_vals.linear_regression
+        puts "\t\trevenue growth: "+
+             "a:#{lr.a.to_s.cap_decimals(4)}, "+
+             "b:#{lr.b.to_s.cap_decimals(4)}, "+
+             "r:#{lr.r.to_s.cap_decimals(4)}, "+
+             "var:#{revenue_growth_row.valid_vals.variance.to_s.cap_decimals(4)}"
+      end
+
+      if fi_over_nfa_row && fi_over_nfa_row.valid_vals.any?
+        lr = fi_over_nfa_row.valid_vals.linear_regression
+        puts "\t\tfi / nfa: "+
+             "a:#{lr.a.to_s.cap_decimals(4)}, "+
+             "b:#{lr.b.to_s.cap_decimals(4)}, "+
+             "r:#{lr.r.to_s.cap_decimals(4)}, "+
+             "var:#{fi_over_nfa_row.valid_vals.variance.to_s.cap_decimals(4)}"
+      end
+    end
+
+    def revenue_growth_row
+      find_row_by_key('Revenue Growth')
+    end
+
+    def operating_pm_row
+      find_row_by_key('Operating PM')
+    end
+
+    def sales_over_noa_row
+      find_row_by_key('Sales / NOA')
+    end
+
+    def fi_over_nfa_row
+      find_row_by_key('FI / NFA')
+    end
+
+    def find_row_by_key(key)
+      self.rows.find{ |x| x.key == key }
+    end
+  end
+
+end
+

data/lib/finmodeling/income_statement_calculation.rb

@@ -0,0 +1,71 @@
+module FinModeling
+  class IncomeStatementCalculation < CompanyFilingCalculation
+
+    def net_income_calculation
+      if @ni.nil?
+        friendly_goal = "net income"
+        label_regexes = [ /^net (income|loss|loss income)/,
+                          /^profit loss$/,
+                          /^allocation.*of.*undistributed.*earnings/ ]
+        id_regexes    = [ /^(|Locator_|loc_)(|us-gaap_)NetIncomeLoss[_0-9a-z]+/,
+                          /^(|Locator_|loc_)(|us-gaap_)NetIncomeLossAvailableToCommonStockholdersBasic[_0-9a-z]+/,
+                          /^(|Locator_|loc_)(|us-gaap_)ProfitLoss[_0-9a-z]+/ ]
+        calc = find_and_verify_calculation_arc(friendly_goal, label_regexes, id_regexes)
+        @ni = NetIncomeCalculation.new(calc)
+      end
+      return @ni
+    end
+
+    def is_valid?
+      has_revenue_item = false
+      has_tax_item     = false
+      net_income_calculation.leaf_items.each do |leaf|
+        if !has_revenue_item and leaf.name.downcase.matches_regexes?([/revenue/, /sales/])
+          has_revenue_item = true
+        end
+        if !has_tax_item and leaf.name.downcase.matches_regexes?([/tax/])
+          has_tax_item     = true
+        end
+      end
+
+      puts "income statement's net income calculation lacks tax item" if !has_tax_item 
+      puts "income statement's net income calculation lacks sales/revenue item" if !has_revenue_item 
+      return (has_revenue_item and has_tax_item)
+    end
+
+    def reformulated(period)
+      return ReformulatedIncomeStatement.new(period, 
+                                             net_income_calculation.summary(:period=>period))
+    end
+
+    def latest_quarterly_reformulated(prev_income_statement)
+      if (net_income_calculation.periods.quarterly.any?) &&
+         (reformulated(net_income_calculation.periods.quarterly.last).operating_revenues.total.abs > 1.0) && # FIXME: make an is_valid here?
+         (reformulated(net_income_calculation.periods.quarterly.last).cost_of_revenues.total.abs > 1.0) # FIXME: make an is_valid here?
+        return reformulated(net_income_calculation.periods.quarterly.last)
+
+      elsif !prev_income_statement
+        return nil
+
+      elsif net_income_calculation.periods.yearly.any? &&
+            prev_income_statement.net_income_calculation.periods.threequarterly.any?
+        is_period = net_income_calculation.periods.yearly.last
+        re_is     = reformulated(is_period)
+  
+        period_1q_thru_3q = prev_income_statement.net_income_calculation.periods.threequarterly.last
+        prev3q  = prev_income_statement.reformulated(period_1q_thru_3q)
+        re_is   = re_is - prev3q
+        return re_is 
+      end
+  
+      return nil
+    end
+ 
+    def write_constructor(file, item_name)
+      item_calc_name = item_name + "_calc"
+      @calculation.write_constructor(file, item_calc_name)
+      file.puts "#{item_name} = FinModeling::IncomeStatementCalculation.new(#{item_calc_name})"
+    end
+
+  end
+end

data/lib/finmodeling/income_statement_item.rb

@@ -0,0 +1,14 @@
+module FinModeling
+  class IncomeStatementItem < String
+    include HasStringClassifier
+
+    BASE_FILENAME = File.join(FinModeling::BASE_PATH, "classifiers/isi_")
+    TYPES         = [ :or, :cogs, :oe, :oibt, :fibt, :tax, :ooiat, :fiat ]
+
+    has_string_classifier(TYPES, IncomeStatementItem)
+
+    def self.load_vectors_and_train
+      self._load_vectors_and_train(BASE_FILENAME, FinModeling::IncomeStatementItem::TRAINING_VECTORS)
+    end
+  end
+end