finmodeling 0.1

data/lib/finmodeling/balance_sheet_analyses.rb

@@ -0,0 +1,33 @@
+module FinModeling
+
+  class BalanceSheetAnalyses < CalculationSummary
+    def initialize(calc_summary)
+      @title              = calc_summary.title
+      @rows               = calc_summary.rows
+      @header_row         = calc_summary.header_row
+      @key_width          = calc_summary.key_width
+      @val_width          = calc_summary.val_width
+      @max_decimals       = calc_summary.max_decimals
+      @totals_row_enabled = false
+    end
+
+    def print_extras # FIXME: rename
+      lr = noa_growth_row.valid_vals.linear_regression
+      puts "\t\tNOA growth: "+
+           "a:#{lr.a.to_s.cap_decimals(4)}, "+
+           "b:#{lr.b.to_s.cap_decimals(4)}, "+
+           "r:#{lr.r.to_s.cap_decimals(4)}, "+
+           "var:#{noa_growth_row.valid_vals.variance.to_s.cap_decimals(4)}"
+    end
+
+    def noa_growth_row
+      find_row_by_key('NOA Growth')
+    end
+
+    def find_row_by_key(key) # FIXME: move this to CalculationSummary
+      self.rows.find{ |x| x.key == key }
+    end
+  end
+
+end
+

data/lib/finmodeling/balance_sheet_calculation.rb

@@ -0,0 +1,68 @@
+module FinModeling
+  class BalanceSheetCalculation < CompanyFilingCalculation
+
+    def assets_calculation
+      if @assets.nil?
+        friendly_goal = "assets"
+        label_regexes = [ /(^total *|^consolidated *|^)assets$/,
+                          /^assets total$/ ]
+        id_regexes    = [ /^(|Locator_|loc_)(|us-gaap_)Assets[_a-z0-9]+/ ]
+  
+        calc = find_and_verify_calculation_arc(friendly_goal, label_regexes, id_regexes)
+        @assets = AssetsCalculation.new(calc)
+      end
+      return @assets
+    end
+  
+    def liabs_and_equity_calculation
+       if @liabs_and_equity.nil?
+        friendly_goal = "liabilities and equity"
+        label_regexes = [ /(^total *|^)liabilities.*and.*equity/ ]
+        id_regexes    = [ /.*/ ] # no checking...
+  
+        calc = find_and_verify_calculation_arc(friendly_goal, label_regexes, id_regexes)
+        @liabs_and_equity = LiabsAndEquityCalculation.new(calc)
+      end
+      return @liabs_and_equity
+    end
+
+    def is_valid?
+      has_cash_item = false
+      assets_calculation.leaf_items.each do |leaf|
+        if !has_cash_item and leaf.name.downcase.matches_regexes?([/cash/])
+          has_cash_item = true
+        end
+      end
+
+      has_equity_item = false
+      liabs_and_equity_calculation.leaf_items.each do |leaf|
+        if !has_equity_item and leaf.name.downcase.matches_regexes?([/equity/, /stock/])
+          has_equity_item = true
+        end
+      end
+
+      left = assets_calculation.leaf_items_sum(:period => periods.last)
+      right = liabs_and_equity_calculation.leaf_items_sum(:period => periods.last)
+      allowed_error = 1.0
+      is_balanced = (left - right) < allowed_error
+
+      puts "balance sheet's assets calculation lacks cash item" if !has_cash_item 
+      puts "balance sheet's liabilities and equity calculation lacks equity item" if !has_equity_item 
+      puts "balance sheet's isn't balanced (#{left}, #{right})" if !is_balanced
+      return (has_cash_item and has_equity_item and is_balanced)
+    end
+
+    def reformulated(period)
+      return ReformulatedBalanceSheet.new(period, 
+                                          assets_calculation.summary(:period=>period), 
+                                          liabs_and_equity_calculation.summary(:period=>period))
+    end
+
+    def write_constructor(file, item_name)
+      item_calc_name = item_name + "_calc"
+      @calculation.write_constructor(file, item_calc_name)
+      file.puts "#{item_name} = FinModeling::BalanceSheetCalculation.new(#{item_calc_name})"
+    end
+
+  end
+end

data/lib/finmodeling/calculation_summary.rb

@@ -0,0 +1,148 @@
+module FinModeling
+
+  class CalculationRow
+    attr_accessor :key, :type, :vals
+
+    def initialize(args = {})
+      @key  = args[:key] || ""
+      @type = args[:type]
+      @vals = args[:vals] || []
+    end
+
+    def valid_vals
+      ArrayWithStats.new(@vals.select{ |val| !val.nil? })
+    end
+
+    def print(key_width=18, max_decimals=4, val_width=12)
+      justified_key = @key.fixed_width_left_justify(key_width)
+  
+      justified_vals = ""
+      @vals.each do |val|
+        val_with_commas = val.to_s.with_thousands_separators
+        justified_vals += "  " + val_with_commas.cap_decimals(max_decimals).fixed_width_right_justify(val_width) 
+      end
+   
+      puts "\t" + justified_key + justified_vals
+    end
+
+    def write_constructor(file, item_name)
+      file.puts "args = { }"
+      file.puts "args[:key] = \"#{@key}\""
+      file.puts "args[:type] = \"#{@type}\""
+      file.puts "args[:vals] = [#{@vals.join(', ')}]"
+      file.puts "#{item_name} = FinModeling::CalculationRow.new(args)"
+    end
+  end
+
+  class CalculationHeader < CalculationRow
+    def print(key_width=18, max_decimals=4, val_width=12)
+      justified_key = @key.fixed_width_left_justify(key_width)
+  
+      justified_vals = ""
+      @vals.each do |val|
+        justified_vals += "  " + val.fixed_width_right_justify(val_width) 
+      end
+  
+      puts "\t" + justified_key + justified_vals
+    end
+
+    def write_constructor(file, item_name)
+      file.puts "args = { }"
+      file.puts "args[:key] = \"#{@key}\""
+      file.puts "args[:vals] = [#{@vals.map{ |val| "\"#{val}\"" }.join(', ')}]"
+      file.puts "#{item_name} = FinModeling::CalculationHeader.new(args)"
+    end
+  end
+
+  class CalculationSummary
+    attr_accessor :title, :header_row, :rows
+    attr_accessor :key_width, :val_width, :max_decimals, :totals_row_enabled
+
+    def initialize
+      @key_width = 18
+      @val_width = 12
+      @max_decimals = 4
+      @totals_row_enabled = true
+    end
+
+    def num_value_columns
+      @rows.map{ |row| row.vals.length }.max
+    end
+
+    def total(col_idx=0)
+      @rows.map{ |row| row.vals[col_idx] }.inject(:+) || 0.0
+    end
+
+    def totals
+      return [] if num_value_columns.nil?
+      0.upto(num_value_columns-1).map { |col_idx| total(col_idx) }
+    end
+
+    def print
+      puts @title
+
+      rows = []
+      rows << @header_row if @header_row
+      rows += @rows
+      rows << CalculationRow.new(:key => "Total", :vals => totals) if @totals_row_enabled
+      rows.each { |row| row.print(@key_width, @max_decimals, @val_width) }
+
+      puts
+    end
+
+    def filter_by_type(type)
+      cs = CalculationSummary.new
+      cs.title = @title
+      cs.rows = @rows.select{ |x| x.type == type }
+      return cs
+    end
+
+    def write_constructor(file, item_name)
+      file.puts "#{item_name} = FinModeling::CalculationSummary.new"
+      file.puts "#{item_name}.title = \"#{@title}\""
+      file.puts "#{item_name}.key_width = #{@key_width}"
+      file.puts "#{item_name}.val_width = #{@val_width}"
+      file.puts "#{item_name}.max_decimals = #{@max_decimals}"
+      file.puts "#{item_name}.totals_row_enabled = #{@totals_row_enabled}"
+
+      if @header_row
+        header_row_item_name = item_name + "_header_row"
+        @header_row.write_constructor(file, header_row_item_name)
+        file.puts "#{item_name}.header_row = #{header_row_item_name}"
+      end
+
+      row_item_names = []
+      @rows.each_with_index do |row, index|
+        row_item_name = item_name + "_row#{index}"
+        row.write_constructor(file, row_item_name)
+        row_item_names << row_item_name
+      end
+      file.puts "#{item_name}.rows = [#{row_item_names.join(',')}]"
+    end
+
+    def +(mccs)
+      raise RuntimeError.new("can't add a CalculationSummary to a #{mccs.class}") if !mccs.is_a?(CalculationSummary)
+      raise RuntimeError.new("can't add CalculationSummaries with different numbers of rows") if @rows.length != mccs.rows.length
+
+      multics = CalculationSummary.new
+      multics.title = @title
+
+      if @header_row
+        multics.header_row = CalculationHeader.new(
+                               :key  => @header_row.key.dup, 
+                               :vals => @header_row.vals.dup + mccs.header_row.vals.dup)
+      end
+
+      multics.rows = []
+      0.upto(@rows.length-1).each do |idx|
+        multics.rows << CalculationRow.new(
+                          :key  => @rows[idx].key.dup, 
+                          :vals => @rows[idx].vals.dup + mccs.rows[idx].vals.dup)
+      end
+
+      return multics
+    end
+
+  end
+
+end

data/lib/finmodeling/can_cache_classifications.rb

@@ -0,0 +1,36 @@
+module FinModeling
+
+  module CanCacheClassifications
+    protected
+
+    def lookup_cached_classifications(base_filename, rows)
+      filename = rows_to_filename(base_filename, rows)
+      return false if !File.exists?(filename) || !Config.caching_enabled?
+
+      f = File.open(filename, "r")
+      rows.each do |row|
+        row.type = f.gets.chomp.to_sym
+      end
+      f.close
+      return true
+    end
+
+    def save_cached_classifications(base_filename, rows)
+      filename = rows_to_filename(base_filename, rows)
+      FileUtils.mkdir_p(File.dirname(filename)) if !File.exists?(File.dirname(filename))
+      f = File.open(filename, "w")
+      rows.each do |row|
+        f.puts row.type.to_s
+      end
+      f.close
+    end
+
+    private
+
+    def rows_to_filename(base_filename, rows)
+      unique_str = Digest::SHA1.hexdigest(rows.map{ |row| row.key }.join)
+      filename = base_filename + unique_str + ".txt"
+    end
+  end
+
+end

data/lib/finmodeling/can_cache_summaries.rb

@@ -0,0 +1,16 @@
+module FinModeling
+
+  module CanCacheSummaries
+    protected
+
+    def lookup_cached_summary(key)
+      @summary_cache = { } if @summary_cache.nil?
+      return @summary_cache[key]
+    end
+
+    def save_cached_summary(key, summary)
+      @summary_cache[key] = summary
+    end
+  end
+
+end

data/lib/finmodeling/can_classify_rows.rb

@@ -0,0 +1,54 @@
+module FinModeling
+
+  module CanClassifyRows # simple viterbi classifier, with N-element lookahead
+    protected
+
+    def classify_rows(all_states, allowed_next_states, rows, row_item_type, lookahead)
+      estimate_of_item_being_in_state = rows.map { |row| row_item_type.new(row.key).classification_estimates }
+
+      prev_state = nil
+      rows.each_with_index do |row, item_index|
+        cur_lookahead = [lookahead, rows.length-item_index-1].min
+        row.type = classify_row(all_states, allowed_next_states, 
+                                estimate_of_item_being_in_state, 
+                                item_index, prev_state, cur_lookahead)[:state]
+        raise RuntimeError.new("couldn't classify....") if row.type.nil?
+
+        prev_state = row.type
+      end
+    end
+
+    private
+
+    def classify_row(all_states, allowed_next_states, estimate_of_item_being_in_state, item_index, prev_state, lookahead)
+      best_overall = { :estimate => -100000000.0, :state => nil }
+      best_allowed = { :estimate => -100000000.0, :state => nil }
+
+      all_states.each do |state|
+        cur = { :estimate => estimate_of_item_being_in_state[item_index][state], :state => state }
+        raise RuntimeError.new("estimate is nil: #{estimate_of_item_being_in_state[item_index]} for state #{state}") if !cur[:estimate]
+
+        if lookahead > 0
+          future_error = classify_row(all_states, allowed_next_states, 
+                                      estimate_of_item_being_in_state, 
+                                      item_index+1, state, lookahead-1)[:error]
+          cur[:estimate] -= future_error
+        end
+
+        if cur[:estimate] > best_overall[:estimate]
+          best_overall = cur
+        end
+
+        is_allowed = allowed_next_states[prev_state] && allowed_next_states[prev_state].include?(state)
+        if is_allowed && (cur[:estimate] > best_allowed[:estimate])
+          best_allowed = cur
+        end
+      end
+
+      return { :state => best_allowed[:state],
+               :error => best_overall[:estimate] - 
+                         best_allowed[:estimate] }
+    end
+  end
+
+end

data/lib/finmodeling/cash_change_calculation.rb

@@ -0,0 +1,67 @@
+module FinModeling
+
+  class CashChangeCalculation < CompanyFilingCalculation
+    include CanCacheClassifications
+    include CanCacheSummaries
+    include CanClassifyRows
+
+    BASE_FILENAME = File.join(FinModeling::BASE_PATH, "summaries/cash_")
+
+    ALL_STATES  =          [ :c, :i, :d, :f ]
+    NEXT_STATES = { nil => [ :c             ],
+                    :c  => [ :c, :i, :d, :f ],
+                    :i  => [     :i, :d, :f ],
+                    :d  => [     :i, :d, :f ],
+                    :f  => [         :d, :f ] }
+
+    def summary(args)
+      summary_cache_key = args[:period].to_pretty_s
+      summary = lookup_cached_summary(summary_cache_key)
+      return summary if !summary.nil? and false # FIXME: get rid of "and false"
+
+      mapping = Xbrlware::ValueMapping.new
+      mapping.policy[:unknown]          = :flip
+      mapping.policy[:credit]           = :flip 
+      mapping.policy[:debit]            = :no_action 
+      mapping.policy[:netincome]        = :no_action 
+      mapping.policy[:taxes]            = :no_action
+      mapping.policy[:proceedsfromdebt] = :no_action
+
+      find_and_tag_special_items(args)
+
+      summary = super(:period => args[:period], :mapping => mapping)
+      if !lookup_cached_classifications(BASE_FILENAME, summary.rows) or true # FIXME: get rid of "or true"
+        lookahead = [4, summary.rows.length-1].min
+        classify_rows(ALL_STATES, NEXT_STATES, summary.rows, FinModeling::CashChangeItem, lookahead)
+        save_cached_classifications(BASE_FILENAME, summary.rows)
+      end
+
+      save_cached_summary(summary_cache_key, summary)
+
+      return summary
+    end
+
+    private
+
+    def find_and_tag_special_items(args)
+      leaf_items(:period => args[:period]).each do |item|
+        if item.name.matches_regexes?([ /NetIncomeLoss/,
+                                        /ProfitLoss/ ])
+          item.def = {} if !item.def
+          item.def["xbrli:balance"] = "netincome"
+        end
+
+        if item.name =~ /IncreaseDecreaseInIncomeTaxes/
+          item.def = {} if !item.def
+          item.def["xbrli:balance"] = "taxes"
+        end
+
+        if item.name =~ /ProceedsFromDebtNetOfIssuanceCosts/
+          item.def = {} if !item.def
+          item.def["xbrli:balance"] = "proceedsfromdebt"
+        end
+      end
+    end
+
+  end
+end

data/lib/finmodeling/cash_change_item.rb

@@ -0,0 +1,14 @@
+module FinModeling
+  class CashChangeItem < String
+    include HasStringClassifier
+
+    BASE_FILENAME = File.join(FinModeling::BASE_PATH, "classifiers/cci_")
+    TYPES         = [ :c, :i, :d, :f ]
+
+    has_string_classifier(TYPES, CashChangeItem)
+
+    def self.load_vectors_and_train
+      self._load_vectors_and_train(BASE_FILENAME, FinModeling::CashChangeItem::TRAINING_VECTORS)
+    end
+  end
+end