finmodeling 0.1 → 0.2

data/lib/finmodeling/cash_change_calculation.rb

@@ -17,7 +17,7 @@ module FinModeling
     def summary(args)
       summary_cache_key = args[:period].to_pretty_s
       summary = lookup_cached_summary(summary_cache_key)
-      return summary if !summary.nil? and false # FIXME: get rid of "and false"
+      return summary if !summary.nil? && false # FIXME: get rid of "and false"
 
       mapping = Xbrlware::ValueMapping.new
       mapping.policy[:unknown]          = :flip
@@ -30,7 +30,7 @@ module FinModeling
       find_and_tag_special_items(args)
 
       summary = super(:period => args[:period], :mapping => mapping)
-      if !lookup_cached_classifications(BASE_FILENAME, summary.rows) or true # FIXME: get rid of "or true"
+      if !lookup_cached_classifications(BASE_FILENAME, summary.rows) || true # FIXME: get rid of "or true"
         lookahead = [4, summary.rows.length-1].min
         classify_rows(ALL_STATES, NEXT_STATES, summary.rows, FinModeling::CashChangeItem, lookahead)
         save_cached_classifications(BASE_FILENAME, summary.rows)
@@ -45,7 +45,7 @@ module FinModeling
 
     def find_and_tag_special_items(args)
       leaf_items(:period => args[:period]).each do |item|
-        if item.name.matches_regexes?([ /NetIncomeLoss/,
+        if item.name.matches_any_regex?([ /NetIncomeLoss/,
                                         /ProfitLoss/ ])
           item.def = {} if !item.def
           item.def["xbrli:balance"] = "netincome"

data/lib/finmodeling/cash_change_summary_from_differences.rb

@@ -0,0 +1,36 @@
+module FinModeling
+  class CashChangeSummaryFromDifferences
+    def initialize(re_cfs1, re_cfs2)
+      @re_cfs1 = re_cfs1
+      @re_cfs2 = re_cfs2
+    end
+    def filter_by_type(key)
+      case key
+        when :c
+          @cs = FinModeling::CalculationSummary.new
+          @cs.title = "Cash from Operations"
+          @cs.rows = [ CalculationRow.new(:key => "First  Row", :vals => [ @re_cfs1.cash_from_operations.total] ),
+                       CalculationRow.new(:key => "Second Row", :vals => [-@re_cfs2.cash_from_operations.total] ) ]
+          return @cs
+        when :i
+          @cs = FinModeling::CalculationSummary.new
+          @cs.title = "Cash Investments in Operations"
+          @cs.rows = [ CalculationRow.new(:key => "First  Row", :vals => [ @re_cfs1.cash_investments_in_operations.total] ),
+                       CalculationRow.new(:key => "Second Row", :vals => [-@re_cfs2.cash_investments_in_operations.total] ) ]
+          return @cs
+        when :d
+          @cs = FinModeling::CalculationSummary.new
+          @cs.title = "Payments to Debtholders"
+          @cs.rows = [ CalculationRow.new(:key => "First  Row", :vals => [ @re_cfs1.payments_to_debtholders.total] ),
+                       CalculationRow.new(:key => "Second Row", :vals => [-@re_cfs2.payments_to_debtholders.total] ) ]
+          return @cs
+        when :f
+          @cs = FinModeling::CalculationSummary.new
+          @cs.title = "Payments to Stockholders"
+          @cs.rows = [ CalculationRow.new(:key => "First  Row", :vals => [ @re_cfs1.payments_to_stockholders.total] ),
+                       CalculationRow.new(:key => "Second Row", :vals => [-@re_cfs2.payments_to_stockholders.total] ) ]
+          return @cs
+      end
+    end
+  end
+end

data/lib/finmodeling/cash_flow_statement_analyses.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+module FinModeling
+
+  class CashFlowStatementAnalyses < CalculationSummary
+    def initialize(calc_summary)
+      @title              = calc_summary.title
+      @rows               = calc_summary.rows
+      @header_row         = calc_summary.header_row
+      @key_width          = calc_summary.key_width
+      @val_width          = calc_summary.val_width
+      @max_decimals       = calc_summary.max_decimals
+      @totals_row_enabled = false
+    end
+
+    def print_regressions # FIXME: rename
+      lr = ni_over_c_row.valid_vals.linear_regression
+      puts "\t\tNI / C: "+
+           "a:#{lr.a.to_s.cap_decimals(4)}, "+
+           "b:#{lr.b.to_s.cap_decimals(4)}, "+
+           "r²:#{lr.r2.to_s.cap_decimals(4)}, "+
+           "σ²:#{ni_over_c_row.valid_vals.variance.to_s.cap_decimals(4)}, " +
+           ( (lr.r2 > 0.6) ? "strong fit" : ( (lr.r2 < 0.2) ? "weak fit [**]" : "avg fit") )
+    end
+
+    def ni_over_c_row
+      find_row_by_key('NI / C')
+    end
+
+    def find_row_by_key(key) # FIXME: move this to CalculationSummary
+      self.rows.find{ |x| x.key == key }
+    end
+  end
+
+end
+

data/lib/finmodeling/cash_flow_statement_calculation.rb

@@ -1,28 +1,32 @@
 module FinModeling
   class CashFlowStatementCalculation < CompanyFilingCalculation
+    include CanChooseSuccessivePeriods
+
+    CASH_GOAL        = "cash change"
+    CASH_LABELS      = [ /^cash and cash equivalents period increase decrease/,
+                         /^(|net )(change|increase|decrease|decrease *increase|increase *decrease) in cash and(| cash) equivalents/,
+                         /^net cash provided by used in (|operating activities )continuing operations/,
+                         /^net cash provided by used in operating activities/]
+    CASH_ANTI_LABELS = [ ]
+    CASH_IDS         = [ /^(|Locator_|loc_)(|us-gaap_)CashAndCashEquivalentsPeriodIncreaseDecrease[_a-z0-9]+/,
+                         /^(|Locator_|loc_)(|us-gaap_)NetCashProvidedByUsedIn(|OperatingActivities)ContinuingOperations[_a-z0-9]+/ ]
 
     def cash_change_calculation
-      if @cash_change.nil?
-        friendly_goal = "cash change"
-        label_regexes = [ /^cash and cash equivalents period increase decrease/,
-                          /^(|net )(change|increase|decrease|decrease *increase|increase *decrease) in cash and cash equivalents/,
-                          /^net cash provided by used in continuing operations/]
-        id_regexes    = [ /^(|Locator_|loc_)(|us-gaap_)CashAndCashEquivalentsPeriodIncreaseDecrease[_a-z0-9]+/,
-                          /^(|Locator_|loc_)(|us-gaap_)NetCashProvidedByUsedInContinuingOperations[_a-z0-9]+/ ]
-  
-        calc = find_and_verify_calculation_arc(friendly_goal, label_regexes, id_regexes)
-        @cash_change = CashChangeCalculation.new(calc)
-      end
-      return @cash_change
+      @cash_change ||= CashChangeCalculation.new(find_calculation_arc(CASH_GOAL, CASH_LABELS, CASH_ANTI_LABELS, CASH_IDS))
     end
   
     def is_valid?
       re_cfs = reformulated(periods.last)
       flows_are_balanced = (re_cfs.free_cash_flow.total == (-1*re_cfs.financing_flows.total))
+      puts "flows are not balanced" if !flows_are_balanced
       none_are_zero = (re_cfs.cash_from_operations.total           != 0) &&
                       (re_cfs.cash_investments_in_operations.total != 0) &&
-                      (re_cfs.payments_to_debtholders.total        != 0) &&
-                      (re_cfs.payments_to_stockholders.total       != 0)
+                      (re_cfs.payments_to_debtholders.total        != 0) #&&
+                      #(re_cfs.payments_to_stockholders.total       != 0) # I relaxed this constraint. Seems it is often legitimately zero
+      puts "(re_cfs.cash_from_operations.total           == 0)" if (re_cfs.cash_from_operations.total           == 0)
+      puts "(re_cfs.cash_investments_in_operations.total == 0)" if (re_cfs.cash_investments_in_operations.total == 0)
+      puts "(re_cfs.payments_to_debtholders.total        == 0)" if (re_cfs.payments_to_debtholders.total        == 0)
+      #puts "(re_cfs.payments_to_stockholders.total       == 0)" if (re_cfs.payments_to_stockholders.total       == 0)
       return (flows_are_balanced && none_are_zero)
     end
 
@@ -30,46 +34,18 @@ module FinModeling
       return ReformulatedCashFlowStatement.new(period, cash_change_calculation.summary(:period => period))
     end
 
-    def latest_quarterly_reformulated(prev_cash_flow_statement)
-      if cash_change_calculation.periods.quarterly.any? &&
-         reformulated(cash_change_calculation.periods.quarterly.last).cash_investments_in_operations.total.abs > 1.0
-        return reformulated(cash_change_calculation.periods.quarterly.last)
-  
-      elsif !prev_cash_flow_statement
-        return nil
-
-      elsif cash_change_calculation.periods.halfyearly.any? &&
-            prev_cash_flow_statement.cash_change_calculation.periods.quarterly.any?
-        cfs_period = cash_change_calculation.periods.halfyearly.last
-        re_cfs     = reformulated(cfs_period)
-  
-        period_1q_thru_1q = prev_cash_flow_statement.cash_change_calculation.periods.quarterly.last
-        prev1q  = prev_cash_flow_statement.reformulated(period_1q_thru_1q)
-        re_cfs  = re_cfs - prev1q
-
-        return re_cfs 
-
-      elsif cash_change_calculation.periods.threequarterly.any? &&
-            prev_cash_flow_statement.cash_change_calculation.periods.halfyearly.any?
-        cfs_period = cash_change_calculation.periods.threequarterly.last
-        re_cfs     = reformulated(cfs_period)
-  
-        period_1q_thru_2q = prev_cash_flow_statement.cash_change_calculation.periods.halfyearly.last
-        prev2q  = prev_cash_flow_statement.reformulated(period_1q_thru_2q)
-        re_cfs  = re_cfs - prev2q
-
-        return re_cfs 
+    def latest_quarterly_reformulated(prev_cfs)
+      if cash_change_calculation.periods.quarterly.any?
+        period = cash_change_calculation.periods.quarterly.last
+        lqr    = reformulated(period)
+        return lqr if lqr.flows_are_plausible?
+      end
 
-      elsif cash_change_calculation.periods.yearly.any? &&
-            prev_cash_flow_statement.cash_change_calculation.periods.threequarterly.any?
-        cfs_period = cash_change_calculation.periods.yearly.last
-        re_cfs     = reformulated(cfs_period)
-  
-        period_1q_thru_3q = prev_cash_flow_statement.cash_change_calculation.periods.threequarterly.last
-        prev3q  = prev_cash_flow_statement.reformulated(period_1q_thru_3q)
-        re_cfs  = re_cfs - prev3q
+      return nil if !prev_cfs
 
-        return re_cfs 
+      cur_period, prev_period = choose_successive_periods(cash_change_calculation, prev_cfs.cash_change_calculation)
+      if cur_period && prev_period
+        return reformulated(cur_period) - prev_cfs.reformulated(prev_period)
       end
   
       return nil

data/lib/finmodeling/classifiers.rb

@@ -5,7 +5,9 @@ module FinModeling
       FinModeling::AssetsItem.load_vectors_and_train
       FinModeling::LiabsAndEquityItem.load_vectors_and_train
       FinModeling::IncomeStatementItem.load_vectors_and_train
+      FinModeling::ComprehensiveIncomeStatementItem.load_vectors_and_train
       FinModeling::CashChangeItem.load_vectors_and_train
+      FinModeling::EquityChangeItem.load_vectors_and_train
     end
   end
 end

data/lib/finmodeling/company_filing.rb

@@ -1,16 +1,39 @@
 module FinModeling
 
   class CachedAnnualFiling 
-    attr_accessor :balance_sheet, :income_statement, :cash_flow_statement, :disclosures
-    def initialize(bs, is, cfs, disclosures)
-      @balance_sheet       = bs
-      @income_statement    = is
-      @cash_flow_statement = cfs
-      @disclosures         = disclosures
+    attr_accessor :balance_sheet, :income_statement, :comprehensive_income_statement, :cash_flow_statement, :shareholder_equity_statement, :disclosures
+    def initialize(bs, is, cis, cfs, ses, disclosures)
+      @balance_sheet                  = bs
+      @income_statement               = is
+      @comprehensive_income_statement = cis
+      @cash_flow_statement            = cfs
+      @shareholder_equity_statement   = ses
+      @disclosures                    = disclosures
+    end
+
+    def has_an_income_statement?
+      !@income_statement.nil?
+    end
+
+    def has_a_comprehensive_income_statement?
+      !@comprehensive_income_statement.nil?
+    end
+
+    def has_a_shareholder_equity_statement?
+      !@shareholder_equity_statement.nil?
     end
 
     def is_valid?
-      return (@income_statement.is_valid? and @balance_sheet.is_valid? and @cash_flow_statement.is_valid?)
+      puts "balance sheet is not valid" if !@balance_sheet.is_valid?
+      puts "income statment is not valid" if has_an_income_statement? && !@income_statement.is_valid?
+      puts "comprehensive income statment is not valid" if has_a_comprehensive_income_statement? && !@comprehensive_income_statement.is_valid?
+      #puts "cash flow statement is not valid" if !cash_flow_statement.is_valid?
+
+      return false if !@balance_sheet.is_valid?
+      return false if has_an_income_statement? && !@income_statement.is_valid?
+      return false if has_a_comprehensive_income_statement? && !@comprehensive_income_statement.is_valid?
+      #return false if !@cash_flow_statement.is_valid? # FIXME: why can't we enable this?
+      return true
     end
   end
 

data/lib/finmodeling/company_filing_calculation.rb

@@ -56,15 +56,25 @@ module FinModeling
 
     protected
 
-    def find_and_verify_calculation_arc(friendly_goal, label_regexes, id_regexes)
-      calc = @calculation.arcs.find{ |x| x.label.downcase.gsub(/[^a-z ]/, '').matches_regexes?(label_regexes) }
+    def find_calculation_arc(friendly_goal, label_regexes, anti_label_regexes, id_regexes, criterion=:first)
+      calcs = @calculation.arcs.select{ |x|  x.label.downcase.gsub(/[^a-z ]/, '').matches_any_regex?(label_regexes) &&
+                                            !x.label.downcase.gsub(/[^a-z ]/, '').matches_any_regex?(anti_label_regexes) }
 
-      if calc.nil?
-        summary_of_arcs = @calculation.arcs.map{ |x| "\"#{x.label}\"" }.join("; ")
-        raise RuntimeError.new("Couldn't find #{friendly_goal} in: " + summary_of_arcs)
+      if calcs.empty?
+        summary_of_arcs = @calculation.arcs.map{ |x| "\t\"#{x.label}\"" }.join("\n")
+        raise InvalidFilingError.new("Couldn't find #{friendly_goal} in:\n" + summary_of_arcs + "\nTried: #{label_regexes.inspect}. (Ignoring: #{anti_label_regexes.inspect}.).")
       end
 
-      if !calc.item_id.matches_regexes?(id_regexes) 
+      calc = case
+        when criterion == :first
+          calcs.first
+        when criterion == :longest
+          calcs.sort{ |x,y| x.leaf_items(periods.last).length <=> y.leaf_items(periods.last).length }.last
+        else
+          raise ArgumentError.new("\"#{criterion}\" is not a valid criterion")
+      end
+
+      if !calc.item_id.matches_any_regex?(id_regexes) 
         puts "Warning: #{friendly_goal} id is not recognized: #{calc.item_id}"
       end
 

data/lib/finmodeling/company_filings.rb

@@ -1,72 +1,138 @@
 module FinModeling
   class CompanyFilings < Array
+    def re_bs_arr
+      @re_bs_arr ||= self.map{ |filing| filing.balance_sheet.reformulated(filing.balance_sheet.periods.last) }
+    end
+
+    def re_is_arr
+      prev_stmt = nil
+      @re_is_arr ||= ([nil] + self).each_cons(2).map do |prev_filing, filing| 
+        cur_re_is = nil
+        cur_ci_calc  = filing.has_a_comprehensive_income_statement? ? filing.comprehensive_income_statement.comprehensive_income_calculation : nil
+        prev_ci_calc = (prev_filing && prev_filing.has_a_comprehensive_income_statement?) ? 
+                       prev_filing.comprehensive_income_statement.comprehensive_income_calculation : nil
+        if filing.has_an_income_statement?
+          cur_re_is = filing.income_statement.latest_quarterly_reformulated(cur_ci_calc, prev_stmt, prev_ci_calc)
+          prev_stmt = filing.income_statement
+        elsif filing.has_a_comprehensive_income_statement? &&
+              filing.comprehensive_income_statement
+                    .comprehensive_income_calculation
+                    .has_revenue_item?
+          cur_re_is = filing.comprehensive_income_statement.latest_quarterly_reformulated(prev_stmt, prev_ci_calc)
+          prev_stmt = filing.comprehensive_income_statement
+        else
+          raise RuntimeError.new("Can't create reformulated income statement")
+        end
+        cur_re_is
+      end
+    end
+
+    def re_cfs_arr
+      @re_cfs_arr ||= ([nil] + self).each_cons(2).map do |prev_filing, filing| 
+        filing.cash_flow_statement.latest_quarterly_reformulated(prev_filing ? prev_filing.cash_flow_statement : nil) 
+      end
+    end
+
     def balance_sheet_analyses
       if !@balance_sheet_analyses
-        re_bs = nil
-        self.each do |filing|
-          prev_re_bs = re_bs
-          re_bs = filing.balance_sheet.reformulated(filing.balance_sheet.periods.last)
-          next_analysis = re_bs.analysis(prev_re_bs)
-    
-          @balance_sheet_analyses = @balance_sheet_analyses + next_analysis if  @balance_sheet_analyses
-          @balance_sheet_analyses =                           next_analysis if !@balance_sheet_analyses
-        end
-        @balance_sheet_analyses = BalanceSheetAnalyses.new(@balance_sheet_analyses)
+        analyses = ([nil] + re_bs_arr).each_cons(2).map { |prev, cur| cur.analysis(prev) }
+        analyses.delete_if{ |x| x.nil? }
+        @balance_sheet_analyses = BalanceSheetAnalyses.new( analyses.inject(:+) )
       end
       return @balance_sheet_analyses
     end
   
-    def income_statement_analyses
+    def income_statement_analyses(expected_rate_of_return)
       if !@income_statement_analyses
-        prev_re_bs, prev_re_is, prev_filing  = [nil, nil, nil]
-        self.each do |filing|
-          re_is = filing.income_statement.latest_quarterly_reformulated(prev_filing ? prev_filing.income_statement : nil)
-          re_bs = filing.balance_sheet.reformulated(filing.balance_sheet.periods.last)
+        analyses = []
+        self.each_with_index do |filing, idx|
+          prev_re_bs = (idx > 0) ? re_bs_arr[idx-1] : nil
+          prev_re_is = (idx > 0) ? re_is_arr[idx-1] : nil
+          re_bs = re_bs_arr[idx]
+          re_is = re_is_arr[idx]
     
-          next_analysis = FinModeling::ReformulatedIncomeStatement.empty_analysis if !re_is
-          next_analysis = re_is.analysis(re_bs, prev_re_is, prev_re_bs)           if  re_is
-        
-          @income_statement_analyses = @income_statement_analyses + next_analysis if  @income_statement_analyses
-          @income_statement_analyses =                              next_analysis if !@income_statement_analyses
-      
-          prev_re_bs, prev_re_is, prev_filing  = [re_bs, re_is, filing]
+          analyses << (re_is ? re_is.analysis(re_bs, prev_re_is, prev_re_bs, expected_rate_of_return) : FinModeling::ReformulatedIncomeStatement.empty_analysis )
         end
-        @income_statement_analyses = IncomeStatementAnalyses.new(@income_statement_analyses)
+
+        analyses.delete_if{ |x| x.nil? }
+        @income_statement_analyses = IncomeStatementAnalyses.new( analyses.inject(:+) )
       end
       return @income_statement_analyses
     end
-  
+   
     def cash_flow_statement_analyses
       if !@cash_flow_statement_analyses
-        prev_filing, prev_re_cfs = [nil, nil]
-        self.each do |filing|
-          re_is = filing.income_statement.latest_quarterly_reformulated(prev_filing ? prev_filing.income_statement : nil)
-          re_cfs = filing.cash_flow_statement.latest_quarterly_reformulated(prev_filing ? prev_filing.cash_flow_statement : nil)
+        analyses = []
+        self.each_with_index do |filing, idx|
+          re_is  = re_is_arr[idx]
+          re_cfs = re_cfs_arr[idx]
       
-          next_analysis = FinModeling::ReformulatedCashFlowStatement.empty_analysis if !re_cfs
-          next_analysis = re_cfs.analysis(re_is)                                    if  re_cfs
-        
-          @cash_flow_statement_analyses = @cash_flow_statement_analyses + next_analysis if  @cash_flow_statement_analyses
-          @cash_flow_statement_analyses =                                 next_analysis if !@cash_flow_statement_analyses
-    
-          prev_filing, prev_re_cfs = [filing, re_cfs]
+          analyses << (re_cfs ? re_cfs.analysis(re_is) : FinModeling::ReformulatedCashFlowStatement.empty_analysis)
         end
+
+        analyses.delete_if{ |x| x.nil? }
+        @cash_flow_statement_analyses = CashFlowStatementAnalyses.new( analyses.inject(:+) )
         @cash_flow_statement_analyses.totals_row_enabled = false
       end
       return @cash_flow_statement_analyses
     end
+
+    def disclosures(title_regex, period_type=nil)
+      ds = nil
+      self.each do |filing|
+        cur_disclosures = filing.disclosures
+        if ( disclosure = filing.disclosures.find{ |disc| disc.summary(:period => disc.periods.last)
+                                                              .title
+                                                              .gsub(/ \(.*/,'') =~ title_regex } )
+
+          period = case period_type
+            when nil        then disclosure.periods.last
+            when :yearly    then disclosure.periods.yearly.last
+            when :quarterly then disclosure.periods.quarterly.last
+            else                 raise RuntimeError.new("bogus period type")
+          end
+
+          if period
+            next_d = disclosure.summary(:period => period )
+            next_d.header_row = CalculationHeader.new(:key => "",   :vals => [period.to_pretty_s.gsub(/.* to /,'')])
+
+            ds = ds + next_d if  ds
+            ds =      next_d if !ds
+          end
+        end
+      end
+      return ds
+    end
   
-    def choose_forecasting_policy
-      if length < 3
-        return FinModeling::GenericForecastingPolicy.new
-      else
-        isa = income_statement_analyses
+    def choose_forecasting_policy(expected_rate_of_return, policy_type=:linear_trend)
+      raise RuntimeError.new("Cannot properly forecast with fewer than 3 filings") if length < 3
+      case policy_type
+      when :trailing_avg
+        isa = income_statement_analyses(expected_rate_of_return)
+        args = { }
+
+        args[:revenue_estimator]        = TimeSeriesEstimator.from_const(ArrayWithStats.new(re_is_arr[1..-1].map{ |re_is| re_is.operating_revenues.total }).mean)
+        args[:sales_pm_estimator]       = TimeSeriesEstimator.from_const(ArrayWithStats.new(isa.operating_pm_row.valid_vals[1..-1]).mean)
+        args[:sales_over_noa_estimator] = TimeSeriesEstimator.from_const(ArrayWithStats.new(isa.sales_over_noa_row.valid_vals[1..-1]).mean)
+        args[:fi_over_nfa_estimator]    = TimeSeriesEstimator.from_const(ArrayWithStats.new(isa.fi_over_nfa_row.valid_vals[1..-1]).mean)
+        return FinModeling::LinearTrendForecastingPolicy.new(args)
+
+      when :linear_trend
+        isa = income_statement_analyses(expected_rate_of_return)
         args = { }
-        args[:revenue_growth] = isa.revenue_growth_row.valid_vals.mean
-        args[:sales_pm      ] = isa.operating_pm_row.valid_vals.mean
-        args[:sales_over_noa] = isa.sales_over_noa_row.valid_vals.mean
-        args[:fi_over_nfa   ] = isa.fi_over_nfa_row.valid_vals.mean
-        return FinModeling::ConstantForecastingPolicy.new(args)
+
+        args[:revenue_estimator]        = TimeSeriesEstimator.from_time_series(re_is_arr[1..-1].map{ |re_is| re_is.period.value["end_date"] },
+                                                                               re_is_arr[1..-1].map{ |re_is| re_is.operating_revenues.total })
+        args[:sales_pm_estimator]       = TimeSeriesEstimator.from_time_series(re_is_arr[1..-1].map{ |re_is| re_is.period.value["end_date"] },
+                                                                               isa.operating_pm_row.vals[1..-1])
+        args[:sales_over_noa_estimator] = TimeSeriesEstimator.from_time_series(re_is_arr[1..-1].map{ |re_is| re_is.period.value["end_date"] },
+                                                                               isa.sales_over_noa_row.vals[1..-1])
+        args[:fi_over_nfa_estimator]    = TimeSeriesEstimator.from_time_series(re_is_arr[1..-1].map{ |re_is| re_is.period.value["end_date"] },
+                                                                               isa.fi_over_nfa_row.vals[1..-1])
+        return FinModeling::LinearTrendForecastingPolicy.new(args)
+
+      else
+        raise ArgumentError.new("\"#{policy_type}\" is not a valid forecasting policy type")
       end
     end
   
@@ -77,7 +143,7 @@ module FinModeling
 
       last_last_is = (self.length >= 2) ? self[-2].income_statement : nil
       puts "warning: last_last_is is nil..." if !last_last_is
-      last_re_is = self.last.income_statement.latest_quarterly_reformulated(last_last_is)
+      last_re_is = self.last.income_statement.latest_quarterly_reformulated(last_cis=nil, last_last_is, last_last_cis=nil)
       raise RuntimeError.new("last_re_is is nil!") if !last_re_is
 
       num_quarters.times do |i|

data/lib/finmodeling/comprehensive_income_calculation.rb

@@ -0,0 +1,60 @@
+module FinModeling
+  class ComprehensiveIncomeCalculation < CompanyFilingCalculation
+    include CanCacheClassifications
+    include CanCacheSummaries
+    include CanClassifyRows
+
+    BASE_FILENAME = File.join(FinModeling::BASE_PATH, "summaries/comprehensive_income_")
+
+    ALL_STATES  =                 [ :or, :cogs, :oe, :oibt, :fibt, :tax, :ooiat, :fiat, :ni, :ooci, :ooci_nci, :foci, :unkoci ]
+    NEXT_STATES = { nil        => [ :or,                                                :ni                                   ],
+                    :or        => [ :or, :cogs, :oe, :oibt, :fibt                                                             ],
+                    :cogs      => [      :cogs, :oe, :oibt, :fibt, :tax                                                       ],
+                    :oe        => [             :oe, :oibt, :fibt, :tax                                                       ],
+                    :oibt      => [                  :oibt, :fibt, :tax                                                       ], # obit/fibt can cycle back/forth
+                    :fibt      => [                  :obit, :fibt, :tax                                                       ], # obit/fibt can cycle back/forth
+                    :tax       => [                                      :ooiat, :fiat,      :ooci, :ooci_nci, :foci, :unkoci ], # 1 tax item. then moves forward.
+                    :ooiat     => [                                      :ooiat, :fiat,      :ooci, :ooci_nci, :foci, :unkoci ], # ooiat/fiat can cycle back/forth
+                    :fiat      => [                                      :ooiat, :fiat,      :ooci, :ooci_nci, :foci, :unkoci ], # ooiat/fiat can cycle back/forth
+
+                    :ni        => [                                                          :ooci, :ooci_nci, :foci, :unkoci ], # after ni, no ordering
+
+                    :ooci      => [                                                          :ooci, :ooci_nci, :foci, :unkoci ], # after ni, no ordering
+                    :ooci_nci  => [                                                          :ooci, :ooci_nci, :foci, :unkoci ], # after ni, no ordering
+                    :foci      => [                                                          :ooci, :ooci_nci, :foci, :unkoci ], # after ni, no ordering
+                    :unkoci    => [                                                          :ooci, :ooci_nci, :foci, :unkoci ] }# after ni, no ordering
+
+    def summary(args)
+      summary_cache_key = args[:period].to_pretty_s
+      thesummary = lookup_cached_summary(summary_cache_key)
+      return thesummary if !thesummary.nil?
+ 
+      mapping = Xbrlware::ValueMapping.new
+      mapping.policy[:debit] = :flip
+
+      thesummary = super(:period => args[:period], :mapping => mapping)
+      if !lookup_cached_classifications(BASE_FILENAME, thesummary.rows)
+        lookahead = [4, thesummary.rows.length-1].min
+        classify_rows(ALL_STATES, NEXT_STATES, thesummary.rows, FinModeling::ComprehensiveIncomeStatementItem, lookahead)
+        save_cached_classifications(BASE_FILENAME, thesummary.rows)
+      end
+
+      save_cached_summary(summary_cache_key, thesummary)
+
+      return thesummary
+    end
+ 
+    def has_revenue_item?
+      @has_revenue_item ||= summary(:period => periods.last).rows.any? do |row|
+        row.type == :or
+      end
+    end
+
+    def has_net_income_item?
+      @has_net_income_item ||= summary(:period => periods.last).rows.any? do |row|
+        row.type == :ni
+      end
+    end
+
+  end
+end

data/lib/finmodeling/comprehensive_income_statement_calculation.rb

@@ -0,0 +1,74 @@
+module FinModeling
+  class ComprehensiveIncomeStatementCalculation < CompanyFilingCalculation
+    include CanChooseSuccessivePeriods
+
+    CI_GOAL        = "comprehensive income"
+    CI_LABELS      = [ /^comprehensive (income|loss|loss income|income loss)(| net of tax)(| attributable to .*)$/ ]
+    CI_ANTI_LABELS = [ /noncontrolling interest/, 
+                       /minority interest/ ]
+    CI_IDS         = [ /^(|Locator_|loc_)(|us-gaap_)ComprehensiveIncomeNetOfTax[_0-9a-z]+/ ] 
+    def comprehensive_income_calculation
+      begin
+        @ci ||= ComprehensiveIncomeCalculation.new(find_calculation_arc(CI_GOAL, CI_LABELS, CI_ANTI_LABELS, CI_IDS))
+      rescue FinModeling::InvalidFilingError => e
+        pre_msg = "calculation tree:\n" + self.calculation.sprint_tree
+        raise e, pre_msg+e.message, e.backtrace
+      end
+    end
+
+    def is_valid?
+      if !comprehensive_income_calculation.has_net_income_item? && !comprehensive_income_calculation.has_revenue_item?
+        puts "comprehensive income statement's comprehensive income calculation lacks net income item"
+        puts "comprehensive income statement's comprehensive income calculation lacks sales/revenue item"
+        if comprehensive_income_calculation
+          puts "summary:"
+          comprehensive_income_calculation.summary(:period => periods.last).print
+        end
+        puts "calculation tree:\n" + self.calculation.sprint_tree(indent_count=0, simplified=true)
+      end
+      return (comprehensive_income_calculation.has_revenue_item? || comprehensive_income_calculation.has_net_income_item?)
+    end
+
+    def reformulated(period, dummy_comprehensive_income_calculation) # 2nd param is just to keep signature consistent w/ IncomeStatement::reformulated
+      # The way ReformulatedIncomeStatement.new() is implemented, it'll just ignore rows with types it 
+      # doesn't know about (like OCI). So this should extract just the NI-related rows.
+      return ReformulatedIncomeStatement.new(period,
+                                             comprehensive_income_calculation.summary(:period=>period), # NI
+                                             comprehensive_income_calculation.summary(:period=>period)) # CI
+    end
+
+    def latest_quarterly_reformulated(dummy_cur_ci_calc, prev_stmt, prev_ci_calc)
+      if comprehensive_income_calculation.periods.quarterly.any?
+        period = comprehensive_income_calculation.periods.quarterly.last
+        lqr = reformulated(period, comprehensive_income_calculation)
+
+        if (lqr.operating_revenues.total.abs > 1.0) && # FIXME: make an is_valid here?
+           (lqr.cost_of_revenues  .total.abs > 1.0)    # FIXME: make an is_valid here?
+          return lqr
+        end
+      end
+
+      return nil if !prev_stmt
+
+      prev_calc = prev_stmt.respond_to?(:net_income_calculation) ? prev_stmt.net_income_calculation : prev_stmt.comprehensive_income_calculation
+
+      cur_period, prev_period = choose_successive_periods(comprehensive_income_calculation, prev_calc)
+      if cur_period && prev_period
+        new_re_is = reformulated(cur_period, comprehensive_income_calculation) - prev_stmt.reformulated(prev_period, prev_ci_calc)
+        # the above subtraction doesn't know what period you want. So let's patch the result to have
+        # a quarterly period with the right end-points
+        new_re_is.period = Xbrlware::Context::Period.new({"start_date"=>prev_period.value["end_date"],
+                                                          "end_date"  =>cur_period.value["end_date"]})
+        return new_re_is
+      end
+
+      return nil
+    end
+
+    def write_constructor(file, item_name)
+      item_calc_name = item_name + "_calc"
+      @calculation.write_constructor(file, item_calc_name)
+      file.puts "#{item_name} = FinModeling::ComprehensiveIncomeStatementCalculation.new(#{item_calc_name})"
+    end
+  end
+end

data/lib/finmodeling/comprehensive_income_statement_item.rb

@@ -0,0 +1,20 @@
+module FinModeling
+  class ComprehensiveIncomeStatementItem < String
+    include HasStringClassifier
+
+    BASE_FILENAME = File.join(FinModeling::BASE_PATH, "classifiers/cisi_")
+    TYPES         = [ :or, :cogs, :oe, :oibt, :fibt, :tax, :ooiat, :fiat, :ni, :ooci, :ooci_nci, :foci, :unkoci ]
+    # same as in IncomeStatementItem, plus four new types:
+    # ni (net income -- optional, for when it is rolled up, versus (more typically) presented in the same detail as in the income statement)
+    # ooci (operating other comperhensive income)
+    # ooci_nci (operating other comperhensive income - non-controling interest)
+    # foci (financial other comperhensive income)
+    # unkoci (unknown (either operating or financial) other comperhensive income)
+
+    has_string_classifier(TYPES, ComprehensiveIncomeStatementItem)
+
+    def self.load_vectors_and_train
+      self._load_vectors_and_train(BASE_FILENAME, FinModeling::ComprehensiveIncomeStatementItem::TRAINING_VECTORS)
+    end
+  end
+end