yahoo_finance_lib 0.7.1 → 0.7.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d983b115acb96dd6eac3720ed79b1118f776444
-  data.tar.gz: c83f71a08f569cfbb5c7a08043d834a4ec87d362
+  metadata.gz: cd71a533f1acc521ce2c0640edca3edf271d4839
+  data.tar.gz: b89733ecc5c7c3e96626b70efb7d83c6ba71318f
 SHA512:
-  metadata.gz: c98697a3fe35489401b736aad6fedef4c677359656d904b0251530a74d40731d36af833f0ea63de2116478f0c2aa6553413b92eda37e733b210f22d3b5ab1c9d
-  data.tar.gz: b5b93dc9c7752c345390a971754c6eaa6ca868f1c216cfbf4b9d38bfb347c12de7e2142a073c4bda07a1edbdac602e4671fd5c78caeeb5d015ac05192f760a4a
+  metadata.gz: f9c5122b36f66b8acd2d615c5824f458b9c71223f2827bd9036c3aa1f41f788a17e69c4edea0d0add7d7f1bfb6848bc49ef6707243de9a3f3f0b30ee9d5123b3
+  data.tar.gz: 8bd86c5c3eac252821d39074c624f53c0f1d2934d379a1f0cc920fa43baec4fa89e9696a0c6d8075cedb754e3a885733cf70df129df9db8c2b5a1e249d512b2b

data/History.md

@@ -1,8 +1,7 @@
-# History of changes
+## History of changes
 
-### Version 0.6
-Original release, supporting key statistics, company events, and analyst opinion
-### Version 0.7
-Added company profile scraping :sector, and :industry
-### Version 0.7.1
-Added :website attribute in company profile; also added YahooFinance::StockHistory abstraction to nas/yahoo_finance
+Version | Description
+------- | -----------
+0.6 | Original release, supporting key statistics, company events, and analyst opinion
+0.7 | Added company profile scraping :sector, and :industry
+0.7.1 |Added :website attribute in company profile; also added YahooFinance::StockHistory abstraction to nas/yahoo_finance

data/README.md

@@ -1,6 +1,6 @@
 # YahooFinance library
 
-YahooFinance lib is a gem that fetches stock quotes, key statistics, company events, and analyst opinion from Yahoo (TM) API and financial HTML pages. The loading of stock quotes uses the excellent nas/yahoo_stock gem. This gem provides a 'unified' attribute based interface, and abstracts the source of the information, i.e. the user only needs to specify an attribute without the need to specify the page source of the attribute. This gem leverages to the highest extend possible the YahooStock (yahoo_stock_nas) gem for all the attributes provided by that gem, and fails over to HTML scraping when the data is unavailable there. Naturally, this has an implication on performance; YahooStock queries bundle 50 stocks in each query, whereas HTML scraping happens one page per stock at a time.
+YahooFinance lib is a gem that fetches stock quotes, key statistics, company profile, company events, and analyst opinion from Yahoo (TM) API and financial HTML pages. Additionally, it has interfaces for historical data, and financial statements. The loading of stock quotes uses the excellent nas/yahoo_stock gem. This gem provides a 'unified' attribute based interface, and abstracts the source of the information, i.e. the user only needs to specify an attribute without the need to specify the page source of the attribute. This gem leverages to the highest extend possible the YahooStock (yahoo_stock_nas) gem for all the attributes provided by that gem, and fails over to HTML scraping when the data is unavailable there. Naturally, this has an implication on performance; YahooStock queries bundle 50 stocks in each query, whereas HTML scraping happens one page per stock at a time.
 
 #####This gem is currently still in development. HTML scraping now supports many attributes from Key Statistics, supports earnings announcements from the Company Events page, and most attributes from Analyst Opinion -- additional pages will be added as I find time.
 
@@ -23,13 +23,20 @@ Or install it yourself as:
 
     $ gem install yahoo_finance_lib
 
+## Test Before You Use
+Web pages get occasionally updated -- updates could break any or all of the classes in the gem. Do not use any classes that don't pass the RSpec tests. To test, run the rspec tests from the root directory:
+```
+	rspec -c -f d
+```
+
 ## Usage
 
  
 
 Example:
+
+```ruby
 	irb
-	require 'rubygems'
 	require 'yahoo_finance'
 	
 	stock = YahooFinance::Stock.new(['AAPL', 'YHOO'], [:market_cap, :bid, :brokers_count, :upgrades_downgrades_history])
@@ -41,15 +48,35 @@ Example:
 	results = stock.fetch
 	aapl_bid = results["AAPL"][:bid]
 	yhoo_last = results["YHOO"][:last_trade_price_only]
+```
 
 A simple interface using yahoo_stock to fetch history. It returns history as an array of hashes, e.g.
+
+```ruby
 	irb
 	require 'yahoo_finance'
 	require 'date'
 	
 	start_date = Date.today - 30
-	history = YahooFinance::StockHistory.new('AAPL', start_date)
+	history = YahooFinance::StockHistory.new('AAPL', start_date) # when you don't specify end date, end date = today - 1
 	history.fetch
+```
+
+A simple interface to scrape financial statements - Income Statement, Balance Sheet, and Cash Flow. It can fetch either quarterly or annual data, and it returns the value in a hash. e.g.
+
+```ruby
+	irb
+	require 'yahoo_finance'
+	
+	income_stmt = YahooFinance::FinancialStatement::QuarterlyIncomeStatementPage.new 'YHOO'
+	result = income_stmt.fetch
+	most_recent_qtr = income_stmt.statement_periods[0]
+	available_fields = income_stmt.available_fields
+	
+	puts "Net income for the most recent quarter ending on #{most_recent_qtr.to_s} is #{result[:net_income][0]}"
+```
+
+The classes for the  financial statement are: QuarterlyIncomeStatementPage, AnnualIncomeStatementPage, AnnualBalanceSheetPage, QuarterlyBalanceSheetPage, QuarterlyCashFlowStatementPage, AnnualCashFlowStatementPage
 	
 !-- ## Contributing
 

data/lib/yahoo_finance.rb

@@ -6,4 +6,5 @@ require 'yahoo_finance/key_statistics'
 require 'yahoo_finance/company_events'
 require 'yahoo_finance/analyst_opinion'
 require 'yahoo_finance/company_profile'
+require 'yahoo_finance/financial_statement'
 

data/lib/yahoo_finance/analyst_opinion.rb

@@ -19,9 +19,9 @@ module YahooFinance
     end
 
     class AnalystOpinionPage
-      @symbol
+      attr_accessor :symbol
     
-      def initialize symbol
+      def initialize symbol=nil
         @symbol = symbol
       end
       

data/lib/yahoo_finance/company_events.rb

@@ -11,9 +11,9 @@ module YahooFinance
     end
 
     class CompanyEventsPage
-      @symbol
+      attr_accessor :symbol
     
-      def initialize symbol
+      def initialize symbol=nil
         @symbol = symbol
       end
       
@@ -38,4 +38,4 @@ module YahooFinance
   
     end
   end
-end
+end

data/lib/yahoo_finance/company_profile.rb

@@ -6,16 +6,18 @@ module YahooFinance
     COMPANY_PROFILE_STATS = {
       :sector => ['Sector:', "Sector classification"],
       :industry => ['Industry:', 'Industry classification'],
-      :website => ['Website:', 'Company website']
+      :website => ['Website:', 'Company website'],
+      :symbol => ['Symbol', 'Company Exchange Symbol'],
+      :company_name => ['Company Name', 'Company Name']
     }
     def CompanyProfile.key_events_available 
       return YahooFinance::CompanyProfile::COMPANY_PROFILE_STATS.keys;
     end
 
     class CompanyProfilePage
-      @symbol
+      attr_accessor :symbol
     
-      def initialize symbol
+      def initialize symbol=nil
         @symbol = symbol
       end
       
@@ -34,6 +36,9 @@ module YahooFinance
         if key_stat == :website
           r = @doc.xpath("//td[contains(., \"#{YahooFinance::CompanyProfile::COMPANY_PROFILE_STATS[key_stat][0]}\")]")
           return r.xpath("./a[contains(., 'http:')]").text
+        elsif key_stat == :company_name
+          name = @doc.xpath('//td[@class="yfnc_modtitlew1"]//b')[0].text
+          return name
         elsif CompanyProfile::key_events_available.include? key_stat
           begin
             value = @doc.xpath("//td[text() = \"#{YahooFinance::CompanyProfile::COMPANY_PROFILE_STATS[key_stat][0]}\"]")[0].parent.children[1].text

data/lib/yahoo_finance/financial_statement.rb

@@ -0,0 +1,324 @@
+require 'open-uri'
+require 'nokogiri'
+require 'date'
+
+module YahooFinance
+  module FinancialStatement
+    INCOME_STMT_FIELDS = {
+      :statement_periods => ['Period Ending', "Ending date per period"],
+      :total_revenue => ['Total Revenue', "Total Revenue (per period)"],
+      :cost_of_revenue => ['Cost of Revenue', 'Cost of Revenue (per period)'],
+      :gross_profit => ['Gross Profit', 'Gross profit (per period)'],
+      :research_development => ['Research Development', 'Research and Development (per period)'],
+      :selling_general_and_administrative => ['Selling General and Administrative', 'Selling General and Administrative (per period)'],
+      :non_recurring => ['Non Recurring', 'Non Recurring (per period)'],
+      :other_expenses => ['Others', 'Other expenses (per period)'],
+      :total_operating_expenses => ['Total Operating Expenses', 'Total Operating Expenses (per period)'],
+      :operating_income_or_loss => ['Operating Income or Loss', 'Operating Income or Loss (per period)'],
+      :total_other_income_expenses_net => ['Total Other Income/Expenses Net', 'Total Other Income/Expenses Net (per period)'],
+      :EBIT => ['Earnings Before Interest And Taxes', '	Earnings Before Interest And Taxes (per period)'],
+      :interest_expense => ['Interest Expense', 'Interest Expense (per period)'],
+      :income_before_tax => ['Income Before Tax', 'Income Before Tax (per period)'],
+      :income_tax_expense => ['Income Tax Expense', 'Income Tax Expense (per period)'],
+      :minority_interest => ['Minority Interest', 'Minority Interest (per period)'],
+      :net_income_from_continuing_operations => ['Net Income From Continuing Ops', 'Net Income From Continuing Ops (per period)'],
+      :discontinued_operations => ['Discontinued Operations', 'Discontinued Operations (per period)'],
+      :extraordinary_items =>['Extraordinary Items', 'Extraordinary Items (per period)'],
+      :effect_of_accounting_changes => ['Effect Of Accounting Changes', 'Effect Of Accounting Changes (per period)'],
+      :other_items => ['Other Items', 'Other Items (per period)'],
+      :net_income => ['Net Income', 'Net Income (per period)'],
+      :preferred_stock_and_other_adjustments => ['Preferred Stock And Other Adjustments', 'Preferred Stock And Other Adjustments (per period)'],
+      :net_income_applicable_to_common_shares => ['Net Income Applicable To Common Shares', 'Net Income Applicable To Common Shares (per period)']
+    }
+    
+    BALANCE_SHEET_FIELDS = {
+      :statement_periods => ['Period Ending', "Ending date per period"],
+      :cash_and_cash_equivalents => ['Cash And Cash Equivalents', 'Cash And Cash Equivalents'],
+      :short_term_investments => ['Short Term Investments', '	Short Term Investments'],
+      :net_receivables => ['Net Receivables', 'Net Receivables'],
+      :inventory => ['Inventory', 'Inventory'],
+      :other_current_assets => ['Other Current Assets', 'Other Current Assets'],
+      :total_current_assets => ['Total Current Assets', 'Total Current Assets'],
+      :long_term_investments => ['Long Term Investments', 'Long Term Investments'],
+      :property_plant_and_equipment => ['Property Plant and Equipment', 'Property Plant and Equipment'],
+      :goodwill => ['Goodwill', 'Goodwill'],
+      :intagible_assets => ['Intangible Assets', 'Intangible Assets'],
+      :accumulated_amortization => ['Accumulated Amortization', 'Accumulated Amortization'],
+      :other_assets => ['Other Assets', 'Other Assets'],
+      :deferred_long_term_asset_charges => ['Deferred Long Term Asset Charges', 'Deferred Long Term Asset Charges'],
+      :total_assets => ['Total Assets', 'Total Assets'],
+      :accounts_payable => ['Accounts Payable', 'Accounts Payable'],
+      :short_current_long_term_debt => ['Short/Current Long Term Debt', 'Short/Current Long Term Debt'],
+      :other_current_liabilities => ['Other Current Liabilities', 'Other Current Liabilities'],
+      :total_current_liabilities => ['Total Current Liabilities', 'Total Current Liabilities'],
+      :long_term_debt => ['Long Term Debt', 'Long Term Debt'],
+      :other_liabilities => ['Other Liabilities', 'Other Liabilities'],
+      :deferred_long_term_liability_charges => ['Deferred Long Term Liability Charges', 'Deferred Long Term Liability Charges'],
+      :minority_interest => ['Minority Interest', 'Minority Interest'],
+      :negative_goodwill => ['Negative Goodwill', 'Negative Goodwill'],
+      :total_liabilities => ['Total Liabilities', 'Total Liabilities'],
+      :misc_stocks_options_warrants => ['Misc Stocks Options Warrants', 'Misc Stocks Options Warrants'],
+      :redeemable_preferred_stock => ['Redeemable Preferred Stock', 'Redeemable Preferred Stock'],
+      :preferred_stock => ['Preferred Stock', 'Preferred Stock'],
+      :common_stock => ['Common Stock', 'Common Stock'],
+      :retained_earnings => ['Retained Earnings', 'Retained Earnings'],
+      :treasury_stock => ['Treasury Stock', 'Treasury Stock'],
+      :capital_surplus => ['Capital Surplus', 'Capital Surplus'],
+      :other_stockholder_equity => ['Other Stockholder Equity', 'Other Stockholder Equity'],
+      :total_stockholder_equity => ['Total Stockholder Equity', 'Total Stockholder Equity'],
+      :net_tangible_assets => ['Net Tangible Assets', 'Net Tangible Assets']
+    }
+    
+    CASH_FLOW_STMT_FIELDS = {
+      :statement_periods => ['Period Ending', "Ending date per period"],
+      :net_income => ['Net Income', 'Net Income'],
+      :depreciation => ['Depreciation', 'Depreciation'],
+      :adjustments_to_net_income => ['Adjustments To Net Income','Adjustments To Net Income'],
+      :changes_in_accounts_receivables => ['Changes In Accounts Receivables', 'Changes In Accounts Receivables'],
+      :changes_in_liabilities => ['Changes In Liabilities','Changes In Liabilities'],
+      :changes_in_inventories => ['Changes In Inventories', 'Changes In Inventories'],
+      :changes_in_other_operating_activities => ['Changes In Other Operating Activities', 'Changes In Other Operating Activities'],
+      :total_cash_flow_from_operating_activities => ['Total Cash Flow From Operating Activities', 'Total Cash Flow From Operating Activities'],
+      :capital_expenditures => ['Capital Expenditures', 'Capital Expenditures'],
+      :investments => ['Investments', 'Investments'],
+      :other_cash_flows_from_investing_activities => ['Other Cash flows from Investing Activities', 'Other Cash flows from Investing Activities'],
+      :total_cash_flows_from_investing_activities => ['Total Cash Flows From Investing Activities', 'Total Cash Flows From Investing Activities'],
+      :dividends_paid => ['Dividends Paid', 'Dividends Paid'],
+      :sale_purchase_of_stock => ['Sale Purchase of Stock', 'Sale Purchase of Stock'],
+      :net_borrowings => ['Net Borrowings', 'Net Borrowings'],
+      :other_cash_flows_from_financing_activities => ['Other Cash Flows from Financing Activities', 'Other Cash Flows from Financing Activities'],
+      :total_cash_flows_from_financing_activities => ['Total Cash Flows From Financing Activities', 'Total Cash Flows From Financing Activities'],
+      :effect_of_exchange_rate_changes => ['Effect Of Exchange Rate Changes', 'Effect Of Exchange Rate Changes'],
+      :change_in_cash_and_cash_equivalents => ['Change In Cash and Cash Equivalents', 'Change In Cash and Cash Equivalents']
+    }
+    
+
+    # Quarterly Data
+    class FinancialStatementPage
+      attr_accessor :symbol
+    
+      def initialize symbol=nil
+        @symbol = symbol
+        @term = :quarterly
+        @periods = []
+        @number_multiplier = 1000.00     # all numbers in the statements are in thousands... I need to verify whether I need to parse that
+        @financial_statement = {}
+        @data_columns = 4
+        @type="is"
+        @field_defs = YahooFinance::FinancialStatement::INCOME_STMT_FIELDS
+      end
+      
+      def term= aValue
+        if term == :quarterly || term == :annual
+          @term = term
+        end
+      end
+      
+     def available_fields 
+        return @field_defs.keys;
+      end
+      
+      def fetch
+        url = "http://finance.yahoo.com/q/#{@type}?s=#{@symbol}&#{@term.to_s}"
+        doc = nil
+        tries = 0
+        begin
+          # SINCE YAHOO HAS SOME TIMES THE TENDENCY TO RETURN AN OOPS PAGE RATHER THAN A HTTP FAILURE
+          # WE ARE USING A TEST OF A HEADER FIELD AS A CONDITION OF A CORRECTLY FETCHED PAGE
+          fetched_ok = false
+          fetch_tries = 1
+          while (fetched_ok == false)  && (fetch_tries <= 3)
+            open(url) do |stream|
+              doc = Nokogiri::HTML(stream)
+            end
+            search_field = ((@type == "is") ? "Income Statement" : ((@type == "bs") ? "Balance Sheet" : "Cash Flow") )
+            fetched_test = doc.xpath("//b[text()[contains(., '" + search_field +"')]]")
+            if fetched_test && fetched_test.size > 0
+              # THERE IS STILL A CHANCE THAT YAHOO WILL NOT HAVE THE DATA FOR THIS STOCK, IN WHICH CASE, IT WILL DISPLAY
+              # a PAGE WITH A STATEMENT LIKE: 'There is no Income Statement data available for AMBR.' --- WE SEARCH FOR THAT AND RETURN NIL IF FOUND
+              nodata_path_search_expr = "//p[text()[contains(., 'There is no "+ search_field + " data available for #{symbol}.')]]"
+              no_statement = doc.xpath(nodata_path_search_expr)
+              # puts "EXPR: #{nodata_path_search_expr}    NO STATEMENT: #{no_statement} AND SIZE: #{no_statement.size}"
+              if (no_statement  && (no_statement.size > 0))
+                puts "WARNING: FOUND There is no #{search_field} data available for #{symbol}. -- RETURNING NIL"
+                return nil        # YAHOO DOESN'T HAVE THE DATA...
+              else
+                fetched_ok = true
+              end
+            else
+              puts "Fetching URL #{url} again - previous one failed. Trying #{fetch_tries} of 3"
+              sleep (1)
+              fetch_tries += 1
+            end
+          end
+        rescue
+          tries += 1
+          if fetch_tries <= 3
+            puts "Exception: #{$!.inspect}  -- Retrying symbol #{@symbol} time: #{fetch_tries.to_s} (total 3)"
+            sleep(2)
+            retry
+          end 
+          
+        end
+        #
+        # %%% OK, WE MUST MAKE THIS MORE ROBUST TO HANDLE FEWER REPORTING PERIODS, e.g. ACT Actavis has 3 quarters reported
+        #
+        # let's initialize periods: by finding 'Period Endinng
+        got_periods = true
+        begin
+          row = doc.xpath("//span[text() = 'Period Ending']")[0].parent.parent.parent
+          period = []
+          1.upto(@data_columns) do |i|
+            begin
+              pd = Date.parse(row.children[i].text)
+              @periods << pd
+            rescue
+              break;    # end of the line here
+            end
+          end
+          @financial_statement[:statement_periods] = @periods
+        rescue
+          puts "Exception: #{$!.inspect}  -- Skipping symbol #{@symbol} URL: #{url}"
+          # puts "DOC: #{doc}"
+          got_periods = false
+        end
+        
+        return nil if !got_periods
+        
+        # YahooFinance.parse_financial_statement_field field, multiplier
+        # let's fetch everything here & store in the income statement
+        @field_defs.keys.each do |incst_key|
+          next if incst_key == :statement_periods
+          path_expr = "//td[text()[contains(., '" + @field_defs[incst_key][0]  + "')]]"
+          row = nil
+          elem = doc.xpath(path_expr)
+          index = 1
+          regex = "\\s\*#{@field_defs[incst_key][0]}\\s\*"
+          if elem && elem.size == 0
+            # this is because we have a strong type; let's try again
+            path_expr = "//td/strong[text()[contains(., '" + @field_defs[incst_key][0]  +"')]]"
+            elem = doc.xpath(path_expr)
+            if elem.size > 0
+              elem.each do |e|
+                if e.text.match regex
+                  # we found it
+                  row = e.parent.parent
+                end
+              end
+              if !row
+                # wasn't found
+                puts "#{@field_defs[incst_key][0]} was not found!"
+                next
+              end
+              index = row.children.index(elem[0].parent)
+            end
+          else
+            elem.each do |e|
+              # puts "Checking #{e}: class #{e.text.class.name} with text <<#{e.text}>> and regex #{regex}"
+              if e.text.match regex
+                # we found it
+                row = e.parent
+                break
+              end
+            end
+            if !row
+              # wasn't found
+              puts "#{@field_defs[incst_key][0]} was not found!"
+              next
+            end
+            index = row.children.index(elem[0])
+          end
+          if elem && elem.size > 0
+            # puts "\tFOUND!!!! PATH EXPRESSION IS #{path_expr} "
+            # puts "\t\tROW: #{row}"
+            rstbl = []
+            1.upto(@periods.size) do |i|
+              field = row.children[index+i].text.tr('^[0-9\,\.\-\(\)]', '')
+              rs = YahooFinance.parse_financial_statement_field(field, @number_multiplier)
+              rstbl << rs
+            end
+            
+            @financial_statement[incst_key] = rstbl
+            # puts "\tPARSED #{incst_key}"
+          else
+            puts "\tNOT FOUND!!!! #{incst_key}"
+          end
+        end
+        @financial_statement
+      end
+      
+      def statement_periods
+        @periods
+      end
+       
+      def value_for key_stat
+        begin
+          return @financial_statement[key_stat]
+        rescue
+        end
+        return nil
+      end  
+    end
+
+    class QuarterlyIncomeStatementPage < FinancialStatementPage
+      def initialize symbol = nil
+        super symbol
+        @term = :quarterly
+        @data_columns = 4
+        @type="is"
+        @field_defs = YahooFinance::FinancialStatement::INCOME_STMT_FIELDS
+      end
+    end
+    
+    class AnnualIncomeStatementPage < FinancialStatementPage
+      def initialize symbol = nil
+        super symbol
+        @term = :annual
+        @data_columns = 3
+        @type="is"
+        @field_defs = YahooFinance::FinancialStatement::INCOME_STMT_FIELDS
+      end
+    end
+    
+    class QuarterlyBalanceSheetPage < FinancialStatementPage
+      def initialize symbol = nil
+        super symbol
+        @term = :quarterly
+        @data_columns = 4
+        @type="bs"
+        @field_defs = YahooFinance::FinancialStatement::BALANCE_SHEET_FIELDS
+      end
+    end
+
+    class AnnualBalanceSheetPage < FinancialStatementPage
+      def initialize symbol = nil
+        super symbol
+        @term = :annual
+        @data_columns = 3
+        @type="bs"
+        @field_defs = YahooFinance::FinancialStatement::BALANCE_SHEET_FIELDS
+      end
+    end
+
+    class QuarterlyCashFlowStatementPage < FinancialStatementPage
+      def initialize symbol = nil
+        super symbol
+        @term = :quarterly
+        @data_columns = 4
+        @type="cf"
+        @field_defs = YahooFinance::FinancialStatement::CASH_FLOW_STMT_FIELDS
+      end
+    end
+
+    class AnnualCashFlowStatementPage < FinancialStatementPage
+      def initialize symbol = nil
+        super symbol
+        @term = :annual
+        @data_columns = 3
+        @type="cf"
+        @field_defs = YahooFinance::FinancialStatement::CASH_FLOW_STMT_FIELDS
+      end
+    end
+
+  end
+end

data/lib/yahoo_finance/key_statistics.rb

@@ -75,11 +75,12 @@ module YahooFinance
     end
 
     class StatsPage
+      attr_accessor :symbol
       @page_keys = []
       @page_values = []
-      @symbol
+
     
-      def initialize symbol
+      def initialize symbol=nil
         @symbol = symbol
       end
       

data/lib/yahoo_finance/stock.rb

@@ -72,6 +72,18 @@ module YahooFinance
       return dt if dt
     end
     
+    m=aField.match /^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[\s]+([\d]{1,2})\s*\-\s*(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[\s]+([\d]{1,2})\,[\s]*([\d]{4})/
+    if m && (m.size == 6)
+      # we found it, and it is a date range
+      dt1 = dt2 = nil
+      begin
+        dt1 = Date.strptime "#{m[1]} #{m[2]}, #{m[5]}", '%b %d, %Y'
+        dt2 = Date.strptime "#{m[3]} #{m[4]}, #{m[5]}", '%b %d, %Y'
+      rescue
+      end
+      return [dt1, dt2]
+    end
+    
     #else make an attempt to parse as date
     dt = nil
     begin
@@ -83,16 +95,42 @@ module YahooFinance
     aField
   end
   
+  # this really parses numeric fields only
+  def YahooFinance.parse_financial_statement_field aField, multiplier = 1000.0
+    aField = aField.strip
+    
+    multiplier = 1000.00 if !multiplier   # just in case a nil is forced...
+    
+    if aField.match /^([\d\,])*(\.[\d]+)*$/
+      # it's a number as far as we care; let's strip the commas....
+      return aField.tr('\,', '').to_f * multiplier
+    end
+    
+    # process the accounting negative field -- parentheses
+    m = aField.match /^\(([\d\,])*(\.[\d]+)*\)$/
+    if m
+      # it's a number as far as we care; let's strip the commas....
+      aField = aField.tr('\(\)\,', '')
+      return aField.to_f * multiplier * -1.0
+    end
+    
+    # anything else becomes zero by definition
+    return 0.0
+    
+  end
+  
   class Stock
-    @@available_fields = (AVL_FIELDS[:YAHOO_STOCK_FIELDS] + AVL_FIELDS[:KEY_STATISTICS] + AVL_FIELDS[:COMPANY_EVENTS] + AVL_FIELDS[:ANALYST_OPINION])
-    @@insert_variable_delays = true
-    @@insert_variable_delay = 4
+    attr_reader :results_hash, :print_progress
+    @@available_fields = (AVL_FIELDS[:YAHOO_STOCK_FIELDS] + AVL_FIELDS[:KEY_STATISTICS] + AVL_FIELDS[:COMPANY_EVENTS] + AVL_FIELDS[:ANALYST_OPINION] + AVL_FIELDS[:COMPANY_PROFILE])
+    @@insert_variable_delays = false
+    @@insert_variable_delay = 2
     @@retry_times = 3
     @@retry_variable_delay = 30
     @symbols = []
     @fields = []
     @fields_hash = {}
     @results_hash = {}
+    @print_progress = false
     
     def self.insert_variable_delays
       @@insert_variable_delays
@@ -188,13 +226,16 @@ module YahooFinance
       end
       # Then we fetch fields from KEY STATISTICS
       DRIVERS.keys.each do |driver|
-        # puts "CHECKING #{driver.to_s}"
         if @fields_hash[driver].size > 0
-          # puts "FETCHING FIELDS FOR #{driver.to_s}"
+          puts "FETCHING FIELDS FOR #{driver.to_s}" if @print_progress
+          stp = DRIVERS[driver].new # will overwrite symbol below
           @symbols.each do |aSymbol|
             tries = 0
+            now = Time.now
+            stp.symbol = aSymbol
+            puts "#{now.hour}:#{now.min}:#{now.sec}\tWorking on symbol #{aSymbol}" if @print_progress
             begin
-              stp = DRIVERS[driver].new(aSymbol)
+              # stp = DRIVERS[driver].new(aSymbol) # will overwrite symbol below
               stp.fetch
               @fields_hash[driver].each do |aField|
                 value = stp.value_for(aField) # this already parses the numbers
@@ -203,12 +244,17 @@ module YahooFinance
               sleep(Random.new(Random.new_seed).rand*@@insert_variable_delay)  if @@insert_variable_delays
             rescue
               tries += 1
-              sleep(@@retry_variable_delay)
-              retry if tries <= @@retry_times
+              puts "Exception: #{$!.inspect}  -- Retrying symbol #{aSymbol} time: #{tries.to_s}"
+              if tries <= @@retry_times
+                sleep(@@retry_variable_delay)
+                retry
+              end 
             end
           end
         end
+        puts "DONE witn #{driver.to_s}" if @print_progress
       end
+      # puts "RIGHT BEFORE I RETURN THE BIG HASH: #{@results_hash}"
       @results_hash
     end
         
@@ -230,7 +276,7 @@ module YahooFinance
           end
         end
       end
-   end
+    end
   end
   
   class StockHistory

data/lib/yahoo_finance/version.rb

@@ -1,3 +1,3 @@
 module YahooFinance
-  VERSION = "0.7.1"
+  VERSION = "0.7.9.8"
 end

data/spec/yahoo_company_events_spec.rb

@@ -10,13 +10,14 @@ describe YahooFinance::CompanyEvents do
     it "available stats should include things like :next_earnings_announcement_date" do
       YahooFinance::CompanyEvents.key_events_available.include?(:next_earnings_announcement_date).should == true
     end
-    it "should fetch a  stat like :next_earnings_announcement_date for AAPL and should be either Date or String" do
+    it "should fetch a  stat like :next_earnings_announcement_date for AAPL and should be either Date, and Array of two dates, or String" do
       pg = YahooFinance::CompanyEvents::CompanyEventsPage.new('AAPL')
       pg.fetch
       aapl_next_earnings_date = pg.value_for :next_earnings_announcement_date
       case aapl_next_earnings_date.class.name
-      when "Date" then aapl_next_earnings_date.class.name.should == "Date"
-      when "String" then aapl_next_earnings_date.class.name.should == "String"
+        when "Date" then aapl_next_earnings_date.class.name.should == "Date"
+        when "Array" then aapl_next_earnings_date.size.should == 2 && aapl_next_earnings_date[0].class.name.should == "Date" && aapl_next_earnings_date[1].class.name.should == "Date"
+        when "String" then aapl_next_earnings_date.class.name.should == "String"
       else
         aapl_next_earnings_date.class.name.should == "Date" #force a failure
       end

data/spec/yahoo_financial_statement_spec.rb

@@ -0,0 +1,124 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+# really more like a quick sample, more concerned about connectivity -- a better test needs to be developed to exhaustively
+# test all fields to make sure Yahoo hasn't changed the format...
+# %%%%%%
+describe YahooFinance::FinancialStatement do
+  # before(:each) do
+  #   @tsymbols = ['AAPL', 'DDD']
+  #   @page = YahooFinance::Page.new(@tsymbols)
+  # end
+  
+  describe "Financial Statement" do
+    it "Income Statement should include things like :next_earnings_announcement_date" do
+      YahooFinance::FinancialStatement::INCOME_STMT_FIELDS.include?(:operating_income_or_loss).should == true
+    end
+    it "Balance Sheet should include things like :intagible_assets" do
+      YahooFinance::FinancialStatement::BALANCE_SHEET_FIELDS.include?(:intagible_assets).should == true
+    end
+    it "Cash Flow Statement should include things like :total_cash_flow_from_operating_activities" do
+      YahooFinance::FinancialStatement::CASH_FLOW_STMT_FIELDS.include?(:total_cash_flow_from_operating_activities).should == true
+    end
+    it "should fetch all fields for e.g. AAPL quarterly Income statement and all values should be float (except for :statement_periods)-- valid values" do
+      is = YahooFinance::FinancialStatement::QuarterlyIncomeStatementPage.new 'AAPL'
+      rs = is.fetch
+      YahooFinance::FinancialStatement::INCOME_STMT_FIELDS.keys.each do |attr|
+        fields = rs[attr]
+        fields.class.name.should == "Array"
+        fields.size.should == 4
+        fields.each do |field|
+          attr == :statement_periods ? field.class.name.should == "Date" : field.class.name.should == "Float"
+        end
+      end
+    end
+    it "should fetch all fields for e.g. AAPL annual Income statement and all values should be float (except for :statement_periods)-- valid values" do
+      is = YahooFinance::FinancialStatement::AnnualIncomeStatementPage.new 'AAPL'
+      rs = is.fetch
+      YahooFinance::FinancialStatement::INCOME_STMT_FIELDS.keys.each do |attr|
+        fields = rs[attr]
+        fields.class.name.should == "Array"
+        fields.size.should == 3
+        fields.each do |field|
+          attr == :statement_periods ? field.class.name.should == "Date" : field.class.name.should == "Float"
+        end
+      end
+    end
+    it "should fetch all fields for e.g. AAPL quarterly Balance Sheet statement and all values should be float (except for :statement_periods)-- valid values" do
+      is = YahooFinance::FinancialStatement::QuarterlyBalanceSheetPage.new 'AAPL'
+      rs = is.fetch
+      YahooFinance::FinancialStatement::BALANCE_SHEET_FIELDS.keys.each do |attr|
+        fields = rs[attr]
+        fields.class.name.should == "Array"
+        fields.size.should == 4
+        fields.each do |field|
+          attr == :statement_periods ? field.class.name.should == "Date" : field.class.name.should == "Float"
+        end
+      end
+    end
+    it "should fetch all fields for e.g. AAPL annual Balance Sheet statement and all values should be float (except for :statement_periods)-- valid values" do
+      is = YahooFinance::FinancialStatement::AnnualBalanceSheetPage.new 'AAPL'
+      rs = is.fetch
+      YahooFinance::FinancialStatement::BALANCE_SHEET_FIELDS.keys.each do |attr|
+        fields = rs[attr]
+        fields.class.name.should == "Array"
+        fields.size.should == 3
+        fields.each do |field|
+          attr == :statement_periods ? field.class.name.should == "Date" : field.class.name.should == "Float"
+        end
+      end
+    end
+    it "should fetch all fields for e.g. AAPL quarterly Cash Flow statement and all values should be float (except for :statement_periods)-- valid values" do
+      is = YahooFinance::FinancialStatement::QuarterlyCashFlowStatementPage.new 'AAPL'
+      rs = is.fetch
+      YahooFinance::FinancialStatement::CASH_FLOW_STMT_FIELDS.keys.each do |attr|
+        fields = rs[attr]
+        fields.class.name.should == "Array"
+        fields.size.should == 4
+        fields.each do |field|
+          attr == :statement_periods ? field.class.name.should == "Date" : field.class.name.should == "Float"
+        end
+      end
+    end
+    it "should fetch all fields for e.g. AAPL annual Cash Flow statement and all values should be float (except for :statement_periods)-- valid values" do
+      is = YahooFinance::FinancialStatement::AnnualCashFlowStatementPage.new 'AAPL'
+      rs = is.fetch
+      YahooFinance::FinancialStatement::CASH_FLOW_STMT_FIELDS.keys.each do |attr|
+        fields = rs[attr]
+        fields.class.name.should == "Array"
+        fields.size.should == 3
+        fields.each do |field|
+          attr == :statement_periods ? field.class.name.should == "Date" : field.class.name.should == "Float"
+        end
+      end
+    end
+    it "should fetch all fields for e.g. ACT (3 quarters only on 11/19/14) quarterly Cash Flow statement and all values should be float (except for :statement_periods)-- valid values" do
+      is = YahooFinance::FinancialStatement::QuarterlyCashFlowStatementPage.new 'ACT'
+      rs = is.fetch
+      YahooFinance::FinancialStatement::CASH_FLOW_STMT_FIELDS.keys.each do |attr|
+        fields = rs[attr]
+        fields.class.name.should == "Array"
+        fields.size.should == 3
+        fields.each do |field|
+          attr == :statement_periods ? field.class.name.should == "Date" : field.class.name.should == "Float"
+        end
+      end
+    end
+    it "should return a nil hash if yahoo doesn't have the statement, e.g. for AMBR" do
+      is = YahooFinance::FinancialStatement::QuarterlyCashFlowStatementPage.new 'AMBR'
+      rs = is.fetch
+      rs.should == nil
+    end
+    
+    #   pg = YahooFinance::CompanyEvents::CompanyEventsPage.new('AAPL')
+    #   pg.fetch
+    #   aapl_next_earnings_date = pg.value_for :next_earnings_announcement_date
+    #   case aapl_next_earnings_date.class.name
+    #     when "Date" then aapl_next_earnings_date.class.name.should == "Date"
+    #     when "Array" then aapl_next_earnings_date.size.should == 2 && aapl_next_earnings_date[0].class.name.should == "Date" && aapl_next_earnings_date[1].class.name.should == "Date"
+    #     when "String" then aapl_next_earnings_date.class.name.should == "String"
+    #   else
+    #     aapl_next_earnings_date.class.name.should == "Date" #force a failure
+    #   end
+    #
+  end
+end

metadata

@@ -1,101 +1,101 @@
 --- !ruby/object:Gem::Specification
 name: yahoo_finance_lib
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.9.8
 platform: ruby
 authors:
 - Takis Mercouris
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-13 00:00:00.000000000 Z
+date: 2014-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nas-yahoo_stock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.8
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.8
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.8
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.1
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.1
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.14.1
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.14.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.14.1
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.14.1
 description: Unified library to stock quotes and various pages in yahoo finance, including
@@ -106,7 +106,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - History.md
 - LICENSE.txt
@@ -116,6 +116,7 @@ files:
 - lib/yahoo_finance/analyst_opinion.rb
 - lib/yahoo_finance/company_events.rb
 - lib/yahoo_finance/company_profile.rb
+- lib/yahoo_finance/financial_statement.rb
 - lib/yahoo_finance/key_statistics.rb
 - lib/yahoo_finance/stock.rb
 - lib/yahoo_finance/version.rb
@@ -124,6 +125,7 @@ files:
 - spec/yahoo_company_events_spec.rb
 - spec/yahoo_company_profile_spec.rb
 - spec/yahoo_finance_spec.rb
+- spec/yahoo_financial_statement_spec.rb
 - spec/yahoo_key_stats_spec.rb
 - yahoo_finance.gemspec
 homepage: http://polymechanus.com/yahoo_finance-ruby-gem-tutorial/
@@ -136,17 +138,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: Fetch stock information from yahoo finance API and HTML pages
@@ -156,4 +158,5 @@ test_files:
 - spec/yahoo_company_events_spec.rb
 - spec/yahoo_company_profile_spec.rb
 - spec/yahoo_finance_spec.rb
+- spec/yahoo_financial_statement_spec.rb
 - spec/yahoo_key_stats_spec.rb