stock_index 0.8.3 → 0.8.4

data/lib/stock_index/bsym_search.rb

@@ -4,17 +4,9 @@ class StockIndex
 
     require 'csv'
 
-    EXCEPTIONS = {
-      'BIDU' => {name: 'BAIDU INC', bbgid: 'BBG000QXWHD1'},
-      'LVNTA' => {name: 'LIBERTY VENTURES', bbgid: 'BBG0038K9G41'},
-      'VIP' => {name: 'VIMPELCOM LTD', bbgid: 'BBG000QCW561'},
-      'VOD' => {name: 'VODAFONE GROUP PLC', bbgid: 'BBG000C4R6H6'}
-    }
-
     class << self
 
       def find(symbol, pricing_source)
-        return {name: EXCEPTIONS[symbol][:name], bbgid: EXCEPTIONS[symbol][:bbgid]} if EXCEPTIONS.keys.include? symbol
         CSV.foreach(files(pricing_source)) do |row|
           return {name: row[1], bbgid: row[2]} if row[0] == symbol
         end

data/lib/stock_index/component.rb

@@ -12,11 +12,9 @@ class StockIndex
     end
 
     def attributes
-      cache_lookup || attributes_lookup
-    end
-
-    def valid?
-      @market && @symbol
+      attrs = cache_lookup || attributes_lookup
+      puts "---- ERROR #{attrs}" unless valid?(attrs)
+      attrs
     end
 
     def cache_lookup
@@ -36,10 +34,20 @@ class StockIndex
     end
 
     def attributes_lookup
-      bsym = lookup_bsym
+      bsym = StockIndex::BsymSearch.find(@symbol, @pricing_source)
       cik = lookup_cik
+      puts "   --- #{@symbol} bsym: #{bsym}"
+      puts "   --- #{@symbol} cik: #{cik}"
       return nil unless bsym
-      a = {market: @market, symbol: @symbol, name: bsym[:name], wikipedia: @wikipedia, cik: cik, bbgid: bsym[:bbgid]}
+      a = {
+        market: @market,
+        share: {
+          symbol: @symbol,
+          name: bsym[:name],
+          bbgid: bsym[:bbgid]
+        },
+        company: {wikipedia: @wikipedia}.merge( cik || {} )
+      }
       cache_write(a)
       a
     end
@@ -48,16 +56,6 @@ class StockIndex
       @pricing_source == :us
     end
 
-    def lookup_bsym
-      bsym = StockIndex::BsymSearch.find(@symbol, @pricing_source)
-      if bsym
-        bsym
-      else
-        puts "bsym --> #{@symbol}" unless testing?
-        return nil
-      end
-    end
-
     def lookup_cik
       if us?
         lookup_cik_us
@@ -69,15 +67,22 @@ class StockIndex
     def lookup_cik_us
       edgar = Cik.lookup(SymbolParser.new(@symbol).bsym_to_cik)
       if edgar
-        edgar[:cik]
+        {
+          cik: edgar[:cik],
+          name: edgar[:name],
+          sic: edgar[:sic]
+        }
       else
-        puts "cik --> #{@symbol}" unless testing?
         return nil
       end
     end
 
-    def testing?
-      @symbol == 'ZZZZ'
+    def valid?(attributes)
+      !attributes[:market].nil? &&
+      !attributes[:share][:symbol].nil? &&
+      !attributes[:share][:name].nil? &&
+      !attributes[:share][:bbgid].nil? &&
+      !attributes[:company][:wikipedia].nil?
     end
 
   end

data/lib/stock_index/scrapers/base_scraper.rb

@@ -10,7 +10,7 @@ class StockIndex
         market = market(tr)
         if symbol && market
           component = StockIndex::Component.new(symbol, market, wikipedia_link(tr))
-          array << component.attributes if component.valid?
+          array << component.attributes
         end
         array
       end
@@ -27,7 +27,7 @@ class StockIndex
     end
 
     def build_wikipedia_link(wikipedia_path)
-      URI::HTTP.build({:host => 'en.wikipedia.org', :path => wikipedia_path}).to_s rescue nil
+      URI::Generic.new('https', nil, 'en.wikipedia.org', nil, nil, wikipedia_path, nil, nil, nil).to_s rescue nil
     end
 
   end

data/lib/stock_index/scrapers/ftse_scraper.rb

@@ -17,7 +17,7 @@ class FtseScraper < StockIndex::BaseScraper
       market = 'XLON'
       if symbol && market
         component = StockIndex::Component.new(symbol, market, @wikipedia_hash[symbol], :ln)
-        array << component.attributes if component.valid?
+        array << component.attributes
       end
       array
     end
@@ -26,7 +26,7 @@ class FtseScraper < StockIndex::BaseScraper
   def symbol(tr)
     symbol_td = td(tr, 0)
     s = symbol_td ? symbol_td.text : nil
-    SymbolParser.new(s).symbol_to_bsym
+    s.nil? ? nil : s.sub('.', '/')
   end
 
   def url(page)
@@ -41,7 +41,49 @@ class FtseScraper < StockIndex::BaseScraper
         hash[symbol_td[1].text] = build_wikipedia_link(link_td.css('a').first.attributes['href'].value)
       end
       hash
-    end
+    end.merge(ftse_not_found_wikipedia_links)
+  end
+
+  def ftse_not_found_wikipedia_links
+    {
+      'III' => 'https://en.wikipedia.org/wiki/3i',
+      'AHT' => 'https://en.wikipedia.org/wiki/Ashtead_Group',
+      'AV/' => 'https://en.wikipedia.org/wiki/Aviva',
+      'BA/' => 'https://en.wikipedia.org/wiki/BAE_Systems',
+      'BDEV' => 'https://en.wikipedia.org/wiki/Barratt_Developments',
+      'BG/' => 'https://en.wikipedia.org/wiki/BG_Group',
+      'BP/' => 'https://en.wikipedia.org/wiki/BP',
+      'BATS' => 'https://en.wikipedia.org/wiki/British_American_Tobacco',
+      'BT/A' => 'https://en.wikipedia.org/wiki/BT_Group',
+      'CCL' => 'https://en.wikipedia.org/wiki/Carnival_Corporation_%26_plc',
+      'DLG' => 'https://en.wikipedia.org/wiki/Direct_Line_Group',
+      'DC/' => 'https://en.wikipedia.org/wiki/Dixons_Carphone',
+      'EZJ' => 'https://en.wikipedia.org/wiki/EasyJet',
+      'HL/' => 'https://en.wikipedia.org/wiki/Hargreaves_Lansdown',
+      'HIK' => 'https://en.wikipedia.org/wiki/Hikma_Pharmaceuticals',
+      'ISAT' => 'https://en.wikipedia.org/wiki/Inmarsat',
+      'INTU' => 'https://en.wikipedia.org/wiki/Intu_Properties',
+      'LSE' => 'https://en.wikipedia.org/wiki/London_Stock_Exchange_Group',
+      'MERL' => 'https://en.wikipedia.org/wiki/Merlin_Entertainments',
+      'MNDI' => 'https://en.wikipedia.org/wiki/Mondi',
+      'NG/' => 'https://en.wikipedia.org/wiki/National_Grid_plc',
+      'PSON' => 'https://en.wikipedia.org/wiki/Pearson_PLC',
+      'PSN' => 'https://en.wikipedia.org/wiki/Persimmon_plc',
+      'RDSB' => 'https://en.wikipedia.org/wiki/Royal_Dutch_Shell',
+      'RB/' => 'https://en.wikipedia.org/wiki/Reckitt_Benckiser',
+      'RR/' => 'https://en.wikipedia.org/wiki/Rolls-Royce_Holdings',
+      'RMG' => 'https://en.wikipedia.org/wiki/Royal_Mail',
+      'SHP' => 'https://en.wikipedia.org/wiki/Shire_(pharmaceutical_company)',
+      'SKY' => 'https://en.wikipedia.org/wiki/Sky_plc',
+      'SN/' => 'https://en.wikipedia.org/wiki/Smith_%26_Nephew',
+      'SPD' => 'https://en.wikipedia.org/wiki/Sports_Direct',
+      'STJ' => 'https://en.wikipedia.org/wiki/St._James%27s_Place_plc',
+      'SL/' => 'https://en.wikipedia.org/wiki/Standard_Life',
+      'TW/' => 'https://en.wikipedia.org/wiki/Taylor_Wimpey',
+      'TPK' => 'https://en.wikipedia.org/wiki/Travis_Perkins',
+      'TUI' => 'https://en.wikipedia.org/wiki/TUI_Group',
+      'UU/' => 'https://en.wikipedia.org/wiki/United_Utilities'
+    }
   end
 
 end

data/lib/stock_index/scrapers/nasdaq_scraper.rb

@@ -8,10 +8,11 @@ class NasdaqScraper < StockIndex::BaseScraper
   def parse_rows(rows)
     rows.inject([]) do |array, li|
       symbol = /.+\((\w+)\)/.match(li)[1]
+      symbol = 'SWKS' if symbol == 'SKWS' # wikipedia erratum
       market = 'XNAS'
       if symbol && market
         component = StockIndex::Component.new(symbol, market, wikipedia(li))
-        array << component.attributes if component.valid?
+        array << component.attributes
       end
       array
     end

data/lib/stock_index/scrapers/nikkei_scraper.rb

@@ -14,18 +14,18 @@ class NikkeiScraper < StockIndex::BaseScraper
       market = 'XJPX'
       if symbol && market
         component = StockIndex::Component.new(symbol, market, @wikipedia_hash[symbol], :jp)
-        array << component.attributes if component.valid?
+        array << component.attributes
       end
       array
     end
   end
 
   def parse_wikipedia_page(wikipedia_doc)
-    wikipedia_doc.css('li').inject({}) do |hash, li|
-      md = li.text.match(/.+\(TYO:\s+(\d{4})/)
+    wikipedia_doc.css('#constituents tr').inject({}) do |hash, tr|
+      md = tr.text.match(/\n(\d{4})\n/)
       if md
         symbol = md[1]
-        link = build_wikipedia_link(li.css('a').first.attributes['href'].value)
+        link = build_wikipedia_link(tr.css('a').first.attributes['href'].value)
         hash[symbol] = link
       end
       hash

data/lib/stock_index/symbol_parser.rb

@@ -6,7 +6,7 @@ class SymbolParser
 
   def symbol_to_bsym
     if @symbol
-      @symbol.sub('.', '/')
+      @symbol.sub('-', '/')
     else
       nil
     end
@@ -20,12 +20,15 @@ class SymbolParser
       # BF/B (Brown-Forman Corp)
       when 'BF/B'
         '0000014693'
+      # QRVO (Qorvo, Inc.)
+      # when 'QRVO'
+      #   '0001604778'
       # QVCA (Liberty Interactive Corp)
-      when 'QVCA'
-        '0001355096'
+      # when 'QVCA'
+      #   '0001355096'
       # LMCK (Liberty Media Corp)
-      when 'LMCK'
-        '0001560385'
+      # when 'LMCK'
+      #   '0001560385'
       else
         @symbol
     end

data/lib/stock_index/version.rb

@@ -1,3 +1,3 @@
 class StockIndex
-  VERSION = "0.8.3"
+  VERSION = "0.8.4"
 end

data/spec/stock_index/bsym_search_spec.rb

@@ -2,14 +2,6 @@ require 'helper'
 
 describe StockIndex::BsymSearch do
 
-  context 'searching a symbol in EXCEPTIONS' do
-
-    it 'returns the component' do
-      expect(StockIndex::BsymSearch.find('BIDU', :us)).to eq({name: 'BAIDU INC', bbgid: 'BBG000QXWHD1'})
-    end
-
-  end
-
   context 'searching a symbol in the CSV file' do
 
     it 'returns the component' do

data/spec/stock_index/component_spec.rb

@@ -10,7 +10,7 @@ describe StockIndex::Component do
 
   it 'returns the correct attributes with a valid symbol' do
     component = StockIndex::Component.new('CSCO', 'XNAS', nil)
-    expect(component.attributes_lookup).to eq({:market=>"XNAS", :symbol=>"CSCO", :name=>"CISCO SYSTEMS INC", :wikipedia=>nil, :cik=>"0000858877", :bbgid=>"BBG000C3J3C9"})
+    expect(component.attributes_lookup).to eq({:market=>"XNAS", :share=>{:symbol=>"CSCO", :name=>"CISCO SYSTEMS INC", :bbgid=>"BBG000C3J3C9"}, :company=>{:wikipedia=>nil, :cik=>"0000858877", :name=>"CISCO SYSTEMS, INC.", :sic=>"3576"}})
   end
 
   it 'returns nil with an invalid symbol' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stock_index
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.8.4
 platform: ruby
 authors:
 - Javier Vidal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-28 00:00:00.000000000 Z
+date: 2015-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -153,14 +153,19 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bsym/.keep
 - bsym/JP.csv
 - bsym/LN.csv
 - bsym/US.csv
+- cache/.keep
 - cache/XJPX.pstore
 - cache/XLON.pstore
 - cache/XNAS.pstore
 - cache/XNYS.pstore
-- cache/test.pstore
+- csv/.keep
+- csv/JP.csv
+- csv/LN.csv
+- csv/US.csv
 - lib/stock_index.rb
 - lib/stock_index/bsym_search.rb
 - lib/stock_index/component.rb