stock_index 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8e6777026b7003122d7f84f12773bee96fe7a071
-  data.tar.gz: e8733b6a09574b44fe59cfc1e4e34485152679ab
+  metadata.gz: 0c587529847bd7aec40fb68c34143c97da5bc63b
+  data.tar.gz: 391c669017f278afacda495baf32cd4c61d37603
 SHA512:
-  metadata.gz: 771b231cabaea517c04ff4df5bfd45c09ce0be016d8bc12685513e042fd128e5c4d631c416d6e8faef8bc70b09bf3fd7f2028ee3f3baedd92b0e8020a5f09e74
-  data.tar.gz: eeaeece0e07c5f074950a3cfaf317bb645769eda7f7e9e71777202ec91752336400f3cf709b659da15da60f229c190c23806f6850ec7ee5db7b32d6e47991a55
+  metadata.gz: 397bc3ba4835c9302f76917d877dfa9217e949f5e178a594cdd3b3a81deb24d6198533214de25cf75a88315071f63de39e183053728544a6e2cda4e82e7da7f4
+  data.tar.gz: 74b3386e6fe3d2815fffad3a7df947140bba9826f39ceea7a4fdd511143cb4646609115b81ab9e2a7da098fe87b9e09532a6a36e2040fe0c24477d0d36a44a2b

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/README.md

@@ -25,7 +25,7 @@ There is no a standard for indices symbols, so this gem uses Yahoo Finance symbo
 | ------ | ---------- | -------------- |
 | ^DJI | DOW JONES INDUSTRIAL AVERAGE | http://en.wikipedia.org/wiki/Dow_Jones_Industrial_Average |
 | ^GSPC | S&P 500 | http://en.wikipedia.org/wiki/List_of_S%26P_500_companies |
-| ^NDX | NASDAQ 100 | https://indexes.nasdaqomx.com/Index/Weighting/NDX |
+| ^NDX | NASDAQ 100 | http://en.wikipedia.org/wiki/NASDAQ-100 |
 | ^N225 | NIKKEI 225 | http://indexes.nikkei.co.jp/en/nkave/index/component?idx=nk225 |
 | ^STOXX50E | EURO STOXX 50 | http://www.stoxx.com/indices/index_information.html?symbol=SX5E |
 | ^FTSE | FTSE 100 | http://www.londonstockexchange.com/exchange/prices-and-markets/stocks/indices/summary/summary-indices-constituents.html?index=UKX |

data/{csv → bsym}/US.csv

@@ -19179,7 +19179,6 @@ HCP,HCP INC,BBG000BKYDP9
 HCN,HEALTH CARE REIT INC,BBG000BKY1G5
 HR,HEALTHCARE REALTY TRUST INC,BBG000BB67F4
 HTA,HEALTHCARE TRUST OF AME-CL A,BBG000PRJCX9
-HREIF,HEALTHLEASE PROPERTIES REAL,BBG0035WSDV4
 HEKRF,HEKTAR REAL ESTATE INVESTMEN,BBG0024571R0
 HT,HERSHA HOSPITALITY TRUST,BBG000BBB8Y5
 HIBRF,HIBERNIA REIT PLC,BBG005ZL7XD9
@@ -19250,6 +19249,7 @@ MAA,MID-AMERICA APARTMENT COMM,BBG000BLMY92
 MSTUF,MILESTONE APARTMENTS REAL ES,BBG00497LRF7
 MRVGF,MIRVAC GROUP,BBG000DS0379
 MNR,MONMOUTH REAL ESTATE INV COR,BBG000BLZD12
+MORE/WD,MONOGRAM RESIDENTIAL TRST-WD,BBG007MKM787
 MORE,MONOGRAM RESIDENTIAL TRUST I,BBG002PWM5J1
 MNARF,MORGUARD NORTH AMERICAN RESI,BBG004T5DWX7
 MGRUF,MORGUARD REAL ESTATE-TR UTS,BBG000G1JPT2

data/lib/stock_index/bsym_search.rb

@@ -5,12 +5,20 @@ class StockIndex
     require 'csv'
 
     FILES = {
-      us: 'csv/US.csv'
+      us: 'bsym/US.csv'
+    }
+
+    EXCEPTIONS = {
+      'BIDU' => {name: 'BAIDU INC', bbgid: 'BBG000QXWHD1'},
+      'LVNTA' => {name: 'LIBERTY VENTURES', bbgid: 'BBG0038K9G41'},
+      'VIP' => {name: 'VIMPELCOM LTD', bbgid: 'BBG000QCW561'},
+      'VOD' => {name: 'VODAFONE GROUP PLC', bbgid: 'BBG000C4R6H6'}
     }
 
     class << self
 
       def find(symbol, pricing_source = :us)
+        return {name: EXCEPTIONS[symbol][:name], bbgid: EXCEPTIONS[symbol][:bbgid]} if EXCEPTIONS.keys.include? symbol
         CSV.foreach(FILES[pricing_source]) do |row|
           return {name: row[1], bbgid: row[2]} if row[0] == symbol
         end

data/lib/stock_index/component.rb

@@ -32,23 +32,18 @@ class StockIndex
     end
 
     def attributes_lookup
-      puts "--> #{@symbol}"
       bsym = StockIndex::BsymSearch.find(@symbol)
-      edgar = Cik.lookup(parse_symbol(@symbol))
-      a = {market: @market, symbol: @symbol, name: bsym[:name], wikipedia: @wikipedia, cik: edgar[:cik], bbgid: bsym[:bbgid]}
-      cache_write(a)
-    end
-
-    def parse_symbol(symbol)
-      case symbol
-        # BRK/B => 0001067983 Berkshire Hathaway Inc
-        # BF/B => 0000014693 Brown-Forman Corp
-        when 'BRK/B'
-          '0001067983'
-        when 'BF/B'
-          '0000014693'
+      if bsym.nil?
+        puts "bsym --> #{@symbol}"
+        return
+      else
+        edgar = Cik.lookup(SymbolParser.new(@symbol).bsym_to_cik)
+        if edgar.nil?
+          puts "cik --> #{@symbol}"
         else
-          symbol
+          a = {market: @market, symbol: @symbol, name: bsym[:name], wikipedia: @wikipedia, cik: edgar[:cik], bbgid: bsym[:bbgid]}
+          cache_write(a)
+        end
       end
     end
 

data/lib/stock_index/indices.rb

@@ -10,7 +10,7 @@ class StockIndex
     },
     '^NDX' => {
       name: 'NASDAQ 100',
-      url: 'https://indexes.nasdaqomx.com/Index/Weighting/NDX'
+      url: 'http://en.wikipedia.org/wiki/NASDAQ-100'
     },
     '^N225' => {
       name: 'NIKKEI 225',

data/lib/stock_index/scrapers/SP500_scraper.rb

@@ -10,7 +10,7 @@ class SP500Scraper < StockIndex::BaseScraper
   def symbol(tr)
     symbol_td = td(tr, 0)
     s = symbol_td ? symbol_td.css('a').first.text : nil
-    parse_symbol(s)
+    SymbolParser.new(s).sp500_to_bsym
   end
 
   def market(tr)
@@ -19,7 +19,7 @@ class SP500Scraper < StockIndex::BaseScraper
     StockIndex::Market.new(parse_market_link(market_link)).to_iso10383
   end
 
-  def wikipedia(tr)
+  def wikipedia_link(tr)
     wikipedia_position(tr, 1)
   end
 
@@ -32,15 +32,4 @@ class SP500Scraper < StockIndex::BaseScraper
     end
   end
 
-  def parse_symbol(symbol)
-    case symbol
-      # BRK.B => BRK/B Berkshire Hathaway Inc
-      # BF.B => BF/B Brown-Forman Corp
-      when /(\w+)\.B/
-        "#{$1}/B"
-      else
-        symbol
-    end
-  end
-
 end

data/lib/stock_index/scrapers/base_scraper.rb

@@ -9,7 +9,7 @@ class StockIndex
         symbol = symbol(tr)
         market = market(tr)
         if symbol && market
-          component = StockIndex::Component.new(symbol, market, wikipedia(tr))
+          component = StockIndex::Component.new(symbol, market, wikipedia_link(tr))
           array << component.attributes if component.valid?
         end
         array
@@ -23,6 +23,10 @@ class StockIndex
     def wikipedia_position(tr, position)
       wikipedia_td = tr.css('td')[position]
       wikipedia_path = wikipedia_td ? wikipedia_td.css('a').first.attributes['href'].value : nil
+      build_wikipedia_link(wikipedia_path)
+    end
+
+    def build_wikipedia_link(wikipedia_path)
       URI::HTTP.build({:host => 'en.wikipedia.org', :path => wikipedia_path}).to_s rescue nil
     end
 

data/lib/stock_index/scrapers/dji_scraper.rb

@@ -18,7 +18,7 @@ class DjiScraper < StockIndex::BaseScraper
     StockIndex::Market.new(market).to_iso10383
   end
 
-  def wikipedia(tr)
+  def wikipedia_link(tr)
     wikipedia_position(tr, 0)
   end
 

data/lib/stock_index/scrapers/nasdaq_scraper.rb

@@ -0,0 +1,25 @@
+class NasdaqScraper < StockIndex::BaseScraper
+
+  def scrape
+    doc = Nokogiri::HTML(open(StockIndex::INDICES['^NDX'][:url]))
+    parse_rows doc.css('ol')[0].css('li')
+  end
+
+  def parse_rows(rows)
+    rows.inject([]) do |array, li|
+      symbol = /.+\((\w+)\)/.match(li)[1]
+      market = 'XNAS'
+      if symbol && market
+        component = StockIndex::Component.new(symbol, market, wikipedia(li))
+        array << component.attributes if component.valid?
+      end
+      array
+    end
+  end
+
+  def wikipedia(li)
+    wikipedia_path = li.css('a')[0].attributes['href'].value
+    build_wikipedia_link(wikipedia_path)
+  end
+
+end

data/lib/stock_index/stock_index.rb

@@ -10,6 +10,8 @@ class StockIndex
         DjiScraper.new.scrape
       when '^GSPC'
         SP500Scraper.new.scrape
+      when '^NDX'
+        NasdaqScraper.new.scrape
       else
         []
     end

data/lib/stock_index/symbol_parser.rb

@@ -0,0 +1,37 @@
+class SymbolParser
+
+  def initialize(symbol)
+    @symbol = symbol
+  end
+
+  def sp500_to_bsym
+    case @symbol
+      # BRK.B => BRK/B (Berkshire Hathaway Inc)
+      # BF.B  => BF/B  (Brown-Forman Corp)
+      when /(\w+)\.B/
+        "#{$1}/B"
+      else
+        @symbol
+    end
+  end
+
+  def bsym_to_cik
+    case @symbol
+      # BRK/B (Berkshire Hathaway Inc)
+      when 'BRK/B'
+        '0001067983'
+      # BF/B (Brown-Forman Corp)
+      when 'BF/B'
+        '0000014693'
+      # QVCA (Liberty Interactive Corp)
+      when 'QVCA'
+        '0001355096'
+      # LMCK (Liberty Media Corp)
+      when 'LMCK'
+        '0001560385'
+      else
+        @symbol
+    end
+  end
+
+end

data/lib/stock_index/version.rb

@@ -1,3 +1,3 @@
 class StockIndex
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

data/lib/stock_index.rb

@@ -5,6 +5,8 @@ require 'stock_index/indices'
 require 'stock_index/market'
 require 'stock_index/component'
 require 'stock_index/bsym_search'
+require 'stock_index/symbol_parser'
 require 'stock_index/scrapers/base_scraper'
 require 'stock_index/scrapers/dji_scraper'
-require 'stock_index/scrapers/SP500_scraper'
+require 'stock_index/scrapers/SP500_scraper'
+require 'stock_index/scrapers/nasdaq_scraper'

data/spec/fixtures/DJI.yaml

@@ -0,0 +1,181 @@
+---
+- :market: XNYS
+  :symbol: MMM
+  :name: 3M CO
+  :wikipedia: http://en.wikipedia.org/wiki/3M
+  :cik: '0000066740'
+  :bbgid: BBG000BP52R2
+- :market: XNYS
+  :symbol: AXP
+  :name: AMERICAN EXPRESS CO
+  :wikipedia: http://en.wikipedia.org/wiki/American_Express_Co
+  :cik: 0000004962
+  :bbgid: BBG000BCQZS4
+- :market: XNYS
+  :symbol: T
+  :name: AT&T INC
+  :wikipedia: http://en.wikipedia.org/wiki/AT%26T_Inc
+  :cik: '0000732717'
+  :bbgid: BBG000BSJK37
+- :market: XNYS
+  :symbol: BA
+  :name: BOEING CO/THE
+  :wikipedia: http://en.wikipedia.org/wiki/Boeing_Company
+  :cik: 0000012927
+  :bbgid: BBG000BCSST7
+- :market: XNYS
+  :symbol: CAT
+  :name: CATERPILLAR INC
+  :wikipedia: http://en.wikipedia.org/wiki/Caterpillar_Inc.
+  :cik: 0000018230
+  :bbgid: BBG000BF0K17
+- :market: XNYS
+  :symbol: CVX
+  :name: CHEVRON CORP
+  :wikipedia: http://en.wikipedia.org/wiki/Chevron_Corp.
+  :cik: 0000093410
+  :bbgid: BBG000K4ND22
+- :market: XNAS
+  :symbol: CSCO
+  :name: CISCO SYSTEMS INC
+  :wikipedia: http://en.wikipedia.org/wiki/Cisco_Systems
+  :cik: 0000858877
+  :bbgid: BBG000C3J3C9
+- :market: XNYS
+  :symbol: KO
+  :name: COCA-COLA CO/THE
+  :wikipedia: http://en.wikipedia.org/wiki/The_Coca_Cola_Company
+  :cik: '0000021344'
+  :bbgid: BBG000BMX289
+- :market: XNYS
+  :symbol: DD
+  :name: DU PONT (E.I.) DE NEMOURS
+  :wikipedia: http://en.wikipedia.org/wiki/Du_Pont_(E.I.)
+  :cik: '0000030554'
+  :bbgid: BBG000BH13K9
+- :market: XNYS
+  :symbol: XOM
+  :name: EXXON MOBIL CORP
+  :wikipedia: http://en.wikipedia.org/wiki/Exxon_Mobil_Corp.
+  :cik: 0000034088
+  :bbgid: BBG000GZQ728
+- :market: XNYS
+  :symbol: GE
+  :name: GENERAL ELECTRIC CO
+  :wikipedia: http://en.wikipedia.org/wiki/General_Electric
+  :cik: '0000040545'
+  :bbgid: BBG000BK6MB5
+- :market: XNYS
+  :symbol: GS
+  :name: GOLDMAN SACHS GROUP INC
+  :wikipedia: http://en.wikipedia.org/wiki/Goldman_Sachs_Group
+  :cik: 0000886982
+  :bbgid: BBG000C6CFJ5
+- :market: XNYS
+  :symbol: HD
+  :name: HOME DEPOT INC
+  :wikipedia: http://en.wikipedia.org/wiki/Home_Depot
+  :cik: 0000354950
+  :bbgid: BBG000BKZB36
+- :market: XNAS
+  :symbol: INTC
+  :name: INTEL CORP
+  :wikipedia: http://en.wikipedia.org/wiki/Intel_Corp.
+  :cik: 0000050863
+  :bbgid: BBG000C0G1D1
+- :market: XNYS
+  :symbol: IBM
+  :name: INTL BUSINESS MACHINES CORP
+  :wikipedia: http://en.wikipedia.org/wiki/International_Bus._Machines
+  :cik: '0000051143'
+  :bbgid: BBG000BLNNH6
+- :market: XNYS
+  :symbol: JNJ
+  :name: JOHNSON & JOHNSON
+  :wikipedia: http://en.wikipedia.org/wiki/Johnson_%26_Johnson
+  :cik: '0000200406'
+  :bbgid: BBG000BMHYD1
+- :market: XNYS
+  :symbol: JPM
+  :name: JPMORGAN CHASE & CO
+  :wikipedia: http://en.wikipedia.org/wiki/JPMorgan_Chase_%26_Co.
+  :cik: 0000019617
+  :bbgid: BBG000DMBXR2
+- :market: XNYS
+  :symbol: MCD
+  :name: MCDONALD'S CORP
+  :wikipedia: http://en.wikipedia.org/wiki/McDonald%27s_Corp.
+  :cik: 0000063908
+  :bbgid: BBG000BNSZP1
+- :market: XNYS
+  :symbol: MRK
+  :name: MERCK & CO. INC.
+  :wikipedia: http://en.wikipedia.org/wiki/Merck_%26_Co.
+  :cik: 0000310158
+  :bbgid: BBG000BPD168
+- :market: XNAS
+  :symbol: MSFT
+  :name: MICROSOFT CORP
+  :wikipedia: http://en.wikipedia.org/wiki/Microsoft_Corp.
+  :cik: 0000789019
+  :bbgid: BBG000BPH459
+- :market: XNYS
+  :symbol: NKE
+  :name: NIKE INC -CL B
+  :wikipedia: http://en.wikipedia.org/wiki/NIKE_Inc.
+  :cik: 0000320187
+  :bbgid: BBG000C5HS04
+- :market: XNYS
+  :symbol: PFE
+  :name: PFIZER INC
+  :wikipedia: http://en.wikipedia.org/wiki/Pfizer_Inc.
+  :cik: 0000078003
+  :bbgid: BBG000BR2B91
+- :market: XNYS
+  :symbol: PG
+  :name: PROCTER & GAMBLE CO/THE
+  :wikipedia: http://en.wikipedia.org/wiki/Procter_%26_Gamble
+  :cik: 0000080424
+  :bbgid: BBG000BR2TH3
+- :market: XNYS
+  :symbol: TRV
+  :name: TRAVELERS COS INC/THE
+  :wikipedia: http://en.wikipedia.org/wiki/The_Travelers_Companies_Inc.
+  :cik: 0000086312
+  :bbgid: BBG000BJ81C1
+- :market: XNYS
+  :symbol: UNH
+  :name: UNITEDHEALTH GROUP INC
+  :wikipedia: http://en.wikipedia.org/wiki/United_Health_Group_Inc.
+  :cik: '0000731766'
+  :bbgid: BBG000CH5208
+- :market: XNYS
+  :symbol: UTX
+  :name: UNITED TECHNOLOGIES CORP
+  :wikipedia: http://en.wikipedia.org/wiki/United_Technologies
+  :cik: 0000101829
+  :bbgid: BBG000BW8S60
+- :market: XNYS
+  :symbol: VZ
+  :name: VERIZON COMMUNICATIONS INC
+  :wikipedia: http://en.wikipedia.org/wiki/Verizon_Communications
+  :cik: '0000732712'
+  :bbgid: BBG000HS77T5
+- :market: XNYS
+  :symbol: V
+  :name: VISA INC-CLASS A SHARES
+  :wikipedia: http://en.wikipedia.org/wiki/Visa_Inc.
+  :cik: '0001403161'
+  :bbgid: BBG000PSKYX7
+- :market: XNYS
+  :symbol: WMT
+  :name: WAL-MART STORES INC
+  :wikipedia: http://en.wikipedia.org/wiki/Wal-Mart_Stores
+  :cik: 0000104169
+  :bbgid: BBG000BWXBC2
+- :market: XNYS
+  :symbol: DIS
+  :name: WALT DISNEY CO/THE
+  :wikipedia: http://en.wikipedia.org/wiki/The_Walt_Disney_Company
+  :cik: 0001001039
+  :bbgid: BBG000BH4R78