ytterb 0.2.150.232042 → 0.2.152.223223

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 02eb13fbdb1991d8669972798c16fe63386b090d
-  data.tar.gz: 1e841defef03d14ca558d51317d6923e5ee3cc27
+  metadata.gz: 4de92e40383cd2fc47c6dd00857bc62ea366000e
+  data.tar.gz: cf77d498119558b2cb956adde78d8e0a5541a7d4
 SHA512:
-  metadata.gz: 5422b931c0951403b0ddbf06c638bc41c9f9cb1c83c300ee5a67205e46905c417c85aacebb3b39350373b622dcb8896224a3589fb1cfa5ed445edc3a904aafaf
-  data.tar.gz: 7a3fe34868ced14473633bb87433c9a83bc29e19c9e70e2ab798d637c922796393eb0de2e0775e695e0613f963e001afebba923a27ac50537d5981a84b1559cc
+  metadata.gz: 850e40a9c4b39e5147bd9c985c29203818b2590d207c8cfa3de1ce363b905eb237f52503dcceb8b78259e5db9937255c7c711e0097ed18bbd2bd4d133105b298
+  data.tar.gz: 61ca6fd0035c30f4e9ab1b54574527422b19e19c70782d4d7237887770da122fb25ad0dbc7a391d8e9b3a3f7e0e73f38de8ef75f6f2353038f371e6fc4441511

data/.gitignore

@@ -32,4 +32,4 @@ Gemfile.lock
 html/
 pkg/
 vendor/cache/*.gem
-lib/ytterb/cached_symbol_data/**/*.yaml
+lib/ytterb/.cache/cached_symbol_data/**/*.yaml

data/Rakefile

@@ -37,5 +37,5 @@ task :default => :spec
 
 task :build_stock_cache do 
   require_relative 'lib/ytterb'
-  Ytterb::StockSymbolCacheBuilder.new
+  Ytterb::StockSymbol::CacheBuilder.new
 end

data/lib/ytterb.rb

@@ -1,4 +1,4 @@
 require 'ytterb/version'
-require 'ytterb/yql_client'
-require 'ytterb/stock_symbol_loader'
-require 'ytterb/stock_symbol_cache_builder'
+require 'ytterb/util'
+require 'ytterb/yql'
+require 'ytterb/stock_symbol'

data/lib/ytterb/stock_symbol.rb

@@ -1,32 +1 @@
-module Ytterb
-  class StockSymbol
-
-    def initialize(options={})
-      @options = {}
-      options.reject {|k,v| !k }.each do |k,v|
-        sym_k = k.gsub(/\s+/,"_").downcase.to_sym
-        val = v.strip
-        val.gsub!(/\^/,"-") if sym_k == :symbol
-        @options[sym_k] = val
-      end
-    end
-
-    def attributes
-      return @options.keys
-    end
-
-    def method_missing(meth, *args, &block)
-      if @options.has_key?(meth)
-        return @options[meth]
-      else
-        # don't know how to handle it
-        super
-      end
-    end
-
-    def to_s
-      @options.to_s
-    end
-
-  end # StockSymbol
-end # Ytterb
+require_relative 'stock_symbol/_req'

data/lib/ytterb/stock_symbol/_req.rb

@@ -0,0 +1,2 @@
+path = File.expand_path(File.dirname(__FILE__))
+Dir.glob(File.join(path,"**","*.rb")).each {|f| require f }

data/lib/ytterb/stock_symbol/cache_builder.rb

@@ -0,0 +1,82 @@
+require 'thread'
+require 'date'
+
+require_relative '../util'
+require_relative '../yql'
+require_relative 'market_loader'
+
+module Ytterb
+  module StockSymbol
+    class CacheBuilder
+
+      def initialize
+        @yql = Yql::Client.new
+        @sml = MarketLoader.new      
+        @local_settings = Util::Settings.new
+
+        # build the queue used when syncing
+        build_stock_sync_queue
+
+        # run the processor that fetches and persists stock info
+        stock_processor_run
+
+        @local_settings.save
+      end
+
+      def build_stock_sync_queue
+        @stock_sync_queue = Queue.new
+        @sml.stock_symbols.shuffle.each do |stock|
+          @stock_sync_queue << stock.symbol
+        end
+      end
+
+      def stock_processor_run
+        while true
+          begin
+            curr = @stock_sync_queue.pop(true) rescue nil
+            break unless curr # queue is empty
+            sleep(3600.0/@local_settings[:api_calls_per_hour])
+            puts "#{@stock_sync_queue.length} symbols in the queue"
+            puts "Processing Symbol: #{curr}"
+            curr_file = @local_settings.get_symbol_store_file(curr)
+            curr_stock_info = Util::DataPersistHelper.load(curr_file) rescue nil
+            curr_stock_info ||= {}
+            curr_stock_info[:sync_to] ||= @local_settings[:sync_from]
+            date_start = Date.parse(curr_stock_info[:sync_to])
+            if (date_start < Date.today)
+              date_end = date_start + @local_settings[:sync_increment]
+              date_end = Date.today if date_end > Date.today
+              result = @yql.get_symbol_historical(curr,date_start.to_s, date_end.to_s)
+              max_end_date = Date.parse(curr_stock_info[:sync_to])
+              symbol_updated = false
+              if result and result.has_key?("quote") and result["quote"].kind_of?(Array)
+                # TODO: need to pull all this logic into the yql client
+                # TODO: this needs some tests!
+                result["quote"].each do |item|
+                  if item["Date"]
+                    curr_stock_info[:data]||={}
+                    curr_stock_info[:data][item["Date"]] = item
+                    if Date.parse(item["Date"]) > max_end_date
+                      max_end_date = Date.parse(item["Date"])
+                      curr_stock_info[:sync_to] = max_end_date.to_s
+                      symbol_updated = true
+                    end
+                  end
+                end
+                @stock_sync_queue << curr if symbol_updated
+                puts curr_stock_info[:sync_to]
+                Util::DataPersistHelper.save(curr_file,curr_stock_info)
+              else
+                puts "Debug result: #{result}"
+              end
+            end
+          rescue StandardError => e
+            puts "error in stock processor run: #{e.message} : #{e.backtrace.join(" | ")}"
+          end
+        end
+      end
+
+
+    end # CacheBuilder
+  end 
+end

data/lib/ytterb/stock_symbol/market_loader.rb

@@ -0,0 +1,42 @@
+require_relative '../util'
+require_relative 'stock'
+
+module Ytterb
+  module StockSymbol
+    class MarketLoader
+      def initialize
+        @stock_symbols = []
+        path = Util::Settings.new.get_raw_symbol_list_data_dir
+        Dir.entries(path).select {|f| !File.directory?(f) and /companylist_[a-zA-Z]+\.csv/ =~ f }.each do |file_to_process|
+          market = /companylist_(?<market>[a-zA-Z]+)\.csv/.match(file_to_process)[:market]
+          Stock.builder_from_csv(File.join(path,file_to_process),market) do |stock_symbol|
+            @stock_symbols << stock_symbol
+          end
+        end
+      end
+
+      def industries
+        return @industries if @industries
+        @raw_industries = {}
+        @stock_symbols.each do |stock_symbol|
+          @raw_industries[stock_symbol.industry] = 1
+        end
+        @industries = @raw_industries.keys
+      end
+
+      def sectors
+        return @sectors if @sectors
+        @raw_sectors = {}
+        @stock_symbols.each do |stock_symbol|
+          @raw_sectors[stock_symbol.sector] = 1
+        end
+        @sectors = @raw_sectors.keys
+      end
+
+      def stock_symbols
+        @stock_symbols
+      end
+    end
+  end
+end
+

data/lib/ytterb/stock_symbol/stock.rb

@@ -0,0 +1,42 @@
+require 'csv'
+
+module Ytterb
+  module StockSymbol
+    class Stock
+
+      def initialize(options={})
+        @options = {}
+        options.reject {|k,v| !k }.each do |k,v|
+          sym_k = k.gsub(/\s+/,"_").downcase.to_sym
+          val = v.strip
+          val.gsub!(/\^/,"-") if sym_k == :symbol
+          @options[sym_k] = val
+        end
+      end
+
+      def attributes
+        return @options.keys
+      end
+
+      def method_missing(meth, *args, &block)
+        if @options.has_key?(meth)
+          return @options[meth]
+        else
+          # don't know how to handle it
+          super
+        end
+      end
+
+      def to_s
+        @options.to_s
+      end
+      
+      def self.builder_from_csv(stock_symbol_file, exchange)
+        CSV.foreach(stock_symbol_file, :headers => true) do |obj|
+          yield Stock.new(obj.to_hash.merge("Exchange" => exchange))
+        end
+      end
+
+    end
+  end
+end # Ytterb

data/lib/ytterb/util.rb

@@ -0,0 +1 @@
+require_relative 'util/_req'

data/lib/ytterb/util/_req.rb

@@ -0,0 +1,2 @@
+path = File.expand_path(File.dirname(__FILE__))
+Dir.glob(File.join(path,"**","*.rb")).each {|f| require f }

data/lib/ytterb/util/data_persist_helper.rb

@@ -0,0 +1,19 @@
+require 'yaml'
+
+module Ytterb
+  module Util
+    class DataPersistHelper
+      def self.save(file, value)
+        File.open(file,"w") {|f| f.write(YAML.dump(value)) }
+      end
+
+      def self.load(file)
+        File.open(file,"r") {|f| YAML.load(f.read()) }
+      end
+
+      def self.get_extension
+        return ".yaml"
+      end
+    end
+  end
+end

data/lib/ytterb/util/settings.rb

@@ -0,0 +1,80 @@
+require_relative 'data_persist_helper'
+
+module Ytterb
+  module Util
+    class Settings
+      def initialize
+        @cache_dir = ".cache"
+        @options = {
+          :settings_file => File.join("settings",DataPersistHelper.get_extension()),
+          :settings_data_dir => [@cache_dir, "local_settings"],
+          :cached_symbol_data_dir => [@cache_dir, "cached_symbol_data"],
+          :raw_symbol_list_data_dir => [@cache_dir, "raw_symbol_data"]
+        }.freeze
+        @local_path =  File.expand_path(File.dirname(__FILE__)).
+          split(File::SEPARATOR).
+          reverse.drop(1).reverse.
+          join(File::SEPARATOR)
+
+        # create the cache dirs if they don's exist
+        @options.each do |option, value|
+          options_s = option.to_s
+          next unless options_s.end_with?("data_dir")
+          FileUtils.mkdir_p(File.join(@local_path, @options[option]))
+        end
+        load
+      end
+
+      def set_defaults_if_uninitialized
+        # defaults to be used if not defined
+        @local_settings ||= {}
+        @local_settings[:sync_from] ||= (Date.today-365).to_s
+        @local_settings[:sync_increment] ||= 120
+        @local_settings[:api_calls_per_hour] ||= 1900
+      end
+
+      def create_full_path_and_file(full_path) 
+        FileUtils.mkdir_p(File.dirname(full_path))
+        unless File.file?(full_path)
+          FileUtils.touch(full_path)
+        end
+      end
+
+      def full_path_settings_file
+        File.join(@local_path,@options[:settings_data_dir],@options[:settings_file])
+      end
+
+      def get_symbol_store_file(symbol, build_path_and_file = true)
+        split_for_storage = symbol.split('')
+        split_for_storage[-1] += DataPersistHelper.get_extension()
+        full_path = File.join(@local_path,@options[:cached_symbol_data_dir],split_for_storage)
+        create_full_path_and_file(full_path)  if build_path_and_file == true
+        full_path
+      end
+
+      def get_raw_symbol_list_data_dir
+        File.join(@local_path,@options[:raw_symbol_list_data_dir])
+      end
+
+      def [](key)
+        @local_settings[key]
+      end
+
+      def []=(key, value)
+        @local_settings[key] = value
+      end
+
+      def load
+        @local_settings = DataPersistHelper.load(full_path_settings_file())
+      rescue
+      ensure
+        set_defaults_if_uninitialized
+      end
+
+      def save
+        create_full_path_and_file(full_path_settings_file())
+        DataPersistHelper.save(full_path_settings_file(), @local_settings)
+      end
+    end
+  end
+end

data/lib/ytterb/yql.rb

@@ -0,0 +1 @@
+require_relative 'yql/_req'

data/lib/ytterb/yql/_req.rb

@@ -0,0 +1,2 @@
+path = File.expand_path(File.dirname(__FILE__))
+Dir.glob(File.join(path,"**","*.rb")).each {|f| require f }

data/lib/ytterb/yql/client.rb

@@ -0,0 +1,40 @@
+require 'rest-client'
+require 'cgi'
+require 'json'
+require 'date'
+
+module Ytterb
+  module Yql
+    class Client
+      def initialize
+        @endpoint = "http://query.yahooapis.com/v1/public/yql"
+        puts "YQL Client is using endpoint @ #{@endpoint}"
+      end
+
+      def get_yql_endpoint
+        @endpoint
+      end
+
+      def run_select_query(query)
+        RestClient.get("#{get_yql_endpoint}?q=#{CGI::escape(query)}&format=json&env=store://datatables.org/alltableswithkeys") do |response, request, result|
+          raise "Failed performing YQLQuery\n Request: #{request.inspect}\n Response #{response.inspect}" unless response.code == 200
+          return JSON.parse(response)
+        end
+      end
+
+      def get_symbol_current(my_symbol)
+        run_select_query("select * from yahoo.finance.quotes where symbol in (\"#{my_symbol}\")")["query"]["results"]
+      end
+
+      def get_symbol_historical(my_symbol, start_date = nil, end_date = nil)
+        requested_end = Date.parse(end_date) if end_date
+        requested_end = Date.today unless requested_end
+
+        requested_start = Date.parse(start_date) if start_date
+        requested_start = requested_end - 30 unless requested_start
+        run_select_query("select * from yahoo.finance.historicaldata where symbol in (\"#{my_symbol}\") and startDate='#{requested_start}' and endDate='#{requested_end}'")["query"]["results"]
+      end
+
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ytterb
 version: !ruby/object:Gem::Version
-  version: 0.2.150.232042
+  version: 0.2.152.223223
 platform: ruby
 authors:
 - thext
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-31 00:00:00.000000000 Z
+date: 2014-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,20 +94,25 @@ files:
 - README.md
 - Rakefile
 - lib/ytterb.rb
-- lib/ytterb/cached_symbol_data/.keepme
-- lib/ytterb/cached_symbol_data/clean.sh
-- lib/ytterb/data_persist_helper.rb
-- lib/ytterb/local_settings/settings.yaml
-- lib/ytterb/raw_symbol_data/companylist_amex.csv
-- lib/ytterb/raw_symbol_data/companylist_nasdaq.csv
-- lib/ytterb/raw_symbol_data/companylist_nyse.csv
-- lib/ytterb/settings.rb
-- lib/ytterb/stock_market_loader.rb
+- lib/ytterb/.cache/cached_symbol_data/.keepme
+- lib/ytterb/.cache/cached_symbol_data/clean.sh
+- lib/ytterb/.cache/local_settings/settings.yaml
+- lib/ytterb/.cache/raw_symbol_data/companylist_amex.csv
+- lib/ytterb/.cache/raw_symbol_data/companylist_nasdaq.csv
+- lib/ytterb/.cache/raw_symbol_data/companylist_nyse.csv
 - lib/ytterb/stock_symbol.rb
-- lib/ytterb/stock_symbol_cache_builder.rb
-- lib/ytterb/stock_symbol_loader.rb
+- lib/ytterb/stock_symbol/_req.rb
+- lib/ytterb/stock_symbol/cache_builder.rb
+- lib/ytterb/stock_symbol/market_loader.rb
+- lib/ytterb/stock_symbol/stock.rb
+- lib/ytterb/util.rb
+- lib/ytterb/util/_req.rb
+- lib/ytterb/util/data_persist_helper.rb
+- lib/ytterb/util/settings.rb
 - lib/ytterb/version.rb
-- lib/ytterb/yql_client.rb
+- lib/ytterb/yql.rb
+- lib/ytterb/yql/_req.rb
+- lib/ytterb/yql/client.rb
 - spec/spec_helper.rb
 - spec/ytterb_spec.rb
 - ytterb.gemspec

data/lib/ytterb/data_persist_helper.rb

@@ -1,17 +0,0 @@
-require 'yaml'
-
-module Ytterb
-  class DataPersistHelper
-    def self.save(file, value)
-      File.open(file,"w") {|f| f.write(YAML.dump(value)) }
-    end
-
-    def self.load(file)
-      File.open(file,"r") {|f| YAML.load(f.read()) }
-    end
-
-    def self.get_extension
-      return ".yaml"
-    end
-  end
-end

data/lib/ytterb/settings.rb

@@ -1,62 +0,0 @@
-require_relative 'data_persist_helper'
-
-module Ytterb
-  class Settings
-    def initialize
-      @options = {
-        :settings_dir => "local_settings",
-        :settings_file => File.join("settings",DataPersistHelper.get_extension()),
-        :cached_symbol_data_dir => "cached_symbol_data"
-      }.freeze
-      @local_path = File.expand_path(File.dirname(__FILE__))
-      load
-    end
-    
-    def set_defaults_if_uninitialized
-      # defaults to be used if not defined
-      @local_settings ||= {}
-      @local_settings[:sync_from] ||= (Date.today-365).to_s
-      @local_settings[:sync_increment] ||= 120
-      @local_settings[:api_calls_per_hour] ||= 1900
-    end
-    
-    def create_full_path_and_file(full_path) 
-      FileUtils.mkdir_p(File.dirname(full_path))
-      unless File.file?(full_path)
-        FileUtils.touch(full_path)
-      end
-    end
-    
-    def full_path_settings_file
-      File.join(@local_path,@options[:settings_dir],@options[:settings_file])
-    end
-    
-    def get_symbol_store_file(symbol, build_path_and_file = true)
-      split_for_storage = symbol.split('')
-      split_for_storage[-1] += DataPersistHelper.get_extension()
-      full_path = File.join(@local_path,@options[:cached_symbol_data_dir],split_for_storage)
-      create_full_path_and_file(full_path)  if build_path_and_file == true
-      full_path
-    end
-    
-    def [](key)
-      @local_settings[key]
-    end
-    
-    def []=(key, value)
-      @local_settings[key] = value
-    end
-    
-    def load
-      @local_settings = DataPersistHelper.load(full_path_settings_file())
-    rescue
-    ensure
-      set_defaults_if_uninitialized
-    end
-
-    def save
-      create_full_path_and_file(full_path_settings_file())
-      DataPersistHelper.load(full_path_settings_file(), @local_settings)
-    end
-  end
-end

data/lib/ytterb/stock_market_loader.rb

@@ -1,41 +0,0 @@
-require_relative 'stock_symbol_loader'
-
-module Ytterb
-
-  class StockMarketLoader
-    def initialize
-      @stock_symbols = []
-      path = File.join(File.expand_path(File.dirname(__FILE__)),"raw_symbol_data")
-      Dir.entries(path).select {|f| !File.directory?(f) and /companylist_[a-zA-Z]+\.csv/ =~ f }.each do |file_to_process|
-        market = /companylist_(?<market>[a-zA-Z]+)\.csv/.match(file_to_process)[:market]
-        StockSymbolLoader.new(File.join(path,file_to_process),market).parse do |stock_symbol|
-          @stock_symbols << stock_symbol
-        end
-      end
-    end
-
-    def industries
-      return @industries if @industries
-      @raw_industries = {}
-      @stock_symbols.each do |stock_symbol|
-        @raw_industries[stock_symbol.industry] = 1
-      end
-      @industries = @raw_industries.keys
-    end
-
-    def sectors
-      return @sectors if @sectors
-      @raw_sectors = {}
-      @stock_symbols.each do |stock_symbol|
-        @raw_sectors[stock_symbol.sector] = 1
-      end
-      @sectors = @raw_sectors.keys
-    end
-
-    def stock_symbols
-      @stock_symbols
-    end
-  end
-
-end
-

data/lib/ytterb/stock_symbol_cache_builder.rb

@@ -1,72 +0,0 @@
-require 'fileutils'
-require 'thread'
-require 'date'
-require_relative 'data_persist_helper'
-require_relative 'settings'
-require_relative 'stock_market_loader'
-require_relative 'yql_client'
-
-module Ytterb
-
-  class StockSymbolCacheBuilder
-
-    def initialize
-      @yql = YQLClient.new
-      @sml = StockMarketLoader.new      
-      @local_settings = Settings.new
-
-      # build the queue used when syncing
-      build_stock_sync_queue
-
-      # run the processor that fetches and persists stock info
-      stock_processor_run
-
-      @local_settings.save
-    end
-
-    def build_stock_sync_queue
-      @stock_sync_queue = Queue.new
-      @sml.stock_symbols.shuffle.each do |stock|
-        @stock_sync_queue << stock.symbol
-      end
-    end
-
-    def stock_processor_run
-      while true
-        begin
-          curr = @stock_sync_queue.pop(true) rescue nil
-          break unless curr # queue is empty
-          sleep(3600.0/@local_settings[:api_calls_per_hour])
-          puts "#{@stock_sync_queue.length} symbols in the queue"
-          puts "Processing Symbol: #{curr}"
-          curr_file = @local_settings.get_symbol_store_file(curr)
-          curr_stock_info = DataPersistHelper.load(curr_file) rescue nil
-          curr_stock_info ||= {}
-          curr_stock_info[:sync_to] ||= @local_settings[:sync_from]
-          date_start = Date.parse(curr_stock_info[:sync_to])
-          if (date_start < Date.today)
-            date_end = date_start + @local_settings[:sync_increment]
-            date_end = Date.today if date_end > Date.today
-            result = @yql.get_symbol_historical(curr,date_start.to_s, date_end.to_s)
-            curr_stock_info[:sync_to] = date_end.to_s
-            if result and result.has_key?("quote")
-              result["quote"].each do |item|
-                curr_stock_info[:data]||={}
-                curr_stock_info[:data][item["Date"]] = item
-              end
-              @stock_sync_queue << curr
-              puts curr_stock_info[:sync_to]
-              DataPersistHelper.save(curr_file,curr_stock_info)
-            else
-              puts "Debug result: #{result}"
-            end
-          end
-        rescue StandardError => e
-          puts "error in stock processor run: #{e.message} : #{e.backtrace.join(" | ")}"
-        end
-      end
-    end
-
-
-  end # StockSymbolCacheBuilder
-end

data/lib/ytterb/stock_symbol_loader.rb

@@ -1,19 +0,0 @@
-require 'csv'
-require_relative 'stock_symbol'
-
-module Ytterb
-
-  class StockSymbolLoader
-    def initialize(stock_symbol_file, exchange)
-      @stock_symbol_file = stock_symbol_file
-      @exchange = exchange
-    end
-
-    def parse
-      CSV.foreach(@stock_symbol_file, :headers => true) do |obj|
-        yield StockSymbol.new(obj.to_hash.merge("Exchange" => @exchange))
-      end
-    end
-  end
-
-end

data/lib/ytterb/yql_client.rb

@@ -1,39 +0,0 @@
-require 'rest-client'
-require 'cgi'
-require 'json'
-require 'date'
-
-module Ytterb
-
-  class YQLClient
-    def initialize
-      @endpoint = "http://query.yahooapis.com/v1/public/yql"
-      puts "YQL Client is using endpoint @ #{@endpoint}"
-    end
-
-    def get_yql_endpoint
-      @endpoint
-    end
-
-    def run_select_query(query)
-      RestClient.get("#{get_yql_endpoint}?q=#{CGI::escape(query)}&format=json&env=store://datatables.org/alltableswithkeys") do |response, request, result|
-        raise "Failed performing YQLQuery\n Request: #{request.inspect}\n Response #{response.inspect}" unless response.code == 200
-        return JSON.parse(response)
-      end
-    end
-
-    def get_symbol_current(my_symbol)
-      run_select_query("select * from yahoo.finance.quotes where symbol in (\"#{my_symbol}\")")["query"]["results"]
-    end
-
-    def get_symbol_historical(my_symbol, start_date = nil, end_date = nil)
-      requested_end = Date.parse(end_date) if end_date
-      requested_end = Date.today unless requested_end
-
-      requested_start = Date.parse(start_date) if start_date
-      requested_start = requested_end - 30 unless requested_start
-      run_select_query("select * from yahoo.finance.historicaldata where symbol in (\"#{my_symbol}\") and startDate='#{requested_start}' and endDate='#{requested_end}'")["query"]["results"]
-    end
-
-  end
-end