barchart_data 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3a15f991395b2a28f41ed757bb6ec6a4d97c741
-  data.tar.gz: 24d413cdabccc7774f07d106f6fe627b7076a27a
+  metadata.gz: a650e81ec1da0f0464f7ca67e29a4e968ee5d5e8
+  data.tar.gz: d66ab9f7f81445f1e75ff37a7e7eead4b9f34fa1
 SHA512:
-  metadata.gz: 906a39e71e7282231017d9e3cc7690f888e8e2093882933af26d19b6ca2b19db1d63afe965551eeda12e05151bf8cda90483635a20c422df5a6dd1b63b525059
-  data.tar.gz: d2964855e8dc43b894ba169763d073aaa4836e0b59a10e4b36ba0503b97cc67fe3f354ec94120617c54eb33120d575ed6cb45b4fd8c8cab956c03f00da3274f4
+  metadata.gz: 2264434c8ae67a12e6357090d516aaa2c893ccc79e5b1bac69a83d3e5a9b4103a7c949991b7f4379dc95b4659f5cc59e0282319d346916a84e08d1ff3b9bfb64
+  data.tar.gz: f4cb381853ead748af91b79a7c15ea7df4eae021fbe3a77d31badea19c9eab99a2c6800abc7ad64116d0bc43955a999cf0f2b005238df116e9f18366ceda0019

data/.gitignore

@@ -8,3 +8,5 @@
 /spec/reports/
 /tmp/
 /vendor/
+.byebug_history
+test/test_files/vcr_cassettes/

data/README.md

@@ -1,3 +1,19 @@
+### UPDATE:
+Version(0.1.2) released. Modification to schema so will break previous version.  URLs work with old.barchart.com.  Working on updating to current site specifics.
+
+### ALERT:
+
+Recently it has come to my attention that this gem is broken as there has been a major over haul to Barchart.
+
+Links in the following format are now invalid: http://www.barchart.com/stocks/athigh.php?_dtp1-0
+
+Links in the following format are currently active http://old.barchart.com/stocks/athigh.php?_dtp1-0
+
+I'm currently working on cleaning up this gem's codebase for an update with the new urls.
+
+Applies to version **(0.1.1)**
+
+
 # BarchartData
 
 **BarchartData:** Screen Scrape Utility to grab and persist stock releated data from the site Barcharts.com  Current implemenation extracts:
@@ -8,7 +24,7 @@
 *  52-Week Lows
 *  New Highs, New Lows
 
-Version (0.1.1) This project is still in its infancy and will be heavily modified over time.
+Version (0.1.2) This project is still in its infancy and will be heavily modified over time.
 
 ## Installation
 

data/barchart_data.gemspec

@@ -27,11 +27,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.9"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency 'fakeweb', '~> 1.3.0'
-  spec.add_development_dependency 'sqlite3', '~> 1.3.10'
-  spec.add_dependency             'activerecord', '~> 4.2.2'
-  spec.add_dependency 'mechanize', '~> 2.7.3'
-  spec.add_dependency 'nokogiri', '~> 1.6.6.2'
+  spec.add_development_dependency "bundler", "~> 1.12.4"
+  spec.add_development_dependency "rake", "~> 11.3.0"
+  spec.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.3'
+  spec.add_development_dependency 'sqlite3', '~> 1.3.11'
+  spec.add_development_dependency 'minitest-coverage', '~> 1.0.0.b1'
+  spec.add_development_dependency 'byebug', '~> 9.0', '>= 9.0.6'
+  spec.add_development_dependency 'webmock', '~> 2.0', '>= 2.0.3'
+  spec.add_dependency             'activerecord', '~> 5.0.1'
+  spec.add_dependency             'mechanize', '~> 2.7.5'
+  spec.add_dependency             'nokogiri', '~> 1.6.8.1'
 end

data/bin/barchart_data

@@ -2,14 +2,37 @@
 
 require "bundler/setup"
 require "barchart_data"
-require_relative '../lib/barchart_data/scraper'
-require_relative '../lib/barchart_data/high_low_scraper'
+require_relative '../lib/barchart_scripts/barchart_connection'
+require_relative '../lib/barchart_scripts/scraper'
+require_relative '../lib/barchart_scripts/scrape_new_highs_new_lows'
 
+BARCHART_URLS = { AllTimeHigh: 'http://old.barchart.com/stocks/athigh.php?_dtp1=0',
+                  AllTimeLow: 'http://old.barchart.com/stocks/atlow.php?_dtp1=0',
+                  NewHigh: 'http://old.barchart.com/stocks/high.php?_dtp1=0',
+                  NewLow: 'http://old.barchart.com/stocks/low.php?_dtp1=0' }
+BARCHART_HIGH_LOWS_URL = 'https://www.barchart.com/stocks/highs-lows/summary'
+
+connection = BarchartData::BarchartConnection.new
 scraper = BarchartData::Scraper.new
-scraper.data_extraction
+high_low = BarchartData::ScrapeNewHighsNewLows.new
 
+BARCHART_URLS.each do |symbol, url|
+  page = connection.fetch_page url
+  data = scraper.extract_symbols_from_page page
+  scraper.insert_data data, symbol
+end
 
-hl_scraper = BarchartData::HighLowScraper.new
-links = hl_scraper.data_extraction
-values = hl_scraper.strip_highs_and_lows_from links
-hl_scraper.insert_data values
+page = connection.fetch_page BARCHART_HIGH_LOWS_URL
+highs = high_low.extract_table_data_with_class_even page
+lows = high_low.extract_table_data_with_class_odd page
+high_values = high_low.extract_values highs
+low_values = high_low.extract_values lows
+high_fields = high_low.convert_stringy_numbers_to_int high_values
+low_fields = high_low.convert_stringy_numbers_to_int low_values
+convert_highs = high_low.convert_field_names_to_symbols(high_fields, "high")
+convert_lows = high_low.convert_field_names_to_symbols(low_fields, "low")
+merged = high_low.merge_high_low(convert_highs, convert_lows)
+to_hash = high_low.hash_data_before_insertion merged
+high_low.add_datestamp to_hash
+high_low.insert_data to_hash
+to_hash

data/lib/barchart_data.rb

@@ -1,8 +1,8 @@
 require "barchart_data/version"
-require 'barchart_data/alltimehigh'
-require 'barchart_data/newhigh'
-require 'barchart_data/scraper'
-require 'barchart_data/high_low_scraper'
+
+require 'barchart_scripts/barchart_connection'
+require 'barchart_scripts/scrape_new_highs_new_lows'
+require 'barchart_scripts/scraper'
 module BarchartData
   # Your code goes here...
 end

data/lib/barchart_data/version.rb

@@ -1,3 +1,3 @@
 module BarchartData
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/barchart_scripts/barchart_connection.rb

@@ -0,0 +1,16 @@
+require 'mechanize'
+
+module BarchartData
+  class BarchartConnection
+    attr_accessor :mechanize
+
+    def initialize
+      @mechanize = Mechanize.new
+      @mechanize.user_agent = 'Mac FireFox'
+    end
+
+    def fetch_page(url)
+      @mechanize.get(url)
+    end
+  end
+end

data/lib/barchart_scripts/scrape_new_highs_new_lows.rb

@@ -0,0 +1,76 @@
+require 'active_record'
+module BarchartData
+  class ScrapeNewHighsNewLows
+    def extract_table_data_with_class_even page
+      array = page.search('.even').map { |e| e.text() }
+      array = prune_array_data array
+      array.map! { |elem| elem.split }
+    end
+
+    def extract_table_data_with_class_odd page
+      array = page.search('.odd').map { |e| e.text() }
+      array = prune_array_data array
+      array.map! { |elem| elem.split }
+    end
+
+    def prune_array_data arr
+      ten = arr[10]
+      arr.slice!(5..15)
+      arr << ten
+    end
+
+    def extract_values array
+      temp = []
+      array.each do |elem|
+        elem.each_with_index do |e, i|
+          temp << e if i == 0 || i == 2
+        end
+      end
+      temp
+      # remove_extraneous_elements temp
+    end
+
+    def convert_stringy_numbers_to_int array
+      array.map! { |elem| elem !~ /\D/ ? elem.to_i : elem }
+      array
+    end
+
+    def validate_data_integrity array
+      return false unless array.partition { |v| v.is_a? String }[0].count == 6
+      return false unless array.partition { |v| v.is_a? Integer }[0].count == 6
+      array
+    end
+
+    def convert_field_names_to_symbols array, arg
+      keyed_array = []
+      hash = { "1-Month" => :"one_month_#{arg}", "3-Month" => :"three_month_#{arg}",
+               "6-Month" => :"six_month_#{arg}", "52-Week" => :"twelve_month_#{arg}",
+               "YTD" => :"ytd_#{arg}", "All-Time" => :"all_time_#{arg}" }
+
+      array.each { |elem| hash.include?(elem) ? keyed_array << hash[elem] : keyed_array << elem }
+      keyed_array
+    end
+
+    def merge_high_low highs, lows
+      highs + lows
+    end
+
+    def hash_data_before_insertion array
+      array.each_slice(2).to_h
+    end
+
+    def add_datestamp hash
+      hash[:saved_on] = Time.now.strftime("%Y-%m-%d")
+    end
+
+    def insert_data high_low
+      HighLow.create(high_low)
+    end
+
+    private
+    def remove_extraneous_elements array
+      array.slice!(10,10)
+      array
+    end
+  end
+end

data/lib/barchart_scripts/scraper.rb

@@ -0,0 +1,37 @@
+require 'active_record'
+module BarchartData
+  class Scraper
+    def extract_symbols_from_page page
+      array = page.search('.ds_symbol').map { |e| e.text() }
+      array.map! { |elem| elem.split }
+    end
+
+    def validate_data_integrity dirty_array
+      dirty_array.reduce([]) { |cleaned_array, i|
+        cleaned_array << i if i.to_s == i.to_s.upcase
+        cleaned_array
+      }
+    end
+
+    def insert_data arr, sym
+      case sym
+      when :AllTimeHigh
+        arr.each do |s|
+          AllTimeHigh.create(symbol: s, saved_on: Time.current)
+        end
+      when :AllTimeLow
+        arr.each do |s|
+          AllTimeLow.create(symbol: s, saved_on: Time.current)
+        end
+      when :NewHigh
+        arr.each do |s|
+          NewHigh.create(symbol: s, saved_on: Time.current)
+        end
+      when :NewLow
+        arr.each do |s|
+          NewLow.create(symbol: s, saved_on: Time.current)
+        end
+      end
+    end
+  end
+end

data/lib/generators/barchart_data/install/install_generator.rb

@@ -34,8 +34,9 @@ module BarchartData
       end
 
       def copy_classes
-        copy_file "../../../../barchart_data/scraper.rb", 'lib/barchart_data/scraper.rb'
-        copy_file "../../../../barchart_data/high_low_scraper.rb", 'lib/barchart_data/high_low_scraper.rb'
+        copy_file "../../../../barchart_scripts/barchart_connection.rb", "lib/barchart_scripts/barchart_connection.rb"
+        copy_file "../../../../barchart_scripts/scraper.rb", 'lib/barchart_scripts/scraper.rb'
+        copy_file "../../../../barchart_scripts/scrape_new_highs_new_lows.rb", 'lib/barchart_scripts/scrape_new_highs_new_lows.rb'
         copy_file "../../../../../bin/barchart_data", 'bin/barchart'
       end
     end

data/lib/generators/barchart_data/install/templates/highlow_migration.rb

@@ -11,6 +11,8 @@ class CreateHighLows < ActiveRecord::Migration
     t.integer     :twelve_month_low
     t.integer     :ytd_high
     t.integer     :ytd_low
+    t.integer     :all_time_high
+    t.integer     :all_time_low
     t.datetime    :saved_on
     t.timestamps  null: false
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: barchart_data
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Michael Becco
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-17 00:00:00.000000000 Z
+date: 2017-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,98 +16,158 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: 1.12.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: 1.12.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 11.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 11.3.0
 - !ruby/object:Gem::Dependency
-  name: fakeweb
+  name: vcr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.3
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.10
+        version: 1.3.11
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.11
+- !ruby/object:Gem::Dependency
+  name: minitest-coverage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0.b1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.10
+        version: 1.0.0.b1
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 9.0.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 9.0.6
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.3
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.2
+        version: 5.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.2.2
+        version: 5.0.1
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.3
+        version: 2.7.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.3
+        version: 2.7.5
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.6.2
+        version: 1.6.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.6.2
+        version: 1.6.8.1
 description: "Extract data for all-time-highs, new-highs, new-lows and store.  Generator
   for schema\n                          and Ruby scripts for integration with Rails. "
 email:
@@ -123,16 +183,16 @@ files:
 - README.md
 - Rakefile
 - barchart_data-0.1.0.gem
+- barchart_data-0.1.1.gem
 - barchart_data.gemspec
 - bin/barchart_data
 - bin/console
 - bin/setup
 - lib/barchart_data.rb
-- lib/barchart_data/alltimehigh.rb
-- lib/barchart_data/high_low_scraper.rb
-- lib/barchart_data/newhigh.rb
-- lib/barchart_data/scraper.rb
 - lib/barchart_data/version.rb
+- lib/barchart_scripts/barchart_connection.rb
+- lib/barchart_scripts/scrape_new_highs_new_lows.rb
+- lib/barchart_scripts/scraper.rb
 - lib/generators/barchart_data/install/install_generator.rb
 - lib/generators/barchart_data/install/templates/alltimehigh_migration.rb
 - lib/generators/barchart_data/install/templates/alltimelow_migration.rb
@@ -165,7 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Scrape Data From BarCharts.com and persist in database

data/lib/barchart_data/alltimehigh.rb

@@ -1,30 +0,0 @@
-require 'barchart_data'
-require 'active_record'
-require 'mechanize'
-
-module BarchartData
-  class AllTimeHigh
-    attr_reader :url, :agent
-
-    def initialize
-      @url = "http://www.barchart.com/stocks/athigh.php?_dtp1-0"
-      @agent = Mechanize.new
-    end
-
-    def parse_url
-      @agent.get(@url).search("input")
-    end
-
-    def strip_symbols(page)
-      symbols = page[6].to_s
-      strip_symbols = symbols.scan(/[A-Z]+,[^a-z]+[A-Z]/)
-      tickers = strip_symbols[0].split(',')
-    end
-
-    def insert_symbols(symbols)
-      symbols.each do |s|
-        :AllTimeHigh.create(symbol: s, saved_on: Time.now.strftime("%m/%d/%Y"))
-      end
-    end
-  end
-end

data/lib/barchart_data/high_low_scraper.rb

@@ -1,56 +0,0 @@
-require 'barchart_data'
-require 'active_record'
-require 'mechanize'
-
-module BarchartData
-  class HighLowScraper
-    attr_reader :url, :agent
-
-    def initialize
-      @url = { high_low: 'http://www.barchart.com/stocks/newhilo.php?dwm=d'}
-      @agent = Mechanize.new
-    end
-
-    def data_extraction
-      page = @agent.get(@url[:high_low]).search("div[id='divContent']")
-      page.css("td[align='right']").to_a
-    end
-
-    def strip_highs_and_lows_from links
-      strip_links = []
-      regex_the_links = []
-      overall_high_low = []
-
-      links.each do |a|
-        strip_links.push(a.to_s)
-      end
-
-      strip_links.slice!(0..3)
-      strip_links.shift(6)
-      strip_links.each { |e| regex_the_links.push(e.match(/>\d+</).to_s) }
-       regex_the_links.map! do |e|
-        if e.blank?
-          e = ">0<"
-        else
-          e = e
-        end
-      end
-
-      regex_the_links.map! { |e| e.gsub(/[><]/,"") }
-      extracted_values = regex_the_links.each_slice(10).to_a
-      extracted_values.each do |value|
-        overall_high_low << value.first.to_i
-      end
-      overall_high_low
-    end
-
-    def insert_data high_low
-      ::HighLow.create(one_month_high: high_low[0], one_month_low: high_low[1],
-                     three_month_high: high_low[2], three_month_low: high_low[3],
-                     six_month_high: high_low[4], six_month_low: high_low[5],
-                     twelve_month_high: high_low[6], twelve_month_low: high_low[7],
-                     ytd_high: high_low[8], ytd_low: high_low[8],
-                     saved_on: Time.current )
-    end
-  end
-end

data/lib/barchart_data/newhigh.rb

@@ -1,30 +0,0 @@
-require 'barchart_data'
-require 'active_record'
-require 'mechanize'
-
-module BarchartData
-  class NewHigh
-    attr_reader :url, :agent
-
-    def initialize
-      @url = 'http://www.barchart.com/stocks/high.php'
-      @agent = Mechanize.new
-    end
-
-     def parse_url
-      @agent.get(@url).search("input")
-    end
-
-    def strip_symbols(page)
-      symbols = page[6].to_s
-      strip_symbols = symbols.scan(/[A-Z]+,[^a-z]+[A-Z]/)
-      tickers = strip_symbols[0].split(',')
-    end
-
-    def insert_symbols(symbols)
-      symbols.each do |s|
-        :NewHigh.create(symbol: s, saved_on: Time.now.strftime("%m/%d/%Y"))
-      end
-    end
-  end
-end

data/lib/barchart_data/scraper.rb

@@ -1,54 +0,0 @@
-require 'barchart_data'
-require 'active_record'
-require 'mechanize'
-
-module BarchartData
-  class Scraper
-    attr_reader :urls, :agent
-
-    def initialize
-      @urls = {
-        all_time_high: 'http://www.barchart.com/stocks/athigh.php?_dtp1-0',
-        new_high: 'http://www.barchart.com/stocks/high.php?_dtp1-0',
-        new_low: 'http://www.barchart.com/stocks/low.php?_dtp1=0',
-        all_time_low: 'http://www.barchart.com/stocks/atlow.php?_dtp1=0'
-      }
-      @agent = Mechanize.new
-    end
-
-    def data_extraction
-      @urls.each do |sym, url|
-        parse_url = @agent.get("#{url}").search("input")
-        symbols = self.strip_symbols(parse_url)
-        self.insert_symbols(sym, symbols)
-      end
-    end
-
-    def strip_symbols page
-      symbols = page[6].to_s
-      strip_symbols = symbols.scan(/[A-Z]+,[^a-z]+[A-Z]/)
-      tickers = strip_symbols[0].split(',')
-    end
-
-    def insert_symbols sym, symbols
-      case sym
-      when :all_time_high
-        symbols.each do |s|
-          ::AllTimeHigh.create(symbol: s, saved_on: Time.current)
-        end
-      when :new_high
-        symbols.each do |s|
-          ::NewHigh.create(symbol: s, saved_on: Time.current)
-        end
-      when :new_low
-        symbols.each do |s|
-          ::NewLow.create(symbol: s, saved_on: Time.current)
-        end
-      when :all_time_low
-        symbols.each do |s|
-          ::AllTimeLow.create(symbol: s, saved_on: Time.current)
-        end
-      end
-    end
-  end
-end