barchart_data 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed65fd6b74e559dedbe41fb2b7a5a924630e736f
-  data.tar.gz: 5ca2f13c1c54d0e2e7d5b830199269636a281263
+  metadata.gz: d3a15f991395b2a28f41ed757bb6ec6a4d97c741
+  data.tar.gz: 24d413cdabccc7774f07d106f6fe627b7076a27a
 SHA512:
-  metadata.gz: 75fa510e7ad7a18474713f1ed2761646825ffbdb488928d32bfb31f73b3cb317045c9516df91b229a210e48e6e106c229f65195e255a1e633da358f1f7d65756
-  data.tar.gz: 090c479d94d05ffa5053d7999d7fc95e21f72247c4f44065bcb68cc98f832f838b3d009adf4aa1f05e8578beb1a4ee92cd4f5f55d52cacba1601f339f5f5abc2
+  metadata.gz: 906a39e71e7282231017d9e3cc7690f888e8e2093882933af26d19b6ca2b19db1d63afe965551eeda12e05151bf8cda90483635a20c422df5a6dd1b63b525059
+  data.tar.gz: d2964855e8dc43b894ba169763d073aaa4836e0b59a10e4b36ba0503b97cc67fe3f354ec94120617c54eb33120d575ed6cb45b4fd8c8cab956c03f00da3274f4

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+***Version 0.1.0***
+
+Initial release.  Screen scrape All Time Highs.
+
+***Version 0.1.1***
+
+Bug Fix: Stray puts line that broke database insertion.
+
+Modified original templates/schema.rb to alltimehigh_migration.rb for consistency with newly added features
+
+Added scraper features:
+
+*  New Highs
+*  New Lows
+*  All Time Lows
+*  HighLowScraper
+

data/README.md

@@ -1,8 +1,14 @@
 # BarchartData
 
-BarchartData: Screen Scrape Utility to grab and persist stock releated data.
+**BarchartData:** Screen Scrape Utility to grab and persist stock releated data from the site Barcharts.com  Current implemenation extracts:
 
-Current version (0.1.0) extracts All Time Highs.  This project is still in its infancy and will be heavily modified over time.
+*  All-Time-Highs
+*  52-Week Highs
+*  All-Time-Lows
+*  52-Week Lows
+*  New Highs, New Lows
+
+Version (0.1.1) This project is still in its infancy and will be heavily modified over time.
 
 ## Installation
 
@@ -30,25 +36,23 @@ Or install it yourself as:
 
 ***Best Run Time:***  7PM PST (2AM GMT) as data stabalizes.
 
-***Be Kind***  Test against the file test/test_files/athigh.php and limit hits against site.
+***Be Kind***  Test against the files in test/test_files/ and limit live test hits against site.
 
-## Prevent Duplicate Records
-There should only be one record per symbol per date.
+## Models
+To prevent duplication of data, the gem installs 4 models with validations.  Ex:
 
 
 **AllTimeHigh**
 
     class AllTimeHigh < ActiveRecord::Base
+      validates :symbol, :saved_on, presence: true
       validates :symbol, uniqueness: { scope: :saved_on }
     end
 
 
 ## Future Features:
-*  52-Week High
-*  All-Time-Low
-*  52-Week Low
-*  Current New Highs/Lows
-*  Error Handling
+*  Improve Error Handling
+*  Add Scraper for Earnings Related Data
 
 
 ## Development
@@ -66,4 +70,4 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 5. Create a new Pull Request
 
 ## Contact:
-Feel free to contract me with questions, collaborations, features and ideas, refactoring and code improvement etc...
+Feel free to contact me with questions, collaborations, features and ideas, refactoring and code improvement etc...

data/bin/barchart_data

@@ -2,9 +2,14 @@
 
 require "bundler/setup"
 require "barchart_data"
+require_relative '../lib/barchart_data/scraper'
+require_relative '../lib/barchart_data/high_low_scraper'
 
-require_relative '../lib/barchart_data/alltimehigh'
-ath = BarchartData::AllTimeHigh.new
-page = ath.parse_url
-symbols = ath.strip_symbols(page)
-puts symbols
+scraper = BarchartData::Scraper.new
+scraper.data_extraction
+
+
+hl_scraper = BarchartData::HighLowScraper.new
+links = hl_scraper.data_extraction
+values = hl_scraper.strip_highs_and_lows_from links
+hl_scraper.insert_data values

data/lib/barchart_data/high_low_scraper.rb

@@ -0,0 +1,56 @@
+require 'barchart_data'
+require 'active_record'
+require 'mechanize'
+
+module BarchartData
+  class HighLowScraper
+    attr_reader :url, :agent
+
+    def initialize
+      @url = { high_low: 'http://www.barchart.com/stocks/newhilo.php?dwm=d'}
+      @agent = Mechanize.new
+    end
+
+    def data_extraction
+      page = @agent.get(@url[:high_low]).search("div[id='divContent']")
+      page.css("td[align='right']").to_a
+    end
+
+    def strip_highs_and_lows_from links
+      strip_links = []
+      regex_the_links = []
+      overall_high_low = []
+
+      links.each do |a|
+        strip_links.push(a.to_s)
+      end
+
+      strip_links.slice!(0..3)
+      strip_links.shift(6)
+      strip_links.each { |e| regex_the_links.push(e.match(/>\d+</).to_s) }
+       regex_the_links.map! do |e|
+        if e.blank?
+          e = ">0<"
+        else
+          e = e
+        end
+      end
+
+      regex_the_links.map! { |e| e.gsub(/[><]/,"") }
+      extracted_values = regex_the_links.each_slice(10).to_a
+      extracted_values.each do |value|
+        overall_high_low << value.first.to_i
+      end
+      overall_high_low
+    end
+
+    def insert_data high_low
+      ::HighLow.create(one_month_high: high_low[0], one_month_low: high_low[1],
+                     three_month_high: high_low[2], three_month_low: high_low[3],
+                     six_month_high: high_low[4], six_month_low: high_low[5],
+                     twelve_month_high: high_low[6], twelve_month_low: high_low[7],
+                     ytd_high: high_low[8], ytd_low: high_low[8],
+                     saved_on: Time.current )
+    end
+  end
+end

data/lib/barchart_data/newhigh.rb

@@ -0,0 +1,30 @@
+require 'barchart_data'
+require 'active_record'
+require 'mechanize'
+
+module BarchartData
+  class NewHigh
+    attr_reader :url, :agent
+
+    def initialize
+      @url = 'http://www.barchart.com/stocks/high.php'
+      @agent = Mechanize.new
+    end
+
+     def parse_url
+      @agent.get(@url).search("input")
+    end
+
+    def strip_symbols(page)
+      symbols = page[6].to_s
+      strip_symbols = symbols.scan(/[A-Z]+,[^a-z]+[A-Z]/)
+      tickers = strip_symbols[0].split(',')
+    end
+
+    def insert_symbols(symbols)
+      symbols.each do |s|
+        :NewHigh.create(symbol: s, saved_on: Time.now.strftime("%m/%d/%Y"))
+      end
+    end
+  end
+end

data/lib/barchart_data/scraper.rb

@@ -0,0 +1,54 @@
+require 'barchart_data'
+require 'active_record'
+require 'mechanize'
+
+module BarchartData
+  class Scraper
+    attr_reader :urls, :agent
+
+    def initialize
+      @urls = {
+        all_time_high: 'http://www.barchart.com/stocks/athigh.php?_dtp1-0',
+        new_high: 'http://www.barchart.com/stocks/high.php?_dtp1-0',
+        new_low: 'http://www.barchart.com/stocks/low.php?_dtp1=0',
+        all_time_low: 'http://www.barchart.com/stocks/atlow.php?_dtp1=0'
+      }
+      @agent = Mechanize.new
+    end
+
+    def data_extraction
+      @urls.each do |sym, url|
+        parse_url = @agent.get("#{url}").search("input")
+        symbols = self.strip_symbols(parse_url)
+        self.insert_symbols(sym, symbols)
+      end
+    end
+
+    def strip_symbols page
+      symbols = page[6].to_s
+      strip_symbols = symbols.scan(/[A-Z]+,[^a-z]+[A-Z]/)
+      tickers = strip_symbols[0].split(',')
+    end
+
+    def insert_symbols sym, symbols
+      case sym
+      when :all_time_high
+        symbols.each do |s|
+          ::AllTimeHigh.create(symbol: s, saved_on: Time.current)
+        end
+      when :new_high
+        symbols.each do |s|
+          ::NewHigh.create(symbol: s, saved_on: Time.current)
+        end
+      when :new_low
+        symbols.each do |s|
+          ::NewLow.create(symbol: s, saved_on: Time.current)
+        end
+      when :all_time_low
+        symbols.each do |s|
+          ::AllTimeLow.create(symbol: s, saved_on: Time.current)
+        end
+      end
+    end
+  end
+end

data/lib/barchart_data/version.rb

@@ -1,3 +1,3 @@
 module BarchartData
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/barchart_data.rb

@@ -1,5 +1,8 @@
 require "barchart_data/version"
 require 'barchart_data/alltimehigh'
+require 'barchart_data/newhigh'
+require 'barchart_data/scraper'
+require 'barchart_data/high_low_scraper'
 module BarchartData
   # Your code goes here...
 end

data/lib/generators/barchart_data/install/install_generator.rb

@@ -8,10 +8,34 @@ module BarchartData
     class InstallGenerator < Rails::Generators::Base
       source_root File.expand_path('../templates', __FILE__)
 
-      def copy_schema
+      def copy_migrations
         time = Time.now.strftime("%Y%m%d%H%M%S")
-        template "schema.rb", "db/migrate/#{time}_create_all_time_highs.rb"
-        copy_file "../../../../barchart_data/alltimehigh.rb", 'lib/barchart_data/alltimehigh.rb'
+        template "alltimehigh_migration.rb", "db/migrate/#{time}_create_all_time_highs.rb"
+        sleep 2
+        time = Time.now.strftime("%Y%m%d%H%M%S")
+        template 'newhigh_migration.rb', "db/migrate/#{time}_create_new_highs.rb"
+        sleep 1
+        time = Time.now.strftime("%Y%m%d%H%M%S")
+        template 'newlow_migration.rb', "db/migrate/#{time}_create_new_lows.rb"
+        sleep 2
+        time = Time.now.strftime("%Y%m%d%H%M%S")
+        template 'alltimelow_migration.rb', "db/migrate/#{time}_create_all_time_lows.rb"
+        sleep 1
+        time = Time.now.strftime("%Y%m%d%H%M%S")
+        template 'highlow_migration.rb', "db/migrate/#{time}_create_high_lows.rb"
+      end
+
+      def copy_models
+        template 'models/all_time_high.rb', 'app/models/all_time_high.rb'
+        template 'models/all_time_low.rb', 'app/models/all_time_low.rb'
+        template 'models/new_high.rb', 'app/models/new_high.rb'
+        template 'models/new_low.rb', 'app/models/new_low.rb'
+        template 'models/high_low.rb', 'app/models/high_low.rb'
+      end
+
+      def copy_classes
+        copy_file "../../../../barchart_data/scraper.rb", 'lib/barchart_data/scraper.rb'
+        copy_file "../../../../barchart_data/high_low_scraper.rb", 'lib/barchart_data/high_low_scraper.rb'
         copy_file "../../../../../bin/barchart_data", 'bin/barchart'
       end
     end

data/lib/generators/barchart_data/install/templates/alltimelow_migration.rb

@@ -0,0 +1,9 @@
+class CreateAllTimeLows < ActiveRecord::Migration
+  def change
+    create_table :all_time_lows, :force => true do |t|
+      t.string :symbol
+      t.datetime :saved_on
+      t.timestamps null: false
+    end
+  end
+end

data/lib/generators/barchart_data/install/templates/highlow_migration.rb

@@ -0,0 +1,18 @@
+class CreateHighLows < ActiveRecord::Migration
+  def change
+    create_table :high_lows, :force => true do |t|
+    t.integer     :one_month_high
+    t.integer     :one_month_low
+    t.integer     :three_month_high
+    t.integer     :three_month_low
+    t.integer     :six_month_high
+    t.integer     :six_month_low
+    t.integer     :twelve_month_high
+    t.integer     :twelve_month_low
+    t.integer     :ytd_high
+    t.integer     :ytd_low
+    t.datetime    :saved_on
+    t.timestamps  null: false
+    end
+  end
+end

data/lib/generators/barchart_data/install/templates/models/all_time_high.rb

@@ -0,0 +1,4 @@
+class AllTimeHigh < ActiveRecord::Base
+  validates :symbol, :saved_on, presence: true
+  validates :symbol, uniqueness: { scope: :saved_on }
+end

data/lib/generators/barchart_data/install/templates/models/all_time_low.rb

@@ -0,0 +1,4 @@
+class AllTimeLow < ActiveRecord::Base
+  validates :symbol, :saved_on, presence: true
+  validates :symbol, uniqueness: { scope: :saved_on }
+end

data/lib/generators/barchart_data/install/templates/models/high_low.rb

@@ -0,0 +1,6 @@
+class HighLow < ActiveRecord::Base
+  validates :saved_on, presence: true, uniqueness: true
+  validates :one_month_high, :one_month_low, :three_month_high, :three_month_low,
+            :six_month_high, :six_month_low, :twelve_month_high, :twelve_month_low,
+            :ytd_high, :ytd_low, presence: true
+end

data/lib/generators/barchart_data/install/templates/models/new_high.rb

@@ -0,0 +1,4 @@
+class NewHigh < ActiveRecord::Base
+  validates :symbol, :saved_on, presence: true
+  validates :symbol, uniqueness: { scope: :saved_on }
+end

data/lib/generators/barchart_data/install/templates/models/new_low.rb

@@ -0,0 +1,4 @@
+class NewLow < ActiveRecord::Base
+  validates :symbol, :saved_on, presence: true
+  validates :symbol, uniqueness: { scope: :saved_on }
+end

data/lib/generators/barchart_data/install/templates/newhigh_migration.rb

@@ -0,0 +1,9 @@
+class CreateNewHighs < ActiveRecord::Migration
+  def change
+    create_table :new_highs, :force => true do |t|
+      t.string :symbol
+      t.datetime :saved_on
+      t.timestamps null: false
+    end
+  end
+end

data/lib/generators/barchart_data/install/templates/newlow_migration.rb

@@ -0,0 +1,9 @@
+class CreateNewLows < ActiveRecord::Migration
+  def change
+    create_table :new_lows, :force => true do |t|
+      t.string :symbol
+      t.datetime :saved_on
+      t.timestamps null: false
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: barchart_data
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Michael Becco
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-09 00:00:00.000000000 Z
+date: 2015-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -118,18 +118,32 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - README.md
 - Rakefile
+- barchart_data-0.1.0.gem
 - barchart_data.gemspec
 - bin/barchart_data
 - bin/console
 - bin/setup
 - lib/barchart_data.rb
 - lib/barchart_data/alltimehigh.rb
+- lib/barchart_data/high_low_scraper.rb
+- lib/barchart_data/newhigh.rb
+- lib/barchart_data/scraper.rb
 - lib/barchart_data/version.rb
 - lib/generators/barchart_data/install/install_generator.rb
-- lib/generators/barchart_data/install/templates/schema.rb
+- lib/generators/barchart_data/install/templates/alltimehigh_migration.rb
+- lib/generators/barchart_data/install/templates/alltimelow_migration.rb
+- lib/generators/barchart_data/install/templates/highlow_migration.rb
+- lib/generators/barchart_data/install/templates/models/all_time_high.rb
+- lib/generators/barchart_data/install/templates/models/all_time_low.rb
+- lib/generators/barchart_data/install/templates/models/high_low.rb
+- lib/generators/barchart_data/install/templates/models/new_high.rb
+- lib/generators/barchart_data/install/templates/models/new_low.rb
+- lib/generators/barchart_data/install/templates/newhigh_migration.rb
+- lib/generators/barchart_data/install/templates/newlow_migration.rb
 homepage: https://github.com/trendwithin/barchart_data
 licenses:
 - MIT