barchart_data 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +15 -11
- data/barchart_data-0.1.0.gem +0 -0
- data/bin/barchart_data +10 -5
- data/lib/barchart_data/high_low_scraper.rb +56 -0
- data/lib/barchart_data/newhigh.rb +30 -0
- data/lib/barchart_data/scraper.rb +54 -0
- data/lib/barchart_data/version.rb +1 -1
- data/lib/barchart_data.rb +3 -0
- data/lib/generators/barchart_data/install/install_generator.rb +27 -3
- data/lib/generators/barchart_data/install/templates/{schema.rb → alltimehigh_migration.rb} +0 -0
- data/lib/generators/barchart_data/install/templates/alltimelow_migration.rb +9 -0
- data/lib/generators/barchart_data/install/templates/highlow_migration.rb +18 -0
- data/lib/generators/barchart_data/install/templates/models/all_time_high.rb +4 -0
- data/lib/generators/barchart_data/install/templates/models/all_time_low.rb +4 -0
- data/lib/generators/barchart_data/install/templates/models/high_low.rb +6 -0
- data/lib/generators/barchart_data/install/templates/models/new_high.rb +4 -0
- data/lib/generators/barchart_data/install/templates/models/new_low.rb +4 -0
- data/lib/generators/barchart_data/install/templates/newhigh_migration.rb +9 -0
- data/lib/generators/barchart_data/install/templates/newlow_migration.rb +9 -0
- metadata +17 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d3a15f991395b2a28f41ed757bb6ec6a4d97c741
|
4
|
+
data.tar.gz: 24d413cdabccc7774f07d106f6fe627b7076a27a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 906a39e71e7282231017d9e3cc7690f888e8e2093882933af26d19b6ca2b19db1d63afe965551eeda12e05151bf8cda90483635a20c422df5a6dd1b63b525059
|
7
|
+
data.tar.gz: d2964855e8dc43b894ba169763d073aaa4836e0b59a10e4b36ba0503b97cc67fe3f354ec94120617c54eb33120d575ed6cb45b4fd8c8cab956c03f00da3274f4
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
***Version 0.1.0***
|
2
|
+
|
3
|
+
Initial release. Screen scrape All Time Highs.
|
4
|
+
|
5
|
+
***Version 0.1.1***
|
6
|
+
|
7
|
+
Bug Fix: Stray puts line that broke database insertion.
|
8
|
+
|
9
|
+
Modified original templates/schema.rb to alltimehigh_migration.rb for consistency with newly added features
|
10
|
+
|
11
|
+
Added scraper features:
|
12
|
+
|
13
|
+
* New Highs
|
14
|
+
* New Lows
|
15
|
+
* All Time Lows
|
16
|
+
* HighLowScraper
|
17
|
+
|
data/README.md
CHANGED
@@ -1,8 +1,14 @@
|
|
1
1
|
# BarchartData
|
2
2
|
|
3
|
-
BarchartData
|
3
|
+
**BarchartData:** Screen Scrape Utility to grab and persist stock releated data from the site Barcharts.com Current implemenation extracts:
|
4
4
|
|
5
|
-
|
5
|
+
* All-Time-Highs
|
6
|
+
* 52-Week Highs
|
7
|
+
* All-Time-Lows
|
8
|
+
* 52-Week Lows
|
9
|
+
* New Highs, New Lows
|
10
|
+
|
11
|
+
Version (0.1.1) This project is still in its infancy and will be heavily modified over time.
|
6
12
|
|
7
13
|
## Installation
|
8
14
|
|
@@ -30,25 +36,23 @@ Or install it yourself as:
|
|
30
36
|
|
31
37
|
***Best Run Time:*** 7PM PST (2AM GMT) as data stabalizes.
|
32
38
|
|
33
|
-
***Be Kind*** Test against the
|
39
|
+
***Be Kind*** Test against the files in test/test_files/ and limit live test hits against site.
|
34
40
|
|
35
|
-
##
|
36
|
-
|
41
|
+
## Models
|
42
|
+
To prevent duplication of data, the gem installs 4 models with validations. Ex:
|
37
43
|
|
38
44
|
|
39
45
|
**AllTimeHigh**
|
40
46
|
|
41
47
|
class AllTimeHigh < ActiveRecord::Base
|
48
|
+
validates :symbol, :saved_on, presence: true
|
42
49
|
validates :symbol, uniqueness: { scope: :saved_on }
|
43
50
|
end
|
44
51
|
|
45
52
|
|
46
53
|
## Future Features:
|
47
|
-
*
|
48
|
-
*
|
49
|
-
* 52-Week Low
|
50
|
-
* Current New Highs/Lows
|
51
|
-
* Error Handling
|
54
|
+
* Improve Error Handling
|
55
|
+
* Add Scraper for Earnings Related Data
|
52
56
|
|
53
57
|
|
54
58
|
## Development
|
@@ -66,4 +70,4 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
66
70
|
5. Create a new Pull Request
|
67
71
|
|
68
72
|
## Contact:
|
69
|
-
Feel free to
|
73
|
+
Feel free to contact me with questions, collaborations, features and ideas, refactoring and code improvement etc...
|
Binary file
|
data/bin/barchart_data
CHANGED
@@ -2,9 +2,14 @@
|
|
2
2
|
|
3
3
|
require "bundler/setup"
|
4
4
|
require "barchart_data"
|
5
|
+
require_relative '../lib/barchart_data/scraper'
|
6
|
+
require_relative '../lib/barchart_data/high_low_scraper'
|
5
7
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
8
|
+
scraper = BarchartData::Scraper.new
|
9
|
+
scraper.data_extraction
|
10
|
+
|
11
|
+
|
12
|
+
hl_scraper = BarchartData::HighLowScraper.new
|
13
|
+
links = hl_scraper.data_extraction
|
14
|
+
values = hl_scraper.strip_highs_and_lows_from links
|
15
|
+
hl_scraper.insert_data values
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'barchart_data'
|
2
|
+
require 'active_record'
|
3
|
+
require 'mechanize'
|
4
|
+
|
5
|
+
module BarchartData
|
6
|
+
class HighLowScraper
|
7
|
+
attr_reader :url, :agent
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@url = { high_low: 'http://www.barchart.com/stocks/newhilo.php?dwm=d'}
|
11
|
+
@agent = Mechanize.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def data_extraction
|
15
|
+
page = @agent.get(@url[:high_low]).search("div[id='divContent']")
|
16
|
+
page.css("td[align='right']").to_a
|
17
|
+
end
|
18
|
+
|
19
|
+
def strip_highs_and_lows_from links
|
20
|
+
strip_links = []
|
21
|
+
regex_the_links = []
|
22
|
+
overall_high_low = []
|
23
|
+
|
24
|
+
links.each do |a|
|
25
|
+
strip_links.push(a.to_s)
|
26
|
+
end
|
27
|
+
|
28
|
+
strip_links.slice!(0..3)
|
29
|
+
strip_links.shift(6)
|
30
|
+
strip_links.each { |e| regex_the_links.push(e.match(/>\d+</).to_s) }
|
31
|
+
regex_the_links.map! do |e|
|
32
|
+
if e.blank?
|
33
|
+
e = ">0<"
|
34
|
+
else
|
35
|
+
e = e
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
regex_the_links.map! { |e| e.gsub(/[><]/,"") }
|
40
|
+
extracted_values = regex_the_links.each_slice(10).to_a
|
41
|
+
extracted_values.each do |value|
|
42
|
+
overall_high_low << value.first.to_i
|
43
|
+
end
|
44
|
+
overall_high_low
|
45
|
+
end
|
46
|
+
|
47
|
+
def insert_data high_low
|
48
|
+
::HighLow.create(one_month_high: high_low[0], one_month_low: high_low[1],
|
49
|
+
three_month_high: high_low[2], three_month_low: high_low[3],
|
50
|
+
six_month_high: high_low[4], six_month_low: high_low[5],
|
51
|
+
twelve_month_high: high_low[6], twelve_month_low: high_low[7],
|
52
|
+
ytd_high: high_low[8], ytd_low: high_low[8],
|
53
|
+
saved_on: Time.current )
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'barchart_data'
|
2
|
+
require 'active_record'
|
3
|
+
require 'mechanize'
|
4
|
+
|
5
|
+
module BarchartData
|
6
|
+
class NewHigh
|
7
|
+
attr_reader :url, :agent
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@url = 'http://www.barchart.com/stocks/high.php'
|
11
|
+
@agent = Mechanize.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def parse_url
|
15
|
+
@agent.get(@url).search("input")
|
16
|
+
end
|
17
|
+
|
18
|
+
def strip_symbols(page)
|
19
|
+
symbols = page[6].to_s
|
20
|
+
strip_symbols = symbols.scan(/[A-Z]+,[^a-z]+[A-Z]/)
|
21
|
+
tickers = strip_symbols[0].split(',')
|
22
|
+
end
|
23
|
+
|
24
|
+
def insert_symbols(symbols)
|
25
|
+
symbols.each do |s|
|
26
|
+
:NewHigh.create(symbol: s, saved_on: Time.now.strftime("%m/%d/%Y"))
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'barchart_data'
|
2
|
+
require 'active_record'
|
3
|
+
require 'mechanize'
|
4
|
+
|
5
|
+
module BarchartData
|
6
|
+
class Scraper
|
7
|
+
attr_reader :urls, :agent
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@urls = {
|
11
|
+
all_time_high: 'http://www.barchart.com/stocks/athigh.php?_dtp1-0',
|
12
|
+
new_high: 'http://www.barchart.com/stocks/high.php?_dtp1-0',
|
13
|
+
new_low: 'http://www.barchart.com/stocks/low.php?_dtp1=0',
|
14
|
+
all_time_low: 'http://www.barchart.com/stocks/atlow.php?_dtp1=0'
|
15
|
+
}
|
16
|
+
@agent = Mechanize.new
|
17
|
+
end
|
18
|
+
|
19
|
+
def data_extraction
|
20
|
+
@urls.each do |sym, url|
|
21
|
+
parse_url = @agent.get("#{url}").search("input")
|
22
|
+
symbols = self.strip_symbols(parse_url)
|
23
|
+
self.insert_symbols(sym, symbols)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def strip_symbols page
|
28
|
+
symbols = page[6].to_s
|
29
|
+
strip_symbols = symbols.scan(/[A-Z]+,[^a-z]+[A-Z]/)
|
30
|
+
tickers = strip_symbols[0].split(',')
|
31
|
+
end
|
32
|
+
|
33
|
+
def insert_symbols sym, symbols
|
34
|
+
case sym
|
35
|
+
when :all_time_high
|
36
|
+
symbols.each do |s|
|
37
|
+
::AllTimeHigh.create(symbol: s, saved_on: Time.current)
|
38
|
+
end
|
39
|
+
when :new_high
|
40
|
+
symbols.each do |s|
|
41
|
+
::NewHigh.create(symbol: s, saved_on: Time.current)
|
42
|
+
end
|
43
|
+
when :new_low
|
44
|
+
symbols.each do |s|
|
45
|
+
::NewLow.create(symbol: s, saved_on: Time.current)
|
46
|
+
end
|
47
|
+
when :all_time_low
|
48
|
+
symbols.each do |s|
|
49
|
+
::AllTimeLow.create(symbol: s, saved_on: Time.current)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/lib/barchart_data.rb
CHANGED
@@ -8,10 +8,34 @@ module BarchartData
|
|
8
8
|
class InstallGenerator < Rails::Generators::Base
|
9
9
|
source_root File.expand_path('../templates', __FILE__)
|
10
10
|
|
11
|
-
def
|
11
|
+
def copy_migrations
|
12
12
|
time = Time.now.strftime("%Y%m%d%H%M%S")
|
13
|
-
template "
|
14
|
-
|
13
|
+
template "alltimehigh_migration.rb", "db/migrate/#{time}_create_all_time_highs.rb"
|
14
|
+
sleep 2
|
15
|
+
time = Time.now.strftime("%Y%m%d%H%M%S")
|
16
|
+
template 'newhigh_migration.rb', "db/migrate/#{time}_create_new_highs.rb"
|
17
|
+
sleep 1
|
18
|
+
time = Time.now.strftime("%Y%m%d%H%M%S")
|
19
|
+
template 'newlow_migration.rb', "db/migrate/#{time}_create_new_lows.rb"
|
20
|
+
sleep 2
|
21
|
+
time = Time.now.strftime("%Y%m%d%H%M%S")
|
22
|
+
template 'alltimelow_migration.rb', "db/migrate/#{time}_create_all_time_lows.rb"
|
23
|
+
sleep 1
|
24
|
+
time = Time.now.strftime("%Y%m%d%H%M%S")
|
25
|
+
template 'highlow_migration.rb', "db/migrate/#{time}_create_high_lows.rb"
|
26
|
+
end
|
27
|
+
|
28
|
+
def copy_models
|
29
|
+
template 'models/all_time_high.rb', 'app/models/all_time_high.rb'
|
30
|
+
template 'models/all_time_low.rb', 'app/models/all_time_low.rb'
|
31
|
+
template 'models/new_high.rb', 'app/models/new_high.rb'
|
32
|
+
template 'models/new_low.rb', 'app/models/new_low.rb'
|
33
|
+
template 'models/high_low.rb', 'app/models/high_low.rb'
|
34
|
+
end
|
35
|
+
|
36
|
+
def copy_classes
|
37
|
+
copy_file "../../../../barchart_data/scraper.rb", 'lib/barchart_data/scraper.rb'
|
38
|
+
copy_file "../../../../barchart_data/high_low_scraper.rb", 'lib/barchart_data/high_low_scraper.rb'
|
15
39
|
copy_file "../../../../../bin/barchart_data", 'bin/barchart'
|
16
40
|
end
|
17
41
|
end
|
File without changes
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class CreateHighLows < ActiveRecord::Migration
|
2
|
+
def change
|
3
|
+
create_table :high_lows, :force => true do |t|
|
4
|
+
t.integer :one_month_high
|
5
|
+
t.integer :one_month_low
|
6
|
+
t.integer :three_month_high
|
7
|
+
t.integer :three_month_low
|
8
|
+
t.integer :six_month_high
|
9
|
+
t.integer :six_month_low
|
10
|
+
t.integer :twelve_month_high
|
11
|
+
t.integer :twelve_month_low
|
12
|
+
t.integer :ytd_high
|
13
|
+
t.integer :ytd_low
|
14
|
+
t.datetime :saved_on
|
15
|
+
t.timestamps null: false
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,6 @@
|
|
1
|
+
class HighLow < ActiveRecord::Base
|
2
|
+
validates :saved_on, presence: true, uniqueness: true
|
3
|
+
validates :one_month_high, :one_month_low, :three_month_high, :three_month_low,
|
4
|
+
:six_month_high, :six_month_low, :twelve_month_high, :twelve_month_low,
|
5
|
+
:ytd_high, :ytd_low, presence: true
|
6
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: barchart_data
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Becco
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -118,18 +118,32 @@ extra_rdoc_files: []
|
|
118
118
|
files:
|
119
119
|
- ".gitignore"
|
120
120
|
- ".travis.yml"
|
121
|
+
- CHANGELOG.md
|
121
122
|
- Gemfile
|
122
123
|
- README.md
|
123
124
|
- Rakefile
|
125
|
+
- barchart_data-0.1.0.gem
|
124
126
|
- barchart_data.gemspec
|
125
127
|
- bin/barchart_data
|
126
128
|
- bin/console
|
127
129
|
- bin/setup
|
128
130
|
- lib/barchart_data.rb
|
129
131
|
- lib/barchart_data/alltimehigh.rb
|
132
|
+
- lib/barchart_data/high_low_scraper.rb
|
133
|
+
- lib/barchart_data/newhigh.rb
|
134
|
+
- lib/barchart_data/scraper.rb
|
130
135
|
- lib/barchart_data/version.rb
|
131
136
|
- lib/generators/barchart_data/install/install_generator.rb
|
132
|
-
- lib/generators/barchart_data/install/templates/
|
137
|
+
- lib/generators/barchart_data/install/templates/alltimehigh_migration.rb
|
138
|
+
- lib/generators/barchart_data/install/templates/alltimelow_migration.rb
|
139
|
+
- lib/generators/barchart_data/install/templates/highlow_migration.rb
|
140
|
+
- lib/generators/barchart_data/install/templates/models/all_time_high.rb
|
141
|
+
- lib/generators/barchart_data/install/templates/models/all_time_low.rb
|
142
|
+
- lib/generators/barchart_data/install/templates/models/high_low.rb
|
143
|
+
- lib/generators/barchart_data/install/templates/models/new_high.rb
|
144
|
+
- lib/generators/barchart_data/install/templates/models/new_low.rb
|
145
|
+
- lib/generators/barchart_data/install/templates/newhigh_migration.rb
|
146
|
+
- lib/generators/barchart_data/install/templates/newlow_migration.rb
|
133
147
|
homepage: https://github.com/trendwithin/barchart_data
|
134
148
|
licenses:
|
135
149
|
- MIT
|