tsp_scraper 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +37 -2
- data/lib/tsp_scraper/converter.rb +25 -0
- data/lib/tsp_scraper/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 49f6c804fc5d5e2610bcf037ccaeab27bbb3e0de
|
|
4
|
+
data.tar.gz: 3fd8e293e0e7406e5d83242d2dbd08fd97e30e73
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4cbb06c487cc7c867c60343ba3449777b0b36ffb59347bb697ef7b341023c2b2321b813d223fa3c03e4ca684ac1d24359abe9bf851ad687aff056c7544f60a99
|
|
7
|
+
data.tar.gz: aca17f786d764989a5bbdfdc9110be85afddcfdf65f739e5f81bea30a9dbea498cf33b36bd34e858fb4d5fb92fbf00a807704dfb26c7c5ce3181ba7a98972526
|
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# tsp_scraper
|
|
2
2
|
|
|
3
|
+
[](https://badge.fury.io/rb/tsp_scraper)
|
|
4
|
+
|
|
3
5
|
Scrapes fund price history from the TSP.gov website.
|
|
4
6
|
|
|
5
7
|
## Installation
|
|
@@ -10,7 +12,7 @@ gem install tsp_scraper
|
|
|
10
12
|
|
|
11
13
|
## Usage
|
|
12
14
|
|
|
13
|
-
The `scrape` method returns
|
|
15
|
+
The `scrape` method returns data in an array of hashes:
|
|
14
16
|
|
|
15
17
|
```ruby
|
|
16
18
|
require 'tsp_scraper'
|
|
@@ -18,7 +20,40 @@ TSPScraper::Client.scrape() # Get trailing one month's prices
|
|
|
18
20
|
TSPScraper::Client.scrape("2016-01-01", "2016-01-31") # Get January prices
|
|
19
21
|
```
|
|
20
22
|
|
|
21
|
-
The
|
|
23
|
+
The returned array of hashes has the format:
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
[
|
|
27
|
+
{
|
|
28
|
+
date: DATE1,
|
|
29
|
+
funds: [
|
|
30
|
+
{ fund: "FUND NAME 1", price: "12.3456" },
|
|
31
|
+
{ fund: "FUND NAME 2", price: "12.3456" },
|
|
32
|
+
{ fund: "FUND NAME 3", price: "12.3456" }
|
|
33
|
+
]
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
date: DATE2,
|
|
37
|
+
funds: [
|
|
38
|
+
{ fund: "FUND NAME 1", price: "12.3456" },
|
|
39
|
+
{ fund: "FUND NAME 2", price: "12.3456" },
|
|
40
|
+
{ fund: "FUND NAME 3", price: "12.3456" }
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
date: DATE3,
|
|
45
|
+
funds: [
|
|
46
|
+
{ fund: "FUND NAME 1", price: "12.3456" },
|
|
47
|
+
{ fund: "FUND NAME 2", price: "12.3456" },
|
|
48
|
+
{ fund: "FUND NAME 3", price: "12.3456" }
|
|
49
|
+
]
|
|
50
|
+
}
|
|
51
|
+
]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
You can get the raw CSV data as returned by the TSP website by using `TSPScraper::Client.scrape_raw()`.
|
|
55
|
+
|
|
56
|
+
The `scrape` and `scrape_raw` methods also accept an options hash for the HTTP request. The options hash should contain [HTTParty](https://github.com/jnunemaker/httparty) options.
|
|
22
57
|
|
|
23
58
|
```ruby
|
|
24
59
|
TSPScraper::Client.scrape("2016-01-01", "2016-01-31", verify: false) # Don't verify SSL certificate
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
module TSPScraper
|
|
2
|
+
class Converter
|
|
3
|
+
# TSP CSV has a janky format:
|
|
4
|
+
# - Each field in a row, except for the first field, is prepended by a space
|
|
5
|
+
# - Each row has a final column containing just a space
|
|
6
|
+
# For this reason, we manually clean up the CSV instead of using the Ruby CSV library, especially since we know TSP
|
|
7
|
+
# does not use quotation marks in their CSV fields... at least for now.
|
|
8
|
+
def self.raw_csv_to_hash(raw_csv)
|
|
9
|
+
raw_data = raw_csv.lines.map { |row| row.strip.chomp(',').split(',').map { |field| field.strip } }
|
|
10
|
+
headers = raw_data.shift
|
|
11
|
+
data = []
|
|
12
|
+
raw_data.each do |d|
|
|
13
|
+
hash = {
|
|
14
|
+
date: Date.parse(d.first),
|
|
15
|
+
funds: []
|
|
16
|
+
}
|
|
17
|
+
headers.each_with_index do |header, index|
|
|
18
|
+
hash[:funds].push({ fund: header, price: BigDecimal.new(d[index]) }) unless index == 0
|
|
19
|
+
end
|
|
20
|
+
data.push(hash)
|
|
21
|
+
end
|
|
22
|
+
data
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
data/lib/tsp_scraper/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tsp_scraper
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jeff Fredrickson
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-09-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: httparty
|
|
@@ -40,6 +40,7 @@ files:
|
|
|
40
40
|
- bin/tsp_scraper
|
|
41
41
|
- lib/tsp_scraper.rb
|
|
42
42
|
- lib/tsp_scraper/client.rb
|
|
43
|
+
- lib/tsp_scraper/converter.rb
|
|
43
44
|
- lib/tsp_scraper/version.rb
|
|
44
45
|
- tsp_scraper.gemspec
|
|
45
46
|
homepage: https://github.com/jfredrickson/tsp_scraper
|
|
@@ -67,4 +68,3 @@ signing_key:
|
|
|
67
68
|
specification_version: 4
|
|
68
69
|
summary: Scrapes fund price history from the TSP.gov website
|
|
69
70
|
test_files: []
|
|
70
|
-
has_rdoc:
|