tsp_scraper 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3fdf116c523aefb82e8d38f43a60b9b2e7e85c46da74e5a05d4f0070353edb9
-  data.tar.gz: 6dbda0c5d9a5172b18c58bc743675977802eaeb6c207468bc195b99de557ab9c
+  metadata.gz: 06f326a47003377a09a52f6a6943355119373cfd73740f645a7e91f7f4583a91
+  data.tar.gz: 0e8f2a09436a87940f7b7fc66b7243d8c4c0b28e88b32e72ebf984b283c28237
 SHA512:
-  metadata.gz: e4de9026afd500e4d1f4c9ddbd2243998fd5e3ae85fc0196e5df92a479d83ea624b066fae944f8e7d8c01cb54d0d7cad3bf8350759a0229163984c85ef74ac41
-  data.tar.gz: fa3d6219627bba8b6f40591354df177ec79c4e28e9a7cabac0551fb87214417e805da6111dc4c4b32749bffcd3b722a16d661b2bc7f64029934811747fcafa1d
+  metadata.gz: 397fa873097f209732982fcdf156b2062e240aaf71ea53470861fca6e6a56715aa2b257099b825a1aa28438fa3206ab4fb775ad81b74eadbe136271dabc12540
+  data.tar.gz: 342ccc4844395e03c838d88fdcc79bf1bbe139406228cb7304738e720f9311cc6f7edd47e1915d16a8a4e32f3a78f806be6cd7a66881ec484858d99ade698805

data/lib/tsp_scraper/client.rb

@@ -4,22 +4,21 @@ module TSPScraper
   class Client
     include HTTParty
     disable_rails_query_string_format
-    base_uri "https://www.tsp.gov/InvestmentFunds/FundPerformance"
+    base_uri "https://secure.tsp.gov/components/CORS"
 
     def self.scrape_raw(start_date = Date.today.prev_month, end_date = Date.today, options = {})
-      funds = ["Linc", "L2010", "L2020", "L2030", "L2040", "L2050", "G", "F", "C", "S", "I"]
       default_options = {
         query: {
-          reloaded: 1,
-          startdate: start_date.strftime("%m/%d/%Y"),
-          enddate: end_date.strftime("%m/%d/%Y"),
-          whichButton: "CSV",
-          fundgroup: funds
+          download: 1,
+          startdate: start_date.strftime("%Y%m%d"),
+          enddate: end_date.strftime("%Y%m%d"),
+          format: "CSV",
+          Lfunds: 1,
+          InvFunds: 1
         }
       }
       options = default_options.merge(options)
-
-      response = self.get("/index.html", options)
+      response = self.get("/getSharePrices.html", options)
       response.body.strip
     end
 

data/lib/tsp_scraper/converter.rb

@@ -2,11 +2,10 @@ module TSPScraper
   class Converter
     # TSP CSV has a janky format:
     #   - Each field in a row, except for the first field, is prepended by a space
-    #   - Each row has a final column containing just a space
     # For this reason, we manually clean up the CSV instead of using the Ruby CSV library, especially since we know TSP
     # does not use quotation marks in their CSV fields... at least for now.
     def self.raw_csv_to_hash(raw_csv)
-      raw_data = raw_csv.lines.map { |row| row.strip.chomp(',').split(',').map { |field| field.strip } }
+      raw_data = raw_csv.lines.map { |row| row.strip.split(',').map { |field| field.strip } }
       headers = raw_data.shift
       data = []
       raw_data.each do |d|
@@ -16,12 +15,7 @@ module TSPScraper
         }
         headers.each_with_index do |header, index|
           next if index == 0
-          if d[index].empty?
-            value = nil
-          else
-            value = BigDecimal(d[index])
-          end
-          hash[:funds][header] = value
+          hash[:funds][header] = BigDecimal(d[index]) unless d[index].empty?
         end
         data.push(hash)
       end

data/lib/tsp_scraper/version.rb

@@ -1,3 +1,3 @@
 module TSPScraper
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

data/tsp_scraper.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
   s.executables = ['tsp_scraper']
 
   s.add_runtime_dependency 'httparty', '~> 0.17.1'
-  s.add_development_dependency 'bundler', '~> 2.0.2'
+  s.add_development_dependency 'bundler', '~> 2.1.4'
   s.add_development_dependency 'rake', '~> 13.0.0'
   s.add_development_dependency 'pry', '~> 0.12.2'
   s.add_development_dependency 'minitest', '~> 5.12.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tsp_scraper
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Jeff Fredrickson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-15 00:00:00.000000000 Z
+date: 2020-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.1.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -107,7 +107,7 @@ homepage: https://github.com/jfredrickson/tsp_scraper
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -123,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Scrapes fund price history from the TSP.gov website
 test_files: []