ytj_client 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 455bb7b28b6be0a244f82d8be90a5a6d1dffa0d7
-  data.tar.gz: f268be3fd7806007f44600fb2d9e3fbb9fb4ab15
+  metadata.gz: 47a276c1b5853c8a646a66cb67b4678615461fe4
+  data.tar.gz: ab7257eef607daca529f9639c68bc0634632b18b
 SHA512:
-  metadata.gz: 26734252f07e86bc96bc022b5b70f0b39d5186e245a6cd5d79e1a9fbaccd5355635c71e082f24fef18526174592f0c4506de40cd37d7e90412861605401a7569
-  data.tar.gz: bcacb0411072814d10fbea61cb6c8c6894a9ea2c84bccddb900a8f71822c06ee786ed27da71a1370ce675abb9b4495eca95b5b42467520e3fae44cd370a02393
+  metadata.gz: 42702fb6d17475bb9d16a077bde5dd652c506313ff58986b2fee390e1f5f74bb4cf97fe503fd94bd9b8ff58ee6f949a3687e0228f52cac4e01fae0482691765f
+  data.tar.gz: 7b092f3f0e14f4dc41b6cdeaf815063d8a8abe17f5164841c19e3a90188fad6623fe214d7e6d42e9f0b4d5cb6cc569c274425c2ad766d8bc3ff1e4d8d5645fd6

data/README.md

@@ -54,6 +54,12 @@ YtjClient.fetch_company('2331972-7')
 ```
 ## Version history
 
+### 0.2.3
+
+- Fixes to fetching all the companies:
+  - fetch one year and 1000 companies at a time
+  - save to csv right away
+
 ### 0.2.2
 
 - fetch all Finnish companies from TR api

data/lib/ytj_client/version.rb

@@ -1,3 +1,3 @@
 module YtjClient
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/lib/ytj_client.rb

@@ -8,11 +8,15 @@ require 'active_support/core_ext/hash/slice'
 require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/string/inflections'
 
+require 'json'
+require 'csv'
+
 module YtjClient
 
   YTJ_API_URL = 'http://avoindata.prh.fi:80/bis/v1/'.freeze
-  TR_API_URL = 'http://avoindata.prh.fi:80/tr/v1?totalResults=false&maxResults=1000&resultsFrom=0&companyRegistrationFrom=
+  TR_API_URL = 'http://avoindata.prh.fi:80/tr/v1?totalResults=false&maxResults=1000&resultsFrom=0
 '.freeze
+  START_YEAR = 1896
 
   class << self
 
@@ -36,27 +40,62 @@ module YtjClient
       logger.error "Error fetching data from YTJ: #{$!.message} - #{$!.backtrace}"
     end
 
-    def fetch_all_companies(to_date = '1896-01-01')
-      all_companies = []
-      url = TR_API_URL+to_date
+    def fetch_all_companies(format = 'csv')
+      overall_fetched_companies = 0
+      end_year = Time.now.year
 
-      while true
-        companies, url = fetch_1000_companies(url)
-        companies.each do |result|
-          all_companies << result.slice("businessId", "name", "companyForm", "registrationDate").symbolize_keys
-        end
-        break if url.blank?
+      end_year.downto(START_YEAR).to_a.each do |year|
+        overall_fetched_companies += fetch_year(year, format)
       end
-      return all_companies
+
+      logger.info "Fetched #{overall_fetched_companies} companies and saved in #{format}"
+      return overall_fetched_companies
+    rescue
+      logger.error "Error fetching data from TR API: #{$!.message} - #{$!.backtrace}"
     end
 
     private
 
+      def fetch_year(year, format)
+        url = url = TR_API_URL+"&companyRegistrationFrom=#{year}-01-01&companyRegistrationTo=#{year}-12-31"
+        fetched_companies = 0
+        while true
+          companies, url = fetch_1000_companies(url)
+          logger.info "Fetched #{companies.size} companies."
+          logger.info "Next URL: #{url}"
+          save_companies(companies, format)
+          fetched_companies += companies.size
+          if url.blank?
+            logger.info "No more companies to get for year #{year}. Last response was #{companies.size} companies: #{companies}"
+            break
+          end
+          logger.info "Got #{fetched_companies} companies now, fetching some more"
+          sleep 5
+        end
+        logger.info "Got #{fetched_companies} for year #{year}. Moving on."
+        return fetched_companies
+      rescue
+        logger.error "Error fetching data for year #{year} from TR API: #{$!.message} - #{$!.backtrace}"
+      end
+
       def fetch_1000_companies(url)
         response = JSON.parse(RestClient.get(url).body)
         return response["results"], response["nextResultsUri"]
       end
 
+      def save_companies(companies, format)
+        case format
+        when 'csv'
+          CSV.open("companies.csv", "ab") do |csv|
+            companies.each do |company|
+              csv << [company["businessId"], company["companyForm"], company["name"], company["registrationDate"]]
+            end
+          end
+        else
+          logger.info "Unknown save format"
+        end
+      end
+
       # Returns a hash with all company data from YTJ
       def api_call(business_id)
         url = "#{YTJ_API_URL}#{business_id}"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ytj_client
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Janne Warén
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-27 00:00:00.000000000 Z
+date: 2017-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler