holistics 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 971907f72fa1cd5696fcb351d9c3ef571a40990e
-  data.tar.gz: 865601ab1b96aba716d23ddec7eb3a13db454100
+SHA256:
+  metadata.gz: d37b04f209ce3578a3d08d7854d6c5ed15950c772ac5298a2b5ce9f591404d82
+  data.tar.gz: dea733b1f41dd4fd48a2216db0f04f9b8d02eed1d31666e3b778dbfdad77bd79
 SHA512:
-  metadata.gz: 47788d94da15628c6100e653c4bb371c07e245fe0733dfbc039a101ff6a8f806123e98a76c188e49d31c3f1b6a2cb8414a3a17a2a77bf8ef51c50395ca651d02
-  data.tar.gz: e9db1824bdd8a394de9cad3e2d592bb4f03891d40539b860d7a8eccd3b54f94fa1bd11db834378a781e80f639bf9ed9ed39350ce074b210af4b539d23312b4fa
+  metadata.gz: 13e878fe25977f683c37444f9b6518b50e0e3e428714e8203df958c1809f6a042575641efb3e3193d37c9960a91b04761f13ad80b7b5833e3acb0610034a27e4
+  data.tar.gz: e63194d1bad41ceb4a39b06c4d83d1b6892a018c0368fc1cb122e8cf47d64b85d1408b3f0e07b38eba4a877898a867295887bc0688b32941430697b145519f33

data/CHANGELOG.md

@@ -23,3 +23,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ## v0.2.6
 * Add error handling and retry mechanism for `fetch_job_details`
 
+## v0.3.0
+* Add option to execute Data Imports in custom range mode
+
+## v0.3.1
+* Add option to split a custom range Data Import execution into smaller ones

data/lib/holistics/api_client.rb

@@ -7,6 +7,8 @@ require 'holistics/helpers/job_manager'
 
 module Holistics
   class ApiClient
+    class ImportError < StandardError
+    end
 
     def import_csv options
       local_filepath = options[:filepath]
@@ -89,6 +91,10 @@ module Holistics
 
       puts "Job submitted. Job ID: #{job_id}."
       job_manager.tail_logs job_id
+      res = job_manager.fetch_job_results job_id
+      unless res['status'] == 'success'
+        raise ImportError.new("Failed Import Job #{job_id}: #{res['error']}")
+      end
     end
 
     def job_show(options)

data/lib/holistics/helpers/job_manager.rb

@@ -9,10 +9,10 @@ module Holistics
 
       def fetch_job_details(job_id, last_id = 0)
         tries ||= MAX_RETRIES
-        
+
         url = auth_helper.api_url_for("jobs/#{job_id}/logs.json", last_id: last_id)
-        response = HttpRequest.new.simple_get url
-        
+        response = http_request.simple_get url
+
         JSON.parse(response.body)
       rescue JSON::ParserError => err
         sleep 2 ** (MAX_RETRIES - tries) unless Holistics.test?
@@ -24,6 +24,11 @@ module Holistics
         raise
       end
 
+      def fetch_job_results(job_id)
+        url = "jobs/#{job_id}/get_results.json"
+        http_request.get(url, "Cannot fetch info of job #{job_id}")
+      end
+
       def job_show options
         job_id = options[:job_id]
         tail_logs(job_id)
@@ -47,6 +52,12 @@ module Holistics
         ts = Time.parse(log['timestamp'])
         Holistics.logger.log(log['level'], log['message'], timestamp: ts)
       end
+
+      private
+
+      def http_request
+        @http_request ||= HttpRequest.new
+      end
     end
   end
 end

data/lib/holistics/version.rb

@@ -1,4 +1,4 @@
 module Holistics
-  VERSION = '0.3.0'
-  DATE = '2017-10-27'
+  VERSION = '0.3.1'
+  DATE = '2018-08-24'
 end

data/lib/import.rb

@@ -1,4 +1,5 @@
 require 'thor'
+require 'colorize'
 
 module Holistics
   class Import < Thor
@@ -18,6 +19,7 @@ module Holistics
     method_option :custom_range, aliases: '--custom-range', type: :boolean, lazy_default: true, required: false, desc: 'Execution type is custom_range. The range of left-closed right-opened range `[start, end)`.'
     method_option :range_start, aliases: '--range-start', type: :string, required: false, desc: 'start range'
     method_option :range_end, aliases: '--range-end', type: :string, required: false, desc: 'end range'
+    method_option :split_mode, aliases: '--split-mode', type: :string, required: false, desc: "Import splitting mode. Use 'daily' to split your custom range import to multiple executions, each covering one day of data."
     desc 'execute', 'Invoke an import job'
 
     def execute
@@ -30,8 +32,14 @@ module Holistics
         params[:range_start] = options[:range_start]
         params[:range_end] = options[:range_end]
         params[:execution_mode] = 'custom_range'
+        if options[:split_mode] == 'daily'
+          return daily_splitted_import(params.dup)
+        end
       end
-      api_client.send_import(params.dup)
+      send_import(params)
+    rescue => e
+      STDERR.puts e.message.red
+      exit 1
     end
 
     no_commands do
@@ -39,5 +47,36 @@ module Holistics
         @api_client ||= ApiClient.new
       end
     end
+
+    private
+
+    def daily_splitted_import(params)
+      start_date = Date.parse(params[:range_start]) rescue raise("Invalid range start")
+      if params[:range_end]
+        end_date = Date.parse(params[:range_end]) rescue raise("Invalid range end")
+      else
+        end_date = Date.today
+      end
+      while start_date + 1 < end_date
+        params[:range_start] = start_date.strftime('%Y-%m-%d')
+        params[:range_end] = (start_date + 1).strftime('%Y-%m-%d')
+        puts "Importing range '#{params[:range_start]}' - '#{params[:range_end]}'".green
+        send_import(params, tries: 2)
+        start_date = start_date + 1
+      end
+    end
+
+    def send_import(params, tries: 0)
+      total_trials ||= tries
+      api_client.send_import(params.dup)
+    rescue ApiClient::ImportError => e
+      sleep 2 ** (total_trials - tries) unless Holistics.test?
+      if (tries -= 1) >= 0
+        puts 'Retrying...'.orange
+        retry
+      end
+      puts "Exceeded retry count of #{total_trials}...".orange if total_trials > 0
+      raise
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: holistics
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Thanh Dinh Khac
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-27 00:00:00.000000000 Z
+date: 2018-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -128,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: CLI interface for Holistics