holistics 0.2.9 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b19979733fe3e8b7e3291a2d191b3fdd6f162274
-  data.tar.gz: d3c4dfcfa4b1b65159c3ba307e50940cd2af5bb8
+SHA256:
+  metadata.gz: 07bec8a025b674144e6fb6e35b9ef2c9a279e917e2187587d83a75e025f369c0
+  data.tar.gz: abe49de6255090ee1c96898df73eeb25ee3cc88dd36eef7233901c12dbc87980
 SHA512:
-  metadata.gz: 855138b11b0725a28f254012e2c60c1765a87e4f41859f8aa6609a644d4dff6a7c7326c0a3c6e11414dfbc72302db5a4e964aca9a423f9471af41bb29e178ae2
-  data.tar.gz: 1a5fe165c00d5bfb606d10e2e6dd9ca7b36ed565d2beb7dacbe68093c62df84aa8cbe0534776e691e4bdc11269ec3a6e9bc26a4a4f2db5f508ec1682d8e5ffd9
+  metadata.gz: d63c018afd38938a15be260b6ccdd63f3168d2ebba073789cba120cab1c52ea2fde9592061d580c1bfd5a166ca2662461bf16462d9b090b5fc1b951a86cec3ec
+  data.tar.gz: 6cef3b620618a6bbc562bc3ec6ed55197036ec18ebefe3ddc1c78e01492695dc17f58906df85b2054ed73f1b7e049832a302d4a64c54e84f30e98a50ba16d1fa

data/CHANGELOG.md

@@ -2,24 +2,39 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
-## v0.0.11
-* Support invocation of transform from CLI: `holistics transform -j <id>`
+## v0.3.4
+* Update retry mechanism for `fetch_job_details`
 
-## v0.0.12
-* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
+## v0.3.3
+* Add "transform list" and "import list" commands
+* Add "version" command
 
-## v0.1.0
-* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
+## v0.3.2
+* Add 'colorize' to gemspec dependencies
 
-## v0.2.0
-* Support command to import csv into table
+## v0.3.1
+* Add option to split a custom range Data Import execution into smaller ones
 
-## v0.2.1
-* Fix dependency error
+## v0.3.0
+* Add option to execute Data Imports in custom range mode
+
+## v0.2.6
+* Add error handling and retry mechanism for `fetch_job_details`
 
 ## v0.2.5
 * Support invoking email schedule with `holistics email_schedule trigger <id>`
 
-## v0.2.6
-* Add error handling and retry mechanism for `fetch_job_details`
+## v0.2.1
+* Fix dependency error
+
+## v0.2.0
+* Support command to import csv into table
+
+## v0.1.0
+* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
 
+## v0.0.12
+* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
+
+## v0.0.11
+* Support invocation of transform from CLI: `holistics transform -j <id>`

data/README.md

@@ -8,6 +8,7 @@ Command-line interface to Holistics API
 First, open `version.rb` and increase the version number. Please also update the changelog too.
 
 Then run the following:
+    $ git tag -a v0.0.3 && git push --tags
 
     $ gem build holistics.gemspec
     WARNING:  description and summary are identical

data/holistics.gemspec

@@ -26,5 +26,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'httparty', '~> 0.13'
   spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'multipart-post', '~> 2.0'
+  spec.add_dependency "colorize", "~> 0.8.1"
 
 end

data/lib/holistics.rb

@@ -1,12 +1,14 @@
 require 'thor'
 require 'vcr_helper'
 
+require 'holistics/version'
 require 'holistics/api_client'
 require 'holistics/custom_logger'
 require 'holistics/auth_api_client'
 
 require 'import'
 require 'transport'
+require 'transform'
 require 'data_sources'
 require 'email_schedule'
 
@@ -43,6 +45,7 @@ module Holistics
 
     register(Holistics::Import, 'import', 'import <command>', "Execute import commands")
     register(Holistics::Transport, 'transport', 'transport <command>', "Execute transport module's commands")
+    register(Holistics::Transform, 'transform', 'transform <command>', "Execute transform module's commands")
     register(Holistics::DataSources, 'data_sources', 'data_sources <command>', "Execute data_sources module's commands")
     register(Holistics::EmailSchedule, 'email_schedule', 'email_schedule <command>', "Execute email schedule's commands")
 
@@ -69,13 +72,6 @@ module Holistics
       api_client.ds_list
     end
 
-    method_option :transform_id, aliases: '-j', type: :string, required: true, desc: 'ID of transform job to be executed'
-    desc 'transform', 'Invoke a transform'
-
-    def transform
-      api_client.send_transform(options.dup)
-    end
-
     method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
     desc 'job_show', 'Show job log'
 
@@ -83,5 +79,9 @@ module Holistics
       api_client.job_show(options.dup)
     end
 
+    desc 'version', "Show current Holistics gem's version"
+    def version
+      puts Holistics::VERSION
+    end
   end
 end

data/lib/holistics/api_client.rb

@@ -7,6 +7,8 @@ require 'holistics/helpers/job_manager'
 
 module Holistics
   class ApiClient
+    class ImportError < StandardError
+    end
 
     def import_csv options
       local_filepath = options[:filepath]
@@ -47,6 +49,26 @@ module Holistics
       puts TabularFormatter.new(table).to_pretty_table
     end
 
+    def import_list
+      result = http_request.get 'data_imports.json', 'Error retrieving list of data imports'
+
+      table = [%w(ID Name)]
+      rows = result.map {|record| [record['id'], record['title']]}
+      table.concat(rows)
+
+      puts TabularFormatter.new(table).to_pretty_table
+    end
+
+    def transform_list
+      result = http_request.get 'data_transforms.json', 'Error retrieving list of data transports'
+
+      table = [%w(ID Name)]
+      rows = result.map {|record| [record['id'], record['title']]}
+      table.concat(rows)
+
+      puts TabularFormatter.new(table).to_pretty_table
+    end
+
     def send_transport(options)
       puts 'Submitting transport job ...'
 
@@ -89,6 +111,10 @@ module Holistics
 
       puts "Job submitted. Job ID: #{job_id}."
       job_manager.tail_logs job_id
+      res = job_manager.fetch_job_results job_id
+      unless res['status'] == 'success'
+        raise ImportError.new("Failed Import Job #{job_id}: #{res['error']}")
+      end
     end
 
     def job_show(options)

data/lib/holistics/helpers/job_manager.rb

@@ -9,19 +9,24 @@ module Holistics
 
       def fetch_job_details(job_id, last_id = 0)
         tries ||= MAX_RETRIES
-        
+
         url = auth_helper.api_url_for("jobs/#{job_id}/logs.json", last_id: last_id)
-        response = HttpRequest.new.simple_get url
-        
+        response = http_request.simple_get url
+
         JSON.parse(response.body)
-      rescue JSON::ParserError => err
+      rescue StandardError => e
         sleep 2 ** (MAX_RETRIES - tries) unless Holistics.test?
         if (tries -= 1) >= 0
           puts 'Retrying...'
           retry
         end
         puts 'Retry exceeded... Raise error'
-        raise
+        raise e
+      end
+
+      def fetch_job_results(job_id)
+        url = "jobs/#{job_id}/get_results.json"
+        http_request.get(url, "Cannot fetch info of job #{job_id}")
       end
 
       def job_show options
@@ -47,6 +52,12 @@ module Holistics
         ts = Time.parse(log['timestamp'])
         Holistics.logger.log(log['level'], log['message'], timestamp: ts)
       end
+
+      private
+
+      def http_request
+        @http_request ||= HttpRequest.new
+      end
     end
   end
 end

data/lib/holistics/version.rb

@@ -1,4 +1,4 @@
 module Holistics
-  VERSION = '0.2.9'
-  DATE = '2017-10-27'
+  VERSION = '0.3.4'
+  DATE = '2018-08-24'
 end

data/lib/import.rb

@@ -1,4 +1,5 @@
 require 'thor'
+require 'colorize'
 
 module Holistics
   class Import < Thor
@@ -15,13 +16,14 @@ module Holistics
 
     # execute import job
     method_option :import_id, aliases: '-j', type: :string, required: true, desc: 'ID of import job to be executed'
-    method_option :range, aliases: '--range', type: :boolean, required: false, desc: 'Execution type is custom_range. The range of left-closed right-opened range `[start, end)`.'
-    method_option :range_start, aliases: '--start', type: :string, required: false, desc: 'start range'
-    method_option :range_end, aliases: '--end', type: :string, required: false, desc: 'end range'
+    method_option :custom_range, aliases: '--custom-range', type: :boolean, lazy_default: true, required: false, desc: 'Execution type is custom_range. The range of left-closed right-opened range `[start, end)`.'
+    method_option :range_start, aliases: '--range-start', type: :string, required: false, desc: 'start range'
+    method_option :range_end, aliases: '--range-end', type: :string, required: false, desc: 'end range'
+    method_option :split_mode, aliases: '--split-mode', type: :string, required: false, desc: "Import splitting mode. Use 'daily' to split your custom range import to multiple executions, each covering one day of data."
     desc 'execute', 'Invoke an import job'
 
     def execute
-      range_mode = options[:range]
+      range_mode = options[:custom_range]
       params = {
         import_id: options[:import_id]
       }
@@ -30,8 +32,19 @@ module Holistics
         params[:range_start] = options[:range_start]
         params[:range_end] = options[:range_end]
         params[:execution_mode] = 'custom_range'
+        if options[:split_mode] == 'daily'
+          return daily_splitted_import(params.dup)
+        end
       end
-      api_client.send_import(params.dup)
+      send_import(params)
+    rescue => e
+      STDERR.puts e.message.red
+      exit 1
+    end
+
+    desc 'list', 'List all data imports'
+    def list
+      api_client.import_list
     end
 
     no_commands do
@@ -39,5 +52,36 @@ module Holistics
         @api_client ||= ApiClient.new
       end
     end
+
+    private
+
+    def daily_splitted_import(params)
+      start_date = Date.parse(params[:range_start]) rescue raise("Invalid range start")
+      if params[:range_end]
+        end_date = Date.parse(params[:range_end]) rescue raise("Invalid range end")
+      else
+        end_date = Date.today
+      end
+      while start_date + 1 < end_date
+        params[:range_start] = start_date.strftime('%Y-%m-%d')
+        params[:range_end] = (start_date + 1).strftime('%Y-%m-%d')
+        puts "Importing range '#{params[:range_start]}' - '#{params[:range_end]}'".green
+        send_import(params, tries: 2)
+        start_date = start_date + 1
+      end
+    end
+
+    def send_import(params, tries: 0)
+      total_trials ||= tries
+      api_client.send_import(params.dup)
+    rescue ApiClient::ImportError => e
+      sleep 2 ** (total_trials - tries) unless Holistics.test?
+      if (tries -= 1) >= 0
+        puts 'Retrying...'.orange
+        retry
+      end
+      puts "Exceeded retry count of #{total_trials}...".orange if total_trials > 0
+      raise
+    end
   end
-end
+end

data/lib/transform.rb

@@ -0,0 +1,29 @@
+require 'thor'
+
+module Holistics
+  class Transform < Thor
+
+    no_commands do
+      def api_client
+        @api_client ||= ApiClient.new
+      end
+    end
+
+    method_option :transform_id, aliases: '-j', type: :string, desc: 'ID of transform job to be executed'
+    desc 'transform', 'Invoke a transform'
+
+    def transform
+      unless options['transform_id']
+        puts "Please include a tranform job ID to execute. Usage: 'holistics transform --transform-id <id>'."
+        exit 1
+      end
+      api_client.send_transform(options.dup)
+    end
+    default_task :transform
+
+    desc 'list', 'List all data transforms'
+    def list
+      api_client.transform_list
+    end
+  end
+end

data/lib/transport.rb

@@ -30,4 +30,4 @@ module Holistics
       api_client.generate_configs(options.dup)
     end
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: holistics
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.3.4
 platform: ruby
 authors:
 - Thanh Dinh Khac
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-27 00:00:00.000000000 Z
+date: 2018-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,6 +81,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
 description: CLI interface for Holistics
 email: huy@holistics.io
 executables:
@@ -105,6 +119,7 @@ files:
 - lib/holistics/version.rb
 - lib/import.rb
 - lib/tabular_formatter.rb
+- lib/transform.rb
 - lib/transport.rb
 - lib/vcr_helper.rb
 homepage: http://rubygems.org/gems/holistics-cli
@@ -128,7 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: CLI interface for Holistics