holistics 0.3.0 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 971907f72fa1cd5696fcb351d9c3ef571a40990e
-  data.tar.gz: 865601ab1b96aba716d23ddec7eb3a13db454100
+  metadata.gz: cab913239d13210e0e3c091b01bc3f9b61cdc78d
+  data.tar.gz: 53b64ba89eb7ffb7134b7b80c15948d291b75efe
 SHA512:
-  metadata.gz: 47788d94da15628c6100e653c4bb371c07e245fe0733dfbc039a101ff6a8f806123e98a76c188e49d31c3f1b6a2cb8414a3a17a2a77bf8ef51c50395ca651d02
-  data.tar.gz: e9db1824bdd8a394de9cad3e2d592bb4f03891d40539b860d7a8eccd3b54f94fa1bd11db834378a781e80f639bf9ed9ed39350ce074b210af4b539d23312b4fa
+  metadata.gz: 89b361bb3763a3c89c59ffe4845d5f3b1625da74292fd6e19164dfb19ee1814566e9c8f246f7d7a753fb56a4e926c7894738099006dad0063f2c4dc1e1753925
+  data.tar.gz: f115d45d5baa6972a7d30d2b9610032aa39b924d1cf97c1ff98101cfc20e9ca8e6b640b5830245c77f39b1a2915e3c424ebdc40d3959eb7d68d90b8646eefa1c

data/CHANGELOG.md

@@ -2,24 +2,39 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
-## v0.0.11
-* Support invocation of transform from CLI: `holistics transform -j <id>`
+## v0.3.4
+* Update retry mechanism for `fetch_job_details`
 
-## v0.0.12
-* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
+## v0.3.3
+* Add "transform list" and "import list" commands
+* Add "version" command
 
-## v0.1.0
-* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
+## v0.3.2
+* Add 'colorize' to gemspec dependencies
 
-## v0.2.0
-* Support command to import csv into table
+## v0.3.1
+* Add option to split a custom range Data Import execution into smaller ones
 
-## v0.2.1
-* Fix dependency error
+## v0.3.0
+* Add option to execute Data Imports in custom range mode
+
+## v0.2.6
+* Add error handling and retry mechanism for `fetch_job_details`
 
 ## v0.2.5
 * Support invoking email schedule with `holistics email_schedule trigger <id>`
 
-## v0.2.6
-* Add error handling and retry mechanism for `fetch_job_details`
+## v0.2.1
+* Fix dependency error
+
+## v0.2.0
+* Support command to import csv into table
+
+## v0.1.0
+* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
 
+## v0.0.12
+* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
+
+## v0.0.11
+* Support invocation of transform from CLI: `holistics transform -j <id>`

data/README.md

@@ -8,6 +8,7 @@ Command-line interface to Holistics API
 First, open `version.rb` and increase the version number. Please also update the changelog too.
 
 Then run the following:
+    $ git tag -a v0.0.3 && git push --tags
 
     $ gem build holistics.gemspec
     WARNING:  description and summary are identical

data/holistics.gemspec

@@ -26,5 +26,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'httparty', '~> 0.13'
   spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'multipart-post', '~> 2.0'
+  spec.add_dependency "colorize", "~> 0.8.1"
 
 end

data/lib/holistics.rb

@@ -1,12 +1,15 @@
 require 'thor'
 require 'vcr_helper'
 
+require 'holistics/version'
 require 'holistics/api_client'
 require 'holistics/custom_logger'
 require 'holistics/auth_api_client'
 
 require 'import'
+require 'import_chunking'
 require 'transport'
+require 'transform'
 require 'data_sources'
 require 'email_schedule'
 
@@ -42,7 +45,9 @@ module Holistics
     end
 
     register(Holistics::Import, 'import', 'import <command>', "Execute import commands")
+    register(Holistics::ImportChunking, 'import_chunking', 'import_chunking <command>', "Generate import chunks for date range imports")
     register(Holistics::Transport, 'transport', 'transport <command>', "Execute transport module's commands")
+    register(Holistics::Transform, 'transform', 'transform <command>', "Execute transform module's commands")
     register(Holistics::DataSources, 'data_sources', 'data_sources <command>', "Execute data_sources module's commands")
     register(Holistics::EmailSchedule, 'email_schedule', 'email_schedule <command>', "Execute email schedule's commands")
 
@@ -69,13 +74,6 @@ module Holistics
       api_client.ds_list
     end
 
-    method_option :transform_id, aliases: '-j', type: :string, required: true, desc: 'ID of transform job to be executed'
-    desc 'transform', 'Invoke a transform'
-
-    def transform
-      api_client.send_transform(options.dup)
-    end
-
     method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
     desc 'job_show', 'Show job log'
 
@@ -83,5 +81,9 @@ module Holistics
       api_client.job_show(options.dup)
     end
 
+    desc 'version', "Show current Holistics gem's version"
+    def version
+      puts Holistics::VERSION
+    end
   end
 end

data/lib/holistics/api_client.rb

@@ -7,6 +7,8 @@ require 'holistics/helpers/job_manager'
 
 module Holistics
   class ApiClient
+    class ImportError < StandardError
+    end
 
     def import_csv options
       local_filepath = options[:filepath]
@@ -47,6 +49,26 @@ module Holistics
       puts TabularFormatter.new(table).to_pretty_table
     end
 
+    def import_list
+      result = http_request.get 'data_imports.json', 'Error retrieving list of data imports'
+
+      table = [%w(ID Name)]
+      rows = result.map {|record| [record['id'], record['title']]}
+      table.concat(rows)
+
+      puts TabularFormatter.new(table).to_pretty_table
+    end
+
+    def transform_list
+      result = http_request.get 'data_transforms.json', 'Error retrieving list of data transports'
+
+      table = [%w(ID Name)]
+      rows = result.map {|record| [record['id'], record['title']]}
+      table.concat(rows)
+
+      puts TabularFormatter.new(table).to_pretty_table
+    end
+
     def send_transport(options)
       puts 'Submitting transport job ...'
 
@@ -89,6 +111,10 @@ module Holistics
 
       puts "Job submitted. Job ID: #{job_id}."
       job_manager.tail_logs job_id
+      res = job_manager.fetch_job_results job_id
+      unless res['status'] == 'success'
+        raise ImportError.new("Failed Import Job #{job_id}: #{res['error']}")
+      end
     end
 
     def job_show(options)

data/lib/holistics/helpers/job_manager.rb

@@ -9,19 +9,24 @@ module Holistics
 
       def fetch_job_details(job_id, last_id = 0)
         tries ||= MAX_RETRIES
-        
+
         url = auth_helper.api_url_for("jobs/#{job_id}/logs.json", last_id: last_id)
-        response = HttpRequest.new.simple_get url
-        
+        response = http_request.simple_get url
+
         JSON.parse(response.body)
-      rescue JSON::ParserError => err
+      rescue StandardError => e
         sleep 2 ** (MAX_RETRIES - tries) unless Holistics.test?
         if (tries -= 1) >= 0
           puts 'Retrying...'
           retry
         end
         puts 'Retry exceeded... Raise error'
-        raise
+        raise e
+      end
+
+      def fetch_job_results(job_id)
+        url = "jobs/#{job_id}/get_results.json"
+        http_request.get(url, "Cannot fetch info of job #{job_id}")
       end
 
       def job_show options
@@ -47,6 +52,12 @@ module Holistics
         ts = Time.parse(log['timestamp'])
         Holistics.logger.log(log['level'], log['message'], timestamp: ts)
       end
+
+      private
+
+      def http_request
+        @http_request ||= HttpRequest.new
+      end
     end
   end
 end

data/lib/holistics/version.rb

@@ -1,4 +1,4 @@
 module Holistics
-  VERSION = '0.3.0'
-  DATE = '2017-10-27'
+  VERSION = '0.3.5'
+  DATE = '2021-04-19'
 end

data/lib/import.rb

@@ -1,4 +1,5 @@
 require 'thor'
+require 'colorize'
 
 module Holistics
   class Import < Thor
@@ -18,6 +19,7 @@ module Holistics
     method_option :custom_range, aliases: '--custom-range', type: :boolean, lazy_default: true, required: false, desc: 'Execution type is custom_range. The range of left-closed right-opened range `[start, end)`.'
     method_option :range_start, aliases: '--range-start', type: :string, required: false, desc: 'start range'
     method_option :range_end, aliases: '--range-end', type: :string, required: false, desc: 'end range'
+    method_option :split_mode, aliases: '--split-mode', type: :string, required: false, desc: "Import splitting mode. Use 'daily' to split your custom range import to multiple executions, each covering one day of data."
     desc 'execute', 'Invoke an import job'
 
     def execute
@@ -30,8 +32,19 @@ module Holistics
         params[:range_start] = options[:range_start]
         params[:range_end] = options[:range_end]
         params[:execution_mode] = 'custom_range'
+        if options[:split_mode] == 'daily'
+          return daily_splitted_import(params.dup)
+        end
       end
-      api_client.send_import(params.dup)
+      send_import(params)
+    rescue => e
+      STDERR.puts e.message.red
+      exit 1
+    end
+
+    desc 'list', 'List all data imports'
+    def list
+      api_client.import_list
     end
 
     no_commands do
@@ -39,5 +52,36 @@ module Holistics
         @api_client ||= ApiClient.new
       end
     end
+
+    private
+
+    def daily_splitted_import(params)
+      start_date = Date.parse(params[:range_start]) rescue raise("Invalid range start")
+      if params[:range_end]
+        end_date = Date.parse(params[:range_end]) rescue raise("Invalid range end")
+      else
+        end_date = Date.today
+      end
+      while start_date + 1 < end_date
+        params[:range_start] = start_date.strftime('%Y-%m-%d')
+        params[:range_end] = (start_date + 1).strftime('%Y-%m-%d')
+        puts "Importing range '#{params[:range_start]}' - '#{params[:range_end]}'".green
+        send_import(params, tries: 2)
+        start_date = start_date + 1
+      end
+    end
+
+    def send_import(params, tries: 0)
+      total_trials ||= tries
+      api_client.send_import(params.dup)
+    rescue ApiClient::ImportError => e
+      sleep 2 ** (total_trials - tries) unless Holistics.test?
+      if (tries -= 1) >= 0
+        puts 'Retrying...'.orange
+        retry
+      end
+      puts "Exceeded retry count of #{total_trials}...".orange if total_trials > 0
+      raise
+    end
   end
-end
+end

data/lib/import_chunking.rb

@@ -0,0 +1,52 @@
+require 'thor'
+require 'colorize'
+require 'time'
+
+module Holistics
+  class ImportChunking < Thor
+
+
+    # Example
+    # holistics import_chunking execute -j 29283 --range-start 2020-8-01T00:00:00Z --range-end 2021-9-01T00:00:00Z --chunk-days 10
+ 
+
+    method_option :import_id, aliases: '-j', type: :string, required: true, desc: 'ID of import job to be executed'
+    method_option :range_start, aliases: '--range-start', type: :string, required: true, desc: 'start range'
+    method_option :range_end, aliases: '--range-end', type: :string, required: true, desc: 'end range'
+    method_option :chunk_days, aliases: '--chunk-days', type: :numeric, required: true, desc: 'The range in days of each import'
+
+    desc 'execute', 'Generate import chunks for a date range'
+    def execute
+      one_day = 86_400
+      chunk_duration = one_day * options[:chunk_days]
+
+      date_start = Time.parse(options[:range_start])
+      date_end = Time.parse(options[:range_end])
+
+      generated_result = []
+
+      current_time = date_start
+
+      while current_time < date_end
+        chunk_start = current_time
+        chunk_end = current_time + chunk_duration
+
+        generated_result << gen_command(options[:import_id],
+                                        chunk_start,
+                                        chunk_end < date_end ? chunk_end : date_end)
+
+        current_time += chunk_duration
+      end
+
+      puts generated_result.join(" && \\ \n")
+    end
+
+    private
+
+    def gen_command(id, range_start, range_end)
+      start_str = range_start.utc.iso8601
+      end_str = range_end.utc.iso8601
+      "holistics import execute -j #{id} --custom-range --range-start #{start_str} --range-end #{end_str}"
+    end
+  end
+end

data/lib/transform.rb

@@ -0,0 +1,29 @@
+require 'thor'
+
+module Holistics
+  class Transform < Thor
+
+    no_commands do
+      def api_client
+        @api_client ||= ApiClient.new
+      end
+    end
+
+    method_option :transform_id, aliases: '-j', type: :string, desc: 'ID of transform job to be executed'
+    desc 'transform', 'Invoke a transform'
+
+    def transform
+      unless options['transform_id']
+        puts "Please include a tranform job ID to execute. Usage: 'holistics transform --transform-id <id>'."
+        exit 1
+      end
+      api_client.send_transform(options.dup)
+    end
+    default_task :transform
+
+    desc 'list', 'List all data transforms'
+    def list
+      api_client.transform_list
+    end
+  end
+end

data/lib/transport.rb

@@ -30,4 +30,4 @@ module Holistics
       api_client.generate_configs(options.dup)
     end
   end
-end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: holistics
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.5
 platform: ruby
 authors:
 - Thanh Dinh Khac
 - Huy Nguyen
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-27 00:00:00.000000000 Z
+date: 2021-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,6 +81,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
 description: CLI interface for Holistics
 email: huy@holistics.io
 executables:
@@ -104,14 +118,16 @@ files:
 - lib/holistics/helpers/job_manager.rb
 - lib/holistics/version.rb
 - lib/import.rb
+- lib/import_chunking.rb
 - lib/tabular_formatter.rb
+- lib/transform.rb
 - lib/transport.rb
 - lib/vcr_helper.rb
 homepage: http://rubygems.org/gems/holistics-cli
 licenses:
 - GPL
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -127,9 +143,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.5.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: CLI interface for Holistics
 test_files: []