holistics 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1bba5c00d378d880776abee2c49de89169d805c446eb568cdd53dd06230875c3
-  data.tar.gz: de6a9892e7731010c961783945cda2ea86ba26bcbda8492ddf53bbf57016029e
+  metadata.gz: 32364a4f3b5aa6da19afdc05c34a6a9c3361721a200a6788bbc81bd4c7039b52
+  data.tar.gz: a3a993895ae584784d2abe4b3ba4e27a21688e8a4222c71be942cad483bfa809
 SHA512:
-  metadata.gz: 16de9d14ab01ab1c6a187ce74ddf00f3c366b026c227b5c3fd0e619084fc4d201063e136a677f59ce64a6dc3cef1db8eda1a60d01418cc8020280ebaf32d7c4b
-  data.tar.gz: 73657415d7f7f527153f34440a01fbf0831505e11e242d0ff589590bed3551638db95167d4eb6002c66eba45963e7c1ff0e6fc87012908f2734d5982416bfdc5
+  metadata.gz: 8d5cfeb9ffc0990025139117aa02c31a0bada08b86c267bfd37e1257f240267221f9acb0d8eb162e7f9f1ab5a60a00a152c1d070c30ff8bc60a18dafd70c3919
+  data.tar.gz: 441660cafeee9c3b74f6a8cd6f9984b1e2b4e13bf7cbfd33c58d0a765ab98c45b5da371a3a36c315ea236287f5110b2a012aa3501e6caa85ba0f2931d84a4ff5

data/CHANGELOG.md

@@ -2,32 +2,42 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
-## v0.0.11
-* Support invocation of transform from CLI: `holistics transform -j <id>`
+## v0.4.0
+* Add `dbt upload` command
 
-## v0.0.12
-* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
+## v0.3.4
+* Update retry mechanism for `fetch_job_details`
 
-## v0.1.0
-* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
+## v0.3.3
+* Add "transform list" and "import list" commands
+* Add "version" command
 
-## v0.2.0
-* Support command to import csv into table
+## v0.3.2
+* Add 'colorize' to gemspec dependencies
 
-## v0.2.1
-* Fix dependency error
+## v0.3.1
+* Add option to split a custom range Data Import execution into smaller ones
 
-## v0.2.5
-* Support invoking email schedule with `holistics email_schedule trigger <id>`
+## v0.3.0
+* Add option to execute Data Imports in custom range mode
 
 ## v0.2.6
 * Add error handling and retry mechanism for `fetch_job_details`
 
-## v0.3.0
-* Add option to execute Data Imports in custom range mode
+## v0.2.5
+* Support invoking email schedule with `holistics email_schedule trigger <id>`
 
-## v0.3.1
-* Add option to split a custom range Data Import execution into smaller ones
+## v0.2.1
+* Fix dependency error
 
-## v0.3.2
-* Add 'colorize' to gemspec dependencies
+## v0.2.0
+* Support command to import csv into table
+
+## v0.1.0
+* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
+
+## v0.0.12
+* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
+
+## v0.0.11
+* Support invocation of transform from CLI: `holistics transform -j <id>`

data/README.md

@@ -8,6 +8,7 @@ Command-line interface to Holistics API
 First, open `version.rb` and increase the version number. Please also update the changelog too.
 
 Then run the following:
+    $ git tag -a v0.0.3 && git push --tags
 
     $ gem build holistics.gemspec
     WARNING:  description and summary are identical
@@ -79,6 +80,11 @@ Example:
     [job:738] Hot-swapping with Redshift table public.users ...
     [job:738] Done.
 
+## dbt commands
+### Upload dbt's manifest.json file to Holistics
+    $ holistics dbt upload --file-path /path/to/manifest.json --data-source demodb
+
+
 ### Custom table build transport
 
 See `samples/clicks_mysql_to_redshift.json` for details of transport configs.

data/holistics.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.0"
 
-  spec.add_dependency 'activesupport', '~> 4.2'
+  spec.add_dependency 'activesupport'
   spec.add_dependency 'httparty', '~> 0.13'
   spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'multipart-post', '~> 2.0'

data/lib/dbt.rb

@@ -0,0 +1,48 @@
+require 'thor'
+require 'holistics/helpers/http_request'
+
+module Holistics
+  class Dbt < Thor
+    method_option :filepath, aliases: '--file-path', type: :string, required: false, desc: 'Path to manifest.json file'
+    method_option :data_source_name, aliases: '--data-source', type: :string, required: false, desc: 'Name of data_source'
+    desc 'upload', 'Upload manifest.json file to Holistics'
+    def upload
+      data_source_name = options[:data_source_name]
+      manifest_file = get_manifest_file_from(options)
+
+      params = {
+        data_source_name: data_source_name
+      }
+
+      http_request.post_file('api/v2/data_sources/upload_dbt_manifest.json', params, manifest_file, 'application/json',
+                             'Error uploading manifest.json file')
+
+      puts 'Upload completed!'
+    end
+
+    private
+
+    def get_manifest_file_from(options)
+      local_filepath =
+        if options[:filepath].present?
+          options[:filepath]
+        else
+          Dir['./**/manifest.json'].first
+        end
+
+      begin
+        file = File.open(local_filepath)
+      rescue StandardError
+        warn "Could not open file at '#{local_filepath}'."
+        puts 'Invalid file path. Please check your file path.'
+        return
+      end
+
+      file
+    end
+
+    def http_request
+      @http_request ||= Helpers::HttpRequest.new
+    end
+  end
+end

data/lib/holistics/api_client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tabular_formatter'
 require 'yaml'
 require 'json'
@@ -10,16 +12,16 @@ module Holistics
     class ImportError < StandardError
     end
 
-    def import_csv options
+    def import_csv(options)
       local_filepath = options[:filepath]
       dest_ds_id = options[:dest_ds_id]
       dest_fqname = options[:dest_table_name]
 
       begin
         file = File.open local_filepath
-      rescue
-        STDERR.puts "Could not open file at '#{local_filepath}'."
-        puts "Invalid file path. Please check your file path."
+      rescue StandardError
+        warn "Could not open file at '#{local_filepath}'."
+        puts 'Invalid file path. Please check your file path.'
 
         return
       end
@@ -31,7 +33,7 @@ module Holistics
         dest_ds_id: dest_ds_id
       }
 
-      json = http_request.post_csv 'data_imports/import_csv.json', params, file, 'Error uploading CSV file'
+      json = http_request.post_file 'data_imports/import_csv.json', params, file, 'text/csv', 'Error uploading CSV file'
 
       job_id = json['message']['job_id']
 
@@ -42,8 +44,28 @@ module Holistics
     def ds_list
       result = http_request.get 'data_sources.json', 'Error retrieving list of data sources'
 
-      table = [%w(ID Type Name)]
-      rows = result.map {|record| [record['id'], record['dbtype'], record['name']]}
+      table = [%w[ID Type Name]]
+      rows = result.map { |record| [record['id'], record['dbtype'], record['name']] }
+      table.concat(rows)
+
+      puts TabularFormatter.new(table).to_pretty_table
+    end
+
+    def import_list
+      result = http_request.get 'data_imports.json', 'Error retrieving list of data imports'
+
+      table = [%w[ID Name]]
+      rows = result.map { |record| [record['id'], record['title']] }
+      table.concat(rows)
+
+      puts TabularFormatter.new(table).to_pretty_table
+    end
+
+    def transform_list
+      result = http_request.get 'data_transforms.json', 'Error retrieving list of data transports'
+
+      table = [%w[ID Name]]
+      rows = result.map { |record| [record['id'], record['title']] }
       table.concat(rows)
 
       puts TabularFormatter.new(table).to_pretty_table
@@ -93,7 +115,7 @@ module Holistics
       job_manager.tail_logs job_id
       res = job_manager.fetch_job_results job_id
       unless res['status'] == 'success'
-        raise ImportError.new("Failed Import Job #{job_id}: #{res['error']}")
+        raise ImportError, "Failed Import Job #{job_id}: #{res['error']}"
       end
     end
 
@@ -103,16 +125,14 @@ module Holistics
     end
 
     def generate_configs(options)
-      unless Dir.exist?(options['output'])
-        raise 'Output location is invalid.'
-      end
+      raise 'Output location is invalid.' unless Dir.exist?(options['output'])
 
       puts "Generating transport JSON config files to #{options['output']} directory..."
 
       result = http_request.post_json 'transports/generate_configs.json', options, 'Error generating transport configs'
 
       result.each do |table_data|
-        File.open("#{options['output']}/#{table_data['filename']}", "w") do |f|
+        File.open("#{options['output']}/#{table_data['filename']}", 'w') do |f|
           f.write(JSON.pretty_generate(table_data['json_content']))
         end
 
@@ -125,24 +145,28 @@ module Holistics
 
     def build_submit_params(options)
       params = options.except(:config_path)
-      params[:configs] = parse_transport_config(options[:config_path]).to_json if options[:config_path]
+      if options[:config_path]
+        params[:configs] = parse_transport_config(options[:config_path]).to_json
+      end
       params
     end
 
     def parse_transport_config(filepath)
       file_ext = File.extname(filepath).downcase
       if file_ext == '.json'
-        return JSON.parse(File.read(filepath))
+        JSON.parse(File.read(filepath))
       elsif file_ext == '.yml'
-        return YAML.load(File.read(filepath))
+        YAML.safe_load(File.read(filepath))
       else
-        raise StandardError.new 'Invalid config file extension. Please use either JSON or YML'
+        raise StandardError, 'Invalid config file extension. Please use either JSON or YML'
       end
     rescue StandardError => e
-      STDERR.puts "Error parsing transport config file: #{e.message}"
+      warn "Error parsing transport config file: #{e.message}"
       exit 1
     end
 
+    def dbt_upload(filepath, ds_name); end
+
     private
 
     def file_manager
@@ -160,7 +184,5 @@ module Holistics
     def auth_info
       @auth_info ||= Helpers::AuthInfo.new
     end
-
-
   end
 end

data/lib/holistics/helpers/http_request.rb

@@ -34,7 +34,7 @@ module Holistics
         exit_if_error(msg_if_error, response)
         JSON.parse response.body
       end
-      
+
       def simple_get(url, token = nil)
         HTTParty
           .get url, headers: { API_KEY_HEADER => token || auth_helper.get_token_from_gconfig }
@@ -61,11 +61,11 @@ module Holistics
       #
       # see this link for setting it
       # https://github.com/nicksieger/multipart-post/issues/18#issuecomment-171479987
-      def post_csv url, params, file, msg_if_error = DEFAULT_ERROR_MSG
+      def post_file url, params, file, content_type, msg_if_error = DEFAULT_ERROR_MSG
         uri = URI.parse(endpoint_for(url))
-        csv = UploadIO.new file, "text/csv", File.basename(file.path)
-        params = params.merge file: csv
-        
+        upload_file = UploadIO.new file, content_type, File.basename(file.path)
+        params = params.merge file: upload_file
+
         post_data = Net::HTTP::Post::Multipart.new uri.path, params
         post_data.add_field(API_KEY_HEADER, auth_helper.get_token_from_gconfig)
 

data/lib/holistics/helpers/job_manager.rb

@@ -14,14 +14,14 @@ module Holistics
         response = http_request.simple_get url
 
         JSON.parse(response.body)
-      rescue JSON::ParserError => err
+      rescue StandardError => e
         sleep 2 ** (MAX_RETRIES - tries) unless Holistics.test?
         if (tries -= 1) >= 0
           puts 'Retrying...'
           retry
         end
         puts 'Retry exceeded... Raise error'
-        raise
+        raise e
       end
 
       def fetch_job_results(job_id)

data/lib/holistics/version.rb

@@ -1,4 +1,4 @@
 module Holistics
-  VERSION = '0.3.2'
-  DATE = '2018-08-24'
+  VERSION = '0.4.0'
+  DATE = '2021-10-15'
 end

data/lib/holistics.rb

@@ -1,14 +1,18 @@
 require 'thor'
 require 'vcr_helper'
 
+require 'holistics/version'
 require 'holistics/api_client'
 require 'holistics/custom_logger'
 require 'holistics/auth_api_client'
 
 require 'import'
+require 'import_chunking'
 require 'transport'
+require 'transform'
 require 'data_sources'
 require 'email_schedule'
+require 'dbt'
 
 module Holistics
   def self.root
@@ -42,9 +46,12 @@ module Holistics
     end
 
     register(Holistics::Import, 'import', 'import <command>', "Execute import commands")
+    register(Holistics::ImportChunking, 'import_chunking', 'import_chunking <command>', "Generate import chunks for date range imports")
     register(Holistics::Transport, 'transport', 'transport <command>', "Execute transport module's commands")
+    register(Holistics::Transform, 'transform', 'transform <command>', "Execute transform module's commands")
     register(Holistics::DataSources, 'data_sources', 'data_sources <command>', "Execute data_sources module's commands")
     register(Holistics::EmailSchedule, 'email_schedule', 'email_schedule <command>', "Execute email schedule's commands")
+    register(Holistics::Dbt, 'dbt', 'dbt <comnand>', 'dbt integration with Holistics')
 
     no_commands do
       def auth_api_client
@@ -69,13 +76,6 @@ module Holistics
       api_client.ds_list
     end
 
-    method_option :transform_id, aliases: '-j', type: :string, required: true, desc: 'ID of transform job to be executed'
-    desc 'transform', 'Invoke a transform'
-
-    def transform
-      api_client.send_transform(options.dup)
-    end
-
     method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
     desc 'job_show', 'Show job log'
 
@@ -83,5 +83,9 @@ module Holistics
       api_client.job_show(options.dup)
     end
 
+    desc 'version', "Show current Holistics gem's version"
+    def version
+      puts Holistics::VERSION
+    end
   end
 end

data/lib/import.rb

@@ -42,6 +42,11 @@ module Holistics
       exit 1
     end
 
+    desc 'list', 'List all data imports'
+    def list
+      api_client.import_list
+    end
+
     no_commands do
       def api_client
         @api_client ||= ApiClient.new
@@ -79,4 +84,4 @@ module Holistics
       raise
     end
   end
-end
+end

data/lib/import_chunking.rb

@@ -0,0 +1,52 @@
+require 'thor'
+require 'colorize'
+require 'time'
+
+module Holistics
+  class ImportChunking < Thor
+
+
+    # Example
+    # holistics import_chunking execute -j 29283 --range-start 2020-8-01T00:00:00Z --range-end 2021-9-01T00:00:00Z --chunk-days 10
+ 
+
+    method_option :import_id, aliases: '-j', type: :string, required: true, desc: 'ID of import job to be executed'
+    method_option :range_start, aliases: '--range-start', type: :string, required: true, desc: 'start range'
+    method_option :range_end, aliases: '--range-end', type: :string, required: true, desc: 'end range'
+    method_option :chunk_days, aliases: '--chunk-days', type: :numeric, required: true, desc: 'The range in days of each import'
+
+    desc 'execute', 'Generate import chunks for a date range'
+    def execute
+      one_day = 86_400
+      chunk_duration = one_day * options[:chunk_days]
+
+      date_start = Time.parse(options[:range_start])
+      date_end = Time.parse(options[:range_end])
+
+      generated_result = []
+
+      current_time = date_start
+
+      while current_time < date_end
+        chunk_start = current_time
+        chunk_end = current_time + chunk_duration
+
+        generated_result << gen_command(options[:import_id],
+                                        chunk_start,
+                                        chunk_end < date_end ? chunk_end : date_end)
+
+        current_time += chunk_duration
+      end
+
+      puts generated_result.join(" && \\ \n")
+    end
+
+    private
+
+    def gen_command(id, range_start, range_end)
+      start_str = range_start.utc.iso8601
+      end_str = range_end.utc.iso8601
+      "holistics import execute -j #{id} --custom-range --range-start #{start_str} --range-end #{end_str}"
+    end
+  end
+end

data/lib/transform.rb

@@ -0,0 +1,29 @@
+require 'thor'
+
+module Holistics
+  class Transform < Thor
+
+    no_commands do
+      def api_client
+        @api_client ||= ApiClient.new
+      end
+    end
+
+    method_option :transform_id, aliases: '-j', type: :string, desc: 'ID of transform job to be executed'
+    desc 'transform', 'Invoke a transform'
+
+    def transform
+      unless options['transform_id']
+        puts "Please include a tranform job ID to execute. Usage: 'holistics transform --transform-id <id>'."
+        exit 1
+      end
+      api_client.send_transform(options.dup)
+    end
+    default_task :transform
+
+    desc 'list', 'List all data transforms'
+    def list
+      api_client.transform_list
+    end
+  end
+end

data/lib/transport.rb

@@ -30,4 +30,4 @@ module Holistics
       api_client.generate_configs(options.dup)
     end
   end
-end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: holistics
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Thanh Dinh Khac
 - Huy Nguyen
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-24 00:00:00.000000000 Z
+date: 2021-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -29,16 +29,16 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +108,7 @@ files:
 - bin/holistics
 - holistics.gemspec
 - lib/data_sources.rb
+- lib/dbt.rb
 - lib/email_schedule.rb
 - lib/holistics.rb
 - lib/holistics/api_client.rb
@@ -118,14 +119,16 @@ files:
 - lib/holistics/helpers/job_manager.rb
 - lib/holistics/version.rb
 - lib/import.rb
+- lib/import_chunking.rb
 - lib/tabular_formatter.rb
+- lib/transform.rb
 - lib/transport.rb
 - lib/vcr_helper.rb
 homepage: http://rubygems.org/gems/holistics-cli
 licenses:
 - GPL
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -141,9 +144,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: CLI interface for Holistics
 test_files: []