holistics 0.0.11 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 355be8beb301cf93d2012d2ccdbd58540c5355c0
-  data.tar.gz: b57530d173f8b22cb95083e777e6ddcc88eac391
+  metadata.gz: ffcfdb6aa3257f5a42c021a116e428316ee4d02d
+  data.tar.gz: 4aa5954b9ab8d2157284d64570e0c085f0ff9f3b
 SHA512:
-  metadata.gz: 3af5780921c7914376eac4c8153e1c18175d65334e25d0190a333ca68e258d981b42a3e062df7974e5b1f593e4b97528a80f9be58d5d0f338ab58f9958295a55
-  data.tar.gz: ed67c37140dd9eb780d43e390cfd4d1de4c1b17a79efeb9f0906421ed678460fb0c86fc391e528145aaf7472c15f1e45350049cb439fd8bc753a98f1466b2b51
+  metadata.gz: 297dffd7ca7867cd1deb9907196768cb1a533254bc057f41d103d09acf93da127cf3704a72e6a5e9078becdbda830da0250cc720c49d84c8ebd90363124d7577
+  data.tar.gz: 56a782a5815716bd7ac88e4680ffb526d09c03b95ed8ded9990f2070e28d8033ab13df84d100386f6a555eb24bb799f762dc67c945dd49caec3823e6583b5520

data/CHANGELOG.md

@@ -4,3 +4,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## v0.0.11
 * Support invocation of transform from CLI: `holistics transform -j <id>`
+
+## v0.0.12
+* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
+
+## v0.1.0
+* Modularized transport and data_sources commands. Rename existing transport command to transport_old.

data/lib/data_sources.rb

@@ -0,0 +1,17 @@
+require 'thor'
+
+module Holistics
+  class DataSources < Thor
+
+    no_commands do
+      def api_client
+        @api_client ||= ApiClient.new
+      end
+    end
+
+    desc 'list', 'List all data sources'
+    def list
+      api_client.ds_list
+    end
+  end
+end

data/lib/holistics.rb

@@ -1,7 +1,12 @@
 require 'thor'
+require 'vcr_helper'
+
 require 'holistics/api_client'
 require 'holistics/custom_logger'
-require 'vcr'
+require 'holistics/auth_api_client'
+
+require 'transport'
+require 'data_sources'
 
 module Holistics
   def self.root
@@ -26,18 +31,30 @@ module Holistics
       super(args, options, config)
     end
 
+    register(Holistics::Transport, 'transport', 'transport <command>', "Execute transport module's commands")
+    register(Holistics::DataSources, 'data_sources', 'data_sources <command>', "Execute data_sources module's commands")
+
+    no_commands do
+      def auth_api_client
+        @auth_api_client ||= AuthApiClient.new
+      end
+
+      def api_client
+        @api_client ||= ApiClient.new
+      end
+    end
+
     desc 'login [auth_token]', 'Perform authentication'
 
     def login token
-      client = ApiClient.new
-      client.login(token)
+      auth_api_client.login(token)
     end
 
-
-    desc 'ds_list', 'List all data sources'
+    desc 'ds_list', '[DEPRECATED] List all data sources'
 
     def ds_list
-      ApiClient.new.ds_list
+      p 'DEPRECATED. This command will be removed in the next version. Please consider using `holistics data_sources list` instead.'
+      api_client.ds_list
     end
 
 
@@ -46,19 +63,33 @@ module Holistics
     method_option :from_table_name, aliases: '-t', type: :string, required: false, desc: 'The table to copy over'
     method_option :dest_table_name, aliases: '-n', type: :string, required: false, desc: '(optional) Rename destination table. Please specify fully qualified name'
     method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
-    method_option :full, type: :boolean, default: true, required: false, desc: 'Full table transport'
+    method_option :full, type: :boolean, default: false, required: false, desc: 'Full table transport'
     method_option :incremental, type: :boolean, default: false, required: false, desc: 'Incremental table transport'
-    desc 'transport', 'Submit a data transport job'
+    desc 'transport_old', '[DEPRECATED] Submit a data transport job'
 
-    def transport
-      ApiClient.new.send_transport(options.dup)
+    def transport_old
+      p 'DEPRECATED. This command will be removed in the next version. Please consider using `holistics transport submit` instead.'
+      api_client.send_transport(options.dup)
     end
 
+
     method_option :transform_id, aliases: '-j', type: :string, required: true, desc: 'ID of transform job to be executed'
     desc 'transform', 'Invoke a transform job'
 
     def transform
-      ApiClient.new.send_transform(options.dup)
+      api_client.send_transform(options.dup)
+    end
+
+
+    method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
+    method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
+    method_option :tables, aliases: '-t', type: :array, required: true, desc: 'List of tables to generate configs from. Wildcards are allowed'
+    method_option :output, aliases: '-o', type: :string, required: false, default: './', desc: 'Location where files will be generated to'
+    desc 'generate_configs', '[DEPRECATED] Generate transport JSON configuration files'
+
+    def generate_configs
+      p 'DEPRECATED. This command will be removed in the next version. Please consider using `holistics transport generate` instead.'
+      api_client.generate_configs(options.dup)
     end
 
 
@@ -66,7 +97,7 @@ module Holistics
     desc 'job_show', 'Show job log'
 
     def job_show
-      ApiClient.new.job_show(options.dup)
+      api_client.job_show(options.dup)
     end
 
   end

data/lib/holistics/api_client.rb

@@ -3,49 +3,29 @@ require 'tabular_formatter'
 require 'yaml'
 require 'json'
 
+require 'holistics/helpers/http_request'
+require 'holistics/helpers/job_manager'
+
 module Holistics
   class ApiClient
-    SERVER_URL = 'https://secure.holistics.io/'
-
-    def initialize
-
-    end
-
-    def login token
-      puts 'Logging in...'
-      response, ok = holistics_authenticate(token)
-      if ok
-        parsed = JSON.parse(response.body)
-        puts 'Authentication successful. Info:'
-        puts "- ID: #{parsed['id']}"
-        puts "- Email: #{parsed['email']}"
 
-        write_token_to_gconfig(token)
-      else
-        puts 'Error logging in. Please check your token again.'
-      end
+    def job_manager
+      @job_helper ||= Helpers::JobManager.new
     end
 
-    def holistics_authenticate(token)
-      url = api_url_for('users/info.json', {}, token)
-      response = HTTParty.get(url)
-      return response, (response.code == 200)
+    def http_request
+      @http_helper ||= Helpers::HttpRequest.new
     end
 
-    def job_show options
-      job_id = options[:job_id]
-      tail_logs(job_id)
+    def auth_info
+      @auth_info ||= Helpers::AuthInfo.new
     end
 
     def ds_list
-      url = api_url_for('data_sources.json')
-      response = HTTParty.get(url)
-      err_and_exit('Error retrieving list of data sources', response) if response_has_error?(response)
-
-      parsed = JSON.parse(response.body)
+      result = http_request.get 'data_sources.json', 'Error retrieving list of data sources'
 
       table = [%w(ID Type Name)]
-      rows = parsed.map { |record| [record['id'], record['dbtype'], record['name']] }
+      rows = result.map { |record| [record['id'], record['dbtype'], record['name']] }
       table.concat(rows)
 
       puts TabularFormatter.new(table).to_pretty_table
@@ -53,84 +33,53 @@ module Holistics
 
     def send_transport(options)
       puts 'Submitting transport job ...'
+
       params = build_submit_params(options)
-      response = submit_transport_job(params)
 
-      err_and_exit('Error submitting transport job', response) if response_has_error?(response)
+      result = http_request.post_json 'transports/submit.json', params, 'Error submitting transport job'
 
-      parsed = JSON.parse(response.body)
-      job_id = parsed['job_id']
+      job_id = result['job_id']
 
       puts "Job submitted. Job ID: #{job_id}."
-      tail_logs(job_id)
+      job_manager.tail_logs job_id
     end
 
     def send_transform(options)
       puts 'Invoking transform job...'
-      params = options.merge(_utoken: get_token_from_gconfig)
-      response = submit_transform_job(params)
 
-      err_and_exit('Error submitting transform job', response) if response_has_error?(response)
+      params = options.merge(_utoken: auth_info.get_token_from_gconfig)
+      result = http_request.post_json "data_transforms/#{params[:transform_id]}/execute.json", params, 'Error submitting transform job'
 
-      parsed = JSON.parse(response.body)
-      job_id = parsed['job_id']
+      job_id = result['job_id']
 
       puts "Job submitted. Job ID: #{job_id}."
-      tail_logs(job_id)
+      job_manager.tail_logs job_id
     end
 
-    def tail_logs(job_id)
-      last_id = 0
-      while true
-        parsed = fetch_job_details(job_id, last_id)
-        logs = parsed['logs']
-        logs.each { |log| print_log(log) }
-        last_id = logs.last['id'] if logs.size > 0
-
-        break unless parsed['has_more']
-
-        sleep 0.5
+    def generate_configs(options)
+      unless Dir.exist?(options['output'])
+        raise 'Output location is invalid.'
       end
-    end
 
-    def fetch_job_details(job_id, last_id = 0)
-      response = HTTParty.get(api_url_for("jobs/#{job_id}/logs.json", last_id: last_id))
-      JSON.parse(response.body)
-    end
+      puts "Generating transport JSON config files to #{options['output']} directory..."
 
-    def submit_transport_job(params)
-      options = {
-        body: params.to_json,
-        headers: {'Content-Type' => 'application/json'}
-      }
-      HTTParty.post(server_url + 'transports/submit.json', options)
-    end
+      params = options.merge(_utoken: auth_info.get_token_from_gconfig)
+      result = http_request.post_json 'transports/generate_configs.json', params, 'Error generating transport configs'
+      
+      result.each do |table_data|
+        File.open("#{options['output']}/#{table_data['filename']}","w") do |f|
+          f.write(JSON.pretty_generate(table_data['json_content']))
+        end
 
-    def submit_transform_job(params)
-      options = {
-        headers: {'Content-Type' => 'application/json'},
-        body: params.to_json
-      }
-      HTTParty.post(server_url + "data_transforms/#{params[:transform_id]}/execute.json", options)
-    end
+        puts "Generated #{table_data['filename']}"
+      end
 
-    def server_url
-      host =
-        if ENV['HOLISTICS_DEV'] || ENV['HOLISTICS_TEST']
-          'http://localhost:3000'
-        elsif ENV['HOLISTICS_STAGING']
-          'https://staging.holistics.io'
-        elsif ENV['HOLISTICS_HOST']
-          ENV['HOLISTICS_HOST']
-        else
-          SERVER_URL
-        end
-      host += '/' if host[-1] != '/'
-      host
+      puts
+      puts "Configs generation succeeded with #{result.length} files in total."
     end
 
     def build_submit_params(options)
-      params = options.except(:config_path).merge(_utoken: get_token_from_gconfig)
+      params = options.except(:config_path).merge(_utoken: auth_info.get_token_from_gconfig)
       params[:configs] = parse_transport_config(options[:config_path]).to_json if options[:config_path]
       params
     end
@@ -148,49 +97,5 @@ module Holistics
       STDERR.puts "Error parsing transport config file: #{e.message}"
       exit 1
     end
-
-    def authenticated?
-      File.exists?(get_gconfig_filepath)
-    end
-
-    def get_token_from_gconfig
-      if authenticated?
-        string = YAML.load_file(get_gconfig_filepath)
-        string['token']
-      else
-        raise StandardError.new 'Holistics config file not found'
-      end
-    end
-
-    def get_gconfig_filepath
-      File.expand_path('~/.holistics.yml', __FILE__)
-    end
-
-    def api_url_for(path, params = {}, token = nil)
-      params[:_utoken] = token || get_token_from_gconfig
-      "#{server_url}#{path}?#{URI.encode_www_form(params)}"
-    end
-
-    def print_log log
-      ts = Time.parse(log['timestamp'])
-      Holistics.logger.log(log['level'], log['message'], timestamp: ts)
-    end
-
-    def err_and_exit(message, response)
-      STDERR.puts message
-      STDERR.puts "Error Response Code: #{response.code}"
-      STDERR.puts response.body
-      exit 1
-    end
-
-    def response_has_error?(response)
-      response.code != 200
-    end
-
-    def write_token_to_gconfig(token)
-      file_path = File.join(ENV['HOME'], '.holistics.yml')
-      h = {'token' => token}
-      File.write(file_path, h.to_yaml)
-    end
   end
 end

data/lib/holistics/auth_api_client.rb

@@ -0,0 +1,38 @@
+require 'httparty'
+require 'holistics/helpers/auth_info'
+
+module Holistics
+  class AuthApiClient
+
+    def auth_info
+      @auth_info ||= Helpers::AuthInfo.new
+    end
+
+    def login token
+      puts 'Logging in...'
+      response, ok = authenticate(token)
+      if ok
+        parsed = JSON.parse(response.body)
+        puts 'Authentication successful. Info:'
+        puts "- ID: #{parsed['id']}"
+        puts "- Email: #{parsed['email']}"
+
+        write_token_to_gconfig(token)
+      else
+        puts 'Error logging in. Please check your token again.'
+      end
+    end
+
+    def authenticate(token)
+      url = auth_info.api_url_for('users/info.json', {}, token)
+      response = HTTParty.get(url)
+      return response, (response.code == 200)
+    end
+
+    def write_token_to_gconfig(token)
+      file_path = File.join(ENV['HOME'], '.holistics.yml')
+      h = {'token' => token}
+      File.write(file_path, h.to_yaml)
+    end
+  end
+end

data/lib/holistics/helpers/auth_info.rb

@@ -0,0 +1,34 @@
+require 'holistics/helpers/http_request'
+
+module Holistics
+  module Helpers
+    class AuthInfo
+
+      def http_request
+        @http_helper ||= Helpers::HttpRequest.new
+      end
+
+      def authenticated?
+        File.exists?(get_gconfig_filepath)
+      end
+
+      def api_url_for(path, params = {}, token = nil)
+        params[:_utoken] = token || get_token_from_gconfig
+        "#{http_request.server_url}#{path}?#{URI.encode_www_form(params)}"
+      end
+
+      def get_token_from_gconfig
+        if authenticated?
+          string = YAML.load_file(get_gconfig_filepath)
+          string['token']
+        else
+          raise StandardError.new 'Holistics config file not found'
+        end
+      end
+
+      def get_gconfig_filepath
+        File.expand_path('~/.holistics.yml', __FILE__)
+      end
+    end
+  end
+end

data/lib/holistics/helpers/http_request.rb

@@ -0,0 +1,70 @@
+require 'httparty'
+
+module Holistics
+  module Helpers
+    class HttpRequest
+
+      DEFAULT_ERROR_MSG = 'Error occurred!'
+      SERVER_URL = 'https://secure.holistics.io/'
+
+      def auth_helper
+        @auth_info ||= Helpers::AuthInfo.new
+      end
+
+      def server_url
+        host =
+            if ENV['HOLISTICS_DEV'] || ENV['HOLISTICS_TEST']
+              'http://localhost:3000'
+            elsif ENV['HOLISTICS_STAGING']
+              'https://staging.holistics.io'
+            elsif ENV['HOLISTICS_HOST']
+              ENV['HOLISTICS_HOST']
+            else
+              SERVER_URL
+            end
+        host += '/' if host[-1] != '/'
+        host
+      end
+
+      def get url, msg_if_error = DEFAULT_ERROR_MSG
+        url = auth_helper.api_url_for url
+
+        response = HTTParty.get url
+
+        exit_if_error(msg_if_error, response)
+
+        JSON.parse response.body
+      end
+
+      def post_json url, params, msg_if_error = DEFAULT_ERROR_MSG
+        options = {
+          body: params.to_json,
+          headers: {'Content-Type' => 'application/json'}
+        }
+
+        response = HTTParty.post("#{server_url}#{url}", options)
+
+        exit_if_error(msg_if_error, response)
+
+        JSON.parse response.body
+      end
+
+      private
+
+      def exit_if_error (message, response)
+        err_and_exit(message, response) if response_has_error?(response)
+      end
+
+      def err_and_exit(message, response)
+        STDERR.puts message
+        STDERR.puts "Error Response Code: #{response.code}"
+        STDERR.puts response.body
+        exit 1
+      end
+
+      def response_has_error?(response)
+        response.code != 200
+      end
+    end
+  end
+end

data/lib/holistics/helpers/job_manager.rb

@@ -0,0 +1,39 @@
+module Holistics
+  module Helpers
+    class JobManager
+
+      def auth_helper
+        @auth_info ||= Helpers::AuthInfo.new
+      end
+
+      def fetch_job_details(job_id, last_id = 0)
+        response = HTTParty.get(auth_helper.api_url_for("jobs/#{job_id}/logs.json", last_id: last_id))
+        JSON.parse(response.body)
+      end
+
+      def job_show options
+        job_id = options[:job_id]
+        tail_logs(job_id)
+      end
+
+      def tail_logs(job_id)
+        last_id = 0
+        while true
+          parsed = fetch_job_details(job_id, last_id)
+          logs = parsed['logs']
+          logs.each { |log| print_log(log) }
+          last_id = logs.last['id'] if logs.size > 0
+
+          break unless parsed['has_more']
+
+          sleep 0.5
+        end
+      end
+
+      def print_log log
+        ts = Time.parse(log['timestamp'])
+        Holistics.logger.log(log['level'], log['message'], timestamp: ts)
+      end
+    end
+  end
+end

data/lib/holistics/version.rb

@@ -1,3 +1,3 @@
 module Holistics
-  VERSION = '0.0.11'
+  VERSION = '0.1.0'
 end

data/lib/transport.rb

@@ -0,0 +1,33 @@
+require 'thor'
+
+module Holistics
+  class Transport < Thor
+
+    no_commands do
+      def api_client
+        @api_client ||= ApiClient.new
+      end
+    end
+
+    method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
+    method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
+    method_option :from_table_name, aliases: '-t', type: :string, required: false, desc: 'The table to copy over'
+    method_option :dest_table_name, aliases: '-n', type: :string, required: false, desc: '(optional) Rename destination table. Please specify fully qualified name'
+    method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
+    method_option :full, type: :boolean, default: false, required: false, desc: 'Full table transport'
+    method_option :incremental, type: :boolean, default: false, required: false, desc: 'Incremental table transport'
+    desc 'submit', 'Submit a data transport job'
+    def submit
+      api_client.send_transport(options.dup)
+    end
+
+    method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
+    method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
+    method_option :tables, aliases: '-t', type: :array, required: true, desc: 'List of tables to generate configs from. Wildcards are allowed'
+    method_option :output, aliases: '-o', type: :string, required: false, default: './', desc: 'Location where files will be generated to'
+    desc 'generate', 'Generate transport JSON configuration files'
+    def generate
+      api_client.generate_configs(options.dup)
+    end
+  end
+end

data/lib/vcr_helper.rb

@@ -0,0 +1,38 @@
+# This is a cucumber test helper.
+# It wrap the running of thor commands around VCR cassette if needed
+
+if ENV['IS_CUCUMBER']
+  ENV['HOME'] = File.join(ENV['GEM_ROOT_PATH'], 'tmp/aruba')
+end
+
+if ENV['VCR_CASSETTE']
+  require 'vcr'
+
+
+  require 'thor'
+  require 'active_support/core_ext/module'
+  class Thor::Task
+
+    def run_with_vcr(instance, args=[])
+      if ENV['VCR_CASSETTE']
+        cassette_name = ENV.delete('VCR_CASSETTE')
+
+        VCR.configure do |c|
+          c.cassette_library_dir = Holistics.root.join 'features/vcr_cassettes'
+          c.default_cassette_options = {:record => :new_episodes}
+          c.hook_into :webmock
+        end
+
+        VCR.use_cassette(cassette_name) do
+          run_without_vcr(instance, args)
+        end
+      else
+        run_without_vcr(instance, args)
+      end
+
+
+    end
+
+    alias_method_chain :run, :vcr
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: holistics
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.1.0
 platform: ruby
 authors:
 - Thanh Dinh Khac
@@ -79,12 +79,18 @@ files:
 - README.md
 - bin/holistics
 - holistics.gemspec
+- lib/data_sources.rb
 - lib/holistics.rb
 - lib/holistics/api_client.rb
+- lib/holistics/auth_api_client.rb
 - lib/holistics/custom_logger.rb
+- lib/holistics/helpers/auth_info.rb
+- lib/holistics/helpers/http_request.rb
+- lib/holistics/helpers/job_manager.rb
 - lib/holistics/version.rb
 - lib/tabular_formatter.rb
-- lib/vcr.rb
+- lib/transport.rb
+- lib/vcr_helper.rb
 homepage: http://rubygems.org/gems/holistics-cli
 licenses:
 - GPL

data/lib/vcr.rb

@@ -1,26 +0,0 @@
-# This is a cucumber test helper.
-# It wrap the running of thor commands around VCR cassette if needed
-
-if ENV['IS_CUCUMBER']
-  ENV['HOME'] = File.join(ENV['GEM_ROOT_PATH'], 'tmp/aruba')
-end
-
-if ENV['VCR_CASSETTE']
-  require 'vcr'
-  require 'thor'
-  class Thor::Task
-    def run_with_vcr(instance, args=[])
-      cassette = ENV.delete('VCR_CASSETTE')
-      VCR.configure do |c|
-        c.cassette_library_dir = Holistics.root.join 'features/vcr_cassettes'
-        c.default_cassette_options = {:record => :new_episodes}
-        c.hook_into :webmock
-      end
-      VCR.use_cassette(cassette) do
-        run_without_vcr(instance, args)
-      end
-    end
-
-    alias_method_chain :run, :vcr
-  end
-end