holistics 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2196254a9a2d43fa37feb6c350bbbd6fa4e9cdd6
-  data.tar.gz: 4731a3d139b7c0d3148892151a198749b8fa3fda
+  metadata.gz: 26e77b302741a732ef4e316c0ec9c1625437e42f
+  data.tar.gz: c194bbd6d10e117bb73326773cc0fff889242b74
 SHA512:
-  metadata.gz: e2b13abef29da4b65c3f9ae6268a573e7015c8e5bbaade0b91a6baa354c32727f6707be3acc2f2da6349703a3504b29a59b233e4d3571305facd949b3c551bf5
-  data.tar.gz: c267f3d5a577f35eaa7a673a5ec5c524dfea69e3301dd1cd533f1b23d529e3ba6c4cef46bca3baf8f4aa0dbf539ddcac5eea376e23823a6bea5932c55c4a4f80
+  metadata.gz: c58d2d82458634296a4654474c6532314cbfcfac8fcd070b4a392f762c3d592df1445c735c1e6471c37b1f8ed602629797a0599bc5eb86e1ae08d80219e0154d
+  data.tar.gz: 8db5b264195102c9df96c86042538f92dddb01e0a850c26786f80be68931ca982e1bb6a7a0fac9a47ead34a9620dbddded6c45e0656c55132e9920bd585a8358

data/README.md

@@ -53,13 +53,13 @@ Then perform authentication:
     | 1  | PostgreSQL | Production DB |
     | 2  | Redshift   | Analytics DB  |
 
-### Transport data from Redshift to Postgres:
+### Perform a data transport
  
-    $ holistics rs_to_pg -s <source_id> -d <dest_id> -t <table_name>
+    $ holistics transport -from-ds-id <from_data_source_id> -dest-ds-id <dest_data_source_id> --from-table-name <table_name>
 
 Example:
 
-    $ holistics rs_to_pg -s 1 -d 2 -t public.users
+    $ holistics transport -s 1 -d 2 -t public.users
     Submitting transport job ...
     Job submitted. Job ID: 738.
     [job:738] Status: queued
@@ -79,5 +79,5 @@ Example:
 
 See `samples/clicks_mysql_to_redshift.json` for details of transport configs.
 
-    $ holistics mysql_to_redshift -c samples/clicks_mysql_to_redshift.json
+    $ holistics transport -c samples/clicks_mysql_to_redshift.json
 

data/bin/holistics

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-ENV['ROOT_PATH'] = File.expand_path('../../', __FILE__)
+ENV['GEM_ROOT_PATH'] = File.expand_path('../../', __FILE__)
 
 require 'holistics'
 HolisticsRunner.start

data/lib/holistics.rb

@@ -20,66 +20,25 @@ class HolisticsRunner < Thor
     client.login(token)
   end
 
+
   desc 'ds_list', 'List all data sources'
 
   def ds_list
     ApiClient.new.ds_list
   end
 
-  method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
-  method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
-  method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
-  method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
-  method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
-  desc 'redshift_to_postgres', 'Transport data from Amazon Redshift data source to PostgreSQL'
-
-  def redshift_to_postgres
-    ApiClient.new.send_transport(:redshift, :postgresql, options.dup)
-  end
-
-  method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
-  method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
-  method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
-  method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
-  method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
-  desc 'postgres_to_redshift', 'Transport data from PostgreSQL to Amazon Redshift'
-
-  def postgres_to_redshift
-    ApiClient.new.send_transport(:postgresql, :redshift, options.dup)
-  end
 
   method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
   method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
-  method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
-  method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
+  method_option :from_table_name, aliases: '-t', type: :string, required: false, desc: 'The table to copy over'
+  method_option :dest_table_name, aliases: '-n', type: :string, required: false, desc: '(optional) Rename destination table. Please specify fully qualified name'
   method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
-  desc 'postgres_to_postgres', 'Transport data from PostgreSQL to PostgreSQL'
+  desc 'transport', 'Submit a data transport job'
 
-  def postgres_to_postgres
-    ApiClient.new.send_transport(:postgresql, :postgresql, options.dup)
+  def transport
+    ApiClient.new.send_transport(options.dup)
   end
 
-  method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
-  method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
-  method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
-  method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
-  method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
-  desc 'mysql_to_postgres', 'Transport data from MySQL to Postgres'
-
-  def mysql_to_postgres
-    ApiClient.new.send_transport(:mysql, :postgresql, options.dup)
-  end
-
-  method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
-  method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
-  method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
-  method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
-  method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
-  desc 'mysql_to_redshift', 'Transport data from MySQL to Redshift'
-
-  def mysql_to_redshift
-    ApiClient.new.send_transport(:mysql, :redshift, options.dup)
-  end
 
   method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
   desc 'job_show', 'Show job log'

data/lib/holistics/api_client.rb

@@ -46,9 +46,9 @@ class ApiClient
     puts TabularFormatter.new(table).to_pretty_table
   end
 
-  def send_transport(from_ds_type, dest_ds_type, options)
+  def send_transport(options)
     puts 'Submitting transport job ...'
-    params = build_submit_params(dest_ds_type, from_ds_type, options)
+    params = build_submit_params(options)
     response = submit_transport_job(params)
 
     err_and_exit("Error submitting transport job", response) if response_has_error?(response)
@@ -91,7 +91,7 @@ class ApiClient
 
   def server_url
     host =
-      if ENV['DEV']
+      if ENV['DEV'] || ENV['TEST']
         'http://localhost:3000'
       elsif ENV['STAGING']
         'https://staging.holistics.io'
@@ -104,18 +104,9 @@ class ApiClient
     host
   end
 
-  def build_submit_params(dest_ds_type, from_ds_type, options)
-    params = options.except(:config_path).merge(from_ds_type: from_ds_type, dest_ds_type: dest_ds_type, _utoken: get_token_from_gconfig)
-
-    configs = {}
-    if options[:config_path]
-      configs = parse_transport_config(options[:config_path])
-    end
-
-    configs['from_table_name'] = options[:table_name] if options[:table_name]
-    configs['dest_table_name'] = options[:rename] if options[:rename]
-
-    params[:configs] = configs.to_json # should be a string
+  def build_submit_params(options)
+    params = options.except(:config_path).merge(_utoken: get_token_from_gconfig)
+    params[:configs] = parse_transport_config(options[:config_path]).to_json if options[:config_path]
     params
   end
 

data/lib/holistics/cucumber/vcr.rb

@@ -2,7 +2,7 @@
 # It wrap the running of thor commands around VCR cassette if needed
 
 if ENV['IS_CUCUMBER']
-  ENV['HOME'] = File.join(ENV['ROOT_PATH'], 'tmp/aruba')
+  ENV['HOME'] = File.join(ENV['GEM_ROOT_PATH'], 'tmp/aruba')
 end
 
 if ENV['VCR_CASSETTE']

data/lib/holistics/version.rb

@@ -1,3 +1,3 @@
 class Holistics
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: holistics
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Thanh Dinh Khac