holistics 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -4
- data/bin/holistics +1 -1
- data/lib/holistics.rb +6 -47
- data/lib/holistics/api_client.rb +6 -15
- data/lib/holistics/cucumber/vcr.rb +1 -1
- data/lib/holistics/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 26e77b302741a732ef4e316c0ec9c1625437e42f
|
4
|
+
data.tar.gz: c194bbd6d10e117bb73326773cc0fff889242b74
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c58d2d82458634296a4654474c6532314cbfcfac8fcd070b4a392f762c3d592df1445c735c1e6471c37b1f8ed602629797a0599bc5eb86e1ae08d80219e0154d
|
7
|
+
data.tar.gz: 8db5b264195102c9df96c86042538f92dddb01e0a850c26786f80be68931ca982e1bb6a7a0fac9a47ead34a9620dbddded6c45e0656c55132e9920bd585a8358
|
data/README.md
CHANGED
@@ -53,13 +53,13 @@ Then perform authentication:
|
|
53
53
|
| 1 | PostgreSQL | Production DB |
|
54
54
|
| 2 | Redshift | Analytics DB |
|
55
55
|
|
56
|
-
###
|
56
|
+
### Perform a data transport
|
57
57
|
|
58
|
-
$ holistics
|
58
|
+
$ holistics transport -from-ds-id <from_data_source_id> -dest-ds-id <dest_data_source_id> --from-table-name <table_name>
|
59
59
|
|
60
60
|
Example:
|
61
61
|
|
62
|
-
$ holistics
|
62
|
+
$ holistics transport -s 1 -d 2 -t public.users
|
63
63
|
Submitting transport job ...
|
64
64
|
Job submitted. Job ID: 738.
|
65
65
|
[job:738] Status: queued
|
@@ -79,5 +79,5 @@ Example:
|
|
79
79
|
|
80
80
|
See `samples/clicks_mysql_to_redshift.json` for details of transport configs.
|
81
81
|
|
82
|
-
$ holistics
|
82
|
+
$ holistics transport -c samples/clicks_mysql_to_redshift.json
|
83
83
|
|
data/bin/holistics
CHANGED
data/lib/holistics.rb
CHANGED
@@ -20,66 +20,25 @@ class HolisticsRunner < Thor
|
|
20
20
|
client.login(token)
|
21
21
|
end
|
22
22
|
|
23
|
+
|
23
24
|
desc 'ds_list', 'List all data sources'
|
24
25
|
|
25
26
|
def ds_list
|
26
27
|
ApiClient.new.ds_list
|
27
28
|
end
|
28
29
|
|
29
|
-
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
30
|
-
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
31
|
-
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
32
|
-
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
33
|
-
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
34
|
-
desc 'redshift_to_postgres', 'Transport data from Amazon Redshift data source to PostgreSQL'
|
35
|
-
|
36
|
-
def redshift_to_postgres
|
37
|
-
ApiClient.new.send_transport(:redshift, :postgresql, options.dup)
|
38
|
-
end
|
39
|
-
|
40
|
-
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
41
|
-
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
42
|
-
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
43
|
-
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
44
|
-
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
45
|
-
desc 'postgres_to_redshift', 'Transport data from PostgreSQL to Amazon Redshift'
|
46
|
-
|
47
|
-
def postgres_to_redshift
|
48
|
-
ApiClient.new.send_transport(:postgresql, :redshift, options.dup)
|
49
|
-
end
|
50
30
|
|
51
31
|
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
52
32
|
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
53
|
-
method_option :
|
54
|
-
method_option :
|
33
|
+
method_option :from_table_name, aliases: '-t', type: :string, required: false, desc: 'The table to copy over'
|
34
|
+
method_option :dest_table_name, aliases: '-n', type: :string, required: false, desc: '(optional) Rename destination table. Please specify fully qualified name'
|
55
35
|
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
56
|
-
desc '
|
36
|
+
desc 'transport', 'Submit a data transport job'
|
57
37
|
|
58
|
-
def
|
59
|
-
ApiClient.new.send_transport(
|
38
|
+
def transport
|
39
|
+
ApiClient.new.send_transport(options.dup)
|
60
40
|
end
|
61
41
|
|
62
|
-
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
63
|
-
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
64
|
-
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
65
|
-
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
66
|
-
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
67
|
-
desc 'mysql_to_postgres', 'Transport data from MySQL to Postgres'
|
68
|
-
|
69
|
-
def mysql_to_postgres
|
70
|
-
ApiClient.new.send_transport(:mysql, :postgresql, options.dup)
|
71
|
-
end
|
72
|
-
|
73
|
-
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
74
|
-
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
75
|
-
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
76
|
-
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
77
|
-
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
78
|
-
desc 'mysql_to_redshift', 'Transport data from MySQL to Redshift'
|
79
|
-
|
80
|
-
def mysql_to_redshift
|
81
|
-
ApiClient.new.send_transport(:mysql, :redshift, options.dup)
|
82
|
-
end
|
83
42
|
|
84
43
|
method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
|
85
44
|
desc 'job_show', 'Show job log'
|
data/lib/holistics/api_client.rb
CHANGED
@@ -46,9 +46,9 @@ class ApiClient
|
|
46
46
|
puts TabularFormatter.new(table).to_pretty_table
|
47
47
|
end
|
48
48
|
|
49
|
-
def send_transport(
|
49
|
+
def send_transport(options)
|
50
50
|
puts 'Submitting transport job ...'
|
51
|
-
params = build_submit_params(
|
51
|
+
params = build_submit_params(options)
|
52
52
|
response = submit_transport_job(params)
|
53
53
|
|
54
54
|
err_and_exit("Error submitting transport job", response) if response_has_error?(response)
|
@@ -91,7 +91,7 @@ class ApiClient
|
|
91
91
|
|
92
92
|
def server_url
|
93
93
|
host =
|
94
|
-
if ENV['DEV']
|
94
|
+
if ENV['DEV'] || ENV['TEST']
|
95
95
|
'http://localhost:3000'
|
96
96
|
elsif ENV['STAGING']
|
97
97
|
'https://staging.holistics.io'
|
@@ -104,18 +104,9 @@ class ApiClient
|
|
104
104
|
host
|
105
105
|
end
|
106
106
|
|
107
|
-
def build_submit_params(
|
108
|
-
params = options.except(:config_path).merge(
|
109
|
-
|
110
|
-
configs = {}
|
111
|
-
if options[:config_path]
|
112
|
-
configs = parse_transport_config(options[:config_path])
|
113
|
-
end
|
114
|
-
|
115
|
-
configs['from_table_name'] = options[:table_name] if options[:table_name]
|
116
|
-
configs['dest_table_name'] = options[:rename] if options[:rename]
|
117
|
-
|
118
|
-
params[:configs] = configs.to_json # should be a string
|
107
|
+
def build_submit_params(options)
|
108
|
+
params = options.except(:config_path).merge(_utoken: get_token_from_gconfig)
|
109
|
+
params[:configs] = parse_transport_config(options[:config_path]).to_json if options[:config_path]
|
119
110
|
params
|
120
111
|
end
|
121
112
|
|
data/lib/holistics/version.rb
CHANGED