holistics 0.0.11 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/lib/data_sources.rb +17 -0
- data/lib/holistics.rb +43 -12
- data/lib/holistics/api_client.rb +35 -130
- data/lib/holistics/auth_api_client.rb +38 -0
- data/lib/holistics/helpers/auth_info.rb +34 -0
- data/lib/holistics/helpers/http_request.rb +70 -0
- data/lib/holistics/helpers/job_manager.rb +39 -0
- data/lib/holistics/version.rb +1 -1
- data/lib/transport.rb +33 -0
- data/lib/vcr_helper.rb +38 -0
- metadata +8 -2
- data/lib/vcr.rb +0 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ffcfdb6aa3257f5a42c021a116e428316ee4d02d
|
4
|
+
data.tar.gz: 4aa5954b9ab8d2157284d64570e0c085f0ff9f3b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 297dffd7ca7867cd1deb9907196768cb1a533254bc057f41d103d09acf93da127cf3704a72e6a5e9078becdbda830da0250cc720c49d84c8ebd90363124d7577
|
7
|
+
data.tar.gz: 56a782a5815716bd7ac88e4680ffb526d09c03b95ed8ded9990f2070e28d8033ab13df84d100386f6a555eb24bb799f762dc67c945dd49caec3823e6583b5520
|
data/CHANGELOG.md
CHANGED
@@ -4,3 +4,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
|
|
4
4
|
|
5
5
|
## v0.0.11
|
6
6
|
* Support invocation of transform from CLI: `holistics transform -j <id>`
|
7
|
+
|
8
|
+
## v0.0.12
|
9
|
+
* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
|
10
|
+
|
11
|
+
## v0.1.0
|
12
|
+
* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
|
data/lib/data_sources.rb
ADDED
data/lib/holistics.rb
CHANGED
@@ -1,7 +1,12 @@
|
|
1
1
|
require 'thor'
|
2
|
+
require 'vcr_helper'
|
3
|
+
|
2
4
|
require 'holistics/api_client'
|
3
5
|
require 'holistics/custom_logger'
|
4
|
-
require '
|
6
|
+
require 'holistics/auth_api_client'
|
7
|
+
|
8
|
+
require 'transport'
|
9
|
+
require 'data_sources'
|
5
10
|
|
6
11
|
module Holistics
|
7
12
|
def self.root
|
@@ -26,18 +31,30 @@ module Holistics
|
|
26
31
|
super(args, options, config)
|
27
32
|
end
|
28
33
|
|
34
|
+
register(Holistics::Transport, 'transport', 'transport <command>', "Execute transport module's commands")
|
35
|
+
register(Holistics::DataSources, 'data_sources', 'data_sources <command>', "Execute data_sources module's commands")
|
36
|
+
|
37
|
+
no_commands do
|
38
|
+
def auth_api_client
|
39
|
+
@auth_api_client ||= AuthApiClient.new
|
40
|
+
end
|
41
|
+
|
42
|
+
def api_client
|
43
|
+
@api_client ||= ApiClient.new
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
29
47
|
desc 'login [auth_token]', 'Perform authentication'
|
30
48
|
|
31
49
|
def login token
|
32
|
-
|
33
|
-
client.login(token)
|
50
|
+
auth_api_client.login(token)
|
34
51
|
end
|
35
52
|
|
36
|
-
|
37
|
-
desc 'ds_list', 'List all data sources'
|
53
|
+
desc 'ds_list', '[DEPRECATED] List all data sources'
|
38
54
|
|
39
55
|
def ds_list
|
40
|
-
|
56
|
+
p 'DEPRECATED. This command will be removed in the next version. Please consider using `holistics data_sources list` instead.'
|
57
|
+
api_client.ds_list
|
41
58
|
end
|
42
59
|
|
43
60
|
|
@@ -46,19 +63,33 @@ module Holistics
|
|
46
63
|
method_option :from_table_name, aliases: '-t', type: :string, required: false, desc: 'The table to copy over'
|
47
64
|
method_option :dest_table_name, aliases: '-n', type: :string, required: false, desc: '(optional) Rename destination table. Please specify fully qualified name'
|
48
65
|
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
49
|
-
method_option :full, type: :boolean, default:
|
66
|
+
method_option :full, type: :boolean, default: false, required: false, desc: 'Full table transport'
|
50
67
|
method_option :incremental, type: :boolean, default: false, required: false, desc: 'Incremental table transport'
|
51
|
-
desc '
|
68
|
+
desc 'transport_old', '[DEPRECATED] Submit a data transport job'
|
52
69
|
|
53
|
-
def
|
54
|
-
|
70
|
+
def transport_old
|
71
|
+
p 'DEPRECATED. This command will be removed in the next version. Please consider using `holistics transport submit` instead.'
|
72
|
+
api_client.send_transport(options.dup)
|
55
73
|
end
|
56
74
|
|
75
|
+
|
57
76
|
method_option :transform_id, aliases: '-j', type: :string, required: true, desc: 'ID of transform job to be executed'
|
58
77
|
desc 'transform', 'Invoke a transform job'
|
59
78
|
|
60
79
|
def transform
|
61
|
-
|
80
|
+
api_client.send_transform(options.dup)
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
|
85
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
|
86
|
+
method_option :tables, aliases: '-t', type: :array, required: true, desc: 'List of tables to generate configs from. Wildcards are allowed'
|
87
|
+
method_option :output, aliases: '-o', type: :string, required: false, default: './', desc: 'Location where files will be generated to'
|
88
|
+
desc 'generate_configs', '[DEPRECATED] Generate transport JSON configuration files'
|
89
|
+
|
90
|
+
def generate_configs
|
91
|
+
p 'DEPRECATED. This command will be removed in the next version. Please consider using `holistics transport generate` instead.'
|
92
|
+
api_client.generate_configs(options.dup)
|
62
93
|
end
|
63
94
|
|
64
95
|
|
@@ -66,7 +97,7 @@ module Holistics
|
|
66
97
|
desc 'job_show', 'Show job log'
|
67
98
|
|
68
99
|
def job_show
|
69
|
-
|
100
|
+
api_client.job_show(options.dup)
|
70
101
|
end
|
71
102
|
|
72
103
|
end
|
data/lib/holistics/api_client.rb
CHANGED
@@ -3,49 +3,29 @@ require 'tabular_formatter'
|
|
3
3
|
require 'yaml'
|
4
4
|
require 'json'
|
5
5
|
|
6
|
+
require 'holistics/helpers/http_request'
|
7
|
+
require 'holistics/helpers/job_manager'
|
8
|
+
|
6
9
|
module Holistics
|
7
10
|
class ApiClient
|
8
|
-
SERVER_URL = 'https://secure.holistics.io/'
|
9
|
-
|
10
|
-
def initialize
|
11
|
-
|
12
|
-
end
|
13
|
-
|
14
|
-
def login token
|
15
|
-
puts 'Logging in...'
|
16
|
-
response, ok = holistics_authenticate(token)
|
17
|
-
if ok
|
18
|
-
parsed = JSON.parse(response.body)
|
19
|
-
puts 'Authentication successful. Info:'
|
20
|
-
puts "- ID: #{parsed['id']}"
|
21
|
-
puts "- Email: #{parsed['email']}"
|
22
11
|
|
23
|
-
|
24
|
-
|
25
|
-
puts 'Error logging in. Please check your token again.'
|
26
|
-
end
|
12
|
+
def job_manager
|
13
|
+
@job_helper ||= Helpers::JobManager.new
|
27
14
|
end
|
28
15
|
|
29
|
-
def
|
30
|
-
|
31
|
-
response = HTTParty.get(url)
|
32
|
-
return response, (response.code == 200)
|
16
|
+
def http_request
|
17
|
+
@http_helper ||= Helpers::HttpRequest.new
|
33
18
|
end
|
34
19
|
|
35
|
-
def
|
36
|
-
|
37
|
-
tail_logs(job_id)
|
20
|
+
def auth_info
|
21
|
+
@auth_info ||= Helpers::AuthInfo.new
|
38
22
|
end
|
39
23
|
|
40
24
|
def ds_list
|
41
|
-
|
42
|
-
response = HTTParty.get(url)
|
43
|
-
err_and_exit('Error retrieving list of data sources', response) if response_has_error?(response)
|
44
|
-
|
45
|
-
parsed = JSON.parse(response.body)
|
25
|
+
result = http_request.get 'data_sources.json', 'Error retrieving list of data sources'
|
46
26
|
|
47
27
|
table = [%w(ID Type Name)]
|
48
|
-
rows =
|
28
|
+
rows = result.map { |record| [record['id'], record['dbtype'], record['name']] }
|
49
29
|
table.concat(rows)
|
50
30
|
|
51
31
|
puts TabularFormatter.new(table).to_pretty_table
|
@@ -53,84 +33,53 @@ module Holistics
|
|
53
33
|
|
54
34
|
def send_transport(options)
|
55
35
|
puts 'Submitting transport job ...'
|
36
|
+
|
56
37
|
params = build_submit_params(options)
|
57
|
-
response = submit_transport_job(params)
|
58
38
|
|
59
|
-
|
39
|
+
result = http_request.post_json 'transports/submit.json', params, 'Error submitting transport job'
|
60
40
|
|
61
|
-
|
62
|
-
job_id = parsed['job_id']
|
41
|
+
job_id = result['job_id']
|
63
42
|
|
64
43
|
puts "Job submitted. Job ID: #{job_id}."
|
65
|
-
tail_logs
|
44
|
+
job_manager.tail_logs job_id
|
66
45
|
end
|
67
46
|
|
68
47
|
def send_transform(options)
|
69
48
|
puts 'Invoking transform job...'
|
70
|
-
params = options.merge(_utoken: get_token_from_gconfig)
|
71
|
-
response = submit_transform_job(params)
|
72
49
|
|
73
|
-
|
50
|
+
params = options.merge(_utoken: auth_info.get_token_from_gconfig)
|
51
|
+
result = http_request.post_json "data_transforms/#{params[:transform_id]}/execute.json", params, 'Error submitting transform job'
|
74
52
|
|
75
|
-
|
76
|
-
job_id = parsed['job_id']
|
53
|
+
job_id = result['job_id']
|
77
54
|
|
78
55
|
puts "Job submitted. Job ID: #{job_id}."
|
79
|
-
tail_logs
|
56
|
+
job_manager.tail_logs job_id
|
80
57
|
end
|
81
58
|
|
82
|
-
def
|
83
|
-
|
84
|
-
|
85
|
-
parsed = fetch_job_details(job_id, last_id)
|
86
|
-
logs = parsed['logs']
|
87
|
-
logs.each { |log| print_log(log) }
|
88
|
-
last_id = logs.last['id'] if logs.size > 0
|
89
|
-
|
90
|
-
break unless parsed['has_more']
|
91
|
-
|
92
|
-
sleep 0.5
|
59
|
+
def generate_configs(options)
|
60
|
+
unless Dir.exist?(options['output'])
|
61
|
+
raise 'Output location is invalid.'
|
93
62
|
end
|
94
|
-
end
|
95
63
|
|
96
|
-
|
97
|
-
response = HTTParty.get(api_url_for("jobs/#{job_id}/logs.json", last_id: last_id))
|
98
|
-
JSON.parse(response.body)
|
99
|
-
end
|
64
|
+
puts "Generating transport JSON config files to #{options['output']} directory..."
|
100
65
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
66
|
+
params = options.merge(_utoken: auth_info.get_token_from_gconfig)
|
67
|
+
result = http_request.post_json 'transports/generate_configs.json', params, 'Error generating transport configs'
|
68
|
+
|
69
|
+
result.each do |table_data|
|
70
|
+
File.open("#{options['output']}/#{table_data['filename']}","w") do |f|
|
71
|
+
f.write(JSON.pretty_generate(table_data['json_content']))
|
72
|
+
end
|
108
73
|
|
109
|
-
|
110
|
-
|
111
|
-
headers: {'Content-Type' => 'application/json'},
|
112
|
-
body: params.to_json
|
113
|
-
}
|
114
|
-
HTTParty.post(server_url + "data_transforms/#{params[:transform_id]}/execute.json", options)
|
115
|
-
end
|
74
|
+
puts "Generated #{table_data['filename']}"
|
75
|
+
end
|
116
76
|
|
117
|
-
|
118
|
-
|
119
|
-
if ENV['HOLISTICS_DEV'] || ENV['HOLISTICS_TEST']
|
120
|
-
'http://localhost:3000'
|
121
|
-
elsif ENV['HOLISTICS_STAGING']
|
122
|
-
'https://staging.holistics.io'
|
123
|
-
elsif ENV['HOLISTICS_HOST']
|
124
|
-
ENV['HOLISTICS_HOST']
|
125
|
-
else
|
126
|
-
SERVER_URL
|
127
|
-
end
|
128
|
-
host += '/' if host[-1] != '/'
|
129
|
-
host
|
77
|
+
puts
|
78
|
+
puts "Configs generation succeeded with #{result.length} files in total."
|
130
79
|
end
|
131
80
|
|
132
81
|
def build_submit_params(options)
|
133
|
-
params = options.except(:config_path).merge(_utoken: get_token_from_gconfig)
|
82
|
+
params = options.except(:config_path).merge(_utoken: auth_info.get_token_from_gconfig)
|
134
83
|
params[:configs] = parse_transport_config(options[:config_path]).to_json if options[:config_path]
|
135
84
|
params
|
136
85
|
end
|
@@ -148,49 +97,5 @@ module Holistics
|
|
148
97
|
STDERR.puts "Error parsing transport config file: #{e.message}"
|
149
98
|
exit 1
|
150
99
|
end
|
151
|
-
|
152
|
-
def authenticated?
|
153
|
-
File.exists?(get_gconfig_filepath)
|
154
|
-
end
|
155
|
-
|
156
|
-
def get_token_from_gconfig
|
157
|
-
if authenticated?
|
158
|
-
string = YAML.load_file(get_gconfig_filepath)
|
159
|
-
string['token']
|
160
|
-
else
|
161
|
-
raise StandardError.new 'Holistics config file not found'
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
def get_gconfig_filepath
|
166
|
-
File.expand_path('~/.holistics.yml', __FILE__)
|
167
|
-
end
|
168
|
-
|
169
|
-
def api_url_for(path, params = {}, token = nil)
|
170
|
-
params[:_utoken] = token || get_token_from_gconfig
|
171
|
-
"#{server_url}#{path}?#{URI.encode_www_form(params)}"
|
172
|
-
end
|
173
|
-
|
174
|
-
def print_log log
|
175
|
-
ts = Time.parse(log['timestamp'])
|
176
|
-
Holistics.logger.log(log['level'], log['message'], timestamp: ts)
|
177
|
-
end
|
178
|
-
|
179
|
-
def err_and_exit(message, response)
|
180
|
-
STDERR.puts message
|
181
|
-
STDERR.puts "Error Response Code: #{response.code}"
|
182
|
-
STDERR.puts response.body
|
183
|
-
exit 1
|
184
|
-
end
|
185
|
-
|
186
|
-
def response_has_error?(response)
|
187
|
-
response.code != 200
|
188
|
-
end
|
189
|
-
|
190
|
-
def write_token_to_gconfig(token)
|
191
|
-
file_path = File.join(ENV['HOME'], '.holistics.yml')
|
192
|
-
h = {'token' => token}
|
193
|
-
File.write(file_path, h.to_yaml)
|
194
|
-
end
|
195
100
|
end
|
196
101
|
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
require 'holistics/helpers/auth_info'
|
3
|
+
|
4
|
+
module Holistics
|
5
|
+
class AuthApiClient
|
6
|
+
|
7
|
+
def auth_info
|
8
|
+
@auth_info ||= Helpers::AuthInfo.new
|
9
|
+
end
|
10
|
+
|
11
|
+
def login token
|
12
|
+
puts 'Logging in...'
|
13
|
+
response, ok = authenticate(token)
|
14
|
+
if ok
|
15
|
+
parsed = JSON.parse(response.body)
|
16
|
+
puts 'Authentication successful. Info:'
|
17
|
+
puts "- ID: #{parsed['id']}"
|
18
|
+
puts "- Email: #{parsed['email']}"
|
19
|
+
|
20
|
+
write_token_to_gconfig(token)
|
21
|
+
else
|
22
|
+
puts 'Error logging in. Please check your token again.'
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def authenticate(token)
|
27
|
+
url = auth_info.api_url_for('users/info.json', {}, token)
|
28
|
+
response = HTTParty.get(url)
|
29
|
+
return response, (response.code == 200)
|
30
|
+
end
|
31
|
+
|
32
|
+
def write_token_to_gconfig(token)
|
33
|
+
file_path = File.join(ENV['HOME'], '.holistics.yml')
|
34
|
+
h = {'token' => token}
|
35
|
+
File.write(file_path, h.to_yaml)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'holistics/helpers/http_request'
|
2
|
+
|
3
|
+
module Holistics
|
4
|
+
module Helpers
|
5
|
+
class AuthInfo
|
6
|
+
|
7
|
+
def http_request
|
8
|
+
@http_helper ||= Helpers::HttpRequest.new
|
9
|
+
end
|
10
|
+
|
11
|
+
def authenticated?
|
12
|
+
File.exists?(get_gconfig_filepath)
|
13
|
+
end
|
14
|
+
|
15
|
+
def api_url_for(path, params = {}, token = nil)
|
16
|
+
params[:_utoken] = token || get_token_from_gconfig
|
17
|
+
"#{http_request.server_url}#{path}?#{URI.encode_www_form(params)}"
|
18
|
+
end
|
19
|
+
|
20
|
+
def get_token_from_gconfig
|
21
|
+
if authenticated?
|
22
|
+
string = YAML.load_file(get_gconfig_filepath)
|
23
|
+
string['token']
|
24
|
+
else
|
25
|
+
raise StandardError.new 'Holistics config file not found'
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def get_gconfig_filepath
|
30
|
+
File.expand_path('~/.holistics.yml', __FILE__)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
|
3
|
+
module Holistics
|
4
|
+
module Helpers
|
5
|
+
class HttpRequest
|
6
|
+
|
7
|
+
DEFAULT_ERROR_MSG = 'Error occurred!'
|
8
|
+
SERVER_URL = 'https://secure.holistics.io/'
|
9
|
+
|
10
|
+
def auth_helper
|
11
|
+
@auth_info ||= Helpers::AuthInfo.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def server_url
|
15
|
+
host =
|
16
|
+
if ENV['HOLISTICS_DEV'] || ENV['HOLISTICS_TEST']
|
17
|
+
'http://localhost:3000'
|
18
|
+
elsif ENV['HOLISTICS_STAGING']
|
19
|
+
'https://staging.holistics.io'
|
20
|
+
elsif ENV['HOLISTICS_HOST']
|
21
|
+
ENV['HOLISTICS_HOST']
|
22
|
+
else
|
23
|
+
SERVER_URL
|
24
|
+
end
|
25
|
+
host += '/' if host[-1] != '/'
|
26
|
+
host
|
27
|
+
end
|
28
|
+
|
29
|
+
def get url, msg_if_error = DEFAULT_ERROR_MSG
|
30
|
+
url = auth_helper.api_url_for url
|
31
|
+
|
32
|
+
response = HTTParty.get url
|
33
|
+
|
34
|
+
exit_if_error(msg_if_error, response)
|
35
|
+
|
36
|
+
JSON.parse response.body
|
37
|
+
end
|
38
|
+
|
39
|
+
def post_json url, params, msg_if_error = DEFAULT_ERROR_MSG
|
40
|
+
options = {
|
41
|
+
body: params.to_json,
|
42
|
+
headers: {'Content-Type' => 'application/json'}
|
43
|
+
}
|
44
|
+
|
45
|
+
response = HTTParty.post("#{server_url}#{url}", options)
|
46
|
+
|
47
|
+
exit_if_error(msg_if_error, response)
|
48
|
+
|
49
|
+
JSON.parse response.body
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def exit_if_error (message, response)
|
55
|
+
err_and_exit(message, response) if response_has_error?(response)
|
56
|
+
end
|
57
|
+
|
58
|
+
def err_and_exit(message, response)
|
59
|
+
STDERR.puts message
|
60
|
+
STDERR.puts "Error Response Code: #{response.code}"
|
61
|
+
STDERR.puts response.body
|
62
|
+
exit 1
|
63
|
+
end
|
64
|
+
|
65
|
+
def response_has_error?(response)
|
66
|
+
response.code != 200
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Holistics
|
2
|
+
module Helpers
|
3
|
+
class JobManager
|
4
|
+
|
5
|
+
def auth_helper
|
6
|
+
@auth_info ||= Helpers::AuthInfo.new
|
7
|
+
end
|
8
|
+
|
9
|
+
def fetch_job_details(job_id, last_id = 0)
|
10
|
+
response = HTTParty.get(auth_helper.api_url_for("jobs/#{job_id}/logs.json", last_id: last_id))
|
11
|
+
JSON.parse(response.body)
|
12
|
+
end
|
13
|
+
|
14
|
+
def job_show options
|
15
|
+
job_id = options[:job_id]
|
16
|
+
tail_logs(job_id)
|
17
|
+
end
|
18
|
+
|
19
|
+
def tail_logs(job_id)
|
20
|
+
last_id = 0
|
21
|
+
while true
|
22
|
+
parsed = fetch_job_details(job_id, last_id)
|
23
|
+
logs = parsed['logs']
|
24
|
+
logs.each { |log| print_log(log) }
|
25
|
+
last_id = logs.last['id'] if logs.size > 0
|
26
|
+
|
27
|
+
break unless parsed['has_more']
|
28
|
+
|
29
|
+
sleep 0.5
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def print_log log
|
34
|
+
ts = Time.parse(log['timestamp'])
|
35
|
+
Holistics.logger.log(log['level'], log['message'], timestamp: ts)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/holistics/version.rb
CHANGED
data/lib/transport.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'thor'
|
2
|
+
|
3
|
+
module Holistics
|
4
|
+
class Transport < Thor
|
5
|
+
|
6
|
+
no_commands do
|
7
|
+
def api_client
|
8
|
+
@api_client ||= ApiClient.new
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
13
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
14
|
+
method_option :from_table_name, aliases: '-t', type: :string, required: false, desc: 'The table to copy over'
|
15
|
+
method_option :dest_table_name, aliases: '-n', type: :string, required: false, desc: '(optional) Rename destination table. Please specify fully qualified name'
|
16
|
+
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
17
|
+
method_option :full, type: :boolean, default: false, required: false, desc: 'Full table transport'
|
18
|
+
method_option :incremental, type: :boolean, default: false, required: false, desc: 'Incremental table transport'
|
19
|
+
desc 'submit', 'Submit a data transport job'
|
20
|
+
def submit
|
21
|
+
api_client.send_transport(options.dup)
|
22
|
+
end
|
23
|
+
|
24
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
|
25
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
|
26
|
+
method_option :tables, aliases: '-t', type: :array, required: true, desc: 'List of tables to generate configs from. Wildcards are allowed'
|
27
|
+
method_option :output, aliases: '-o', type: :string, required: false, default: './', desc: 'Location where files will be generated to'
|
28
|
+
desc 'generate', 'Generate transport JSON configuration files'
|
29
|
+
def generate
|
30
|
+
api_client.generate_configs(options.dup)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/vcr_helper.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
# This is a cucumber test helper.
|
2
|
+
# It wrap the running of thor commands around VCR cassette if needed
|
3
|
+
|
4
|
+
if ENV['IS_CUCUMBER']
|
5
|
+
ENV['HOME'] = File.join(ENV['GEM_ROOT_PATH'], 'tmp/aruba')
|
6
|
+
end
|
7
|
+
|
8
|
+
if ENV['VCR_CASSETTE']
|
9
|
+
require 'vcr'
|
10
|
+
|
11
|
+
|
12
|
+
require 'thor'
|
13
|
+
require 'active_support/core_ext/module'
|
14
|
+
class Thor::Task
|
15
|
+
|
16
|
+
def run_with_vcr(instance, args=[])
|
17
|
+
if ENV['VCR_CASSETTE']
|
18
|
+
cassette_name = ENV.delete('VCR_CASSETTE')
|
19
|
+
|
20
|
+
VCR.configure do |c|
|
21
|
+
c.cassette_library_dir = Holistics.root.join 'features/vcr_cassettes'
|
22
|
+
c.default_cassette_options = {:record => :new_episodes}
|
23
|
+
c.hook_into :webmock
|
24
|
+
end
|
25
|
+
|
26
|
+
VCR.use_cassette(cassette_name) do
|
27
|
+
run_without_vcr(instance, args)
|
28
|
+
end
|
29
|
+
else
|
30
|
+
run_without_vcr(instance, args)
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
alias_method_chain :run, :vcr
|
37
|
+
end
|
38
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: holistics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thanh Dinh Khac
|
@@ -79,12 +79,18 @@ files:
|
|
79
79
|
- README.md
|
80
80
|
- bin/holistics
|
81
81
|
- holistics.gemspec
|
82
|
+
- lib/data_sources.rb
|
82
83
|
- lib/holistics.rb
|
83
84
|
- lib/holistics/api_client.rb
|
85
|
+
- lib/holistics/auth_api_client.rb
|
84
86
|
- lib/holistics/custom_logger.rb
|
87
|
+
- lib/holistics/helpers/auth_info.rb
|
88
|
+
- lib/holistics/helpers/http_request.rb
|
89
|
+
- lib/holistics/helpers/job_manager.rb
|
85
90
|
- lib/holistics/version.rb
|
86
91
|
- lib/tabular_formatter.rb
|
87
|
-
- lib/
|
92
|
+
- lib/transport.rb
|
93
|
+
- lib/vcr_helper.rb
|
88
94
|
homepage: http://rubygems.org/gems/holistics-cli
|
89
95
|
licenses:
|
90
96
|
- GPL
|
data/lib/vcr.rb
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
# This is a cucumber test helper.
|
2
|
-
# It wrap the running of thor commands around VCR cassette if needed
|
3
|
-
|
4
|
-
if ENV['IS_CUCUMBER']
|
5
|
-
ENV['HOME'] = File.join(ENV['GEM_ROOT_PATH'], 'tmp/aruba')
|
6
|
-
end
|
7
|
-
|
8
|
-
if ENV['VCR_CASSETTE']
|
9
|
-
require 'vcr'
|
10
|
-
require 'thor'
|
11
|
-
class Thor::Task
|
12
|
-
def run_with_vcr(instance, args=[])
|
13
|
-
cassette = ENV.delete('VCR_CASSETTE')
|
14
|
-
VCR.configure do |c|
|
15
|
-
c.cassette_library_dir = Holistics.root.join 'features/vcr_cassettes'
|
16
|
-
c.default_cassette_options = {:record => :new_episodes}
|
17
|
-
c.hook_into :webmock
|
18
|
-
end
|
19
|
-
VCR.use_cassette(cassette) do
|
20
|
-
run_without_vcr(instance, args)
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
alias_method_chain :run, :vcr
|
25
|
-
end
|
26
|
-
end
|