holistics 0.2.7 → 0.2.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +7 -3
- data/lib/holistics/api_client.rb +25 -15
- data/lib/holistics/auth_api_client.rb +3 -2
- data/lib/holistics/helpers/auth_info.rb +1 -2
- data/lib/holistics/helpers/http_request.rb +14 -6
- data/lib/holistics/helpers/job_manager.rb +4 -1
- data/lib/holistics/version.rb +2 -2
- data/lib/holistics.rb +0 -29
- data/lib/import.rb +9 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cdc4e0f5e9b4d1d66a0a9d911a102169c6ecaeb8
|
4
|
+
data.tar.gz: 8b5ea3eaafe961fc7b31bbdb81b4f9bebdd114a3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b5eecacd26f55c6e34ab9f3ae2a6360a36c33c8f3f8e8dac8185d890e028535b749349bec7c4febf76319841d9b99df244a00839ff3c5ec68b69dcdfaa1d348f
|
7
|
+
data.tar.gz: 8735748beb6f7163711d97b3cae2c6e38926fb9eb040475c406760a59995567928a8e8b9595ccfd184f662ee351bd05ae45f4bac0ceac2b89dc74616f31c7ed8
|
data/README.md
CHANGED
@@ -5,6 +5,10 @@ Command-line interface to Holistics API
|
|
5
5
|
|
6
6
|
## Build and Publish
|
7
7
|
|
8
|
+
First, open `version.rb` and increase the version number. Please also update the changelog too.
|
9
|
+
|
10
|
+
Then run the following:
|
11
|
+
|
8
12
|
$ gem build holistics.gemspec
|
9
13
|
WARNING: description and summary are identical
|
10
14
|
WARNING: See http://guides.rubygems.org/specification-reference/ for help
|
@@ -32,14 +36,14 @@ Then perform authentication:
|
|
32
36
|
|
33
37
|
### To change remote server for testing purpose
|
34
38
|
|
35
|
-
$ HOLISTICS_DEV=1 holistics
|
39
|
+
$ HOLISTICS_DEV=1 holistics login [token]
|
36
40
|
# Will point to http://localhost:3000
|
37
41
|
|
38
|
-
$ HOLISTICS_STAGING=1 holistics
|
42
|
+
$ HOLISTICS_STAGING=1 holistics login [token]
|
39
43
|
# Will point to https://staging.holistics.io
|
40
44
|
|
41
45
|
# or custom host
|
42
|
-
$ HOLISTICS_HOST=https://blah.com holistics
|
46
|
+
$ HOLISTICS_HOST=https://blah.com holistics login [token]
|
43
47
|
|
44
48
|
|
45
49
|
## Data Transport commands
|
data/lib/holistics/api_client.rb
CHANGED
@@ -26,8 +26,7 @@ module Holistics
|
|
26
26
|
|
27
27
|
params = {
|
28
28
|
dest_fqname: dest_fqname,
|
29
|
-
dest_ds_id: dest_ds_id
|
30
|
-
_utoken: auth_info.get_token_from_gconfig
|
29
|
+
dest_ds_id: dest_ds_id
|
31
30
|
}
|
32
31
|
|
33
32
|
json = http_request.post_csv 'data_imports/import_csv.json', params, file, 'Error uploading CSV file'
|
@@ -42,7 +41,7 @@ module Holistics
|
|
42
41
|
result = http_request.get 'data_sources.json', 'Error retrieving list of data sources'
|
43
42
|
|
44
43
|
table = [%w(ID Type Name)]
|
45
|
-
rows = result.map {
|
44
|
+
rows = result.map {|record| [record['id'], record['dbtype'], record['name']]}
|
46
45
|
table.concat(rows)
|
47
46
|
|
48
47
|
puts TabularFormatter.new(table).to_pretty_table
|
@@ -52,11 +51,9 @@ module Holistics
|
|
52
51
|
puts 'Submitting transport job ...'
|
53
52
|
|
54
53
|
params = build_submit_params(options)
|
55
|
-
|
56
|
-
result = http_request.post_json 'transports/submit.json', params, 'Error submitting transport job'
|
54
|
+
result = http_request.post_json('transports/submit.json', params, 'Error submitting transport job')
|
57
55
|
|
58
56
|
job_id = result['job_id']
|
59
|
-
|
60
57
|
puts "Job submitted. Job ID: #{job_id}."
|
61
58
|
job_manager.tail_logs job_id
|
62
59
|
end
|
@@ -64,8 +61,7 @@ module Holistics
|
|
64
61
|
def invoke_email_schedule(es_id, options)
|
65
62
|
puts "Invoking email schedule ID #{es_id}"
|
66
63
|
|
67
|
-
|
68
|
-
result = http_request.post_json "email_schedules/#{es_id}/execute.json", params, 'Error submitting email schedule job'
|
64
|
+
result = http_request.post_json "email_schedules/#{es_id}/execute.json", options, 'Error submitting email schedule job'
|
69
65
|
|
70
66
|
job_id = result['job_id']
|
71
67
|
|
@@ -76,8 +72,7 @@ module Holistics
|
|
76
72
|
def send_transform(options)
|
77
73
|
puts 'Invoking transform job...'
|
78
74
|
|
79
|
-
|
80
|
-
result = http_request.post_json "data_transforms/#{params[:transform_id]}/execute.json", params, 'Error submitting transform job'
|
75
|
+
result = http_request.post_json "data_transforms/#{options[:transform_id]}/execute.json", options, 'Error submitting transform job'
|
81
76
|
|
82
77
|
job_id = result['job_id']
|
83
78
|
|
@@ -85,6 +80,22 @@ module Holistics
|
|
85
80
|
job_manager.tail_logs job_id
|
86
81
|
end
|
87
82
|
|
83
|
+
def send_import(options)
|
84
|
+
puts 'Invoking import job...'
|
85
|
+
|
86
|
+
result = http_request.post_json "data_imports/#{options[:import_id]}/execute.json", options, 'Error submitting import job'
|
87
|
+
|
88
|
+
job_id = result['job_id']
|
89
|
+
|
90
|
+
puts "Job submitted. Job ID: #{job_id}."
|
91
|
+
job_manager.tail_logs job_id
|
92
|
+
end
|
93
|
+
|
94
|
+
def job_show(options)
|
95
|
+
puts 'Getting job log...'
|
96
|
+
job_manager.tail_logs options[:job_id]
|
97
|
+
end
|
98
|
+
|
88
99
|
def generate_configs(options)
|
89
100
|
unless Dir.exist?(options['output'])
|
90
101
|
raise 'Output location is invalid.'
|
@@ -92,11 +103,10 @@ module Holistics
|
|
92
103
|
|
93
104
|
puts "Generating transport JSON config files to #{options['output']} directory..."
|
94
105
|
|
95
|
-
|
96
|
-
|
97
|
-
|
106
|
+
result = http_request.post_json 'transports/generate_configs.json', options, 'Error generating transport configs'
|
107
|
+
|
98
108
|
result.each do |table_data|
|
99
|
-
File.open("#{options['output']}/#{table_data['filename']}","w") do |f|
|
109
|
+
File.open("#{options['output']}/#{table_data['filename']}", "w") do |f|
|
100
110
|
f.write(JSON.pretty_generate(table_data['json_content']))
|
101
111
|
end
|
102
112
|
|
@@ -108,7 +118,7 @@ module Holistics
|
|
108
118
|
end
|
109
119
|
|
110
120
|
def build_submit_params(options)
|
111
|
-
params = options.except(:config_path)
|
121
|
+
params = options.except(:config_path)
|
112
122
|
params[:configs] = parse_transport_config(options[:config_path]).to_json if options[:config_path]
|
113
123
|
params
|
114
124
|
end
|
@@ -20,12 +20,13 @@ module Holistics
|
|
20
20
|
write_token_to_gconfig(token)
|
21
21
|
else
|
22
22
|
puts 'Error logging in. Please check your token again.'
|
23
|
+
puts response
|
23
24
|
end
|
24
25
|
end
|
25
26
|
|
26
27
|
def authenticate(token)
|
27
|
-
url = auth_info.api_url_for('users/info.json', {}
|
28
|
-
response =
|
28
|
+
url = auth_info.api_url_for('users/info.json', {})
|
29
|
+
response = Helpers::HttpRequest.new.simple_get url, token
|
29
30
|
return response, (response.code == 200)
|
30
31
|
end
|
31
32
|
|
@@ -12,8 +12,7 @@ module Holistics
|
|
12
12
|
File.exists?(get_gconfig_filepath)
|
13
13
|
end
|
14
14
|
|
15
|
-
def api_url_for(path, params = {}
|
16
|
-
params[:_utoken] = token || get_token_from_gconfig
|
15
|
+
def api_url_for(path, params = {})
|
17
16
|
"#{http_request.server_url}#{path}?#{URI.encode_www_form(params)}"
|
18
17
|
end
|
19
18
|
|
@@ -7,6 +7,7 @@ module Holistics
|
|
7
7
|
|
8
8
|
DEFAULT_ERROR_MSG = 'Error occurred!'
|
9
9
|
SERVER_URL = 'https://api.holistics.io/'
|
10
|
+
API_KEY_HEADER = 'X-Holistics-Key' # must match Holistics::API_KEY_HEADER in main repo
|
10
11
|
|
11
12
|
def auth_helper
|
12
13
|
@auth_info ||= Helpers::AuthInfo.new
|
@@ -14,7 +15,7 @@ module Holistics
|
|
14
15
|
|
15
16
|
def server_url
|
16
17
|
host =
|
17
|
-
if %w(development test).include?(ENV['HOLISTICS_ENV'])
|
18
|
+
if ENV['HOLISTICS_DEV'] || %w(development test).include?(ENV['HOLISTICS_ENV'])
|
18
19
|
'http://localhost:3000'
|
19
20
|
elsif ENV['HOLISTICS_STAGING'] || ENV['HOLISTICS_ENV'] == 'staging'
|
20
21
|
'https://staging.holistics.io'
|
@@ -29,18 +30,23 @@ module Holistics
|
|
29
30
|
|
30
31
|
def get url, msg_if_error = DEFAULT_ERROR_MSG
|
31
32
|
url = auth_helper.api_url_for url
|
32
|
-
|
33
|
-
response = HTTParty.get url
|
34
|
-
|
33
|
+
response = simple_get url
|
35
34
|
exit_if_error(msg_if_error, response)
|
36
|
-
|
37
35
|
JSON.parse response.body
|
38
36
|
end
|
37
|
+
|
38
|
+
def simple_get(url, token = nil)
|
39
|
+
HTTParty
|
40
|
+
.get url, headers: { API_KEY_HEADER => token || auth_helper.get_token_from_gconfig }
|
41
|
+
end
|
39
42
|
|
40
43
|
def post_json url, params, msg_if_error = DEFAULT_ERROR_MSG
|
41
44
|
options = {
|
42
45
|
body: params.to_json,
|
43
|
-
headers: {
|
46
|
+
headers: {
|
47
|
+
'Content-Type' => 'application/json',
|
48
|
+
API_KEY_HEADER => auth_helper.get_token_from_gconfig
|
49
|
+
}
|
44
50
|
}
|
45
51
|
|
46
52
|
response = HTTParty.post("#{server_url}#{url}", options)
|
@@ -59,7 +65,9 @@ module Holistics
|
|
59
65
|
uri = URI.parse(endpoint_for(url))
|
60
66
|
csv = UploadIO.new file, "text/csv", File.basename(file.path)
|
61
67
|
params = params.merge file: csv
|
68
|
+
|
62
69
|
post_data = Net::HTTP::Post::Multipart.new uri.path, params
|
70
|
+
post_data.add_field(API_KEY_HEADER, auth_helper.get_token_from_gconfig)
|
63
71
|
|
64
72
|
https_request = Net::HTTP.new uri.host, uri.port
|
65
73
|
https_request.use_ssl = true if uri.scheme == 'https'
|
@@ -9,7 +9,10 @@ module Holistics
|
|
9
9
|
|
10
10
|
def fetch_job_details(job_id, last_id = 0)
|
11
11
|
tries ||= MAX_RETRIES
|
12
|
-
|
12
|
+
|
13
|
+
url = auth_helper.api_url_for("jobs/#{job_id}/logs.json", last_id: last_id)
|
14
|
+
response = HttpRequest.new.simple_get url
|
15
|
+
|
13
16
|
JSON.parse(response.body)
|
14
17
|
rescue JSON::ParserError => err
|
15
18
|
sleep 2 ** (MAX_RETRIES - tries) unless Holistics.test?
|
data/lib/holistics/version.rb
CHANGED
data/lib/holistics.rb
CHANGED
@@ -69,22 +69,6 @@ module Holistics
|
|
69
69
|
api_client.ds_list
|
70
70
|
end
|
71
71
|
|
72
|
-
|
73
|
-
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
74
|
-
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
75
|
-
method_option :from_table_name, aliases: '-t', type: :string, required: false, desc: 'The table to copy over'
|
76
|
-
method_option :dest_table_name, aliases: '-n', type: :string, required: false, desc: '(optional) Rename destination table. Please specify fully qualified name'
|
77
|
-
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
78
|
-
method_option :full, type: :boolean, default: false, required: false, desc: 'Full table transport'
|
79
|
-
method_option :incremental, type: :boolean, default: false, required: false, desc: 'Incremental table transport'
|
80
|
-
desc 'transport_old', '[DEPRECATED] Submit a data transport job'
|
81
|
-
|
82
|
-
def transport_old
|
83
|
-
p 'DEPRECATED. This command will be removed in the next version. Please consider using `holistics transport submit` instead.'
|
84
|
-
api_client.send_transport(options.dup)
|
85
|
-
end
|
86
|
-
|
87
|
-
|
88
72
|
method_option :transform_id, aliases: '-j', type: :string, required: true, desc: 'ID of transform job to be executed'
|
89
73
|
desc 'transform', 'Invoke a transform'
|
90
74
|
|
@@ -92,19 +76,6 @@ module Holistics
|
|
92
76
|
api_client.send_transform(options.dup)
|
93
77
|
end
|
94
78
|
|
95
|
-
|
96
|
-
method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
|
97
|
-
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
|
98
|
-
method_option :tables, aliases: '-t', type: :array, required: true, desc: 'List of tables to generate configs from. Wildcards are allowed'
|
99
|
-
method_option :output, aliases: '-o', type: :string, required: false, default: './', desc: 'Location where files will be generated to'
|
100
|
-
desc 'generate_configs', '[DEPRECATED] Generate transport JSON configuration files'
|
101
|
-
|
102
|
-
def generate_configs
|
103
|
-
p 'DEPRECATED. This command will be removed in the next version. Please consider using `holistics transport generate` instead.'
|
104
|
-
api_client.generate_configs(options.dup)
|
105
|
-
end
|
106
|
-
|
107
|
-
|
108
79
|
method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
|
109
80
|
desc 'job_show', 'Show job log'
|
110
81
|
|
data/lib/import.rb
CHANGED
@@ -3,6 +3,7 @@ require 'thor'
|
|
3
3
|
module Holistics
|
4
4
|
class Import < Thor
|
5
5
|
|
6
|
+
# import from csv file
|
6
7
|
method_option :filepath, aliases: '-f', type: :string, required: true, desc: 'Path to CSV file'
|
7
8
|
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'Destination data source'
|
8
9
|
method_option :dest_table_name, aliases: '-t', type: :string, required: true, desc: 'Specify destination table to write to'
|
@@ -12,6 +13,14 @@ module Holistics
|
|
12
13
|
api_client.import_csv(options.dup)
|
13
14
|
end
|
14
15
|
|
16
|
+
# execute import job
|
17
|
+
method_option :import_id, aliases: '-j', type: :string, required: true, desc: 'ID of import job to be executed'
|
18
|
+
desc 'execute', 'Invoke an import job'
|
19
|
+
|
20
|
+
def execute
|
21
|
+
api_client.send_import(options.dup)
|
22
|
+
end
|
23
|
+
|
15
24
|
no_commands do
|
16
25
|
def api_client
|
17
26
|
@api_client ||= ApiClient.new
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: holistics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thanh Dinh Khac
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-10-20 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|