holistics 0.0.1 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/holistics.gemspec +6 -3
- data/lib/holistics/api_client.rb +67 -37
- data/lib/holistics/version.rb +1 -1
- data/lib/holistics.rb +32 -19
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a84dcb9a88918717b220f217e544a804ee83ac8d
|
4
|
+
data.tar.gz: cedcd80882d85ae412fe1e810ef13afc3e1e7e16
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 36a8b5e6de2e778028f8b919b8667439269f41650d874b3177bb05e826d13a407c54238885ff41b5e458fa92daac93d6e9134bbd8ef5b9d70f6c9ae926b7d8a0
|
7
|
+
data.tar.gz: 85075130ff88b1fd518a3ab1c5f91eae3403e1e0bb7e37fc8f76e6c3304edbf1a38f8ce304f9e9112152c4f93f430530b2045f67f6ac74f12bc0bd9f4658b1c4
|
data/holistics.gemspec
CHANGED
@@ -14,11 +14,14 @@ Gem::Specification.new do |spec|
|
|
14
14
|
spec.homepage = 'http://rubygems.org/gems/holistics-cli'
|
15
15
|
spec.license = 'GPL'
|
16
16
|
|
17
|
-
spec.files = %w[.document holistics.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
|
18
|
-
spec.require_paths
|
19
|
-
|
17
|
+
spec.files = %w[.document holistics.gemspec] + Dir['*.md', 'bin/*', 'lib/*', 'lib/**/*.rb']
|
18
|
+
spec.require_paths << 'lib'
|
20
19
|
spec.executables = ['holistics']
|
21
20
|
|
21
|
+
spec.required_ruby_version = '>= 2.0'
|
22
|
+
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.0"
|
24
|
+
|
22
25
|
spec.add_dependency 'activesupport', '~> 4.2'
|
23
26
|
spec.add_dependency 'httparty', '~> 0.13'
|
24
27
|
spec.add_dependency 'thor', '~> 0.19'
|
data/lib/holistics/api_client.rb
CHANGED
@@ -1,27 +1,30 @@
|
|
1
1
|
require 'httparty'
|
2
2
|
require 'holistics/tabular_formatter'
|
3
|
+
require 'yaml'
|
4
|
+
require 'json'
|
3
5
|
|
4
6
|
class ApiClient
|
5
7
|
SERVER_URL = 'https://secure.holistics.io/'
|
6
8
|
|
7
9
|
def login token
|
8
|
-
puts '
|
9
|
-
|
10
|
-
|
10
|
+
puts 'Logging in...'
|
11
|
+
response, ok = holistics_authenticate(token)
|
12
|
+
if ok
|
13
|
+
parsed = JSON.parse(response.body)
|
14
|
+
puts 'Authentication successful. Info:'
|
15
|
+
puts "- ID: #{parsed['id']}"
|
16
|
+
puts "- Email: #{parsed['email']}"
|
11
17
|
|
12
|
-
|
13
|
-
|
14
|
-
|
18
|
+
write_token_to_gconfig(token)
|
19
|
+
else
|
20
|
+
puts 'Error logging in. Please check your token again.'
|
15
21
|
end
|
22
|
+
end
|
16
23
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
file_path = File.join(ENV['HOME'], '.holistics.yml')
|
23
|
-
|
24
|
-
File.write(file_path, token)
|
24
|
+
def holistics_authenticate(token)
|
25
|
+
url = api_url_for('users/info.json', token)
|
26
|
+
response = HTTParty.get(url)
|
27
|
+
return response, (response.code == 200)
|
25
28
|
end
|
26
29
|
|
27
30
|
def job_show options
|
@@ -32,12 +35,7 @@ class ApiClient
|
|
32
35
|
def ds_list
|
33
36
|
url = api_url_for('data_sources.json')
|
34
37
|
response = HTTParty.get(url)
|
35
|
-
|
36
|
-
if response.code != 200
|
37
|
-
puts "Error retrieving list of data sources. Code: #{response.code}"
|
38
|
-
puts response.body
|
39
|
-
exit 1
|
40
|
-
end
|
38
|
+
err_and_exit("Error retrieving list of data sources", response) if response_has_error?(response)
|
41
39
|
|
42
40
|
parsed = JSON.parse(response.body)
|
43
41
|
|
@@ -53,16 +51,12 @@ class ApiClient
|
|
53
51
|
params = build_submit_params(dest_ds_type, from_ds_type, options)
|
54
52
|
response = submit_transport_job(params)
|
55
53
|
|
56
|
-
|
57
|
-
puts "Error submitting transport job. Code: #{response.code}"
|
58
|
-
puts response.body
|
59
|
-
exit 1
|
60
|
-
end
|
54
|
+
err_and_exit("Error submitting transport job", response) if response_has_error?(response)
|
61
55
|
|
62
56
|
parsed = JSON.parse(response.body)
|
63
57
|
job_id = parsed['job_id']
|
64
|
-
puts "Job submitted. Job ID: #{job_id}."
|
65
58
|
|
59
|
+
puts "Job submitted. Job ID: #{job_id}."
|
66
60
|
tail_job(job_id)
|
67
61
|
end
|
68
62
|
|
@@ -102,39 +96,75 @@ class ApiClient
|
|
102
96
|
end
|
103
97
|
|
104
98
|
def build_submit_params(dest_ds_type, from_ds_type, options)
|
105
|
-
params = options.merge(from_ds_type: from_ds_type, dest_ds_type: dest_ds_type, _utoken:
|
99
|
+
params = options.except(:config_path).merge(from_ds_type: from_ds_type, dest_ds_type: dest_ds_type, _utoken: get_token_from_gconfig)
|
106
100
|
|
107
101
|
configs = {}
|
108
102
|
if options[:config_path]
|
109
|
-
|
110
|
-
configs =
|
103
|
+
config_filepath = File.join(ENV['ROOT_PATH'], options[:config_path])
|
104
|
+
configs = parse_transport_config(config_filepath)
|
111
105
|
end
|
112
106
|
|
113
|
-
configs[
|
114
|
-
configs[
|
107
|
+
configs['from_table_name'] = options[:table_name] if options[:table_name]
|
108
|
+
configs['dest_table_name'] = options[:rename] if options[:rename]
|
115
109
|
|
116
|
-
params[:
|
110
|
+
params[:configs] = configs.to_json # should be a string
|
117
111
|
params
|
118
112
|
end
|
119
113
|
|
114
|
+
def parse_transport_config(filepath)
|
115
|
+
file_ext = File.extname(filepath).downcase
|
116
|
+
if file_ext == '.json'
|
117
|
+
return JSON.parse(File.read(filepath))
|
118
|
+
elsif file_ext == '.yml'
|
119
|
+
return YAML.load(File.read(filepath))
|
120
|
+
else
|
121
|
+
raise StandardError.new 'Invalid config file extension. Please use either JSON or YML'
|
122
|
+
end
|
123
|
+
rescue StandardError => e
|
124
|
+
STDERR.puts "Error parsing transport config file: #{e.message}"
|
125
|
+
exit 1
|
126
|
+
end
|
127
|
+
|
120
128
|
def authenticated?
|
121
|
-
File.exists?(
|
129
|
+
File.exists?(get_gconfig_filepath)
|
122
130
|
end
|
123
131
|
|
124
|
-
def
|
125
|
-
|
132
|
+
def get_token_from_gconfig
|
133
|
+
if authenticated?
|
134
|
+
string = YAML.load_file(get_gconfig_filepath)
|
135
|
+
string['token']
|
136
|
+
else
|
137
|
+
raise StandardError.new 'Holistics config file not found'
|
138
|
+
end
|
126
139
|
end
|
127
140
|
|
128
|
-
def
|
141
|
+
def get_gconfig_filepath
|
129
142
|
File.expand_path('~/.holistics.yml', __FILE__)
|
130
143
|
end
|
131
144
|
|
132
145
|
def api_url_for(path, token = nil)
|
133
|
-
"#{server_url}#{path}?_utoken=#{token ||
|
146
|
+
"#{server_url}#{path}?_utoken=#{token || get_token_from_gconfig}"
|
134
147
|
end
|
135
148
|
|
136
149
|
def print_log log
|
137
150
|
puts "#{log['created_at'][0..18]} - #{log['level']} - #{log['message']}"
|
138
151
|
end
|
139
152
|
|
153
|
+
def err_and_exit(message, response)
|
154
|
+
STDERR.puts message
|
155
|
+
STDERR.puts "Error Response Code: #{response.code}"
|
156
|
+
STDERR.puts response.body
|
157
|
+
exit 1
|
158
|
+
end
|
159
|
+
|
160
|
+
def response_has_error?(response)
|
161
|
+
response.code != 200
|
162
|
+
end
|
163
|
+
|
164
|
+
def write_token_to_gconfig(token)
|
165
|
+
file_path = File.join(ENV['HOME'], '.holistics.yml')
|
166
|
+
h = {'token' => token}
|
167
|
+
File.write(file_path, h.to_yaml)
|
168
|
+
end
|
169
|
+
|
140
170
|
end
|
data/lib/holistics/version.rb
CHANGED
data/lib/holistics.rb
CHANGED
@@ -13,7 +13,7 @@ end
|
|
13
13
|
# Either we use other libs, or figure out an elegant way to do it without
|
14
14
|
|
15
15
|
class HolisticsRunner < Thor
|
16
|
-
desc 'login', 'Perform authentication'
|
16
|
+
desc 'login [auth_token]', 'Perform authentication'
|
17
17
|
|
18
18
|
def login token
|
19
19
|
client = ApiClient.new
|
@@ -26,42 +26,55 @@ class HolisticsRunner < Thor
|
|
26
26
|
ApiClient.new.ds_list
|
27
27
|
end
|
28
28
|
|
29
|
-
method_option :
|
30
|
-
method_option :
|
31
|
-
method_option :
|
32
|
-
|
29
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
30
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
31
|
+
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
32
|
+
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
33
|
+
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
34
|
+
desc 'redshift_to_postgres', 'Transport data from Amazon Redshift data source to PostgreSQL'
|
33
35
|
|
34
|
-
def
|
36
|
+
def redshift_to_postgres
|
35
37
|
ApiClient.new.send_transport(:redshift, :postgresql, options.dup)
|
36
38
|
end
|
37
39
|
|
38
|
-
method_option :from_ds_id, aliases: '-s', type: :string, required:
|
39
|
-
method_option :dest_ds_id, aliases: '-d', type: :string, required:
|
40
|
-
method_option :
|
41
|
-
|
40
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
41
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
42
|
+
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
43
|
+
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
44
|
+
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
45
|
+
desc 'postgres_to_redshift', 'Transport data from PostgreSQL to Amazon Redshift'
|
42
46
|
|
43
|
-
def
|
47
|
+
def postgres_to_redshift
|
44
48
|
ApiClient.new.send_transport(:postgresql, :redshift, options.dup)
|
45
49
|
end
|
46
50
|
|
47
|
-
method_option :from_ds_id, aliases: '-s', type: :string, required:
|
48
|
-
method_option :dest_ds_id, aliases: '-d', type: :string, required:
|
49
|
-
method_option :
|
50
|
-
method_option :
|
51
|
-
|
51
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
52
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
53
|
+
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
54
|
+
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
55
|
+
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
56
|
+
desc 'postgres_to_postgres', 'Transport data from PostgreSQL to PostgreSQL'
|
52
57
|
|
53
|
-
def
|
58
|
+
def postgres_to_postgres
|
54
59
|
ApiClient.new.send_transport(:postgresql, :postgresql, options.dup)
|
55
60
|
end
|
56
61
|
|
57
|
-
method_option :
|
62
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
63
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
64
|
+
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
65
|
+
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
66
|
+
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
58
67
|
desc 'mysql_to_postgres', 'Transport data from MySQL to Postgres'
|
59
68
|
|
60
69
|
def mysql_to_postgres
|
61
70
|
ApiClient.new.send_transport(:mysql, :postgresql, options.dup)
|
62
71
|
end
|
63
72
|
|
64
|
-
method_option :
|
73
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: false, desc: 'From data source'
|
74
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: false, desc: 'To data source'
|
75
|
+
method_option :from_table, aliases: '-t', type: :string, required: false, desc: 'Table names'
|
76
|
+
method_option :dest_table, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
77
|
+
method_option :config_path, aliases: '-c', type: :string, required: false, desc: 'Path to transport config (JSON/YML)̄ file'
|
65
78
|
desc 'mysql_to_redshift', 'Transport data from MySQL to Redshift'
|
66
79
|
|
67
80
|
def mysql_to_redshift
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: holistics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thanh Dinh Khac
|
@@ -11,6 +11,20 @@ bindir: bin
|
|
11
11
|
cert_chain: []
|
12
12
|
date: 2015-11-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '1.0'
|
21
|
+
type: :development
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '1.0'
|
14
28
|
- !ruby/object:Gem::Dependency
|
15
29
|
name: activesupport
|
16
30
|
requirement: !ruby/object:Gem::Requirement
|
@@ -77,11 +91,12 @@ post_install_message:
|
|
77
91
|
rdoc_options: []
|
78
92
|
require_paths:
|
79
93
|
- lib
|
94
|
+
- lib
|
80
95
|
required_ruby_version: !ruby/object:Gem::Requirement
|
81
96
|
requirements:
|
82
97
|
- - ">="
|
83
98
|
- !ruby/object:Gem::Version
|
84
|
-
version: '0'
|
99
|
+
version: '2.0'
|
85
100
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
86
101
|
requirements:
|
87
102
|
- - ">="
|