holistics 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.document +4 -0
- data/README.md +67 -0
- data/bin/holistics +5 -0
- data/holistics.gemspec +26 -0
- data/lib/holistics/api_client.rb +140 -0
- data/lib/holistics/cucumber/vcr.rb +26 -0
- data/lib/holistics/tabular_formatter.rb +58 -0
- data/lib/holistics/version.rb +3 -0
- data/lib/holistics.rb +78 -0
- metadata +96 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2f4151e165215a9fb3c9b3a6317cd9b9ef829f5e
|
4
|
+
data.tar.gz: 0f0078f67112c8176971d6cfbce5a8094e901f07
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 654ee243f6fb1415300f32f1004563574aef12b3cbdca659dbbe186a34c6f11b98e2792d75847bf8814af6d516eeaacbf661410b1e1d319c8d78e51605c81d60
|
7
|
+
data.tar.gz: 7a892fa848ef8b5b5ac65b9c5cb55c8166a09d5bc61dfe8a1395fe3a201504fa6a7849afe6a1f87efdef7f85032a9f165db366689102f68047e3ded7346ef297
|
data/.document
ADDED
data/README.md
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
# Holistics CLI
|
2
|
+
|
3
|
+
Command-line interface to Holistics API
|
4
|
+
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
Install gem from RubyGems
|
9
|
+
|
10
|
+
$ gem install holistics
|
11
|
+
|
12
|
+
|
13
|
+
Then perform authentication:
|
14
|
+
|
15
|
+
$ holistics login [token]
|
16
|
+
Authenticating token...
|
17
|
+
Authentication successful. Info:
|
18
|
+
- ID: 1
|
19
|
+
- Email: admin@taxibooking.com
|
20
|
+
|
21
|
+
## Data Transport commands
|
22
|
+
|
23
|
+
### List existing data sources:
|
24
|
+
|
25
|
+
$ holistics ds_list
|
26
|
+
Listing all data sources...
|
27
|
+
| ID | Type | Name |
|
28
|
+
|----+------------+---------------|
|
29
|
+
| 1 | PostgreSQL | Production DB |
|
30
|
+
| 2 | Redshift | Analytics DB |
|
31
|
+
|
32
|
+
### Transport data from Redshift to Postgres:
|
33
|
+
|
34
|
+
$ holistics rs_to_pg -s <source_id> -d <dest_id> -t <table_name>
|
35
|
+
|
36
|
+
Example:
|
37
|
+
|
38
|
+
$ holistics rs_to_pg -s 1 -d 2 -t public.users
|
39
|
+
Submitting transport job ...
|
40
|
+
Job submitted. Job ID: 738.
|
41
|
+
[job:738] Status: queued
|
42
|
+
[job:738] Status: queued
|
43
|
+
[job:738] Status: queued
|
44
|
+
[job:738] Status: running
|
45
|
+
[job:738] Reading schema of table public.users ...
|
46
|
+
[job:738] Counting table's records ...
|
47
|
+
[job:738] - Record count: 6,783
|
48
|
+
[job:738] Creating temporary table public.temp_users_blah on destination ...
|
49
|
+
[job:738] Getting table data from source ...
|
50
|
+
[job:738] Loading data into Redshift table public.temp_users_blah ...
|
51
|
+
[job:738] Hot-swapping with Redshift table public.users ...
|
52
|
+
[job:738] Done.
|
53
|
+
|
54
|
+
### Custom table build transport
|
55
|
+
|
56
|
+
See `samples/clicks_mysql_to_redshift.json` for details of transport configs.
|
57
|
+
|
58
|
+
$ holistics mysql_to_redshift -c samples/clicks_mysql_to_redshift.json
|
59
|
+
|
60
|
+
|
61
|
+
|
62
|
+
|
63
|
+
### Mechanism:
|
64
|
+
|
65
|
+
* Submit a POST request to `/transports`. Will check and return a Job ID.
|
66
|
+
* Keep pinging
|
67
|
+
|
data/bin/holistics
ADDED
data/holistics.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
$LOAD_PATH << File.expand_path('../lib', __FILE__)
|
2
|
+
require 'holistics/version'
|
3
|
+
|
4
|
+
Gem::Specification.new do |spec|
|
5
|
+
spec.name = 'holistics'
|
6
|
+
spec.version = Holistics::VERSION
|
7
|
+
spec.date = '2015-11-16'
|
8
|
+
spec.description = 'CLI interface for Holistics'
|
9
|
+
spec.summary = spec.description
|
10
|
+
|
11
|
+
spec.authors = ['Thanh Dinh Khac', 'Huy Nguyen']
|
12
|
+
spec.email = 'huy@holistics.io'
|
13
|
+
|
14
|
+
spec.homepage = 'http://rubygems.org/gems/holistics-cli'
|
15
|
+
spec.license = 'GPL'
|
16
|
+
|
17
|
+
spec.files = %w[.document holistics.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
|
18
|
+
spec.require_paths = ['lib']
|
19
|
+
|
20
|
+
spec.executables = ['holistics']
|
21
|
+
|
22
|
+
spec.add_dependency 'activesupport', '~> 4.2'
|
23
|
+
spec.add_dependency 'httparty', '~> 0.13'
|
24
|
+
spec.add_dependency 'thor', '~> 0.19'
|
25
|
+
|
26
|
+
end
|
@@ -0,0 +1,140 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
require 'holistics/tabular_formatter'
|
3
|
+
|
4
|
+
class ApiClient
|
5
|
+
SERVER_URL = 'https://secure.holistics.io/'
|
6
|
+
|
7
|
+
def login token
|
8
|
+
puts 'Authenticating token...'
|
9
|
+
url = api_url_for('users/info.json', token)
|
10
|
+
response = HTTParty.get(url)
|
11
|
+
|
12
|
+
if response.code != 200
|
13
|
+
puts 'Error authenticating. Please check your token again.'
|
14
|
+
exit 1
|
15
|
+
end
|
16
|
+
|
17
|
+
parsed = JSON.parse(response.body)
|
18
|
+
puts 'Authentication successful. Info:'
|
19
|
+
puts "- ID: #{parsed['id']}"
|
20
|
+
puts "- Email: #{parsed['email']}"
|
21
|
+
|
22
|
+
file_path = File.join(ENV['HOME'], '.holistics.yml')
|
23
|
+
|
24
|
+
File.write(file_path, token)
|
25
|
+
end
|
26
|
+
|
27
|
+
def job_show options
|
28
|
+
job_id = options[:job_id]
|
29
|
+
tail_job(job_id)
|
30
|
+
end
|
31
|
+
|
32
|
+
def ds_list
|
33
|
+
url = api_url_for('data_sources.json')
|
34
|
+
response = HTTParty.get(url)
|
35
|
+
|
36
|
+
if response.code != 200
|
37
|
+
puts "Error retrieving list of data sources. Code: #{response.code}"
|
38
|
+
puts response.body
|
39
|
+
exit 1
|
40
|
+
end
|
41
|
+
|
42
|
+
parsed = JSON.parse(response.body)
|
43
|
+
|
44
|
+
table = [%w(ID Type Name)]
|
45
|
+
rows = parsed.map { |record| [record['id'], record['dbtype'], record['name']] }
|
46
|
+
table.concat(rows)
|
47
|
+
|
48
|
+
puts TabularFormatter.new(table).to_pretty_table
|
49
|
+
end
|
50
|
+
|
51
|
+
def send_transport(from_ds_type, dest_ds_type, options)
|
52
|
+
puts 'Submitting transport job ...'
|
53
|
+
params = build_submit_params(dest_ds_type, from_ds_type, options)
|
54
|
+
response = submit_transport_job(params)
|
55
|
+
|
56
|
+
if response.code != 200
|
57
|
+
puts "Error submitting transport job. Code: #{response.code}"
|
58
|
+
puts response.body
|
59
|
+
exit 1
|
60
|
+
end
|
61
|
+
|
62
|
+
parsed = JSON.parse(response.body)
|
63
|
+
job_id = parsed['job_id']
|
64
|
+
puts "Job submitted. Job ID: #{job_id}."
|
65
|
+
|
66
|
+
tail_job(job_id)
|
67
|
+
end
|
68
|
+
|
69
|
+
def tail_job(job_id)
|
70
|
+
last_ts = ''
|
71
|
+
while true
|
72
|
+
response = fetch_job_status(job_id)
|
73
|
+
parsed = JSON.parse(response.body)
|
74
|
+
logs = parsed['job_logs']
|
75
|
+
|
76
|
+
select_logs = logs.select { |log| log['created_at'] > last_ts }
|
77
|
+
select_logs.each do |log|
|
78
|
+
print_log(log)
|
79
|
+
end
|
80
|
+
last_ts = logs.last['created_at'] if logs.size > 0
|
81
|
+
break unless has_more?(parsed['status'])
|
82
|
+
sleep 1
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def fetch_job_status(job_id)
|
87
|
+
HTTParty.get(api_url_for("jobs/#{job_id}.json"))
|
88
|
+
end
|
89
|
+
|
90
|
+
def has_more?(status)
|
91
|
+
%w(running created).include?(status)
|
92
|
+
end
|
93
|
+
|
94
|
+
def submit_transport_job(params)
|
95
|
+
HTTParty.post(server_url + 'transports/submit.json', body: params.to_json, headers: {'Content-Type' => 'application/json'})
|
96
|
+
end
|
97
|
+
|
98
|
+
def server_url
|
99
|
+
host = (ENV['HOST'] || SERVER_URL).dup
|
100
|
+
host += '/' if host[-1] != '/'
|
101
|
+
host
|
102
|
+
end
|
103
|
+
|
104
|
+
def build_submit_params(dest_ds_type, from_ds_type, options)
|
105
|
+
params = options.merge(from_ds_type: from_ds_type, dest_ds_type: dest_ds_type, _utoken: get_key)
|
106
|
+
|
107
|
+
configs = {}
|
108
|
+
if options[:config_path]
|
109
|
+
config_content = File.read(File.join(ENV['ROOT_PATH'], options[:config_path]))
|
110
|
+
configs = JSON.parse(config_content)
|
111
|
+
end
|
112
|
+
|
113
|
+
configs[:from_table_name] = options[:table_name] if options[:table_name]
|
114
|
+
configs[:dest_table_name] = options[:rename] if options[:rename]
|
115
|
+
|
116
|
+
params[:config] = configs.to_json # should be a string
|
117
|
+
params
|
118
|
+
end
|
119
|
+
|
120
|
+
def authenticated?
|
121
|
+
File.exists?(config_filepath)
|
122
|
+
end
|
123
|
+
|
124
|
+
def get_key
|
125
|
+
File.read(config_filepath).strip
|
126
|
+
end
|
127
|
+
|
128
|
+
def config_filepath
|
129
|
+
File.expand_path('~/.holistics.yml', __FILE__)
|
130
|
+
end
|
131
|
+
|
132
|
+
def api_url_for(path, token = nil)
|
133
|
+
"#{server_url}#{path}?_utoken=#{token || get_key}"
|
134
|
+
end
|
135
|
+
|
136
|
+
def print_log log
|
137
|
+
puts "#{log['created_at'][0..18]} - #{log['level']} - #{log['message']}"
|
138
|
+
end
|
139
|
+
|
140
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# This is a cucumber test helper.
|
2
|
+
# It wrap the running of thor commands around VCR cassette if needed
|
3
|
+
|
4
|
+
if ENV['IS_CUCUMBER']
|
5
|
+
ENV['HOME'] = File.join(ENV['ROOT_PATH'], 'tmp/aruba')
|
6
|
+
end
|
7
|
+
|
8
|
+
if ENV['VCR_CASSETTE']
|
9
|
+
require 'vcr'
|
10
|
+
require 'thor'
|
11
|
+
class Thor::Task
|
12
|
+
def run_with_vcr(instance, args=[])
|
13
|
+
cassette = ENV.delete('VCR_CASSETTE')
|
14
|
+
VCR.configure do |c|
|
15
|
+
c.cassette_library_dir = Holistics.root.join 'features/vcr_cassettes'
|
16
|
+
c.default_cassette_options = {:record => :new_episodes}
|
17
|
+
c.hook_into :webmock
|
18
|
+
end
|
19
|
+
VCR.use_cassette(cassette) do
|
20
|
+
run_without_vcr(instance, args)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
alias_method_chain :run, :vcr
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'active_support/all'
|
3
|
+
|
4
|
+
|
5
|
+
class TabularFormatter
|
6
|
+
|
7
|
+
def initialize input_data, options = {}
|
8
|
+
@input_data = input_data
|
9
|
+
@options = options
|
10
|
+
@options[:headers] = true if @options[:headers].nil?
|
11
|
+
end
|
12
|
+
|
13
|
+
def to_csv(col_sep = ',')
|
14
|
+
CSV.generate(col_sep: col_sep) do |csv|
|
15
|
+
@input_data.each { |row| csv << row }
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_tsv
|
20
|
+
to_csv("\t")
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_pretty_table
|
24
|
+
return '' if @input_data.blank? or @input_data.first.empty?
|
25
|
+
|
26
|
+
max_column_lengths = Array.new(@input_data.first.length, 0)
|
27
|
+
@input_data.each do |row|
|
28
|
+
row.each_with_index do |cell, index|
|
29
|
+
max_column_lengths[index] = [cell.to_s.length, max_column_lengths[index]].max
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
output = []
|
34
|
+
@input_data.each do |row|
|
35
|
+
output << print_table_row(max_column_lengths, row)
|
36
|
+
end
|
37
|
+
|
38
|
+
divider = max_column_lengths.inject("") do |str, l|
|
39
|
+
str << "-#{'-' * l}-+"
|
40
|
+
end.chomp!("+")
|
41
|
+
divider = divider[1..-1]
|
42
|
+
output.insert(1,divider)
|
43
|
+
output.join("\n").concat("\n")
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def print_table_row column_lengths, row
|
49
|
+
row_string = ''
|
50
|
+
row.each_with_index do |cell, index|
|
51
|
+
padding = column_lengths[index] - cell.to_s.length
|
52
|
+
row_string << " #{cell}#{' ' * padding} |"
|
53
|
+
end
|
54
|
+
row_string.chomp!("|")
|
55
|
+
row_string.strip!
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
data/lib/holistics.rb
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'holistics/api_client'
|
3
|
+
require 'holistics/cucumber/vcr'
|
4
|
+
|
5
|
+
class Holistics
|
6
|
+
def self.root
|
7
|
+
Pathname.new(__FILE__).parent.parent
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
# Right now we can't do `holistics ds:list`
|
12
|
+
# Known issue: https://github.com/erikhuda/thor/issues/249
|
13
|
+
# Either we use other libs, or figure out an elegant way to do it without
|
14
|
+
|
15
|
+
class HolisticsRunner < Thor
|
16
|
+
desc 'login', 'Perform authentication'
|
17
|
+
|
18
|
+
def login token
|
19
|
+
client = ApiClient.new
|
20
|
+
client.login(token)
|
21
|
+
end
|
22
|
+
|
23
|
+
desc 'ds_list', 'List all data sources'
|
24
|
+
|
25
|
+
def ds_list
|
26
|
+
ApiClient.new.ds_list
|
27
|
+
end
|
28
|
+
|
29
|
+
method_option :from_ds, aliases: '-s', type: :string, required: true, desc: 'From data source (ID)'
|
30
|
+
method_option :dest_ds, aliases: '-d', type: :string, required: true, desc: 'To data source (ID)'
|
31
|
+
method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table name'
|
32
|
+
desc 'rs_to_pg', 'Transport data from Amazon Redshift data source to PostgreSQL'
|
33
|
+
|
34
|
+
def rs_to_pg
|
35
|
+
ApiClient.new.send_transport(:redshift, :postgresql, options.dup)
|
36
|
+
end
|
37
|
+
|
38
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
|
39
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
|
40
|
+
method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table names'
|
41
|
+
desc 'pg_to_rs', 'Transport data from PostgreSQL to Amazon Redshift'
|
42
|
+
|
43
|
+
def pg_to_rs
|
44
|
+
ApiClient.new.send_transport(:postgresql, :redshift, options.dup)
|
45
|
+
end
|
46
|
+
|
47
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
|
48
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
|
49
|
+
method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table names'
|
50
|
+
method_option :rename, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
51
|
+
desc 'pg_to_pg', 'Transport data from PostgreSQL to PostgreSQL'
|
52
|
+
|
53
|
+
def pg_to_pg
|
54
|
+
ApiClient.new.send_transport(:postgresql, :postgresql, options.dup)
|
55
|
+
end
|
56
|
+
|
57
|
+
method_option :config_path, aliases: '-c', type: :string, required: true, desc: 'Path to transport config (JSON) file'
|
58
|
+
desc 'mysql_to_postgres', 'Transport data from MySQL to Postgres'
|
59
|
+
|
60
|
+
def mysql_to_postgres
|
61
|
+
ApiClient.new.send_transport(:mysql, :postgresql, options.dup)
|
62
|
+
end
|
63
|
+
|
64
|
+
method_option :config_path, aliases: '-c', type: :string, required: true, desc: 'Path to transport config (JSON) file'
|
65
|
+
desc 'mysql_to_redshift', 'Transport data from MySQL to Redshift'
|
66
|
+
|
67
|
+
def mysql_to_redshift
|
68
|
+
ApiClient.new.send_transport(:mysql, :redshift, options.dup)
|
69
|
+
end
|
70
|
+
|
71
|
+
method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
|
72
|
+
desc 'job_show', 'Show job log'
|
73
|
+
|
74
|
+
def job_show
|
75
|
+
ApiClient.new.job_show(options.dup)
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
metadata
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: holistics
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Thanh Dinh Khac
|
8
|
+
- Huy Nguyen
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2015-11-16 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: activesupport
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '4.2'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '4.2'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: httparty
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0.13'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0.13'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: thor
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0.19'
|
49
|
+
type: :runtime
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0.19'
|
56
|
+
description: CLI interface for Holistics
|
57
|
+
email: huy@holistics.io
|
58
|
+
executables:
|
59
|
+
- holistics
|
60
|
+
extensions: []
|
61
|
+
extra_rdoc_files: []
|
62
|
+
files:
|
63
|
+
- ".document"
|
64
|
+
- README.md
|
65
|
+
- bin/holistics
|
66
|
+
- holistics.gemspec
|
67
|
+
- lib/holistics.rb
|
68
|
+
- lib/holistics/api_client.rb
|
69
|
+
- lib/holistics/cucumber/vcr.rb
|
70
|
+
- lib/holistics/tabular_formatter.rb
|
71
|
+
- lib/holistics/version.rb
|
72
|
+
homepage: http://rubygems.org/gems/holistics-cli
|
73
|
+
licenses:
|
74
|
+
- GPL
|
75
|
+
metadata: {}
|
76
|
+
post_install_message:
|
77
|
+
rdoc_options: []
|
78
|
+
require_paths:
|
79
|
+
- lib
|
80
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
85
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
requirements: []
|
91
|
+
rubyforge_project:
|
92
|
+
rubygems_version: 2.4.8
|
93
|
+
signing_key:
|
94
|
+
specification_version: 4
|
95
|
+
summary: CLI interface for Holistics
|
96
|
+
test_files: []
|