holistics 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.document +4 -0
- data/README.md +67 -0
- data/bin/holistics +5 -0
- data/holistics.gemspec +26 -0
- data/lib/holistics/api_client.rb +140 -0
- data/lib/holistics/cucumber/vcr.rb +26 -0
- data/lib/holistics/tabular_formatter.rb +58 -0
- data/lib/holistics/version.rb +3 -0
- data/lib/holistics.rb +78 -0
- metadata +96 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2f4151e165215a9fb3c9b3a6317cd9b9ef829f5e
|
4
|
+
data.tar.gz: 0f0078f67112c8176971d6cfbce5a8094e901f07
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 654ee243f6fb1415300f32f1004563574aef12b3cbdca659dbbe186a34c6f11b98e2792d75847bf8814af6d516eeaacbf661410b1e1d319c8d78e51605c81d60
|
7
|
+
data.tar.gz: 7a892fa848ef8b5b5ac65b9c5cb55c8166a09d5bc61dfe8a1395fe3a201504fa6a7849afe6a1f87efdef7f85032a9f165db366689102f68047e3ded7346ef297
|
data/.document
ADDED
data/README.md
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
# Holistics CLI
|
2
|
+
|
3
|
+
Command-line interface to Holistics API
|
4
|
+
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
Install gem from RubyGems
|
9
|
+
|
10
|
+
$ gem install holistics
|
11
|
+
|
12
|
+
|
13
|
+
Then perform authentication:
|
14
|
+
|
15
|
+
$ holistics login [token]
|
16
|
+
Authenticating token...
|
17
|
+
Authentication successful. Info:
|
18
|
+
- ID: 1
|
19
|
+
- Email: admin@taxibooking.com
|
20
|
+
|
21
|
+
## Data Transport commands
|
22
|
+
|
23
|
+
### List existing data sources:
|
24
|
+
|
25
|
+
$ holistics ds_list
|
26
|
+
Listing all data sources...
|
27
|
+
| ID | Type | Name |
|
28
|
+
|----+------------+---------------|
|
29
|
+
| 1 | PostgreSQL | Production DB |
|
30
|
+
| 2 | Redshift | Analytics DB |
|
31
|
+
|
32
|
+
### Transport data from Redshift to Postgres:
|
33
|
+
|
34
|
+
$ holistics rs_to_pg -s <source_id> -d <dest_id> -t <table_name>
|
35
|
+
|
36
|
+
Example:
|
37
|
+
|
38
|
+
$ holistics rs_to_pg -s 1 -d 2 -t public.users
|
39
|
+
Submitting transport job ...
|
40
|
+
Job submitted. Job ID: 738.
|
41
|
+
[job:738] Status: queued
|
42
|
+
[job:738] Status: queued
|
43
|
+
[job:738] Status: queued
|
44
|
+
[job:738] Status: running
|
45
|
+
[job:738] Reading schema of table public.users ...
|
46
|
+
[job:738] Counting table's records ...
|
47
|
+
[job:738] - Record count: 6,783
|
48
|
+
[job:738] Creating temporary table public.temp_users_blah on destination ...
|
49
|
+
[job:738] Getting table data from source ...
|
50
|
+
[job:738] Loading data into Redshift table public.temp_users_blah ...
|
51
|
+
[job:738] Hot-swapping with Redshift table public.users ...
|
52
|
+
[job:738] Done.
|
53
|
+
|
54
|
+
### Custom table build transport
|
55
|
+
|
56
|
+
See `samples/clicks_mysql_to_redshift.json` for details of transport configs.
|
57
|
+
|
58
|
+
$ holistics mysql_to_redshift -c samples/clicks_mysql_to_redshift.json
|
59
|
+
|
60
|
+
|
61
|
+
|
62
|
+
|
63
|
+
### Mechanism:
|
64
|
+
|
65
|
+
* Submit a POST request to `/transports`. Will check and return a Job ID.
|
66
|
+
* Keep pinging
|
67
|
+
|
data/bin/holistics
ADDED
data/holistics.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
$LOAD_PATH << File.expand_path('../lib', __FILE__)
|
2
|
+
require 'holistics/version'
|
3
|
+
|
4
|
+
Gem::Specification.new do |spec|
|
5
|
+
spec.name = 'holistics'
|
6
|
+
spec.version = Holistics::VERSION
|
7
|
+
spec.date = '2015-11-16'
|
8
|
+
spec.description = 'CLI interface for Holistics'
|
9
|
+
spec.summary = spec.description
|
10
|
+
|
11
|
+
spec.authors = ['Thanh Dinh Khac', 'Huy Nguyen']
|
12
|
+
spec.email = 'huy@holistics.io'
|
13
|
+
|
14
|
+
spec.homepage = 'http://rubygems.org/gems/holistics-cli'
|
15
|
+
spec.license = 'GPL'
|
16
|
+
|
17
|
+
spec.files = %w[.document holistics.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
|
18
|
+
spec.require_paths = ['lib']
|
19
|
+
|
20
|
+
spec.executables = ['holistics']
|
21
|
+
|
22
|
+
spec.add_dependency 'activesupport', '~> 4.2'
|
23
|
+
spec.add_dependency 'httparty', '~> 0.13'
|
24
|
+
spec.add_dependency 'thor', '~> 0.19'
|
25
|
+
|
26
|
+
end
|
@@ -0,0 +1,140 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
require 'holistics/tabular_formatter'
|
3
|
+
|
4
|
+
class ApiClient
|
5
|
+
SERVER_URL = 'https://secure.holistics.io/'
|
6
|
+
|
7
|
+
def login token
|
8
|
+
puts 'Authenticating token...'
|
9
|
+
url = api_url_for('users/info.json', token)
|
10
|
+
response = HTTParty.get(url)
|
11
|
+
|
12
|
+
if response.code != 200
|
13
|
+
puts 'Error authenticating. Please check your token again.'
|
14
|
+
exit 1
|
15
|
+
end
|
16
|
+
|
17
|
+
parsed = JSON.parse(response.body)
|
18
|
+
puts 'Authentication successful. Info:'
|
19
|
+
puts "- ID: #{parsed['id']}"
|
20
|
+
puts "- Email: #{parsed['email']}"
|
21
|
+
|
22
|
+
file_path = File.join(ENV['HOME'], '.holistics.yml')
|
23
|
+
|
24
|
+
File.write(file_path, token)
|
25
|
+
end
|
26
|
+
|
27
|
+
def job_show options
|
28
|
+
job_id = options[:job_id]
|
29
|
+
tail_job(job_id)
|
30
|
+
end
|
31
|
+
|
32
|
+
def ds_list
|
33
|
+
url = api_url_for('data_sources.json')
|
34
|
+
response = HTTParty.get(url)
|
35
|
+
|
36
|
+
if response.code != 200
|
37
|
+
puts "Error retrieving list of data sources. Code: #{response.code}"
|
38
|
+
puts response.body
|
39
|
+
exit 1
|
40
|
+
end
|
41
|
+
|
42
|
+
parsed = JSON.parse(response.body)
|
43
|
+
|
44
|
+
table = [%w(ID Type Name)]
|
45
|
+
rows = parsed.map { |record| [record['id'], record['dbtype'], record['name']] }
|
46
|
+
table.concat(rows)
|
47
|
+
|
48
|
+
puts TabularFormatter.new(table).to_pretty_table
|
49
|
+
end
|
50
|
+
|
51
|
+
def send_transport(from_ds_type, dest_ds_type, options)
|
52
|
+
puts 'Submitting transport job ...'
|
53
|
+
params = build_submit_params(dest_ds_type, from_ds_type, options)
|
54
|
+
response = submit_transport_job(params)
|
55
|
+
|
56
|
+
if response.code != 200
|
57
|
+
puts "Error submitting transport job. Code: #{response.code}"
|
58
|
+
puts response.body
|
59
|
+
exit 1
|
60
|
+
end
|
61
|
+
|
62
|
+
parsed = JSON.parse(response.body)
|
63
|
+
job_id = parsed['job_id']
|
64
|
+
puts "Job submitted. Job ID: #{job_id}."
|
65
|
+
|
66
|
+
tail_job(job_id)
|
67
|
+
end
|
68
|
+
|
69
|
+
def tail_job(job_id)
|
70
|
+
last_ts = ''
|
71
|
+
while true
|
72
|
+
response = fetch_job_status(job_id)
|
73
|
+
parsed = JSON.parse(response.body)
|
74
|
+
logs = parsed['job_logs']
|
75
|
+
|
76
|
+
select_logs = logs.select { |log| log['created_at'] > last_ts }
|
77
|
+
select_logs.each do |log|
|
78
|
+
print_log(log)
|
79
|
+
end
|
80
|
+
last_ts = logs.last['created_at'] if logs.size > 0
|
81
|
+
break unless has_more?(parsed['status'])
|
82
|
+
sleep 1
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def fetch_job_status(job_id)
|
87
|
+
HTTParty.get(api_url_for("jobs/#{job_id}.json"))
|
88
|
+
end
|
89
|
+
|
90
|
+
def has_more?(status)
|
91
|
+
%w(running created).include?(status)
|
92
|
+
end
|
93
|
+
|
94
|
+
def submit_transport_job(params)
|
95
|
+
HTTParty.post(server_url + 'transports/submit.json', body: params.to_json, headers: {'Content-Type' => 'application/json'})
|
96
|
+
end
|
97
|
+
|
98
|
+
def server_url
|
99
|
+
host = (ENV['HOST'] || SERVER_URL).dup
|
100
|
+
host += '/' if host[-1] != '/'
|
101
|
+
host
|
102
|
+
end
|
103
|
+
|
104
|
+
def build_submit_params(dest_ds_type, from_ds_type, options)
|
105
|
+
params = options.merge(from_ds_type: from_ds_type, dest_ds_type: dest_ds_type, _utoken: get_key)
|
106
|
+
|
107
|
+
configs = {}
|
108
|
+
if options[:config_path]
|
109
|
+
config_content = File.read(File.join(ENV['ROOT_PATH'], options[:config_path]))
|
110
|
+
configs = JSON.parse(config_content)
|
111
|
+
end
|
112
|
+
|
113
|
+
configs[:from_table_name] = options[:table_name] if options[:table_name]
|
114
|
+
configs[:dest_table_name] = options[:rename] if options[:rename]
|
115
|
+
|
116
|
+
params[:config] = configs.to_json # should be a string
|
117
|
+
params
|
118
|
+
end
|
119
|
+
|
120
|
+
def authenticated?
|
121
|
+
File.exists?(config_filepath)
|
122
|
+
end
|
123
|
+
|
124
|
+
def get_key
|
125
|
+
File.read(config_filepath).strip
|
126
|
+
end
|
127
|
+
|
128
|
+
def config_filepath
|
129
|
+
File.expand_path('~/.holistics.yml', __FILE__)
|
130
|
+
end
|
131
|
+
|
132
|
+
def api_url_for(path, token = nil)
|
133
|
+
"#{server_url}#{path}?_utoken=#{token || get_key}"
|
134
|
+
end
|
135
|
+
|
136
|
+
def print_log log
|
137
|
+
puts "#{log['created_at'][0..18]} - #{log['level']} - #{log['message']}"
|
138
|
+
end
|
139
|
+
|
140
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# This is a cucumber test helper.
|
2
|
+
# It wrap the running of thor commands around VCR cassette if needed
|
3
|
+
|
4
|
+
if ENV['IS_CUCUMBER']
|
5
|
+
ENV['HOME'] = File.join(ENV['ROOT_PATH'], 'tmp/aruba')
|
6
|
+
end
|
7
|
+
|
8
|
+
if ENV['VCR_CASSETTE']
|
9
|
+
require 'vcr'
|
10
|
+
require 'thor'
|
11
|
+
class Thor::Task
|
12
|
+
def run_with_vcr(instance, args=[])
|
13
|
+
cassette = ENV.delete('VCR_CASSETTE')
|
14
|
+
VCR.configure do |c|
|
15
|
+
c.cassette_library_dir = Holistics.root.join 'features/vcr_cassettes'
|
16
|
+
c.default_cassette_options = {:record => :new_episodes}
|
17
|
+
c.hook_into :webmock
|
18
|
+
end
|
19
|
+
VCR.use_cassette(cassette) do
|
20
|
+
run_without_vcr(instance, args)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
alias_method_chain :run, :vcr
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'active_support/all'
|
3
|
+
|
4
|
+
|
5
|
+
class TabularFormatter
|
6
|
+
|
7
|
+
def initialize input_data, options = {}
|
8
|
+
@input_data = input_data
|
9
|
+
@options = options
|
10
|
+
@options[:headers] = true if @options[:headers].nil?
|
11
|
+
end
|
12
|
+
|
13
|
+
def to_csv(col_sep = ',')
|
14
|
+
CSV.generate(col_sep: col_sep) do |csv|
|
15
|
+
@input_data.each { |row| csv << row }
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_tsv
|
20
|
+
to_csv("\t")
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_pretty_table
|
24
|
+
return '' if @input_data.blank? or @input_data.first.empty?
|
25
|
+
|
26
|
+
max_column_lengths = Array.new(@input_data.first.length, 0)
|
27
|
+
@input_data.each do |row|
|
28
|
+
row.each_with_index do |cell, index|
|
29
|
+
max_column_lengths[index] = [cell.to_s.length, max_column_lengths[index]].max
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
output = []
|
34
|
+
@input_data.each do |row|
|
35
|
+
output << print_table_row(max_column_lengths, row)
|
36
|
+
end
|
37
|
+
|
38
|
+
divider = max_column_lengths.inject("") do |str, l|
|
39
|
+
str << "-#{'-' * l}-+"
|
40
|
+
end.chomp!("+")
|
41
|
+
divider = divider[1..-1]
|
42
|
+
output.insert(1,divider)
|
43
|
+
output.join("\n").concat("\n")
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def print_table_row column_lengths, row
|
49
|
+
row_string = ''
|
50
|
+
row.each_with_index do |cell, index|
|
51
|
+
padding = column_lengths[index] - cell.to_s.length
|
52
|
+
row_string << " #{cell}#{' ' * padding} |"
|
53
|
+
end
|
54
|
+
row_string.chomp!("|")
|
55
|
+
row_string.strip!
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
data/lib/holistics.rb
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'holistics/api_client'
|
3
|
+
require 'holistics/cucumber/vcr'
|
4
|
+
|
5
|
+
class Holistics
|
6
|
+
def self.root
|
7
|
+
Pathname.new(__FILE__).parent.parent
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
# Right now we can't do `holistics ds:list`
|
12
|
+
# Known issue: https://github.com/erikhuda/thor/issues/249
|
13
|
+
# Either we use other libs, or figure out an elegant way to do it without
|
14
|
+
|
15
|
+
class HolisticsRunner < Thor
|
16
|
+
desc 'login', 'Perform authentication'
|
17
|
+
|
18
|
+
def login token
|
19
|
+
client = ApiClient.new
|
20
|
+
client.login(token)
|
21
|
+
end
|
22
|
+
|
23
|
+
desc 'ds_list', 'List all data sources'
|
24
|
+
|
25
|
+
def ds_list
|
26
|
+
ApiClient.new.ds_list
|
27
|
+
end
|
28
|
+
|
29
|
+
method_option :from_ds, aliases: '-s', type: :string, required: true, desc: 'From data source (ID)'
|
30
|
+
method_option :dest_ds, aliases: '-d', type: :string, required: true, desc: 'To data source (ID)'
|
31
|
+
method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table name'
|
32
|
+
desc 'rs_to_pg', 'Transport data from Amazon Redshift data source to PostgreSQL'
|
33
|
+
|
34
|
+
def rs_to_pg
|
35
|
+
ApiClient.new.send_transport(:redshift, :postgresql, options.dup)
|
36
|
+
end
|
37
|
+
|
38
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
|
39
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
|
40
|
+
method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table names'
|
41
|
+
desc 'pg_to_rs', 'Transport data from PostgreSQL to Amazon Redshift'
|
42
|
+
|
43
|
+
def pg_to_rs
|
44
|
+
ApiClient.new.send_transport(:postgresql, :redshift, options.dup)
|
45
|
+
end
|
46
|
+
|
47
|
+
method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
|
48
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
|
49
|
+
method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table names'
|
50
|
+
method_option :rename, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
|
51
|
+
desc 'pg_to_pg', 'Transport data from PostgreSQL to PostgreSQL'
|
52
|
+
|
53
|
+
def pg_to_pg
|
54
|
+
ApiClient.new.send_transport(:postgresql, :postgresql, options.dup)
|
55
|
+
end
|
56
|
+
|
57
|
+
method_option :config_path, aliases: '-c', type: :string, required: true, desc: 'Path to transport config (JSON) file'
|
58
|
+
desc 'mysql_to_postgres', 'Transport data from MySQL to Postgres'
|
59
|
+
|
60
|
+
def mysql_to_postgres
|
61
|
+
ApiClient.new.send_transport(:mysql, :postgresql, options.dup)
|
62
|
+
end
|
63
|
+
|
64
|
+
method_option :config_path, aliases: '-c', type: :string, required: true, desc: 'Path to transport config (JSON) file'
|
65
|
+
desc 'mysql_to_redshift', 'Transport data from MySQL to Redshift'
|
66
|
+
|
67
|
+
def mysql_to_redshift
|
68
|
+
ApiClient.new.send_transport(:mysql, :redshift, options.dup)
|
69
|
+
end
|
70
|
+
|
71
|
+
method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
|
72
|
+
desc 'job_show', 'Show job log'
|
73
|
+
|
74
|
+
def job_show
|
75
|
+
ApiClient.new.job_show(options.dup)
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
metadata
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: holistics
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Thanh Dinh Khac
|
8
|
+
- Huy Nguyen
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2015-11-16 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: activesupport
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '4.2'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '4.2'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: httparty
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0.13'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0.13'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: thor
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0.19'
|
49
|
+
type: :runtime
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0.19'
|
56
|
+
description: CLI interface for Holistics
|
57
|
+
email: huy@holistics.io
|
58
|
+
executables:
|
59
|
+
- holistics
|
60
|
+
extensions: []
|
61
|
+
extra_rdoc_files: []
|
62
|
+
files:
|
63
|
+
- ".document"
|
64
|
+
- README.md
|
65
|
+
- bin/holistics
|
66
|
+
- holistics.gemspec
|
67
|
+
- lib/holistics.rb
|
68
|
+
- lib/holistics/api_client.rb
|
69
|
+
- lib/holistics/cucumber/vcr.rb
|
70
|
+
- lib/holistics/tabular_formatter.rb
|
71
|
+
- lib/holistics/version.rb
|
72
|
+
homepage: http://rubygems.org/gems/holistics-cli
|
73
|
+
licenses:
|
74
|
+
- GPL
|
75
|
+
metadata: {}
|
76
|
+
post_install_message:
|
77
|
+
rdoc_options: []
|
78
|
+
require_paths:
|
79
|
+
- lib
|
80
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
85
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
requirements: []
|
91
|
+
rubyforge_project:
|
92
|
+
rubygems_version: 2.4.8
|
93
|
+
signing_key:
|
94
|
+
specification_version: 4
|
95
|
+
summary: CLI interface for Holistics
|
96
|
+
test_files: []
|