holistics 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2f4151e165215a9fb3c9b3a6317cd9b9ef829f5e
4
+ data.tar.gz: 0f0078f67112c8176971d6cfbce5a8094e901f07
5
+ SHA512:
6
+ metadata.gz: 654ee243f6fb1415300f32f1004563574aef12b3cbdca659dbbe186a34c6f11b98e2792d75847bf8814af6d516eeaacbf661410b1e1d319c8d78e51605c81d60
7
+ data.tar.gz: 7a892fa848ef8b5b5ac65b9c5cb55c8166a09d5bc61dfe8a1395fe3a201504fa6a7849afe6a1f87efdef7f85032a9f165db366689102f68047e3ded7346ef297
data/.document ADDED
@@ -0,0 +1,4 @@
1
+ lib/*.rb
2
+ lib/**/*.rb
3
+ -
4
+
data/README.md ADDED
@@ -0,0 +1,67 @@
1
+ # Holistics CLI
2
+
3
+ Command-line interface to Holistics API
4
+
5
+
6
+ ## Installation
7
+
8
+ Install gem from RubyGems
9
+
10
+ $ gem install holistics
11
+
12
+
13
+ Then perform authentication:
14
+
15
+ $ holistics login [token]
16
+ Authenticating token...
17
+ Authentication successful. Info:
18
+ - ID: 1
19
+ - Email: admin@taxibooking.com
20
+
21
+ ## Data Transport commands
22
+
23
+ ### List existing data sources:
24
+
25
+ $ holistics ds_list
26
+ Listing all data sources...
27
+ | ID | Type | Name |
28
+ |----+------------+---------------|
29
+ | 1 | PostgreSQL | Production DB |
30
+ | 2 | Redshift | Analytics DB |
31
+
32
+ ### Transport data from Redshift to Postgres:
33
+
34
+ $ holistics rs_to_pg -s <source_id> -d <dest_id> -t <table_name>
35
+
36
+ Example:
37
+
38
+ $ holistics rs_to_pg -s 1 -d 2 -t public.users
39
+ Submitting transport job ...
40
+ Job submitted. Job ID: 738.
41
+ [job:738] Status: queued
42
+ [job:738] Status: queued
43
+ [job:738] Status: queued
44
+ [job:738] Status: running
45
+ [job:738] Reading schema of table public.users ...
46
+ [job:738] Counting table's records ...
47
+ [job:738] - Record count: 6,783
48
+ [job:738] Creating temporary table public.temp_users_blah on destination ...
49
+ [job:738] Getting table data from source ...
50
+ [job:738] Loading data into Redshift table public.temp_users_blah ...
51
+ [job:738] Hot-swapping with Redshift table public.users ...
52
+ [job:738] Done.
53
+
54
+ ### Custom table build transport
55
+
56
+ See `samples/clicks_mysql_to_redshift.json` for details of transport configs.
57
+
58
+ $ holistics mysql_to_redshift -c samples/clicks_mysql_to_redshift.json
59
+
60
+
61
+
62
+
63
+ ### Mechanism:
64
+
65
+ * Submit a POST request to `/transports`. Will check and return a Job ID.
66
+ * Keep pinging
67
+
data/bin/holistics ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+ ENV['ROOT_PATH'] = File.expand_path('../../', __FILE__)
3
+
4
+ require 'holistics'
5
+ HolisticsRunner.start
data/holistics.gemspec ADDED
@@ -0,0 +1,26 @@
1
+ $LOAD_PATH << File.expand_path('../lib', __FILE__)
2
+ require 'holistics/version'
3
+
4
+ Gem::Specification.new do |spec|
5
+ spec.name = 'holistics'
6
+ spec.version = Holistics::VERSION
7
+ spec.date = '2015-11-16'
8
+ spec.description = 'CLI interface for Holistics'
9
+ spec.summary = spec.description
10
+
11
+ spec.authors = ['Thanh Dinh Khac', 'Huy Nguyen']
12
+ spec.email = 'huy@holistics.io'
13
+
14
+ spec.homepage = 'http://rubygems.org/gems/holistics-cli'
15
+ spec.license = 'GPL'
16
+
17
+ spec.files = %w[.document holistics.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
18
+ spec.require_paths = ['lib']
19
+
20
+ spec.executables = ['holistics']
21
+
22
+ spec.add_dependency 'activesupport', '~> 4.2'
23
+ spec.add_dependency 'httparty', '~> 0.13'
24
+ spec.add_dependency 'thor', '~> 0.19'
25
+
26
+ end
@@ -0,0 +1,140 @@
1
+ require 'httparty'
2
+ require 'holistics/tabular_formatter'
3
+
4
+ class ApiClient
5
+ SERVER_URL = 'https://secure.holistics.io/'
6
+
7
+ def login token
8
+ puts 'Authenticating token...'
9
+ url = api_url_for('users/info.json', token)
10
+ response = HTTParty.get(url)
11
+
12
+ if response.code != 200
13
+ puts 'Error authenticating. Please check your token again.'
14
+ exit 1
15
+ end
16
+
17
+ parsed = JSON.parse(response.body)
18
+ puts 'Authentication successful. Info:'
19
+ puts "- ID: #{parsed['id']}"
20
+ puts "- Email: #{parsed['email']}"
21
+
22
+ file_path = File.join(ENV['HOME'], '.holistics.yml')
23
+
24
+ File.write(file_path, token)
25
+ end
26
+
27
+ def job_show options
28
+ job_id = options[:job_id]
29
+ tail_job(job_id)
30
+ end
31
+
32
+ def ds_list
33
+ url = api_url_for('data_sources.json')
34
+ response = HTTParty.get(url)
35
+
36
+ if response.code != 200
37
+ puts "Error retrieving list of data sources. Code: #{response.code}"
38
+ puts response.body
39
+ exit 1
40
+ end
41
+
42
+ parsed = JSON.parse(response.body)
43
+
44
+ table = [%w(ID Type Name)]
45
+ rows = parsed.map { |record| [record['id'], record['dbtype'], record['name']] }
46
+ table.concat(rows)
47
+
48
+ puts TabularFormatter.new(table).to_pretty_table
49
+ end
50
+
51
+ def send_transport(from_ds_type, dest_ds_type, options)
52
+ puts 'Submitting transport job ...'
53
+ params = build_submit_params(dest_ds_type, from_ds_type, options)
54
+ response = submit_transport_job(params)
55
+
56
+ if response.code != 200
57
+ puts "Error submitting transport job. Code: #{response.code}"
58
+ puts response.body
59
+ exit 1
60
+ end
61
+
62
+ parsed = JSON.parse(response.body)
63
+ job_id = parsed['job_id']
64
+ puts "Job submitted. Job ID: #{job_id}."
65
+
66
+ tail_job(job_id)
67
+ end
68
+
69
+ def tail_job(job_id)
70
+ last_ts = ''
71
+ while true
72
+ response = fetch_job_status(job_id)
73
+ parsed = JSON.parse(response.body)
74
+ logs = parsed['job_logs']
75
+
76
+ select_logs = logs.select { |log| log['created_at'] > last_ts }
77
+ select_logs.each do |log|
78
+ print_log(log)
79
+ end
80
+ last_ts = logs.last['created_at'] if logs.size > 0
81
+ break unless has_more?(parsed['status'])
82
+ sleep 1
83
+ end
84
+ end
85
+
86
+ def fetch_job_status(job_id)
87
+ HTTParty.get(api_url_for("jobs/#{job_id}.json"))
88
+ end
89
+
90
+ def has_more?(status)
91
+ %w(running created).include?(status)
92
+ end
93
+
94
+ def submit_transport_job(params)
95
+ HTTParty.post(server_url + 'transports/submit.json', body: params.to_json, headers: {'Content-Type' => 'application/json'})
96
+ end
97
+
98
+ def server_url
99
+ host = (ENV['HOST'] || SERVER_URL).dup
100
+ host += '/' if host[-1] != '/'
101
+ host
102
+ end
103
+
104
+ def build_submit_params(dest_ds_type, from_ds_type, options)
105
+ params = options.merge(from_ds_type: from_ds_type, dest_ds_type: dest_ds_type, _utoken: get_key)
106
+
107
+ configs = {}
108
+ if options[:config_path]
109
+ config_content = File.read(File.join(ENV['ROOT_PATH'], options[:config_path]))
110
+ configs = JSON.parse(config_content)
111
+ end
112
+
113
+ configs[:from_table_name] = options[:table_name] if options[:table_name]
114
+ configs[:dest_table_name] = options[:rename] if options[:rename]
115
+
116
+ params[:config] = configs.to_json # should be a string
117
+ params
118
+ end
119
+
120
+ def authenticated?
121
+ File.exists?(config_filepath)
122
+ end
123
+
124
+ def get_key
125
+ File.read(config_filepath).strip
126
+ end
127
+
128
+ def config_filepath
129
+ File.expand_path('~/.holistics.yml', __FILE__)
130
+ end
131
+
132
+ def api_url_for(path, token = nil)
133
+ "#{server_url}#{path}?_utoken=#{token || get_key}"
134
+ end
135
+
136
+ def print_log log
137
+ puts "#{log['created_at'][0..18]} - #{log['level']} - #{log['message']}"
138
+ end
139
+
140
+ end
@@ -0,0 +1,26 @@
1
+ # This is a cucumber test helper.
2
+ # It wrap the running of thor commands around VCR cassette if needed
3
+
4
+ if ENV['IS_CUCUMBER']
5
+ ENV['HOME'] = File.join(ENV['ROOT_PATH'], 'tmp/aruba')
6
+ end
7
+
8
+ if ENV['VCR_CASSETTE']
9
+ require 'vcr'
10
+ require 'thor'
11
+ class Thor::Task
12
+ def run_with_vcr(instance, args=[])
13
+ cassette = ENV.delete('VCR_CASSETTE')
14
+ VCR.configure do |c|
15
+ c.cassette_library_dir = Holistics.root.join 'features/vcr_cassettes'
16
+ c.default_cassette_options = {:record => :new_episodes}
17
+ c.hook_into :webmock
18
+ end
19
+ VCR.use_cassette(cassette) do
20
+ run_without_vcr(instance, args)
21
+ end
22
+ end
23
+
24
+ alias_method_chain :run, :vcr
25
+ end
26
+ end
@@ -0,0 +1,58 @@
1
+ require 'csv'
2
+ require 'active_support/all'
3
+
4
+
5
+ class TabularFormatter
6
+
7
+ def initialize input_data, options = {}
8
+ @input_data = input_data
9
+ @options = options
10
+ @options[:headers] = true if @options[:headers].nil?
11
+ end
12
+
13
+ def to_csv(col_sep = ',')
14
+ CSV.generate(col_sep: col_sep) do |csv|
15
+ @input_data.each { |row| csv << row }
16
+ end
17
+ end
18
+
19
+ def to_tsv
20
+ to_csv("\t")
21
+ end
22
+
23
+ def to_pretty_table
24
+ return '' if @input_data.blank? or @input_data.first.empty?
25
+
26
+ max_column_lengths = Array.new(@input_data.first.length, 0)
27
+ @input_data.each do |row|
28
+ row.each_with_index do |cell, index|
29
+ max_column_lengths[index] = [cell.to_s.length, max_column_lengths[index]].max
30
+ end
31
+ end
32
+
33
+ output = []
34
+ @input_data.each do |row|
35
+ output << print_table_row(max_column_lengths, row)
36
+ end
37
+
38
+ divider = max_column_lengths.inject("") do |str, l|
39
+ str << "-#{'-' * l}-+"
40
+ end.chomp!("+")
41
+ divider = divider[1..-1]
42
+ output.insert(1,divider)
43
+ output.join("\n").concat("\n")
44
+ end
45
+
46
+ private
47
+
48
+ def print_table_row column_lengths, row
49
+ row_string = ''
50
+ row.each_with_index do |cell, index|
51
+ padding = column_lengths[index] - cell.to_s.length
52
+ row_string << " #{cell}#{' ' * padding} |"
53
+ end
54
+ row_string.chomp!("|")
55
+ row_string.strip!
56
+ end
57
+
58
+ end
@@ -0,0 +1,3 @@
1
+ class Holistics
2
+ VERSION = '0.0.1'
3
+ end
data/lib/holistics.rb ADDED
@@ -0,0 +1,78 @@
1
+ require 'thor'
2
+ require 'holistics/api_client'
3
+ require 'holistics/cucumber/vcr'
4
+
5
+ class Holistics
6
+ def self.root
7
+ Pathname.new(__FILE__).parent.parent
8
+ end
9
+ end
10
+
11
+ # Right now we can't do `holistics ds:list`
12
+ # Known issue: https://github.com/erikhuda/thor/issues/249
13
+ # Either we use other libs, or figure out an elegant way to do it without
14
+
15
+ class HolisticsRunner < Thor
16
+ desc 'login', 'Perform authentication'
17
+
18
+ def login token
19
+ client = ApiClient.new
20
+ client.login(token)
21
+ end
22
+
23
+ desc 'ds_list', 'List all data sources'
24
+
25
+ def ds_list
26
+ ApiClient.new.ds_list
27
+ end
28
+
29
+ method_option :from_ds, aliases: '-s', type: :string, required: true, desc: 'From data source (ID)'
30
+ method_option :dest_ds, aliases: '-d', type: :string, required: true, desc: 'To data source (ID)'
31
+ method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table name'
32
+ desc 'rs_to_pg', 'Transport data from Amazon Redshift data source to PostgreSQL'
33
+
34
+ def rs_to_pg
35
+ ApiClient.new.send_transport(:redshift, :postgresql, options.dup)
36
+ end
37
+
38
+ method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
39
+ method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
40
+ method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table names'
41
+ desc 'pg_to_rs', 'Transport data from PostgreSQL to Amazon Redshift'
42
+
43
+ def pg_to_rs
44
+ ApiClient.new.send_transport(:postgresql, :redshift, options.dup)
45
+ end
46
+
47
+ method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
48
+ method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
49
+ method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table names'
50
+ method_option :rename, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
51
+ desc 'pg_to_pg', 'Transport data from PostgreSQL to PostgreSQL'
52
+
53
+ def pg_to_pg
54
+ ApiClient.new.send_transport(:postgresql, :postgresql, options.dup)
55
+ end
56
+
57
+ method_option :config_path, aliases: '-c', type: :string, required: true, desc: 'Path to transport config (JSON) file'
58
+ desc 'mysql_to_postgres', 'Transport data from MySQL to Postgres'
59
+
60
+ def mysql_to_postgres
61
+ ApiClient.new.send_transport(:mysql, :postgresql, options.dup)
62
+ end
63
+
64
+ method_option :config_path, aliases: '-c', type: :string, required: true, desc: 'Path to transport config (JSON) file'
65
+ desc 'mysql_to_redshift', 'Transport data from MySQL to Redshift'
66
+
67
+ def mysql_to_redshift
68
+ ApiClient.new.send_transport(:mysql, :redshift, options.dup)
69
+ end
70
+
71
+ method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
72
+ desc 'job_show', 'Show job log'
73
+
74
+ def job_show
75
+ ApiClient.new.job_show(options.dup)
76
+ end
77
+
78
+ end
metadata ADDED
@@ -0,0 +1,96 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: holistics
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Thanh Dinh Khac
8
+ - Huy Nguyen
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2015-11-16 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: activesupport
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - "~>"
19
+ - !ruby/object:Gem::Version
20
+ version: '4.2'
21
+ type: :runtime
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - "~>"
26
+ - !ruby/object:Gem::Version
27
+ version: '4.2'
28
+ - !ruby/object:Gem::Dependency
29
+ name: httparty
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - "~>"
33
+ - !ruby/object:Gem::Version
34
+ version: '0.13'
35
+ type: :runtime
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - "~>"
40
+ - !ruby/object:Gem::Version
41
+ version: '0.13'
42
+ - !ruby/object:Gem::Dependency
43
+ name: thor
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - "~>"
47
+ - !ruby/object:Gem::Version
48
+ version: '0.19'
49
+ type: :runtime
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - "~>"
54
+ - !ruby/object:Gem::Version
55
+ version: '0.19'
56
+ description: CLI interface for Holistics
57
+ email: huy@holistics.io
58
+ executables:
59
+ - holistics
60
+ extensions: []
61
+ extra_rdoc_files: []
62
+ files:
63
+ - ".document"
64
+ - README.md
65
+ - bin/holistics
66
+ - holistics.gemspec
67
+ - lib/holistics.rb
68
+ - lib/holistics/api_client.rb
69
+ - lib/holistics/cucumber/vcr.rb
70
+ - lib/holistics/tabular_formatter.rb
71
+ - lib/holistics/version.rb
72
+ homepage: http://rubygems.org/gems/holistics-cli
73
+ licenses:
74
+ - GPL
75
+ metadata: {}
76
+ post_install_message:
77
+ rdoc_options: []
78
+ require_paths:
79
+ - lib
80
+ required_ruby_version: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - ">="
83
+ - !ruby/object:Gem::Version
84
+ version: '0'
85
+ required_rubygems_version: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ requirements: []
91
+ rubyforge_project:
92
+ rubygems_version: 2.4.8
93
+ signing_key:
94
+ specification_version: 4
95
+ summary: CLI interface for Holistics
96
+ test_files: []