holistics 0.1.0 → 0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -1
- data/lib/holistics/api_client.rb +46 -9
- data/lib/holistics/helpers/http_request.rb +21 -1
- data/lib/holistics/version.rb +1 -1
- data/lib/holistics.rb +2 -0
- data/lib/import.rb +20 -0
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f326702c9d1af543ba76f9790aff1b1944fb6e4d
|
4
|
+
data.tar.gz: cf04ab410f5a2de32dc9693490bbfcc5364acceb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: abf41026bc233126273fb08f90f94d0f7295c9dcc53c5589c1acf72563cfc01ca410c2beb98bda90d1b4fc486867b67283119eaab969d9a6a494c6d3e3244df3
|
7
|
+
data.tar.gz: f8c7f05f08afcb88b8ec19c01600bd80a12bb8536004b435fbf0ba9faed3947483249a7dc19e1566ef2ccc6eaa92426705f88f7f9374a1e44f8ddc6ac861ad69
|
data/CHANGELOG.md
CHANGED
@@ -9,4 +9,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
|
|
9
9
|
* Support generation of transport config files from CLI: `holistics generate_configs -s <src_ds_id> -d <dest_ds_id> -t <tables> [-o <output>]`
|
10
10
|
|
11
11
|
## v0.1.0
|
12
|
-
* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
|
12
|
+
* Modularized transport and data_sources commands. Rename existing transport command to transport_old.
|
13
|
+
|
14
|
+
## v0.2
|
15
|
+
* Support command to import csv into table
|
16
|
+
|
17
|
+
|
data/lib/holistics/api_client.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
require 'httparty'
|
2
1
|
require 'tabular_formatter'
|
3
2
|
require 'yaml'
|
4
3
|
require 'json'
|
@@ -9,16 +8,34 @@ require 'holistics/helpers/job_manager'
|
|
9
8
|
module Holistics
|
10
9
|
class ApiClient
|
11
10
|
|
12
|
-
def
|
13
|
-
|
14
|
-
|
11
|
+
def import_csv options
|
12
|
+
local_filepath = options[:filepath]
|
13
|
+
dest_ds_id = options[:dest_ds_id]
|
14
|
+
dest_fqname = options[:dest_table_name]
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
16
|
+
begin
|
17
|
+
file = File.open local_filepath
|
18
|
+
rescue
|
19
|
+
STDERR.puts "Could not open file at '#{local_filepath}'."
|
20
|
+
puts "Invalid file path. Please check your file path."
|
19
21
|
|
20
|
-
|
21
|
-
|
22
|
+
return
|
23
|
+
end
|
24
|
+
|
25
|
+
puts "Read csv file '#{local_filepath}'."
|
26
|
+
|
27
|
+
params = {
|
28
|
+
dest_fqname: dest_fqname,
|
29
|
+
dest_ds_id: dest_ds_id,
|
30
|
+
_utoken: auth_info.get_token_from_gconfig
|
31
|
+
}
|
32
|
+
|
33
|
+
json = http_request.post_csv 'data_imports/import_csv.json', params, file, 'Error uploading CSV file'
|
34
|
+
|
35
|
+
job_id = json['message']['job_id']
|
36
|
+
|
37
|
+
puts "Job submitted. Job ID: #{job_id}."
|
38
|
+
job_manager.tail_logs job_id.to_i
|
22
39
|
end
|
23
40
|
|
24
41
|
def ds_list
|
@@ -97,5 +114,25 @@ module Holistics
|
|
97
114
|
STDERR.puts "Error parsing transport config file: #{e.message}"
|
98
115
|
exit 1
|
99
116
|
end
|
117
|
+
|
118
|
+
private
|
119
|
+
|
120
|
+
def file_manager
|
121
|
+
@file_manager ||= Helpers::FileManager.new
|
122
|
+
end
|
123
|
+
|
124
|
+
def job_manager
|
125
|
+
@job_helper ||= Helpers::JobManager.new
|
126
|
+
end
|
127
|
+
|
128
|
+
def http_request
|
129
|
+
@http_helper ||= Helpers::HttpRequest.new
|
130
|
+
end
|
131
|
+
|
132
|
+
def auth_info
|
133
|
+
@auth_info ||= Helpers::AuthInfo.new
|
134
|
+
end
|
135
|
+
|
136
|
+
|
100
137
|
end
|
101
138
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'httparty'
|
2
|
+
require 'net/http/post/multipart'
|
2
3
|
|
3
4
|
module Holistics
|
4
5
|
module Helpers
|
@@ -49,6 +50,21 @@ module Holistics
|
|
49
50
|
JSON.parse response.body
|
50
51
|
end
|
51
52
|
|
53
|
+
def post_csv url, params, file, msg_if_error = DEFAULT_ERROR_MSG
|
54
|
+
uri = URI.parse endpoint_for url
|
55
|
+
csv = UploadIO.new file, "text/csv", File.basename(file.path)
|
56
|
+
params = params.merge file: csv
|
57
|
+
request = Net::HTTP::Post::Multipart.new uri.path, params
|
58
|
+
|
59
|
+
response = Net::HTTP.start(uri.host, uri.port) do |http|
|
60
|
+
http.request(request)
|
61
|
+
end
|
62
|
+
|
63
|
+
exit_if_error(msg_if_error, response)
|
64
|
+
|
65
|
+
JSON.parse response.body
|
66
|
+
end
|
67
|
+
|
52
68
|
private
|
53
69
|
|
54
70
|
def exit_if_error (message, response)
|
@@ -63,7 +79,11 @@ module Holistics
|
|
63
79
|
end
|
64
80
|
|
65
81
|
def response_has_error?(response)
|
66
|
-
response.code != 200
|
82
|
+
response.code.to_i != 200
|
83
|
+
end
|
84
|
+
|
85
|
+
def endpoint_for route
|
86
|
+
"#{server_url}/#{route}"
|
67
87
|
end
|
68
88
|
end
|
69
89
|
end
|
data/lib/holistics/version.rb
CHANGED
data/lib/holistics.rb
CHANGED
@@ -5,6 +5,7 @@ require 'holistics/api_client'
|
|
5
5
|
require 'holistics/custom_logger'
|
6
6
|
require 'holistics/auth_api_client'
|
7
7
|
|
8
|
+
require 'import'
|
8
9
|
require 'transport'
|
9
10
|
require 'data_sources'
|
10
11
|
|
@@ -31,6 +32,7 @@ module Holistics
|
|
31
32
|
super(args, options, config)
|
32
33
|
end
|
33
34
|
|
35
|
+
register(Holistics::Import, 'import', 'import<command>', "Execute import commands")
|
34
36
|
register(Holistics::Transport, 'transport', 'transport <command>', "Execute transport module's commands")
|
35
37
|
register(Holistics::DataSources, 'data_sources', 'data_sources <command>', "Execute data_sources module's commands")
|
36
38
|
|
data/lib/import.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'thor'
|
2
|
+
|
3
|
+
module Holistics
|
4
|
+
class Import < Thor
|
5
|
+
|
6
|
+
no_commands do
|
7
|
+
def api_client
|
8
|
+
@api_client ||= ApiClient.new
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
method_option :filepath, aliases: '-f', type: :string, required: true, desc: 'Path to CSV file'
|
13
|
+
method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'Destination data source'
|
14
|
+
method_option :dest_table_name, aliases: '-t', type: :string, required: true, desc: 'Specify destination table to write to'
|
15
|
+
desc 'csv', 'Import a local CSV file to destination server'
|
16
|
+
def csv
|
17
|
+
api_client.import_csv(options.dup)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: holistics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: '0.2'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thanh Dinh Khac
|
@@ -88,6 +88,7 @@ files:
|
|
88
88
|
- lib/holistics/helpers/http_request.rb
|
89
89
|
- lib/holistics/helpers/job_manager.rb
|
90
90
|
- lib/holistics/version.rb
|
91
|
+
- lib/import.rb
|
91
92
|
- lib/tabular_formatter.rb
|
92
93
|
- lib/transport.rb
|
93
94
|
- lib/vcr_helper.rb
|