idata 0.2.5 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/iload +102 -72
- data/idata.gemspec +1 -1
- data/lib/idata/version.rb +1 -1
- metadata +2 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: aea47287cb8de9e1f47ed792a229bc894134f27c
|
4
|
+
data.tar.gz: f417b2e5b617104aa140f573a5a6cad3c863227d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 21fc196f3067f17dade972c9ed3776b9fc04fd3efd505ae9e25f8178c7111bfce36a45f2e5e77258f41839cf0c076887d736c041b5e0522535775c8bcb7832a3
|
7
|
+
data.tar.gz: 6dbd7e130723fdcd2b764612e65da385bcdc9b023417afa3589d5e7b65b4629f4e5dcef026643cfe8d0a0b5c366dec8a6dc3a4ac241417750ab3fee49d3cca7f
|
data/bin/iload
CHANGED
@@ -17,8 +17,16 @@ require 'fileutils'
|
|
17
17
|
|
18
18
|
SUPPORTED_INPUT_FORMATS = ['CSV', 'FX', 'RPT']
|
19
19
|
POSTGRESQL_PORT = 5432
|
20
|
+
POSTGRESQL_USERNAME = 'postgres'
|
20
21
|
CSV_DEFAULT_DELIMITER = ','
|
21
22
|
CSV_DEFAULT_QUOTE = '"'
|
23
|
+
CSV_DEFAULT_FORMAT = 'CSV'
|
24
|
+
|
25
|
+
|
26
|
+
def error(msg)
|
27
|
+
puts "Error: #{msg}"
|
28
|
+
exit(-1)
|
29
|
+
end
|
22
30
|
|
23
31
|
$options = {}
|
24
32
|
parser = OptionParser.new("", 24) do |opts|
|
@@ -75,6 +83,10 @@ parser = OptionParser.new("", 24) do |opts|
|
|
75
83
|
opts.on("-l", "--listen PORT", "PostgreSQL listen port (default to 5432)") do |v|
|
76
84
|
$options[:listen] = v
|
77
85
|
end
|
86
|
+
|
87
|
+
opts.on("-c", "--client CLIENT", "Client name") do |v|
|
88
|
+
$options[:client] = v
|
89
|
+
end
|
78
90
|
|
79
91
|
opts.on_tail('--help', 'Displays this help') do
|
80
92
|
puts opts, "", help
|
@@ -104,76 +116,105 @@ rescue SystemExit => ex
|
|
104
116
|
exit
|
105
117
|
end
|
106
118
|
|
107
|
-
#
|
108
|
-
$options[:
|
109
|
-
$options[:
|
110
|
-
$options[:
|
111
|
-
$options[:
|
112
|
-
$options[:
|
119
|
+
# default
|
120
|
+
$options[:format].upcase! if $options[:format]
|
121
|
+
$options[:format] ||= CSV_DEFAULT_FORMAT
|
122
|
+
$options[:listen] ||= POSTGRESQL_PORT unless $options[:client]
|
123
|
+
$options[:username] ||= POSTGRESQL_USERNAME unless $options[:client]
|
124
|
+
$options[:delim] ||= CSV_DEFAULT_DELIMITER
|
125
|
+
$options[:quote] ||= CSV_DEFAULT_QUOTE
|
126
|
+
$options[:drop] ||= false
|
113
127
|
|
114
128
|
# validate parameters
|
115
129
|
if $options[:input].nil?
|
116
|
-
|
117
|
-
exit
|
130
|
+
error "please specify input file: -i"
|
118
131
|
end
|
119
132
|
|
120
|
-
if $options[:
|
121
|
-
|
122
|
-
exit
|
133
|
+
if File.exists?($options[:input]) && File.directory?($options[:input])
|
134
|
+
error "`#{$options[:input]}` is a directory! input must be a file"
|
123
135
|
end
|
124
136
|
|
125
|
-
|
126
|
-
$options[:
|
137
|
+
unless File.exists?($options[:input])
|
138
|
+
error "file `#{$options[:input]}` not found!"
|
139
|
+
end
|
127
140
|
|
128
141
|
if !SUPPORTED_INPUT_FORMATS.include?($options[:format])
|
129
|
-
|
130
|
-
exit
|
142
|
+
error "invalid input file format, supported formats are: #{SUPPORTED_INPUT_FORMATS.join(', ')}"
|
131
143
|
end
|
132
144
|
|
133
145
|
if $options[:table].nil?
|
134
|
-
|
135
|
-
exit
|
146
|
+
error "Please specify table name: -t"
|
136
147
|
end
|
137
148
|
|
138
|
-
|
139
|
-
|
140
|
-
exit
|
149
|
+
if $options[:client] and ($options[:host] or $options[:database] or $options[:username] or $options[:password])
|
150
|
+
error "once -c is specified, -h/-u/-d/-p is no longer needed"
|
141
151
|
end
|
142
152
|
|
143
|
-
if $options[:
|
144
|
-
|
145
|
-
|
146
|
-
end
|
153
|
+
if $options[:client]
|
154
|
+
if ENV['MAINDB'].blank?
|
155
|
+
error "The environment variable MAINDB (used by -c/--client) is not present"
|
156
|
+
end
|
157
|
+
|
158
|
+
if ENV['MAINDBHOST'].blank?
|
159
|
+
error "The environment variable MAINDBHOST (used by -c/--client) is not present"
|
160
|
+
end
|
161
|
+
|
162
|
+
ActiveRecord::Base.establish_connection(
|
163
|
+
'adapter' => 'postgresql',
|
164
|
+
'host' => ENV['MAINDBHOST'],
|
165
|
+
'database' => ENV['MAINDB'],
|
166
|
+
'username' => ENV['MAINDBUSER'] || POSTGRESQL_USERNAME,
|
167
|
+
'port' => ENV['MAINDBPORT'] || POSTGRESQL_PORT,
|
168
|
+
'timeout' => 15000
|
169
|
+
)
|
170
|
+
|
171
|
+
class Organization < ActiveRecord::Base
|
172
|
+
end
|
173
|
+
|
174
|
+
client = Organization.where("name ilike ? AND name NOT ilike 'MSSS'", $options[:client]).first
|
175
|
+
unless client
|
176
|
+
error "cannot find client with such name: `#{$options[:client]}`"
|
177
|
+
else
|
178
|
+
Organization.establish_connection(
|
179
|
+
'adapter' => 'postgresql',
|
180
|
+
'host' => client.db_ipaddress,
|
181
|
+
'database' => client.db_name,
|
182
|
+
'username' => client.db_username,
|
183
|
+
'port' => client.db_port,
|
184
|
+
'timeout' => 15000
|
185
|
+
)
|
186
|
+
|
187
|
+
$options[:database] = client.db_name
|
188
|
+
$options[:username] = client.db_username
|
189
|
+
$options[:host] = client.db_ipaddress
|
190
|
+
$options[:listen] = client.db_port
|
191
|
+
end
|
192
|
+
else
|
193
|
+
if $options[:host].nil?
|
194
|
+
error "please specify host name: -h"
|
195
|
+
end
|
147
196
|
|
148
|
-
if $options[:database].nil?
|
149
|
-
|
150
|
-
|
151
|
-
end
|
197
|
+
if $options[:database].nil?
|
198
|
+
error "please specify database name: -d"
|
199
|
+
end
|
152
200
|
|
153
|
-
if $options[:username].nil?
|
154
|
-
|
155
|
-
|
156
|
-
end
|
201
|
+
if $options[:username].nil?
|
202
|
+
error "please specify username: -u"
|
203
|
+
end
|
157
204
|
|
158
|
-
#
|
159
|
-
|
160
|
-
|
161
|
-
$options[:
|
162
|
-
$options[:
|
205
|
+
# Database dump
|
206
|
+
ActiveRecord::Base.establish_connection(
|
207
|
+
'adapter' => 'postgresql',
|
208
|
+
'host' => $options[:host],
|
209
|
+
'database' => $options[:database],
|
210
|
+
'username' => $options[:username],
|
211
|
+
'password' => $options[:password],
|
212
|
+
'port' => $options[:listen],
|
213
|
+
'timeout' => 15000
|
214
|
+
)
|
215
|
+
end
|
163
216
|
|
164
217
|
$tmpfile = "/tmp/#{Digest::SHA1.hexdigest(rand(100000).to_s)}.csv"
|
165
|
-
|
166
|
-
# Database dump
|
167
|
-
ActiveRecord::Base.establish_connection(
|
168
|
-
'adapter' => 'postgresql',
|
169
|
-
'host' => $options[:host],
|
170
|
-
'database' => $options[:database],
|
171
|
-
'username' => $options[:username],
|
172
|
-
'password' => $options[:password],
|
173
|
-
'port' => $options[:listen],
|
174
|
-
'timeout' => 15000
|
175
|
-
)
|
176
|
-
|
177
218
|
$csv_converters = [:stripper]
|
178
219
|
|
179
220
|
CSV::Converters[:stripper] = lambda{ |s|
|
@@ -202,10 +243,6 @@ end
|
|
202
243
|
|
203
244
|
class MyParser
|
204
245
|
def initialize
|
205
|
-
# remote server always requires password
|
206
|
-
if !local? and $options[:password].nil?
|
207
|
-
raise "You are connecting to a remote server\nPlease make sure you specify SQL password: --password "
|
208
|
-
end
|
209
246
|
end
|
210
247
|
|
211
248
|
def run
|
@@ -263,7 +300,7 @@ class MyParser
|
|
263
300
|
|
264
301
|
def create_table_from_csv(csv_path)
|
265
302
|
# Get headers
|
266
|
-
csv = CSV.open(csv_path, :headers => true, :col_sep =>
|
303
|
+
csv = CSV.open(csv_path, :headers => true, :col_sep => $options[:delim], :quote_char => CSV_DEFAULT_QUOTE)
|
267
304
|
|
268
305
|
first = csv.first
|
269
306
|
unless first
|
@@ -281,7 +318,7 @@ class MyParser
|
|
281
318
|
|
282
319
|
# check if every field name is unique
|
283
320
|
if headers.count != headers.uniq.count
|
284
|
-
|
321
|
+
error "duplicate field name: [#{headers.sort.join(', ')}]"
|
285
322
|
end
|
286
323
|
|
287
324
|
# Create table
|
@@ -297,34 +334,27 @@ class MyParser
|
|
297
334
|
|
298
335
|
insert_data_sql = headers.map{|e| "\"#{e}\""}.join(",")
|
299
336
|
insert_data_sql = "COPY #{$options[:table]}( #{insert_data_sql} ) FROM '#{csv_path}' DELIMITER ',' CSV HEADER"
|
337
|
+
insert_data_sql = "PGPASSWORD=#{$options[:password]} psql -U #{$options[:username]} -h #{$options[:host]} -p #{$options[:listen]} #{$options[:database]} -c \"\\#{insert_data_sql}\""
|
300
338
|
|
301
339
|
# Change output file permission so that postgres user can read it
|
302
340
|
begin
|
303
341
|
FileUtils.chmod 0755, csv_path
|
304
342
|
rescue Exception => ex
|
305
|
-
|
306
|
-
end
|
307
|
-
|
308
|
-
if local?
|
309
|
-
query(insert_data_sql)
|
310
|
-
else
|
311
|
-
puts "\nWARNING: pushing data to remote server [#{$options[:host]}].\nBe sure you have the correct version of `psql` command installed\n\n"
|
312
|
-
insert_data_sql = "PGPASSWORD=#{$options[:username]} psql -U #{$options[:username]} -h #{$options[:host]} -p #{$options[:listen]} #{$options[:database]} -c \"\\#{insert_data_sql}\""
|
313
|
-
|
314
|
-
`#{insert_data_sql}`
|
315
|
-
`PGPASSWORD=""`
|
343
|
+
error "cannot change file permission"
|
316
344
|
end
|
317
345
|
|
318
|
-
|
346
|
+
# Execute
|
347
|
+
`#{insert_data_sql}`
|
348
|
+
|
349
|
+
# Clean up
|
350
|
+
File.delete(csv_path) if File.exists?(csv_path)
|
351
|
+
|
352
|
+
puts "Table `#{$options[:table]}` loaded\n"
|
319
353
|
end
|
320
354
|
|
321
355
|
private
|
322
356
|
def query(*query_str)
|
323
|
-
|
324
|
-
end
|
325
|
-
|
326
|
-
def local?
|
327
|
-
return ['localhost', '127.0.0.1'].include?($options[:host])
|
357
|
+
Organization.connection.execute(query_str.join("; "))
|
328
358
|
end
|
329
359
|
end
|
330
360
|
|
data/idata.gemspec
CHANGED
@@ -22,6 +22,6 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_development_dependency "bundler", "~> 1.3"
|
23
23
|
spec.add_development_dependency "rake", "~> 10.0"
|
24
24
|
|
25
|
-
spec.add_dependency "rails", ">= 4.0"
|
25
|
+
#spec.add_dependency "rails", ">= 4.0"
|
26
26
|
spec.add_dependency "pg", "~> 0.16"
|
27
27
|
end
|
data/lib/idata/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: idata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nghi Pham
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-03-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -38,20 +38,6 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '10.0'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: rails
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ">="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '4.0'
|
48
|
-
type: :runtime
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - ">="
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '4.0'
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: pg
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|