idata 0.1.6 → 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/iexport +23 -2
- data/bin/iload +22 -3
- data/bin/ivalidate +6 -6
- data/lib/idata/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 63a5dd0a3ae8f3eddca79de0caf67073852e0e13
|
4
|
+
data.tar.gz: dcd2ffe755ea9e61e07a0e419489b983e52addc8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6c5b90be452b015cb5e4f422667555fe80d0fffc0b77007c1ac8bc1589f77557ca3aa3bd86b402ce21a381a7df65a5572251fd7a57585f8a42556ead6d42e658
|
7
|
+
data.tar.gz: d7aa54e1ad652cf60889069b5c73c53676176da2f89b586abeae6029c4c79d0c9d87ae61b865a653df8c6e203bafb337cf78bb8fce121a32baca2ff9a2bf3f21
|
data/bin/iexport
CHANGED
@@ -47,6 +47,14 @@ parser = OptionParser.new("", 24) do |opts|
|
|
47
47
|
opts.on("--delim DELIMITER", "CSV delimiter") do |v|
|
48
48
|
$options[:delim] = v
|
49
49
|
end
|
50
|
+
|
51
|
+
opts.on("--include FIELDS", "") do |v|
|
52
|
+
$options[:include] = v
|
53
|
+
end
|
54
|
+
|
55
|
+
opts.on("--exclude FIELDS", "") do |v|
|
56
|
+
$options[:exclude] = v
|
57
|
+
end
|
50
58
|
|
51
59
|
opts.on("--linebreak CHAR", "") do |v|
|
52
60
|
$options[:linebreak] = v
|
@@ -163,6 +171,19 @@ $options[:quote_empty] ||= false
|
|
163
171
|
$options[:select] ||= '*'
|
164
172
|
$options[:where] ||= 'true'
|
165
173
|
|
174
|
+
$options[:include] ||= ""
|
175
|
+
|
176
|
+
|
177
|
+
if $options[:exclude]
|
178
|
+
$options[:exclude] = $options[:exclude].split(/\s*,\s*/)
|
179
|
+
else
|
180
|
+
$options[:exclude] = []
|
181
|
+
end
|
182
|
+
|
183
|
+
$options[:select] = [$options[:select].split(/\s*,\s*/) + $options[:include].split(/\s*,\s*/)].join(", ")
|
184
|
+
|
185
|
+
p $options[:select]
|
186
|
+
|
166
187
|
#$options = {host: 'localhost', database: 'db', username: 'postgres', password: 'postgres', table: 'products', listen: 5432}
|
167
188
|
ActiveRecord::Base.establish_connection(
|
168
189
|
'adapter' => 'postgresql',
|
@@ -183,14 +204,14 @@ CSV.open($options[:output], "wb", :quote_char => $options[:quote_char], :col_sep
|
|
183
204
|
first = scope.first
|
184
205
|
|
185
206
|
if first
|
186
|
-
csv << first.attributes.keys if $options[:headers]
|
207
|
+
csv << first.attributes.delete_if{|k,v| k.nil? || $options[:exclude].include?(k) }.keys if $options[:headers]
|
187
208
|
else
|
188
209
|
# only use .column_names when table is empty, to ensure fields match values
|
189
210
|
csv << Product.column_names if $options[:headers]
|
190
211
|
end
|
191
212
|
|
192
213
|
scope.each do |item|
|
193
|
-
attrs = item.attributes.values
|
214
|
+
attrs = item.attributes.delete_if{|k,v| k.nil? || $options[:exclude].include?(k) }.values
|
194
215
|
if !$options[:quote_empty]
|
195
216
|
attrs.map!{|e| e == "" ? nil : e }
|
196
217
|
end
|
data/bin/iload
CHANGED
@@ -14,7 +14,7 @@ require 'rubygems'
|
|
14
14
|
require 'digest/sha1'
|
15
15
|
require 'fileutils'
|
16
16
|
|
17
|
-
SUPPORTED_INPUT_FORMATS = ['CSV', 'FX']
|
17
|
+
SUPPORTED_INPUT_FORMATS = ['CSV', 'FX', 'RPT']
|
18
18
|
POSTGRESQL_PORT = 5432
|
19
19
|
CSV_DEFAULT_DELIMITER = ','
|
20
20
|
CSV_DEFAULT_QUOTE = '"'
|
@@ -34,6 +34,10 @@ parser = OptionParser.new("", 24) do |opts|
|
|
34
34
|
opts.on("--delim DELIMITER", "Field DELIMITER (for CSV format only - default to COMMA ',')") do |v|
|
35
35
|
$options[:delim] = v
|
36
36
|
end
|
37
|
+
|
38
|
+
opts.on("--null NULL", "") do |v|
|
39
|
+
$options[:null] = v
|
40
|
+
end
|
37
41
|
|
38
42
|
opts.on("--quote QUOTE", "Default to '\"'") do |v|
|
39
43
|
$options[:quote] = v
|
@@ -161,6 +165,14 @@ ActiveRecord::Base.establish_connection(
|
|
161
165
|
'timeout' => 15000
|
162
166
|
)
|
163
167
|
|
168
|
+
$csv_converters = []
|
169
|
+
$csv_converters << :null_converter if $options[:null]
|
170
|
+
|
171
|
+
CSV::Converters[:null_converter] = lambda{ |s|
|
172
|
+
return nil if s == $options[:null]
|
173
|
+
return s
|
174
|
+
}
|
175
|
+
|
164
176
|
class String
|
165
177
|
def underscore
|
166
178
|
return self if self.nil?
|
@@ -177,14 +189,14 @@ class MyParser
|
|
177
189
|
end
|
178
190
|
|
179
191
|
def run
|
180
|
-
load_fx if $options[:format] == 'FX'
|
192
|
+
load_fx if $options[:format] == 'FX' || $options[:format] == 'RPT'
|
181
193
|
load_csv if $options[:format] == 'CSV'
|
182
194
|
end
|
183
195
|
|
184
196
|
def load_csv
|
185
197
|
# Load CSV data from input file to a temp array
|
186
198
|
csv_data = []
|
187
|
-
CSV.foreach($options[:input], :col_sep => $options[:delim], :quote_char => $options[:quote]) do |csv|
|
199
|
+
CSV.foreach($options[:input], :col_sep => $options[:delim], :quote_char => $options[:quote], :converters => $csv_converters) do |csv|
|
188
200
|
csv_data << csv
|
189
201
|
end
|
190
202
|
|
@@ -214,6 +226,13 @@ class MyParser
|
|
214
226
|
csv << headers
|
215
227
|
data.each_with_index{|s, index|
|
216
228
|
record = s.unpack(ranges).map{|e| e.strip}
|
229
|
+
|
230
|
+
# take advantage of CSV converters
|
231
|
+
$csv_converters.each {|converter|
|
232
|
+
converter_lambda = CSV::Converters[converter]
|
233
|
+
record.map!(&converter_lambda)
|
234
|
+
}
|
235
|
+
|
217
236
|
csv << record
|
218
237
|
}
|
219
238
|
end
|
data/bin/ivalidate
CHANGED
@@ -135,7 +135,7 @@ if $options[:database].nil?
|
|
135
135
|
end
|
136
136
|
|
137
137
|
if $options[:username].nil?
|
138
|
-
puts "\nPlease specify PostgreSQL username: -
|
138
|
+
puts "\nPlease specify PostgreSQL username: -u\n\n"
|
139
139
|
exit
|
140
140
|
end
|
141
141
|
|
@@ -179,7 +179,7 @@ $options[:unique].each do |field|
|
|
179
179
|
puts "Checking unique fields: #{field}"
|
180
180
|
|
181
181
|
uniq_sql = <<-eos
|
182
|
-
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}
|
182
|
+
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} not unique', ' || '), ' || ')
|
183
183
|
WHERE #{field} IN (
|
184
184
|
SELECT #{field} FROM #{$options[:table]} GROUP BY #{field}
|
185
185
|
HAVING count(*) > 1
|
@@ -196,7 +196,7 @@ $options[:not_null].each do |field|
|
|
196
196
|
puts "Checking not-null fields: #{field}"
|
197
197
|
|
198
198
|
not_null_sql = <<-eos
|
199
|
-
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}
|
199
|
+
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} null', ' || '), ' || ')
|
200
200
|
WHERE #{field} IS NULL OR length(trim(#{field})) = 0;
|
201
201
|
eos
|
202
202
|
|
@@ -212,7 +212,7 @@ $options[:match].each do |value|
|
|
212
212
|
puts "Checking REGEXP matching: #{regexp}"
|
213
213
|
|
214
214
|
match_sql = <<-eos
|
215
|
-
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{regexp.gsub("'", "''")}
|
215
|
+
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} not match [#{regexp.gsub("'", "''")}]', ' || '), ' || ')
|
216
216
|
WHERE #{field} IS NOT NULL AND length(trim(#{field})) <> 0 AND #{field} !~ '#{regexp}';
|
217
217
|
eos
|
218
218
|
|
@@ -228,7 +228,7 @@ $options[:not_match].each do |value|
|
|
228
228
|
puts "Checking REGEXP not matching: #{regexp}"
|
229
229
|
|
230
230
|
not_match_sql = <<-eos
|
231
|
-
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{regexp.gsub("'", "''")}
|
231
|
+
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} match [#{regexp.gsub("'", "''")}]', ' || '), ' || ')
|
232
232
|
WHERE #{field} IS NOT NULL AND length(trim(#{field})) <> 0 AND #{field} ~ '#{regexp}';
|
233
233
|
eos
|
234
234
|
|
@@ -254,7 +254,7 @@ $options[:cross_reference].each do |value|
|
|
254
254
|
|
255
255
|
# @todo: poor performance here, think of a better SQL!!!
|
256
256
|
ref_sql = <<-eos
|
257
|
-
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field}
|
257
|
+
UPDATE #{$options[:table]} SET #{$options[:log_to]} = array_to_string(string_to_array(#{$options[:log_to]}, ' || ') || string_to_array('#{field} not referenced to #{values[1]}.#{values[2]}', ' || '), ' || ')
|
258
258
|
WHERE #{field} NOT IN (
|
259
259
|
SELECT #{ref_field} FROM #{ref_table}
|
260
260
|
) AND #{field} IS NOT NULL AND length(trim(#{field})) <> 0;
|
data/lib/idata/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: idata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nghi Pham
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|