idata 0.0.5 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/iload +20 -25
- data/lib/idata/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 819b2d2fdfbb0c55e7e0fad65c6bbdb28c5e5b5d
|
4
|
+
data.tar.gz: 553beaa45ad12795547c87902c18d6d44da58a34
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c3e289852ac767f0948e5f835cb0bf937309699c9e6d04ac50386888327de5abc1cfe8465a8ff403f5695bced3ad45e7b5e06f4bbdf91b16a4f4fc2d55c0133f
|
7
|
+
data.tar.gz: 5fad9fdd602ad71e584e98b0dea174dfa74f9027be7d3d7cac00be55f7f6c231218a7480bcfdbfb948e7b9fdc228db5bb083e13be193b974b8392f2d73d010ed
|
data/bin/iload
CHANGED
@@ -19,14 +19,6 @@ POSTGRESQL_PORT = 5432
|
|
19
19
|
CSV_DEFAULT_DELIMITER = ','
|
20
20
|
CSV_DEFAULT_QUOTE = '"'
|
21
21
|
|
22
|
-
PG_TAB = "E'\t'"
|
23
|
-
PG_BACKSPACE = "E'\b'"
|
24
|
-
|
25
|
-
PG_CHAR_MAP = {
|
26
|
-
"\t" => PG_TAB,
|
27
|
-
"\b" => PG_BACKSPACE
|
28
|
-
}
|
29
|
-
|
30
22
|
$options = {}
|
31
23
|
parser = OptionParser.new("", 24) do |opts|
|
32
24
|
opts.banner = "\nProgram: Data Loader\nAuthor: MCKI\n\n"
|
@@ -195,8 +187,20 @@ class MyParser
|
|
195
187
|
end
|
196
188
|
|
197
189
|
def load_csv
|
198
|
-
#
|
199
|
-
|
190
|
+
# Load CSV data from input file to a temp array
|
191
|
+
csv_data = []
|
192
|
+
CSV.foreach($options[:input], :col_sep => $options[:delim], :quote_char => $options[:quote]) do |csv|
|
193
|
+
csv_data << csv
|
194
|
+
end
|
195
|
+
|
196
|
+
# Serialize array into a new CSV (with standard delimiter, quote) for later use with PostgreSQL
|
197
|
+
CSV.open($tmpfile, "wb", :col_sep => CSV_DEFAULT_DELIMITER, :quote_char => CSV_DEFAULT_QUOTE) do |writer|
|
198
|
+
csv_data.each do |csv|
|
199
|
+
writer << csv
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
# Send to PostgreSQL
|
200
204
|
create_table_from_csv($tmpfile)
|
201
205
|
end
|
202
206
|
|
@@ -211,21 +215,21 @@ class MyParser
|
|
211
215
|
headers.map!{|s| s.downcase.strip }
|
212
216
|
|
213
217
|
# Write
|
214
|
-
CSV.open($tmpfile, "wb", :col_sep => CSV_DEFAULT_DELIMITER) do |csv|
|
218
|
+
CSV.open($tmpfile, "wb", :col_sep => CSV_DEFAULT_DELIMITER, :quote_char => CSV_DEFAULT_QUOTE) do |csv|
|
215
219
|
csv << headers
|
216
|
-
|
217
220
|
data.each_with_index{|s, index|
|
218
221
|
record = s.unpack(ranges).map{|e| e.strip}
|
219
222
|
csv << record
|
220
223
|
}
|
221
224
|
end
|
222
|
-
|
225
|
+
|
226
|
+
# Send to PostgreSQL
|
223
227
|
create_table_from_csv($tmpfile)
|
224
228
|
end
|
225
229
|
|
226
230
|
def create_table_from_csv(csv_path)
|
227
231
|
# Get headers
|
228
|
-
csv = CSV.open(csv_path, :headers => true, :col_sep =>
|
232
|
+
csv = CSV.open(csv_path, :headers => true, :col_sep => CSV_DEFAULT_DELIMITER, :quote_char => CSV_DEFAULT_QUOTE)
|
229
233
|
|
230
234
|
first = csv.first
|
231
235
|
unless first
|
@@ -256,19 +260,10 @@ class MyParser
|
|
256
260
|
create_table_sql = headers.map{|e| "\"#{e}\" text"}.join(",")
|
257
261
|
create_table_sql = "create table if not exists #{$options[:table]}( id serial not null, #{create_table_sql} );"
|
258
262
|
query(drop_table_sql, create_table_sql)
|
259
|
-
|
260
|
-
# Dump data
|
261
|
-
#pg_tab = "E'\t'" if $options[:delim] == "\t"
|
262
|
-
|
263
|
-
tab = PG_CHAR_MAP[$options[:delim]]
|
264
|
-
tab ||= "'#{$options[:delim]}'"
|
265
|
-
|
266
|
-
quote = PG_CHAR_MAP[$options[:quote]]
|
267
|
-
quote = "'#{$options[:quote]}'"
|
268
263
|
|
269
264
|
insert_data_sql = headers.map{|e| "\"#{e}\""}.join(",")
|
270
|
-
insert_data_sql = "COPY #{$options[:table]}( #{insert_data_sql} ) FROM '#{csv_path}' DELIMITER
|
271
|
-
|
265
|
+
insert_data_sql = "COPY #{$options[:table]}( #{insert_data_sql} ) FROM '#{csv_path}' DELIMITER ',' CSV HEADER"
|
266
|
+
|
272
267
|
# Change output file permission so that postgres user can read it
|
273
268
|
begin
|
274
269
|
FileUtils.chmod 0755, csv_path
|
data/lib/idata/version.rb
CHANGED