idata 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b369eb5215e497e66d4364240f9fb7de6da8c62
-  data.tar.gz: 37d2fdc825b364a26f48df4172f1546be61ce301
+  metadata.gz: 819b2d2fdfbb0c55e7e0fad65c6bbdb28c5e5b5d
+  data.tar.gz: 553beaa45ad12795547c87902c18d6d44da58a34
 SHA512:
-  metadata.gz: 2ee2a70707b45a9d1e163fd735ad88b47bc37c117039e4b1686b669352a75b420569a8b3f72977c567e13ecc5e301f0addb5c3b694f592dc072e194a249e95b3
-  data.tar.gz: e0ce31954906f7c468ce945e83165ab6091fe77eb657dc4c0cc11b0b443cf5e089ae46a762b19321328abdf8540b4650d4e742a3d435b41d000f499513ee53e3
+  metadata.gz: c3e289852ac767f0948e5f835cb0bf937309699c9e6d04ac50386888327de5abc1cfe8465a8ff403f5695bced3ad45e7b5e06f4bbdf91b16a4f4fc2d55c0133f
+  data.tar.gz: 5fad9fdd602ad71e584e98b0dea174dfa74f9027be7d3d7cac00be55f7f6c231218a7480bcfdbfb948e7b9fdc228db5bb083e13be193b974b8392f2d73d010ed

data/bin/iload

@@ -19,14 +19,6 @@ POSTGRESQL_PORT = 5432
 CSV_DEFAULT_DELIMITER = ','
 CSV_DEFAULT_QUOTE = '"'
 
-PG_TAB = "E'\t'"
-PG_BACKSPACE = "E'\b'"
-
-PG_CHAR_MAP = {
-  "\t" => PG_TAB,
-  "\b" => PG_BACKSPACE
-}
-
 $options = {}
 parser = OptionParser.new("", 24) do |opts|
   opts.banner = "\nProgram: Data Loader\nAuthor: MCKI\n\n"
@@ -195,8 +187,20 @@ class MyParser
   end
 
   def load_csv
-    # Copy file to /tmp/ folder to prevent Permission Error 
-    FileUtils.cp $options[:input], $tmpfile
+    # Load CSV data from input file to a temp array    
+    csv_data = []
+    CSV.foreach($options[:input], :col_sep => $options[:delim], :quote_char => $options[:quote]) do |csv|
+      csv_data << csv
+    end
+    
+    # Serialize array into a new CSV (with standard delimiter, quote) for later use with PostgreSQL
+    CSV.open($tmpfile, "wb", :col_sep => CSV_DEFAULT_DELIMITER, :quote_char => CSV_DEFAULT_QUOTE) do |writer|
+      csv_data.each do |csv|
+        writer << csv
+      end
+    end
+    
+    # Send to PostgreSQL
     create_table_from_csv($tmpfile)
   end
 
@@ -211,21 +215,21 @@ class MyParser
     headers.map!{|s| s.downcase.strip }
 
     # Write
-    CSV.open($tmpfile, "wb", :col_sep => CSV_DEFAULT_DELIMITER) do |csv|
+    CSV.open($tmpfile, "wb", :col_sep => CSV_DEFAULT_DELIMITER, :quote_char => CSV_DEFAULT_QUOTE) do |csv|
       csv << headers
-
       data.each_with_index{|s, index|
         record = s.unpack(ranges).map{|e| e.strip}
         csv << record
       }
     end
-
+    
+    # Send to PostgreSQL
     create_table_from_csv($tmpfile)
   end
 
   def create_table_from_csv(csv_path)
     # Get headers
-    csv = CSV.open(csv_path, :headers => true, :col_sep => $options[:delim], quote_char: "\b")
+    csv = CSV.open(csv_path, :headers => true, :col_sep => CSV_DEFAULT_DELIMITER, :quote_char => CSV_DEFAULT_QUOTE)
     
     first = csv.first
     unless first
@@ -256,19 +260,10 @@ class MyParser
     create_table_sql = headers.map{|e| "\"#{e}\" text"}.join(",")
     create_table_sql = "create table if not exists #{$options[:table]}( id serial not null, #{create_table_sql} );"
     query(drop_table_sql, create_table_sql)
-
-    # Dump data
-    #pg_tab = "E'\t'" if $options[:delim] == "\t"
-
-    tab = PG_CHAR_MAP[$options[:delim]]
-    tab ||= "'#{$options[:delim]}'"
-
-    quote = PG_CHAR_MAP[$options[:quote]]
-    quote = "'#{$options[:quote]}'"
     
     insert_data_sql = headers.map{|e| "\"#{e}\""}.join(",")
-    insert_data_sql = "COPY #{$options[:table]}( #{insert_data_sql} ) FROM '#{csv_path}' DELIMITER #{tab} CSV HEADER QUOTE #{quote};"
-
+    insert_data_sql = "COPY #{$options[:table]}( #{insert_data_sql} ) FROM '#{csv_path}' DELIMITER ',' CSV HEADER"
+    
     # Change output file permission so that postgres user can read it
     begin
       FileUtils.chmod 0755, csv_path

data/lib/idata/version.rb

@@ -1,3 +1,3 @@
 module Idata
-  VERSION = "0.0.5"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: idata
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
 - Nghi Pham