csv2sqlite 0.0.4 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 956c1cf32dc30af45bdb67def38869b993344bd6
-  data.tar.gz: b34f2d4eb15ad7944ec90906a68aef04612a3407
+  metadata.gz: ff49a31a357c2e5acc7c7f79ac6b20e0d0dd5d72
+  data.tar.gz: f3f4ad9cc5aebaee7fc47ef5f140fd071334127a
 SHA512:
-  metadata.gz: dd8ad64c158d96f931d52362776732b056007bd5eb3e300463153398932d7ff1fe6a229ead6bdb2a3616643e412c41cd866ec512a24bd19e74a1eb59114b1e80
-  data.tar.gz: fd3b7dec22c93e8c9012362c4b70ff4aab131ad13fbfadaafc016a7051cb10c7b6146f16d1fded6449e25e5d30688b14b9e9e26841453db0153f66b35c1b6e59
+  metadata.gz: f2ac564da8ceca6fe8a45b7bcf47c64f9a57383e01494ff0f19016f70df492e58623461f0563a492766247e5308a352f6268eafa013e2b426d72fbbfe618ee59
+  data.tar.gz: 4ccb998011ac61c2ae1c68abe43477fc6cd558698cdfa18bd5d33318cbb9d939cdcb4067d6f1cb700991998cd305cf3297ce1837b0d153b0e99bcacd19d1849f

data/bin/csv2sqlite

@@ -4,6 +4,7 @@ require 'csv'
 require 'active_record'
 require 'active_support/all'
 require 'sqlite3'
+require 'fileutils'
 
 db = ARGV[0]
 csvs = ARGV[1..-1]
@@ -20,72 +21,93 @@ development:
 }
 ActiveRecord::Base.configurations = YAML::load(config)
 ActiveRecord::Base.establish_connection('development')
-
+RESERVED_COLUMN_NAMES = ["abort", "action", "add", "after", "all", "alter", "analyze", "and", "as", "asc", "attach", "autoincrement", "before", "begin", "between", "by", "cascade", "case", "cast", "check", "collate", "column", "commit", "conflict", "constraint", "create", "cross", "current_date", "current_time", "current_timestamp", "database", "default", "deferrable", "deferred", "delete", "desc", "detach", "distinct", "drop", "each", "else", "end", "escape", "except", "exclusive", "exists", "explain", "fail", "for", "foreign", "from", "full", "glob", "group", "having", "if", "ignore", "immediate", "in", "index", "indexed", "initially", "inner", "insert", "instead", "intersect", "into", "is", "isnull", "join", "key", "left", "like", "limit", "match", "natural", "no", "not", "notnull", "null", "of", "offset", "on", "or", "order", "outer", "plan", "pragma", "primary", "query", "raise", "recursive", "references", "regexp", "reindex", "release", "rename", "replace", "restrict", "right", "rollback", "row", "savepoint", "select", "set", "table", "temp", "temporary", "then", "to", "transaction", "trigger", "union", "unique", "update", "using", "vacuum", "values", "view", "virtual", "when", "where", "with", "without"]
 csvs.each {|file|
+  files = [{path: file, name: nil}]
   file_io = nil
-  if file.match(/\.xlsx$/).present?
-    name = File.basename(file, ".xlsx").classify
-    temp_file = Tempfile.new(["conv_", ".csv"])
-    `ssconvert --import-type=Gnumeric_Excel:xlsx #{file} #{temp_file.path} > /dev/null 2>&1`
-
-    temp_file.rewind
-    file = temp_file.path
-  elsif file.match(/\.xls$/).present?
-    name = File.basename(file, ".xls").classify
+  name = nil
+  if file.match(/\.xlsx?$/).present?
+    
     temp_file = Tempfile.new(["conv_", ".csv"])
-    `ssconvert --import-type=Gnumeric_Excel:excel #{file} #{temp_file.path} > /dev/null 2>&1`
+    if file.match(/\.xlsx$/).present?
+      name = File.basename(file, ".xlsx")
+      `ssconvert --import-type=Gnumeric_Excel:xlsx -S #{file} #{temp_file.path} > /dev/null 2>&1`
+    else
+      name = File.basename(file, ".xls")
+      `ssconvert --import-type=Gnumeric_Excel:excel -S #{file} #{temp_file.path} > /dev/null 2>&1`
+    end
 
-    temp_file.rewind
-    file = temp_file.path
+    files = Dir["#{temp_file.path}.*"]
+    if files.size > 1
+      files = Dir["#{temp_file.path}.*"].map {|path|
+        ending = path.match(/\.([0-9]+)/)[1]
+        {path: path, name: "#{name}Sheet#{ending}".classify.gsub(/[^a-zA-Z0-9]/, ""), delete: true}
+      }
+    else
+      files = Dir["#{temp_file.path}.*"].map {|path|
+        {path: path, name: name.classify.gsub(/[^a-zA-Z0-9]/, ""), delete: true}
+      }
+    end
   else
-    name = File.basename(file, ".csv").classify
+    files[0][:name] = File.basename(file, ".csv").gsub(/[^a-zA-Z0-9]/, "").classify
   end
-  puts name
-  count = 0
+  
   ActiveRecord::Base.transaction do
-    cols = {}
     
-    CSV.foreach file, :headers => true, encoding: "ISO-8859-1:UTF-8",  header_converters: ->(col){
-      col = col.downcase.gsub(/[^a-z0-9_]/, "") 
-      if cols[col]  
-        col += "#{cols.size}"
-      end
-      if col == "type"
-        col += "_#{cols.size}"
-      end
-      if col == "add"
-        col += "_#{cols.size}"
-      end
-      if col == "order"
-        col += "_#{cols.size}"
-      end
-      if col == "guid"
-        col += "_#{cols.size}"
+    
+    files.each {|options|
+      cols = {}
+      file = options[:path]
+      name = options[:name]
+      p name
+      count = 0
+      klass = nil
+      CSV.foreach(file, :headers => true, encoding: "ISO-8859-1:UTF-8",  header_converters: ->(col){
+        col = (col || "").downcase.gsub(/[^a-z0-9_]/, "") 
+        if col == ""
+          col += "col#{cols.size}"
+        end
+        if ["class", "guid", 'type'].include?(col)
+          col += "#{cols.size}"
+        end
+        if RESERVED_COLUMN_NAMES.include?(col)
+          col += "#{cols.size}"
+        end
+        if cols[col]  
+          col += "#{cols.size}"
+        end
+        cols[col] = true 
+        #p col
+        col
+      }) do |row|
+        if count == 0
+          columns = (row.to_hash.keys + ['guid']).map {|r| "#{r} varchar" }.join(", ")
+          insert_statement = "CREATE TABLE #{name.downcase.pluralize} (#{columns});" 
+          ActiveRecord::Base.connection.execute(insert_statement)
+          s = %{ 
+            class Csv2Sqlite#{name} < ActiveRecord::Base
+              self.table_name = "#{name.downcase.pluralize}"
+            end
+          }
+          Kernel.eval(s)
+          klass = "Csv2Sqlite#{name}".constantize
+          klass.reset_column_information
+        end
+        
+        
+        print "."
+        hash = row.to_hash
+        hash['guid'] = SecureRandom.uuid
+        klass.create!(hash)
+        
+        count += 1
       end
-      cols[col] = true 
-      #p col
-      col
-    } do |row|
-      if count == 0
-        columns = (row.to_hash.keys + ['guid']).map {|r| "#{r} varchar" }.join(", ")
-        insert_statement = "CREATE TABLE #{name.downcase.pluralize} (#{columns});" 
-        ActiveRecord::Base.connection.execute(insert_statement)
+      if options[:delete] == true
+        FileUtils.rm(file)
       end
-      s = %{ 
-        class Csv2Sqlite#{name} < ActiveRecord::Base
-          self.table_name = "#{name.downcase.pluralize}"
-        end
-      }
-      Kernel.eval(s)
-      klass = "Csv2Sqlite#{name}".constantize
-      print "."
-      hash = Hash[row]
-      hash['guid'] = SecureRandom.uuid
-      #p hash
-      klass.reset_column_information
-      klass.create! hash
-      count += 1
-    end
+    }
+
+    
   end 
   puts "\nDone."
 }

data/lib/csv2sqlite/version.rb

@@ -1,3 +1,3 @@
 module Csv2sqlite
-  VERSION = "0.0.4"
+  VERSION = "0.0.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csv2sqlite
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.6
 platform: ruby
 authors:
 - Pete Brumm