csv2sqlite 0.0.4 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 956c1cf32dc30af45bdb67def38869b993344bd6
4
- data.tar.gz: b34f2d4eb15ad7944ec90906a68aef04612a3407
3
+ metadata.gz: ff49a31a357c2e5acc7c7f79ac6b20e0d0dd5d72
4
+ data.tar.gz: f3f4ad9cc5aebaee7fc47ef5f140fd071334127a
5
5
  SHA512:
6
- metadata.gz: dd8ad64c158d96f931d52362776732b056007bd5eb3e300463153398932d7ff1fe6a229ead6bdb2a3616643e412c41cd866ec512a24bd19e74a1eb59114b1e80
7
- data.tar.gz: fd3b7dec22c93e8c9012362c4b70ff4aab131ad13fbfadaafc016a7051cb10c7b6146f16d1fded6449e25e5d30688b14b9e9e26841453db0153f66b35c1b6e59
6
+ metadata.gz: f2ac564da8ceca6fe8a45b7bcf47c64f9a57383e01494ff0f19016f70df492e58623461f0563a492766247e5308a352f6268eafa013e2b426d72fbbfe618ee59
7
+ data.tar.gz: 4ccb998011ac61c2ae1c68abe43477fc6cd558698cdfa18bd5d33318cbb9d939cdcb4067d6f1cb700991998cd305cf3297ce1837b0d153b0e99bcacd19d1849f
data/bin/csv2sqlite CHANGED
@@ -4,6 +4,7 @@ require 'csv'
4
4
  require 'active_record'
5
5
  require 'active_support/all'
6
6
  require 'sqlite3'
7
+ require 'fileutils'
7
8
 
8
9
  db = ARGV[0]
9
10
  csvs = ARGV[1..-1]
@@ -20,72 +21,93 @@ development:
20
21
  }
21
22
  ActiveRecord::Base.configurations = YAML::load(config)
22
23
  ActiveRecord::Base.establish_connection('development')
23
-
24
+ RESERVED_COLUMN_NAMES = ["abort", "action", "add", "after", "all", "alter", "analyze", "and", "as", "asc", "attach", "autoincrement", "before", "begin", "between", "by", "cascade", "case", "cast", "check", "collate", "column", "commit", "conflict", "constraint", "create", "cross", "current_date", "current_time", "current_timestamp", "database", "default", "deferrable", "deferred", "delete", "desc", "detach", "distinct", "drop", "each", "else", "end", "escape", "except", "exclusive", "exists", "explain", "fail", "for", "foreign", "from", "full", "glob", "group", "having", "if", "ignore", "immediate", "in", "index", "indexed", "initially", "inner", "insert", "instead", "intersect", "into", "is", "isnull", "join", "key", "left", "like", "limit", "match", "natural", "no", "not", "notnull", "null", "of", "offset", "on", "or", "order", "outer", "plan", "pragma", "primary", "query", "raise", "recursive", "references", "regexp", "reindex", "release", "rename", "replace", "restrict", "right", "rollback", "row", "savepoint", "select", "set", "table", "temp", "temporary", "then", "to", "transaction", "trigger", "union", "unique", "update", "using", "vacuum", "values", "view", "virtual", "when", "where", "with", "without"]
24
25
  csvs.each {|file|
26
+ files = [{path: file, name: nil}]
25
27
  file_io = nil
26
- if file.match(/\.xlsx$/).present?
27
- name = File.basename(file, ".xlsx").classify
28
- temp_file = Tempfile.new(["conv_", ".csv"])
29
- `ssconvert --import-type=Gnumeric_Excel:xlsx #{file} #{temp_file.path} > /dev/null 2>&1`
30
-
31
- temp_file.rewind
32
- file = temp_file.path
33
- elsif file.match(/\.xls$/).present?
34
- name = File.basename(file, ".xls").classify
28
+ name = nil
29
+ if file.match(/\.xlsx?$/).present?
30
+
35
31
  temp_file = Tempfile.new(["conv_", ".csv"])
36
- `ssconvert --import-type=Gnumeric_Excel:excel #{file} #{temp_file.path} > /dev/null 2>&1`
32
+ if file.match(/\.xlsx$/).present?
33
+ name = File.basename(file, ".xlsx")
34
+ `ssconvert --import-type=Gnumeric_Excel:xlsx -S #{file} #{temp_file.path} > /dev/null 2>&1`
35
+ else
36
+ name = File.basename(file, ".xls")
37
+ `ssconvert --import-type=Gnumeric_Excel:excel -S #{file} #{temp_file.path} > /dev/null 2>&1`
38
+ end
37
39
 
38
- temp_file.rewind
39
- file = temp_file.path
40
+ files = Dir["#{temp_file.path}.*"]
41
+ if files.size > 1
42
+ files = Dir["#{temp_file.path}.*"].map {|path|
43
+ ending = path.match(/\.([0-9]+)/)[1]
44
+ {path: path, name: "#{name}Sheet#{ending}".classify.gsub(/[^a-zA-Z0-9]/, ""), delete: true}
45
+ }
46
+ else
47
+ files = Dir["#{temp_file.path}.*"].map {|path|
48
+ {path: path, name: name.classify.gsub(/[^a-zA-Z0-9]/, ""), delete: true}
49
+ }
50
+ end
40
51
  else
41
- name = File.basename(file, ".csv").classify
52
+ files[0][:name] = File.basename(file, ".csv").gsub(/[^a-zA-Z0-9]/, "").classify
42
53
  end
43
- puts name
44
- count = 0
54
+
45
55
  ActiveRecord::Base.transaction do
46
- cols = {}
47
56
 
48
- CSV.foreach file, :headers => true, encoding: "ISO-8859-1:UTF-8", header_converters: ->(col){
49
- col = col.downcase.gsub(/[^a-z0-9_]/, "")
50
- if cols[col]
51
- col += "#{cols.size}"
52
- end
53
- if col == "type"
54
- col += "_#{cols.size}"
55
- end
56
- if col == "add"
57
- col += "_#{cols.size}"
58
- end
59
- if col == "order"
60
- col += "_#{cols.size}"
61
- end
62
- if col == "guid"
63
- col += "_#{cols.size}"
57
+
58
+ files.each {|options|
59
+ cols = {}
60
+ file = options[:path]
61
+ name = options[:name]
62
+ p name
63
+ count = 0
64
+ klass = nil
65
+ CSV.foreach(file, :headers => true, encoding: "ISO-8859-1:UTF-8", header_converters: ->(col){
66
+ col = (col || "").downcase.gsub(/[^a-z0-9_]/, "")
67
+ if col == ""
68
+ col += "col#{cols.size}"
69
+ end
70
+ if ["class", "guid", 'type'].include?(col)
71
+ col += "#{cols.size}"
72
+ end
73
+ if RESERVED_COLUMN_NAMES.include?(col)
74
+ col += "#{cols.size}"
75
+ end
76
+ if cols[col]
77
+ col += "#{cols.size}"
78
+ end
79
+ cols[col] = true
80
+ #p col
81
+ col
82
+ }) do |row|
83
+ if count == 0
84
+ columns = (row.to_hash.keys + ['guid']).map {|r| "#{r} varchar" }.join(", ")
85
+ insert_statement = "CREATE TABLE #{name.downcase.pluralize} (#{columns});"
86
+ ActiveRecord::Base.connection.execute(insert_statement)
87
+ s = %{
88
+ class Csv2Sqlite#{name} < ActiveRecord::Base
89
+ self.table_name = "#{name.downcase.pluralize}"
90
+ end
91
+ }
92
+ Kernel.eval(s)
93
+ klass = "Csv2Sqlite#{name}".constantize
94
+ klass.reset_column_information
95
+ end
96
+
97
+
98
+ print "."
99
+ hash = row.to_hash
100
+ hash['guid'] = SecureRandom.uuid
101
+ klass.create!(hash)
102
+
103
+ count += 1
64
104
  end
65
- cols[col] = true
66
- #p col
67
- col
68
- } do |row|
69
- if count == 0
70
- columns = (row.to_hash.keys + ['guid']).map {|r| "#{r} varchar" }.join(", ")
71
- insert_statement = "CREATE TABLE #{name.downcase.pluralize} (#{columns});"
72
- ActiveRecord::Base.connection.execute(insert_statement)
105
+ if options[:delete] == true
106
+ FileUtils.rm(file)
73
107
  end
74
- s = %{
75
- class Csv2Sqlite#{name} < ActiveRecord::Base
76
- self.table_name = "#{name.downcase.pluralize}"
77
- end
78
- }
79
- Kernel.eval(s)
80
- klass = "Csv2Sqlite#{name}".constantize
81
- print "."
82
- hash = Hash[row]
83
- hash['guid'] = SecureRandom.uuid
84
- #p hash
85
- klass.reset_column_information
86
- klass.create! hash
87
- count += 1
88
- end
108
+ }
109
+
110
+
89
111
  end
90
112
  puts "\nDone."
91
113
  }
@@ -1,3 +1,3 @@
1
1
  module Csv2sqlite
2
- VERSION = "0.0.4"
2
+ VERSION = "0.0.6"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: csv2sqlite
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pete Brumm