queryable_pstore 0.0.10 → 0.0.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/csv_converter.rb +2 -0
- data/lib/csv_importer.rb +62 -8
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3b23893d697cf9c96836e24472b374949340eeb8
|
4
|
+
data.tar.gz: 8413c0ab01438990d730789c768226a9c298e6a3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 586899fbb166d695f06c48e93896c1db697f9d57ce3465ae5a9be60def4963219708c019b0af183ffb8cdbc20ea95da567bb5d4de4b8130ff86a870d21fd97f7
|
7
|
+
data.tar.gz: 7c6530e9745705aaf10fd772b95d29f3cd19e5bb2e348d51d54fd19693b5ff6597e46e8b26eebf247ff18ec844b0419296134785b4460e87f3741d092e6eb5b2
|
data/lib/csv_converter.rb
CHANGED
data/lib/csv_importer.rb
CHANGED
@@ -1,28 +1,33 @@
|
|
1
|
+
require 'pry'
|
2
|
+
|
1
3
|
class CSVImporter
|
2
4
|
CURRENT_DIRECTORY = "./".freeze
|
3
5
|
|
4
6
|
attr_reader :original_headers, :headers_and_type
|
5
7
|
|
6
8
|
def import_csv(filename, opts = {})
|
7
|
-
csv = CSV.read(File.open(filename), headers: true, header_converters: -> (header) { convert_header_to_methodable_name(header) })
|
9
|
+
csv = CSV.read(File.open(filename), headers: true, header_converters: -> (header) { convert_header_to_methodable_name(header, track_original: true) })
|
8
10
|
store = create_pstore(csv, opts, filename)
|
9
11
|
store.csv_importer = self
|
10
|
-
set_headers_and_type(@original_headers, store)
|
11
12
|
|
12
13
|
store
|
13
14
|
end
|
14
15
|
|
15
16
|
def import_csv_from_string(string, opts = {})
|
16
|
-
csv = CSV.parse(string, headers: true, header_converters: -> (header) { convert_header_to_methodable_name(header) })
|
17
|
+
csv = CSV.parse(string, headers: true, header_converters: -> (header) { convert_header_to_methodable_name(header, track_original: true) })
|
17
18
|
store = create_pstore(csv, opts)
|
18
19
|
store.csv_importer = self
|
19
|
-
set_headers_and_type(@original_headers, store)
|
20
20
|
|
21
21
|
store
|
22
22
|
end
|
23
23
|
|
24
24
|
def create_pstore(csv, opts = {}, filename = SecureRandom.uuid)
|
25
|
-
|
25
|
+
conversion_required = opts.fetch(:convert, [])
|
26
|
+
if opts.fetch(:convert, nil) == :best_guess
|
27
|
+
conversion_required = guess_conversion_types(csv)
|
28
|
+
end
|
29
|
+
|
30
|
+
csv_converters = conversion_required.map { |header, desired_format| CSVConverter.new(header => desired_format) }
|
26
31
|
file_save_location = opts[:save_to_location] || CURRENT_DIRECTORY
|
27
32
|
|
28
33
|
store = QueryablePStore.new("#{file_save_location}#{filename}.pstore")
|
@@ -32,12 +37,61 @@ class CSVImporter
|
|
32
37
|
end
|
33
38
|
end
|
34
39
|
|
40
|
+
if opts.fetch(:convert, nil) == :best_guess
|
41
|
+
original_headers = @original_headers.clone
|
42
|
+
|
43
|
+
original_headers.each do |original_header|
|
44
|
+
@headers_and_type ||= {}
|
45
|
+
@headers_and_type[original_header] = symbol_to_class(
|
46
|
+
conversion_required[
|
47
|
+
convert_header_to_methodable_name(original_header)
|
48
|
+
]
|
49
|
+
)
|
50
|
+
end
|
51
|
+
else
|
52
|
+
set_headers_and_type(@original_headers, store)
|
53
|
+
end
|
54
|
+
|
35
55
|
store
|
36
56
|
end
|
37
57
|
|
38
|
-
def
|
39
|
-
|
40
|
-
|
58
|
+
def symbol_to_class(symbol)
|
59
|
+
case symbol
|
60
|
+
when :integer
|
61
|
+
Integer
|
62
|
+
when :float
|
63
|
+
Float
|
64
|
+
when :string
|
65
|
+
String
|
66
|
+
else
|
67
|
+
raise "Unknown symbol: #{symbol}"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def guess_conversion_types(csv)
|
72
|
+
headers = csv.headers
|
73
|
+
headers_and_type = {}
|
74
|
+
|
75
|
+
csv.each do |row|
|
76
|
+
headers.each do |header|
|
77
|
+
if row[header].match(/\A(\d)+\z/) && (headers_and_type[header].nil? || headers_and_type[header] == :integer)
|
78
|
+
headers_and_type[header] = :integer
|
79
|
+
elsif row[header].match(/^\d*\.\d+$/) && (headers_and_type[header].nil? || headers_and_type[header] == :float)
|
80
|
+
headers_and_type[header] = :float
|
81
|
+
else
|
82
|
+
headers_and_type[header] = :string
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
headers_and_type
|
88
|
+
end
|
89
|
+
|
90
|
+
def convert_header_to_methodable_name(header, track_original: false)
|
91
|
+
if track_original
|
92
|
+
@original_headers ||= []
|
93
|
+
@original_headers << header
|
94
|
+
end
|
41
95
|
|
42
96
|
header.downcase.gsub(/[^a-z]/, "_").to_sym
|
43
97
|
end
|