table_importer 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d97c5be89ed1115ef5f4ca473c650e80fc94a5c2
|
4
|
+
data.tar.gz: 0ce496477bfffbeb34ed62df57d584dec0e4cb9b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7511ab5023f3acc49988d034551173be2635edb79a6e09564ea0cfd58cfb8fd4d4618baeb2c3330666dcf4a9a89ccc001da2d69c4aec98155cc8f2dd8aa7c4d3
|
7
|
+
data.tar.gz: c884a44b264ddb101a7455c339e7a8a544bcfdc45ef0e0cf3d2bfc90ba757dc95ea55bd84d20fac85989d12f81e720b318a89fca3e7bb3b74a37406b6ecb91ad
|
data/lib/table_importer/csv.rb
CHANGED
@@ -17,6 +17,7 @@ module TableImporter
|
|
17
17
|
end
|
18
18
|
get_column_separator(first_line)
|
19
19
|
raise TableImporter::EmptyFileImportError.new unless file_has_content
|
20
|
+
@mapping = data[:user_headers]
|
20
21
|
@headers = @headers_present ? first_line.split(@column_separator) : default_headers(100)
|
21
22
|
rescue ArgumentError
|
22
23
|
@file = clean_file(@file)
|
@@ -114,12 +115,12 @@ module TableImporter
|
|
114
115
|
end
|
115
116
|
|
116
117
|
def chunks_with_headers(chunk_size)
|
117
|
-
key_mapping = convert_headers(SmarterCSV.process(@file.path, default_options).first.keys, @
|
118
|
+
key_mapping = convert_headers(SmarterCSV.process(@file.path, default_options).first.keys, @mapping, @headers_present).delete_if{ |key, value| value.blank?}
|
118
119
|
SmarterCSV.process(@file.path, default_options({:chunk_size => chunk_size, :key_mapping => key_mapping, :remove_unmapped_keys => true, :user_provided_headers => nil}))
|
119
120
|
end
|
120
121
|
|
121
122
|
def chunks_without_headers(chunk_size)
|
122
|
-
user_provided_headers = convert_headers(SmarterCSV.process(@file.path, default_options).first.keys, @
|
123
|
+
user_provided_headers = convert_headers(SmarterCSV.process(@file.path, default_options).first.keys, @mapping, @headers_present).values
|
123
124
|
SmarterCSV.process(@file.path, default_options({:chunk_size => chunk_size, :user_provided_headers => user_provided_headers, :remove_empty_values => true}))
|
124
125
|
end
|
125
126
|
|
@@ -6,10 +6,11 @@ module TableImporter
|
|
6
6
|
@headers
|
7
7
|
end
|
8
8
|
|
9
|
-
def get_preview_lines(start_point =
|
9
|
+
def get_preview_lines(start_point = 1, end_point = 10)
|
10
10
|
begin
|
11
11
|
@headers = @mapping.present? && @mapping != false ? convert_headers : @headers
|
12
|
-
|
12
|
+
start_point += 1 if @headers_present == true
|
13
|
+
lines = clean_chunks([get_lines(start_point, end_point, true)], @compulsory_headers)[0][:lines]
|
13
14
|
if lines.first.nil?
|
14
15
|
get_preview_lines(start_point+10, end_point+10)
|
15
16
|
else
|
@@ -20,12 +21,13 @@ module TableImporter
|
|
20
21
|
end
|
21
22
|
end
|
22
23
|
|
23
|
-
def get_lines(start, number_of_lines)
|
24
|
+
def get_lines(start, number_of_lines, preview_lines = false)
|
24
25
|
@last_row ||= @file.last_row
|
25
26
|
finish = [@last_row, start + number_of_lines].min
|
26
27
|
mapped_lines = []
|
27
|
-
(start
|
28
|
-
|
28
|
+
range = preview_lines ? (start..finish) : (start...finish)
|
29
|
+
range.each do |row_number|
|
30
|
+
mapped_lines << Hash[@headers.zip(@file.row(row_number))]
|
29
31
|
end
|
30
32
|
mapped_lines
|
31
33
|
end
|
Binary file
|
data/spec/sources/csv_spec.rb
CHANGED
@@ -6,7 +6,7 @@ describe TableImporter::Source do
|
|
6
6
|
|
7
7
|
context 'when source is a csv file with headers' do
|
8
8
|
before(:each) do
|
9
|
-
@source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/with_headers.csv"].join), :headers_present => true, :user_headers =>
|
9
|
+
@source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/with_headers.csv"].join), :headers_present => true, :user_headers => {"email" => "5"}, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
|
10
10
|
end
|
11
11
|
|
12
12
|
it "has the correct headers" do
|
@@ -86,7 +86,7 @@ describe TableImporter::Source do
|
|
86
86
|
|
87
87
|
context 'when source is a badly encoded file' do
|
88
88
|
it 'can still get the correct chunks' do
|
89
|
-
source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/mexico2013_pressdoc.csv"].join), :headers_present => true, :user_headers =>
|
89
|
+
source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/mexico2013_pressdoc.csv"].join), :headers_present => true, :user_headers => {"email" => "5"}, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
|
90
90
|
source.get_chunks.first[:lines].count.should eql(49)
|
91
91
|
end
|
92
92
|
end
|
@@ -120,7 +120,7 @@ describe TableImporter::Source do
|
|
120
120
|
context 'when source is badly encoded partway through the file' do
|
121
121
|
|
122
122
|
before(:each) do
|
123
|
-
@source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/partway.csv"].join), :headers_present => false, :user_headers =>
|
123
|
+
@source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/csv/partway.csv"].join), :headers_present => false, :user_headers => {"email" => "0"}, :type => "csv", :column_separator => "", :record_separator => "", :compulsory_headers => {:email => true}})
|
124
124
|
end
|
125
125
|
|
126
126
|
it "Gets the first chunk without error" do
|
data/spec/sources/excel_spec.rb
CHANGED
@@ -38,6 +38,10 @@ describe TableImporter::Source do
|
|
38
38
|
@source.get_chunks(1).count.should eql(6)
|
39
39
|
end
|
40
40
|
|
41
|
+
it "gets the correct chunk content" do
|
42
|
+
expect(@source.get_chunks[0][:lines].first[:email]).to eql("darrenp@fabrikam.com")
|
43
|
+
end
|
44
|
+
|
41
45
|
it "has the correct number of chunks" do
|
42
46
|
@source.get_chunks(2).count.should eql(4)
|
43
47
|
end
|
@@ -78,6 +82,10 @@ describe TableImporter::Source do
|
|
78
82
|
@source.get_chunks(1).count.should eql(6)
|
79
83
|
end
|
80
84
|
|
85
|
+
it "gets the correct chunk content" do
|
86
|
+
expect(@source.get_chunks[0][:lines].first[:email]).to eql("darrenp@fabrikam.com")
|
87
|
+
end
|
88
|
+
|
81
89
|
it "has the correct number of chunks" do
|
82
90
|
@source.get_chunks(2).count.should eql(4)
|
83
91
|
end
|
@@ -197,4 +205,38 @@ describe TableImporter::Source do
|
|
197
205
|
@source = nil
|
198
206
|
end
|
199
207
|
end
|
208
|
+
|
209
|
+
context "it has only one line" do
|
210
|
+
context "when mapping has not been set" do
|
211
|
+
before(:each) do
|
212
|
+
@source = TableImporter::Source.new({:content => File.open([Dir.pwd, "/spec/files/excel/one_line.xls"].join), :headers_present => "true", :type => "xls", :column_separator => "", :record_separator => "",
|
213
|
+
:compulsory_headers => {:email => true}
|
214
|
+
})
|
215
|
+
end
|
216
|
+
|
217
|
+
it 'has the correct preview lines' do
|
218
|
+
expect(@source.get_preview_lines.first[:Email]).to eql("mailto:john.smith@example.com")
|
219
|
+
end
|
220
|
+
|
221
|
+
after(:each) do
|
222
|
+
@source = nil
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
context "when mapping has been set" do
|
227
|
+
before(:each) do
|
228
|
+
@source = TableImporter::Source.new({content: File.open([Dir.pwd, "/spec/files/excel/one_line.xls"].join), headers_present: "true", type: "xls", column_separator: "", record_separator: "",
|
229
|
+
compulsory_headers: {email: true}, user_headers: {"email" => "13"}
|
230
|
+
})
|
231
|
+
end
|
232
|
+
|
233
|
+
it 'has the correct chunks' do
|
234
|
+
expect(@source.get_chunks[0][:lines].first[:email]).to eql("mailto:john.smith@example.com")
|
235
|
+
end
|
236
|
+
|
237
|
+
after(:each) do
|
238
|
+
@source = nil
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|
200
242
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: table_importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nick Dowse
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-12-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: spreadsheet
|
@@ -215,6 +215,7 @@ files:
|
|
215
215
|
- spec/files/excel/empty_lines.xlsx
|
216
216
|
- spec/files/excel/empty_lines_at_start.xlsx
|
217
217
|
- spec/files/excel/no_content.xlsx
|
218
|
+
- spec/files/excel/one_line.xls
|
218
219
|
- spec/files/excel/premapped_1.xls
|
219
220
|
- spec/files/excel/premapped_2.xls
|
220
221
|
- spec/files/excel/with_headers.xls
|
@@ -265,6 +266,7 @@ test_files:
|
|
265
266
|
- spec/files/excel/empty_lines.xlsx
|
266
267
|
- spec/files/excel/empty_lines_at_start.xlsx
|
267
268
|
- spec/files/excel/no_content.xlsx
|
269
|
+
- spec/files/excel/one_line.xls
|
268
270
|
- spec/files/excel/premapped_1.xls
|
269
271
|
- spec/files/excel/premapped_2.xls
|
270
272
|
- spec/files/excel/with_headers.xls
|