rails_csv_importer 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- data/MIT-LICENSE +21 -0
- data/README.rdoc +123 -0
- data/lib/rails_csv_importer.rb +229 -0
- data/rails/init.rb +2 -0
- metadata +68 -0
data/MIT-LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (c) 2012 Ben Li
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
21
|
+
|
data/README.rdoc
ADDED
@@ -0,0 +1,123 @@
|
|
1
|
+
= RailsCsvImporter
|
2
|
+
|
3
|
+
Define configuration in a hash and then import Ruby on Rails model data from CSV with one method call.
|
4
|
+
|
5
|
+
Both Rails 2 and 3 are supported.
|
6
|
+
|
7
|
+
== Usage
|
8
|
+
|
9
|
+
1. Add acts_as_rails_csv_importer to your model
|
10
|
+
2. Call YourModel.get_csv_import_template to generate the CSV template.
|
11
|
+
3. Call YourModel.import_from_csv in your controller to import data from a CSV file uploaded.
|
12
|
+
|
13
|
+
=== In a Rails Model
|
14
|
+
|
15
|
+
class Foo < ActiveRecord::Base
|
16
|
+
acts_as_rails_csv_importer
|
17
|
+
end
|
18
|
+
|
19
|
+
=== Generate CSV Template
|
20
|
+
|
21
|
+
Call Foo.get_csv_import_template(import_config). See below for details on the parameter.
|
22
|
+
|
23
|
+
=== Import From CSV
|
24
|
+
|
25
|
+
Call Foo.import_from_csv(import_config, content, options = {})
|
26
|
+
|
27
|
+
* import_config
|
28
|
+
|
29
|
+
A hash that defines what data are imported and how.
|
30
|
+
|
31
|
+
[:mapping] A hash that defines the columns in the csv. Each hash key is the model attribute name for the column.
|
32
|
+
Each value is a hash defining how the column is processed:
|
33
|
+
|
34
|
+
[:name] The heading that identify the column in the csv.
|
35
|
+
If omitted, the heading will be the humanized form of the key.
|
36
|
+
(_Optional_)
|
37
|
+
|
38
|
+
There are four different ways to specify how the column is processed:
|
39
|
+
|
40
|
+
[:value_method] A lambda to convert the column value from csv into the attribute value in the model.
|
41
|
+
Class Acts::RailsCsvImporter::ValueMethods provides some pre-defined value methods.
|
42
|
+
(_Optional_)
|
43
|
+
|
44
|
+
[:record_method] A lambda to find the associated record if this attribue is a foreign key.
|
45
|
+
(_Optional_)
|
46
|
+
|
47
|
+
[:virtual] If set to true, the column is not mapped to a model attribute.
|
48
|
+
Usually the column will be used together with other columns.
|
49
|
+
|
50
|
+
[None of the above options present] The column value will be used without any conversion.
|
51
|
+
|
52
|
+
[:find_existing] A lambda to find the existing record to update for a row.
|
53
|
+
A new record is created if the lambda could not find an existing record and return nil.
|
54
|
+
The row is passed as a hash to the lambda.
|
55
|
+
If this parameter is omitted, a new record is created for each row.
|
56
|
+
(_Optional_)
|
57
|
+
|
58
|
+
* content
|
59
|
+
|
60
|
+
A string that contains the csv content to import from.
|
61
|
+
|
62
|
+
* options
|
63
|
+
|
64
|
+
A hash of options for the import. (_Optional_)
|
65
|
+
|
66
|
+
[:partial_save] If true, any valid rows in the csv will be saved even if there are some invalid rows.
|
67
|
+
Otherwise, data will be imported only when all rows are valid.
|
68
|
+
|
69
|
+
Exception +Acts::RailsCsvImporter::RailsCsvImportError+ is thrown when there are errors duing the importing.
|
70
|
+
|
71
|
+
== Example
|
72
|
+
|
73
|
+
class Material < ActiveRecord::Base
|
74
|
+
acts_as_rails_csv_importer
|
75
|
+
end
|
76
|
+
|
77
|
+
class MaterialsController < ApplicationController
|
78
|
+
IMPORT_CONFIG = {
|
79
|
+
:mapping => {
|
80
|
+
'name' => {},
|
81
|
+
'fragile' => {
|
82
|
+
:name => "Fragile?",
|
83
|
+
:value_method => Acts::RailsCsvImporter::ValueMethods.boolean_value_method
|
84
|
+
},
|
85
|
+
'category_id' => {:record_method => lambda { |v, row, mapping| Category.find_by_name(v) } },
|
86
|
+
},
|
87
|
+
:find_existing => lambda { |row| Material.find_by_name(row['name']) }
|
88
|
+
}
|
89
|
+
|
90
|
+
def download_template
|
91
|
+
headers["Content-Type"] = 'text/csv'
|
92
|
+
headers["Content-Disposition"] = 'attachment; filename="template.csv"'
|
93
|
+
render :text => Material.get_import_template(IMPORT_CONFIG)
|
94
|
+
end
|
95
|
+
|
96
|
+
def import
|
97
|
+
num_imported = Material.import_from_csv(IMPORT_CONFIG, params[:file_upload])
|
98
|
+
flash[:notice] = num_imported.to_s + " records imported successfully."
|
99
|
+
rescue Acts::RailsCsvImporter::RailsCsvImportError => ex
|
100
|
+
@err_message = ex.errors.map { |err, row| err.is_a?(String) ? err : err.full_messages }.join(';')
|
101
|
+
render :template => :error
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
For more examples, refer to +test/rails_csv_importer_test.rb+
|
106
|
+
|
107
|
+
== Download and installation
|
108
|
+
|
109
|
+
The latest version of RailsCsvImporter can be installed with RubyGems:
|
110
|
+
|
111
|
+
% [sudo] gem install rails_csv_importer
|
112
|
+
|
113
|
+
Source code can be downloaded on GitHub
|
114
|
+
|
115
|
+
* https://github.com/benli/rails_csv_importer/tree/master
|
116
|
+
|
117
|
+
|
118
|
+
== License
|
119
|
+
|
120
|
+
RailsCsvImporter is released under the MIT license:
|
121
|
+
|
122
|
+
* http://www.opensource.org/licenses/MIT
|
123
|
+
|
@@ -0,0 +1,229 @@
|
|
1
|
+
if RUBY_VERSION >= "1.9"
|
2
|
+
# CSV in ruby 1.9 is FasterCSV plus support for Ruby 1.9's m17n encoding engine
|
3
|
+
require 'csv'
|
4
|
+
FasterCSV = CSV
|
5
|
+
else
|
6
|
+
require 'faster_csv'
|
7
|
+
end
|
8
|
+
require 'iconv'
|
9
|
+
|
10
|
+
module Acts # :nodoc
|
11
|
+
module RailsCsvImporter
|
12
|
+
#
|
13
|
+
# This class is the placeholder of methods common used.
|
14
|
+
#
|
15
|
+
class ValueMethods
|
16
|
+
#
|
17
|
+
# A value method that can be used on boolean type columns.
|
18
|
+
# Accept 'yes' and 'no' instead of the default 'true" and 'false'
|
19
|
+
#
|
20
|
+
def self.yes_no_value_method
|
21
|
+
lambda { |v, row, mapping|
|
22
|
+
case v.downcase
|
23
|
+
when 'yes' then true
|
24
|
+
when 'no' then false
|
25
|
+
else
|
26
|
+
raise "Value must be Yes or No"
|
27
|
+
end
|
28
|
+
}
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
#
|
33
|
+
# The exception thrown when there are errors in the import.
|
34
|
+
#
|
35
|
+
class RailsCsvImportError < RuntimeError
|
36
|
+
#
|
37
|
+
# An array of errors occurred during the import.
|
38
|
+
#
|
39
|
+
# Each error is an array with two elements:
|
40
|
+
# 1. Either a string of error message or the ActiveRecord::Errors object.
|
41
|
+
# 2. An array of columns in the row that is associated with this error.
|
42
|
+
#
|
43
|
+
attr_reader :errors
|
44
|
+
|
45
|
+
# An array of column headings.
|
46
|
+
attr_reader :header_row
|
47
|
+
|
48
|
+
# Number of rows already imported.
|
49
|
+
attr_reader :num_imported
|
50
|
+
|
51
|
+
def initialize(errors, header_row, num_imported) # :nodoc:
|
52
|
+
@errors = errors
|
53
|
+
@header_row = header_row
|
54
|
+
@num_imported = num_imported
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.included(base) # :nodoc:
|
59
|
+
base.extend(ClassMethods)
|
60
|
+
end
|
61
|
+
|
62
|
+
module ClassMethods
|
63
|
+
#
|
64
|
+
# Called in a Rails model to bring it CSV Import funcationality provided in this gem.
|
65
|
+
#
|
66
|
+
def acts_as_rails_csv_importer
|
67
|
+
class_eval do
|
68
|
+
extend Acts::RailsCsvImporter::SingletonMethods
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
module SingletonMethods
|
74
|
+
#
|
75
|
+
# Import model data from csv content
|
76
|
+
#
|
77
|
+
# Options:
|
78
|
+
#
|
79
|
+
# * +import_config+ - specifies how the csv content is parsed.
|
80
|
+
# * +content+ - the csv content in a string that can be parsed by FasterCSV
|
81
|
+
# * +options+ - none for now
|
82
|
+
#
|
83
|
+
# Throws: +RailsCsvImportError+
|
84
|
+
#
|
85
|
+
# See +README.rdoc+ for details.
|
86
|
+
#
|
87
|
+
def import_from_csv(import_config, content, options = {})
|
88
|
+
ic = Iconv.new('UTF-8', 'UTF-8')
|
89
|
+
|
90
|
+
num_rows_saved=0
|
91
|
+
errors = []
|
92
|
+
header_row = []
|
93
|
+
|
94
|
+
mapping = import_config[:mapping]
|
95
|
+
# a hash for finding the key for a column heading quickly
|
96
|
+
name_to_column_hash = mapping.keys.inject({}) { |acc, key|
|
97
|
+
acc[translate_column(key, mapping).downcase] = key
|
98
|
+
acc
|
99
|
+
}
|
100
|
+
|
101
|
+
self.transaction do
|
102
|
+
all_rows = []
|
103
|
+
# the column keys in the order of the column headings
|
104
|
+
col_names = []
|
105
|
+
row_num = 0
|
106
|
+
col_num = 0
|
107
|
+
|
108
|
+
# first phase: parse the csv and store the result in all_rows
|
109
|
+
|
110
|
+
first_row = true
|
111
|
+
begin
|
112
|
+
FasterCSV.parse(content, :skip_blanks => true) do |row|
|
113
|
+
row_num += 1
|
114
|
+
col_num = 0
|
115
|
+
row = row.map { |col| col_num += 1; ic.iconv(col) }
|
116
|
+
if first_row == true
|
117
|
+
header_row = row
|
118
|
+
row.each { |column_name| col_names << name_to_column_hash[column_name.downcase] }
|
119
|
+
first_row = false
|
120
|
+
else
|
121
|
+
all_rows << row
|
122
|
+
row_hash = {}
|
123
|
+
row.each_with_index { |column, x| row_hash[col_names[x]] = column if col_names[x] }
|
124
|
+
end
|
125
|
+
end
|
126
|
+
rescue Iconv::IllegalSequence => ex
|
127
|
+
all_rows = header_row = []
|
128
|
+
errors << ["Invalid character encountered in row #{row_num}, column #{col_num} in the CSV file: #{ex.message}", []]
|
129
|
+
rescue FasterCSV::MalformedCSVError => ex
|
130
|
+
all_rows = header_row = []
|
131
|
+
errors << ["Invalid CSV format: #{ex.message}", []]
|
132
|
+
end
|
133
|
+
|
134
|
+
# second phase: process the rows
|
135
|
+
|
136
|
+
all_rows.each do |row|
|
137
|
+
row_hash = {}
|
138
|
+
row.each_with_index { |column, x| row_hash[col_names[x]] = column if col_names[x] }
|
139
|
+
|
140
|
+
find_existing = import_config[:find_existing]
|
141
|
+
if find_existing
|
142
|
+
record = find_existing.call(row_hash) || self.new
|
143
|
+
else
|
144
|
+
record = self.new
|
145
|
+
end
|
146
|
+
|
147
|
+
begin
|
148
|
+
row.each_with_index do |column, x|
|
149
|
+
col_name = col_names[x]
|
150
|
+
if col_name
|
151
|
+
col_config = mapping[col_name]
|
152
|
+
unless column.blank?
|
153
|
+
begin
|
154
|
+
# assign the correct value to the attribute according to the config
|
155
|
+
record[col_name] = if col_config[:value_method]
|
156
|
+
col_config[:value_method].call(column, row_hash, mapping)
|
157
|
+
elsif col_config[:record_method]
|
158
|
+
r = col_config[:record_method].call(column, row_hash, mapping)
|
159
|
+
raise "Unable to find referred record of value #{column}" if r.nil?
|
160
|
+
r.id
|
161
|
+
else
|
162
|
+
column
|
163
|
+
end
|
164
|
+
rescue Exception => e
|
165
|
+
raise "Failed to import column '#{header_row[x]}': #{e.message}"
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
rescue Exception => e
|
171
|
+
errors << [e.message, row]
|
172
|
+
next
|
173
|
+
end
|
174
|
+
if record.save
|
175
|
+
num_rows_saved += 1
|
176
|
+
else
|
177
|
+
errors << [record.errors, row]
|
178
|
+
end
|
179
|
+
end # all_rows
|
180
|
+
|
181
|
+
raise RailsCsvImportError.new(errors, header_row, num_rows_saved) if errors.any? && !options[:partial_save]
|
182
|
+
end # transaction
|
183
|
+
|
184
|
+
raise RailsCsvImportError.new(errors, header_row, num_rows_saved) if errors.any? && options[:partial_save]
|
185
|
+
|
186
|
+
num_rows_saved
|
187
|
+
end
|
188
|
+
|
189
|
+
#
|
190
|
+
# Return the csv import template in a string.
|
191
|
+
#
|
192
|
+
# options:
|
193
|
+
#
|
194
|
+
# * +import_config+ - specifies how the csv content is parsed when importing from the template.
|
195
|
+
# See +README.rdoc+ for details
|
196
|
+
#
|
197
|
+
def get_csv_import_template(import_config)
|
198
|
+
mapping = import_config[:mapping]
|
199
|
+
FasterCSV.generate do |csv|
|
200
|
+
csv << mapping.keys.map { |key| translate_column(key, mapping) }
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
private
|
205
|
+
|
206
|
+
#
|
207
|
+
# Translate a column key into its heading
|
208
|
+
#
|
209
|
+
# options:
|
210
|
+
#
|
211
|
+
# * +col+ - the key of the column
|
212
|
+
# * +mapping+ - the mapping parameter in the config
|
213
|
+
#
|
214
|
+
def translate_column(col, mapping)
|
215
|
+
if mapping[col][:name]
|
216
|
+
mapping[col][:name]
|
217
|
+
else
|
218
|
+
if col[-3,3] == "_id"
|
219
|
+
col = col[0, col.length - 3]
|
220
|
+
end
|
221
|
+
col.humanize
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
ActiveRecord::Base.send(:include, Acts::RailsCsvImporter) if defined?(ActiveRecord)
|
229
|
+
|
data/rails/init.rb
ADDED
metadata
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rails_csv_importer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 29
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- 3
|
10
|
+
version: 0.1.3
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Ben Li
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-06-25 00:00:00 Z
|
19
|
+
dependencies: []
|
20
|
+
|
21
|
+
description: Define configuration in a hash and then import Ruby on Rails model data from CSV with one method call.
|
22
|
+
email: libin1231@gmail.com
|
23
|
+
executables: []
|
24
|
+
|
25
|
+
extensions: []
|
26
|
+
|
27
|
+
extra_rdoc_files: []
|
28
|
+
|
29
|
+
files:
|
30
|
+
- MIT-LICENSE
|
31
|
+
- README.rdoc
|
32
|
+
- lib/rails_csv_importer.rb
|
33
|
+
- rails/init.rb
|
34
|
+
homepage: http://rubygems.org/gems/rails_csv_importer
|
35
|
+
licenses: []
|
36
|
+
|
37
|
+
post_install_message:
|
38
|
+
rdoc_options: []
|
39
|
+
|
40
|
+
require_paths:
|
41
|
+
- lib
|
42
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
43
|
+
none: false
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
hash: 3
|
48
|
+
segments:
|
49
|
+
- 0
|
50
|
+
version: "0"
|
51
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
hash: 3
|
57
|
+
segments:
|
58
|
+
- 0
|
59
|
+
version: "0"
|
60
|
+
requirements: []
|
61
|
+
|
62
|
+
rubyforge_project:
|
63
|
+
rubygems_version: 1.8.24
|
64
|
+
signing_key:
|
65
|
+
specification_version: 3
|
66
|
+
summary: A little gem to ease data importing in Ruby on Rails
|
67
|
+
test_files: []
|
68
|
+
|