active_admin_import 2.1.2 → 3.0.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.hound.yml +4 -0
- data/.travis.yml +9 -0
- data/Gemfile +16 -0
- data/README.md +73 -173
- data/Rakefile +7 -2
- data/active_admin_import.gemspec +5 -3
- data/app/views/admin/import.html.erb +1 -1
- data/config/locales/en.yml +11 -4
- data/config/locales/es.yml +18 -0
- data/config/locales/it.yml +3 -1
- data/config/locales/zh-CN.yml +24 -0
- data/lib/active_admin_import/dsl.rb +39 -31
- data/lib/active_admin_import/import_result.rb +39 -0
- data/lib/active_admin_import/importer.rb +91 -44
- data/lib/active_admin_import/model.rb +80 -29
- data/lib/active_admin_import/options.rb +44 -0
- data/lib/active_admin_import/version.rb +1 -1
- data/lib/active_admin_import.rb +3 -0
- data/spec/fixtures/files/author.csv +2 -0
- data/spec/fixtures/files/author_broken_header.csv +2 -0
- data/spec/fixtures/files/author_invalid.csv +2 -0
- data/spec/fixtures/files/authors.csv +3 -0
- data/spec/fixtures/files/authors_bom.csv +3 -0
- data/spec/fixtures/files/authors_invalid_db.csv +3 -0
- data/spec/fixtures/files/authors_invalid_model.csv +3 -0
- data/spec/fixtures/files/authors_no_headers.csv +2 -0
- data/spec/fixtures/files/authors_win1251_win_endline.csv +3 -0
- data/spec/fixtures/files/authors_with_ids.csv +3 -0
- data/spec/fixtures/files/authors_with_semicolons.csv +3 -0
- data/spec/fixtures/files/empty.csv +0 -0
- data/spec/fixtures/files/only_headers.csv +1 -0
- data/spec/fixtures/files/posts.csv +4 -0
- data/spec/fixtures/files/posts_for_author.csv +3 -0
- data/spec/fixtures/files/posts_for_author_no_headers.csv +2 -0
- data/spec/import_result_spec.rb +32 -0
- data/spec/import_spec.rb +432 -0
- data/spec/model_spec.rb +5 -0
- data/spec/spec_helper.rb +72 -0
- data/spec/support/active_model_lint.rb +14 -0
- data/spec/support/admin.rb +20 -0
- data/spec/support/rails_template.rb +29 -0
- data/tasks/test.rake +6 -0
- metadata +80 -19
@@ -2,47 +2,33 @@ require 'csv'
|
|
2
2
|
module ActiveAdminImport
|
3
3
|
class Importer
|
4
4
|
|
5
|
-
attr_reader :resource,
|
6
|
-
|
7
|
-
def store
|
8
|
-
result = @resource.transaction do
|
9
|
-
options[:before_batch_import].call(self) if options[:before_batch_import].is_a?(Proc)
|
10
|
-
|
11
|
-
result = resource.import headers.values, csv_lines, {
|
12
|
-
validate: options[:validate],
|
13
|
-
on_duplicate_key_update: options[:on_duplicate_key_update],
|
14
|
-
ignore: options[:ignore],
|
15
|
-
timestamps: options[:timestamps]
|
16
|
-
}
|
17
|
-
options[:after_batch_import].call(self) if options[:after_batch_import].is_a?(Proc)
|
18
|
-
result
|
19
|
-
end
|
20
|
-
{imported: csv_lines.count - result.failed_instances.count, failed: result.failed_instances}
|
21
|
-
end
|
5
|
+
attr_reader :resource, :options, :result, :model
|
6
|
+
attr_accessor :csv_lines, :headers
|
22
7
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
8
|
+
OPTIONS = [
|
9
|
+
:validate,
|
10
|
+
:on_duplicate_key_update,
|
11
|
+
:ignore,
|
12
|
+
:timestamps,
|
13
|
+
:before_import,
|
14
|
+
:after_import,
|
15
|
+
:before_batch_import,
|
16
|
+
:after_batch_import,
|
17
|
+
:headers_rewrites,
|
18
|
+
:batch_size,
|
19
|
+
:batch_transaction,
|
20
|
+
:csv_options
|
21
|
+
].freeze
|
29
22
|
|
30
23
|
def initialize(resource, model, options)
|
31
24
|
@resource = resource
|
32
25
|
@model = model
|
33
|
-
@options = {batch_size: 1000, validate: true}.merge(options)
|
34
26
|
@headers = model.respond_to?(:csv_headers) ? model.csv_headers : []
|
35
|
-
|
36
|
-
|
37
|
-
ActiveSupport::Deprecation.warn "row_sep and col_sep options are deprecated, use csv_options to override default CSV options"
|
38
|
-
@csv_options = @options.slice(:col_sep, :row_sep)
|
39
|
-
else
|
40
|
-
@csv_options = @options[:csv_options] || {}
|
41
|
-
end
|
42
|
-
#override csv options from model if it respond_to csv_options
|
43
|
-
@csv_options = model.csv_options if model.respond_to?(:csv_options)
|
44
|
-
@csv_options.reject! {| key, value | value.blank? }
|
27
|
+
assign_options(options)
|
28
|
+
end
|
45
29
|
|
30
|
+
def import_result
|
31
|
+
@import_result ||= ImportResult.new
|
46
32
|
end
|
47
33
|
|
48
34
|
def file
|
@@ -51,28 +37,89 @@ module ActiveAdminImport
|
|
51
37
|
|
52
38
|
def cycle(lines)
|
53
39
|
@csv_lines = CSV.parse(lines.join, @csv_options)
|
54
|
-
|
40
|
+
import_result.add(batch_import, lines.count)
|
55
41
|
end
|
56
42
|
|
57
43
|
def import
|
58
|
-
|
59
|
-
|
60
|
-
|
44
|
+
run_callback(:before_import)
|
45
|
+
process_file
|
46
|
+
run_callback(:after_import)
|
47
|
+
import_result
|
48
|
+
end
|
49
|
+
|
50
|
+
def import_options
|
51
|
+
@import_options ||= options.slice(:validate, :on_duplicate_key_update, :ignore, :timestamps, :batch_transaction)
|
52
|
+
end
|
53
|
+
|
54
|
+
def batch_replace(header_key, options)
|
55
|
+
index = header_index(header_key)
|
56
|
+
csv_lines.map! do |line|
|
57
|
+
from = line[index]
|
58
|
+
line[index] = options[from] if options.has_key?(from)
|
59
|
+
line
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def values_at(header_key)
|
64
|
+
csv_lines.collect { |line| line[header_index(header_key)] }.uniq
|
65
|
+
end
|
66
|
+
|
67
|
+
def header_index(header_key)
|
68
|
+
headers.values.index(header_key)
|
69
|
+
end
|
70
|
+
|
71
|
+
protected
|
72
|
+
|
73
|
+
def process_file
|
74
|
+
lines, batch_size = [], options[:batch_size].to_i
|
61
75
|
File.open(file.path) do |f|
|
62
76
|
# capture headers if not exist
|
63
|
-
prepare_headers
|
77
|
+
prepare_headers { CSV.parse(f.readline, @csv_options).first }
|
64
78
|
f.each_line do |line|
|
65
|
-
|
66
|
-
lines << line
|
79
|
+
lines << line if line.present?
|
67
80
|
if lines.size == batch_size || f.eof?
|
68
|
-
cycle
|
81
|
+
cycle(lines)
|
69
82
|
lines = []
|
70
83
|
end
|
71
84
|
end
|
72
85
|
end
|
73
86
|
cycle(lines) unless lines.blank?
|
74
|
-
options[:after_import].call(self) if options[:after_import].is_a?(Proc)
|
75
|
-
result
|
76
87
|
end
|
88
|
+
|
89
|
+
def prepare_headers
|
90
|
+
headers = self.headers.present? ? self.headers.map(&:to_s) : yield
|
91
|
+
@headers = Hash[headers.zip(headers.map { |el| el.underscore.gsub(/\s+/, '_') })].with_indifferent_access
|
92
|
+
@headers.merge!(options[:headers_rewrites].symbolize_keys.slice(*@headers.symbolize_keys.keys))
|
93
|
+
@headers
|
94
|
+
end
|
95
|
+
|
96
|
+
def run_callback(name)
|
97
|
+
options[name].call(self) if options[name].is_a?(Proc)
|
98
|
+
end
|
99
|
+
|
100
|
+
def batch_import
|
101
|
+
batch_result = nil
|
102
|
+
@resource.transaction do
|
103
|
+
run_callback(:before_batch_import)
|
104
|
+
batch_result = resource.import(headers.values, csv_lines, import_options)
|
105
|
+
raise ActiveRecord::Rollback if import_options[:batch_transaction] && batch_result.failed_instances.any?
|
106
|
+
run_callback(:after_batch_import)
|
107
|
+
end
|
108
|
+
batch_result
|
109
|
+
end
|
110
|
+
|
111
|
+
def assign_options(options)
|
112
|
+
@options = {batch_size: 1000, validate: true}.merge(options.slice(*OPTIONS))
|
113
|
+
detect_csv_options
|
114
|
+
end
|
115
|
+
|
116
|
+
def detect_csv_options
|
117
|
+
@csv_options = if model.respond_to?(:csv_options)
|
118
|
+
model.csv_options
|
119
|
+
else
|
120
|
+
options[:csv_options] || {}
|
121
|
+
end.reject { |_, value| value.blank? }
|
122
|
+
end
|
123
|
+
|
77
124
|
end
|
78
125
|
end
|
@@ -1,19 +1,23 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rchardet'
|
4
|
+
|
1
5
|
module ActiveAdminImport
|
2
6
|
class Model
|
3
|
-
|
4
|
-
include ActiveModel::
|
7
|
+
|
8
|
+
include ActiveModel::Model
|
5
9
|
include ActiveModel::Validations
|
6
10
|
include ActiveModel::Validations::Callbacks
|
7
11
|
|
8
|
-
validates :file, presence: {
|
9
|
-
|
10
|
-
|
11
|
-
validate :correct_content_type, if: proc { |me| me.file.present? }
|
12
|
-
validate :file_contents_present, if: proc { |me| me.file.present? }
|
12
|
+
validates :file, presence: {
|
13
|
+
message: ->(*_){ I18n.t("active_admin_import.no_file_error") }
|
14
|
+
}, unless: ->(me){ me.new_record? }
|
13
15
|
|
16
|
+
validate :correct_content_type, if: ->(me) { me.file.present? }
|
17
|
+
validate :file_contents_present, if: ->(me) { me.file.present? }
|
14
18
|
|
15
|
-
before_validation :
|
16
|
-
before_validation :encode_file,
|
19
|
+
before_validation :unzip_file, if: ->(me) { me.archive? && me.allow_archive? }
|
20
|
+
before_validation :encode_file, if: ->(me) { me.force_encoding? && me.file.present? }
|
17
21
|
|
18
22
|
attr_reader :attributes
|
19
23
|
|
@@ -27,12 +31,7 @@ module ActiveAdminImport
|
|
27
31
|
@attributes.merge!(args)
|
28
32
|
@new_record = new_record
|
29
33
|
args.keys.each do |key|
|
30
|
-
key
|
31
|
-
#generate methods for instance object by attributes
|
32
|
-
singleton_class.class_eval do
|
33
|
-
define_method(key) { self.attributes[key] } unless method_defined? key
|
34
|
-
define_method("#{key}=") { |new_value| @attributes[key] = new_value } unless method_defined? "#{key}="
|
35
|
-
end
|
34
|
+
define_methods_for(key.to_sym)
|
36
35
|
end if args.is_a?(Hash)
|
37
36
|
end
|
38
37
|
|
@@ -41,11 +40,17 @@ module ActiveAdminImport
|
|
41
40
|
end
|
42
41
|
|
43
42
|
def default_attributes
|
44
|
-
{
|
43
|
+
{
|
44
|
+
allow_archive: true,
|
45
|
+
csv_headers: [],
|
46
|
+
file: nil,
|
47
|
+
force_encoding: "UTF-8",
|
48
|
+
hint: ""
|
49
|
+
}
|
45
50
|
end
|
46
51
|
|
47
52
|
def allow_archive?
|
48
|
-
|
53
|
+
!!attributes[:allow_archive]
|
49
54
|
end
|
50
55
|
|
51
56
|
def new_record?
|
@@ -53,11 +58,7 @@ module ActiveAdminImport
|
|
53
58
|
end
|
54
59
|
|
55
60
|
def force_encoding?
|
56
|
-
|
57
|
-
end
|
58
|
-
|
59
|
-
def to_hash
|
60
|
-
@attributes
|
61
|
+
!!attributes[:force_encoding]
|
61
62
|
end
|
62
63
|
|
63
64
|
def persisted?
|
@@ -68,6 +69,8 @@ module ActiveAdminImport
|
|
68
69
|
file_type == 'application/zip'
|
69
70
|
end
|
70
71
|
|
72
|
+
alias :to_hash :attributes
|
73
|
+
|
71
74
|
protected
|
72
75
|
|
73
76
|
def file_path
|
@@ -79,23 +82,21 @@ module ActiveAdminImport
|
|
79
82
|
end
|
80
83
|
|
81
84
|
def encode_file
|
82
|
-
data = File.read(file_path)
|
85
|
+
data = File.read(file_path)
|
83
86
|
File.open(file_path, 'w') do |f|
|
84
|
-
|
87
|
+
f.write(encode(data))
|
85
88
|
end
|
86
89
|
end
|
87
90
|
|
88
|
-
def
|
91
|
+
def unzip_file
|
89
92
|
Zip::File.open(file_path) do |zip_file|
|
90
|
-
self.file = Tempfile.new(
|
93
|
+
self.file = Tempfile.new('active-admin-import-unzipped')
|
91
94
|
data = zip_file.entries.select { |f| f.file? }.first.get_input_stream.read
|
92
|
-
data = data.encode(force_encoding, invalid: :replace, undef: :replace) if self.force_encoding?
|
93
95
|
self.file << data
|
94
96
|
self.file.close
|
95
97
|
end
|
96
98
|
end
|
97
99
|
|
98
|
-
|
99
100
|
def csv_allowed_types
|
100
101
|
[
|
101
102
|
'text/csv',
|
@@ -107,7 +108,6 @@ module ActiveAdminImport
|
|
107
108
|
]
|
108
109
|
end
|
109
110
|
|
110
|
-
|
111
111
|
def correct_content_type
|
112
112
|
unless file.blank? || file.is_a?(Tempfile)
|
113
113
|
errors.add(:file, I18n.t('active_admin_import.file_format_error')) unless csv_allowed_types.include? file_type
|
@@ -125,6 +125,57 @@ module ActiveAdminImport
|
|
125
125
|
''
|
126
126
|
end
|
127
127
|
end
|
128
|
+
|
129
|
+
protected
|
130
|
+
|
131
|
+
def define_methods_for(attr_name)
|
132
|
+
#generate methods for instance object by attributes
|
133
|
+
singleton_class.class_eval do
|
134
|
+
define_set_method(attr_name)
|
135
|
+
define_get_method(attr_name)
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
def encode(data)
|
140
|
+
data = content_encode(data) if force_encoding?
|
141
|
+
data = data.encode(
|
142
|
+
'UTF-8',
|
143
|
+
invalid: :replace, undef: :replace, universal_newline: true
|
144
|
+
)
|
145
|
+
begin
|
146
|
+
data.sub("\xEF\xBB\xBF", '') # bom
|
147
|
+
rescue StandardError => _
|
148
|
+
data
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def detect_encoding?
|
153
|
+
force_encoding == :auto
|
154
|
+
end
|
155
|
+
|
156
|
+
def dynamic_encoding(data)
|
157
|
+
CharDet.detect(data)['encoding']
|
158
|
+
end
|
159
|
+
|
160
|
+
def content_encode(data)
|
161
|
+
encoding_name = if detect_encoding?
|
162
|
+
dynamic_encoding(data)
|
163
|
+
else
|
164
|
+
force_encoding.to_s
|
165
|
+
end
|
166
|
+
data.force_encoding(encoding_name)
|
167
|
+
end
|
168
|
+
|
169
|
+
class <<self
|
170
|
+
def define_set_method(attr_name)
|
171
|
+
define_method(attr_name) { self.attributes[attr_name] } unless method_defined? attr_name
|
172
|
+
end
|
173
|
+
|
174
|
+
def define_get_method(attr_name)
|
175
|
+
define_method("#{attr_name}=") { |new_value| @attributes[attr_name] = new_value } unless method_defined? "#{attr_name}="
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
128
179
|
end
|
129
180
|
end
|
130
181
|
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module ActiveAdminImport
|
2
|
+
|
3
|
+
module Options
|
4
|
+
VALID_OPTIONS = [
|
5
|
+
:back,
|
6
|
+
:csv_options,
|
7
|
+
:validate,
|
8
|
+
:batch_size,
|
9
|
+
:batch_transaction,
|
10
|
+
:before_import,
|
11
|
+
:after_import,
|
12
|
+
:before_batch_import,
|
13
|
+
:after_batch_import,
|
14
|
+
:on_duplicate_key_update,
|
15
|
+
:timestamps,
|
16
|
+
:ignore,
|
17
|
+
:template,
|
18
|
+
:template_object,
|
19
|
+
:resource_class,
|
20
|
+
:resource_label,
|
21
|
+
:plural_resource_label,
|
22
|
+
:headers_rewrites
|
23
|
+
].freeze
|
24
|
+
|
25
|
+
|
26
|
+
def self.options_for(config, options= {})
|
27
|
+
options[:template_object] = ActiveAdminImport::Model.new unless options.has_key? :template_object
|
28
|
+
|
29
|
+
{
|
30
|
+
back: {action: :import},
|
31
|
+
csv_options: {},
|
32
|
+
template: "admin/import",
|
33
|
+
resource_class: config.resource_class,
|
34
|
+
resource_label: config.resource_label,
|
35
|
+
plural_resource_label: config.plural_resource_label,
|
36
|
+
headers_rewrites: {}
|
37
|
+
}.deep_merge(options)
|
38
|
+
|
39
|
+
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
data/lib/active_admin_import.rb
CHANGED
@@ -2,9 +2,12 @@ require 'activerecord-import'
|
|
2
2
|
require 'active_admin'
|
3
3
|
require 'active_admin_import/version'
|
4
4
|
require 'active_admin_import/engine'
|
5
|
+
require 'active_admin_import/import_result'
|
6
|
+
require 'active_admin_import/options'
|
5
7
|
require 'active_admin_import/dsl'
|
6
8
|
require 'active_admin_import/importer'
|
7
9
|
require 'active_admin_import/model'
|
8
10
|
require 'active_admin_import/authorization'
|
9
11
|
::ActiveAdmin::DSL.send(:include, ActiveAdminImport::DSL)
|
10
12
|
|
13
|
+
|
File without changes
|
@@ -0,0 +1 @@
|
|
1
|
+
Name,Last name,Birthday
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe ActiveAdminImport::ImportResult do
|
4
|
+
context "failed_message" do
|
5
|
+
let(:import_result) { ActiveAdminImport::ImportResult.new }
|
6
|
+
|
7
|
+
before do
|
8
|
+
Author.create(name: 'John', last_name: 'Doe')
|
9
|
+
Author.create(name: 'Jane', last_name: 'Roe')
|
10
|
+
|
11
|
+
@result = double \
|
12
|
+
failed_instances: [
|
13
|
+
Author.create(name: 'Jim', last_name: "Doe"), # {:last_name=>["has already been taken"]}
|
14
|
+
Author.create(name: nil, last_name: 'Doe') # {:name=>["can't be blank"], :last_name=>["has already been taken"]}
|
15
|
+
]
|
16
|
+
end
|
17
|
+
|
18
|
+
it "should work without any failed instances" do
|
19
|
+
expect(import_result.failed_message).to eq("")
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should work" do
|
23
|
+
import_result.add(@result, 4)
|
24
|
+
expect(import_result.failed_message).to eq("Last name has already been taken - Doe ; Name can't be blank - , Last name has already been taken - Doe")
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should work on limit param" do
|
28
|
+
import_result.add(@result, 4)
|
29
|
+
expect(import_result.failed_message(limit: 1)).to eq("Last name has already been taken - Doe")
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|