active_admin_import 2.1.2 → 3.0.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.hound.yml +4 -0
  3. data/.travis.yml +9 -0
  4. data/Gemfile +16 -0
  5. data/README.md +73 -173
  6. data/Rakefile +7 -2
  7. data/active_admin_import.gemspec +5 -3
  8. data/app/views/admin/import.html.erb +1 -1
  9. data/config/locales/en.yml +11 -4
  10. data/config/locales/es.yml +18 -0
  11. data/config/locales/it.yml +3 -1
  12. data/config/locales/zh-CN.yml +24 -0
  13. data/lib/active_admin_import/dsl.rb +39 -31
  14. data/lib/active_admin_import/import_result.rb +39 -0
  15. data/lib/active_admin_import/importer.rb +91 -44
  16. data/lib/active_admin_import/model.rb +80 -29
  17. data/lib/active_admin_import/options.rb +44 -0
  18. data/lib/active_admin_import/version.rb +1 -1
  19. data/lib/active_admin_import.rb +3 -0
  20. data/spec/fixtures/files/author.csv +2 -0
  21. data/spec/fixtures/files/author_broken_header.csv +2 -0
  22. data/spec/fixtures/files/author_invalid.csv +2 -0
  23. data/spec/fixtures/files/authors.csv +3 -0
  24. data/spec/fixtures/files/authors_bom.csv +3 -0
  25. data/spec/fixtures/files/authors_invalid_db.csv +3 -0
  26. data/spec/fixtures/files/authors_invalid_model.csv +3 -0
  27. data/spec/fixtures/files/authors_no_headers.csv +2 -0
  28. data/spec/fixtures/files/authors_win1251_win_endline.csv +3 -0
  29. data/spec/fixtures/files/authors_with_ids.csv +3 -0
  30. data/spec/fixtures/files/authors_with_semicolons.csv +3 -0
  31. data/spec/fixtures/files/empty.csv +0 -0
  32. data/spec/fixtures/files/only_headers.csv +1 -0
  33. data/spec/fixtures/files/posts.csv +4 -0
  34. data/spec/fixtures/files/posts_for_author.csv +3 -0
  35. data/spec/fixtures/files/posts_for_author_no_headers.csv +2 -0
  36. data/spec/import_result_spec.rb +32 -0
  37. data/spec/import_spec.rb +432 -0
  38. data/spec/model_spec.rb +5 -0
  39. data/spec/spec_helper.rb +72 -0
  40. data/spec/support/active_model_lint.rb +14 -0
  41. data/spec/support/admin.rb +20 -0
  42. data/spec/support/rails_template.rb +29 -0
  43. data/tasks/test.rake +6 -0
  44. metadata +80 -19
@@ -2,47 +2,33 @@ require 'csv'
2
2
  module ActiveAdminImport
3
3
  class Importer
4
4
 
5
- attr_reader :resource, :options, :result, :headers, :csv_lines, :model
6
-
7
- def store
8
- result = @resource.transaction do
9
- options[:before_batch_import].call(self) if options[:before_batch_import].is_a?(Proc)
10
-
11
- result = resource.import headers.values, csv_lines, {
12
- validate: options[:validate],
13
- on_duplicate_key_update: options[:on_duplicate_key_update],
14
- ignore: options[:ignore],
15
- timestamps: options[:timestamps]
16
- }
17
- options[:after_batch_import].call(self) if options[:after_batch_import].is_a?(Proc)
18
- result
19
- end
20
- {imported: csv_lines.count - result.failed_instances.count, failed: result.failed_instances}
21
- end
5
+ attr_reader :resource, :options, :result, :model
6
+ attr_accessor :csv_lines, :headers
22
7
 
23
- #
24
- def prepare_headers(headers)
25
- @headers = Hash[headers.zip(headers.map { |el| el.underscore.gsub(/\s+/, '_') })]
26
- @headers.merge!(options[:headers_rewrites])
27
- @headers
28
- end
8
+ OPTIONS = [
9
+ :validate,
10
+ :on_duplicate_key_update,
11
+ :ignore,
12
+ :timestamps,
13
+ :before_import,
14
+ :after_import,
15
+ :before_batch_import,
16
+ :after_batch_import,
17
+ :headers_rewrites,
18
+ :batch_size,
19
+ :batch_transaction,
20
+ :csv_options
21
+ ].freeze
29
22
 
30
23
  def initialize(resource, model, options)
31
24
  @resource = resource
32
25
  @model = model
33
- @options = {batch_size: 1000, validate: true}.merge(options)
34
26
  @headers = model.respond_to?(:csv_headers) ? model.csv_headers : []
35
- @result= {failed: [], imported: 0}
36
- if @options.has_key?(:col_sep) || @options.has_key?(:row_sep)
37
- ActiveSupport::Deprecation.warn "row_sep and col_sep options are deprecated, use csv_options to override default CSV options"
38
- @csv_options = @options.slice(:col_sep, :row_sep)
39
- else
40
- @csv_options = @options[:csv_options] || {}
41
- end
42
- #override csv options from model if it respond_to csv_options
43
- @csv_options = model.csv_options if model.respond_to?(:csv_options)
44
- @csv_options.reject! {| key, value | value.blank? }
27
+ assign_options(options)
28
+ end
45
29
 
30
+ def import_result
31
+ @import_result ||= ImportResult.new
46
32
  end
47
33
 
48
34
  def file
@@ -51,28 +37,89 @@ module ActiveAdminImport
51
37
 
52
38
  def cycle(lines)
53
39
  @csv_lines = CSV.parse(lines.join, @csv_options)
54
- @result.merge!(self.store) { |key, val1, val2| val1+val2 }
40
+ import_result.add(batch_import, lines.count)
55
41
  end
56
42
 
57
43
  def import
58
- options[:before_import].call(self) if options[:before_import].is_a?(Proc)
59
- lines = []
60
- batch_size = options[:batch_size].to_i
44
+ run_callback(:before_import)
45
+ process_file
46
+ run_callback(:after_import)
47
+ import_result
48
+ end
49
+
50
+ def import_options
51
+ @import_options ||= options.slice(:validate, :on_duplicate_key_update, :ignore, :timestamps, :batch_transaction)
52
+ end
53
+
54
+ def batch_replace(header_key, options)
55
+ index = header_index(header_key)
56
+ csv_lines.map! do |line|
57
+ from = line[index]
58
+ line[index] = options[from] if options.has_key?(from)
59
+ line
60
+ end
61
+ end
62
+
63
+ def values_at(header_key)
64
+ csv_lines.collect { |line| line[header_index(header_key)] }.uniq
65
+ end
66
+
67
+ def header_index(header_key)
68
+ headers.values.index(header_key)
69
+ end
70
+
71
+ protected
72
+
73
+ def process_file
74
+ lines, batch_size = [], options[:batch_size].to_i
61
75
  File.open(file.path) do |f|
62
76
  # capture headers if not exist
63
- prepare_headers(headers.any? ? headers : CSV.parse(f.readline, @csv_options).first)
77
+ prepare_headers { CSV.parse(f.readline, @csv_options).first }
64
78
  f.each_line do |line|
65
- next if line.blank?
66
- lines << line
79
+ lines << line if line.present?
67
80
  if lines.size == batch_size || f.eof?
68
- cycle lines
81
+ cycle(lines)
69
82
  lines = []
70
83
  end
71
84
  end
72
85
  end
73
86
  cycle(lines) unless lines.blank?
74
- options[:after_import].call(self) if options[:after_import].is_a?(Proc)
75
- result
76
87
  end
88
+
89
+ def prepare_headers
90
+ headers = self.headers.present? ? self.headers.map(&:to_s) : yield
91
+ @headers = Hash[headers.zip(headers.map { |el| el.underscore.gsub(/\s+/, '_') })].with_indifferent_access
92
+ @headers.merge!(options[:headers_rewrites].symbolize_keys.slice(*@headers.symbolize_keys.keys))
93
+ @headers
94
+ end
95
+
96
+ def run_callback(name)
97
+ options[name].call(self) if options[name].is_a?(Proc)
98
+ end
99
+
100
+ def batch_import
101
+ batch_result = nil
102
+ @resource.transaction do
103
+ run_callback(:before_batch_import)
104
+ batch_result = resource.import(headers.values, csv_lines, import_options)
105
+ raise ActiveRecord::Rollback if import_options[:batch_transaction] && batch_result.failed_instances.any?
106
+ run_callback(:after_batch_import)
107
+ end
108
+ batch_result
109
+ end
110
+
111
+ def assign_options(options)
112
+ @options = {batch_size: 1000, validate: true}.merge(options.slice(*OPTIONS))
113
+ detect_csv_options
114
+ end
115
+
116
+ def detect_csv_options
117
+ @csv_options = if model.respond_to?(:csv_options)
118
+ model.csv_options
119
+ else
120
+ options[:csv_options] || {}
121
+ end.reject { |_, value| value.blank? }
122
+ end
123
+
77
124
  end
78
125
  end
@@ -1,19 +1,23 @@
1
+ # encoding: utf-8
2
+
3
+ require 'rchardet'
4
+
1
5
  module ActiveAdminImport
2
6
  class Model
3
- extend ActiveModel::Naming
4
- include ActiveModel::Conversion
7
+
8
+ include ActiveModel::Model
5
9
  include ActiveModel::Validations
6
10
  include ActiveModel::Validations::Callbacks
7
11
 
8
- validates :file, presence: {message: Proc.new { I18n.t('active_admin_import.no_file_error') }},
9
- unless: proc { |me| me.new_record? }
10
-
11
- validate :correct_content_type, if: proc { |me| me.file.present? }
12
- validate :file_contents_present, if: proc { |me| me.file.present? }
12
+ validates :file, presence: {
13
+ message: ->(*_){ I18n.t("active_admin_import.no_file_error") }
14
+ }, unless: ->(me){ me.new_record? }
13
15
 
16
+ validate :correct_content_type, if: ->(me) { me.file.present? }
17
+ validate :file_contents_present, if: ->(me) { me.file.present? }
14
18
 
15
- before_validation :uncompress_file, if: proc { |me| me.archive? && me.allow_archive? }
16
- before_validation :encode_file, if: proc { |me| me.force_encoding? && me.file.present? }
19
+ before_validation :unzip_file, if: ->(me) { me.archive? && me.allow_archive? }
20
+ before_validation :encode_file, if: ->(me) { me.force_encoding? && me.file.present? }
17
21
 
18
22
  attr_reader :attributes
19
23
 
@@ -27,12 +31,7 @@ module ActiveAdminImport
27
31
  @attributes.merge!(args)
28
32
  @new_record = new_record
29
33
  args.keys.each do |key|
30
- key = key.to_sym
31
- #generate methods for instance object by attributes
32
- singleton_class.class_eval do
33
- define_method(key) { self.attributes[key] } unless method_defined? key
34
- define_method("#{key}=") { |new_value| @attributes[key] = new_value } unless method_defined? "#{key}="
35
- end
34
+ define_methods_for(key.to_sym)
36
35
  end if args.is_a?(Hash)
37
36
  end
38
37
 
@@ -41,11 +40,17 @@ module ActiveAdminImport
41
40
  end
42
41
 
43
42
  def default_attributes
44
- {hint: '', file: nil, csv_headers: [], allow_archive: true, force_encoding: 'UTF-8'}
43
+ {
44
+ allow_archive: true,
45
+ csv_headers: [],
46
+ file: nil,
47
+ force_encoding: "UTF-8",
48
+ hint: ""
49
+ }
45
50
  end
46
51
 
47
52
  def allow_archive?
48
- !!@attributes[:allow_archive]
53
+ !!attributes[:allow_archive]
49
54
  end
50
55
 
51
56
  def new_record?
@@ -53,11 +58,7 @@ module ActiveAdminImport
53
58
  end
54
59
 
55
60
  def force_encoding?
56
- !!@attributes[:force_encoding]
57
- end
58
-
59
- def to_hash
60
- @attributes
61
+ !!attributes[:force_encoding]
61
62
  end
62
63
 
63
64
  def persisted?
@@ -68,6 +69,8 @@ module ActiveAdminImport
68
69
  file_type == 'application/zip'
69
70
  end
70
71
 
72
+ alias :to_hash :attributes
73
+
71
74
  protected
72
75
 
73
76
  def file_path
@@ -79,23 +82,21 @@ module ActiveAdminImport
79
82
  end
80
83
 
81
84
  def encode_file
82
- data = File.read(file_path).encode(force_encoding, invalid: :replace, undef: :replace)
85
+ data = File.read(file_path)
83
86
  File.open(file_path, 'w') do |f|
84
- f.write(data)
87
+ f.write(encode(data))
85
88
  end
86
89
  end
87
90
 
88
- def uncompress_file
91
+ def unzip_file
89
92
  Zip::File.open(file_path) do |zip_file|
90
- self.file = Tempfile.new("active-admin-import-unzipped")
93
+ self.file = Tempfile.new('active-admin-import-unzipped')
91
94
  data = zip_file.entries.select { |f| f.file? }.first.get_input_stream.read
92
- data = data.encode(force_encoding, invalid: :replace, undef: :replace) if self.force_encoding?
93
95
  self.file << data
94
96
  self.file.close
95
97
  end
96
98
  end
97
99
 
98
-
99
100
  def csv_allowed_types
100
101
  [
101
102
  'text/csv',
@@ -107,7 +108,6 @@ module ActiveAdminImport
107
108
  ]
108
109
  end
109
110
 
110
-
111
111
  def correct_content_type
112
112
  unless file.blank? || file.is_a?(Tempfile)
113
113
  errors.add(:file, I18n.t('active_admin_import.file_format_error')) unless csv_allowed_types.include? file_type
@@ -125,6 +125,57 @@ module ActiveAdminImport
125
125
  ''
126
126
  end
127
127
  end
128
+
129
+ protected
130
+
131
+ def define_methods_for(attr_name)
132
+ #generate methods for instance object by attributes
133
+ singleton_class.class_eval do
134
+ define_set_method(attr_name)
135
+ define_get_method(attr_name)
136
+ end
137
+ end
138
+
139
+ def encode(data)
140
+ data = content_encode(data) if force_encoding?
141
+ data = data.encode(
142
+ 'UTF-8',
143
+ invalid: :replace, undef: :replace, universal_newline: true
144
+ )
145
+ begin
146
+ data.sub("\xEF\xBB\xBF", '') # bom
147
+ rescue StandardError => _
148
+ data
149
+ end
150
+ end
151
+
152
+ def detect_encoding?
153
+ force_encoding == :auto
154
+ end
155
+
156
+ def dynamic_encoding(data)
157
+ CharDet.detect(data)['encoding']
158
+ end
159
+
160
+ def content_encode(data)
161
+ encoding_name = if detect_encoding?
162
+ dynamic_encoding(data)
163
+ else
164
+ force_encoding.to_s
165
+ end
166
+ data.force_encoding(encoding_name)
167
+ end
168
+
169
+ class <<self
170
+ def define_set_method(attr_name)
171
+ define_method(attr_name) { self.attributes[attr_name] } unless method_defined? attr_name
172
+ end
173
+
174
+ def define_get_method(attr_name)
175
+ define_method("#{attr_name}=") { |new_value| @attributes[attr_name] = new_value } unless method_defined? "#{attr_name}="
176
+ end
177
+ end
178
+
128
179
  end
129
180
  end
130
181
 
@@ -0,0 +1,44 @@
1
+ module ActiveAdminImport
2
+
3
+ module Options
4
+ VALID_OPTIONS = [
5
+ :back,
6
+ :csv_options,
7
+ :validate,
8
+ :batch_size,
9
+ :batch_transaction,
10
+ :before_import,
11
+ :after_import,
12
+ :before_batch_import,
13
+ :after_batch_import,
14
+ :on_duplicate_key_update,
15
+ :timestamps,
16
+ :ignore,
17
+ :template,
18
+ :template_object,
19
+ :resource_class,
20
+ :resource_label,
21
+ :plural_resource_label,
22
+ :headers_rewrites
23
+ ].freeze
24
+
25
+
26
+ def self.options_for(config, options= {})
27
+ options[:template_object] = ActiveAdminImport::Model.new unless options.has_key? :template_object
28
+
29
+ {
30
+ back: {action: :import},
31
+ csv_options: {},
32
+ template: "admin/import",
33
+ resource_class: config.resource_class,
34
+ resource_label: config.resource_label,
35
+ plural_resource_label: config.plural_resource_label,
36
+ headers_rewrites: {}
37
+ }.deep_merge(options)
38
+
39
+
40
+
41
+ end
42
+
43
+ end
44
+ end
@@ -1,3 +1,3 @@
1
1
  module ActiveAdminImport
2
- VERSION = "2.1.2"
2
+ VERSION = "3.0.0.pre"
3
3
  end
@@ -2,9 +2,12 @@ require 'activerecord-import'
2
2
  require 'active_admin'
3
3
  require 'active_admin_import/version'
4
4
  require 'active_admin_import/engine'
5
+ require 'active_admin_import/import_result'
6
+ require 'active_admin_import/options'
5
7
  require 'active_admin_import/dsl'
6
8
  require 'active_admin_import/importer'
7
9
  require 'active_admin_import/model'
8
10
  require 'active_admin_import/authorization'
9
11
  ::ActiveAdmin::DSL.send(:include, ActiveAdminImport::DSL)
10
12
 
13
+
@@ -0,0 +1,2 @@
1
+ Name,Last name,Birthday
2
+ John,Doe,1986-05-01
@@ -0,0 +1,2 @@
1
+ Name,Second name,Birthday
2
+ John,Doe,1986-05-01
@@ -0,0 +1,2 @@
1
+ Name,Last name,Birthday
2
+ ,Doe,1986-05-01
@@ -0,0 +1,3 @@
1
+ Name,Last name,Birthday
2
+ John,Doe,1986-05-01
3
+ Jane,Roe,1988-11-16
@@ -0,0 +1,3 @@
1
+ Name,Last name,Birthday
2
+ John,Doe,1986-05-01
3
+ Jane,Roe,1988-11-16
@@ -0,0 +1,3 @@
1
+ Name,Last name,Birthday
2
+ John,Doe,1986-05-01
3
+ John,Doe,1986-05-01
@@ -0,0 +1,3 @@
1
+ Name,Last name,Birthday
2
+ Jim,Doe,1986-05-01
3
+ Jane,Roe,1986-05-01
@@ -0,0 +1,2 @@
1
+ John,Doe,1986-05-01
2
+ Jane,Roe,1988-11-16
@@ -0,0 +1,3 @@
1
+ Name,Last name,Birthday
2
+ �������,���������,1986-05-01
3
+ Jane,Roe,1988-11-16
@@ -0,0 +1,3 @@
1
+ Id,Name,Last name,Birthday
2
+ 1,John,Doe,1986-05-01
3
+ 2,Jane,Roe,1988-11-16
@@ -0,0 +1,3 @@
1
+ Name;Last name;Birthday
2
+ John;Doe;1986-05-01
3
+ Jane;Roe;1988-11-16
File without changes
@@ -0,0 +1 @@
1
+ Name,Last name,Birthday
@@ -0,0 +1,4 @@
1
+ "Title","Body","Author Name"
2
+ "title1","some body","John"
3
+ "title2","some body","Jane"
4
+
@@ -0,0 +1,3 @@
1
+ "Title","Body"
2
+ "title1","some body"
3
+ "title2","some body"
@@ -0,0 +1,2 @@
1
+ "title1","some body"
2
+ "title2","some body"
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+
3
+ describe ActiveAdminImport::ImportResult do
4
+ context "failed_message" do
5
+ let(:import_result) { ActiveAdminImport::ImportResult.new }
6
+
7
+ before do
8
+ Author.create(name: 'John', last_name: 'Doe')
9
+ Author.create(name: 'Jane', last_name: 'Roe')
10
+
11
+ @result = double \
12
+ failed_instances: [
13
+ Author.create(name: 'Jim', last_name: "Doe"), # {:last_name=>["has already been taken"]}
14
+ Author.create(name: nil, last_name: 'Doe') # {:name=>["can't be blank"], :last_name=>["has already been taken"]}
15
+ ]
16
+ end
17
+
18
+ it "should work without any failed instances" do
19
+ expect(import_result.failed_message).to eq("")
20
+ end
21
+
22
+ it "should work" do
23
+ import_result.add(@result, 4)
24
+ expect(import_result.failed_message).to eq("Last name has already been taken - Doe ; Name can't be blank - , Last name has already been taken - Doe")
25
+ end
26
+
27
+ it "should work on limit param" do
28
+ import_result.add(@result, 4)
29
+ expect(import_result.failed_message(limit: 1)).to eq("Last name has already been taken - Doe")
30
+ end
31
+ end
32
+ end