active_record_importer 0.2.1 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +41 -2
- data/lib/active_record_importer/attribute/find_options_builder.rb +1 -11
- data/lib/active_record_importer/batch_importer.rb +30 -17
- data/lib/active_record_importer/data_processor.rb +38 -24
- data/lib/active_record_importer/dispatcher.rb +16 -12
- data/lib/active_record_importer/errors.rb +6 -0
- data/lib/active_record_importer/importable.rb +52 -3
- data/lib/active_record_importer/instance_builder.rb +15 -16
- data/lib/active_record_importer/options_builder.rb +2 -1
- data/lib/active_record_importer/version.rb +1 -1
- data/lib/app/models/import.rb +2 -2
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0b0c2019e863558605d2703fb80890c8ba209192
|
4
|
+
data.tar.gz: 480be20d7fc0440427e4107f7b50e7e19572ad81
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f9191b603ceb3e77bff828ac70ad0a9301b854a1956cc1ef218e00177c2f8d5b20b0722e98bcc99cb9f99663126e29e53d3b57eebe60d14c88eb248008092441
|
7
|
+
data.tar.gz: 4be44dcf15fcd9a9981ff0942c63a1a1e6d21691cf27832cb3d3755a78636f4fe79193fbe26eb5bb4ccc437cbd0e6c0ab5583d73e37f25e67e78b1a8d7d53265
|
data/README.md
CHANGED
@@ -25,8 +25,32 @@ Or install it yourself as:
|
|
25
25
|
$ gem install active_record_importer
|
26
26
|
|
27
27
|
## Usage
|
28
|
+
### For version 0.3.0
|
28
29
|
|
29
|
-
|
30
|
+
For the newest version (0.3.0), you don't have to create Import table/model and controller.
|
31
|
+
You just need to add the `acts_as_importable` in your model you want to be importable, and you may now run:
|
32
|
+
|
33
|
+
```ruby
|
34
|
+
User.import!(file: File.open(PATH_TO_FILE))
|
35
|
+
```
|
36
|
+
|
37
|
+
`insert` will be the default insert method for this
|
38
|
+
If you want to use `upsert` or `error_duplicate`, define it in your importer options:
|
39
|
+
|
40
|
+
```ruby
|
41
|
+
class User < ActiveRecord::Base
|
42
|
+
acts_as_importable insert_method: 'upsert',
|
43
|
+
find_options: [:email]
|
44
|
+
end
|
45
|
+
```
|
46
|
+
|
47
|
+
Or you may use in your console:
|
48
|
+
|
49
|
+
```ruby
|
50
|
+
User.acts_as_importable insert_method: 'error_duplicate', find_options: ['email']
|
51
|
+
```
|
52
|
+
|
53
|
+
### If you don't want to record the status of your import, you don't have to do the remaining steps
|
30
54
|
|
31
55
|
### Create Import table/model
|
32
56
|
I'll add a generator on my next release
|
@@ -76,15 +100,27 @@ class Import < ActiveRecord::Base
|
|
76
100
|
|
77
101
|
# I'll add import options in the next major release
|
78
102
|
# accepts_nested_attributes_for :import_options, allow_destroy: true
|
79
|
-
|
103
|
+
### THIS IS VERSION 0.2.1 and below
|
80
104
|
def execute
|
81
105
|
resource_class.import!(self, execute_on_create)
|
82
106
|
end
|
83
107
|
|
108
|
+
### THIS IS VERSION 0.2.1 and below
|
109
|
+
def execute
|
110
|
+
resource_class.import!(object: self, execute: execute_on_create)
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
### THIS IS VERSION 0.2.1 and below
|
84
115
|
def execute!
|
85
116
|
resource_class.import!(self, true)
|
86
117
|
end
|
87
118
|
|
119
|
+
### THIS IS VERSION 0.3.0
|
120
|
+
def execute!
|
121
|
+
resource_class.import!(object: self, execute: true)
|
122
|
+
end
|
123
|
+
|
88
124
|
def resource_class
|
89
125
|
resource.safe_constantize
|
90
126
|
end
|
@@ -226,6 +262,9 @@ end
|
|
226
262
|
@import.execute!
|
227
263
|
```
|
228
264
|
|
265
|
+
###REMINDER:
|
266
|
+
Headers of your csv file should be formatted/transformed to column names of your IMPORTABLE model
|
267
|
+
|
229
268
|
|
230
269
|
## Development
|
231
270
|
|
@@ -4,7 +4,7 @@ module ActiveRecordImporter
|
|
4
4
|
include Virtus.model
|
5
5
|
include Helpers
|
6
6
|
|
7
|
-
attribute :
|
7
|
+
attribute :importable
|
8
8
|
attribute :attrs, Hash, default: {}
|
9
9
|
attribute :find_options, String
|
10
10
|
attribute :prefix, String
|
@@ -16,18 +16,8 @@ module ActiveRecordImporter
|
|
16
16
|
|
17
17
|
private
|
18
18
|
|
19
|
-
def klass
|
20
|
-
resource.safe_constantize
|
21
|
-
end
|
22
|
-
|
23
|
-
delegate :importer_options, to: :klass
|
24
|
-
|
25
|
-
delegate :required_attributes, to: :importer_options
|
26
|
-
|
27
19
|
def get_find_opts
|
28
20
|
@options = strip_and_symbolize
|
29
|
-
@options ||= importer_options.find_options || required_attributes
|
30
|
-
@options
|
31
21
|
end
|
32
22
|
|
33
23
|
def slice_attributes
|
@@ -1,19 +1,18 @@
|
|
1
1
|
module ActiveRecordImporter
|
2
2
|
class BatchImporter
|
3
|
+
include Virtus.model
|
3
4
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@data = data
|
9
|
-
@failed_file = FailedFileBuilder.new(import)
|
10
|
-
end
|
5
|
+
attribute :import, Import
|
6
|
+
attribute :importable
|
7
|
+
attribute :data, Array, default: []
|
8
|
+
attribute :failed_file, FailedFileBuilder, default: :initialize_failed_file
|
11
9
|
|
12
10
|
def process!
|
13
11
|
@imported_count, @failed_count = 0, 0
|
14
|
-
|
15
|
-
|
16
|
-
|
12
|
+
|
13
|
+
data.each do |row_attrs|
|
14
|
+
next if row_attrs.blank?
|
15
|
+
process_row(row_attrs.symbolize_keys!)
|
17
16
|
end
|
18
17
|
|
19
18
|
set_import_count
|
@@ -22,25 +21,39 @@ module ActiveRecordImporter
|
|
22
21
|
|
23
22
|
private
|
24
23
|
|
25
|
-
def
|
26
|
-
|
24
|
+
def initialize_failed_file
|
25
|
+
return unless import
|
26
|
+
FailedFileBuilder.new(import)
|
27
|
+
end
|
28
|
+
|
29
|
+
def process_row(row_attrs)
|
30
|
+
processor =
|
31
|
+
DataProcessor.new(
|
32
|
+
import: import,
|
33
|
+
importable: importable,
|
34
|
+
row_attrs: row_attrs
|
35
|
+
)
|
27
36
|
return @imported_count += 1 if processor.process
|
28
37
|
|
29
|
-
|
38
|
+
collect_failed_rows(row_attrs, processor.row_errors)
|
30
39
|
@failed_count += 1
|
31
40
|
end
|
32
41
|
|
33
42
|
def set_import_count
|
43
|
+
return unless import
|
44
|
+
|
34
45
|
Import.update_counters(import.id, imported_rows: @imported_count)
|
35
46
|
Import.update_counters(import.id, failed_rows: @failed_count)
|
36
47
|
end
|
37
48
|
|
38
|
-
def
|
39
|
-
|
49
|
+
def collect_failed_rows(row_attrs, errors)
|
50
|
+
return puts errors.inspect unless failed_file
|
51
|
+
@failed_file.failed_rows << row_attrs.merge(import_errors: errors)
|
40
52
|
end
|
41
53
|
|
42
|
-
def
|
43
|
-
|
54
|
+
def finalize_batch_import
|
55
|
+
return unless failed_file
|
56
|
+
@failed_file.build
|
44
57
|
end
|
45
58
|
|
46
59
|
delegate :importer_options, to: :importable
|
@@ -1,16 +1,16 @@
|
|
1
1
|
module ActiveRecordImporter
|
2
2
|
class DataProcessor
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
3
|
+
include Virtus.model
|
4
|
+
|
5
|
+
attribute :import, Import
|
6
|
+
attribute :importable, Class
|
7
|
+
attribute :insert_method, String, default: :set_insert_method
|
8
|
+
attribute :row_attrs, Hash
|
9
|
+
attribute :instance_attrs, Hash
|
10
|
+
attribute :find_options, String, default: :set_find_options
|
11
|
+
attribute :find_attributes, Hash
|
12
|
+
attribute :row_errors, Array
|
13
|
+
attribute :instance
|
14
14
|
|
15
15
|
def process
|
16
16
|
fetch_instance_attributes
|
@@ -24,10 +24,12 @@ module ActiveRecordImporter
|
|
24
24
|
ActiveRecord::Base.transaction do
|
25
25
|
begin
|
26
26
|
@instance =
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
27
|
+
InstanceBuilder.new(
|
28
|
+
importable: importable,
|
29
|
+
insert_method: insert_method,
|
30
|
+
find_attributes: find_attributes,
|
31
|
+
instance_attrs: attributes_without_state_machine_attrs
|
32
|
+
).build
|
31
33
|
|
32
34
|
methods_after_upsert
|
33
35
|
true
|
@@ -38,18 +40,18 @@ module ActiveRecordImporter
|
|
38
40
|
end
|
39
41
|
|
40
42
|
def fetch_instance_attributes
|
41
|
-
@
|
42
|
-
|
43
|
-
|
43
|
+
@instance_attrs = Attribute::AttributesBuilder.new(
|
44
|
+
importable, row_attrs
|
45
|
+
).build
|
44
46
|
rescue => exception
|
45
47
|
append_errors(exception)
|
46
48
|
end
|
47
49
|
|
48
50
|
def fetch_find_attributes
|
49
51
|
@find_attributes = Attribute::FindOptionsBuilder.new(
|
50
|
-
|
51
|
-
find_options:
|
52
|
-
attrs:
|
52
|
+
importable: importable,
|
53
|
+
find_options: find_options,
|
54
|
+
attrs: instance_attrs
|
53
55
|
).build
|
54
56
|
rescue => exception
|
55
57
|
append_errors(exception)
|
@@ -62,8 +64,10 @@ module ActiveRecordImporter
|
|
62
64
|
run_after_save_callbacks
|
63
65
|
end
|
64
66
|
|
65
|
-
delegate :
|
66
|
-
:
|
67
|
+
delegate :importer_options,
|
68
|
+
to: :importable
|
69
|
+
|
70
|
+
delegate :after_save, :state_machine_attr,
|
67
71
|
to: :importer_options
|
68
72
|
|
69
73
|
def state_transitions
|
@@ -77,7 +81,7 @@ module ActiveRecordImporter
|
|
77
81
|
end
|
78
82
|
|
79
83
|
def attributes_without_state_machine_attrs
|
80
|
-
|
84
|
+
instance_attrs.except(*state_machine_attr)
|
81
85
|
end
|
82
86
|
|
83
87
|
def skip_callbacks?
|
@@ -97,5 +101,15 @@ module ActiveRecordImporter
|
|
97
101
|
@row_errors = message
|
98
102
|
fail ActiveRecord::Rollback if rollback
|
99
103
|
end
|
104
|
+
|
105
|
+
def set_insert_method
|
106
|
+
@insert_method = import.try(:insert_method)
|
107
|
+
@insert_method ||= importer_options.insert_method
|
108
|
+
@insert_method ||= 'insert'
|
109
|
+
end
|
110
|
+
|
111
|
+
def set_find_options
|
112
|
+
import.try(:find_options) || importer_options.find_options.join(',')
|
113
|
+
end
|
100
114
|
end
|
101
115
|
end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
module ActiveRecordImporter
|
2
2
|
class Dispatcher
|
3
|
-
|
3
|
+
include Virtus.model
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
attribute :import, Import
|
6
|
+
attribute :importable, Class
|
7
|
+
attribute :execute, Boolean, default: true
|
8
|
+
attribute :import_file
|
9
9
|
|
10
10
|
def call
|
11
11
|
divide_and_conquer
|
@@ -14,22 +14,22 @@ module ActiveRecordImporter
|
|
14
14
|
private
|
15
15
|
|
16
16
|
def divide_and_conquer
|
17
|
-
File.open(
|
17
|
+
File.open(import_file, 'r:bom|utf-8') do |file|
|
18
18
|
SmarterCSV.process(file, csv_options) do |collection|
|
19
19
|
queue_or_execute(collection)
|
20
20
|
end
|
21
21
|
end
|
22
|
+
true
|
22
23
|
end
|
23
24
|
|
24
25
|
def csv_options
|
25
|
-
|
26
|
-
opts
|
27
|
-
return opts if import.batch_size.blank? || import.batch_size < 1
|
26
|
+
opts = klass_csv_opts
|
27
|
+
return opts if import.nil? || import.batch_size.blank?
|
28
28
|
opts.merge(chunk_size: import.batch_size)
|
29
29
|
end
|
30
30
|
|
31
|
-
def klass_csv_opts
|
32
|
-
|
31
|
+
def klass_csv_opts
|
32
|
+
importable.importer_options.csv_opts.to_hash
|
33
33
|
end
|
34
34
|
|
35
35
|
def queue_or_execute(collection)
|
@@ -38,7 +38,11 @@ module ActiveRecordImporter
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def process_import(collection)
|
41
|
-
BatchImporter.new(
|
41
|
+
BatchImporter.new(
|
42
|
+
import: import,
|
43
|
+
importable: importable,
|
44
|
+
data: collection
|
45
|
+
).process!
|
42
46
|
end
|
43
47
|
|
44
48
|
def queue(collection)
|
@@ -6,6 +6,8 @@ module ActiveRecordImporter
|
|
6
6
|
|
7
7
|
module ClassMethods
|
8
8
|
##
|
9
|
+
# #acts_as_importable
|
10
|
+
#
|
9
11
|
# Make a model importable
|
10
12
|
# This will allow a model to use the importer
|
11
13
|
#
|
@@ -29,13 +31,60 @@ module ActiveRecordImporter
|
|
29
31
|
@@importer_options
|
30
32
|
end
|
31
33
|
|
32
|
-
def
|
33
|
-
|
34
|
-
|
34
|
+
def importable?
|
35
|
+
::IMPORTABLES.include?(self.name)
|
36
|
+
end
|
37
|
+
|
38
|
+
##
|
39
|
+
# #import!
|
40
|
+
#
|
41
|
+
# This method is called in the Import instance during execution of import
|
42
|
+
# You may also call this method without any import instance
|
43
|
+
# e.g.
|
44
|
+
#
|
45
|
+
# User.import!(file: File.open(PATH_TO_FILE))
|
46
|
+
#
|
47
|
+
# "insert" will be the default insert method for this
|
48
|
+
# If you want to use "upsert" or "error_duplicate",
|
49
|
+
# define it in your importer options:
|
50
|
+
#
|
51
|
+
# class User < ActiveRecord::Base
|
52
|
+
# acts_as_importable insert_method: 'upsert',
|
53
|
+
# find_options: ['email']
|
54
|
+
# end
|
55
|
+
#
|
56
|
+
# Or you may use:
|
57
|
+
# User.acts_as_importable insert_method: 'error_duplicate', find_options: ['email']
|
58
|
+
#
|
59
|
+
##
|
60
|
+
def import!(options = {})
|
61
|
+
fail "#{self.name} is not importable" unless importable?
|
62
|
+
|
63
|
+
import_object = options.fetch(:object, nil)
|
64
|
+
execute = options.fetch(:execute, true)
|
65
|
+
import_file = get_import_file(import_object, options)
|
66
|
+
|
67
|
+
call_dispatcher(import_object, execute, import_file)
|
35
68
|
end
|
36
69
|
|
37
70
|
private
|
38
71
|
|
72
|
+
def call_dispatcher(import_object = nil, execute = true, file = nil)
|
73
|
+
ActiveRecordImporter::Dispatcher.new(
|
74
|
+
importable: self,
|
75
|
+
import: import_object,
|
76
|
+
execute: execute,
|
77
|
+
import_file: file
|
78
|
+
).call
|
79
|
+
end
|
80
|
+
|
81
|
+
def get_import_file(import, options = {})
|
82
|
+
file = options.fetch(:file, nil) || import.try(:import_file)
|
83
|
+
fail Errors::MissingImportFile.new unless file
|
84
|
+
file
|
85
|
+
end
|
86
|
+
|
87
|
+
|
39
88
|
def allowed_columns_hash(options = {})
|
40
89
|
{
|
41
90
|
importable_columns: allowed_columns(options)
|
@@ -1,12 +1,11 @@
|
|
1
1
|
module ActiveRecordImporter
|
2
2
|
class InstanceBuilder
|
3
|
-
|
3
|
+
include Virtus.model
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
end
|
5
|
+
attribute :importable, ActiveRecord::Base
|
6
|
+
attribute :insert_method, String, default: 'insert'
|
7
|
+
attribute :instance_attrs, Hash, default: {}
|
8
|
+
attribute :find_attributes, Hash, default: {}
|
10
9
|
|
11
10
|
def build
|
12
11
|
instance = initialize_instance
|
@@ -15,20 +14,16 @@ module ActiveRecordImporter
|
|
15
14
|
|
16
15
|
private
|
17
16
|
|
18
|
-
delegate :
|
17
|
+
delegate :error_duplicate, :insert?, to: :insert_method_inquiry
|
19
18
|
|
20
19
|
def initialize_instance
|
21
|
-
return
|
20
|
+
return importable.new if insert?
|
22
21
|
|
23
22
|
fail Errors::MissingFindByOption if find_attributes.blank?
|
24
|
-
|
25
|
-
end
|
26
|
-
|
27
|
-
def klass
|
28
|
-
import.resource.safe_constantize
|
23
|
+
importable.find_or_initialize_by(find_attributes)
|
29
24
|
end
|
30
25
|
|
31
|
-
delegate :importer_options, to: :
|
26
|
+
delegate :importer_options, to: :importable
|
32
27
|
delegate :before_save, to: :importer_options
|
33
28
|
|
34
29
|
def process_data(instance)
|
@@ -44,13 +39,17 @@ module ActiveRecordImporter
|
|
44
39
|
end
|
45
40
|
|
46
41
|
def assign_attrs_and_save!(instance)
|
47
|
-
instance.attributes =
|
42
|
+
instance.attributes = instance_attrs
|
48
43
|
instance.save!
|
49
44
|
instance
|
50
45
|
end
|
51
46
|
|
47
|
+
def insert_method_inquiry
|
48
|
+
insert_method.inquiry
|
49
|
+
end
|
50
|
+
|
52
51
|
def error_duplicate?(instance)
|
53
|
-
instance.persisted? &&
|
52
|
+
instance.persisted? && error_duplicate?
|
54
53
|
end
|
55
54
|
end
|
56
55
|
end
|
@@ -31,9 +31,10 @@ module ActiveRecordImporter
|
|
31
31
|
class ImporterOptions
|
32
32
|
include Virtus.model
|
33
33
|
|
34
|
-
attribute :find_options, Array
|
34
|
+
attribute :find_options, Array, default: []
|
35
35
|
attribute :exclude_from_find_options, Array
|
36
36
|
attribute :scope, Symbol
|
37
|
+
attribute :insert_method, String
|
37
38
|
attribute :importable_columns, Array
|
38
39
|
attribute :default_attributes, Hash
|
39
40
|
attribute :csv_opts, CsvOptions, default: CsvOptions.new
|
data/lib/app/models/import.rb
CHANGED
@@ -28,11 +28,11 @@ module ActiveRecordImporter
|
|
28
28
|
# accepts_nested_attributes_for :import_options, allow_destroy: true
|
29
29
|
|
30
30
|
def execute
|
31
|
-
resource_class.import!(self, execute_on_create)
|
31
|
+
resource_class.import!(object: self, execute: execute_on_create)
|
32
32
|
end
|
33
33
|
|
34
34
|
def execute!
|
35
|
-
resource_class.import!(self, true)
|
35
|
+
resource_class.import!(object: self, execute: true)
|
36
36
|
end
|
37
37
|
|
38
38
|
def resource_class
|