active_record_importer 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -2
- data/active_record_importer.gemspec +1 -1
- data/lib/active_record_importer/batch_importer.rb +8 -15
- data/lib/active_record_importer/data_processor.rb +1 -1
- data/lib/active_record_importer/dispatcher.rb +32 -2
- data/lib/active_record_importer/failed_file_builder.rb +12 -55
- data/lib/active_record_importer/importable.rb +8 -2
- data/lib/active_record_importer/options_builder.rb +1 -1
- data/lib/active_record_importer/version.rb +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 71fdf968e3955c0ad63d10e6bab1f97ca4554aea
|
4
|
+
data.tar.gz: 891b854bfd76d3ea41c2596a03af4a4494e2c470
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bbd93f935866c4ba427e942ca90fe0d2012df09cf7d064ad44642cfb981cb22f9c03cc541a0a16b71a0c0d64ecef3705e61c056e34e501a963689ce07e25a123
|
7
|
+
data.tar.gz: 8235e2a1afb32b3d4cf689c08afbeb478b88034dcd14a90cff9ecd437451493b40a38f6c672ad266a141f6f922d55b78377ddf0374a1a486e31df14587c676e0
|
data/README.md
CHANGED
@@ -25,9 +25,10 @@ Or install it yourself as:
|
|
25
25
|
$ gem install active_record_importer
|
26
26
|
|
27
27
|
## Usage
|
28
|
-
### For version 0.
|
28
|
+
### For version 0.4.0
|
29
29
|
|
30
|
-
For the newest version (0.
|
30
|
+
For the newest version (0.4.0), you don't have to create Import table/model and controller.
|
31
|
+
I already fixed the errors when there's no Import table/model on version 0.3.0.
|
31
32
|
You just need to add the `acts_as_importable` in your model you want to be importable, and you may now run:
|
32
33
|
|
33
34
|
```ruby
|
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.email = ['kapitan_03@yahoo.com']
|
11
11
|
|
12
12
|
spec.summary = 'Active Record Importer'
|
13
|
-
spec.description = 'Smart gem for importing
|
13
|
+
spec.description = 'Smart gem for importing CSV files to ActiveRecord models'
|
14
14
|
spec.homepage = 'https://github.com/michaelnera/active_record_importer'
|
15
15
|
spec.license = 'MIT'
|
16
16
|
|
@@ -2,10 +2,9 @@ module ActiveRecordImporter
|
|
2
2
|
class BatchImporter
|
3
3
|
include Virtus.model
|
4
4
|
|
5
|
-
attribute :import, Import
|
6
5
|
attribute :importable
|
6
|
+
attribute :import
|
7
7
|
attribute :data, Array, default: []
|
8
|
-
attribute :failed_file, FailedFileBuilder, default: :initialize_failed_file
|
9
8
|
|
10
9
|
def process!
|
11
10
|
@imported_count, @failed_count = 0, 0
|
@@ -16,16 +15,10 @@ module ActiveRecordImporter
|
|
16
15
|
end
|
17
16
|
|
18
17
|
set_import_count
|
19
|
-
finalize_batch_import
|
20
18
|
end
|
21
19
|
|
22
20
|
private
|
23
21
|
|
24
|
-
def initialize_failed_file
|
25
|
-
return unless import
|
26
|
-
FailedFileBuilder.new(import)
|
27
|
-
end
|
28
|
-
|
29
22
|
def process_row(row_attrs)
|
30
23
|
processor =
|
31
24
|
DataProcessor.new(
|
@@ -35,7 +28,7 @@ module ActiveRecordImporter
|
|
35
28
|
)
|
36
29
|
return @imported_count += 1 if processor.process
|
37
30
|
|
38
|
-
|
31
|
+
write_failed_row(row_attrs, processor.row_errors)
|
39
32
|
@failed_count += 1
|
40
33
|
end
|
41
34
|
|
@@ -46,14 +39,14 @@ module ActiveRecordImporter
|
|
46
39
|
Import.update_counters(import.id, failed_rows: @failed_count)
|
47
40
|
end
|
48
41
|
|
49
|
-
def
|
50
|
-
return
|
51
|
-
@failed_file
|
42
|
+
def failed_file
|
43
|
+
return unless import.present? || import.respond_to?(:failed_file)
|
44
|
+
@failed_file ||= FailedFileBuilder.new(import)
|
52
45
|
end
|
53
46
|
|
54
|
-
def
|
55
|
-
return unless failed_file
|
56
|
-
|
47
|
+
def write_failed_row(row_attrs, errors)
|
48
|
+
return puts errors.inspect unless failed_file
|
49
|
+
failed_file.write(row_attrs.merge(import_errors: errors))
|
57
50
|
end
|
58
51
|
|
59
52
|
delegate :importer_options, to: :importable
|
@@ -2,8 +2,8 @@ module ActiveRecordImporter
|
|
2
2
|
class DataProcessor
|
3
3
|
include Virtus.model
|
4
4
|
|
5
|
-
attribute :import, Import
|
6
5
|
attribute :importable, Class
|
6
|
+
attribute :import
|
7
7
|
attribute :insert_method, String, default: :set_insert_method
|
8
8
|
attribute :row_attrs, Hash
|
9
9
|
attribute :instance_attrs, Hash
|
@@ -2,13 +2,14 @@ module ActiveRecordImporter
|
|
2
2
|
class Dispatcher
|
3
3
|
include Virtus.model
|
4
4
|
|
5
|
-
attribute :import, Import
|
6
5
|
attribute :importable, Class
|
7
6
|
attribute :execute, Boolean, default: true
|
7
|
+
attribute :import
|
8
8
|
attribute :import_file
|
9
9
|
|
10
10
|
def call
|
11
11
|
divide_and_conquer
|
12
|
+
create_import_failed_file
|
12
13
|
end
|
13
14
|
|
14
15
|
private
|
@@ -19,6 +20,7 @@ module ActiveRecordImporter
|
|
19
20
|
queue_or_execute(collection)
|
20
21
|
end
|
21
22
|
end
|
23
|
+
|
22
24
|
true
|
23
25
|
end
|
24
26
|
|
@@ -45,8 +47,36 @@ module ActiveRecordImporter
|
|
45
47
|
).process!
|
46
48
|
end
|
47
49
|
|
48
|
-
def queue(
|
50
|
+
def queue(_collection)
|
49
51
|
# To follow
|
50
52
|
end
|
53
|
+
|
54
|
+
def create_import_failed_file
|
55
|
+
return if import.nil? || !File.exists?(temp_failed_file_path)
|
56
|
+
File.open(temp_failed_file_path) do |file|
|
57
|
+
import.failed_file = file
|
58
|
+
|
59
|
+
# I forced to save it as 'text/csv' because
|
60
|
+
# the file is being saved as 'text/x-pascal'
|
61
|
+
# and I still have no idea why?!?
|
62
|
+
|
63
|
+
import.failed_file_content_type = 'text/csv'
|
64
|
+
import.save!
|
65
|
+
end
|
66
|
+
|
67
|
+
destroy_temp_failed_file
|
68
|
+
end
|
69
|
+
|
70
|
+
def destroy_temp_failed_file
|
71
|
+
FileUtils.rm(temp_failed_file_path)
|
72
|
+
end
|
73
|
+
|
74
|
+
def temp_failed_file_path
|
75
|
+
"/tmp/#{target_file_name}"
|
76
|
+
end
|
77
|
+
|
78
|
+
def target_file_name
|
79
|
+
"failed_file_#{import.id}.csv"
|
80
|
+
end
|
51
81
|
end
|
52
82
|
end
|
@@ -5,74 +5,31 @@ module ActiveRecordImporter
|
|
5
5
|
|
6
6
|
def initialize(import)
|
7
7
|
@import = import
|
8
|
-
@failed_rows = []
|
9
8
|
end
|
10
|
-
|
11
|
-
def build
|
12
|
-
return if failed_rows.blank?
|
13
9
|
|
14
|
-
|
15
|
-
|
16
|
-
destroy_temp_file
|
17
|
-
end
|
18
|
-
|
19
|
-
private
|
10
|
+
def write(failed_row = {})
|
11
|
+
return if failed_row.blank?
|
20
12
|
|
21
|
-
|
22
|
-
|
23
|
-
|
13
|
+
if File.exists?(temp_failed_file_path)
|
14
|
+
File.open(temp_failed_file_path, 'a') do |file|
|
15
|
+
file.write failed_row.values.to_csv
|
16
|
+
end
|
24
17
|
else
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
def write_csv_file
|
30
|
-
CSV.open(temp_file_path, 'wb') do |csv|
|
31
|
-
csv << failed_rows.first.keys
|
32
|
-
insert_failed_rows(csv)
|
18
|
+
File.open(temp_failed_file_path, 'w') do |file|
|
19
|
+
file.write failed_row.keys
|
20
|
+
file.write failed_row.values.to_csv
|
21
|
+
end
|
33
22
|
end
|
34
23
|
end
|
35
24
|
|
36
|
-
|
37
|
-
return if import.failed_file.blank?
|
38
|
-
CSV.open(import.failed_file_path, 'a+') do |csv|
|
39
|
-
insert_failed_rows(csv)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
|
44
|
-
def insert_failed_rows(csv)
|
45
|
-
failed_rows.each do |hash|
|
46
|
-
csv << hash.values
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def destroy_temp_file
|
51
|
-
return unless File.exists?(temp_file_path)
|
52
|
-
FileUtils.rm(temp_file_path)
|
53
|
-
end
|
25
|
+
private
|
54
26
|
|
55
|
-
def
|
27
|
+
def temp_failed_file_path
|
56
28
|
"/tmp/#{target_file_name}"
|
57
29
|
end
|
58
30
|
|
59
31
|
def target_file_name
|
60
32
|
"failed_file_#{import.id}.csv"
|
61
33
|
end
|
62
|
-
|
63
|
-
def create_import_failed_file
|
64
|
-
return if import.failed_file.present?
|
65
|
-
File.open(temp_file_path) do |file|
|
66
|
-
import.failed_file = file
|
67
|
-
|
68
|
-
# I forced to save it as 'text/csv' because
|
69
|
-
# the file is being saved as 'text/x-pascal'
|
70
|
-
# and I still have no idea why?!?
|
71
|
-
|
72
|
-
import.failed_file_content_type = 'text/csv'
|
73
|
-
import.save!
|
74
|
-
end
|
75
|
-
|
76
|
-
end
|
77
34
|
end
|
78
35
|
end
|
@@ -22,9 +22,9 @@ module ActiveRecordImporter
|
|
22
22
|
##
|
23
23
|
def acts_as_importable(options = {})
|
24
24
|
@@importer_options = OptionsBuilder.build(options.merge(allowed_columns_hash(options)))
|
25
|
-
::IMPORTABLES << self.name unless ::IMPORTABLES.include?(self.name)
|
26
25
|
|
27
26
|
include ActiveRecordImporter::Importable::InstanceMethods
|
27
|
+
extend ActiveRecordImporter::Importable::SingletonMethods
|
28
28
|
end
|
29
29
|
|
30
30
|
def importer_options
|
@@ -32,7 +32,7 @@ module ActiveRecordImporter
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def importable?
|
35
|
-
|
35
|
+
false
|
36
36
|
end
|
37
37
|
|
38
38
|
##
|
@@ -121,5 +121,11 @@ module ActiveRecordImporter
|
|
121
121
|
@importing ||= false
|
122
122
|
end
|
123
123
|
end
|
124
|
+
|
125
|
+
module SingletonMethods
|
126
|
+
def importable?
|
127
|
+
true
|
128
|
+
end
|
129
|
+
end
|
124
130
|
end
|
125
131
|
end
|
@@ -20,7 +20,7 @@ module ActiveRecordImporter
|
|
20
20
|
attribute :remove_empty_values, Boolean, default: false
|
21
21
|
attribute :comment_regexp, Regexp, default: Regexp.new(/^#=>/)
|
22
22
|
attribute :force_utf8, Boolean, default: true
|
23
|
-
attribute :chunk_size, Integer, default:
|
23
|
+
attribute :chunk_size, Integer, default: nil
|
24
24
|
attribute :col_sep, String, default: ','
|
25
25
|
end
|
26
26
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_record_importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Nera
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -132,7 +132,7 @@ dependencies:
|
|
132
132
|
- - "~>"
|
133
133
|
- !ruby/object:Gem::Version
|
134
134
|
version: '4.0'
|
135
|
-
description: Smart gem for importing
|
135
|
+
description: Smart gem for importing CSV files to ActiveRecord models
|
136
136
|
email:
|
137
137
|
- kapitan_03@yahoo.com
|
138
138
|
executables: []
|
@@ -191,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
191
191
|
version: '0'
|
192
192
|
requirements: []
|
193
193
|
rubyforge_project:
|
194
|
-
rubygems_version: 2.
|
194
|
+
rubygems_version: 2.2.2
|
195
195
|
signing_key:
|
196
196
|
specification_version: 4
|
197
197
|
summary: Active Record Importer
|