importo 2.0.5 → 3.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +22 -5
- data/Rakefile +16 -17
- data/app/controllers/concerns/maintenance_standards.rb +1 -1
- data/app/controllers/importo/imports_controller.rb +15 -12
- data/app/helpers/importo/application_helper.rb +11 -11
- data/app/importers/concerns/exportable.rb +31 -24
- data/app/importers/concerns/importable.rb +144 -62
- data/app/importers/concerns/importer_dsl.rb +6 -14
- data/app/importers/concerns/original.rb +23 -23
- data/app/importers/concerns/result_feedback.rb +62 -44
- data/app/importers/concerns/revertable.rb +11 -12
- data/app/importers/importo/base_importer.rb +16 -4
- data/app/importers/importo/import_job_callback.rb +29 -0
- data/app/jobs/importo/import_job.rb +40 -0
- data/app/mailers/importo/application_mailer.rb +2 -2
- data/app/models/importo/import.rb +35 -8
- data/app/models/importo/result.rb +5 -0
- data/app/services/importo/import_service.rb +1 -3
- data/app/services/importo/revert_service.rb +1 -1
- data/app/tables/importo/imports_table.rb +13 -17
- data/app/tables/importo/mensa_imports_table.rb +35 -0
- data/app/views/importo/imports/index.html.slim +6 -2
- data/app/views/importo/imports/new.html.slim +30 -20
- data/config/locales/en.yml +21 -9
- data/config/locales/nl.yml +10 -4
- data/config/routes.rb +4 -4
- data/db/migrate/20180409151031_create_importo_import.rb +9 -9
- data/db/migrate/20180628175535_add_locale_importo_import.rb +1 -1
- data/db/migrate/20230510051447_remove_result_from_imports.rb +5 -0
- data/db/migrate/20230510083043_create_importo_results.rb +11 -0
- data/lib/generators/importo/importer_generator.rb +1 -1
- data/lib/generators/importo/install_generator.rb +7 -8
- data/lib/generators/importo/templates/importo.rb +11 -0
- data/lib/generators/satis/install_generator.rb +22 -0
- data/lib/generators/satis/tailwind_config_generator.rb +24 -0
- data/lib/generators/satis/templates/config/initializers/satis.rb +13 -0
- data/lib/importo/acts_as_import_owner.rb +1 -1
- data/lib/importo/configuration.rb +52 -49
- data/lib/importo/engine.rb +11 -9
- data/lib/importo/import_column.rb +18 -6
- data/lib/importo/test_helpers.rb +19 -0
- data/lib/importo/version.rb +1 -1
- data/lib/importo.rb +12 -17
- metadata +75 -24
- data/app/services/importo/callback_service.rb +0 -14
- data/lib/generators/templates/importo.rb +0 -21
- /data/lib/generators/{templates → importo/templates}/README +0 -0
- /data/lib/generators/{templates → importo/templates}/application_importer.rb +0 -0
- /data/lib/generators/{templates → importo/templates}/importer.rb +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 223e8f60e766e765e6b1933f05938e87a0edf05c02a307b931892266113802ad
|
4
|
+
data.tar.gz: e87886f568dd191c231e50fc03978e36c646c63d0acdfedb903deb35731fc60b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 553a215bc4f9b34194bd0405a22d53f9e28621fe9d5bdeadd19365acee1129b686f8b4f43db44def508b327ddba5b29d6efd4276d345904cca43acc5f827b4fe
|
7
|
+
data.tar.gz: 75c8a8d51b112b7b302c7b5607023a0f405e856435c31531ec39ab9264df8f51ca26e50b8e3c15ee26cc586802b7b632688c70c52ce1fb910d823200712db181
|
data/README.md
CHANGED
@@ -26,9 +26,11 @@ class ProductsImporter < ApplicationImporter
|
|
26
26
|
column attribute: :id
|
27
27
|
|
28
28
|
# attributes
|
29
|
-
column attribute: :name
|
30
|
-
column attribute: :number
|
29
|
+
column attribute: :name, required: true
|
31
30
|
column attribute: :description, strip_tags: false
|
31
|
+
column attribute: :number, export: { format: 'text', value: ->(record) { record.number }, example: 'FLAG-NLD-001' }, style: {b: true}
|
32
|
+
column attribute: :expires_on, export: { format: 'dd/mm/yyyy h:mm'}
|
33
|
+
column name: :price, export: { format: 'number', value: ->(record) { record.price } }
|
32
34
|
column attribute: :images do |value|
|
33
35
|
value.split(',').map do |image|
|
34
36
|
uri = URI.parse(image)
|
@@ -36,12 +38,14 @@ class ProductsImporter < ApplicationImporter
|
|
36
38
|
{ filename: File.basename(uri.to_s), io: URI.open(uri) }
|
37
39
|
end
|
38
40
|
end
|
41
|
+
column name: :kitting_component_product, delay: ->(value) {value.present? ? 5 : 0 }
|
39
42
|
|
40
43
|
def export_scope
|
41
44
|
Current.account.products
|
42
45
|
end
|
43
46
|
end
|
44
47
|
```
|
48
|
+
export args for column is optional, format takes excel custom format codes default is General
|
45
49
|
|
46
50
|
You should add translations to your locale files:
|
47
51
|
|
@@ -62,12 +66,22 @@ en:
|
|
62
66
|
number: Number
|
63
67
|
description: Description
|
64
68
|
images: Images
|
65
|
-
|
66
|
-
|
69
|
+
kitting_component_product: Component Product
|
70
|
+
# Shown in note in import sheet
|
67
71
|
hint:
|
68
72
|
id: 36 characters, existing of hexadecimal numbers, separated by dashes
|
69
73
|
images: Allows multiple image urls, separated by comma
|
70
|
-
|
74
|
+
# Below items are show in explanation sheet
|
75
|
+
explanation:
|
76
|
+
id: Record-id, only needed if you want to update an existing record
|
77
|
+
example:
|
78
|
+
id: 12345678-1234-1234-1234-123456789012
|
79
|
+
name: TEST-123
|
80
|
+
number: TEST-123
|
81
|
+
description: Test product
|
82
|
+
value:
|
83
|
+
id: Optional
|
84
|
+
|
71
85
|
```
|
72
86
|
|
73
87
|
## Installation
|
@@ -78,6 +92,9 @@ Add this line to your application's Gemfile:
|
|
78
92
|
gem 'importo'
|
79
93
|
```
|
80
94
|
|
95
|
+
Importo depends on Sidekiq Pro's batch functionality,
|
96
|
+
though you can use [sidekiq-batch](https://github.com/entdec/sidekiq-batch) as a drop-in for that.
|
97
|
+
|
81
98
|
And then execute:
|
82
99
|
|
83
100
|
```bash
|
data/Rakefile
CHANGED
@@ -1,39 +1,38 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
-
require
|
4
|
+
require "bundler/setup"
|
5
5
|
rescue LoadError
|
6
|
-
puts
|
6
|
+
puts "You must `gem install bundler` and `bundle install` to run rake tasks"
|
7
7
|
end
|
8
8
|
|
9
|
-
require
|
9
|
+
require "rdoc/task"
|
10
10
|
|
11
11
|
RDoc::Task.new(:rdoc) do |rdoc|
|
12
|
-
rdoc.rdoc_dir =
|
13
|
-
rdoc.title
|
14
|
-
rdoc.options <<
|
15
|
-
rdoc.rdoc_files.include(
|
16
|
-
rdoc.rdoc_files.include(
|
12
|
+
rdoc.rdoc_dir = "rdoc"
|
13
|
+
rdoc.title = "Importo"
|
14
|
+
rdoc.options << "--line-numbers"
|
15
|
+
rdoc.rdoc_files.include("README.md")
|
16
|
+
rdoc.rdoc_files.include("lib/**/*.rb")
|
17
17
|
end
|
18
18
|
|
19
|
-
APP_RAKEFILE = File.expand_path(
|
20
|
-
load
|
19
|
+
APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
|
20
|
+
load "rails/tasks/engine.rake"
|
21
21
|
|
22
|
-
load
|
22
|
+
load "rails/tasks/statistics.rake"
|
23
23
|
|
24
|
-
require
|
24
|
+
require "bundler/gem_tasks"
|
25
25
|
|
26
|
-
require
|
26
|
+
require "rake/testtask"
|
27
27
|
|
28
28
|
Rake::TestTask.new(:test) do |t|
|
29
|
-
t.libs <<
|
30
|
-
t.pattern =
|
29
|
+
t.libs << "test"
|
30
|
+
t.pattern = "test/**/*_test.rb"
|
31
31
|
t.verbose = false
|
32
32
|
end
|
33
33
|
|
34
34
|
task default: :test
|
35
35
|
|
36
36
|
# Adds the Auxilium semver task
|
37
|
-
spec = Gem::Specification.find_by_name
|
37
|
+
spec = Gem::Specification.find_by_name "auxilium"
|
38
38
|
load "#{spec.gem_dir}/lib/tasks/semver.rake"
|
39
|
-
|
@@ -11,7 +11,7 @@ module MaintenanceStandards
|
|
11
11
|
#
|
12
12
|
def flash_and_redirect(result, path, notice, error, render_action = :edit)
|
13
13
|
if result
|
14
|
-
if params[:commit] ==
|
14
|
+
if params[:commit] == "continue"
|
15
15
|
flash.now[:notice] = notice
|
16
16
|
else
|
17
17
|
redirect_to(path, notice: notice) && return
|
@@ -11,22 +11,22 @@ module Importo
|
|
11
11
|
def create
|
12
12
|
unless import_params
|
13
13
|
@import = Import.new(kind: params[:kind], locale: I18n.locale)
|
14
|
-
|
14
|
+
Signum.error(Current.user, text: t('.flash.no_file'))
|
15
15
|
render :new
|
16
16
|
return
|
17
17
|
end
|
18
|
-
@import = Import.new(import_params.merge(locale: I18n.locale,
|
18
|
+
@import = Import.new(import_params.merge(locale: I18n.locale,
|
19
|
+
importo_ownable: Importo.config.current_import_owner.call))
|
19
20
|
if @import.valid? && @import.schedule!
|
20
|
-
|
21
|
-
redirect_to action: :index
|
21
|
+
redirect_to importo.new_import_path(params[:kind] || @import.kind)
|
22
22
|
else
|
23
|
-
|
23
|
+
Signum.error(Current.user, text: t('.flash.error', error: @import.errors&.full_messages&.join('.')))
|
24
24
|
render :new
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
28
|
def undo
|
29
|
-
@import = Import.where(importo_ownable: Importo.config.current_import_owner).find(params[:id])
|
29
|
+
@import = Import.where(importo_ownable: Importo.config.current_import_owner.call).find(params[:id])
|
30
30
|
if @import.can_revert? && @import.revert
|
31
31
|
redirect_to action: :index, notice: 'Import reverted'
|
32
32
|
else
|
@@ -35,8 +35,8 @@ module Importo
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def destroy
|
38
|
-
@import = Import.
|
39
|
-
redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy(@import)
|
38
|
+
@import = Import.find(params[:id])
|
39
|
+
redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy.call(@import)
|
40
40
|
|
41
41
|
@import.destroy
|
42
42
|
redirect_to action: :index
|
@@ -44,22 +44,25 @@ module Importo
|
|
44
44
|
|
45
45
|
def sample
|
46
46
|
import = Import.new(kind: params[:kind], locale: I18n.locale)
|
47
|
-
send_data import.importer.sample_file.read,
|
47
|
+
send_data import.importer.sample_file.read,
|
48
|
+
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('sample')
|
48
49
|
end
|
49
50
|
|
50
51
|
def export
|
51
52
|
import = Import.new(kind: params[:kind], locale: I18n.locale)
|
52
|
-
send_data import.importer.export_file.read,
|
53
|
+
send_data import.importer.export_file.read,
|
54
|
+
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('export')
|
53
55
|
end
|
54
56
|
|
55
57
|
def index
|
56
|
-
@imports = Importo.config.admin_visible_imports.order(created_at: :desc).limit(50)
|
58
|
+
@imports = Importo.config.admin_visible_imports.call.order(created_at: :desc).limit(50)
|
57
59
|
end
|
58
60
|
|
59
61
|
private
|
60
62
|
|
61
63
|
def import_params
|
62
|
-
params.require(:import).permit(:original, :kind, :column_overrides,
|
64
|
+
params.require(:import).permit(:original, :kind, :column_overrides,
|
65
|
+
column_overrides: params.dig(:import, :column_overrides)&.keys)
|
63
66
|
end
|
64
67
|
end
|
65
68
|
end
|
@@ -2,16 +2,16 @@
|
|
2
2
|
|
3
3
|
module Importo
|
4
4
|
module ApplicationHelper
|
5
|
-
def respond_to_missing?(method)
|
6
|
-
|
7
|
-
end
|
8
|
-
|
9
|
-
def method_missing(method, *args, &block)
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
end
|
5
|
+
# def respond_to_missing?(method)
|
6
|
+
# method.ends_with?('_url') || method.ends_with?('_path')
|
7
|
+
# end
|
8
|
+
#
|
9
|
+
# def method_missing(method, *args, &block)
|
10
|
+
# if main_app.respond_to?(method)
|
11
|
+
# main_app.send(method, *args)
|
12
|
+
# else
|
13
|
+
# super
|
14
|
+
# end
|
15
|
+
# end
|
16
16
|
end
|
17
17
|
end
|
@@ -1,7 +1,6 @@
|
|
1
|
-
|
2
1
|
# frozen_string_literal: true
|
3
2
|
|
4
|
-
require
|
3
|
+
require "active_support/concern"
|
5
4
|
|
6
5
|
module Exportable
|
7
6
|
extend ActiveSupport::Concern
|
@@ -14,7 +13,9 @@ module Exportable
|
|
14
13
|
end
|
15
14
|
|
16
15
|
def sample_data
|
17
|
-
|
16
|
+
sample_data = []
|
17
|
+
100.times { sample_data << export_columns.map { |_, c| c.options[:example] || "" } }
|
18
|
+
sample_data
|
18
19
|
end
|
19
20
|
|
20
21
|
#
|
@@ -38,15 +39,16 @@ module Exportable
|
|
38
39
|
|
39
40
|
def export_row(record)
|
40
41
|
export_columns.map do |_, c|
|
41
|
-
value =
|
42
|
+
value = ""
|
42
43
|
|
43
44
|
if c.options[:attribute]
|
44
|
-
if
|
45
|
+
if c.options[:export]&.key?(:value)
|
46
|
+
value = c.options[:export][:value].call(record) if c.options[:export][:value].is_a?(Proc)
|
47
|
+
elsif record.respond_to?(c.options[:attribute])
|
45
48
|
value = record.send(c.options[:attribute])
|
46
49
|
value = value&.body&.to_html if value.is_a?(ActionText::RichText)
|
47
50
|
end
|
48
|
-
|
49
|
-
value ||= record.attributes[c.options[:attribute].to_s]
|
51
|
+
value ||= record.attributes[c.options[:attribute].to_s] if value.nil?
|
50
52
|
end
|
51
53
|
|
52
54
|
value
|
@@ -59,9 +61,9 @@ module Exportable
|
|
59
61
|
workbook = xls.workbook
|
60
62
|
sheet = workbook.add_worksheet(name: friendly_name&.pluralize || model.name.demodulize.pluralize)
|
61
63
|
workbook.styles do |style|
|
62
|
-
introduction_style = style.add_style(bg_color:
|
63
|
-
header_style = style.add_style(b: true, bg_color:
|
64
|
-
header_required_style = style.add_style(b: true, bg_color:
|
64
|
+
introduction_style = style.add_style(bg_color: "E2EEDA")
|
65
|
+
header_style = style.add_style(b: true, bg_color: "A8D08E", border: {style: :thin, color: "000000"})
|
66
|
+
header_required_style = style.add_style(b: true, bg_color: "A8D08E", fg_color: "C00100", border: {style: :thin, color: "000000"})
|
65
67
|
|
66
68
|
# Introduction
|
67
69
|
introduction.each_with_index do |intro, i|
|
@@ -75,14 +77,19 @@ module Exportable
|
|
75
77
|
|
76
78
|
export_columns.each.with_index do |f, i|
|
77
79
|
field = f.last
|
78
|
-
sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author:
|
80
|
+
sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author: "", text: field.hint, visible: false if field.hint.present?
|
81
|
+
end
|
82
|
+
styles = export_columns.map do |_, c|
|
83
|
+
if c.options.dig(:export, :format) == "number" || (c.options.dig(:export, :format).nil? && c.options.dig(:export, :example).is_a?(Numeric))
|
84
|
+
number = workbook.styles.add_style format_code: "#"
|
85
|
+
elsif c.options.dig(:export, :format) == "text" || (c.options.dig(:export, :format).nil? && c.options.dig(:export, :example).is_a?(String))
|
86
|
+
text = workbook.styles.add_style format_code: "@"
|
87
|
+
elsif c.options.dig(:export, :format)
|
88
|
+
workbook.styles.add_style format_code: c.options.dig(:export, :format).to_s
|
89
|
+
else
|
90
|
+
workbook.styles.add_style format_code: "General"
|
91
|
+
end
|
79
92
|
end
|
80
|
-
|
81
|
-
number = workbook.styles.add_style format_code: '#'
|
82
|
-
text = workbook.styles.add_style format_code: '@'
|
83
|
-
|
84
|
-
styles = export_columns.map { |_, c| c.options[:example].is_a?(Numeric) ? number : text }
|
85
|
-
|
86
93
|
# Examples
|
87
94
|
data_rows.each do |data|
|
88
95
|
sheet.add_row data, style: styles
|
@@ -91,15 +98,15 @@ module Exportable
|
|
91
98
|
|
92
99
|
sheet.column_info[0].width = 10
|
93
100
|
|
94
|
-
sheet = workbook.add_worksheet(name: I18n.t(
|
101
|
+
sheet = workbook.add_worksheet(name: I18n.t("importo.sheet.explanation.name"))
|
95
102
|
|
96
103
|
workbook.styles do |style|
|
97
|
-
introduction_style = style.add_style(bg_color:
|
98
|
-
header_style = style.add_style(b: true, bg_color:
|
104
|
+
introduction_style = style.add_style(bg_color: "E2EEDA")
|
105
|
+
header_style = style.add_style(b: true, bg_color: "A8D08E", border: {style: :thin, color: "000000"})
|
99
106
|
|
100
107
|
column_style = style.add_style(b: true)
|
101
|
-
required_style = style.add_style(b: true, fg_color:
|
102
|
-
wrap_style = workbook.styles.add_style alignment: {
|
108
|
+
required_style = style.add_style(b: true, fg_color: "C00100")
|
109
|
+
wrap_style = workbook.styles.add_style alignment: {wrap_text: true}
|
103
110
|
|
104
111
|
# Introduction
|
105
112
|
introduction.each_with_index do |intro, i|
|
@@ -109,10 +116,10 @@ module Exportable
|
|
109
116
|
end
|
110
117
|
|
111
118
|
# Header row
|
112
|
-
sheet.add_row [I18n.t(
|
119
|
+
sheet.add_row [I18n.t("importo.sheet.explanation.column"), I18n.t("importo.sheet.explanation.value"), I18n.t("importo.sheet.explanation.explanation"), I18n.t("importo.sheet.explanation.example")], style: [header_style] * 4
|
113
120
|
export_columns.each do |_, c|
|
114
121
|
styles = [c.options[:required] ? required_style : column_style, wrap_style]
|
115
|
-
sheet.add_row [c.name, c.explanation], style: styles
|
122
|
+
sheet.add_row [c.name, c.value, c.explanation, c.example], style: styles
|
116
123
|
end
|
117
124
|
end
|
118
125
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "active_support/concern"
|
4
4
|
|
5
5
|
module Importable
|
6
6
|
extend ActiveSupport::Concern
|
@@ -14,15 +14,13 @@ module Importable
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def convert_values(row)
|
17
|
-
return row if row.instance_variable_get(
|
17
|
+
return row if row.instance_variable_get(:@importo_converted_values)
|
18
18
|
|
19
|
-
row.instance_variable_set(
|
19
|
+
row.instance_variable_set(:@importo_converted_values, true)
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
end
|
21
|
+
columns.each do |k, col|
|
22
|
+
next if col.proc.blank? || row[k].nil?
|
24
23
|
|
25
|
-
cols_to_populate.each do |k, col|
|
26
24
|
attr = col.options[:attribute]
|
27
25
|
|
28
26
|
row[k] = import.column_overrides[col.attribute] if import.column_overrides[col.attribute]
|
@@ -52,26 +50,36 @@ module Importable
|
|
52
50
|
# It wil try and find the record by id, or initialize a new record with it's attributes set based on the mapping from columns
|
53
51
|
#
|
54
52
|
def populate(row, record = nil)
|
55
|
-
raise
|
53
|
+
raise "No attributes set for columns" unless columns.any? { |_, v| v.options[:attribute].present? }
|
56
54
|
|
57
55
|
row = convert_values(row)
|
58
56
|
|
59
57
|
result = if record
|
60
|
-
|
61
|
-
|
62
|
-
|
58
|
+
record
|
59
|
+
else
|
60
|
+
raise "No model set" unless model
|
63
61
|
|
64
|
-
|
65
|
-
|
62
|
+
model.find_or_initialize_by(id: row["id"])
|
63
|
+
end
|
66
64
|
|
67
65
|
attributes = {}
|
68
66
|
cols_to_populate = columns.select do |_, v|
|
69
67
|
v.options[:attribute].present?
|
70
68
|
end
|
71
69
|
|
70
|
+
cols_to_populate = cols_to_populate.deep_symbolize_keys
|
71
|
+
|
72
72
|
cols_to_populate.each do |k, col|
|
73
73
|
attr = col.options[:attribute]
|
74
|
-
|
74
|
+
|
75
|
+
next unless row.key? k
|
76
|
+
next if !row[k].present? && col.options[:default].nil?
|
77
|
+
|
78
|
+
attributes = if !row[k].present? && !col.options[:default].nil?
|
79
|
+
set_attribute(attributes, attr, col.options[:default])
|
80
|
+
else
|
81
|
+
set_attribute(attributes, attr, row[k])
|
82
|
+
end
|
75
83
|
end
|
76
84
|
|
77
85
|
result.assign_attributes(attributes)
|
@@ -80,89 +88,163 @@ module Importable
|
|
80
88
|
end
|
81
89
|
|
82
90
|
#
|
83
|
-
#
|
91
|
+
# Callbakcs
|
84
92
|
#
|
93
|
+
def before_build(_record, _row)
|
94
|
+
end
|
95
|
+
|
96
|
+
def around_build(_record, _row)
|
97
|
+
yield
|
98
|
+
end
|
99
|
+
|
100
|
+
def after_build(_record, _row)
|
101
|
+
end
|
102
|
+
|
85
103
|
def before_save(_record, _row)
|
86
|
-
# Implement optionally in child class to mangle
|
87
104
|
end
|
88
105
|
|
89
|
-
|
90
|
-
|
91
|
-
|
106
|
+
def around_save(_record, _row)
|
107
|
+
yield
|
108
|
+
end
|
109
|
+
|
92
110
|
def after_save(_record, _row)
|
93
|
-
|
111
|
+
end
|
112
|
+
|
113
|
+
def before_validate(_record, _row)
|
114
|
+
end
|
115
|
+
|
116
|
+
def around_validate(_record, _row)
|
117
|
+
yield
|
118
|
+
end
|
119
|
+
|
120
|
+
def after_validate(_record, _row)
|
94
121
|
end
|
95
122
|
|
96
123
|
#
|
97
124
|
# Does the actual import
|
98
125
|
#
|
99
126
|
def import!
|
100
|
-
raise ArgumentError,
|
101
|
-
|
102
|
-
|
103
|
-
|
127
|
+
raise ArgumentError, "Invalid data structure" unless structure_valid?
|
128
|
+
|
129
|
+
batch = Sidekiq::Batch.new
|
130
|
+
batch.description = "#{import.original.filename} - #{import.kind}"
|
131
|
+
batch.on(:success, Importo::ImportJobCallback, import_id: import.id)
|
132
|
+
|
133
|
+
batch.jobs do
|
134
|
+
column_with_delay = columns.select { |k, v| v.delay.present? }
|
135
|
+
loop_data_rows do |attributes, index|
|
136
|
+
if column_with_delay.present?
|
137
|
+
delay = column_with_delay.map do |k, v|
|
138
|
+
next unless attributes[k].present?
|
139
|
+
v.delay.call(attributes[k])
|
140
|
+
end.compact
|
141
|
+
end
|
142
|
+
Importo::ImportJob.set(wait_until: (delay.max * index).seconds.from_now).perform_async(JSON.dump(attributes), index, import.id) if delay.present?
|
143
|
+
Importo::ImportJob.perform_async(JSON.dump(attributes), index, import.id) unless delay.present?
|
144
|
+
end
|
104
145
|
end
|
105
|
-
@import.result.attach(io: results_file, filename: @import.importer.file_name('results'),
|
106
|
-
content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
|
107
|
-
|
108
|
-
@import.result_message = I18n.t('importo.importers.result_message', nr: results.compact.count, of: results.count,
|
109
|
-
start_row: data_start_row)
|
110
|
-
@import.complete!
|
111
146
|
|
112
147
|
true
|
113
|
-
rescue
|
148
|
+
rescue => e
|
114
149
|
@import.result_message = "Exception: #{e.message}"
|
115
|
-
Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(
|
150
|
+
Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(";")}"
|
116
151
|
@import.failure!
|
117
152
|
|
118
153
|
false
|
119
154
|
end
|
120
155
|
|
121
|
-
|
122
|
-
|
123
|
-
##
|
124
|
-
# Overridable failure method
|
125
|
-
#
|
126
|
-
def failure(_row, _record, index, exception)
|
127
|
-
Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(';')}"
|
128
|
-
end
|
129
|
-
|
130
|
-
def process_data_row(attributes, index)
|
156
|
+
def process_data_row(attributes, index, last_attempt: true)
|
131
157
|
record = nil
|
132
158
|
row_hash = Digest::SHA256.base64digest(attributes.inspect)
|
133
159
|
duplicate_import = nil
|
134
160
|
|
135
|
-
|
136
|
-
|
161
|
+
run_callbacks :row_import do
|
162
|
+
record = nil
|
163
|
+
|
164
|
+
ActiveRecord::Base.transaction(isolation: :read_committed) do
|
137
165
|
register_result(index, hash: row_hash, state: :processing)
|
138
166
|
|
139
|
-
record
|
140
|
-
record
|
141
|
-
|
142
|
-
|
143
|
-
|
167
|
+
before_build(record, attributes)
|
168
|
+
around_build(record, attributes) do
|
169
|
+
record = build(attributes)
|
170
|
+
end
|
171
|
+
after_build(record, attributes)
|
172
|
+
|
173
|
+
before_validate(record, attributes)
|
174
|
+
around_validate(record, attributes) do
|
175
|
+
record.validate!
|
176
|
+
end
|
177
|
+
after_validate(record, attributes)
|
178
|
+
|
179
|
+
model.with_advisory_lock(:importo) do
|
180
|
+
before_save(record, attributes)
|
181
|
+
around_save(record, attributes) do
|
182
|
+
record.save!
|
183
|
+
end
|
184
|
+
after_save(record, attributes)
|
185
|
+
end
|
186
|
+
|
144
187
|
duplicate_import = duplicate?(row_hash, record.id)
|
145
188
|
raise Importo::DuplicateRowError if duplicate_import
|
146
189
|
|
147
190
|
register_result(index, class: record.class.name, id: record.id, state: :success)
|
148
191
|
end
|
149
|
-
record
|
150
|
-
rescue Importo::DuplicateRowError
|
151
|
-
record_id = duplicate_import.results.find { |data| data['hash'] == row_hash }['id']
|
152
|
-
register_result(index, id: record_id, state: :duplicate,
|
153
|
-
message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
|
154
|
-
nil
|
155
|
-
rescue StandardError => e
|
156
|
-
errors = record.respond_to?(:errors) && record.errors.full_messages.join(', ')
|
157
|
-
error_message = "#{e.message} (#{e.backtrace.first.split('/').last})"
|
158
|
-
failure(attributes, record, index, e)
|
159
|
-
register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
|
160
|
-
nil
|
161
192
|
end
|
193
|
+
|
194
|
+
record
|
195
|
+
rescue Importo::DuplicateRowError
|
196
|
+
record_id = duplicate_import.results.find { |data| data["hash"] == row_hash }["id"]
|
197
|
+
register_result(index, id: record_id, state: :duplicate,
|
198
|
+
message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
|
199
|
+
|
200
|
+
run_callbacks(:row_import, :after)
|
201
|
+
nil
|
202
|
+
rescue Importo::RetryError => e
|
203
|
+
raise e unless last_attempt
|
204
|
+
|
205
|
+
errors = record.respond_to?(:errors) && record.errors.full_messages.join(", ")
|
206
|
+
error_message = "#{e.message} (#{e.backtrace.first.split("/").last})"
|
207
|
+
failure(attributes, record, index, e)
|
208
|
+
register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
|
209
|
+
nil
|
210
|
+
rescue => e
|
211
|
+
# We rescue ActiveRecord::RecordNotUnique here, due to how transactions work, some row imports may have started the transaction and the current row doesn't see the results
|
212
|
+
raise Importo::RetryError.new("ActiveRecord::RecordNotUnique", 2) if !last_attempt && e.is_a?(ActiveRecord::RecordNotUnique)
|
213
|
+
errors = record.respond_to?(:errors) && record.errors.full_messages.join(", ")
|
214
|
+
error_message = "#{e.message} (#{e.backtrace.first.split("/").last})"
|
215
|
+
failure(attributes, record, index, e)
|
216
|
+
register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
|
217
|
+
run_callbacks(:row_import, :after)
|
218
|
+
nil
|
219
|
+
end
|
220
|
+
|
221
|
+
private
|
222
|
+
|
223
|
+
##
|
224
|
+
# Overridable failure method
|
225
|
+
#
|
226
|
+
def failure(_row, _record, index, exception)
|
227
|
+
Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(";")}"
|
162
228
|
end
|
163
229
|
|
164
230
|
def set_attribute(hash, path, value)
|
165
|
-
tmp_hash = path.to_s.split(
|
231
|
+
tmp_hash = path.to_s.split(".").reverse.inject(value) { |h, s| {s => h} }
|
166
232
|
hash.deep_merge(tmp_hash)
|
167
233
|
end
|
234
|
+
|
235
|
+
# Only when the block returns a record created in this import, it returns that record, otherwise nil
|
236
|
+
#
|
237
|
+
# record ||= only_current_import! do
|
238
|
+
# User.find_by(email: row[:email].downcase) if row[:email].present?
|
239
|
+
# end
|
240
|
+
#
|
241
|
+
# Only if the user was created this import will the block return the user found.
|
242
|
+
#
|
243
|
+
# @return [Object]
|
244
|
+
def only_current_import!(&block)
|
245
|
+
ActiveRecord::Base.uncached do
|
246
|
+
record = yield
|
247
|
+
record if record && @import.results.where("details->>'state' = ?", "success").where("details->>'id' = ? ", record.id).exists?
|
248
|
+
end
|
249
|
+
end
|
168
250
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "active_support/concern"
|
4
4
|
|
5
5
|
module ImporterDsl
|
6
6
|
extend ActiveSupport::Concern
|
@@ -31,19 +31,11 @@ module ImporterDsl
|
|
31
31
|
#
|
32
32
|
# @param [Object] args
|
33
33
|
# @param [Object] block which will filter the results before storing the value in the attribute, this is useful for lookups or reformatting
|
34
|
-
def column(
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
name
|
39
|
-
name ||= options[:attribute]
|
40
|
-
|
41
|
-
hint = args[1]
|
42
|
-
hint ||= options[:hint]
|
43
|
-
|
44
|
-
options[:scope] = self.name.underscore.to_s.tr('/', '.').to_sym
|
45
|
-
|
46
|
-
columns[name] = Importo::ImportColumn.new(name, hint, options[:explanation], options, &block)
|
34
|
+
def column(**options, &block)
|
35
|
+
column_name ||= options[:name]
|
36
|
+
column_name ||= options[:attribute]
|
37
|
+
options[:scope] = name.to_s.underscore.to_s.tr("/", ".").to_sym
|
38
|
+
columns[column_name] = Importo::ImportColumn.new(name: column_name, **options, &block)
|
47
39
|
end
|
48
40
|
|
49
41
|
def model(model = nil)
|