importo 2.0.5 → 3.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +22 -5
- data/Rakefile +16 -17
- data/app/controllers/concerns/maintenance_standards.rb +1 -1
- data/app/controllers/importo/imports_controller.rb +15 -12
- data/app/helpers/importo/application_helper.rb +11 -11
- data/app/importers/concerns/exportable.rb +31 -24
- data/app/importers/concerns/importable.rb +144 -62
- data/app/importers/concerns/importer_dsl.rb +6 -14
- data/app/importers/concerns/original.rb +23 -23
- data/app/importers/concerns/result_feedback.rb +62 -44
- data/app/importers/concerns/revertable.rb +11 -12
- data/app/importers/importo/base_importer.rb +16 -4
- data/app/importers/importo/import_job_callback.rb +29 -0
- data/app/jobs/importo/import_job.rb +40 -0
- data/app/mailers/importo/application_mailer.rb +2 -2
- data/app/models/importo/import.rb +35 -8
- data/app/models/importo/result.rb +5 -0
- data/app/services/importo/import_service.rb +1 -3
- data/app/services/importo/revert_service.rb +1 -1
- data/app/tables/importo/imports_table.rb +13 -17
- data/app/tables/importo/mensa_imports_table.rb +35 -0
- data/app/views/importo/imports/index.html.slim +6 -2
- data/app/views/importo/imports/new.html.slim +30 -20
- data/config/locales/en.yml +21 -9
- data/config/locales/nl.yml +10 -4
- data/config/routes.rb +4 -4
- data/db/migrate/20180409151031_create_importo_import.rb +9 -9
- data/db/migrate/20180628175535_add_locale_importo_import.rb +1 -1
- data/db/migrate/20230510051447_remove_result_from_imports.rb +5 -0
- data/db/migrate/20230510083043_create_importo_results.rb +11 -0
- data/lib/generators/importo/importer_generator.rb +1 -1
- data/lib/generators/importo/install_generator.rb +7 -8
- data/lib/generators/importo/templates/importo.rb +11 -0
- data/lib/generators/satis/install_generator.rb +22 -0
- data/lib/generators/satis/tailwind_config_generator.rb +24 -0
- data/lib/generators/satis/templates/config/initializers/satis.rb +13 -0
- data/lib/importo/acts_as_import_owner.rb +1 -1
- data/lib/importo/configuration.rb +52 -49
- data/lib/importo/engine.rb +11 -9
- data/lib/importo/import_column.rb +18 -6
- data/lib/importo/test_helpers.rb +19 -0
- data/lib/importo/version.rb +1 -1
- data/lib/importo.rb +12 -17
- metadata +75 -24
- data/app/services/importo/callback_service.rb +0 -14
- data/lib/generators/templates/importo.rb +0 -21
- /data/lib/generators/{templates → importo/templates}/README +0 -0
- /data/lib/generators/{templates → importo/templates}/application_importer.rb +0 -0
- /data/lib/generators/{templates → importo/templates}/importer.rb +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 208e74497d020c5798661a0c0280d0952ab0a2381ede5317e3460f57b6ea8aa2
|
4
|
+
data.tar.gz: 78d02a8842863216adc84411c2ee71bd16202f40b7c50734248cf35aaa76a244
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d954737dd202b1e068fdcae143e06f49a3cb7515d04b7da85bc0111c80cbeed7f384a6bf3ba968a0a742bfe76efbc6d554af9bc57b01777be3756968b8ae8e88
|
7
|
+
data.tar.gz: 8796b9d5746d0995e8cff516f30eb1662598ee3d34f5c4fd32c19d0604e379ffa79cae747a15b4502e4b0ecef59eed0662fda3bd42f0d4d0dee5fdd030c93fe1
|
data/README.md
CHANGED
@@ -26,9 +26,11 @@ class ProductsImporter < ApplicationImporter
|
|
26
26
|
column attribute: :id
|
27
27
|
|
28
28
|
# attributes
|
29
|
-
column attribute: :name
|
30
|
-
column attribute: :number
|
29
|
+
column attribute: :name, required: true
|
31
30
|
column attribute: :description, strip_tags: false
|
31
|
+
column attribute: :number, export: { format: 'text', value: ->(record) { record.number }, example: 'FLAG-NLD-001' }, style: {b: true}
|
32
|
+
column attribute: :expires_on, export: { format: 'dd/mm/yyyy h:mm'}
|
33
|
+
column name: :price, export: { format: 'number', value: ->(record) { record.price } }
|
32
34
|
column attribute: :images do |value|
|
33
35
|
value.split(',').map do |image|
|
34
36
|
uri = URI.parse(image)
|
@@ -36,12 +38,14 @@ class ProductsImporter < ApplicationImporter
|
|
36
38
|
{ filename: File.basename(uri.to_s), io: URI.open(uri) }
|
37
39
|
end
|
38
40
|
end
|
41
|
+
column name: :kitting_component_product, delay: ->(value) {value.present? ? 5 : 0 }
|
39
42
|
|
40
43
|
def export_scope
|
41
44
|
Current.account.products
|
42
45
|
end
|
43
46
|
end
|
44
47
|
```
|
48
|
+
export args for column is optional, format takes excel custom format codes default is General
|
45
49
|
|
46
50
|
You should add translations to your locale files:
|
47
51
|
|
@@ -62,12 +66,22 @@ en:
|
|
62
66
|
number: Number
|
63
67
|
description: Description
|
64
68
|
images: Images
|
65
|
-
|
66
|
-
|
69
|
+
kitting_component_product: Component Product
|
70
|
+
# Shown in note in import sheet
|
67
71
|
hint:
|
68
72
|
id: 36 characters, existing of hexadecimal numbers, separated by dashes
|
69
73
|
images: Allows multiple image urls, separated by comma
|
70
|
-
|
74
|
+
# Below items are show in explanation sheet
|
75
|
+
explanation:
|
76
|
+
id: Record-id, only needed if you want to update an existing record
|
77
|
+
example:
|
78
|
+
id: 12345678-1234-1234-1234-123456789012
|
79
|
+
name: TEST-123
|
80
|
+
number: TEST-123
|
81
|
+
description: Test product
|
82
|
+
value:
|
83
|
+
id: Optional
|
84
|
+
|
71
85
|
```
|
72
86
|
|
73
87
|
## Installation
|
@@ -78,6 +92,9 @@ Add this line to your application's Gemfile:
|
|
78
92
|
gem 'importo'
|
79
93
|
```
|
80
94
|
|
95
|
+
Importo depends on Sidekiq Pro's batch functionality,
|
96
|
+
though you can use [sidekiq-batch](https://github.com/entdec/sidekiq-batch) as a drop-in for that.
|
97
|
+
|
81
98
|
And then execute:
|
82
99
|
|
83
100
|
```bash
|
data/Rakefile
CHANGED
@@ -1,39 +1,38 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
-
require
|
4
|
+
require "bundler/setup"
|
5
5
|
rescue LoadError
|
6
|
-
puts
|
6
|
+
puts "You must `gem install bundler` and `bundle install` to run rake tasks"
|
7
7
|
end
|
8
8
|
|
9
|
-
require
|
9
|
+
require "rdoc/task"
|
10
10
|
|
11
11
|
RDoc::Task.new(:rdoc) do |rdoc|
|
12
|
-
rdoc.rdoc_dir =
|
13
|
-
rdoc.title
|
14
|
-
rdoc.options <<
|
15
|
-
rdoc.rdoc_files.include(
|
16
|
-
rdoc.rdoc_files.include(
|
12
|
+
rdoc.rdoc_dir = "rdoc"
|
13
|
+
rdoc.title = "Importo"
|
14
|
+
rdoc.options << "--line-numbers"
|
15
|
+
rdoc.rdoc_files.include("README.md")
|
16
|
+
rdoc.rdoc_files.include("lib/**/*.rb")
|
17
17
|
end
|
18
18
|
|
19
|
-
APP_RAKEFILE = File.expand_path(
|
20
|
-
load
|
19
|
+
APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
|
20
|
+
load "rails/tasks/engine.rake"
|
21
21
|
|
22
|
-
load
|
22
|
+
load "rails/tasks/statistics.rake"
|
23
23
|
|
24
|
-
require
|
24
|
+
require "bundler/gem_tasks"
|
25
25
|
|
26
|
-
require
|
26
|
+
require "rake/testtask"
|
27
27
|
|
28
28
|
Rake::TestTask.new(:test) do |t|
|
29
|
-
t.libs <<
|
30
|
-
t.pattern =
|
29
|
+
t.libs << "test"
|
30
|
+
t.pattern = "test/**/*_test.rb"
|
31
31
|
t.verbose = false
|
32
32
|
end
|
33
33
|
|
34
34
|
task default: :test
|
35
35
|
|
36
36
|
# Adds the Auxilium semver task
|
37
|
-
spec = Gem::Specification.find_by_name
|
37
|
+
spec = Gem::Specification.find_by_name "auxilium"
|
38
38
|
load "#{spec.gem_dir}/lib/tasks/semver.rake"
|
39
|
-
|
@@ -11,7 +11,7 @@ module MaintenanceStandards
|
|
11
11
|
#
|
12
12
|
def flash_and_redirect(result, path, notice, error, render_action = :edit)
|
13
13
|
if result
|
14
|
-
if params[:commit] ==
|
14
|
+
if params[:commit] == "continue"
|
15
15
|
flash.now[:notice] = notice
|
16
16
|
else
|
17
17
|
redirect_to(path, notice: notice) && return
|
@@ -11,22 +11,22 @@ module Importo
|
|
11
11
|
def create
|
12
12
|
unless import_params
|
13
13
|
@import = Import.new(kind: params[:kind], locale: I18n.locale)
|
14
|
-
|
14
|
+
Signum.error(Current.user, text: t('.flash.no_file'))
|
15
15
|
render :new
|
16
16
|
return
|
17
17
|
end
|
18
|
-
@import = Import.new(import_params.merge(locale: I18n.locale,
|
18
|
+
@import = Import.new(import_params.merge(locale: I18n.locale,
|
19
|
+
importo_ownable: Importo.config.current_import_owner.call))
|
19
20
|
if @import.valid? && @import.schedule!
|
20
|
-
|
21
|
-
redirect_to action: :index
|
21
|
+
redirect_to importo.new_import_path(params[:kind] || @import.kind)
|
22
22
|
else
|
23
|
-
|
23
|
+
Signum.error(Current.user, text: t('.flash.error', error: @import.errors&.full_messages&.join('.')))
|
24
24
|
render :new
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
28
|
def undo
|
29
|
-
@import = Import.where(importo_ownable: Importo.config.current_import_owner).find(params[:id])
|
29
|
+
@import = Import.where(importo_ownable: Importo.config.current_import_owner.call).find(params[:id])
|
30
30
|
if @import.can_revert? && @import.revert
|
31
31
|
redirect_to action: :index, notice: 'Import reverted'
|
32
32
|
else
|
@@ -35,8 +35,8 @@ module Importo
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def destroy
|
38
|
-
@import = Import.
|
39
|
-
redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy(@import)
|
38
|
+
@import = Import.find(params[:id])
|
39
|
+
redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy.call(@import)
|
40
40
|
|
41
41
|
@import.destroy
|
42
42
|
redirect_to action: :index
|
@@ -44,22 +44,25 @@ module Importo
|
|
44
44
|
|
45
45
|
def sample
|
46
46
|
import = Import.new(kind: params[:kind], locale: I18n.locale)
|
47
|
-
send_data import.importer.sample_file.read,
|
47
|
+
send_data import.importer.sample_file.read,
|
48
|
+
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('sample')
|
48
49
|
end
|
49
50
|
|
50
51
|
def export
|
51
52
|
import = Import.new(kind: params[:kind], locale: I18n.locale)
|
52
|
-
send_data import.importer.export_file.read,
|
53
|
+
send_data import.importer.export_file.read,
|
54
|
+
type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('export')
|
53
55
|
end
|
54
56
|
|
55
57
|
def index
|
56
|
-
@imports = Importo.config.admin_visible_imports.order(created_at: :desc).limit(50)
|
58
|
+
@imports = Importo.config.admin_visible_imports.call.order(created_at: :desc).limit(50)
|
57
59
|
end
|
58
60
|
|
59
61
|
private
|
60
62
|
|
61
63
|
def import_params
|
62
|
-
params.require(:import).permit(:original, :kind, :column_overrides,
|
64
|
+
params.require(:import).permit(:original, :kind, :column_overrides,
|
65
|
+
column_overrides: params.dig(:import, :column_overrides)&.keys)
|
63
66
|
end
|
64
67
|
end
|
65
68
|
end
|
@@ -2,16 +2,16 @@
|
|
2
2
|
|
3
3
|
module Importo
|
4
4
|
module ApplicationHelper
|
5
|
-
def respond_to_missing?(method)
|
6
|
-
|
7
|
-
end
|
8
|
-
|
9
|
-
def method_missing(method, *args, &block)
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
end
|
5
|
+
# def respond_to_missing?(method)
|
6
|
+
# method.ends_with?('_url') || method.ends_with?('_path')
|
7
|
+
# end
|
8
|
+
#
|
9
|
+
# def method_missing(method, *args, &block)
|
10
|
+
# if main_app.respond_to?(method)
|
11
|
+
# main_app.send(method, *args)
|
12
|
+
# else
|
13
|
+
# super
|
14
|
+
# end
|
15
|
+
# end
|
16
16
|
end
|
17
17
|
end
|
@@ -1,7 +1,6 @@
|
|
1
|
-
|
2
1
|
# frozen_string_literal: true
|
3
2
|
|
4
|
-
require
|
3
|
+
require "active_support/concern"
|
5
4
|
|
6
5
|
module Exportable
|
7
6
|
extend ActiveSupport::Concern
|
@@ -14,7 +13,9 @@ module Exportable
|
|
14
13
|
end
|
15
14
|
|
16
15
|
def sample_data
|
17
|
-
|
16
|
+
sample_data = []
|
17
|
+
100.times { sample_data << export_columns.map { |_, c| c.options[:example] || "" } }
|
18
|
+
sample_data
|
18
19
|
end
|
19
20
|
|
20
21
|
#
|
@@ -38,15 +39,16 @@ module Exportable
|
|
38
39
|
|
39
40
|
def export_row(record)
|
40
41
|
export_columns.map do |_, c|
|
41
|
-
value =
|
42
|
+
value = ""
|
42
43
|
|
43
44
|
if c.options[:attribute]
|
44
|
-
if
|
45
|
+
if c.options[:export]&.key?(:value)
|
46
|
+
value = c.options[:export][:value].call(record) if c.options[:export][:value].is_a?(Proc)
|
47
|
+
elsif record.respond_to?(c.options[:attribute])
|
45
48
|
value = record.send(c.options[:attribute])
|
46
49
|
value = value&.body&.to_html if value.is_a?(ActionText::RichText)
|
47
50
|
end
|
48
|
-
|
49
|
-
value ||= record.attributes[c.options[:attribute].to_s]
|
51
|
+
value ||= record.attributes[c.options[:attribute].to_s] if value.nil?
|
50
52
|
end
|
51
53
|
|
52
54
|
value
|
@@ -59,9 +61,9 @@ module Exportable
|
|
59
61
|
workbook = xls.workbook
|
60
62
|
sheet = workbook.add_worksheet(name: friendly_name&.pluralize || model.name.demodulize.pluralize)
|
61
63
|
workbook.styles do |style|
|
62
|
-
introduction_style = style.add_style(bg_color:
|
63
|
-
header_style = style.add_style(b: true, bg_color:
|
64
|
-
header_required_style = style.add_style(b: true, bg_color:
|
64
|
+
introduction_style = style.add_style(bg_color: "E2EEDA")
|
65
|
+
header_style = style.add_style(b: true, bg_color: "A8D08E", border: {style: :thin, color: "000000"})
|
66
|
+
header_required_style = style.add_style(b: true, bg_color: "A8D08E", fg_color: "C00100", border: {style: :thin, color: "000000"})
|
65
67
|
|
66
68
|
# Introduction
|
67
69
|
introduction.each_with_index do |intro, i|
|
@@ -75,14 +77,19 @@ module Exportable
|
|
75
77
|
|
76
78
|
export_columns.each.with_index do |f, i|
|
77
79
|
field = f.last
|
78
|
-
sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author:
|
80
|
+
sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author: "", text: field.hint, visible: false if field.hint.present?
|
81
|
+
end
|
82
|
+
styles = export_columns.map do |_, c|
|
83
|
+
if c.options.dig(:export, :format) == "number" || (c.options.dig(:export, :format).nil? && c.options.dig(:export, :example).is_a?(Numeric))
|
84
|
+
number = workbook.styles.add_style format_code: "#"
|
85
|
+
elsif c.options.dig(:export, :format) == "text" || (c.options.dig(:export, :format).nil? && c.options.dig(:export, :example).is_a?(String))
|
86
|
+
text = workbook.styles.add_style format_code: "@"
|
87
|
+
elsif c.options.dig(:export, :format)
|
88
|
+
workbook.styles.add_style format_code: c.options.dig(:export, :format).to_s
|
89
|
+
else
|
90
|
+
workbook.styles.add_style format_code: "General"
|
91
|
+
end
|
79
92
|
end
|
80
|
-
|
81
|
-
number = workbook.styles.add_style format_code: '#'
|
82
|
-
text = workbook.styles.add_style format_code: '@'
|
83
|
-
|
84
|
-
styles = export_columns.map { |_, c| c.options[:example].is_a?(Numeric) ? number : text }
|
85
|
-
|
86
93
|
# Examples
|
87
94
|
data_rows.each do |data|
|
88
95
|
sheet.add_row data, style: styles
|
@@ -91,15 +98,15 @@ module Exportable
|
|
91
98
|
|
92
99
|
sheet.column_info[0].width = 10
|
93
100
|
|
94
|
-
sheet = workbook.add_worksheet(name: I18n.t(
|
101
|
+
sheet = workbook.add_worksheet(name: I18n.t("importo.sheet.explanation.name"))
|
95
102
|
|
96
103
|
workbook.styles do |style|
|
97
|
-
introduction_style = style.add_style(bg_color:
|
98
|
-
header_style = style.add_style(b: true, bg_color:
|
104
|
+
introduction_style = style.add_style(bg_color: "E2EEDA")
|
105
|
+
header_style = style.add_style(b: true, bg_color: "A8D08E", border: {style: :thin, color: "000000"})
|
99
106
|
|
100
107
|
column_style = style.add_style(b: true)
|
101
|
-
required_style = style.add_style(b: true, fg_color:
|
102
|
-
wrap_style = workbook.styles.add_style alignment: {
|
108
|
+
required_style = style.add_style(b: true, fg_color: "C00100")
|
109
|
+
wrap_style = workbook.styles.add_style alignment: {wrap_text: true}
|
103
110
|
|
104
111
|
# Introduction
|
105
112
|
introduction.each_with_index do |intro, i|
|
@@ -109,10 +116,10 @@ module Exportable
|
|
109
116
|
end
|
110
117
|
|
111
118
|
# Header row
|
112
|
-
sheet.add_row [I18n.t(
|
119
|
+
sheet.add_row [I18n.t("importo.sheet.explanation.column"), I18n.t("importo.sheet.explanation.value"), I18n.t("importo.sheet.explanation.explanation"), I18n.t("importo.sheet.explanation.example")], style: [header_style] * 4
|
113
120
|
export_columns.each do |_, c|
|
114
121
|
styles = [c.options[:required] ? required_style : column_style, wrap_style]
|
115
|
-
sheet.add_row [c.name, c.explanation], style: styles
|
122
|
+
sheet.add_row [c.name, c.value, c.explanation, c.example], style: styles
|
116
123
|
end
|
117
124
|
end
|
118
125
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "active_support/concern"
|
4
4
|
|
5
5
|
module Importable
|
6
6
|
extend ActiveSupport::Concern
|
@@ -14,15 +14,13 @@ module Importable
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def convert_values(row)
|
17
|
-
return row if row.instance_variable_get(
|
17
|
+
return row if row.instance_variable_get(:@importo_converted_values)
|
18
18
|
|
19
|
-
row.instance_variable_set(
|
19
|
+
row.instance_variable_set(:@importo_converted_values, true)
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
end
|
21
|
+
columns.each do |k, col|
|
22
|
+
next if col.proc.blank? || row[k].nil?
|
24
23
|
|
25
|
-
cols_to_populate.each do |k, col|
|
26
24
|
attr = col.options[:attribute]
|
27
25
|
|
28
26
|
row[k] = import.column_overrides[col.attribute] if import.column_overrides[col.attribute]
|
@@ -52,26 +50,36 @@ module Importable
|
|
52
50
|
# It wil try and find the record by id, or initialize a new record with it's attributes set based on the mapping from columns
|
53
51
|
#
|
54
52
|
def populate(row, record = nil)
|
55
|
-
raise
|
53
|
+
raise "No attributes set for columns" unless columns.any? { |_, v| v.options[:attribute].present? }
|
56
54
|
|
57
55
|
row = convert_values(row)
|
58
56
|
|
59
57
|
result = if record
|
60
|
-
|
61
|
-
|
62
|
-
|
58
|
+
record
|
59
|
+
else
|
60
|
+
raise "No model set" unless model
|
63
61
|
|
64
|
-
|
65
|
-
|
62
|
+
model.find_or_initialize_by(id: row["id"])
|
63
|
+
end
|
66
64
|
|
67
65
|
attributes = {}
|
68
66
|
cols_to_populate = columns.select do |_, v|
|
69
67
|
v.options[:attribute].present?
|
70
68
|
end
|
71
69
|
|
70
|
+
cols_to_populate = cols_to_populate.deep_symbolize_keys
|
71
|
+
|
72
72
|
cols_to_populate.each do |k, col|
|
73
73
|
attr = col.options[:attribute]
|
74
|
-
|
74
|
+
|
75
|
+
next unless row.key? k
|
76
|
+
next if !row[k].present? && col.options[:default].nil?
|
77
|
+
|
78
|
+
attributes = if !row[k].present? && !col.options[:default].nil?
|
79
|
+
set_attribute(attributes, attr, col.options[:default])
|
80
|
+
else
|
81
|
+
set_attribute(attributes, attr, row[k])
|
82
|
+
end
|
75
83
|
end
|
76
84
|
|
77
85
|
result.assign_attributes(attributes)
|
@@ -80,89 +88,163 @@ module Importable
|
|
80
88
|
end
|
81
89
|
|
82
90
|
#
|
83
|
-
#
|
91
|
+
# Callbakcs
|
84
92
|
#
|
93
|
+
def before_build(_record, _row)
|
94
|
+
end
|
95
|
+
|
96
|
+
def around_build(_record, _row)
|
97
|
+
yield
|
98
|
+
end
|
99
|
+
|
100
|
+
def after_build(_record, _row)
|
101
|
+
end
|
102
|
+
|
85
103
|
def before_save(_record, _row)
|
86
|
-
# Implement optionally in child class to mangle
|
87
104
|
end
|
88
105
|
|
89
|
-
|
90
|
-
|
91
|
-
|
106
|
+
def around_save(_record, _row)
|
107
|
+
yield
|
108
|
+
end
|
109
|
+
|
92
110
|
def after_save(_record, _row)
|
93
|
-
|
111
|
+
end
|
112
|
+
|
113
|
+
def before_validate(_record, _row)
|
114
|
+
end
|
115
|
+
|
116
|
+
def around_validate(_record, _row)
|
117
|
+
yield
|
118
|
+
end
|
119
|
+
|
120
|
+
def after_validate(_record, _row)
|
94
121
|
end
|
95
122
|
|
96
123
|
#
|
97
124
|
# Does the actual import
|
98
125
|
#
|
99
126
|
def import!
|
100
|
-
raise ArgumentError,
|
101
|
-
|
102
|
-
|
103
|
-
|
127
|
+
raise ArgumentError, "Invalid data structure" unless structure_valid?
|
128
|
+
|
129
|
+
batch = Sidekiq::Batch.new
|
130
|
+
batch.description = "#{import.original.filename} - #{import.kind}"
|
131
|
+
batch.on(:success, Importo::ImportJobCallback, import_id: import.id)
|
132
|
+
|
133
|
+
batch.jobs do
|
134
|
+
column_with_delay = columns.select { |k, v| v.delay.present? }
|
135
|
+
loop_data_rows do |attributes, index|
|
136
|
+
if column_with_delay.present?
|
137
|
+
delay = column_with_delay.map do |k, v|
|
138
|
+
next unless attributes[k].present?
|
139
|
+
v.delay.call(attributes[k])
|
140
|
+
end.compact
|
141
|
+
end
|
142
|
+
Importo::ImportJob.set(wait_until: (delay.max * index).seconds.from_now).perform_async(JSON.dump(attributes), index, import.id) if delay.present?
|
143
|
+
Importo::ImportJob.perform_async(JSON.dump(attributes), index, import.id) unless delay.present?
|
144
|
+
end
|
104
145
|
end
|
105
|
-
@import.result.attach(io: results_file, filename: @import.importer.file_name('results'),
|
106
|
-
content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
|
107
|
-
|
108
|
-
@import.result_message = I18n.t('importo.importers.result_message', nr: results.compact.count, of: results.count,
|
109
|
-
start_row: data_start_row)
|
110
|
-
@import.complete!
|
111
146
|
|
112
147
|
true
|
113
|
-
rescue
|
148
|
+
rescue => e
|
114
149
|
@import.result_message = "Exception: #{e.message}"
|
115
|
-
Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(
|
150
|
+
Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(";")}"
|
116
151
|
@import.failure!
|
117
152
|
|
118
153
|
false
|
119
154
|
end
|
120
155
|
|
121
|
-
|
122
|
-
|
123
|
-
##
|
124
|
-
# Overridable failure method
|
125
|
-
#
|
126
|
-
def failure(_row, _record, index, exception)
|
127
|
-
Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(';')}"
|
128
|
-
end
|
129
|
-
|
130
|
-
def process_data_row(attributes, index)
|
156
|
+
def process_data_row(attributes, index, last_attempt: true)
|
131
157
|
record = nil
|
132
158
|
row_hash = Digest::SHA256.base64digest(attributes.inspect)
|
133
159
|
duplicate_import = nil
|
134
160
|
|
135
|
-
|
136
|
-
|
161
|
+
run_callbacks :row_import do
|
162
|
+
record = nil
|
163
|
+
|
164
|
+
ActiveRecord::Base.transaction do
|
137
165
|
register_result(index, hash: row_hash, state: :processing)
|
138
166
|
|
139
|
-
record
|
140
|
-
record
|
141
|
-
|
142
|
-
|
143
|
-
|
167
|
+
before_build(record, attributes)
|
168
|
+
around_build(record, attributes) do
|
169
|
+
record = build(attributes)
|
170
|
+
end
|
171
|
+
after_build(record, attributes)
|
172
|
+
|
173
|
+
before_validate(record, attributes)
|
174
|
+
around_validate(record, attributes) do
|
175
|
+
record.validate!
|
176
|
+
end
|
177
|
+
after_validate(record, attributes)
|
178
|
+
|
179
|
+
model.with_advisory_lock(:importo) do
|
180
|
+
before_save(record, attributes)
|
181
|
+
around_save(record, attributes) do
|
182
|
+
record.save!
|
183
|
+
end
|
184
|
+
after_save(record, attributes)
|
185
|
+
end
|
186
|
+
|
144
187
|
duplicate_import = duplicate?(row_hash, record.id)
|
145
188
|
raise Importo::DuplicateRowError if duplicate_import
|
146
189
|
|
147
190
|
register_result(index, class: record.class.name, id: record.id, state: :success)
|
148
191
|
end
|
149
|
-
record
|
150
|
-
rescue Importo::DuplicateRowError
|
151
|
-
record_id = duplicate_import.results.find { |data| data['hash'] == row_hash }['id']
|
152
|
-
register_result(index, id: record_id, state: :duplicate,
|
153
|
-
message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
|
154
|
-
nil
|
155
|
-
rescue StandardError => e
|
156
|
-
errors = record.respond_to?(:errors) && record.errors.full_messages.join(', ')
|
157
|
-
error_message = "#{e.message} (#{e.backtrace.first.split('/').last})"
|
158
|
-
failure(attributes, record, index, e)
|
159
|
-
register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
|
160
|
-
nil
|
161
192
|
end
|
193
|
+
|
194
|
+
record
|
195
|
+
rescue Importo::DuplicateRowError
|
196
|
+
record_id = duplicate_import.results.find { |data| data["hash"] == row_hash }["id"]
|
197
|
+
register_result(index, id: record_id, state: :duplicate,
|
198
|
+
message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
|
199
|
+
|
200
|
+
run_callbacks(:row_import, :after)
|
201
|
+
nil
|
202
|
+
rescue Importo::RetryError => e
|
203
|
+
raise e unless last_attempt
|
204
|
+
|
205
|
+
errors = record.respond_to?(:errors) && record.errors.full_messages.join(", ")
|
206
|
+
error_message = "#{e.message} (#{e.backtrace.first.split("/").last})"
|
207
|
+
failure(attributes, record, index, e)
|
208
|
+
register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
|
209
|
+
nil
|
210
|
+
rescue => e
|
211
|
+
# We rescue ActiveRecord::RecordNotUnique here, due to how transactions work, some row imports may have started the transaction and the current row doesn't see the results
|
212
|
+
raise Importo::RetryError.new("ActiveRecord::RecordNotUnique", 2) if !last_attempt && e.is_a?(ActiveRecord::RecordNotUnique)
|
213
|
+
errors = record.respond_to?(:errors) && record.errors.full_messages.join(", ")
|
214
|
+
error_message = "#{e.message} (#{e.backtrace.first.split("/").last})"
|
215
|
+
failure(attributes, record, index, e)
|
216
|
+
register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
|
217
|
+
run_callbacks(:row_import, :after)
|
218
|
+
nil
|
219
|
+
end
|
220
|
+
|
221
|
+
private
|
222
|
+
|
223
|
+
##
|
224
|
+
# Overridable failure method
|
225
|
+
#
|
226
|
+
def failure(_row, _record, index, exception)
|
227
|
+
Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(";")}"
|
162
228
|
end
|
163
229
|
|
164
230
|
def set_attribute(hash, path, value)
|
165
|
-
tmp_hash = path.to_s.split(
|
231
|
+
tmp_hash = path.to_s.split(".").reverse.inject(value) { |h, s| {s => h} }
|
166
232
|
hash.deep_merge(tmp_hash)
|
167
233
|
end
|
234
|
+
|
235
|
+
# Only when the block returns a record created in this import, it returns that record, otherwise nil
|
236
|
+
#
|
237
|
+
# record ||= only_current_import! do
|
238
|
+
# User.find_by(email: row[:email].downcase) if row[:email].present?
|
239
|
+
# end
|
240
|
+
#
|
241
|
+
# Only if the user was created this import will the block return the user found.
|
242
|
+
#
|
243
|
+
# @return [Object]
|
244
|
+
def only_current_import!(&block)
|
245
|
+
ActiveRecord::Base.uncached do
|
246
|
+
record = yield
|
247
|
+
record if record && @import.results.where("details->>'state' = ?", "success").where("details->>'id' = ? ", record.id).exists?
|
248
|
+
end
|
249
|
+
end
|
168
250
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "active_support/concern"
|
4
4
|
|
5
5
|
module ImporterDsl
|
6
6
|
extend ActiveSupport::Concern
|
@@ -31,19 +31,11 @@ module ImporterDsl
|
|
31
31
|
#
|
32
32
|
# @param [Object] args
|
33
33
|
# @param [Object] block which will filter the results before storing the value in the attribute, this is useful for lookups or reformatting
|
34
|
-
def column(
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
name
|
39
|
-
name ||= options[:attribute]
|
40
|
-
|
41
|
-
hint = args[1]
|
42
|
-
hint ||= options[:hint]
|
43
|
-
|
44
|
-
options[:scope] = self.name.underscore.to_s.tr('/', '.').to_sym
|
45
|
-
|
46
|
-
columns[name] = Importo::ImportColumn.new(name, hint, options[:explanation], options, &block)
|
34
|
+
def column(**options, &block)
|
35
|
+
column_name ||= options[:name]
|
36
|
+
column_name ||= options[:attribute]
|
37
|
+
options[:scope] = name.to_s.underscore.to_s.tr("/", ".").to_sym
|
38
|
+
columns[column_name] = Importo::ImportColumn.new(name: column_name, **options, &block)
|
47
39
|
end
|
48
40
|
|
49
41
|
def model(model = nil)
|