importo 2.0.5 → 3.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +22 -5
  3. data/Rakefile +16 -17
  4. data/app/controllers/concerns/maintenance_standards.rb +1 -1
  5. data/app/controllers/importo/imports_controller.rb +15 -12
  6. data/app/helpers/importo/application_helper.rb +11 -11
  7. data/app/importers/concerns/exportable.rb +31 -24
  8. data/app/importers/concerns/importable.rb +144 -62
  9. data/app/importers/concerns/importer_dsl.rb +6 -14
  10. data/app/importers/concerns/original.rb +23 -23
  11. data/app/importers/concerns/result_feedback.rb +62 -44
  12. data/app/importers/concerns/revertable.rb +11 -12
  13. data/app/importers/importo/base_importer.rb +16 -4
  14. data/app/importers/importo/import_job_callback.rb +29 -0
  15. data/app/jobs/importo/import_job.rb +40 -0
  16. data/app/mailers/importo/application_mailer.rb +2 -2
  17. data/app/models/importo/import.rb +35 -8
  18. data/app/models/importo/result.rb +5 -0
  19. data/app/services/importo/import_service.rb +1 -3
  20. data/app/services/importo/revert_service.rb +1 -1
  21. data/app/tables/importo/imports_table.rb +13 -17
  22. data/app/tables/importo/mensa_imports_table.rb +35 -0
  23. data/app/views/importo/imports/index.html.slim +6 -2
  24. data/app/views/importo/imports/new.html.slim +30 -20
  25. data/config/locales/en.yml +21 -9
  26. data/config/locales/nl.yml +10 -4
  27. data/config/routes.rb +4 -4
  28. data/db/migrate/20180409151031_create_importo_import.rb +9 -9
  29. data/db/migrate/20180628175535_add_locale_importo_import.rb +1 -1
  30. data/db/migrate/20230510051447_remove_result_from_imports.rb +5 -0
  31. data/db/migrate/20230510083043_create_importo_results.rb +11 -0
  32. data/lib/generators/importo/importer_generator.rb +1 -1
  33. data/lib/generators/importo/install_generator.rb +7 -8
  34. data/lib/generators/importo/templates/importo.rb +11 -0
  35. data/lib/generators/satis/install_generator.rb +22 -0
  36. data/lib/generators/satis/tailwind_config_generator.rb +24 -0
  37. data/lib/generators/satis/templates/config/initializers/satis.rb +13 -0
  38. data/lib/importo/acts_as_import_owner.rb +1 -1
  39. data/lib/importo/configuration.rb +52 -49
  40. data/lib/importo/engine.rb +11 -9
  41. data/lib/importo/import_column.rb +18 -6
  42. data/lib/importo/test_helpers.rb +19 -0
  43. data/lib/importo/version.rb +1 -1
  44. data/lib/importo.rb +12 -17
  45. metadata +75 -24
  46. data/app/services/importo/callback_service.rb +0 -14
  47. data/lib/generators/templates/importo.rb +0 -21
  48. /data/lib/generators/{templates → importo/templates}/README +0 -0
  49. /data/lib/generators/{templates → importo/templates}/application_importer.rb +0 -0
  50. /data/lib/generators/{templates → importo/templates}/importer.rb +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6005b4d69612846b48c464ea6dfe53d763498518201576610e26e5ee7b99a1a9
4
- data.tar.gz: bf6b4711e93260528cb440217588155dd815874666e7296715e2878559153001
3
+ metadata.gz: 208e74497d020c5798661a0c0280d0952ab0a2381ede5317e3460f57b6ea8aa2
4
+ data.tar.gz: 78d02a8842863216adc84411c2ee71bd16202f40b7c50734248cf35aaa76a244
5
5
  SHA512:
6
- metadata.gz: 3728c3639169ec860471b1a80f3b293c4ee987748f0004f26978a9f716ef1b9ad69ac48bb49a7cc2a42b90083ca89322100aab6ba3164b329accdb3a036a19d2
7
- data.tar.gz: 047d674853e4facae068811e41579bc3c01af12ef9680c06151a80816bc8d0639e809607763d9be76d96cfb00a392236d623186887b22716ea2d0e63fabd81d8
6
+ metadata.gz: d954737dd202b1e068fdcae143e06f49a3cb7515d04b7da85bc0111c80cbeed7f384a6bf3ba968a0a742bfe76efbc6d554af9bc57b01777be3756968b8ae8e88
7
+ data.tar.gz: 8796b9d5746d0995e8cff516f30eb1662598ee3d34f5c4fd32c19d0604e379ffa79cae747a15b4502e4b0ecef59eed0662fda3bd42f0d4d0dee5fdd030c93fe1
data/README.md CHANGED
@@ -26,9 +26,11 @@ class ProductsImporter < ApplicationImporter
26
26
  column attribute: :id
27
27
 
28
28
  # attributes
29
- column attribute: :name
30
- column attribute: :number
29
+ column attribute: :name, required: true
31
30
  column attribute: :description, strip_tags: false
31
+ column attribute: :number, export: { format: 'text', value: ->(record) { record.number }, example: 'FLAG-NLD-001' }, style: {b: true}
32
+ column attribute: :expires_on, export: { format: 'dd/mm/yyyy h:mm'}
33
+ column name: :price, export: { format: 'number', value: ->(record) { record.price } }
32
34
  column attribute: :images do |value|
33
35
  value.split(',').map do |image|
34
36
  uri = URI.parse(image)
@@ -36,12 +38,14 @@ class ProductsImporter < ApplicationImporter
36
38
  { filename: File.basename(uri.to_s), io: URI.open(uri) }
37
39
  end
38
40
  end
41
+ column name: :kitting_component_product, delay: ->(value) {value.present? ? 5 : 0 }
39
42
 
40
43
  def export_scope
41
44
  Current.account.products
42
45
  end
43
46
  end
44
47
  ```
48
+ export args for column is optional, format takes excel custom format codes default is General
45
49
 
46
50
  You should add translations to your locale files:
47
51
 
@@ -62,12 +66,22 @@ en:
62
66
  number: Number
63
67
  description: Description
64
68
  images: Images
65
- explanation:
66
- id: Record-id, only needed if you want to update an existing record
69
+ kitting_component_product: Component Product
70
+ # Shown in note in import sheet
67
71
  hint:
68
72
  id: 36 characters, existing of hexadecimal numbers, separated by dashes
69
73
  images: Allows multiple image urls, separated by comma
70
- introduction: null
74
+ # Below items are show in explanation sheet
75
+ explanation:
76
+ id: Record-id, only needed if you want to update an existing record
77
+ example:
78
+ id: 12345678-1234-1234-1234-123456789012
79
+ name: TEST-123
80
+ number: TEST-123
81
+ description: Test product
82
+ value:
83
+ id: Optional
84
+
71
85
  ```
72
86
 
73
87
  ## Installation
@@ -78,6 +92,9 @@ Add this line to your application's Gemfile:
78
92
  gem 'importo'
79
93
  ```
80
94
 
95
+ Importo depends on Sidekiq Pro's batch functionality,
96
+ though you can use [sidekiq-batch](https://github.com/entdec/sidekiq-batch) as a drop-in for that.
97
+
81
98
  And then execute:
82
99
 
83
100
  ```bash
data/Rakefile CHANGED
@@ -1,39 +1,38 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  begin
4
- require 'bundler/setup'
4
+ require "bundler/setup"
5
5
  rescue LoadError
6
- puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
6
+ puts "You must `gem install bundler` and `bundle install` to run rake tasks"
7
7
  end
8
8
 
9
- require 'rdoc/task'
9
+ require "rdoc/task"
10
10
 
11
11
  RDoc::Task.new(:rdoc) do |rdoc|
12
- rdoc.rdoc_dir = 'rdoc'
13
- rdoc.title = 'Importo'
14
- rdoc.options << '--line-numbers'
15
- rdoc.rdoc_files.include('README.md')
16
- rdoc.rdoc_files.include('lib/**/*.rb')
12
+ rdoc.rdoc_dir = "rdoc"
13
+ rdoc.title = "Importo"
14
+ rdoc.options << "--line-numbers"
15
+ rdoc.rdoc_files.include("README.md")
16
+ rdoc.rdoc_files.include("lib/**/*.rb")
17
17
  end
18
18
 
19
- APP_RAKEFILE = File.expand_path('test/dummy/Rakefile', __dir__)
20
- load 'rails/tasks/engine.rake'
19
+ APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
20
+ load "rails/tasks/engine.rake"
21
21
 
22
- load 'rails/tasks/statistics.rake'
22
+ load "rails/tasks/statistics.rake"
23
23
 
24
- require 'bundler/gem_tasks'
24
+ require "bundler/gem_tasks"
25
25
 
26
- require 'rake/testtask'
26
+ require "rake/testtask"
27
27
 
28
28
  Rake::TestTask.new(:test) do |t|
29
- t.libs << 'test'
30
- t.pattern = 'test/**/*_test.rb'
29
+ t.libs << "test"
30
+ t.pattern = "test/**/*_test.rb"
31
31
  t.verbose = false
32
32
  end
33
33
 
34
34
  task default: :test
35
35
 
36
36
  # Adds the Auxilium semver task
37
- spec = Gem::Specification.find_by_name 'auxilium'
37
+ spec = Gem::Specification.find_by_name "auxilium"
38
38
  load "#{spec.gem_dir}/lib/tasks/semver.rake"
39
-
@@ -11,7 +11,7 @@ module MaintenanceStandards
11
11
  #
12
12
  def flash_and_redirect(result, path, notice, error, render_action = :edit)
13
13
  if result
14
- if params[:commit] == 'continue'
14
+ if params[:commit] == "continue"
15
15
  flash.now[:notice] = notice
16
16
  else
17
17
  redirect_to(path, notice: notice) && return
@@ -11,22 +11,22 @@ module Importo
11
11
  def create
12
12
  unless import_params
13
13
  @import = Import.new(kind: params[:kind], locale: I18n.locale)
14
- flash[:error] = t('.flash.no_file')
14
+ Signum.error(Current.user, text: t('.flash.no_file'))
15
15
  render :new
16
16
  return
17
17
  end
18
- @import = Import.new(import_params.merge(locale: I18n.locale, importo_ownable: Importo.config.current_import_owner))
18
+ @import = Import.new(import_params.merge(locale: I18n.locale,
19
+ importo_ownable: Importo.config.current_import_owner.call))
19
20
  if @import.valid? && @import.schedule!
20
- flash[:notice] = t('.flash.success', id: @import.id)
21
- redirect_to action: :index
21
+ redirect_to importo.new_import_path(params[:kind] || @import.kind)
22
22
  else
23
- flash[:error] = t('.flash.error')
23
+ Signum.error(Current.user, text: t('.flash.error', error: @import.errors&.full_messages&.join('.')))
24
24
  render :new
25
25
  end
26
26
  end
27
27
 
28
28
  def undo
29
- @import = Import.where(importo_ownable: Importo.config.current_import_owner).find(params[:id])
29
+ @import = Import.where(importo_ownable: Importo.config.current_import_owner.call).find(params[:id])
30
30
  if @import.can_revert? && @import.revert
31
31
  redirect_to action: :index, notice: 'Import reverted'
32
32
  else
@@ -35,8 +35,8 @@ module Importo
35
35
  end
36
36
 
37
37
  def destroy
38
- @import = Import.where(importo_ownable: Importo.config.current_import_owner).find(params[:id])
39
- redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy(@import)
38
+ @import = Import.find(params[:id])
39
+ redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy.call(@import)
40
40
 
41
41
  @import.destroy
42
42
  redirect_to action: :index
@@ -44,22 +44,25 @@ module Importo
44
44
 
45
45
  def sample
46
46
  import = Import.new(kind: params[:kind], locale: I18n.locale)
47
- send_data import.importer.sample_file.read, type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('sample')
47
+ send_data import.importer.sample_file.read,
48
+ type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('sample')
48
49
  end
49
50
 
50
51
  def export
51
52
  import = Import.new(kind: params[:kind], locale: I18n.locale)
52
- send_data import.importer.export_file.read, type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('export')
53
+ send_data import.importer.export_file.read,
54
+ type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('export')
53
55
  end
54
56
 
55
57
  def index
56
- @imports = Importo.config.admin_visible_imports.order(created_at: :desc).limit(50)
58
+ @imports = Importo.config.admin_visible_imports.call.order(created_at: :desc).limit(50)
57
59
  end
58
60
 
59
61
  private
60
62
 
61
63
  def import_params
62
- params.require(:import).permit(:original, :kind, :column_overrides, column_overrides: params.dig(:import, :column_overrides)&.keys)
64
+ params.require(:import).permit(:original, :kind, :column_overrides,
65
+ column_overrides: params.dig(:import, :column_overrides)&.keys)
63
66
  end
64
67
  end
65
68
  end
@@ -2,16 +2,16 @@
2
2
 
3
3
  module Importo
4
4
  module ApplicationHelper
5
- def respond_to_missing?(method)
6
- method.ends_with?('_url') || method.ends_with?('_path')
7
- end
8
-
9
- def method_missing(method, *args, &block)
10
- if main_app.respond_to?(method)
11
- main_app.send(method, *args)
12
- else
13
- super
14
- end
15
- end
5
+ # def respond_to_missing?(method)
6
+ # method.ends_with?('_url') || method.ends_with?('_path')
7
+ # end
8
+ #
9
+ # def method_missing(method, *args, &block)
10
+ # if main_app.respond_to?(method)
11
+ # main_app.send(method, *args)
12
+ # else
13
+ # super
14
+ # end
15
+ # end
16
16
  end
17
17
  end
@@ -1,7 +1,6 @@
1
-
2
1
  # frozen_string_literal: true
3
2
 
4
- require 'active_support/concern'
3
+ require "active_support/concern"
5
4
 
6
5
  module Exportable
7
6
  extend ActiveSupport::Concern
@@ -14,7 +13,9 @@ module Exportable
14
13
  end
15
14
 
16
15
  def sample_data
17
- [export_columns.map { |_, c| c.options[:example] || '' }]
16
+ sample_data = []
17
+ 100.times { sample_data << export_columns.map { |_, c| c.options[:example] || "" } }
18
+ sample_data
18
19
  end
19
20
 
20
21
  #
@@ -38,15 +39,16 @@ module Exportable
38
39
 
39
40
  def export_row(record)
40
41
  export_columns.map do |_, c|
41
- value = ''
42
+ value = ""
42
43
 
43
44
  if c.options[:attribute]
44
- if record.respond_to?(c.options[:attribute])
45
+ if c.options[:export]&.key?(:value)
46
+ value = c.options[:export][:value].call(record) if c.options[:export][:value].is_a?(Proc)
47
+ elsif record.respond_to?(c.options[:attribute])
45
48
  value = record.send(c.options[:attribute])
46
49
  value = value&.body&.to_html if value.is_a?(ActionText::RichText)
47
50
  end
48
-
49
- value ||= record.attributes[c.options[:attribute].to_s]
51
+ value ||= record.attributes[c.options[:attribute].to_s] if value.nil?
50
52
  end
51
53
 
52
54
  value
@@ -59,9 +61,9 @@ module Exportable
59
61
  workbook = xls.workbook
60
62
  sheet = workbook.add_worksheet(name: friendly_name&.pluralize || model.name.demodulize.pluralize)
61
63
  workbook.styles do |style|
62
- introduction_style = style.add_style(bg_color: 'E2EEDA')
63
- header_style = style.add_style(b: true, bg_color: 'A8D08E', border: { style: :thin, color: '000000' })
64
- header_required_style = style.add_style(b: true, bg_color: 'A8D08E', fg_color: 'C00100', border: { style: :thin, color: '000000' })
64
+ introduction_style = style.add_style(bg_color: "E2EEDA")
65
+ header_style = style.add_style(b: true, bg_color: "A8D08E", border: {style: :thin, color: "000000"})
66
+ header_required_style = style.add_style(b: true, bg_color: "A8D08E", fg_color: "C00100", border: {style: :thin, color: "000000"})
65
67
 
66
68
  # Introduction
67
69
  introduction.each_with_index do |intro, i|
@@ -75,14 +77,19 @@ module Exportable
75
77
 
76
78
  export_columns.each.with_index do |f, i|
77
79
  field = f.last
78
- sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author: '', text: field.hint, visible: false if field.hint.present?
80
+ sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author: "", text: field.hint, visible: false if field.hint.present?
81
+ end
82
+ styles = export_columns.map do |_, c|
83
+ if c.options.dig(:export, :format) == "number" || (c.options.dig(:export, :format).nil? && c.options.dig(:export, :example).is_a?(Numeric))
84
+ number = workbook.styles.add_style format_code: "#"
85
+ elsif c.options.dig(:export, :format) == "text" || (c.options.dig(:export, :format).nil? && c.options.dig(:export, :example).is_a?(String))
86
+ text = workbook.styles.add_style format_code: "@"
87
+ elsif c.options.dig(:export, :format)
88
+ workbook.styles.add_style format_code: c.options.dig(:export, :format).to_s
89
+ else
90
+ workbook.styles.add_style format_code: "General"
91
+ end
79
92
  end
80
-
81
- number = workbook.styles.add_style format_code: '#'
82
- text = workbook.styles.add_style format_code: '@'
83
-
84
- styles = export_columns.map { |_, c| c.options[:example].is_a?(Numeric) ? number : text }
85
-
86
93
  # Examples
87
94
  data_rows.each do |data|
88
95
  sheet.add_row data, style: styles
@@ -91,15 +98,15 @@ module Exportable
91
98
 
92
99
  sheet.column_info[0].width = 10
93
100
 
94
- sheet = workbook.add_worksheet(name: I18n.t('importo.sheet.explanation.name'))
101
+ sheet = workbook.add_worksheet(name: I18n.t("importo.sheet.explanation.name"))
95
102
 
96
103
  workbook.styles do |style|
97
- introduction_style = style.add_style(bg_color: 'E2EEDA')
98
- header_style = style.add_style(b: true, bg_color: 'A8D08E', border: { style: :thin, color: '000000' })
104
+ introduction_style = style.add_style(bg_color: "E2EEDA")
105
+ header_style = style.add_style(b: true, bg_color: "A8D08E", border: {style: :thin, color: "000000"})
99
106
 
100
107
  column_style = style.add_style(b: true)
101
- required_style = style.add_style(b: true, fg_color: 'C00100')
102
- wrap_style = workbook.styles.add_style alignment: { wrap_text: true }
108
+ required_style = style.add_style(b: true, fg_color: "C00100")
109
+ wrap_style = workbook.styles.add_style alignment: {wrap_text: true}
103
110
 
104
111
  # Introduction
105
112
  introduction.each_with_index do |intro, i|
@@ -109,10 +116,10 @@ module Exportable
109
116
  end
110
117
 
111
118
  # Header row
112
- sheet.add_row [I18n.t('importo.sheet.explanation.column'), I18n.t('importo.sheet.explanation.explanation')], style: [header_style] * 2
119
+ sheet.add_row [I18n.t("importo.sheet.explanation.column"), I18n.t("importo.sheet.explanation.value"), I18n.t("importo.sheet.explanation.explanation"), I18n.t("importo.sheet.explanation.example")], style: [header_style] * 4
113
120
  export_columns.each do |_, c|
114
121
  styles = [c.options[:required] ? required_style : column_style, wrap_style]
115
- sheet.add_row [c.name, c.explanation], style: styles
122
+ sheet.add_row [c.name, c.value, c.explanation, c.example], style: styles
116
123
  end
117
124
  end
118
125
 
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'active_support/concern'
3
+ require "active_support/concern"
4
4
 
5
5
  module Importable
6
6
  extend ActiveSupport::Concern
@@ -14,15 +14,13 @@ module Importable
14
14
  end
15
15
 
16
16
  def convert_values(row)
17
- return row if row.instance_variable_get('@importo_converted_values')
17
+ return row if row.instance_variable_get(:@importo_converted_values)
18
18
 
19
- row.instance_variable_set('@importo_converted_values', true)
19
+ row.instance_variable_set(:@importo_converted_values, true)
20
20
 
21
- cols_to_populate = columns.select do |_, v|
22
- v.options[:attribute].present?
23
- end
21
+ columns.each do |k, col|
22
+ next if col.proc.blank? || row[k].nil?
24
23
 
25
- cols_to_populate.each do |k, col|
26
24
  attr = col.options[:attribute]
27
25
 
28
26
  row[k] = import.column_overrides[col.attribute] if import.column_overrides[col.attribute]
@@ -52,26 +50,36 @@ module Importable
52
50
  # It wil try and find the record by id, or initialize a new record with it's attributes set based on the mapping from columns
53
51
  #
54
52
  def populate(row, record = nil)
55
- raise 'No attributes set for columns' unless columns.any? { |_, v| v.options[:attribute].present? }
53
+ raise "No attributes set for columns" unless columns.any? { |_, v| v.options[:attribute].present? }
56
54
 
57
55
  row = convert_values(row)
58
56
 
59
57
  result = if record
60
- record
61
- else
62
- raise 'No model set' unless model
58
+ record
59
+ else
60
+ raise "No model set" unless model
63
61
 
64
- model.find_or_initialize_by(id: row['id'])
65
- end
62
+ model.find_or_initialize_by(id: row["id"])
63
+ end
66
64
 
67
65
  attributes = {}
68
66
  cols_to_populate = columns.select do |_, v|
69
67
  v.options[:attribute].present?
70
68
  end
71
69
 
70
+ cols_to_populate = cols_to_populate.deep_symbolize_keys
71
+
72
72
  cols_to_populate.each do |k, col|
73
73
  attr = col.options[:attribute]
74
- attributes = set_attribute(attributes, attr, row[k]) if row[k].present?
74
+
75
+ next unless row.key? k
76
+ next if !row[k].present? && col.options[:default].nil?
77
+
78
+ attributes = if !row[k].present? && !col.options[:default].nil?
79
+ set_attribute(attributes, attr, col.options[:default])
80
+ else
81
+ set_attribute(attributes, attr, row[k])
82
+ end
75
83
  end
76
84
 
77
85
  result.assign_attributes(attributes)
@@ -80,89 +88,163 @@ module Importable
80
88
  end
81
89
 
82
90
  #
83
- # Mangle the record before saving
91
+ # Callbakcs
84
92
  #
93
+ def before_build(_record, _row)
94
+ end
95
+
96
+ def around_build(_record, _row)
97
+ yield
98
+ end
99
+
100
+ def after_build(_record, _row)
101
+ end
102
+
85
103
  def before_save(_record, _row)
86
- # Implement optionally in child class to mangle
87
104
  end
88
105
 
89
- #
90
- # Any updates that have to be done after saving
91
- #
106
+ def around_save(_record, _row)
107
+ yield
108
+ end
109
+
92
110
  def after_save(_record, _row)
93
- # Implement optionally in child class to perform other updates
111
+ end
112
+
113
+ def before_validate(_record, _row)
114
+ end
115
+
116
+ def around_validate(_record, _row)
117
+ yield
118
+ end
119
+
120
+ def after_validate(_record, _row)
94
121
  end
95
122
 
96
123
  #
97
124
  # Does the actual import
98
125
  #
99
126
  def import!
100
- raise ArgumentError, 'Invalid data structure' unless structure_valid?
101
-
102
- results = loop_data_rows do |attributes, index|
103
- process_data_row(attributes, index)
127
+ raise ArgumentError, "Invalid data structure" unless structure_valid?
128
+
129
+ batch = Sidekiq::Batch.new
130
+ batch.description = "#{import.original.filename} - #{import.kind}"
131
+ batch.on(:success, Importo::ImportJobCallback, import_id: import.id)
132
+
133
+ batch.jobs do
134
+ column_with_delay = columns.select { |k, v| v.delay.present? }
135
+ loop_data_rows do |attributes, index|
136
+ if column_with_delay.present?
137
+ delay = column_with_delay.map do |k, v|
138
+ next unless attributes[k].present?
139
+ v.delay.call(attributes[k])
140
+ end.compact
141
+ end
142
+ Importo::ImportJob.set(wait_until: (delay.max * index).seconds.from_now).perform_async(JSON.dump(attributes), index, import.id) if delay.present?
143
+ Importo::ImportJob.perform_async(JSON.dump(attributes), index, import.id) unless delay.present?
144
+ end
104
145
  end
105
- @import.result.attach(io: results_file, filename: @import.importer.file_name('results'),
106
- content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
107
-
108
- @import.result_message = I18n.t('importo.importers.result_message', nr: results.compact.count, of: results.count,
109
- start_row: data_start_row)
110
- @import.complete!
111
146
 
112
147
  true
113
- rescue StandardError => e
148
+ rescue => e
114
149
  @import.result_message = "Exception: #{e.message}"
115
- Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(';')}"
150
+ Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(";")}"
116
151
  @import.failure!
117
152
 
118
153
  false
119
154
  end
120
155
 
121
- private
122
-
123
- ##
124
- # Overridable failure method
125
- #
126
- def failure(_row, _record, index, exception)
127
- Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(';')}"
128
- end
129
-
130
- def process_data_row(attributes, index)
156
+ def process_data_row(attributes, index, last_attempt: true)
131
157
  record = nil
132
158
  row_hash = Digest::SHA256.base64digest(attributes.inspect)
133
159
  duplicate_import = nil
134
160
 
135
- begin
136
- ActiveRecord::Base.transaction(requires_new: true) do
161
+ run_callbacks :row_import do
162
+ record = nil
163
+
164
+ ActiveRecord::Base.transaction do
137
165
  register_result(index, hash: row_hash, state: :processing)
138
166
 
139
- record = build(attributes)
140
- record.validate!
141
- before_save(record, attributes)
142
- record.save!
143
- after_save(record, attributes)
167
+ before_build(record, attributes)
168
+ around_build(record, attributes) do
169
+ record = build(attributes)
170
+ end
171
+ after_build(record, attributes)
172
+
173
+ before_validate(record, attributes)
174
+ around_validate(record, attributes) do
175
+ record.validate!
176
+ end
177
+ after_validate(record, attributes)
178
+
179
+ model.with_advisory_lock(:importo) do
180
+ before_save(record, attributes)
181
+ around_save(record, attributes) do
182
+ record.save!
183
+ end
184
+ after_save(record, attributes)
185
+ end
186
+
144
187
  duplicate_import = duplicate?(row_hash, record.id)
145
188
  raise Importo::DuplicateRowError if duplicate_import
146
189
 
147
190
  register_result(index, class: record.class.name, id: record.id, state: :success)
148
191
  end
149
- record
150
- rescue Importo::DuplicateRowError
151
- record_id = duplicate_import.results.find { |data| data['hash'] == row_hash }['id']
152
- register_result(index, id: record_id, state: :duplicate,
153
- message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
154
- nil
155
- rescue StandardError => e
156
- errors = record.respond_to?(:errors) && record.errors.full_messages.join(', ')
157
- error_message = "#{e.message} (#{e.backtrace.first.split('/').last})"
158
- failure(attributes, record, index, e)
159
- register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
160
- nil
161
192
  end
193
+
194
+ record
195
+ rescue Importo::DuplicateRowError
196
+ record_id = duplicate_import.results.find { |data| data["hash"] == row_hash }["id"]
197
+ register_result(index, id: record_id, state: :duplicate,
198
+ message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
199
+
200
+ run_callbacks(:row_import, :after)
201
+ nil
202
+ rescue Importo::RetryError => e
203
+ raise e unless last_attempt
204
+
205
+ errors = record.respond_to?(:errors) && record.errors.full_messages.join(", ")
206
+ error_message = "#{e.message} (#{e.backtrace.first.split("/").last})"
207
+ failure(attributes, record, index, e)
208
+ register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
209
+ nil
210
+ rescue => e
211
+ # We rescue ActiveRecord::RecordNotUnique here, due to how transactions work, some row imports may have started the transaction and the current row doesn't see the results
212
+ raise Importo::RetryError.new("ActiveRecord::RecordNotUnique", 2) if !last_attempt && e.is_a?(ActiveRecord::RecordNotUnique)
213
+ errors = record.respond_to?(:errors) && record.errors.full_messages.join(", ")
214
+ error_message = "#{e.message} (#{e.backtrace.first.split("/").last})"
215
+ failure(attributes, record, index, e)
216
+ register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
217
+ run_callbacks(:row_import, :after)
218
+ nil
219
+ end
220
+
221
+ private
222
+
223
+ ##
224
+ # Overridable failure method
225
+ #
226
+ def failure(_row, _record, index, exception)
227
+ Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(";")}"
162
228
  end
163
229
 
164
230
  def set_attribute(hash, path, value)
165
- tmp_hash = path.to_s.split('.').reverse.inject(value) { |h, s| { s => h } }
231
+ tmp_hash = path.to_s.split(".").reverse.inject(value) { |h, s| {s => h} }
166
232
  hash.deep_merge(tmp_hash)
167
233
  end
234
+
235
+ # Only when the block returns a record created in this import, it returns that record, otherwise nil
236
+ #
237
+ # record ||= only_current_import! do
238
+ # User.find_by(email: row[:email].downcase) if row[:email].present?
239
+ # end
240
+ #
241
+ # Only if the user was created this import will the block return the user found.
242
+ #
243
+ # @return [Object]
244
+ def only_current_import!(&block)
245
+ ActiveRecord::Base.uncached do
246
+ record = yield
247
+ record if record && @import.results.where("details->>'state' = ?", "success").where("details->>'id' = ? ", record.id).exists?
248
+ end
249
+ end
168
250
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'active_support/concern'
3
+ require "active_support/concern"
4
4
 
5
5
  module ImporterDsl
6
6
  extend ActiveSupport::Concern
@@ -31,19 +31,11 @@ module ImporterDsl
31
31
  #
32
32
  # @param [Object] args
33
33
  # @param [Object] block which will filter the results before storing the value in the attribute, this is useful for lookups or reformatting
34
- def column(*args, &block)
35
- options = args.extract_options!
36
-
37
- name = args[0]
38
- name ||= options[:name]
39
- name ||= options[:attribute]
40
-
41
- hint = args[1]
42
- hint ||= options[:hint]
43
-
44
- options[:scope] = self.name.underscore.to_s.tr('/', '.').to_sym
45
-
46
- columns[name] = Importo::ImportColumn.new(name, hint, options[:explanation], options, &block)
34
+ def column(**options, &block)
35
+ column_name ||= options[:name]
36
+ column_name ||= options[:attribute]
37
+ options[:scope] = name.to_s.underscore.to_s.tr("/", ".").to_sym
38
+ columns[column_name] = Importo::ImportColumn.new(name: column_name, **options, &block)
47
39
  end
48
40
 
49
41
  def model(model = nil)