importo 2.0.4 → 3.0.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +22 -5
  3. data/Rakefile +16 -17
  4. data/app/controllers/concerns/maintenance_standards.rb +1 -1
  5. data/app/controllers/importo/imports_controller.rb +15 -12
  6. data/app/helpers/importo/application_helper.rb +11 -11
  7. data/app/importers/concerns/exportable.rb +31 -24
  8. data/app/importers/concerns/importable.rb +144 -62
  9. data/app/importers/concerns/importer_dsl.rb +6 -14
  10. data/app/importers/concerns/original.rb +23 -23
  11. data/app/importers/concerns/result_feedback.rb +62 -44
  12. data/app/importers/concerns/revertable.rb +11 -12
  13. data/app/importers/importo/base_importer.rb +16 -4
  14. data/app/importers/importo/import_job_callback.rb +29 -0
  15. data/app/jobs/importo/application_job.rb +4 -0
  16. data/app/jobs/importo/import_job.rb +40 -0
  17. data/app/jobs/importo/purge_import_job.rb +14 -0
  18. data/app/mailers/importo/application_mailer.rb +2 -2
  19. data/app/models/importo/import.rb +35 -8
  20. data/app/models/importo/result.rb +5 -0
  21. data/app/services/importo/import_service.rb +1 -3
  22. data/app/services/importo/revert_service.rb +1 -1
  23. data/app/tables/importo/imports_table.rb +13 -17
  24. data/app/tables/importo/mensa_imports_table.rb +35 -0
  25. data/app/views/importo/imports/index.html.slim +6 -2
  26. data/app/views/importo/imports/new.html.slim +30 -20
  27. data/config/locales/en.yml +21 -9
  28. data/config/locales/nl.yml +10 -4
  29. data/config/routes.rb +4 -4
  30. data/db/migrate/20180409151031_create_importo_import.rb +9 -9
  31. data/db/migrate/20180628175535_add_locale_importo_import.rb +1 -1
  32. data/db/migrate/20230510051447_remove_result_from_imports.rb +5 -0
  33. data/db/migrate/20230510083043_create_importo_results.rb +11 -0
  34. data/lib/generators/importo/importer_generator.rb +1 -1
  35. data/lib/generators/importo/install_generator.rb +7 -8
  36. data/lib/generators/importo/templates/importo.rb +11 -0
  37. data/lib/generators/satis/install_generator.rb +22 -0
  38. data/lib/generators/satis/tailwind_config_generator.rb +24 -0
  39. data/lib/generators/satis/templates/config/initializers/satis.rb +13 -0
  40. data/lib/importo/acts_as_import_owner.rb +1 -1
  41. data/lib/importo/configuration.rb +52 -49
  42. data/lib/importo/engine.rb +11 -9
  43. data/lib/importo/import_column.rb +18 -6
  44. data/lib/importo/test_helpers.rb +19 -0
  45. data/lib/importo/version.rb +1 -1
  46. data/lib/importo.rb +12 -17
  47. metadata +77 -24
  48. data/app/services/importo/callback_service.rb +0 -14
  49. data/lib/generators/templates/importo.rb +0 -21
  50. /data/lib/generators/{templates → importo/templates}/README +0 -0
  51. /data/lib/generators/{templates → importo/templates}/application_importer.rb +0 -0
  52. /data/lib/generators/{templates → importo/templates}/importer.rb +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a20759f65aaabdc3812973d96d19ba69854eeeb75ec421b779b6da6890b2d916
4
- data.tar.gz: faaba7cb0ceef82f36b161049fdd0eda76c0b14363e7a78636756624cecd6b00
3
+ metadata.gz: 223e8f60e766e765e6b1933f05938e87a0edf05c02a307b931892266113802ad
4
+ data.tar.gz: e87886f568dd191c231e50fc03978e36c646c63d0acdfedb903deb35731fc60b
5
5
  SHA512:
6
- metadata.gz: 59e08eb163844c774f4cdf74bcbb1365b971049c7d986687e80d2acc99512c509907a1b148dd41f4d3e99dd1345c09b2a80595e9987fca06cf8479dff48e34bf
7
- data.tar.gz: 0e750e8752d3467e4ffedd4305c1b81d17f981c604a02d48200464278a9914b87b946e6fc7a9a74b0f73900ad79508ecaf9c6e0ba44e9ac7fe8e63519e3a2f59
6
+ metadata.gz: 553a215bc4f9b34194bd0405a22d53f9e28621fe9d5bdeadd19365acee1129b686f8b4f43db44def508b327ddba5b29d6efd4276d345904cca43acc5f827b4fe
7
+ data.tar.gz: 75c8a8d51b112b7b302c7b5607023a0f405e856435c31531ec39ab9264df8f51ca26e50b8e3c15ee26cc586802b7b632688c70c52ce1fb910d823200712db181
data/README.md CHANGED
@@ -26,9 +26,11 @@ class ProductsImporter < ApplicationImporter
26
26
  column attribute: :id
27
27
 
28
28
  # attributes
29
- column attribute: :name
30
- column attribute: :number
29
+ column attribute: :name, required: true
31
30
  column attribute: :description, strip_tags: false
31
+ column attribute: :number, export: { format: 'text', value: ->(record) { record.number }, example: 'FLAG-NLD-001' }, style: {b: true}
32
+ column attribute: :expires_on, export: { format: 'dd/mm/yyyy h:mm'}
33
+ column name: :price, export: { format: 'number', value: ->(record) { record.price } }
32
34
  column attribute: :images do |value|
33
35
  value.split(',').map do |image|
34
36
  uri = URI.parse(image)
@@ -36,12 +38,14 @@ class ProductsImporter < ApplicationImporter
36
38
  { filename: File.basename(uri.to_s), io: URI.open(uri) }
37
39
  end
38
40
  end
41
+ column name: :kitting_component_product, delay: ->(value) {value.present? ? 5 : 0 }
39
42
 
40
43
  def export_scope
41
44
  Current.account.products
42
45
  end
43
46
  end
44
47
  ```
48
+ export args for column is optional, format takes excel custom format codes default is General
45
49
 
46
50
  You should add translations to your locale files:
47
51
 
@@ -62,12 +66,22 @@ en:
62
66
  number: Number
63
67
  description: Description
64
68
  images: Images
65
- explanation:
66
- id: Record-id, only needed if you want to update an existing record
69
+ kitting_component_product: Component Product
70
+ # Shown in note in import sheet
67
71
  hint:
68
72
  id: 36 characters, existing of hexadecimal numbers, separated by dashes
69
73
  images: Allows multiple image urls, separated by comma
70
- introduction: null
74
+ # Below items are show in explanation sheet
75
+ explanation:
76
+ id: Record-id, only needed if you want to update an existing record
77
+ example:
78
+ id: 12345678-1234-1234-1234-123456789012
79
+ name: TEST-123
80
+ number: TEST-123
81
+ description: Test product
82
+ value:
83
+ id: Optional
84
+
71
85
  ```
72
86
 
73
87
  ## Installation
@@ -78,6 +92,9 @@ Add this line to your application's Gemfile:
78
92
  gem 'importo'
79
93
  ```
80
94
 
95
+ Importo depends on Sidekiq Pro's batch functionality,
96
+ though you can use [sidekiq-batch](https://github.com/entdec/sidekiq-batch) as a drop-in for that.
97
+
81
98
  And then execute:
82
99
 
83
100
  ```bash
data/Rakefile CHANGED
@@ -1,39 +1,38 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  begin
4
- require 'bundler/setup'
4
+ require "bundler/setup"
5
5
  rescue LoadError
6
- puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
6
+ puts "You must `gem install bundler` and `bundle install` to run rake tasks"
7
7
  end
8
8
 
9
- require 'rdoc/task'
9
+ require "rdoc/task"
10
10
 
11
11
  RDoc::Task.new(:rdoc) do |rdoc|
12
- rdoc.rdoc_dir = 'rdoc'
13
- rdoc.title = 'Importo'
14
- rdoc.options << '--line-numbers'
15
- rdoc.rdoc_files.include('README.md')
16
- rdoc.rdoc_files.include('lib/**/*.rb')
12
+ rdoc.rdoc_dir = "rdoc"
13
+ rdoc.title = "Importo"
14
+ rdoc.options << "--line-numbers"
15
+ rdoc.rdoc_files.include("README.md")
16
+ rdoc.rdoc_files.include("lib/**/*.rb")
17
17
  end
18
18
 
19
- APP_RAKEFILE = File.expand_path('test/dummy/Rakefile', __dir__)
20
- load 'rails/tasks/engine.rake'
19
+ APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
20
+ load "rails/tasks/engine.rake"
21
21
 
22
- load 'rails/tasks/statistics.rake'
22
+ load "rails/tasks/statistics.rake"
23
23
 
24
- require 'bundler/gem_tasks'
24
+ require "bundler/gem_tasks"
25
25
 
26
- require 'rake/testtask'
26
+ require "rake/testtask"
27
27
 
28
28
  Rake::TestTask.new(:test) do |t|
29
- t.libs << 'test'
30
- t.pattern = 'test/**/*_test.rb'
29
+ t.libs << "test"
30
+ t.pattern = "test/**/*_test.rb"
31
31
  t.verbose = false
32
32
  end
33
33
 
34
34
  task default: :test
35
35
 
36
36
  # Adds the Auxilium semver task
37
- spec = Gem::Specification.find_by_name 'auxilium'
37
+ spec = Gem::Specification.find_by_name "auxilium"
38
38
  load "#{spec.gem_dir}/lib/tasks/semver.rake"
39
-
@@ -11,7 +11,7 @@ module MaintenanceStandards
11
11
  #
12
12
  def flash_and_redirect(result, path, notice, error, render_action = :edit)
13
13
  if result
14
- if params[:commit] == 'continue'
14
+ if params[:commit] == "continue"
15
15
  flash.now[:notice] = notice
16
16
  else
17
17
  redirect_to(path, notice: notice) && return
@@ -11,22 +11,22 @@ module Importo
11
11
  def create
12
12
  unless import_params
13
13
  @import = Import.new(kind: params[:kind], locale: I18n.locale)
14
- flash[:error] = t('.flash.no_file')
14
+ Signum.error(Current.user, text: t('.flash.no_file'))
15
15
  render :new
16
16
  return
17
17
  end
18
- @import = Import.new(import_params.merge(locale: I18n.locale, importo_ownable: Importo.config.current_import_owner))
18
+ @import = Import.new(import_params.merge(locale: I18n.locale,
19
+ importo_ownable: Importo.config.current_import_owner.call))
19
20
  if @import.valid? && @import.schedule!
20
- flash[:notice] = t('.flash.success', id: @import.id)
21
- redirect_to action: :index
21
+ redirect_to importo.new_import_path(params[:kind] || @import.kind)
22
22
  else
23
- flash[:error] = t('.flash.error')
23
+ Signum.error(Current.user, text: t('.flash.error', error: @import.errors&.full_messages&.join('.')))
24
24
  render :new
25
25
  end
26
26
  end
27
27
 
28
28
  def undo
29
- @import = Import.where(importo_ownable: Importo.config.current_import_owner).find(params[:id])
29
+ @import = Import.where(importo_ownable: Importo.config.current_import_owner.call).find(params[:id])
30
30
  if @import.can_revert? && @import.revert
31
31
  redirect_to action: :index, notice: 'Import reverted'
32
32
  else
@@ -35,8 +35,8 @@ module Importo
35
35
  end
36
36
 
37
37
  def destroy
38
- @import = Import.where(importo_ownable: Importo.config.current_import_owner).find(params[:id])
39
- redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy(@import)
38
+ @import = Import.find(params[:id])
39
+ redirect_to(action: :index, alert: 'Not allowed') && return unless Importo.config.admin_can_destroy.call(@import)
40
40
 
41
41
  @import.destroy
42
42
  redirect_to action: :index
@@ -44,22 +44,25 @@ module Importo
44
44
 
45
45
  def sample
46
46
  import = Import.new(kind: params[:kind], locale: I18n.locale)
47
- send_data import.importer.sample_file.read, type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('sample')
47
+ send_data import.importer.sample_file.read,
48
+ type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('sample')
48
49
  end
49
50
 
50
51
  def export
51
52
  import = Import.new(kind: params[:kind], locale: I18n.locale)
52
- send_data import.importer.export_file.read, type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('export')
53
+ send_data import.importer.export_file.read,
54
+ type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', filename: import.importer.file_name('export')
53
55
  end
54
56
 
55
57
  def index
56
- @imports = Importo.config.admin_visible_imports.order(created_at: :desc).limit(50)
58
+ @imports = Importo.config.admin_visible_imports.call.order(created_at: :desc).limit(50)
57
59
  end
58
60
 
59
61
  private
60
62
 
61
63
  def import_params
62
- params.require(:import).permit(:original, :kind, :column_overrides, column_overrides: params.dig(:import, :column_overrides)&.keys)
64
+ params.require(:import).permit(:original, :kind, :column_overrides,
65
+ column_overrides: params.dig(:import, :column_overrides)&.keys)
63
66
  end
64
67
  end
65
68
  end
@@ -2,16 +2,16 @@
2
2
 
3
3
  module Importo
4
4
  module ApplicationHelper
5
- def respond_to_missing?(method)
6
- method.ends_with?('_url') || method.ends_with?('_path')
7
- end
8
-
9
- def method_missing(method, *args, &block)
10
- if main_app.respond_to?(method)
11
- main_app.send(method, *args)
12
- else
13
- super
14
- end
15
- end
5
+ # def respond_to_missing?(method)
6
+ # method.ends_with?('_url') || method.ends_with?('_path')
7
+ # end
8
+ #
9
+ # def method_missing(method, *args, &block)
10
+ # if main_app.respond_to?(method)
11
+ # main_app.send(method, *args)
12
+ # else
13
+ # super
14
+ # end
15
+ # end
16
16
  end
17
17
  end
@@ -1,7 +1,6 @@
1
-
2
1
  # frozen_string_literal: true
3
2
 
4
- require 'active_support/concern'
3
+ require "active_support/concern"
5
4
 
6
5
  module Exportable
7
6
  extend ActiveSupport::Concern
@@ -14,7 +13,9 @@ module Exportable
14
13
  end
15
14
 
16
15
  def sample_data
17
- [export_columns.map { |_, c| c.options[:example] || '' }]
16
+ sample_data = []
17
+ 100.times { sample_data << export_columns.map { |_, c| c.options[:example] || "" } }
18
+ sample_data
18
19
  end
19
20
 
20
21
  #
@@ -38,15 +39,16 @@ module Exportable
38
39
 
39
40
  def export_row(record)
40
41
  export_columns.map do |_, c|
41
- value = ''
42
+ value = ""
42
43
 
43
44
  if c.options[:attribute]
44
- if record.respond_to?(c.options[:attribute])
45
+ if c.options[:export]&.key?(:value)
46
+ value = c.options[:export][:value].call(record) if c.options[:export][:value].is_a?(Proc)
47
+ elsif record.respond_to?(c.options[:attribute])
45
48
  value = record.send(c.options[:attribute])
46
49
  value = value&.body&.to_html if value.is_a?(ActionText::RichText)
47
50
  end
48
-
49
- value ||= record.attributes[c.options[:attribute].to_s]
51
+ value ||= record.attributes[c.options[:attribute].to_s] if value.nil?
50
52
  end
51
53
 
52
54
  value
@@ -59,9 +61,9 @@ module Exportable
59
61
  workbook = xls.workbook
60
62
  sheet = workbook.add_worksheet(name: friendly_name&.pluralize || model.name.demodulize.pluralize)
61
63
  workbook.styles do |style|
62
- introduction_style = style.add_style(bg_color: 'E2EEDA')
63
- header_style = style.add_style(b: true, bg_color: 'A8D08E', border: { style: :thin, color: '000000' })
64
- header_required_style = style.add_style(b: true, bg_color: 'A8D08E', fg_color: 'C00100', border: { style: :thin, color: '000000' })
64
+ introduction_style = style.add_style(bg_color: "E2EEDA")
65
+ header_style = style.add_style(b: true, bg_color: "A8D08E", border: {style: :thin, color: "000000"})
66
+ header_required_style = style.add_style(b: true, bg_color: "A8D08E", fg_color: "C00100", border: {style: :thin, color: "000000"})
65
67
 
66
68
  # Introduction
67
69
  introduction.each_with_index do |intro, i|
@@ -75,14 +77,19 @@ module Exportable
75
77
 
76
78
  export_columns.each.with_index do |f, i|
77
79
  field = f.last
78
- sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author: '', text: field.hint, visible: false if field.hint.present?
80
+ sheet.add_comment ref: "#{nr_to_col(i)}#{introduction.count + 1}", author: "", text: field.hint, visible: false if field.hint.present?
81
+ end
82
+ styles = export_columns.map do |_, c|
83
+ if c.options.dig(:export, :format) == "number" || (c.options.dig(:export, :format).nil? && c.options.dig(:export, :example).is_a?(Numeric))
84
+ number = workbook.styles.add_style format_code: "#"
85
+ elsif c.options.dig(:export, :format) == "text" || (c.options.dig(:export, :format).nil? && c.options.dig(:export, :example).is_a?(String))
86
+ text = workbook.styles.add_style format_code: "@"
87
+ elsif c.options.dig(:export, :format)
88
+ workbook.styles.add_style format_code: c.options.dig(:export, :format).to_s
89
+ else
90
+ workbook.styles.add_style format_code: "General"
91
+ end
79
92
  end
80
-
81
- number = workbook.styles.add_style format_code: '#'
82
- text = workbook.styles.add_style format_code: '@'
83
-
84
- styles = export_columns.map { |_, c| c.options[:example].is_a?(Numeric) ? number : text }
85
-
86
93
  # Examples
87
94
  data_rows.each do |data|
88
95
  sheet.add_row data, style: styles
@@ -91,15 +98,15 @@ module Exportable
91
98
 
92
99
  sheet.column_info[0].width = 10
93
100
 
94
- sheet = workbook.add_worksheet(name: I18n.t('importo.sheet.explanation.name'))
101
+ sheet = workbook.add_worksheet(name: I18n.t("importo.sheet.explanation.name"))
95
102
 
96
103
  workbook.styles do |style|
97
- introduction_style = style.add_style(bg_color: 'E2EEDA')
98
- header_style = style.add_style(b: true, bg_color: 'A8D08E', border: { style: :thin, color: '000000' })
104
+ introduction_style = style.add_style(bg_color: "E2EEDA")
105
+ header_style = style.add_style(b: true, bg_color: "A8D08E", border: {style: :thin, color: "000000"})
99
106
 
100
107
  column_style = style.add_style(b: true)
101
- required_style = style.add_style(b: true, fg_color: 'C00100')
102
- wrap_style = workbook.styles.add_style alignment: { wrap_text: true }
108
+ required_style = style.add_style(b: true, fg_color: "C00100")
109
+ wrap_style = workbook.styles.add_style alignment: {wrap_text: true}
103
110
 
104
111
  # Introduction
105
112
  introduction.each_with_index do |intro, i|
@@ -109,10 +116,10 @@ module Exportable
109
116
  end
110
117
 
111
118
  # Header row
112
- sheet.add_row [I18n.t('importo.sheet.explanation.column'), I18n.t('importo.sheet.explanation.explanation')], style: [header_style] * 2
119
+ sheet.add_row [I18n.t("importo.sheet.explanation.column"), I18n.t("importo.sheet.explanation.value"), I18n.t("importo.sheet.explanation.explanation"), I18n.t("importo.sheet.explanation.example")], style: [header_style] * 4
113
120
  export_columns.each do |_, c|
114
121
  styles = [c.options[:required] ? required_style : column_style, wrap_style]
115
- sheet.add_row [c.name, c.explanation], style: styles
122
+ sheet.add_row [c.name, c.value, c.explanation, c.example], style: styles
116
123
  end
117
124
  end
118
125
 
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'active_support/concern'
3
+ require "active_support/concern"
4
4
 
5
5
  module Importable
6
6
  extend ActiveSupport::Concern
@@ -14,15 +14,13 @@ module Importable
14
14
  end
15
15
 
16
16
  def convert_values(row)
17
- return row if row.instance_variable_get('@importo_converted_values')
17
+ return row if row.instance_variable_get(:@importo_converted_values)
18
18
 
19
- row.instance_variable_set('@importo_converted_values', true)
19
+ row.instance_variable_set(:@importo_converted_values, true)
20
20
 
21
- cols_to_populate = columns.select do |_, v|
22
- v.options[:attribute].present?
23
- end
21
+ columns.each do |k, col|
22
+ next if col.proc.blank? || row[k].nil?
24
23
 
25
- cols_to_populate.each do |k, col|
26
24
  attr = col.options[:attribute]
27
25
 
28
26
  row[k] = import.column_overrides[col.attribute] if import.column_overrides[col.attribute]
@@ -52,26 +50,36 @@ module Importable
52
50
  # It wil try and find the record by id, or initialize a new record with it's attributes set based on the mapping from columns
53
51
  #
54
52
  def populate(row, record = nil)
55
- raise 'No attributes set for columns' unless columns.any? { |_, v| v.options[:attribute].present? }
53
+ raise "No attributes set for columns" unless columns.any? { |_, v| v.options[:attribute].present? }
56
54
 
57
55
  row = convert_values(row)
58
56
 
59
57
  result = if record
60
- record
61
- else
62
- raise 'No model set' unless model
58
+ record
59
+ else
60
+ raise "No model set" unless model
63
61
 
64
- model.find_or_initialize_by(id: row['id'])
65
- end
62
+ model.find_or_initialize_by(id: row["id"])
63
+ end
66
64
 
67
65
  attributes = {}
68
66
  cols_to_populate = columns.select do |_, v|
69
67
  v.options[:attribute].present?
70
68
  end
71
69
 
70
+ cols_to_populate = cols_to_populate.deep_symbolize_keys
71
+
72
72
  cols_to_populate.each do |k, col|
73
73
  attr = col.options[:attribute]
74
- attributes = set_attribute(attributes, attr, row[k]) if row[k].present?
74
+
75
+ next unless row.key? k
76
+ next if !row[k].present? && col.options[:default].nil?
77
+
78
+ attributes = if !row[k].present? && !col.options[:default].nil?
79
+ set_attribute(attributes, attr, col.options[:default])
80
+ else
81
+ set_attribute(attributes, attr, row[k])
82
+ end
75
83
  end
76
84
 
77
85
  result.assign_attributes(attributes)
@@ -80,89 +88,163 @@ module Importable
80
88
  end
81
89
 
82
90
  #
83
- # Mangle the record before saving
91
+ # Callbakcs
84
92
  #
93
+ def before_build(_record, _row)
94
+ end
95
+
96
+ def around_build(_record, _row)
97
+ yield
98
+ end
99
+
100
+ def after_build(_record, _row)
101
+ end
102
+
85
103
  def before_save(_record, _row)
86
- # Implement optionally in child class to mangle
87
104
  end
88
105
 
89
- #
90
- # Any updates that have to be done after saving
91
- #
106
+ def around_save(_record, _row)
107
+ yield
108
+ end
109
+
92
110
  def after_save(_record, _row)
93
- # Implement optionally in child class to perform other updates
111
+ end
112
+
113
+ def before_validate(_record, _row)
114
+ end
115
+
116
+ def around_validate(_record, _row)
117
+ yield
118
+ end
119
+
120
+ def after_validate(_record, _row)
94
121
  end
95
122
 
96
123
  #
97
124
  # Does the actual import
98
125
  #
99
126
  def import!
100
- raise ArgumentError, 'Invalid data structure' unless structure_valid?
101
-
102
- results = loop_data_rows do |attributes, index|
103
- process_data_row(attributes, index)
127
+ raise ArgumentError, "Invalid data structure" unless structure_valid?
128
+
129
+ batch = Sidekiq::Batch.new
130
+ batch.description = "#{import.original.filename} - #{import.kind}"
131
+ batch.on(:success, Importo::ImportJobCallback, import_id: import.id)
132
+
133
+ batch.jobs do
134
+ column_with_delay = columns.select { |k, v| v.delay.present? }
135
+ loop_data_rows do |attributes, index|
136
+ if column_with_delay.present?
137
+ delay = column_with_delay.map do |k, v|
138
+ next unless attributes[k].present?
139
+ v.delay.call(attributes[k])
140
+ end.compact
141
+ end
142
+ Importo::ImportJob.set(wait_until: (delay.max * index).seconds.from_now).perform_async(JSON.dump(attributes), index, import.id) if delay.present?
143
+ Importo::ImportJob.perform_async(JSON.dump(attributes), index, import.id) unless delay.present?
144
+ end
104
145
  end
105
- @import.result.attach(io: results_file, filename: @import.importer.file_name('results'),
106
- content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
107
-
108
- @import.result_message = I18n.t('importo.importers.result_message', nr: results.compact.count, of: results.count,
109
- start_row: data_start_row)
110
- @import.complete!
111
146
 
112
147
  true
113
- rescue StandardError => e
148
+ rescue => e
114
149
  @import.result_message = "Exception: #{e.message}"
115
- Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(';')}"
150
+ Rails.logger.error "Importo exception: #{e.message} backtrace #{e.backtrace.join(";")}"
116
151
  @import.failure!
117
152
 
118
153
  false
119
154
  end
120
155
 
121
- private
122
-
123
- ##
124
- # Overridable failure method
125
- #
126
- def failure(_row, _record, index, exception)
127
- Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(';')}"
128
- end
129
-
130
- def process_data_row(attributes, index)
156
+ def process_data_row(attributes, index, last_attempt: true)
131
157
  record = nil
132
158
  row_hash = Digest::SHA256.base64digest(attributes.inspect)
133
159
  duplicate_import = nil
134
160
 
135
- begin
136
- ActiveRecord::Base.transaction(requires_new: true) do
161
+ run_callbacks :row_import do
162
+ record = nil
163
+
164
+ ActiveRecord::Base.transaction(isolation: :read_committed) do
137
165
  register_result(index, hash: row_hash, state: :processing)
138
166
 
139
- record = build(attributes)
140
- record.validate!
141
- before_save(record, attributes)
142
- record.save!
143
- after_save(record, attributes)
167
+ before_build(record, attributes)
168
+ around_build(record, attributes) do
169
+ record = build(attributes)
170
+ end
171
+ after_build(record, attributes)
172
+
173
+ before_validate(record, attributes)
174
+ around_validate(record, attributes) do
175
+ record.validate!
176
+ end
177
+ after_validate(record, attributes)
178
+
179
+ model.with_advisory_lock(:importo) do
180
+ before_save(record, attributes)
181
+ around_save(record, attributes) do
182
+ record.save!
183
+ end
184
+ after_save(record, attributes)
185
+ end
186
+
144
187
  duplicate_import = duplicate?(row_hash, record.id)
145
188
  raise Importo::DuplicateRowError if duplicate_import
146
189
 
147
190
  register_result(index, class: record.class.name, id: record.id, state: :success)
148
191
  end
149
- record
150
- rescue Importo::DuplicateRowError
151
- record_id = duplicate_import.results.find { |data| data['hash'] == row_hash }['id']
152
- register_result(index, id: record_id, state: :duplicate,
153
- message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
154
- nil
155
- rescue StandardError => e
156
- errors = record.respond_to?(:errors) && record.errors.full_messages.join(', ')
157
- error_message = "#{e.message} (#{e.backtrace.first.split('/').last})"
158
- failure(attributes, record, index, e)
159
- register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
160
- nil
161
192
  end
193
+
194
+ record
195
+ rescue Importo::DuplicateRowError
196
+ record_id = duplicate_import.results.find { |data| data["hash"] == row_hash }["id"]
197
+ register_result(index, id: record_id, state: :duplicate,
198
+ message: "Row already imported successfully on #{duplicate_import.created_at.to_date}")
199
+
200
+ run_callbacks(:row_import, :after)
201
+ nil
202
+ rescue Importo::RetryError => e
203
+ raise e unless last_attempt
204
+
205
+ errors = record.respond_to?(:errors) && record.errors.full_messages.join(", ")
206
+ error_message = "#{e.message} (#{e.backtrace.first.split("/").last})"
207
+ failure(attributes, record, index, e)
208
+ register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
209
+ nil
210
+ rescue => e
211
+ # We rescue ActiveRecord::RecordNotUnique here, due to how transactions work, some row imports may have started the transaction and the current row doesn't see the results
212
+ raise Importo::RetryError.new("ActiveRecord::RecordNotUnique", 2) if !last_attempt && e.is_a?(ActiveRecord::RecordNotUnique)
213
+ errors = record.respond_to?(:errors) && record.errors.full_messages.join(", ")
214
+ error_message = "#{e.message} (#{e.backtrace.first.split("/").last})"
215
+ failure(attributes, record, index, e)
216
+ register_result(index, class: record.class.name, state: :failure, message: error_message, errors: errors)
217
+ run_callbacks(:row_import, :after)
218
+ nil
219
+ end
220
+
221
+ private
222
+
223
+ ##
224
+ # Overridable failure method
225
+ #
226
+ def failure(_row, _record, index, exception)
227
+ Rails.logger.error "#{exception.message} processing row #{index}: #{exception.backtrace.join(";")}"
162
228
  end
163
229
 
164
230
  def set_attribute(hash, path, value)
165
- tmp_hash = path.to_s.split('.').reverse.inject(value) { |h, s| { s => h } }
231
+ tmp_hash = path.to_s.split(".").reverse.inject(value) { |h, s| {s => h} }
166
232
  hash.deep_merge(tmp_hash)
167
233
  end
234
+
235
+ # Only when the block returns a record created in this import, it returns that record, otherwise nil
236
+ #
237
+ # record ||= only_current_import! do
238
+ # User.find_by(email: row[:email].downcase) if row[:email].present?
239
+ # end
240
+ #
241
+ # Only if the user was created this import will the block return the user found.
242
+ #
243
+ # @return [Object]
244
+ def only_current_import!(&block)
245
+ ActiveRecord::Base.uncached do
246
+ record = yield
247
+ record if record && @import.results.where("details->>'state' = ?", "success").where("details->>'id' = ? ", record.id).exists?
248
+ end
249
+ end
168
250
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'active_support/concern'
3
+ require "active_support/concern"
4
4
 
5
5
  module ImporterDsl
6
6
  extend ActiveSupport::Concern
@@ -31,19 +31,11 @@ module ImporterDsl
31
31
  #
32
32
  # @param [Object] args
33
33
  # @param [Object] block which will filter the results before storing the value in the attribute, this is useful for lookups or reformatting
34
- def column(*args, &block)
35
- options = args.extract_options!
36
-
37
- name = args[0]
38
- name ||= options[:name]
39
- name ||= options[:attribute]
40
-
41
- hint = args[1]
42
- hint ||= options[:hint]
43
-
44
- options[:scope] = self.name.underscore.to_s.tr('/', '.').to_sym
45
-
46
- columns[name] = Importo::ImportColumn.new(name, hint, options[:explanation], options, &block)
34
+ def column(**options, &block)
35
+ column_name ||= options[:name]
36
+ column_name ||= options[:attribute]
37
+ options[:scope] = name.to_s.underscore.to_s.tr("/", ".").to_sym
38
+ columns[column_name] = Importo::ImportColumn.new(name: column_name, **options, &block)
47
39
  end
48
40
 
49
41
  def model(model = nil)