nochmal 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module Adapters
5
+ # collect common things for the carrierwave analysis and migration
6
+ class Carrierwave < Base
7
+ PREFIX = "carrierwave_"
8
+
9
+ def models_with_attachments
10
+ @models_with_attachments ||= begin
11
+ Rails.application.eager_load!
12
+
13
+ ActiveRecord::Base
14
+ .descendants
15
+ .reject(&:abstract_class?)
16
+ .select { |model| carrierwave?(model) }
17
+ end
18
+ end
19
+
20
+ def collection(model, uploader)
21
+ maybe_sti_scope(model).where.not(db_column(uploader) => nil)
22
+ end
23
+
24
+ def blob(attachment)
25
+ Pathname.new(
26
+ attachment.file.file.gsub(
27
+ attachment.mounted_as.to_s,
28
+ not_prefixed(attachment.mounted_as).to_s
29
+ )
30
+ )
31
+ end
32
+
33
+ private
34
+
35
+ def db_column(uploader_name)
36
+ not_prefixed(uploader_name)
37
+ end
38
+
39
+ def not_prefixed(type)
40
+ type.to_s.delete_prefix(PREFIX).to_sym
41
+ end
42
+
43
+ def prefixed(type)
44
+ :"#{PREFIX}#{not_prefixed(type)}"
45
+ end
46
+
47
+ def uploader(model, type)
48
+ @uploaders[model][type]
49
+ end
50
+
51
+ def carrierwave?(model)
52
+ model.uploaders.any? do |_name, uploader|
53
+ uploader.new.is_a? CarrierWave::Uploader::Base
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,222 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module Adapters
5
+ # Wrap ActiveStorageHelper and use carrierwave as simulated from_storage_service
6
+ class CarrierwaveAnalyze < Carrierwave # rubocop:disable Metrics/ClassLength
7
+ def initialize
8
+ super
9
+
10
+ @carrierwave_changed = []
11
+ @variants_present = false
12
+ end
13
+
14
+ def attachment_types_for(model)
15
+ @types[model] ||= model.uploaders.map do |uploader, uploader_class|
16
+ @uploaders[model] = { uploader => uploader_class }
17
+ model.has_one_attached prefixed(uploader), service: @to_storage_service.name
18
+
19
+ uploader
20
+ end
21
+ end
22
+
23
+ def empty_collection?(_model, _uploader)
24
+ false # simulate that uploads exist
25
+ end
26
+
27
+ # actions
28
+
29
+ def reupload(_record, _type)
30
+ { status: :ok } # like count
31
+ end
32
+
33
+ # hooks
34
+
35
+ def general_notes
36
+ [
37
+ display_helper_notes,
38
+ gemfile_additions
39
+ ].join("\n")
40
+ end
41
+
42
+ def type_notes(model = nil, type = nil)
43
+ return nil if @carrierwave_changed.include?(model.base_class.sti_name)
44
+
45
+ @carrierwave_changed << model.base_class.sti_name
46
+ uploader = uploader(model, type)
47
+
48
+ [
49
+ carrierwave_change(model, type, uploader),
50
+ active_storage_change(type, uploader),
51
+ validation_notes(model, type, uploader),
52
+ uploader_change(uploader), "\n"
53
+ ].join
54
+ end
55
+
56
+ private
57
+
58
+ def carrierwave_change(model, type, uploader)
59
+ <<~TEXT
60
+ # replace #{model.name.underscore}.#{type}_url in your views
61
+ # replace #{model.name.underscore}.#{type} in your views
62
+ # maybe search for #{type} in your codebase to find everything...
63
+ #
64
+ # Change carrierwave-uploader in #{model.name}:
65
+ class #{model.name}
66
+ mount_uploader :#{prefixed(type)}, #{uploader.name}, mount_on: '#{type}'
67
+ TEXT
68
+ end
69
+
70
+ def active_storage_change(type, uploader)
71
+ versions = uploader.versions.map do |name, version|
72
+ "attachable.variant :#{name}, #{version.processors.map(&:compact).to_h}"
73
+ end
74
+
75
+ service = ", service: :#{@to}" unless @to.nil?
76
+
77
+ return " has_one_attached :#{type}#{service}" if versions.none?
78
+
79
+ @variants_present = true
80
+
81
+ <<~TEXT
82
+ has_one_attached :#{type}#{service} do |attachable|
83
+ #{versions.join}
84
+ end
85
+ # uploader #{type} has #{versions.size} versions
86
+
87
+ # allow removal, carrierwave-style
88
+ def remove_#{type}; false end
89
+ def remove_#{type}=(delete_it); #{type}.purge_later if delete_it; end
90
+ TEXT
91
+ end
92
+
93
+ def validation_notes(model, type, uploader)
94
+ <<~TEXT
95
+ # Check for #{type} validations in #{model} and #{uploader}
96
+ # Take a look at https://github.com/igorkasyanchuk/active_storage_validations
97
+ #
98
+ # Please make your validation switchable, the validation does not
99
+ # work properly for the migration.
100
+ if ENV['NOCHMAL_MIGRATION'].blank? # if not migrating RIGHT NOW, i.e. normal case
101
+ validates :picture, dimension: { width: { max: 8_000 }, height: { max: 8_000 } },
102
+ content_type: ['image/jpeg', 'image/gif', 'image/png']
103
+ end
104
+ end
105
+ TEXT
106
+ end
107
+
108
+ def uploader_change(uploader)
109
+ <<~TEXT
110
+ # Ensure that #{uploader}#store_dir does not have a prefix of
111
+ # "carrierwave_". To find the existing files, you need to add
112
+ # "mounted_as.to_s.delete_prefix('carrierwave_')" at the appropriate
113
+ # location. If there is no "carrierwave_"-prefix in the generated path,
114
+ # everything is fine.
115
+ TEXT
116
+ end
117
+
118
+ def display_helper_notes
119
+ <<~RUBY
120
+ module UploadDisplayHelper
121
+ # This method provides a facade to serve uploads either from ActiveStorage or
122
+ # CarrierWave
123
+ #
124
+ # Usage:
125
+ #
126
+ # upload_url(person, :picture)
127
+ # upload_url(person, :picture, size: '72x72')
128
+ # upload_url(person, :picture, size: '72x72')
129
+ # upload_url(person, :picture, variant: :thumb)
130
+ # upload_url(person, :picture, variant: :thumb, default: 'profil')
131
+ #
132
+ # could be
133
+ #
134
+ # person.picture or
135
+ # person.picture.variant(resize_to_limit: [72, 72]) or
136
+ # person.picture.variant(:thumb)
137
+ #
138
+ # This helper returns a suitable first argument for image_tag (the image location),
139
+ # but also for the second arg of link_to (the target).
140
+ def upload_url(model, name, size: nil, default: model.class.name.underscore, variant: nil) # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity
141
+ return upload_variant(model, name, variant, default: default) if variant.present?
142
+
143
+ if model.send(name.to_sym).attached?
144
+ model.send(name.to_sym).yield_self do |pic|
145
+ if size
146
+ # variant passes to mini_magick or vips, I assume mini_magick here
147
+ pic.variant(resize_to_limit: extract_image_dimensions(size))
148
+ else
149
+ pic
150
+ end
151
+ end
152
+ elsif model.respond_to?(:"carrierwave_#{name}") && model.send(:"carrierwave_#{name}")
153
+ model.send(:"carrierwave_#{name}_url")
154
+ else
155
+ upload_default(default)
156
+ end
157
+ end
158
+
159
+ # return the filename of the uploaded file
160
+ def upload_name(model, name)
161
+ if model.send(name.to_sym).attached?
162
+ model.send(name.to_sym).filename.to_s
163
+ elsif model.respond_to?(:"carrierwave_#{name}_identifier")
164
+ model.send(:"carrierwave_#{name}_identifier")
165
+ end
166
+ end
167
+
168
+ def upload_exists?(model, name)
169
+ return true if model.send(name.to_sym).attached?
170
+
171
+ if model.respond_to?(:"carrierwave_#{name}")
172
+ model.send(:"carrierwave_#{name}").present?
173
+ else
174
+ false
175
+ end
176
+ end
177
+
178
+ private
179
+
180
+ def upload_variant(model, name, variant, default: model.name.underscore)
181
+ if model.send(name.to_sym).attached?
182
+ model.send(name.to_sym).variant(variant.to_sym)
183
+ elsif model.respond_to?(:"carrierwave_#{name}")
184
+ model.send(:"carrierwave_#{name}").send(variant.to_sym).url
185
+ else
186
+ upload_default([default, variant].compact.map(&:to_s).join('_'))
187
+ end
188
+ end
189
+
190
+ def upload_default(png_name = 'profil')
191
+ ActionController::Base.helpers.asset_pack_path("media/images/#{png_name}.png")
192
+ end
193
+
194
+ def extract_image_dimensions(width_x_height)
195
+ case width_x_height
196
+ when /^\d+x\d+$/ then width_x_height.split('x')
197
+ end
198
+ end
199
+ end
200
+ RUBY
201
+ end
202
+
203
+ def gemfile_additions
204
+ variants_dependencies = <<~TEXT
205
+ gem 'active_storage_variant' # provides person.avatar.variant(:thumb) for Rails < 7
206
+ TEXT
207
+
208
+ validation_dependencies = <<~TEXT
209
+ gem 'active_storage_validations' # validate filesize, dimensions and content-type
210
+ TEXT
211
+
212
+ <<~TEXT
213
+ The following gems are suggested to have in your Gemfile:
214
+
215
+ gem 'nochmal' # only needed until the migration to the desired ActiveStorage-Backend is complete
216
+ #{variants_dependencies if @variants_present}
217
+ #{validation_dependencies}
218
+ TEXT
219
+ end
220
+ end
221
+ end
222
+ end
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module Adapters
5
+ # Wrap ActiveStorageHelper and use carrierwave as simulated from_storage_service
6
+ class CarrierwaveMigration < Carrierwave
7
+ def attachment_types_for(model)
8
+ @types[model] ||= model.uploaders.map do |uploader, uploader_class|
9
+ @uploaders[model] = { uploader => uploader_class }
10
+ uploader
11
+ end
12
+ end
13
+
14
+ def reupload(record, type)
15
+ pathname = blob(record.send(type))
16
+
17
+ if pathname.exist?
18
+ StringIO.open(pathname.read) do |temp|
19
+ record.send(not_prefixed(type)).attach(io: temp, filename: pathname.basename)
20
+ end
21
+
22
+ { status: :ok }
23
+ else
24
+ { status: :missing,
25
+ message: MigrationData::Status.new(filename: pathname, record: record).missing_message }
26
+ end
27
+ end
28
+
29
+ def collection(model, uploader)
30
+ super(model, uploader).tap do |scope|
31
+ type_started(model, uploader, scope.count)
32
+ end
33
+ end
34
+
35
+ # hooks
36
+
37
+ def setup(action)
38
+ @mode = action
39
+
40
+ return if @mode == :count
41
+ raise MigrationData::StatusExists if MigrationData::Status.table_exists?
42
+
43
+ MigrationData::CreateMigrationTables.new.up
44
+ end
45
+
46
+ def teardown
47
+ return if @mode == :count
48
+ raise MigrationData::Incomplete unless completely_done?
49
+ return if ENV["NOCHMAL_KEEP_METADATA"].present?
50
+
51
+ MigrationData::CreateMigrationTables.new.down
52
+ end
53
+
54
+ def item_completed(record, type, status)
55
+ return if @mode == :count
56
+ return unless %i[ok missing].include? status
57
+
58
+ MigrationData::Status.find_or_create_by(
59
+ record_id: record.id,
60
+ record_type: record.class.sti_name,
61
+ uploader_type: type,
62
+ filename: blob(record.send(type)).to_s
63
+ ).update(status: status)
64
+ end
65
+
66
+ def type_completed(model, type)
67
+ return if @mode == :count
68
+
69
+ MigrationData::Meta
70
+ .find_by(record_type: model.sti_name, uploader_type: type)
71
+ .update(migrated: migrated(model, type))
72
+ end
73
+
74
+ private
75
+
76
+ def type_started(model, uploader, count)
77
+ return if @mode == :count
78
+
79
+ MigrationData::Meta.find_or_create_by(
80
+ record_type: model.sti_name,
81
+ uploader_type: uploader
82
+ ).update(expected: count)
83
+ end
84
+
85
+ def completely_done?
86
+ MigrationData::Meta.all.all?(&:done?)
87
+ end
88
+
89
+ def migrated(model, type)
90
+ MigrationData::Status
91
+ .where(record_type: model.sti_name, uploader_type: type)
92
+ .where.not(status: nil)
93
+ .count
94
+ end
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module Adapters
5
+ # Resume a started migration
6
+ class CarrierwaveResume < CarrierwaveMigration
7
+ # action
8
+
9
+ def reupload(record, type)
10
+ status = MigrationData::Status.find_by(
11
+ record_id: record.id, record_type: record.class.sti_name,
12
+ uploader_type: type, filename: blob(record.send(type)).to_s
13
+ )
14
+
15
+ if status&.migrated?
16
+ message = status.missing_message if status.missing?
17
+ { status: :skip, message: message }
18
+ else
19
+ super(record, type)
20
+ end
21
+ end
22
+
23
+ # hooks
24
+
25
+ def setup(action)
26
+ if MigrationData::Status.table_exists? && MigrationData::Meta.table_exists?
27
+ @mode = action
28
+ return true
29
+ end
30
+
31
+ Output.notes [
32
+ "It appears that no previous migration has been running.",
33
+ 'Creating the needed tables and "resuming from square 1"...'
34
+ ]
35
+ super
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module MigrationData
5
+ class CreateMigrationTables < ActiveRecord::Migration[6.0] # :nodoc:
6
+ def up # rubocop:disable Metrics/MethodLength
7
+ create_table :nochmal_migration_data_status do |t|
8
+ t.belongs_to :record, polymorphic: true
9
+
10
+ t.string :uploader_type
11
+ t.string :filename
12
+
13
+ t.string :status
14
+ end
15
+
16
+ return if table_exists?(:nochmal_migration_data_meta)
17
+
18
+ create_table :nochmal_migration_data_meta do |t|
19
+ t.string :record_type
20
+ t.string :uploader_type
21
+ t.integer :expected
22
+ t.integer :migrated
23
+ t.string :status
24
+ end
25
+ end
26
+
27
+ def down
28
+ drop_table :nochmal_migration_data_status, if_exists: true
29
+ drop_table :nochmal_migration_data_meta, if_exists: true
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module MigrationData
5
+ # A migration may not be complete...
6
+ class Incomplete < StandardError
7
+ def initialize(*_args)
8
+ super <<~MESSAGE
9
+ This did not end well...
10
+
11
+ #{Meta.all.map(&:to_s).join("\n ")}
12
+
13
+ Care to clean up the mess?
14
+ MESSAGE
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module MigrationData
5
+ # Track the status of the whole migration
6
+ class Meta < ActiveRecord::Base
7
+ self.table_name = :nochmal_migration_data_meta
8
+
9
+ before_save :update_status
10
+
11
+ def done?
12
+ status.to_s == "done"
13
+ end
14
+
15
+ def to_s
16
+ [
17
+ record_type, "#", uploader_type, ": ",
18
+ migrated, "/", expected, " -> ", status
19
+ ].join
20
+ end
21
+
22
+ private
23
+
24
+ def update_status
25
+ self.status = current_status
26
+ end
27
+
28
+ def current_status # rubocop:disable Metrics/CyclomaticComplexity
29
+ return nil if migrated.nil? && expected.nil?
30
+
31
+ if expected.positive? && migrated.nil?
32
+ "not migrated"
33
+ else
34
+ case migrated.to_i <=> expected
35
+ when -1 then "partial"
36
+ when 0 then "done"
37
+ when 1 then "too much"
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module MigrationData
5
+ # Track the status of individual uploads to be migrated
6
+ class Status < ActiveRecord::Base
7
+ self.table_name = :nochmal_migration_data_status
8
+
9
+ belongs_to :record, polymorphic: true
10
+
11
+ scope :missing, -> { where(status: "missing") }
12
+ scope :ok, -> { where(status: "ok") }
13
+
14
+ def migrated?
15
+ status.present?
16
+ end
17
+
18
+ def missing?
19
+ status.to_s == "missing"
20
+ end
21
+
22
+ def missing_message
23
+ "#{filename} was not found, but was attached to #{record}"
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nochmal
4
+ module MigrationData
5
+ # Provide some help when a migration got aborted.
6
+ #
7
+ # This might happen if the OS or the user kills the process.
8
+ #
9
+ # I suspect an OOM-Kill by the OS or Container-Runtime (K8s/OCP).
10
+ class StatusExists < StandardError
11
+ def initialize
12
+ super <<~ERROR
13
+ It seems like the migration has already been started.
14
+
15
+ You may want to resume with
16
+ rails nochmal:carrierwave:resume
17
+
18
+ Alternatively, you can manually delete the tables
19
+ - #{Status.table_name}
20
+ - #{Meta.table_name}
21
+ and rerun the migration completely.
22
+
23
+ If you want, you can examine the status by interacting with
24
+ - Nochmal::MigrationData::Meta # class/attachment-level stats
25
+ - Nochmal::MigrationData::Status # individual upload stats
26
+ in your rails console.
27
+ ERROR
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "pastel"
4
+
5
+ module Nochmal
6
+ # Handles output for the Reupload Task
7
+ class Output
8
+ class << self
9
+ def reupload(models)
10
+ puts reupload_header(models)
11
+ yield
12
+ puts reupload_footer
13
+ end
14
+
15
+ def model(model, skipping: false)
16
+ if skipping
17
+ puts "Skipping #{pastel.green(model)}"
18
+ return true
19
+ end
20
+
21
+ puts model_header(model)
22
+ yield
23
+ puts model_footer
24
+ end
25
+
26
+ def type(type, count, action)
27
+ puts type_header(type)
28
+ print attachment_summary(count, action)
29
+ yield
30
+ puts type_footer
31
+ end
32
+
33
+ def attachment(filename)
34
+ print attachment_detail(filename)
35
+ end
36
+
37
+ def notes(notes)
38
+ notes = Array.wrap(notes)
39
+ return unless notes.any?
40
+
41
+ puts reupload_notes(notes)
42
+ end
43
+
44
+ def print_result_indicator(status)
45
+ case status
46
+ when :ok then print_progress_indicator
47
+ when :missing then print_failure_indicator
48
+ when :skip then print_skip_indicator
49
+ when :noop then nil
50
+ else print_unknown_indicator
51
+ end
52
+ end
53
+
54
+ def print_progress_indicator
55
+ print pastel.green(".")
56
+ end
57
+
58
+ def print_failure_indicator
59
+ print pastel.red("F")
60
+ end
61
+
62
+ def print_skip_indicator
63
+ print pastel.yellow("*")
64
+ end
65
+
66
+ def print_unknown_indicator
67
+ print pastel.blue("?")
68
+ end
69
+
70
+ private
71
+
72
+ def reupload_header(models)
73
+ model_text = "model".pluralize(models.count)
74
+ model_names = models.map { |model| pastel.green(model) }.join(", ")
75
+
76
+ <<~HEADER
77
+
78
+
79
+ ================================================================================
80
+ I have found #{models.count} #{model_text} to process: #{model_names}
81
+
82
+ HEADER
83
+ end
84
+
85
+ def model_header(model)
86
+ "Model #{pastel.green(model)}"
87
+ end
88
+
89
+ def type_header(type)
90
+ " Type #{pastel.green(type)}"
91
+ end
92
+
93
+ def attachment_summary(count, action)
94
+ " Going to #{action} #{count} #{"attachment".pluralize(count)}: "
95
+ end
96
+
97
+ def attachment_detail(filename)
98
+ "\n - #{filename}"
99
+ end
100
+
101
+ def type_footer
102
+ "\n Done!"
103
+ end
104
+
105
+ def model_footer
106
+ "Done!"
107
+ end
108
+
109
+ def reupload_footer
110
+ "\nAll attachments have been processed!"
111
+ end
112
+
113
+ def reupload_notes(notes)
114
+ <<~NOTES
115
+
116
+ ================================================================================
117
+ #{notes.join("\n")}
118
+ ================================================================================
119
+ NOTES
120
+ end
121
+
122
+ def pastel
123
+ Pastel.new
124
+ end
125
+ end
126
+ end
127
+ end