nochmal 0.2.0

data/lib/nochmal/adapters/carrierwave.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module Adapters
+    # collect common things for the carrierwave analysis and migration
+    class Carrierwave < Base
+      PREFIX = "carrierwave_"
+
+      def models_with_attachments
+        @models_with_attachments ||= begin
+          Rails.application.eager_load!
+
+          ActiveRecord::Base
+            .descendants
+            .reject(&:abstract_class?)
+            .select { |model| carrierwave?(model) }
+        end
+      end
+
+      def collection(model, uploader)
+        maybe_sti_scope(model).where.not(db_column(uploader) => nil)
+      end
+
+      def blob(attachment)
+        Pathname.new(
+          attachment.file.file.gsub(
+            attachment.mounted_as.to_s,
+            not_prefixed(attachment.mounted_as).to_s
+          )
+        )
+      end
+
+      private
+
+      def db_column(uploader_name)
+        not_prefixed(uploader_name)
+      end
+
+      def not_prefixed(type)
+        type.to_s.delete_prefix(PREFIX).to_sym
+      end
+
+      def prefixed(type)
+        :"#{PREFIX}#{not_prefixed(type)}"
+      end
+
+      def uploader(model, type)
+        @uploaders[model][type]
+      end
+
+      def carrierwave?(model)
+        model.uploaders.any? do |_name, uploader|
+          uploader.new.is_a? CarrierWave::Uploader::Base
+        end
+      end
+    end
+  end
+end

data/lib/nochmal/adapters/carrierwave_analyze.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module Adapters
+    # Wrap ActiveStorageHelper and use carrierwave as simulated from_storage_service
+    class CarrierwaveAnalyze < Carrierwave # rubocop:disable Metrics/ClassLength
+      def initialize
+        super
+
+        @carrierwave_changed = []
+        @variants_present = false
+      end
+
+      def attachment_types_for(model)
+        @types[model] ||= model.uploaders.map do |uploader, uploader_class|
+          @uploaders[model] = { uploader => uploader_class }
+          model.has_one_attached prefixed(uploader), service: @to_storage_service.name
+
+          uploader
+        end
+      end
+
+      def empty_collection?(_model, _uploader)
+        false # simulate that uploads exist
+      end
+
+      # actions
+
+      def reupload(_record, _type)
+        { status: :ok } # like count
+      end
+
+      # hooks
+
+      def general_notes
+        [
+          display_helper_notes,
+          gemfile_additions
+        ].join("\n")
+      end
+
+      def type_notes(model = nil, type = nil)
+        return nil if @carrierwave_changed.include?(model.base_class.sti_name)
+
+        @carrierwave_changed << model.base_class.sti_name
+        uploader = uploader(model, type)
+
+        [
+          carrierwave_change(model, type, uploader),
+          active_storage_change(type, uploader),
+          validation_notes(model, type, uploader),
+          uploader_change(uploader), "\n"
+        ].join
+      end
+
+      private
+
+      def carrierwave_change(model, type, uploader)
+        <<~TEXT
+          # replace #{model.name.underscore}.#{type}_url in your views
+          # replace #{model.name.underscore}.#{type} in your views
+          # maybe search for #{type} in your codebase to find everything...
+          #
+          # Change carrierwave-uploader in #{model.name}:
+          class #{model.name}
+            mount_uploader :#{prefixed(type)}, #{uploader.name}, mount_on: '#{type}'
+        TEXT
+      end
+
+      def active_storage_change(type, uploader)
+        versions = uploader.versions.map do |name, version|
+          "attachable.variant :#{name}, #{version.processors.map(&:compact).to_h}"
+        end
+
+        service = ", service: :#{@to}" unless @to.nil?
+
+        return "  has_one_attached :#{type}#{service}" if versions.none?
+
+        @variants_present = true
+
+        <<~TEXT
+            has_one_attached :#{type}#{service} do |attachable|
+              #{versions.join}
+            end
+          # uploader #{type} has #{versions.size} versions
+
+            # allow removal, carrierwave-style
+            def remove_#{type}; false end
+            def remove_#{type}=(delete_it); #{type}.purge_later if delete_it; end
+        TEXT
+      end
+
+      def validation_notes(model, type, uploader)
+        <<~TEXT
+          # Check for #{type} validations in #{model} and #{uploader}
+          # Take a look at https://github.com/igorkasyanchuk/active_storage_validations
+          #
+          # Please make your validation switchable, the validation does not
+          # work properly for the migration.
+            if ENV['NOCHMAL_MIGRATION'].blank? # if not migrating RIGHT NOW, i.e. normal case
+              validates :picture, dimension: { width: { max: 8_000 }, height: { max: 8_000 } },
+                                  content_type: ['image/jpeg', 'image/gif', 'image/png']
+            end
+          end
+        TEXT
+      end
+
+      def uploader_change(uploader)
+        <<~TEXT
+          # Ensure that #{uploader}#store_dir does not have a prefix of
+          # "carrierwave_".  To find the existing files, you need to add
+          # "mounted_as.to_s.delete_prefix('carrierwave_')" at the appropriate
+          # location. If there is no "carrierwave_"-prefix in the generated path,
+          # everything is fine.
+        TEXT
+      end
+
+      def display_helper_notes
+        <<~RUBY
+          module UploadDisplayHelper
+            # This method provides a facade to serve uploads either from ActiveStorage or
+            # CarrierWave
+            #
+            # Usage:
+            #
+            # upload_url(person, :picture)
+            # upload_url(person, :picture, size: '72x72')
+            # upload_url(person, :picture, size: '72x72')
+            # upload_url(person, :picture, variant: :thumb)
+            # upload_url(person, :picture, variant: :thumb, default: 'profil')
+            #
+            # could be
+            #
+            # person.picture or
+            # person.picture.variant(resize_to_limit: [72, 72]) or
+            # person.picture.variant(:thumb)
+            #
+            # This helper returns a suitable first argument for image_tag (the image location),
+            # but also for the second arg of link_to (the target).
+            def upload_url(model, name, size: nil, default: model.class.name.underscore, variant: nil) # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity
+              return upload_variant(model, name, variant, default: default) if variant.present?
+
+              if model.send(name.to_sym).attached?
+                model.send(name.to_sym).yield_self do |pic|
+                  if size
+                    # variant passes to mini_magick or vips, I assume mini_magick here
+                    pic.variant(resize_to_limit: extract_image_dimensions(size))
+                  else
+                    pic
+                  end
+                end
+              elsif model.respond_to?(:"carrierwave_#{name}") && model.send(:"carrierwave_#{name}")
+                model.send(:"carrierwave_#{name}_url")
+              else
+                upload_default(default)
+              end
+            end
+
+            # return the filename of the uploaded file
+            def upload_name(model, name)
+              if model.send(name.to_sym).attached?
+                model.send(name.to_sym).filename.to_s
+              elsif model.respond_to?(:"carrierwave_#{name}_identifier")
+                model.send(:"carrierwave_#{name}_identifier")
+              end
+            end
+
+            def upload_exists?(model, name)
+              return true if model.send(name.to_sym).attached?
+
+              if model.respond_to?(:"carrierwave_#{name}")
+                model.send(:"carrierwave_#{name}").present?
+              else
+                false
+              end
+            end
+
+            private
+
+            def upload_variant(model, name, variant, default: model.name.underscore)
+              if model.send(name.to_sym).attached?
+                model.send(name.to_sym).variant(variant.to_sym)
+              elsif model.respond_to?(:"carrierwave_#{name}")
+                model.send(:"carrierwave_#{name}").send(variant.to_sym).url
+              else
+                upload_default([default, variant].compact.map(&:to_s).join('_'))
+              end
+            end
+
+            def upload_default(png_name = 'profil')
+              ActionController::Base.helpers.asset_pack_path("media/images/#{png_name}.png")
+            end
+
+            def extract_image_dimensions(width_x_height)
+              case width_x_height
+              when /^\d+x\d+$/ then width_x_height.split('x')
+              end
+            end
+          end
+        RUBY
+      end
+
+      def gemfile_additions
+        variants_dependencies = <<~TEXT
+          gem 'active_storage_variant' # provides person.avatar.variant(:thumb) for Rails < 7
+        TEXT
+
+        validation_dependencies = <<~TEXT
+          gem 'active_storage_validations' # validate filesize, dimensions and content-type
+        TEXT
+
+        <<~TEXT
+          The following gems are suggested to have in your Gemfile:
+
+          gem 'nochmal' # only needed until the migration to the desired ActiveStorage-Backend is complete
+          #{variants_dependencies if @variants_present}
+          #{validation_dependencies}
+        TEXT
+      end
+    end
+  end
+end

data/lib/nochmal/adapters/carrierwave_migration.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module Adapters
+    # Wrap ActiveStorageHelper and use carrierwave as simulated from_storage_service
+    class CarrierwaveMigration < Carrierwave
+      def attachment_types_for(model)
+        @types[model] ||= model.uploaders.map do |uploader, uploader_class|
+          @uploaders[model] = { uploader => uploader_class }
+          uploader
+        end
+      end
+
+      def reupload(record, type)
+        pathname = blob(record.send(type))
+
+        if pathname.exist?
+          StringIO.open(pathname.read) do |temp|
+            record.send(not_prefixed(type)).attach(io: temp, filename: pathname.basename)
+          end
+
+          { status: :ok }
+        else
+          { status: :missing,
+            message: MigrationData::Status.new(filename: pathname, record: record).missing_message }
+        end
+      end
+
+      def collection(model, uploader)
+        super(model, uploader).tap do |scope|
+          type_started(model, uploader, scope.count)
+        end
+      end
+
+      # hooks
+
+      def setup(action)
+        @mode = action
+
+        return if @mode == :count
+        raise MigrationData::StatusExists if MigrationData::Status.table_exists?
+
+        MigrationData::CreateMigrationTables.new.up
+      end
+
+      def teardown
+        return if @mode == :count
+        raise MigrationData::Incomplete unless completely_done?
+        return if ENV["NOCHMAL_KEEP_METADATA"].present?
+
+        MigrationData::CreateMigrationTables.new.down
+      end
+
+      def item_completed(record, type, status)
+        return if @mode == :count
+        return unless %i[ok missing].include? status
+
+        MigrationData::Status.find_or_create_by(
+          record_id: record.id,
+          record_type: record.class.sti_name,
+          uploader_type: type,
+          filename: blob(record.send(type)).to_s
+        ).update(status: status)
+      end
+
+      def type_completed(model, type)
+        return if @mode == :count
+
+        MigrationData::Meta
+          .find_by(record_type: model.sti_name, uploader_type: type)
+          .update(migrated: migrated(model, type))
+      end
+
+      private
+
+      def type_started(model, uploader, count)
+        return if @mode == :count
+
+        MigrationData::Meta.find_or_create_by(
+          record_type: model.sti_name,
+          uploader_type: uploader
+        ).update(expected: count)
+      end
+
+      def completely_done?
+        MigrationData::Meta.all.all?(&:done?)
+      end
+
+      def migrated(model, type)
+        MigrationData::Status
+          .where(record_type: model.sti_name, uploader_type: type)
+          .where.not(status: nil)
+          .count
+      end
+    end
+  end
+end

data/lib/nochmal/adapters/carrierwave_resume.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module Adapters
+    # Resume a started migration
+    class CarrierwaveResume < CarrierwaveMigration
+      # action
+
+      def reupload(record, type)
+        status = MigrationData::Status.find_by(
+          record_id: record.id, record_type: record.class.sti_name,
+          uploader_type: type, filename: blob(record.send(type)).to_s
+        )
+
+        if status&.migrated?
+          message = status.missing_message if status.missing?
+          { status: :skip, message: message }
+        else
+          super(record, type)
+        end
+      end
+
+      # hooks
+
+      def setup(action)
+        if MigrationData::Status.table_exists? && MigrationData::Meta.table_exists?
+          @mode = action
+          return true
+        end
+
+        Output.notes [
+          "It appears that no previous migration has been running.",
+          'Creating the needed tables and "resuming from square 1"...'
+        ]
+        super
+      end
+    end
+  end
+end

data/lib/nochmal/migration_data/create_tables.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module MigrationData
+    class CreateMigrationTables < ActiveRecord::Migration[6.0] # :nodoc:
+      def up # rubocop:disable Metrics/MethodLength
+        create_table :nochmal_migration_data_status do |t|
+          t.belongs_to :record, polymorphic: true
+
+          t.string :uploader_type
+          t.string :filename
+
+          t.string :status
+        end
+
+        return if table_exists?(:nochmal_migration_data_meta)
+
+        create_table :nochmal_migration_data_meta do |t|
+          t.string :record_type
+          t.string :uploader_type
+          t.integer :expected
+          t.integer :migrated
+          t.string :status
+        end
+      end
+
+      def down
+        drop_table :nochmal_migration_data_status, if_exists: true
+        drop_table :nochmal_migration_data_meta, if_exists: true
+      end
+    end
+  end
+end

data/lib/nochmal/migration_data/incomplete.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module MigrationData
+    # A migration may not be complete...
+    class Incomplete < StandardError
+      def initialize(*_args)
+        super <<~MESSAGE
+          This did not end well...
+
+            #{Meta.all.map(&:to_s).join("\n  ")}
+
+          Care to clean up the mess?
+        MESSAGE
+      end
+    end
+  end
+end

data/lib/nochmal/migration_data/meta.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module MigrationData
+    # Track the status of the whole migration
+    class Meta < ActiveRecord::Base
+      self.table_name = :nochmal_migration_data_meta
+
+      before_save :update_status
+
+      def done?
+        status.to_s == "done"
+      end
+
+      def to_s
+        [
+          record_type, "#", uploader_type, ": ",
+          migrated, "/", expected, " -> ", status
+        ].join
+      end
+
+      private
+
+      def update_status
+        self.status = current_status
+      end
+
+      def current_status # rubocop:disable Metrics/CyclomaticComplexity
+        return nil if migrated.nil? && expected.nil?
+
+        if expected.positive? && migrated.nil?
+          "not migrated"
+        else
+          case migrated.to_i <=> expected
+          when -1 then "partial"
+          when 0  then "done"
+          when 1  then "too much"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nochmal/migration_data/status.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module MigrationData
+    # Track the status of individual uploads to be migrated
+    class Status < ActiveRecord::Base
+      self.table_name = :nochmal_migration_data_status
+
+      belongs_to :record, polymorphic: true
+
+      scope :missing, -> { where(status: "missing") }
+      scope :ok, -> { where(status: "ok") }
+
+      def migrated?
+        status.present?
+      end
+
+      def missing?
+        status.to_s == "missing"
+      end
+
+      def missing_message
+        "#{filename} was not found, but was attached to #{record}"
+      end
+    end
+  end
+end

data/lib/nochmal/migration_data/status_exists.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Nochmal
+  module MigrationData
+    # Provide some help when a migration got aborted.
+    #
+    # This might happen if the OS or the user kills the process.
+    #
+    # I suspect an OOM-Kill by the OS or Container-Runtime (K8s/OCP).
+    class StatusExists < StandardError
+      def initialize
+        super <<~ERROR
+          It seems like the migration has already been started.
+
+          You may want to resume with
+            rails nochmal:carrierwave:resume
+
+          Alternatively, you can manually delete the tables
+            - #{Status.table_name}
+            - #{Meta.table_name}
+          and rerun the migration completely.
+
+          If you want, you can examine the status by interacting with
+            - Nochmal::MigrationData::Meta # class/attachment-level stats
+            - Nochmal::MigrationData::Status # individual upload stats
+          in your rails console.
+        ERROR
+      end
+    end
+  end
+end

data/lib/nochmal/output.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "pastel"
+
+module Nochmal
+  # Handles output for the Reupload Task
+  class Output
+    class << self
+      def reupload(models)
+        puts reupload_header(models)
+        yield
+        puts reupload_footer
+      end
+
+      def model(model, skipping: false)
+        if skipping
+          puts "Skipping #{pastel.green(model)}"
+          return true
+        end
+
+        puts model_header(model)
+        yield
+        puts model_footer
+      end
+
+      def type(type, count, action)
+        puts type_header(type)
+        print attachment_summary(count, action)
+        yield
+        puts type_footer
+      end
+
+      def attachment(filename)
+        print attachment_detail(filename)
+      end
+
+      def notes(notes)
+        notes = Array.wrap(notes)
+        return unless notes.any?
+
+        puts reupload_notes(notes)
+      end
+
+      def print_result_indicator(status)
+        case status
+        when :ok      then print_progress_indicator
+        when :missing then print_failure_indicator
+        when :skip    then print_skip_indicator
+        when :noop    then nil
+        else print_unknown_indicator
+        end
+      end
+
+      def print_progress_indicator
+        print pastel.green(".")
+      end
+
+      def print_failure_indicator
+        print pastel.red("F")
+      end
+
+      def print_skip_indicator
+        print pastel.yellow("*")
+      end
+
+      def print_unknown_indicator
+        print pastel.blue("?")
+      end
+
+      private
+
+      def reupload_header(models)
+        model_text = "model".pluralize(models.count)
+        model_names = models.map { |model| pastel.green(model) }.join(", ")
+
+        <<~HEADER
+
+
+          ================================================================================
+          I have found #{models.count} #{model_text} to process: #{model_names}
+
+        HEADER
+      end
+
+      def model_header(model)
+        "Model #{pastel.green(model)}"
+      end
+
+      def type_header(type)
+        "  Type #{pastel.green(type)}"
+      end
+
+      def attachment_summary(count, action)
+        "    Going to #{action} #{count} #{"attachment".pluralize(count)}: "
+      end
+
+      def attachment_detail(filename)
+        "\n      - #{filename}"
+      end
+
+      def type_footer
+        "\n  Done!"
+      end
+
+      def model_footer
+        "Done!"
+      end
+
+      def reupload_footer
+        "\nAll attachments have been processed!"
+      end
+
+      def reupload_notes(notes)
+        <<~NOTES
+
+          ================================================================================
+          #{notes.join("\n")}
+          ================================================================================
+        NOTES
+      end
+
+      def pastel
+        Pastel.new
+      end
+    end
+  end
+end