archive_storage 0.1.0

data/lib/archive_storage/registry.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+require_relative "models/file_record"
+
+module ArchiveStorage
+    class Registry
+      def available?
+        defined?(::ActiveRecord::Base) &&
+          ::ActiveRecord::Base.connected? &&
+          record_class.table_exists?
+      rescue StandardError
+        false
+      end
+
+      def find_for_uploader(uploader, identifier:, storage_key:)
+        return nil unless available?
+        return nil unless uploader_identity_available?(uploader)
+
+        record_class.find_by(
+          record_type: uploader.model.class.name,
+          record_id: uploader.model.id,
+          mounted_as: uploader.mounted_as.to_s,
+          identifier: identifier.to_s,
+          storage_key: storage_key.to_s
+        )
+      end
+
+      def current_storage_for(uploader, identifier:, storage_key:, default:)
+        find_for_uploader(
+          uploader,
+          identifier: identifier,
+          storage_key: storage_key
+        )&.current_storage&.to_sym || default
+      end
+
+      def upsert_for_uploader(uploader, identifier:, storage_key:, current_storage:, metadata: {})
+        return nil unless available?
+        return nil unless uploader_identity_available?(uploader)
+
+        record = record_class.find_or_initialize_by(
+          record_type: uploader.model.class.name,
+          record_id: uploader.model.id,
+          mounted_as: uploader.mounted_as.to_s,
+          identifier: identifier.to_s,
+          storage_key: storage_key.to_s
+        )
+
+        record.uploader = uploader.class.name
+        record.current_storage = current_storage.to_s
+        record.byte_size ||= metadata[:byte_size]
+        record.content_type ||= metadata[:content_type]
+        record.checksum ||= metadata[:checksum]
+        record.save!
+        record
+      end
+
+      def claim_candidate(candidate)
+        raise RegistryUnavailableError, "archive_storage_files table is not available" unless available?
+
+        record = record_class.find_or_initialize_by(
+          record_type: candidate.record.class.name,
+          record_id: candidate.record.id,
+          mounted_as: candidate.mounted_as.to_s,
+          identifier: candidate.identifier.to_s,
+          storage_key: candidate.storage_key.to_s
+        )
+        return nil unless claimable?(record)
+
+        record.uploader = candidate.uploader.class.name
+        record.current_storage = candidate.current_storage.to_s
+        record.source_storage = candidate.current_storage.to_s
+        record.target_storage = candidate.target_storage.to_s
+        record.source_storage_key = candidate.source_storage_key.to_s if record.respond_to?(:source_storage_key=)
+        record.target_storage_key = candidate.target_storage_key.to_s if record.respond_to?(:target_storage_key=)
+        record.enqueued_at = Time.now if record.respond_to?(:enqueued_at=)
+        record.source_delete_pending = false if record.respond_to?(:source_delete_pending=) && record.new_record?
+        record.byte_size ||= candidate.byte_size
+        record.content_type ||= candidate.content_type
+        record.save!
+        record
+      end
+
+      alias ensure_for_candidate claim_candidate
+
+      private
+
+      def record_class
+        ArchiveStorage.configuration.registry_class
+      end
+
+      def claimable?(record)
+        return false if record.respond_to?(:migrated_at) && record.migrated_at
+        return true unless record.respond_to?(:enqueued_at)
+        return true unless record.enqueued_at
+
+        record.enqueued_at <= Time.now - ArchiveStorage.configuration.enqueue_claim_ttl
+      end
+
+      def uploader_identity_available?(uploader)
+        uploader.respond_to?(:model) &&
+          uploader.model &&
+          uploader.model.respond_to?(:id) &&
+          uploader.model.id &&
+          uploader.respond_to?(:mounted_as) &&
+          uploader.mounted_as
+      end
+    end
+end

data/lib/archive_storage/schedule_config.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+require_relative "migration_rate"
+
+module ArchiveStorage
+  class ScheduleConfig
+    attr_reader :name, :cron, :model, :mounted_as, :uploaders, :migration_rate
+
+    def initialize(name, cron:, model: nil, mounted_as: nil, uploaders: [], migration_rate: nil)
+      @name = name.to_sym
+      @cron = cron
+      @model = model
+      @mounted_as = mounted_as&.to_sym
+      @uploaders = uploaders.flatten.compact.map(&:to_s)
+      @migration_rate = MigrationRate.new(migration_rate) if migration_rate
+    end
+
+    def entry_name
+      name
+    end
+
+    def job_arguments
+      {}.tap do |args|
+        if mount_schedule?
+          args[:model] = model_name
+          args[:mounted_as] = mounted_as.to_s
+        else
+          args[:uploaders] = uploaders
+        end
+
+        args[:migration_rate] = migration_rate.max_files_per_run if migration_rate
+      end
+    end
+
+    def validate!
+      raise ConfigurationError, "archive_storage schedule #{name.inspect} requires cron" if cron.nil? || cron == ""
+      return if mount_schedule? || uploaders.any?
+
+      raise ConfigurationError, "archive_storage schedule #{name.inspect} requires model/mounted_as or uploader"
+    end
+
+    def good_job_entry
+      validate!
+
+      {
+        cron: cron,
+        class: "ArchiveStorage::Jobs::QueueJob",
+        set: { queue: ArchiveStorage.configuration.schedule_queue },
+        args: [job_arguments]
+      }
+    end
+
+    def sidekiq_cron_entry
+      validate!
+
+      {
+        "cron" => cron,
+        "class" => "ArchiveStorage::Jobs::SidekiqQueueWorker",
+        "queue" => ArchiveStorage.configuration.schedule_queue.to_s,
+        "args" => [stringify_keys(job_arguments)]
+      }
+    end
+
+    private
+
+    def mount_schedule?
+      model && mounted_as
+    end
+
+    def model_name
+      model.respond_to?(:name) ? model.name : model.to_s
+    end
+
+    def stringify_keys(hash)
+      hash.transform_keys(&:to_s)
+    end
+  end
+end

data/lib/archive_storage/scheduler.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  class << self
+    def install_scheduled_jobs!(rails_config: nil)
+      if configuration.job_backend.to_sym == :sidekiq
+        install_sidekiq_schedules!
+      else
+        return false unless good_job_available?
+
+        install_good_job_cron!(rails_config: rails_config)
+      end
+    end
+
+    def install_good_job_cron!(rails_config: nil)
+      entries = good_job_cron
+      return false if entries.empty?
+
+      config = rails_config || rails_application_config
+      return false unless config&.respond_to?(:good_job)
+
+      good_job_config = config.good_job
+      good_job_config.cron = cron_hash(good_job_config.cron).merge(entries)
+      true
+    end
+
+    def install_sidekiq_schedules!
+      install_sidekiq_cron! || install_sidekiq_scheduler!
+    end
+
+    def install_sidekiq_cron!
+      require "sidekiq"
+      require "sidekiq-cron"
+      require_relative "jobs/sidekiq_queue_worker"
+
+      ::Sidekiq.configure_server do |config|
+        config.on(:startup) do
+          entries = ArchiveStorage.sidekiq_cron
+          ::Sidekiq::Cron::Job.load_from_hash(entries) if entries.any?
+        end
+      end
+
+      true
+    rescue LoadError
+      false
+    end
+
+    def install_sidekiq_scheduler!
+      require "sidekiq"
+      require "sidekiq-scheduler"
+      require_relative "jobs/sidekiq_queue_worker"
+
+      ::Sidekiq.configure_server do |config|
+        config.on(:startup) do
+          entries = ArchiveStorage.sidekiq_cron
+          entries.each { |name, entry| ::Sidekiq.set_schedule(name, entry) }
+          reload_sidekiq_scheduler if entries.any?
+        end
+      end
+
+      true
+    rescue LoadError
+      false
+    end
+
+    private
+
+    def cron_hash(value)
+      return {} if value.nil?
+      return value.to_h if value.respond_to?(:to_h)
+
+      {}
+    end
+
+    def good_job_available?
+      defined?(::GoodJob) || Gem.loaded_specs.key?("good_job")
+    end
+
+    def reload_sidekiq_scheduler
+      if defined?(::SidekiqScheduler::Scheduler)
+        ::SidekiqScheduler::Scheduler.instance.reload_schedule!
+      elsif defined?(::Sidekiq::Scheduler) && ::Sidekiq::Scheduler.respond_to?(:reload_schedule!)
+        ::Sidekiq::Scheduler.reload_schedule!
+      end
+    end
+
+    def rails_application_config
+      return nil unless defined?(::Rails) && ::Rails.respond_to?(:application)
+
+      ::Rails.application&.config
+    end
+  end
+end

data/lib/archive_storage/storage.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative "stored_file"
+
+module ArchiveStorage
+  class Storage
+    attr_reader :uploader, :identifier
+
+    def initialize(uploader)
+      @uploader = uploader
+    end
+
+    def store!(file)
+      identifier = identifier_for(file)
+      @identifier = identifier
+      storage_key = uploader.store_path(identifier)
+      primary_storage = policy.primary_storage_key
+      adapter = ArchiveStorage.adapter(primary_storage)
+
+      adapter.upload(storage_key, file)
+      metadata = safe_head(adapter, storage_key)
+
+      ArchiveStorage.registry.upsert_for_uploader(
+        uploader,
+        identifier: identifier,
+        storage_key: storage_key,
+        current_storage: primary_storage,
+        metadata: {
+          byte_size: metadata&.byte_size,
+          content_type: metadata&.content_type,
+          checksum: metadata&.etag
+        }
+      )
+
+      StoredFile.new(uploader, identifier, storage_key: storage_key)
+    end
+
+    def retrieve!(identifier)
+      @identifier = identifier
+      StoredFile.new(
+        uploader,
+        identifier,
+        storage_key: uploader.store_path(identifier)
+      )
+    end
+
+    def cache!(new_file)
+      file_cache_storage.cache!(new_file)
+    end
+
+    def retrieve_from_cache!(identifier)
+      file_cache_storage.retrieve_from_cache!(identifier)
+    end
+
+    def delete_dir!(path)
+      file_cache_storage.delete_dir!(path)
+    end
+
+    def clean_cache!(seconds)
+      file_cache_storage.clean_cache!(seconds)
+    end
+
+    private
+
+    def identifier_for(file)
+      return file.filename if file.respond_to?(:filename) && file.filename
+      return uploader.filename if uploader.respond_to?(:filename) && uploader.filename
+
+      raise ArgumentError, "cannot infer upload identifier for archive storage upload"
+    end
+
+    def policy
+      ArchiveStorage.policy_for_uploader(uploader) ||
+        raise(ConfigurationError, "archive_storage policy is not configured for #{uploader.class.name}")
+    end
+
+    def safe_head(adapter, storage_key)
+      adapter.head(storage_key)
+    rescue StandardError
+      nil
+    end
+
+    def file_cache_storage
+      unless defined?(::CarrierWave::Storage::File)
+        raise ConfigurationError, "CarrierWave file storage is required for archive_storage cache storage"
+      end
+
+      @file_cache_storage ||= ::CarrierWave::Storage::File.new(uploader)
+    end
+  end
+end

data/lib/archive_storage/storage_config.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  class StorageConfig
+    attr_accessor :name,
+                  :provider,
+                  :endpoint,
+                  :bucket,
+                  :access_key_id,
+                  :secret_access_key,
+                  :region,
+                  :path_style,
+                  :public,
+                  :public_host,
+                  :root_path,
+                  :base_url,
+                  :adapter,
+                  :options
+
+    def initialize(name)
+      @name = name.to_sym
+      @provider = :s3
+      @region = "us-east-1"
+      @path_style = false
+      @public = false
+      @options = {}
+    end
+
+    def path_style?
+      !!path_style
+    end
+
+    def public?
+      !!public
+    end
+  end
+end

data/lib/archive_storage/storage_rule.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  class StorageRule
+    attr_reader :role, :storage_key, :after, :condition, :scope
+
+    def initialize(role, storage_key, after: nil, condition: nil, scope: nil)
+      @role = role.to_sym
+      @storage_key = storage_key.to_sym
+      @after = after
+      @condition = condition
+      @scope = scope
+    end
+
+    def eligible?(record, now:, timestamp_attribute:)
+      old_enough?(record, now: now, timestamp_attribute: timestamp_attribute) &&
+        condition_matches?(record)
+    end
+
+    def scoped?
+      !scope.nil?
+    end
+
+    def apply_scope(relation)
+      case scope
+      when Symbol, String
+        relation.public_send(scope)
+      when Proc
+        scope.arity.zero? ? relation.instance_exec(&scope) : scope.call(relation)
+      else
+        relation.respond_to?(:merge) ? relation.merge(scope) : scope
+      end
+    end
+
+    private
+
+    def old_enough?(record, now:, timestamp_attribute:)
+      return true unless after
+      return false unless record
+
+      timestamp = record.public_send(timestamp_attribute) if record.respond_to?(timestamp_attribute)
+      return false unless timestamp
+
+      timestamp <= now - seconds_after
+    end
+
+    def condition_matches?(record)
+      return true unless condition
+
+      condition.arity.zero? ? condition.call : condition.call(record)
+    end
+
+    def seconds_after
+      after.respond_to?(:to_i) ? after.to_i : after
+    end
+  end
+end

data/lib/archive_storage/stored_file.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module ArchiveStorage
+  class StoredFile
+    attr_reader :uploader, :identifier, :storage_key
+
+    def initialize(uploader, identifier, storage_key: nil)
+      @uploader = uploader
+      @identifier = identifier.to_s
+      @storage_key = storage_key || uploader.store_path(identifier)
+    end
+
+    def path
+      storage_key
+    end
+
+    def filename
+      ::File.basename(identifier)
+    end
+
+    def url(*args, **options)
+      positional_options = args.last.is_a?(Hash) ? args.pop : {}
+      url_options = positional_options.merge(options)
+
+      with_read_adapter { |adapter| adapter.url(storage_key, **url_options) }
+    end
+
+    def read
+      with_read_adapter { |adapter| adapter.read(storage_key) }
+    end
+
+    def size
+      with_read_adapter { |adapter| adapter.head(storage_key).byte_size }
+    end
+
+    def content_type
+      with_read_adapter { |adapter| adapter.head(storage_key).content_type }
+    end
+
+    def exists?
+      candidate_storages.any? do |storage|
+        ArchiveStorage.adapter(storage).exists?(storage_key)
+      rescue *fallback_errors
+        false
+      end
+    end
+
+    def delete
+      adapter_for(current_storage).delete(storage_key)
+    end
+
+    def current_storage
+      ArchiveStorage.registry.current_storage_for(
+        uploader,
+        identifier: identifier,
+        storage_key: storage_key,
+        default: policy.primary_storage_key
+      )
+    end
+
+    def candidate_storages
+      ([current_storage] + policy.read_fallbacks + [policy.primary_storage_key]).compact.uniq
+    end
+
+    private
+
+    def with_read_adapter
+      last_error = nil
+
+      candidate_storages.each do |storage|
+        return yield adapter_for(storage)
+      rescue *fallback_errors => error
+        last_error = error
+      end
+
+      raise(last_error || NotFoundError, "object #{storage_key.inspect} not found")
+    end
+
+    def adapter_for(storage)
+      ArchiveStorage.adapter(storage)
+    end
+
+    def policy
+      ArchiveStorage.policy_for_uploader(uploader) ||
+        raise(ConfigurationError, "archive_storage policy is not configured for #{uploader.class.name}")
+    end
+
+    def fallback_errors
+      ArchiveStorage.configuration.fallback_on_read_errors
+    end
+  end
+end

data/lib/archive_storage/tasks.rake

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "archive_storage/planner"
+require "archive_storage/migrator"
+require "archive_storage/jobs/migration_job"
+
+namespace :archive_storage do
+  desc "Dry-run ArchiveStorage migration"
+  task plan: :environment do
+    planner = ArchiveStorage::Planner.new(
+      uploader: ENV["UPLOADER"],
+      model: ENV["MODEL"],
+      mounted_as: ENV["MOUNT"],
+      older_than: ENV["OLDER_THAN"],
+      limit: ENV["LIMIT"],
+      estimate_sizes: ENV.fetch("ESTIMATE_SIZES", "true") != "false"
+    )
+
+    puts planner.call.to_text
+  end
+
+  desc "Enqueue or run ArchiveStorage migration"
+  task migrate: :environment do
+    planner = ArchiveStorage::Planner.new(
+      uploader: ENV["UPLOADER"],
+      model: ENV["MODEL"],
+      mounted_as: ENV["MOUNT"],
+      older_than: ENV["OLDER_THAN"],
+      limit: ENV["LIMIT"],
+      estimate_sizes: true
+    )
+
+    inline = ENV.fetch("INLINE", "false") == "true"
+    count = ArchiveStorage::Migrator
+            .new(planner: planner)
+            .enqueue_or_migrate!(inline: inline)
+
+    puts "#{inline ? "Migrated" : "Enqueued"} #{count} files"
+  end
+
+  desc "Enqueue ArchiveStorage migration jobs"
+  task enqueue: :migrate
+
+  desc "Verify migrated ArchiveStorage files"
+  task verify: :environment do
+    scope = ArchiveStorage.configuration.registry_class.where.not(migrated_at: nil)
+    count = 0
+
+    scope.find_each do |file_record|
+      ArchiveStorage::Migrator.new.verify_record!(file_record)
+      count += 1
+    end
+
+    puts "Verified #{count} files"
+  end
+
+  desc "Delete verified source copies after cleanup delay"
+  task cleanup_source: :environment do
+    scope = ArchiveStorage.configuration.registry_class.pending_cleanup
+    count = 0
+
+    scope.find_each do |file_record|
+      count += 1 if ArchiveStorage::Migrator.new.cleanup_source!(file_record)
+    end
+
+    puts "Deleted #{count} source copies"
+  end
+
+  desc "Show ArchiveStorage migration status"
+  task status: :environment do
+    records = ArchiveStorage.configuration.registry_class
+
+    puts "ArchiveStorage Status"
+    puts ""
+    puts "Tracked:          #{records.count}"
+    puts "Migrated:         #{records.where.not(migrated_at: nil).count}"
+    puts "Verified:         #{records.where.not(verified_at: nil).count}"
+    puts "Pending cleanup:  #{records.pending_cleanup.count}"
+    puts "Source deleted:   #{records.where.not(source_deleted_at: nil).count}"
+    puts "Failed:           #{records.where.not(last_error: [nil, ""]).count}"
+  end
+end

data/lib/archive_storage/verification_result.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  VerificationResult = Struct.new(
+    :strategy,
+    :matched_by,
+    :source_metadata,
+    :target_metadata,
+    keyword_init: true
+  )
+end