archive_storage 0.1.0

data/lib/archive_storage/migrator.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+require_relative "enqueuer"
+require_relative "planner"
+require_relative "verifier"
+
+module ArchiveStorage
+  class Migrator
+    def initialize(planner: nil)
+      @planner = planner
+    end
+
+    def enqueue_or_migrate!(inline: false)
+      count = 0
+
+      planner.each_candidate do |candidate|
+        file_record = ArchiveStorage.registry.claim_candidate(candidate)
+        next unless file_record
+
+        if inline
+          migrate_record!(file_record)
+        else
+          Enqueuer.new.enqueue_migration(file_record.id)
+        end
+
+        count += 1
+      end
+
+      count
+    end
+
+    def migrate_record!(file_record)
+      file_record.with_lock do
+        source_storage = (file_record.source_storage || file_record.current_storage).to_sym
+        target_storage = file_record.target_storage.to_sym
+        source_key = file_record_source_key(file_record)
+        target_key = file_record_target_key(file_record)
+
+        return file_record if source_storage == target_storage && file_record.verified_at
+
+        file_record.update!(
+          migration_started_at: Time.now,
+          attempts: file_record.attempts.to_i + 1,
+          last_error: nil
+        )
+
+        source = ArchiveStorage.adapter(source_storage)
+        target = ArchiveStorage.adapter(target_storage)
+
+        target.copy_from(source, source_key, target_key)
+        verification = Verifier.new.verify!(
+          source_adapter: source,
+          target_adapter: target,
+          source_key: source_key,
+          target_key: target_key
+        )
+        target_metadata = verification.target_metadata
+
+        file_record.update!(
+          current_storage: target_storage.to_s,
+          source_storage: source_storage.to_s,
+          target_storage: target_storage.to_s,
+          storage_key: target_key,
+          byte_size: target_metadata.byte_size,
+          content_type: target_metadata.content_type,
+          checksum: target_metadata.checksum || target_metadata.etag,
+          migrated_at: Time.now,
+          verified_at: Time.now,
+          source_delete_pending: source_storage != target_storage,
+          last_error: nil
+        )
+      end
+
+      file_record
+    rescue StandardError => error
+      safe_update_error(file_record, error)
+      raise
+    end
+
+    def verify_record!(file_record)
+      target_storage = (file_record.target_storage || file_record.current_storage).to_sym
+      metadata = ArchiveStorage.adapter(target_storage).head(file_record.storage_key)
+
+      file_record.update!(
+        byte_size: metadata.byte_size,
+        content_type: metadata.content_type,
+        checksum: metadata.etag,
+        verified_at: Time.now,
+        last_error: nil
+      )
+    end
+
+    def cleanup_source!(file_record)
+      return false unless cleanup_ready?(file_record)
+
+      source_storage = file_record.source_storage.to_sym
+      ArchiveStorage.adapter(source_storage).delete(file_record_source_key(file_record))
+      file_record.update!(source_deleted_at: Time.now, source_delete_pending: false)
+      true
+    end
+
+    private
+
+    attr_reader :planner
+
+    def cleanup_ready?(file_record)
+      return false unless ArchiveStorage.configuration.delete_source_enabled
+      return false unless file_record.source_storage
+      return false if file_record.source_deleted_at
+
+      policy = policy_for(file_record)
+      policy_delay = cleanup_delay_for(file_record)
+      requires_verification = policy.nil? || policy.delete_requires_verification
+      verified = !file_record.verified_at.nil?
+      reference_time = requires_verification ? file_record.verified_at : (file_record.verified_at || file_record.migrated_at)
+      old_enough = reference_time && reference_time <= Time.now - policy_delay
+
+      (!requires_verification || verified) && old_enough
+    end
+
+    def cleanup_delay_for(file_record)
+      policy_for(file_record)&.delete_source_delay&.to_i ||
+        ArchiveStorage.configuration.default_cleanup_delay
+    end
+
+    def policy_for(file_record)
+      ArchiveStorage.policy_for_record(file_record.record_type, file_record.mounted_as)
+    rescue StandardError
+      nil
+    end
+
+    def safe_update_error(file_record, error)
+      file_record.update!(last_error: "#{error.class}: #{error.message}") if file_record.respond_to?(:update!)
+    rescue StandardError
+      nil
+    end
+
+    def file_record_source_key(file_record)
+      return file_record.source_storage_key if file_record.respond_to?(:source_storage_key) && file_record.source_storage_key
+
+      file_record.storage_key
+    end
+
+    def file_record_target_key(file_record)
+      return file_record.target_storage_key if file_record.respond_to?(:target_storage_key) && file_record.target_storage_key
+
+      file_record.storage_key
+    end
+  end
+end

data/lib/archive_storage/model.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  module Model
+    def archive_storage_for(mounted_as, &block)
+      raise ArgumentError, "archive_storage_for requires a block" unless block
+
+      policy = PolicyBuilder.build(&block)
+      uploader_class = archive_storage_uploader_for(mounted_as)
+
+      ArchiveStorage.wire_carrierwave_uploader!(uploader_class)
+      ArchiveStorage.register_mount(self, mounted_as, uploader: uploader_class, policy: policy)
+
+      archive_storage_policies[mounted_as.to_sym] = policy
+    end
+
+    def archive_storage_policy_for(mounted_as)
+      archive_storage_policies[mounted_as.to_sym]
+    end
+
+    def archive_storage_policies
+      @archive_storage_policies ||= {}
+    end
+
+    private
+
+    def archive_storage_uploader_for(mounted_as)
+      if respond_to?(:uploaders) && uploaders[mounted_as.to_sym]
+        uploaders[mounted_as.to_sym]
+      else
+        raise ConfigurationError, "cannot find CarrierWave uploader for #{name}##{mounted_as}"
+      end
+    end
+  end
+end

data/lib/archive_storage/models/file_record.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+begin
+  require "active_record"
+rescue LoadError
+  # ActiveRecord is loaded in Rails apps. The registry object handles absence.
+end
+
+module ArchiveStorage
+    module Models
+      if defined?(::ActiveRecord::Base)
+        class FileRecord < ::ActiveRecord::Base
+          self.table_name = "archive_storage_files"
+
+          scope :verified, -> { where.not(verified_at: nil) }
+          scope :pending_cleanup, -> {
+            verified.where(source_delete_pending: true, source_deleted_at: nil).where.not(source_storage: nil)
+          }
+
+          def verified?
+            !!verified_at
+          end
+        end
+      end
+    end
+end

data/lib/archive_storage/mount_config.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  class MountConfig
+    attr_reader :model, :mounted_as, :uploader, :policy
+
+    def initialize(model, mounted_as, uploader: nil, policy: nil)
+      @model = model
+      @mounted_as = mounted_as.to_sym
+      @uploader = uploader
+      @policy = policy
+    end
+
+    def model_class
+      constantize(model)
+    end
+
+    def model_name
+      model.respond_to?(:name) ? model.name : model.to_s
+    end
+
+    def uploader_class
+      constantize(uploader) if uploader
+    end
+
+    def matches_model?(value, mounted_as_value = nil)
+      model_matches = model_name == (value.respond_to?(:name) ? value.name : value.to_s)
+      mount_matches = mounted_as_value.nil? || mounted_as == mounted_as_value.to_sym
+
+      model_matches && mount_matches
+    end
+
+    def matches_uploader?(value)
+      return true if value.nil?
+
+      expected = uploader_class || value
+      expected.to_s == value.to_s ||
+        (expected.respond_to?(:name) && expected.name == value.to_s)
+    end
+
+    private
+
+    def constantize(value)
+      return value if value.respond_to?(:name) && value.respond_to?(:all)
+      return value if value.is_a?(Class)
+
+      value.to_s.split("::").inject(Object) { |namespace, name| namespace.const_get(name) }
+    end
+  end
+end

data/lib/archive_storage/plan_result.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+    class PlanResult
+      attr_accessor :uploader_name, :destination
+      attr_reader :candidates, :byte_size, :by_model
+
+      def initialize(uploader_name: nil, destination: nil)
+        @uploader_name = uploader_name
+        @destination = destination
+        @candidates = 0
+        @byte_size = 0
+        @by_model = Hash.new { |hash, key| hash[key] = { count: 0, byte_size: 0 } }
+      end
+
+      def add(candidate)
+        @candidates += 1
+        @byte_size += candidate.byte_size.to_i
+        row = by_model[candidate.record.class.name]
+        row[:count] += 1
+        row[:byte_size] += candidate.byte_size.to_i
+      end
+
+      def to_text
+        lines = []
+        lines << "ArchiveStorage Migration Plan"
+        lines << ""
+        lines << "Uploader: #{uploader_name || "all registered uploaders"}"
+        lines << "Candidates: #{candidates}"
+        lines << "Estimated size: #{format_bytes(byte_size)}"
+        lines << ""
+
+        if by_model.any?
+          lines << "By model:"
+          by_model.each do |model, stats|
+            lines << "- #{model}: #{stats[:count]} files, #{format_bytes(stats[:byte_size])}"
+          end
+          lines << ""
+        end
+
+        lines << "Destination: #{destination}" if destination
+        lines << "No files were moved."
+        lines.join("\n")
+      end
+
+      private
+
+      def format_bytes(value)
+        units = %w[B KB MB GB TB PB]
+        size = value.to_f
+        unit = units.shift
+
+        while size >= 1024 && units.any?
+          size /= 1024.0
+          unit = units.shift
+        end
+
+        "#{format("%.2f", size)} #{unit}"
+      end
+    end
+end

data/lib/archive_storage/planner.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require_relative "duration_parser"
+require_relative "plan_result"
+
+module ArchiveStorage
+  Candidate = Struct.new(
+    :record,
+    :mounted_as,
+    :uploader,
+    :identifier,
+    :storage_key,
+    :source_storage_key,
+    :target_storage_key,
+    :current_storage,
+    :target_storage,
+    :byte_size,
+    :content_type,
+    keyword_init: true
+  )
+
+  class Planner
+    attr_reader :uploader_name,
+                :model_name,
+                :mounted_as,
+                :older_than,
+                :limit,
+                :estimate_sizes
+
+    def initialize(uploader: nil, model: nil, mounted_as: nil, older_than: nil, limit: nil, estimate_sizes: false)
+      @uploader_name = uploader
+      @model_name = model
+      @mounted_as = mounted_as&.to_sym
+      @older_than = DurationParser.parse(older_than)
+      @limit = limit&.to_i
+      @estimate_sizes = estimate_sizes
+    end
+
+    def call
+      result = PlanResult.new(uploader_name: uploader_name)
+      each_candidate { |candidate| result.add(candidate) }
+      result
+    end
+
+    def each_candidate
+      return enum_for(:each_candidate) unless block_given?
+
+      count = 0
+      mounts.each do |mount|
+        each_record(mount) do |record|
+          candidates_for(record, mount).each do |candidate|
+            yield candidate
+            count += 1
+            return if limit && count >= limit
+          end
+        end
+      end
+    end
+
+    private
+
+    def mounts
+      if model_name && mounted_as
+        [ArchiveStorage.configuration.find_mount(model_name, mounted_as) ||
+          MountConfig.new(model_name, mounted_as, uploader: uploader_name)]
+      else
+        ArchiveStorage.configuration.mounts.select do |mount|
+          mount.matches_uploader?(uploader_name)
+        end
+      end
+    end
+
+    def each_record(mount)
+      klass = mount.model_class
+      scope = scoped_records_for(klass, mount)
+
+      if klass.respond_to?(:column_names) && klass.column_names.include?(mount.mounted_as.to_s)
+        scope = scope.where.not(mount.mounted_as => [nil, ""])
+      end
+
+      scope.find_each(batch_size: ArchiveStorage.configuration.default_batch_size) do |record|
+        yield record
+      end
+    end
+
+    def candidates_for(record, mount)
+      return [] if too_young?(record)
+
+      uploader = record.public_send(mount.mounted_as)
+      identifier = identifier_for(record, uploader, mount)
+      return [] if identifier.nil? || identifier == ""
+
+      policy = policy_for(record, mount, uploader)
+      return [] unless policy
+
+      uploaders_for(uploader, policy).filter_map do |mounted_uploader|
+        candidate_for_uploader(record, mount, mounted_uploader, identifier, policy)
+      end
+    end
+
+    def scoped_records_for(klass, mount)
+      scope = klass.all
+      policy = ArchiveStorage.policy_for_mount(klass, mount.mounted_as) || uploader_policy_for(mount)
+
+      policy ? policy.apply_rule_scopes(scope) : scope
+    end
+
+    def uploader_policy_for(mount)
+      uploader_class = mount.uploader_class
+      uploader_class.archive_storage_policy if uploader_class.respond_to?(:archive_storage_policy)
+    rescue NameError
+      nil
+    end
+
+    def policy_for(record, mount, uploader)
+      mount.policy ||
+        ArchiveStorage.policy_for_mount(record.class, mount.mounted_as) ||
+        ArchiveStorage.policy_for_uploader(uploader)
+    end
+
+    def candidate_for_uploader(record, mount, uploader, identifier, policy)
+      storage_key = uploader.store_path(identifier)
+      current_storage = ArchiveStorage.registry.current_storage_for(
+        uploader,
+        identifier: identifier,
+        storage_key: storage_key,
+        default: policy.primary_storage_key
+      )
+      target_storage = policy.target_storage_for(record)
+      return nil unless target_storage
+      return nil if current_storage.to_sym == target_storage.to_sym
+
+      metadata = estimate_metadata(target_storage, policy.primary_storage_key, storage_key)
+
+      Candidate.new(
+        record: record,
+        mounted_as: mount.mounted_as,
+        uploader: uploader,
+        identifier: identifier,
+        storage_key: storage_key,
+        source_storage_key: storage_key,
+        target_storage_key: storage_key,
+        current_storage: current_storage,
+        target_storage: target_storage,
+        byte_size: metadata&.byte_size,
+        content_type: metadata&.content_type
+      )
+    end
+
+    def uploaders_for(uploader, policy)
+      uploaders = [uploader]
+      return uploaders unless uploader.respond_to?(:versions)
+
+      version_names = policy.selected_versions
+      version_names ||= uploader.versions.keys if policy.include_versions
+      return uploaders unless version_names
+
+      version_names.each do |name|
+        next unless uploader.versions.key?(name.to_sym)
+        next if uploader.respond_to?(:version_active?) && !uploader.version_active?(name)
+
+        uploaders << uploader.versions.fetch(name.to_sym)
+      end
+
+      uploaders
+    end
+
+    def too_young?(record)
+      return false unless older_than
+
+      timestamp = record.created_at if record.respond_to?(:created_at)
+      timestamp && timestamp > Time.now - older_than
+    end
+
+    def identifier_for(record, uploader, mount)
+      return uploader.identifier if uploader.respond_to?(:identifier) && uploader.identifier
+      return record.public_send(mount.mounted_as) if record.respond_to?(mount.mounted_as)
+
+      nil
+    end
+
+    def estimate_metadata(_target_storage, source_storage, storage_key)
+      return nil unless estimate_sizes
+
+      ArchiveStorage.adapter(source_storage).head(storage_key)
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/archive_storage/policy.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative "storage_rule"
+
+module ArchiveStorage
+  class Policy
+    attr_accessor :primary_storage,
+                  :rules,
+                  :read_fallbacks,
+                  :delete_source_delay,
+                  :delete_requires_verification,
+                  :include_versions,
+                  :selected_versions,
+                  :timestamp_attribute
+
+    def initialize
+      @rules = []
+      @read_fallbacks = []
+      @delete_source_delay = nil
+      @delete_requires_verification = true
+      @include_versions = false
+      @selected_versions = nil
+      @timestamp_attribute = :created_at
+    end
+
+    def primary_storage_key
+      primary_storage&.storage_key
+    end
+
+    def target_storage_for(record, now: Time.now)
+      eligible_rules = rules.select do |rule|
+        rule.eligible?(record, now: now, timestamp_attribute: timestamp_attribute)
+      end
+
+      eligible_rules.last&.storage_key
+    end
+
+    def apply_rule_scopes(scope)
+      scoped_rules = rules.select(&:scoped?)
+      return scope if scoped_rules.empty?
+
+      relations = scoped_rules.map { |rule| rule.apply_scope(scope) }
+      relations.reduce do |combined, relation|
+        combined.respond_to?(:or) ? combined.or(relation) : combined
+      end
+    end
+  end
+end

data/lib/archive_storage/policy_builder.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require_relative "policy"
+require_relative "storage_rule"
+require_relative "errors"
+
+module ArchiveStorage
+  class PolicyBuilder
+    def self.build(&block)
+      new.tap { |builder| builder.instance_eval(&block) }.tap(&:validate!).policy
+    end
+
+    attr_reader :policy
+
+    def initialize
+      @policy = Policy.new
+    end
+
+    def validate!
+      raise ConfigurationError, "archive_storage requires a primary storage" unless policy.primary_storage_key
+    end
+
+    def primary(name)
+      policy.primary_storage = StorageRule.new(:primary, name)
+    end
+
+    alias hot primary
+
+    def warm(name, **options)
+      rule(:warm, name, **options)
+    end
+
+    def archive(name, **options)
+      rule(:archive, name, **options)
+    end
+
+    def storage(name, **options)
+      rule(:storage, name, **options)
+    end
+
+    def rule(role, name, **options)
+      policy.rules << StorageRule.new(
+        role,
+        name,
+        after: options[:after],
+        condition: options[:if],
+        scope: options[:scope]
+      )
+    end
+
+    def read_fallbacks(*names)
+      policy.read_fallbacks = names.flatten.compact.map(&:to_sym)
+    end
+
+    def delete_source_after(verification: true, delay:)
+      policy.delete_requires_verification = verification
+      policy.delete_source_delay = delay
+    end
+
+    def include_versions(value = true)
+      policy.include_versions = value
+    end
+
+    def versions(*names)
+      policy.selected_versions = names.flatten.compact.map(&:to_sym)
+    end
+
+    def timestamp_attribute(name)
+      policy.timestamp_attribute = name.to_sym
+    end
+  end
+end

data/lib/archive_storage/railtie.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module ArchiveStorage
+  class Railtie < ::Rails::Railtie
+    initializer "archive_storage.active_record" do
+      ActiveSupport.on_load(:active_record) do
+        extend ArchiveStorage::Model
+      end
+    end
+
+    initializer "archive_storage.install_scheduled_jobs" do |app|
+      app.config.after_initialize do
+        ArchiveStorage.install_scheduled_jobs!(rails_config: app.config)
+      end
+    end
+
+    rake_tasks do
+      load File.expand_path("tasks.rake", __dir__)
+    end
+  end
+end