archive_storage 0.1.0

data/lib/archive_storage/adapters/s3.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "stringio"
+require "tempfile"
+require_relative "../errors"
+require_relative "metadata"
+
+module ArchiveStorage
+    module Adapters
+      class S3
+        attr_reader :config
+
+        def initialize(config)
+          @config = config
+        end
+
+        def upload(key, file, content_type: nil)
+          body, close_body = upload_body(file)
+
+          client.put_object(
+            bucket: config.bucket,
+            key: key,
+            body: body,
+            content_type: content_type || detect_content_type(file)
+          )
+        ensure
+          body.close if close_body && body.respond_to?(:close)
+        end
+
+        def upload_path(key, path, content_type: nil)
+          ::File.open(path, "rb") do |file|
+            client.put_object(
+              bucket: config.bucket,
+              key: key,
+              body: file,
+              content_type: content_type
+            )
+          end
+        end
+
+        def read(key)
+          client.get_object(bucket: config.bucket, key: key).body.read
+        rescue not_found_errors => error
+          raise NotFoundError, error.message
+        end
+
+        def download_to(key, path)
+          ::File.open(path, "wb") do |file|
+            client.get_object(bucket: config.bucket, key: key) do |chunk|
+              file.write(chunk)
+            end
+          end
+        rescue not_found_errors => error
+          raise NotFoundError, error.message
+        end
+
+        def copy_from(source_adapter, source_key, target_key)
+          source_metadata = source_adapter.head(source_key)
+
+          Tempfile.create("archive-storage-copy") do |tempfile|
+            tempfile.binmode
+            source_adapter.download_to(source_key, tempfile.path)
+            upload_path(target_key, tempfile.path, content_type: source_metadata.content_type)
+          end
+        end
+
+        def head(key)
+          response = client.head_object(bucket: config.bucket, key: key)
+
+          Metadata.new(
+            byte_size: response.content_length,
+            content_type: response.content_type,
+            etag: clean_etag(response.etag),
+            checksum: response_checksum(response),
+            checksum_algorithm: response_checksum_algorithm(response),
+            metadata: response.metadata || {}
+          )
+        rescue not_found_errors => error
+          raise NotFoundError, error.message
+        end
+
+        def exists?(key)
+          head(key)
+          true
+        rescue NotFoundError
+          false
+        end
+
+        def delete(key)
+          client.delete_object(bucket: config.bucket, key: key)
+          true
+        end
+
+        def url(key, expires_in: 3600, public: nil, **_options)
+          if public || config.public?
+            public_url(key)
+          else
+            presigner.presigned_url(
+              :get_object,
+              bucket: config.bucket,
+              key: key,
+              expires_in: expires_in
+            )
+          end
+        end
+
+        def client
+          @client ||= begin
+            require "aws-sdk-s3"
+
+            Aws::S3::Client.new(client_options)
+          end
+        end
+
+        private
+
+        def client_options
+          {
+            access_key_id: config.access_key_id,
+            secret_access_key: config.secret_access_key,
+            region: config.region,
+            endpoint: config.endpoint,
+            force_path_style: config.path_style?
+          }.compact.merge(config.options || {})
+        end
+
+        def presigner
+          require "aws-sdk-s3"
+          Aws::S3::Presigner.new(client: client)
+        end
+
+        def public_url(key)
+          host = config.public_host || "#{config.endpoint}/#{config.bucket}"
+          "#{host.to_s.delete_suffix("/")}/#{escape_key(key)}"
+        end
+
+        def upload_body(file)
+          if file.respond_to?(:path) && file.path
+            [::File.open(file.path, "rb"), true]
+          elsif file.respond_to?(:to_file) && file.to_file
+            [file.to_file, false]
+          elsif file.respond_to?(:read)
+            [file, false]
+          else
+            [StringIO.new(file.to_s), true]
+          end
+        end
+
+        def detect_content_type(file)
+          file.content_type if file.respond_to?(:content_type)
+        end
+
+        def clean_etag(etag)
+          etag&.delete_prefix("\"")&.delete_suffix("\"")
+        end
+
+        def response_checksum(response)
+          checksum_algorithm = response_checksum_algorithm(response)
+          return nil unless checksum_algorithm
+
+          response.public_send("checksum_#{checksum_algorithm}") if response.respond_to?("checksum_#{checksum_algorithm}")
+        end
+
+        def response_checksum_algorithm(response)
+          %w[sha256 sha1 crc32c crc32].find do |algorithm|
+            response.respond_to?("checksum_#{algorithm}") &&
+              response.public_send("checksum_#{algorithm}")
+          end
+        end
+
+        def escape_key(key)
+          key.to_s.split("/").map { |part| CGI.escape(part) }.join("/")
+        end
+
+        def not_found_errors
+          require "aws-sdk-s3"
+          [
+            Aws::S3::Errors::NoSuchKey,
+            Aws::S3::Errors::NotFound,
+            Aws::S3::Errors::NoSuchBucket
+          ]
+        end
+      end
+    end
+end

data/lib/archive_storage/configuration.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require_relative "storage_config"
+require_relative "mount_config"
+require_relative "schedule_config"
+require_relative "errors"
+
+module ArchiveStorage
+  class Configuration
+    attr_accessor :migration_queue,
+                  :schedule_queue,
+                  :job_backend,
+                  :verification_strategy,
+                  :default_batch_size,
+                  :default_cleanup_delay,
+                  :enqueue_claim_ttl,
+                  :delete_source_enabled,
+                  :registry_class_name,
+                  :fallback_on_read_errors
+
+    attr_reader :verify_checksums
+    attr_reader :storages, :mounts, :schedules
+
+    def initialize
+      @storages = {}
+      @mounts = []
+      @schedules = []
+      @adapter_cache = {}
+      @migration_queue = :default
+      @schedule_queue = :default
+      @job_backend = :active_job
+      @verification_strategy = :auto
+      @verify_checksums = false
+      @default_batch_size = 500
+      @default_cleanup_delay = 7 * 24 * 60 * 60
+      @enqueue_claim_ttl = 6 * 60 * 60
+      @delete_source_enabled = false
+      @registry_class_name = "ArchiveStorage::Models::FileRecord"
+      @fallback_on_read_errors = [NotFoundError]
+    end
+
+    def verify_checksums=(value)
+      @verify_checksums = value
+      @verification_strategy = :checksum if value
+    end
+
+    def storage(name, &block)
+      config = (@storages[name.to_sym] ||= StorageConfig.new(name))
+      block.call(config) if block
+      @adapter_cache.delete(name.to_sym)
+      config
+    end
+
+    def storage!(name)
+      @storages.fetch(name.to_sym) do
+        raise ConfigurationError, "unknown archive storage #{name.inspect}"
+      end
+    end
+
+    def adapter(name)
+      @adapter_cache[name.to_sym] ||= build_adapter(storage!(name))
+    end
+
+    def mount(model, mounted_as, uploader: nil, policy: nil)
+      MountConfig.new(model, mounted_as, uploader: uploader, policy: policy).tap do |mount|
+        @mounts.reject! { |existing| existing.matches_model?(model, mounted_as) }
+        @mounts << mount
+      end
+    end
+
+    def find_mount(model, mounted_as)
+      @mounts.find { |mount| mount.matches_model?(model, mounted_as) }
+    end
+
+    def schedule(name, cron:, model: nil, mounted_as: nil, uploader: nil, uploaders: nil, migration_rate: nil)
+      ScheduleConfig.new(
+        name,
+        cron: cron,
+        model: model,
+        mounted_as: mounted_as,
+        uploaders: Array(uploaders || uploader),
+        migration_rate: migration_rate
+      ).tap do |schedule|
+        schedule.validate!
+        @schedules << schedule
+      end
+    end
+
+    def registry_class
+      registry_class_name.to_s.split("::").inject(Object) do |namespace, const_name|
+        namespace.const_get(const_name)
+      end
+    end
+
+    private
+
+    def build_adapter(config)
+      return config.adapter if config.adapter
+
+      case config.provider.to_sym
+      when :s3
+        require_relative "adapters/s3"
+        Adapters::S3.new(config)
+      when :memory
+        require_relative "adapters/memory"
+        Adapters::Memory.new(config)
+      when :filesystem, :file, :nfs
+        require_relative "adapters/filesystem"
+        Adapters::FileSystem.new(config)
+      else
+        raise ConfigurationError, "unsupported storage provider #{config.provider.inspect}"
+      end
+    end
+  end
+end

data/lib/archive_storage/duration_parser.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+    class DurationParser
+      UNITS = {
+        "s" => 1,
+        "m" => 60,
+        "h" => 60 * 60,
+        "d" => 24 * 60 * 60,
+        "w" => 7 * 24 * 60 * 60,
+        "mo" => 30 * 24 * 60 * 60,
+        "y" => 365 * 24 * 60 * 60
+      }.freeze
+
+      def self.parse(value)
+        return nil if value.nil? || value == ""
+        return value if value.is_a?(Numeric)
+        return value.to_i if value.respond_to?(:to_i) && !value.is_a?(String)
+
+        match = value.to_s.strip.match(/\A(\d+)\s*(mo|[smhdwy])\z/i)
+        raise ArgumentError, "invalid duration #{value.inspect}" unless match
+
+        match[1].to_i * UNITS.fetch(match[2].downcase)
+      end
+    end
+end

data/lib/archive_storage/enqueuer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  class Enqueuer
+    def initialize(backend: ArchiveStorage.configuration.job_backend)
+      @backend = backend.to_sym
+    end
+
+    def enqueue_migration(file_record_id)
+      case backend
+      when :inline
+        require_relative "migrator"
+        ArchiveStorage::Migrator.new.migrate_record!(ArchiveStorage.configuration.registry_class.find(file_record_id))
+      when :active_job, :good_job
+        require_relative "jobs/migration_job"
+        Jobs::MigrationJob.perform_later(file_record_id)
+      when :sidekiq
+        require_relative "jobs/sidekiq_migration_worker"
+        Jobs::SidekiqMigrationWorker.perform_async(file_record_id)
+      else
+        raise ConfigurationError, "unknown job backend #{backend.inspect}"
+      end
+    end
+
+    private
+
+    attr_reader :backend
+  end
+end

data/lib/archive_storage/errors.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+    Error = Class.new(StandardError)
+    ConfigurationError = Class.new(Error)
+    NotFoundError = Class.new(Error)
+    VerificationError = Class.new(Error)
+    RegistryUnavailableError = Class.new(Error)
+end

data/lib/archive_storage/jobs/migration_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+begin
+  require "active_job"
+rescue LoadError
+  # ActiveJob is available in Rails apps.
+end
+
+require_relative "../migrator"
+
+module ArchiveStorage
+    module Jobs
+      if defined?(::ActiveJob::Base)
+        class MigrationJob < ::ActiveJob::Base
+          queue_as do
+            ArchiveStorage.configuration.migration_queue
+          end
+
+          def perform(file_record_id)
+            record = ArchiveStorage.configuration.registry_class.find_by(id: file_record_id)
+            return unless record
+
+            Migrator.new.migrate_record!(record)
+          end
+        end
+      end
+    end
+end

data/lib/archive_storage/jobs/queue_job.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+begin
+  require "active_job"
+rescue LoadError
+  # ActiveJob is available in Rails apps.
+end
+
+require_relative "../errors"
+require_relative "../migrator"
+require_relative "../planner"
+
+module ArchiveStorage
+  module Jobs
+    if defined?(::ActiveJob::Base)
+      class QueueJob < ::ActiveJob::Base
+        queue_as do
+          ArchiveStorage.configuration.schedule_queue
+        end
+
+        def perform(options = {})
+          options = symbolize_keys(options)
+          remaining = migration_limit(options)
+          total = 0
+
+          planner_options(options).each do |planner_options|
+            break if remaining && remaining <= 0
+
+            planner = Planner.new(**planner_options.merge(limit: remaining))
+            count = Migrator.new(planner: planner).enqueue_or_migrate!
+            remaining -= count if remaining
+            total += count
+          end
+
+          total
+        end
+
+        private
+
+        def symbolize_keys(hash)
+          hash.to_h.transform_keys(&:to_sym)
+        end
+
+        def scheduled_uploaders(options)
+          Array(options[:uploaders] || options[:uploader]).flatten.compact.tap do |uploaders|
+            raise ConfigurationError, "ArchiveStorage::Jobs::QueueJob requires uploader or uploaders" if uploaders.empty?
+          end
+        end
+
+        def planner_options(options)
+          if options[:model] && options[:mounted_as]
+            [{ model: options[:model], mounted_as: options[:mounted_as] }]
+          else
+            scheduled_uploaders(options).map { |uploader| { uploader: uploader } }
+          end
+        end
+
+        def migration_limit(options)
+          limit = options[:migration_rate] || options[:limit]
+          limit&.to_i
+        end
+      end
+    end
+  end
+end

data/lib/archive_storage/jobs/sidekiq_migration_worker.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+begin
+  require "sidekiq"
+rescue LoadError
+  # Sidekiq is optional. This worker is only used when job_backend is :sidekiq.
+end
+
+require_relative "../migrator"
+
+module ArchiveStorage
+  module Jobs
+    if defined?(::Sidekiq)
+      class SidekiqMigrationWorker
+        include ::Sidekiq::Worker
+
+        sidekiq_options queue: ArchiveStorage.configuration.migration_queue.to_s
+
+        def perform(file_record_id)
+          record = ArchiveStorage.configuration.registry_class.find_by(id: file_record_id)
+          return unless record
+
+          Migrator.new.migrate_record!(record)
+        end
+      end
+    end
+  end
+end

data/lib/archive_storage/jobs/sidekiq_queue_worker.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+begin
+  require "sidekiq"
+rescue LoadError
+  # Sidekiq is optional. Require this file only in Sidekiq-backed apps.
+end
+
+require_relative "../errors"
+require_relative "../migrator"
+require_relative "../planner"
+
+module ArchiveStorage
+  module Jobs
+    if defined?(::Sidekiq)
+      class SidekiqQueueWorker
+        include ::Sidekiq::Worker
+
+        sidekiq_options queue: ArchiveStorage.configuration.schedule_queue.to_s
+
+        def perform(options = {})
+          options = symbolize_keys(options)
+          remaining = migration_limit(options)
+          total = 0
+
+          planner_options(options).each do |planner_options|
+            break if remaining && remaining <= 0
+
+            planner = Planner.new(**planner_options.merge(limit: remaining))
+            count = Migrator.new(planner: planner).enqueue_or_migrate!
+            remaining -= count if remaining
+            total += count
+          end
+
+          total
+        end
+
+        private
+
+        def symbolize_keys(hash)
+          hash.to_h.transform_keys(&:to_sym)
+        end
+
+        def scheduled_uploaders(options)
+          Array(options[:uploaders] || options[:uploader]).flatten.compact.tap do |uploaders|
+            raise ConfigurationError, "ArchiveStorage::Jobs::SidekiqQueueWorker requires uploader or uploaders" if uploaders.empty?
+          end
+        end
+
+        def planner_options(options)
+          if options[:model] && options[:mounted_as]
+            [{ model: options[:model], mounted_as: options[:mounted_as] }]
+          else
+            scheduled_uploaders(options).map { |uploader| { uploader: uploader } }
+          end
+        end
+
+        def migration_limit(options)
+          limit = options[:migration_rate] || options[:limit]
+          limit&.to_i
+        end
+      end
+    end
+  end
+end

data/lib/archive_storage/migration_rate.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  class MigrationRate
+    attr_reader :files
+
+    def initialize(files)
+      @files = Integer(files)
+      raise ArgumentError, "migration rate must be greater than zero" unless files.positive?
+    end
+
+    def max_files_per_run
+      files
+    end
+  end
+end