archive_storage 0.1.0

data/lib/archive_storage/verifier.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+require_relative "adapters/metadata"
+require_relative "verification_result"
+
+module ArchiveStorage
+  class Verifier
+    def initialize(strategy: ArchiveStorage.configuration.verification_strategy)
+      @strategy = strategy.to_sym
+    end
+
+    def verify!(source_adapter:, target_adapter:, source_key:, target_key:)
+      source_metadata = source_adapter.head(source_key)
+      target_metadata = target_adapter.head(target_key)
+
+      verify_metadata!(source_metadata, target_metadata)
+      verify_bytes!(source_adapter, target_adapter, source_key, target_key) if strategy == :byte_compare
+
+      VerificationResult.new(
+        strategy: strategy,
+        matched_by: matched_by(source_metadata, target_metadata),
+        source_metadata: source_metadata,
+        target_metadata: target_metadata
+      )
+    end
+
+    private
+
+    attr_reader :strategy
+
+    def verify_metadata!(source_metadata, target_metadata)
+      verify_size!(source_metadata, target_metadata)
+
+      case strategy
+      when :auto
+        verify_auto!(source_metadata, target_metadata)
+      when :size
+        true
+      when :etag
+        verify_etag!(source_metadata, target_metadata, allow_multipart: true)
+      when :safe_etag
+        verify_safe_etag!(source_metadata, target_metadata)
+      when :checksum
+        verify_checksum!(source_metadata, target_metadata)
+      when :byte_compare
+        true
+      else
+        raise ConfigurationError, "unknown verification strategy #{strategy.inspect}"
+      end
+    end
+
+    def verify_size!(source_metadata, target_metadata)
+      return if source_metadata.byte_size == target_metadata.byte_size
+
+      raise VerificationError,
+            "byte size mismatch: #{source_metadata.byte_size} != #{target_metadata.byte_size}"
+    end
+
+    def verify_auto!(source_metadata, target_metadata)
+      if comparable_checksums?(source_metadata, target_metadata)
+        verify_checksum!(source_metadata, target_metadata)
+      elsif comparable_safe_etags?(source_metadata, target_metadata)
+        verify_safe_etag!(source_metadata, target_metadata)
+      else
+        true
+      end
+    end
+
+    def verify_checksum!(source_metadata, target_metadata)
+      unless comparable_checksums?(source_metadata, target_metadata)
+        raise VerificationError, "checksums are not available or use different algorithms"
+      end
+
+      return if source_metadata.checksum == target_metadata.checksum
+
+      raise VerificationError,
+            "checksum mismatch: #{source_metadata.checksum.inspect} != #{target_metadata.checksum.inspect}"
+    end
+
+    def verify_safe_etag!(source_metadata, target_metadata)
+      unless comparable_safe_etags?(source_metadata, target_metadata)
+        raise VerificationError, "safe etags are not available"
+      end
+
+      verify_etag!(source_metadata, target_metadata, allow_multipart: false)
+    end
+
+    def verify_etag!(source_metadata, target_metadata, allow_multipart:)
+      unless source_metadata.etag && target_metadata.etag
+        raise VerificationError, "etags are not available"
+      end
+
+      if !allow_multipart && (source_metadata.multipart_etag? || target_metadata.multipart_etag?)
+        raise VerificationError, "multipart etags are not stable content checksums"
+      end
+
+      return if source_metadata.etag == target_metadata.etag
+
+      raise VerificationError,
+            "etag mismatch: #{source_metadata.etag.inspect} != #{target_metadata.etag.inspect}"
+    end
+
+    def verify_bytes!(source_adapter, target_adapter, source_key, target_key)
+      source_body = source_adapter.read(source_key)
+      target_body = target_adapter.read(target_key)
+      return if source_body == target_body
+
+      raise VerificationError, "byte comparison mismatch"
+    end
+
+    def comparable_checksums?(source_metadata, target_metadata)
+      source_metadata.checksum &&
+        target_metadata.checksum &&
+        source_metadata.checksum_algorithm &&
+        source_metadata.checksum_algorithm == target_metadata.checksum_algorithm
+    end
+
+    def comparable_safe_etags?(source_metadata, target_metadata)
+      source_metadata.safe_etag? &&
+        target_metadata.safe_etag?
+    end
+
+    def matched_by(source_metadata, target_metadata)
+      case strategy
+      when :checksum
+        :checksum
+      when :etag
+        :etag
+      when :safe_etag
+        :safe_etag
+      when :byte_compare
+        :byte_compare
+      when :auto
+        return :checksum if comparable_checksums?(source_metadata, target_metadata)
+        return :safe_etag if comparable_safe_etags?(source_metadata, target_metadata)
+
+        :size
+      else
+        :size
+      end
+    end
+  end
+end

data/lib/archive_storage/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  VERSION = "0.1.0"
+end

data/lib/archive_storage.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+begin
+  require "carrierwave"
+rescue LoadError
+  # ArchiveStorage can be loaded without CarrierWave. CarrierWave integration is optional at runtime.
+end
+
+require_relative "archive_storage/version"
+require_relative "archive_storage/errors"
+require_relative "archive_storage/configuration"
+require_relative "archive_storage/policy_builder"
+require_relative "archive_storage/model"
+require_relative "archive_storage/registry"
+require_relative "archive_storage/storage"
+require_relative "archive_storage/planner"
+require_relative "archive_storage/enqueuer"
+require_relative "archive_storage/verifier"
+require_relative "archive_storage/migrator"
+require_relative "archive_storage/jobs/queue_job"
+require_relative "archive_storage/jobs/migration_job"
+
+module ArchiveStorage
+  class << self
+    attr_writer :configuration, :registry
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield configuration
+    end
+
+    def reset_configuration!
+      @configuration = Configuration.new
+      @registry = Registry.new
+    end
+
+    def adapter(name)
+      configuration.adapter(name)
+    end
+
+    def registry
+      @registry ||= Registry.new
+    end
+
+    def register_uploader(uploader_class)
+      uploaders << uploader_class
+    end
+
+    def register_mount(model, mounted_as, uploader:, policy:)
+      configuration.mount(model, mounted_as, uploader: uploader, policy: policy)
+    end
+
+    def wire_carrierwave_uploader!(uploader_class)
+      return unless uploader_class
+
+      uploader_class.include(CarrierWave) unless uploader_class < CarrierWave
+      uploader_class.storage(:archive_storage) if uploader_class.respond_to?(:storage)
+    end
+
+    def policy_for_uploader(uploader)
+      mount_policy_for_uploader(uploader) ||
+        (uploader.class.archive_storage_policy if uploader.class.respond_to?(:archive_storage_policy))
+    end
+
+    def policy_for_mount(model, mounted_as)
+      configuration.find_mount(model, mounted_as)&.policy ||
+        model_policy(model, mounted_as)
+    end
+
+    def policy_for_record(record_type, mounted_as)
+      policy_for_mount(record_type, mounted_as)
+    end
+
+    def uploaders
+      @uploaders ||= []
+    end
+
+    def good_job_cron
+      configuration.schedules.each_with_object({}) do |schedule, entries|
+        entries[schedule.entry_name] = schedule.good_job_entry
+      end
+    end
+
+    def sidekiq_cron
+      configuration.schedules.each_with_object({}) do |schedule, entries|
+        entries[schedule.entry_name.to_s] = schedule.sidekiq_cron_entry
+      end
+    end
+
+    private
+
+    def model_policy(model, mounted_as)
+      model_class = model.is_a?(Class) ? model : constantize(model)
+      return nil unless model_class.respond_to?(:archive_storage_policy_for)
+
+      model_class.archive_storage_policy_for(mounted_as)
+    rescue NameError
+      nil
+    end
+
+    def mount_policy_for_uploader(uploader)
+      return nil unless uploader.respond_to?(:model) && uploader.model
+      return nil unless uploader.respond_to?(:mounted_as) && uploader.mounted_as
+
+      policy_for_mount(uploader.model.class, uploader.mounted_as)
+    end
+
+    def constantize(value)
+      value.to_s.split("::").inject(Object) { |namespace, name| namespace.const_get(name) }
+    end
+  end
+
+  module CarrierWave
+    def self.included(base)
+      ArchiveStorage.register_uploader(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def archive_storage(&block)
+        if block
+          @archive_storage_policy = PolicyBuilder.build(&block)
+        else
+          archive_storage_policy
+        end
+      end
+
+      def archive_storage_policy
+        @archive_storage_policy ||
+          (superclass.archive_storage_policy if superclass.respond_to?(:archive_storage_policy))
+      end
+    end
+  end
+end
+
+if defined?(::CarrierWave) && ::CarrierWave.respond_to?(:configure)
+  ::CarrierWave.configure do |config|
+    if config.respond_to?(:storage_engines)
+      config.storage_engines[:archive_storage] = "ArchiveStorage::Storage"
+    end
+  end
+end
+
+require_relative "archive_storage/scheduler"
+require_relative "archive_storage/railtie" if defined?(::Rails::Railtie)

data/lib/generators/archive_storage/install_generator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module ArchiveStorage
+  class InstallGenerator < ::Rails::Generators::Base
+    include ::Rails::Generators::Migration
+    namespace "archive_storage:install"
+
+    source_root File.expand_path("templates", __dir__)
+
+    def copy_migration
+      migration_template(
+        "create_archive_storage_files.rb",
+        "db/migrate/create_archive_storage_files.rb"
+      )
+    end
+
+    def self.next_migration_number(dirname)
+      if ::ActiveRecord::Base.timestamped_migrations
+        Time.now.utc.strftime("%Y%m%d%H%M%S")
+      else
+        "%.3d" % (current_migration_number(dirname) + 1)
+      end
+    end
+  end
+end

data/lib/generators/archive_storage/templates/create_archive_storage_files.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class CreateArchiveStorageFiles < ActiveRecord::Migration[7.0]
+  def change
+    create_table :archive_storage_files do |t|
+      t.string :record_type, null: false
+      t.bigint :record_id, null: false
+      t.string :mounted_as, null: false
+      t.string :uploader, null: false
+
+      t.string :identifier, null: false
+      t.string :storage_key, null: false
+      t.string :source_storage_key
+      t.string :target_storage_key
+
+      t.string :current_storage, null: false
+      t.string :source_storage
+      t.string :target_storage
+
+      t.bigint :byte_size
+      t.string :checksum
+      t.string :content_type
+
+      t.datetime :migration_started_at
+      t.datetime :enqueued_at
+      t.datetime :migrated_at
+      t.datetime :verified_at
+      t.datetime :source_deleted_at
+
+      t.boolean :source_delete_pending, null: false, default: false
+      t.string :last_error
+      t.integer :attempts, null: false, default: 0
+
+      t.timestamps
+    end
+
+    add_index :archive_storage_files,
+              [:record_type, :record_id, :mounted_as, :identifier],
+              name: "idx_archive_storage_identity"
+
+    add_index :archive_storage_files,
+              [:uploader, :current_storage],
+              name: "idx_archive_storage_uploader_storage"
+
+    add_index :archive_storage_files,
+              [:source_storage, :source_deleted_at],
+              name: "idx_archive_storage_cleanup"
+
+    add_index :archive_storage_files,
+              [:source_delete_pending, :source_deleted_at],
+              name: "idx_archive_storage_delete_pending"
+  end
+end

metadata

@@ -0,0 +1,227 @@
+--- !ruby/object:Gem::Specification
+name: archive_storage
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- E. Tashkovyan
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: carrierwave
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: Move uploads across storage backends such as filesystem, NFS, MinIO,
+  and S3 without downtime.
+email: []
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- archive_storage.gemspec
+- lib/archive_storage.rb
+- lib/archive_storage/adapters/filesystem.rb
+- lib/archive_storage/adapters/memory.rb
+- lib/archive_storage/adapters/metadata.rb
+- lib/archive_storage/adapters/s3.rb
+- lib/archive_storage/configuration.rb
+- lib/archive_storage/duration_parser.rb
+- lib/archive_storage/enqueuer.rb
+- lib/archive_storage/errors.rb
+- lib/archive_storage/jobs/migration_job.rb
+- lib/archive_storage/jobs/queue_job.rb
+- lib/archive_storage/jobs/sidekiq_migration_worker.rb
+- lib/archive_storage/jobs/sidekiq_queue_worker.rb
+- lib/archive_storage/migration_rate.rb
+- lib/archive_storage/migrator.rb
+- lib/archive_storage/model.rb
+- lib/archive_storage/models/file_record.rb
+- lib/archive_storage/mount_config.rb
+- lib/archive_storage/plan_result.rb
+- lib/archive_storage/planner.rb
+- lib/archive_storage/policy.rb
+- lib/archive_storage/policy_builder.rb
+- lib/archive_storage/railtie.rb
+- lib/archive_storage/registry.rb
+- lib/archive_storage/schedule_config.rb
+- lib/archive_storage/scheduler.rb
+- lib/archive_storage/storage.rb
+- lib/archive_storage/storage_config.rb
+- lib/archive_storage/storage_rule.rb
+- lib/archive_storage/stored_file.rb
+- lib/archive_storage/tasks.rake
+- lib/archive_storage/verification_result.rb
+- lib/archive_storage/verifier.rb
+- lib/archive_storage/version.rb
+- lib/generators/archive_storage/install_generator.rb
+- lib/generators/archive_storage/templates/create_archive_storage_files.rb
+homepage: https://github.com/estashkovyan/archive_storage
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/estashkovyan/archive_storage
+  source_code_uri: https://github.com/estashkovyan/archive_storage/tree/main
+  changelog_uri: https://github.com/estashkovyan/archive_storage/releases
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Policy-based archive storage and zero-downtime file migration.
+test_files: []