archive_storage 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 32099fdbd406f02b06d31a562372c848c0d520aa6d4da0278793aecb4ed6cfe5
+  data.tar.gz: e28f3f328bcefdf82382b854a7dc7841367f9b6f42e83c5a65fb62082a6985a7
+SHA512:
+  metadata.gz: 2db3a3be0b2d4300e53d00c6ae1e60f450e9201dd37efe62b2c703f1f50a075e212e9ac09265c1b1814a567b0b43ed4e142aa873eac32c20f3d240dab3860146
+  data.tar.gz: 3fd152d37cb0f5e4f4f0566f9c1b0577fc877cddf728abde9b6a35fcbce5fa0834c1c2dbfa0c686f431e8537126a7aef8fe20606aa2b88694108cedff895a431

data/README.md

@@ -0,0 +1,348 @@
+# archive_storage
+
+Zero-downtime archival storage for CarrierWave uploads.
+
+`archive_storage` moves older uploaded files from one storage backend to another, keeps a registry of the current file location, and routes reads to the right backend. It currently integrates with CarrierWave; support for other uploader libraries can be added later without changing the registry model.
+
+Supported storage adapters:
+
+- S3-compatible object storage, including MinIO and AWS S3
+- filesystem/NFS
+- memory adapter for tests
+
+Typical use cases:
+
+- `main` S3/MinIO bucket -> `archive_001` cold bucket
+- `archive_001` -> `archive_002` when the first archive fills up
+- NFS/local disk -> S3-compatible archive storage
+
+## Features
+
+- model-first DSL: `archive_storage_for :file`
+- automatic CarrierWave storage wiring
+- ActiveRecord registry table: `archive_storage_files`
+- dry-run planning
+- scheduled enqueueing
+- background migration jobs
+- copy, verify, read switch, fallback read, delayed source cleanup
+- optional CarrierWave versions/thumbs migration
+- GoodJob, ActiveJob, Sidekiq, `sidekiq-cron`, and `sidekiq-scheduler` support
+
+## Installation
+
+Add the gem:
+
+```ruby
+gem "archive_storage"
+```
+
+For S3-compatible storage:
+
+```ruby
+gem "aws-sdk-s3"
+```
+
+Install the registry table:
+
+```bash
+bin/rails generate archive_storage:install
+bin/rails db:migrate
+```
+
+## Configuration
+
+Define the storage backends and scheduled archive jobs.
+
+```ruby
+# config/initializers/archive_storage.rb
+
+ArchiveStorage.configure do |config|
+  config.storage :main do |s|
+    s.provider = :s3
+    s.endpoint = ENV.fetch("MAIN_STORAGE_ENDPOINT")
+    s.bucket = "production-main"
+    s.access_key_id = ENV.fetch("MAIN_STORAGE_ACCESS_KEY")
+    s.secret_access_key = ENV.fetch("MAIN_STORAGE_SECRET_KEY")
+    s.region = "us-east-1"
+    s.path_style = true
+  end
+
+  config.storage :archive_001 do |s|
+    s.provider = :s3
+    s.endpoint = ENV.fetch("ARCHIVE_001_ENDPOINT")
+    s.bucket = "production-archive-001"
+    s.access_key_id = ENV.fetch("ARCHIVE_001_ACCESS_KEY")
+    s.secret_access_key = ENV.fetch("ARCHIVE_001_SECRET_KEY")
+    s.region = "us-east-1"
+    s.path_style = true
+  end
+
+  config.storage :archive_002 do |s|
+    s.provider = :s3
+    s.endpoint = ENV.fetch("ARCHIVE_002_ENDPOINT")
+    s.bucket = "production-archive-002"
+    s.access_key_id = ENV.fetch("ARCHIVE_002_ACCESS_KEY")
+    s.secret_access_key = ENV.fetch("ARCHIVE_002_SECRET_KEY")
+    s.region = "us-east-1"
+    s.path_style = true
+  end
+
+  config.schedule :archive_documents,
+                  cron: "0 0-6,22,23 * * 1-5",
+                  model: "ProjectDocument",
+                  mounted_as: :file,
+                  migration_rate: 10_000
+
+  # Optional defaults:
+  #
+  # config.job_backend = :active_job # :active_job, :good_job, :sidekiq, or :inline
+  # config.migration_queue = :default
+  # config.schedule_queue = :default
+  # config.default_batch_size = 500
+  # config.verification_strategy = :auto
+  # config.delete_source_enabled = false
+  # config.default_cleanup_delay = 7.days
+end
+```
+
+Filesystem/NFS storage can be mixed with S3-compatible storage:
+
+```ruby
+config.storage :nfs_main do |s|
+  s.provider = :filesystem
+  s.root_path = "/mnt/uploads"
+end
+```
+
+## Model Policy
+
+Put archive policy next to the model that owns the file.
+
+```ruby
+class ProjectDocument < ApplicationRecord
+  mount_uploader :file, DocumentUploader
+
+  archive_storage_for :file do
+    primary :main
+
+    archive :archive_001,
+      after: 90.days,
+      scope: :ready_for_archive,
+      if: ->(record) { record.closed? }
+
+    archive :archive_002,
+      after: 2.years,
+      scope: ->(records) { records.where(priority: "low") },
+      if: ->(record) { record.closed? }
+
+    read_fallbacks :main, :archive_001, :archive_002
+
+    # Optional:
+    #
+    # delete_source_after verification: true, delay: 7.days
+    # include_versions true
+    # versions :thumb, :preview
+    # timestamp_attribute :created_at
+  end
+end
+```
+
+`archive_storage_for` automatically wires the mounted CarrierWave uploader to `storage :archive_storage`. The uploader can stay focused on path, filename, and version behavior:
+
+```ruby
+class DocumentUploader < CarrierWave::Uploader::Base
+  def store_dir
+    "uploads/#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
+  end
+end
+```
+
+Policy notes:
+
+- `primary` is where new uploads are stored.
+- `archive` rules are checked in order; the last eligible rule wins.
+- `scope` narrows the model relation before records are scanned. It can be a model scope name, a relation, or a callable that receives the current relation.
+- `read_fallbacks` is the read-recovery order when registry metadata is missing or a configured fallback error is raised.
+- By default only the original CarrierWave file is planned. Use `include_versions true` or `versions ...` when thumbnails/previews must move too.
+
+## Scheduled Jobs
+
+Schedules are declared in global configuration:
+
+```ruby
+ArchiveStorage.configure do |config|
+  config.schedule :archive_documents,
+                  cron: "0 0-6,22,23 * * 1-5",
+                  model: "ProjectDocument",
+                  mounted_as: :file,
+                  migration_rate: 10_000
+end
+```
+
+`migration_rate` means at most this many files are enqueued by one scheduled run.
+
+`archive_storage` registers scheduler entries automatically. You do not need to merge `ArchiveStorage.good_job_cron` or `ArchiveStorage.sidekiq_cron` into your application config.
+
+### GoodJob
+
+When `good_job` is present, `archive_storage` appends its entries to `config.good_job.cron` after Rails initialization. Existing GoodJob cron entries are preserved.
+
+Enable GoodJob cron in the app environment where the scheduler should run:
+
+```ruby
+# config/environments/production.rb
+
+Rails.application.configure do
+  config.good_job.enable_cron = true
+end
+```
+
+### Sidekiq
+
+Use Sidekiq for migration jobs:
+
+```ruby
+# config/initializers/archive_storage.rb
+
+ArchiveStorage.configure do |config|
+  config.job_backend = :sidekiq
+end
+```
+
+Add one scheduler gem:
+
+```ruby
+gem "sidekiq-cron"
+# or
+gem "sidekiq-scheduler"
+```
+
+On Sidekiq server startup, `archive_storage` adds its own schedules without deleting existing jobs:
+
+- with `sidekiq-cron`, it uses non-destructive `Sidekiq::Cron::Job.load_from_hash`
+- with `sidekiq-scheduler`, it uses `Sidekiq.set_schedule` and reloads the scheduler
+
+Existing jobs from `sidekiq.yml`, `config/schedule.yml`, or custom initializers remain in place.
+
+## Commands
+
+```bash
+bin/rails archive_storage:plan MODEL=ProjectDocument MOUNT=file
+bin/rails archive_storage:enqueue MODEL=ProjectDocument MOUNT=file
+bin/rails archive_storage:migrate MODEL=ProjectDocument MOUNT=file
+bin/rails archive_storage:verify
+bin/rails archive_storage:cleanup_source
+bin/rails archive_storage:status
+```
+
+Options:
+
+```bash
+MODEL=ProjectDocument
+MOUNT=file
+OLDER_THAN=90d
+LIMIT=10000
+INLINE=true
+ESTIMATE_SIZES=false
+```
+
+`UPLOADER=DocumentUploader` is still accepted for advanced/legacy uploader-level configurations.
+
+Command behavior:
+
+- `plan` prints a dry-run plan.
+- `enqueue` and `migrate` enqueue migration jobs by default.
+- `migrate INLINE=true` runs migration inline.
+- `verify` re-checks already migrated files.
+- `cleanup_source` deletes verified source copies that are past the cleanup delay.
+- `status` prints registry counters.
+
+## Migration Flow
+
+```text
+source only
+source + destination copied
+destination verified
+registry points reads to destination
+reads can fallback to source
+source deleted later when cleanup is enabled
+```
+
+Source deletion is disabled by default:
+
+```ruby
+config.delete_source_enabled = false
+```
+
+Turn it on only after the migration path has been verified in production:
+
+```ruby
+config.delete_source_enabled = true
+```
+
+Per-mount cleanup delay:
+
+```ruby
+archive_storage_for :file do
+  delete_source_after verification: true, delay: 7.days
+end
+```
+
+## Verification
+
+The default strategy is `:auto`.
+
+`archive_storage` does not blindly trust S3 ETags. Multipart S3 uploads can have ETags like `hash-3`, and uploading the same bytes to another storage can produce a different ETag.
+
+Strategies:
+
+- `:auto` - size check, then checksum when available, then non-multipart ETag, otherwise size-only
+- `:checksum` - require matching checksums
+- `:safe_etag` - require matching non-multipart ETags
+- `:etag` - require matching ETags, including multipart-looking values
+- `:byte_compare` - compare full file bytes after size check
+- `:size` - compare content length only
+
+```ruby
+ArchiveStorage.configure do |config|
+  config.verification_strategy = :auto
+end
+```
+
+## Registry
+
+The generated migration creates `archive_storage_files`.
+
+The registry stores:
+
+- model identity: `record_type`, `record_id`, `mounted_as`, `uploader`
+- object identity: `identifier`, `storage_key`, source/target keys
+- storage state: `current_storage`, `source_storage`, `target_storage`
+- migration state: enqueue, migration, verification, cleanup timestamps
+- metadata: byte size, checksum, content type, attempts, last error
+
+Business tables do not need extra columns for archive location.
+
+## CarrierWave Versions
+
+CarrierWave versions are disabled by default.
+
+```ruby
+archive_storage_for :file do
+  include_versions true
+end
+```
+
+To migrate only selected versions:
+
+```ruby
+archive_storage_for :file do
+  versions :thumb, :preview
+end
+```
+
+Use this only when those files are stored and read as part of the same archival policy. It can multiply the number of objects planned for migration.
+
+## Current Scope
+
+This MVP is focused on Rails, ActiveRecord, and CarrierWave. The storage and registry layers are not CarrierWave-specific, so other uploader integrations can be added later.

data/archive_storage.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "lib/archive_storage/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "archive_storage"
+  spec.version = ArchiveStorage::VERSION
+  spec.authors = ["E. Tashkovyan"]
+  spec.email = []
+
+  spec.summary = "Policy-based archive storage and zero-downtime file migration."
+  spec.description = "Move uploads across storage backends such as filesystem, NFS, MinIO, and S3 without downtime."
+  spec.homepage = "https://github.com/estashkovyan/archive_storage"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.1.0"
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "#{spec.homepage}/tree/main"
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/releases"
+
+  spec.files = Dir.chdir(__dir__) do
+    Dir[
+      "lib/**/*",
+      "LICENSE.txt",
+      "README.md",
+      "archive_storage.gemspec"
+    ]
+  end
+  spec.bindir = "exe"
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activejob", ">= 6.1", "< 9.0"
+  spec.add_dependency "activerecord", ">= 6.1", "< 9.0"
+  spec.add_dependency "activesupport", ">= 6.1", "< 9.0"
+  spec.add_dependency "railties", ">= 6.1", "< 9.0"
+
+  spec.add_development_dependency "aws-sdk-s3", "~> 1"
+  spec.add_development_dependency "carrierwave", ">= 2.2", "< 4.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+end

data/lib/archive_storage/adapters/filesystem.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require_relative "../errors"
+require_relative "metadata"
+
+module ArchiveStorage
+  module Adapters
+    class FileSystem
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      def upload(key, file, content_type: nil)
+        write(key, read_upload_body(file), content_type: content_type || detect_content_type(file))
+      end
+
+      def write(key, body, content_type: nil, metadata: {})
+        path = path_for(key)
+        ::FileUtils.mkdir_p(::File.dirname(path))
+        ::File.binwrite(path, body.to_s.b)
+        write_metadata(key, content_type: content_type, metadata: metadata)
+        true
+      end
+
+      def read(key)
+        raise_not_found(key) unless exists?(key)
+
+        ::File.binread(path_for(key))
+      end
+
+      def download_to(key, path)
+        raise_not_found(key) unless exists?(key)
+
+        ::FileUtils.mkdir_p(::File.dirname(path))
+        ::FileUtils.cp(path_for(key), path)
+      end
+
+      def copy_from(source_adapter, source_key, target_key)
+        metadata = source_adapter.head(source_key)
+        write(target_key, source_adapter.read(source_key), content_type: metadata.content_type, metadata: metadata.metadata || {})
+      end
+
+      def head(key)
+        raise_not_found(key) unless exists?(key)
+
+        body = ::File.binread(path_for(key))
+        stored_metadata = read_metadata(key)
+
+        checksum = Digest::MD5.hexdigest(body)
+
+        Metadata.new(
+          byte_size: body.bytesize,
+          content_type: stored_metadata[:content_type],
+          etag: nil,
+          checksum: checksum,
+          checksum_algorithm: "md5",
+          metadata: stored_metadata[:metadata] || {}
+        )
+      end
+
+      def exists?(key)
+        ::File.file?(path_for(key))
+      end
+
+      def delete(key)
+        ::FileUtils.rm_f(path_for(key))
+        ::FileUtils.rm_f(metadata_path_for(key))
+        true
+      end
+
+      def url(key, **_options)
+        raise_not_found(key) unless exists?(key)
+
+        if config.base_url
+          "#{config.base_url.to_s.delete_suffix("/")}/#{key}"
+        else
+          path_for(key)
+        end
+      end
+
+      private
+
+      def path_for(key)
+        root = config.root_path || raise(ConfigurationError, "filesystem storage #{config.name.inspect} requires root_path")
+        expanded_root = ::File.expand_path(root)
+        expanded_path = ::File.expand_path(::File.join(expanded_root, key.to_s))
+
+        unless expanded_path == expanded_root || expanded_path.start_with?("#{expanded_root}#{::File::SEPARATOR}")
+          raise ConfigurationError, "storage key escapes filesystem root: #{key.inspect}"
+        end
+
+        expanded_path
+      end
+
+      def metadata_path_for(key)
+        "#{path_for(key)}.archive_storage.json"
+      end
+
+      def read_upload_body(file)
+        return ::File.binread(file.path) if file.respond_to?(:path) && file.path
+        return file.read if file.respond_to?(:read)
+
+        file.to_s
+      end
+
+      def detect_content_type(file)
+        file.content_type if file.respond_to?(:content_type)
+      end
+
+      def write_metadata(key, content_type:, metadata:)
+        return if content_type.nil? && metadata.empty?
+
+        require "json"
+        ::File.binwrite(metadata_path_for(key), JSON.dump(content_type: content_type, metadata: metadata))
+      end
+
+      def read_metadata(key)
+        return {} unless ::File.file?(metadata_path_for(key))
+
+        require "json"
+        JSON.parse(::File.binread(metadata_path_for(key)), symbolize_names: true)
+      rescue JSON::ParserError
+        {}
+      end
+
+      def raise_not_found(key)
+        raise NotFoundError, "object #{key.inspect} not found"
+      end
+    end
+  end
+end

data/lib/archive_storage/adapters/memory.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "digest"
+require_relative "../errors"
+require_relative "metadata"
+
+module ArchiveStorage
+    module Adapters
+      class Memory
+        attr_reader :config
+
+        def initialize(config)
+          @config = config
+          @objects = {}
+        end
+
+        def upload(key, file, content_type: nil)
+          body = read_upload_body(file)
+          write(
+            key,
+            body,
+            content_type: content_type || detect_content_type(file)
+          )
+        end
+
+        def write(key, body, content_type: nil, metadata: {})
+          @objects[key] = {
+            body: body.to_s.b,
+            content_type: content_type,
+            metadata: metadata
+          }
+        end
+
+        def read(key)
+          object_for(key).fetch(:body)
+        end
+
+        def download_to(key, path)
+          ::File.binwrite(path, read(key))
+        end
+
+        def copy_from(source_adapter, source_key, target_key)
+          source_metadata = source_adapter.head(source_key)
+          write(
+            target_key,
+            source_adapter.read(source_key),
+            content_type: source_metadata.content_type,
+            metadata: source_metadata.metadata || {}
+          )
+        end
+
+        def head(key)
+          object = object_for(key)
+          body = object.fetch(:body)
+
+          Metadata.new(
+            byte_size: body.bytesize,
+            content_type: object[:content_type],
+            etag: Digest::MD5.hexdigest(body),
+            checksum: Digest::MD5.hexdigest(body),
+            checksum_algorithm: "md5",
+            metadata: object[:metadata] || {}
+          )
+        end
+
+        def exists?(key)
+          @objects.key?(key)
+        end
+
+        def delete(key)
+          @objects.delete(key)
+          true
+        end
+
+        def url(key, **_options)
+          raise NotFoundError, "object #{key.inspect} not found" unless exists?(key)
+
+          "memory://#{config.name}/#{key}"
+        end
+
+        private
+
+        def object_for(key)
+          @objects.fetch(key) do
+            raise NotFoundError, "object #{key.inspect} not found"
+          end
+        end
+
+        def read_upload_body(file)
+          return ::File.binread(file.path) if file.respond_to?(:path) && file.path
+          return file.read if file.respond_to?(:read)
+
+          file.to_s
+        end
+
+        def detect_content_type(file)
+          file.content_type if file.respond_to?(:content_type)
+        end
+      end
+    end
+end

data/lib/archive_storage/adapters/metadata.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module ArchiveStorage
+  module Adapters
+    Metadata = Struct.new(
+      :byte_size,
+      :content_type,
+      :etag,
+      :checksum,
+      :checksum_algorithm,
+      :metadata,
+      keyword_init: true
+    ) do
+      def multipart_etag?
+        etag.to_s.include?("-")
+      end
+
+      def safe_etag?
+        etag && !multipart_etag?
+      end
+    end
+  end
+end