cold_storage 0.1.0

data/lib/cold_storage/configuration.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # Global defaults. Every model can override most of these through its own
+  # `archivable` options.
+  class Configuration
+    TYPE_MISMATCH_STRATEGIES = %i[warn raise change ignore].freeze
+
+    # Master switch. When false, nothing is copied or deleted: scheduled runs
+    # report themselves as skipped and the on_destroy hook stands down. Handy
+    # for the test environment, which has no archive database.
+    # Schema tasks keep working, so `schema:check` can still run in CI.
+    attr_accessor :enabled
+    # Name of the database.yml entry holding the archive database.
+    attr_accessor :archive_database
+    # Rows copied (and deleted) per round trip.
+    attr_accessor :batch_size
+    # Default column used to decide how old a record is.
+    attr_accessor :timestamp_column
+    # Default column used by `deleted: true`.
+    attr_accessor :deleted_column
+    # Extra column added to every archive table, holding the archiving time.
+    # Set to nil to disable.
+    attr_accessor :archived_at_column
+    # Delete rows from the primary database once they are safely copied.
+    attr_accessor :delete_after_archive
+    # :delete_all (fast, no callbacks) or :destroy_all (slow, runs callbacks).
+    attr_accessor :delete_method
+    # Copy indexes to the archive tables.
+    attr_accessor :mirror_indexes
+    # Copy NOT NULL constraints. Off by default: an archive table should never
+    # reject historical rows because the primary schema grew stricter later.
+    attr_accessor :mirror_null_constraints
+    # Copy column defaults. Off by default, rows are always inserted complete.
+    attr_accessor :mirror_defaults
+    # Drop archive columns that no longer exist in the primary database. Off by
+    # default so that already archived data keeps its columns.
+    attr_accessor :drop_removed_columns
+    # What to do when a column type differs: :warn, :raise, :change or :ignore.
+    attr_reader :on_type_mismatch
+    # What to do when `on_destroy:` cannot reach the archive database:
+    # :raise (block the delete, keep the data) or :log (let the delete through).
+    attr_reader :on_destroy_error
+    # Run the schema mirror automatically after db:migrate / db:schema:load.
+    attr_accessor :sync_schema_after_migrate
+    # Seconds to sleep between batches, to keep long runs off the hot path.
+    attr_accessor :throttle
+    # ActiveJob queue used by the bundled jobs.
+    attr_accessor :job_queue
+    # Parent class of the bundled jobs, as a string.
+    attr_accessor :job_parent_class
+    # Log every statement/plan the gem produces without touching any data.
+    attr_accessor :dry_run
+    attr_accessor :logger
+
+    def initialize
+      @enabled                   = true
+      @archive_database          = :archive
+      @batch_size                = 1_000
+      @timestamp_column          = :created_at
+      @deleted_column            = :deleted_at
+      @archived_at_column        = :archived_at
+      @delete_after_archive      = true
+      @delete_method             = :delete_all
+      @mirror_indexes            = true
+      @mirror_null_constraints   = false
+      @mirror_defaults           = false
+      @drop_removed_columns      = false
+      @on_type_mismatch          = :warn
+      @on_destroy_error          = :raise
+      @sync_schema_after_migrate = true
+      @throttle                  = 0
+      @job_queue                 = :default
+      @job_parent_class          = 'ActiveJob::Base'
+      @dry_run                   = false
+      @logger                    = nil
+    end
+
+    def on_type_mismatch=(strategy)
+      strategy = strategy.to_sym
+      unless TYPE_MISMATCH_STRATEGIES.include?(strategy)
+        raise ConfigurationError,
+              "on_type_mismatch must be one of #{TYPE_MISMATCH_STRATEGIES.join(', ')}, got #{strategy.inspect}"
+      end
+
+      @on_type_mismatch = strategy
+    end
+
+    def on_destroy_error=(strategy)
+      strategy = strategy.to_sym
+      raise ConfigurationError, 'on_destroy_error must be :raise or :log' unless %i[raise log].include?(strategy)
+
+      @on_destroy_error = strategy
+    end
+
+    def delete_method=(method)
+      method = method.to_sym
+      unless %i[delete_all destroy_all].include?(method)
+        raise ConfigurationError, "delete_method must be :delete_all or :destroy_all, got #{method.inspect}"
+      end
+
+      @delete_method = method
+    end
+  end
+end

data/lib/cold_storage/internal_schema.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # The gem's own tables inside the archive database. They are created on
+  # demand: the archive database needs no migrations of its own.
+  module InternalSchema
+    RUNS_TABLE     = 'cold_storage_runs'
+    METADATA_TABLE = 'cold_storage_metadata'
+
+    class << self
+      def ensure!
+        ArchiveRecord.with_archive_connection do |connection|
+          create_runs_table(connection)
+          create_metadata_table(connection)
+        end
+        true
+      end
+
+      def ready?
+        ArchiveRecord.with_archive_connection do |connection|
+          connection.table_exists?(RUNS_TABLE) && connection.table_exists?(METADATA_TABLE)
+        end
+      end
+
+      private
+
+      def create_runs_table(connection)
+        return if connection.table_exists?(RUNS_TABLE)
+
+        connection.create_table(RUNS_TABLE) do |t|
+          t.string   :archived_model, null: false
+          t.string   :archived_table
+          t.string   :status, null: false, default: 'running'
+          t.integer  :archived_count, null: false, default: 0
+          t.integer  :deleted_count, null: false, default: 0
+          t.datetime :started_at
+          t.datetime :finished_at
+          t.text     :error_message
+          t.timestamps
+        end
+        connection.add_index(RUNS_TABLE, %i[archived_model status finished_at],
+                             name: 'index_cold_storage_runs_on_model_and_status')
+      end
+
+      def create_metadata_table(connection)
+        return if connection.table_exists?(METADATA_TABLE)
+
+        connection.create_table(METADATA_TABLE, id: false) do |t|
+          t.string   :key, null: false, primary_key: true
+          t.text     :value
+          t.datetime :updated_at
+        end
+      end
+    end
+  end
+end

data/lib/cold_storage/jobs.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # ActiveJob entry points.
+  #
+  # The classes are built on first use (or when ActiveJob loads) so that the
+  # gem does not force ActiveJob on applications that do not use it, and so
+  # that `config.job_parent_class` is honoured.
+  #
+  #   ColdStorage::ArchiveModelJob.perform_later('Payroll')
+  #   ColdStorage::ArchiveAllJob.perform_later
+  module Jobs
+    JOBS = %i[ArchiveModelJob ArchiveAllJob].freeze
+
+    class << self
+      # @return [Boolean] whether the job classes exist now
+      def define!
+        return false unless defined?(ActiveJob::Base)
+        return true if ColdStorage.const_defined?(:ArchiveModelJob, false)
+
+        parent = ColdStorage.config.job_parent_class.constantize
+        ColdStorage.const_set(:ArchiveModelJob, build_model_job(parent))
+        ColdStorage.const_set(:ArchiveAllJob, build_all_job(parent))
+        true
+      end
+
+      private
+
+      # Archives one model.
+      def build_model_job(parent)
+        Class.new(parent) do
+          queue_as { ColdStorage.config.job_queue }
+
+          def perform(model_name, **options)
+            ColdStorage.archive(model_name.to_s.constantize, **options.symbolize_keys)
+          end
+        end
+      end
+
+      # Fans out over every archivable model. Models whose `every:` window has
+      # not elapsed skip themselves, so this is safe to schedule daily.
+      def build_all_job(parent)
+        Class.new(parent) do
+          queue_as { ColdStorage.config.job_queue }
+
+          def perform(**options)
+            ColdStorage.models.each do |model|
+              ColdStorage::ArchiveModelJob.perform_later(model.name, **options.symbolize_keys)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def self.const_missing(name)
+    return super unless Jobs::JOBS.include?(name) && Jobs.define!
+
+    const_get(name)
+  end
+end

data/lib/cold_storage/logging.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # Shared, prefixed logging.
+  module Logging
+    PREFIX = '[ColdStorage]'
+
+    private
+
+    def log(message, level: :info)
+      ColdStorage.logger.public_send(level) { "#{PREFIX} #{message}" }
+    end
+
+    def warn_log(message)
+      log(message, level: :warn)
+    end
+  end
+end

data/lib/cold_storage/metadata.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # Key/value store in the archive database. Holds the migration version the
+  # mirrored schema was last synced against.
+  class Metadata < ArchiveRecord
+    self.table_name = InternalSchema::METADATA_TABLE
+    self.primary_key = 'key'
+
+    SCHEMA_VERSION_KEY = 'schema_version'
+
+    class << self
+      def get(key)
+        InternalSchema.ensure!
+        where(key: key.to_s).pick(:value)
+      end
+
+      def set(key, value)
+        InternalSchema.ensure!
+        upsert({ key: key.to_s, value: value.to_s, updated_at: Time.current }, unique_by: :key)
+        value
+      end
+
+      def schema_version
+        get(SCHEMA_VERSION_KEY)
+      end
+
+      def schema_version=(version)
+        set(SCHEMA_VERSION_KEY, version)
+      end
+    end
+  end
+end

data/lib/cold_storage/model.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # Extended onto ActiveRecord::Base, so every model can declare itself
+  # archivable.
+  module Model
+    # Declares which rows of this model move to the archive database.
+    #
+    #   class Payroll < ApplicationRecord
+    #     archivable after: 18.months, every: 1.month
+    #   end
+    #
+    # Options:
+    #   after:                Duration. Rows older than this are archivable.
+    #                         (alias: older_than:)
+    #   on:                   Column the age is measured on.
+    #                         Default: :created_at, or the deleted column when
+    #                         deleted: true.
+    #   deleted:              true to archive soft-deleted rows only.
+    #   deleted_column:       Column holding the soft-delete timestamp.
+    #                         Default: ColdStorage.config.deleted_column.
+    #   on_destroy:           true to copy a row to the archive whenever it is
+    #                         really deleted (destroy/destroy_all), so hard
+    #                         deletes are kept too. (alias: hard_delete:)
+    #   every:                Duration. Minimum time between two runs.
+    #   scope:                Symbol, proc or relation narrowing the selection.
+    #   cascade:              has_many/has_one association names to archive
+    #                         together with their parent.
+    #   batch_size:           Rows moved per round trip.
+    #   delete_after_archive: false to copy without deleting the source rows.
+    #   delete_method:        :delete_all (default) or :destroy_all.
+    #
+    # @return [ColdStorage::Policy]
+    def archivable(**options)
+      include ColdStorage::Archivable unless include?(ColdStorage::Archivable)
+
+      self.archiving_policy = Policy.new(self, **options)
+      install_archive_on_destroy! if archiving_policy.on_destroy?
+      Registry.register(self)
+      archiving_policy
+    end
+
+    # Every model answers this, so callers never have to rescue NoMethodError.
+    def archivable?
+      false
+    end
+  end
+end

data/lib/cold_storage/policy.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # The `archivable` options of a single model, and the relation they describe.
+  #
+  #   archivable after: 18.months                  # older than 18 months
+  #   archivable after: 30.days, on: :closed_at    # ... measured on another column
+  #   archivable deleted: true                     # soft-deleted rows only
+  #   archivable deleted: true, after: 90.days     # soft-deleted 90+ days ago
+  #   archivable after: 1.year, every: 1.month     # ... and only run monthly
+  #   archivable after: 2.years, cascade: [:items] # take the children along
+  #   archivable after: 2.years, cascade: { items: [:taxes] } # ... and theirs
+  #   archivable on_destroy: true                  # catch hard deletes
+  class Policy
+    OPTIONS = %i[
+      after older_than on deleted deleted_column every scope
+      batch_size delete_after_archive delete_method cascade
+      on_destroy hard_delete
+    ].freeze
+
+    attr_reader :model_name, :after, :on, :every, :scope, :batch_size,
+                :delete_method, :cascade, :deleted_column
+
+    def initialize(model, **options)
+      unknown = options.keys - OPTIONS
+      unless unknown.empty?
+        raise InvalidPolicyError,
+              "unknown archivable option(s) #{unknown.map(&:inspect).join(', ')} for #{model}. " \
+              "Known options: #{OPTIONS.map(&:inspect).join(', ')}"
+      end
+
+      @model_name           = model.name
+      @after                = normalize_duration(options[:after] || options[:older_than])
+      @deleted              = options.fetch(:deleted, false)
+      @deleted_column       = (options[:deleted_column] || ColdStorage.config.deleted_column).to_sym
+      @on                   = (options[:on] || default_column).to_sym
+      @every                = normalize_duration(options[:every])
+      @scope                = options[:scope]
+      @batch_size           = options[:batch_size]
+      @delete_after_archive = options[:delete_after_archive]
+      @delete_method        = options[:delete_method]
+      @cascade              = AssociationTree.normalize(options[:cascade])
+      @on_destroy           = options.fetch(:on_destroy) { options.fetch(:hard_delete, false) }
+
+      validate!
+    end
+
+    def deleted?
+      @deleted
+    end
+
+    # Copy the row to the archive when it is really destroyed.
+    def on_destroy?
+      @on_destroy
+    end
+
+    # Does this policy describe rows a scheduled run should sweep up?
+    # `on_destroy: true` on its own does not: it only reacts to deletes.
+    def sweeps?
+      !after.nil? || deleted? || !scope.nil?
+    end
+
+    def model
+      @model_name.constantize
+    end
+
+    def batch_size_value
+      batch_size || ColdStorage.config.batch_size
+    end
+
+    def delete_after_archive?
+      @delete_after_archive.nil? ? ColdStorage.config.delete_after_archive : @delete_after_archive
+    end
+
+    def delete_method_value
+      delete_method || ColdStorage.config.delete_method
+    end
+
+    # The rows that are archivable right now.
+    #
+    # Always starts from `unscoped`: a soft-delete default scope would
+    # otherwise hide exactly the rows we are asked to archive.
+    #
+    # @return [ActiveRecord::Relation]
+    def relation(klass = model)
+      return klass.none unless sweeps?
+
+      relation = klass.unscoped
+      relation = relation.where.not(deleted_column => nil) if deleted?
+      relation = relation.where(klass.arel_table[on].lt(cutoff)) if after
+      apply_scope(relation)
+    end
+
+    # @return [Time, nil] rows on the `on:` column older than this are archivable
+    def cutoff(now = Time.current)
+      return nil unless after
+
+      now - after
+    end
+
+    # Has enough time passed since the last successful run?
+    def due?(last_run_at)
+      return true if every.nil? || last_run_at.nil?
+
+      last_run_at <= Time.current - every
+    end
+
+    # Checks the options against the real table. Raises with an actionable
+    # message instead of failing later with a SQL error.
+    def validate_against_schema!(klass = model)
+      columns = klass.column_names
+      check_column!(klass, on, columns)
+      check_column!(klass, deleted_column, columns) if deleted?
+
+      if klass.primary_key.nil? || klass.primary_key.is_a?(Array)
+        raise InvalidPolicyError,
+              "#{klass} must have a single-column primary key to be archivable (got #{klass.primary_key.inspect})"
+      end
+
+      validate_cascade!(klass, cascade)
+      true
+    end
+
+    def cascade_reflection!(klass, name)
+      self.class.cascade_reflection!(klass, name)
+    end
+
+    # @return [ActiveRecord::Reflection::AssociationReflection]
+    def self.cascade_reflection!(klass, name)
+      reflection = klass.reflect_on_association(name)
+      raise InvalidPolicyError, "#{klass} has no association #{name.inspect} to cascade to" if reflection.nil?
+
+      if reflection.through_reflection?
+        raise InvalidPolicyError, "cascade does not support :through associations (#{klass}##{name})"
+      end
+
+      unless reflection.collection? || reflection.has_one?
+        raise InvalidPolicyError,
+              "cascade only supports has_many / has_one associations (#{klass}##{name} is a #{reflection.macro})"
+      end
+
+      reflection
+    end
+
+    # @return [String] one-line summary used by the status report
+    def to_s
+      parts = []
+      parts << 'deleted' if deleted?
+      parts << "#{on} < #{humanize_duration(after)} ago" if after
+      parts << "scope: #{scope.is_a?(Symbol) ? scope : 'custom'}" if scope
+      parts << 'on destroy' if on_destroy?
+      parts << "every #{humanize_duration(every)}" if every && sweeps?
+      parts << "cascade: #{cascade.keys.join(', ')}" if cascade.any?
+      parts << 'keeps source rows' unless delete_after_archive?
+      parts.join(', ')
+    end
+
+    private
+
+    def default_column
+      @deleted ? @deleted_column : ColdStorage.config.timestamp_column
+    end
+
+    def apply_scope(relation)
+      case scope
+      when nil    then relation
+      when Symbol then relation.public_send(scope)
+      when Proc   then relation.instance_exec(&scope)
+      else relation.merge(scope)
+      end
+    end
+
+    def validate!
+      return if sweeps? || on_destroy?
+
+      raise InvalidPolicyError,
+            "#{model_name}: archivable needs at least one of after:, deleted:, scope: or on_destroy:. " \
+            'Without a criterion every row of the table would be archived.'
+    end
+
+    # Walks the whole cascade tree, so a typo deep in it is reported before
+    # anything moves.
+    def validate_cascade!(klass, tree)
+      tree.each do |name, nested|
+        reflection = self.class.cascade_reflection!(klass, name)
+        validate_cascade!(reflection.klass, AssociationTree.normalize(nested))
+      end
+    end
+
+    def check_column!(klass, column, columns)
+      return if columns.include?(column.to_s)
+
+      raise InvalidPolicyError,
+            "#{klass} has no column #{column.inspect} (archivable on:/deleted_column:). " \
+            "Available: #{columns.sort.join(', ')}"
+    end
+
+    def normalize_duration(value)
+      case value
+      when nil then nil
+      when ActiveSupport::Duration then value
+      when Numeric then value.seconds
+      else
+        raise InvalidPolicyError, "expected a duration like 6.months, got #{value.inspect}"
+      end
+    end
+
+    def humanize_duration(duration)
+      duration.inspect
+    end
+  end
+end

data/lib/cold_storage/railtie.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # Rails wiring: the `archivable` macro, the logger, the rake tasks, and the
+  # automatic schema sync after migrations.
+  class Railtie < ::Rails::Railtie
+    config.cold_storage = ActiveSupport::OrderedOptions.new
+
+    initializer 'cold_storage.model' do
+      ActiveSupport.on_load(:active_record) do
+        extend ColdStorage::Model
+      end
+    end
+
+    initializer 'cold_storage.jobs' do
+      ActiveSupport.on_load(:active_job) do
+        ColdStorage::Jobs.define!
+      end
+    end
+
+    initializer 'cold_storage.logger' do |app|
+      app.config.after_initialize do
+        ColdStorage.config.logger ||= Rails.logger
+        app.config.cold_storage.each { |key, value| ColdStorage.config.public_send(:"#{key}=", value) }
+      end
+    end
+
+    rake_tasks do
+      load File.expand_path('tasks/cold_storage.rake', __dir__)
+      ColdStorage::Railtie.hook_into_migrations!
+    end
+
+    # After the primary database changes, bring the archive schema along - for
+    # the archivable models only.
+    def self.hook_into_migrations!
+      %w[db:migrate db:migrate:up db:migrate:down db:rollback db:schema:load].each do |name|
+        next unless Rake::Task.task_defined?(name)
+
+        Rake::Task[name].enhance do
+          next unless ColdStorage.config.sync_schema_after_migrate
+
+          sync = Rake::Task['cold_storage:schema:sync']
+          sync.reenable
+          sync.invoke
+        rescue ColdStorage::ConfigurationError => e
+          Rails.logger&.info { "#{ColdStorage::Logging::PREFIX} skipping schema sync: #{e.message}" }
+        end
+      end
+    end
+  end
+end

data/lib/cold_storage/registry.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module ColdStorage
+  # Keeps track of the models that called `archivable`.
+  #
+  # Model names are stored as strings, never as class objects, so that code
+  # reloading in development does not pin stale constants.
+  module Registry
+    MUTEX = Mutex.new
+    private_constant :MUTEX
+
+    class << self
+      def register(model)
+        MUTEX.synchronize { names << model.name } if model.name
+        model
+      end
+
+      def registered?(model)
+        MUTEX.synchronize { names.include?(model.to_s) }
+      end
+
+      # @return [Array<String>]
+      def model_names(eager_load: true)
+        load_application! if eager_load
+        MUTEX.synchronize { names.to_a }.sort
+      end
+
+      # @return [Array<Class>] the archivable models, skipping names that no
+      #   longer resolve (removed or renamed classes).
+      def models(eager_load: true)
+        model_names(eager_load: eager_load).filter_map do |name|
+          klass = name.safe_constantize
+          klass if klass.respond_to?(:archivable?) && klass.archivable?
+        end
+      end
+
+      def clear!
+        MUTEX.synchronize { @names = Set.new }
+      end
+
+      private
+
+      def names
+        @names ||= Set.new
+      end
+
+      def load_application!
+        return if @eager_loaded
+        return unless defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+
+        Rails.application.eager_load!
+        @eager_loaded = true
+      rescue StandardError => e
+        ColdStorage.logger.warn { "#{Logging::PREFIX} eager load failed: #{e.class}: #{e.message}" }
+      end
+    end
+  end
+end