cold_storage 0.1.0

data/lib/cold_storage/report.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # Human readable status of every archivable model: what its rule is, how many
+  # rows are waiting, how many are already archived, when it last ran.
+  class Report
+    HEADERS = ['Model', 'Rule', 'Pending', 'Archived', 'Last run'].freeze
+
+    def initialize(models: nil, counts: true)
+      @models = models || ColdStorage.models
+      @counts = counts
+    end
+
+    def rows
+      @rows ||= @models.map { |model| row_for(model) }
+    end
+
+    def to_s
+      table = [HEADERS, *rows]
+      widths = HEADERS.each_index.map { |i| table.map { |row| row[i].to_s.length }.max }
+      separator = widths.map { |width| '-' * width }.join('-+-')
+
+      lines = table.map { |row| row.each_with_index.map { |cell, i| cell.to_s.ljust(widths[i]) }.join(' | ') }
+      lines.insert(1, separator)
+      lines.join("\n")
+    end
+
+    private
+
+    def row_for(model)
+      [
+        model.name,
+        model.archiving_policy.to_s,
+        count { model.archivable_records.count },
+        count { model.archived.count },
+        Run.last_success_at(model)&.strftime('%Y-%m-%d %H:%M') || 'never'
+      ]
+    rescue StandardError => e
+      [model.name, 'error', e.class.to_s, '-', '-']
+    end
+
+    def count
+      return '-' unless @counts
+
+      yield
+    rescue ActiveRecord::StatementInvalid
+      'n/a'
+    end
+  end
+end

data/lib/cold_storage/restorer.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # Moves rows back from the archive database into the primary one, optionally
+  # taking their archived children along.
+  #
+  #   ColdStorage.restore(Invoice, [1, 2])
+  #   ColdStorage.restore(Invoice, [1, 2], with: :all)
+  #   ColdStorage.restore(Invoice, [1, 2], with: [:invoice_lines])
+  #   ColdStorage.restore(Invoice, [1, 2], with: { invoice_lines: [:taxes] })
+  #
+  # Parents are written before their children, so foreign keys in the primary
+  # database hold at every step.
+  class Restorer
+    include Logging
+    include RowReader
+
+    attr_reader :model
+
+    # @param with [Symbol, Array, Hash, nil] associations to restore too;
+    #   `:all` walks every has_many/has_one that has an archive table
+    # @param delete_from_archive [Boolean] remove the rows from the archive
+    #   once they are back in the primary database
+    def initialize(model, with: nil, delete_from_archive: true, batch_size: nil,
+                   dry_run: ColdStorage.config.dry_run, visited: [])
+      @model               = model
+      @with                = with
+      @delete_from_archive = delete_from_archive
+      @batch_size          = batch_size || ColdStorage.config.batch_size
+      @dry_run             = dry_run
+      @visited             = visited
+    end
+
+    # @param ids [Array, ActiveRecord::Relation] ids to restore, or a relation
+    #   on the archive model (Invoice.archived.where(...))
+    # @return [Integer] number of restored rows, children included
+    def call(ids)
+      validate_associations!
+
+      relation = normalize(ids)
+      restored = 0
+      own = 0
+
+      relation.in_batches(of: @batch_size) do |batch|
+        rows = read(batch)
+        next if rows.empty?
+
+        own += write(rows)
+        restored_ids = rows.map { |row| row[primary_key] }
+        restored += restore_associations(restored_ids)
+        delete(restored_ids)
+      end
+
+      log("#{model}: restored #{own} row(s) from the archive#{' [dry run]' if @dry_run}")
+      own + restored
+    end
+
+    # The associations `with:` resolves to, as association name => nested with.
+    # @return [Hash{Symbol => Object}]
+    def associations
+      @associations ||= expand(@with)
+    end
+
+    private
+
+    def primary_key
+      @primary_key ||= model.primary_key
+    end
+
+    def archive_model
+      @archive_model ||= ArchiveModel.for(model)
+    end
+
+    def normalize(ids)
+      return ids if ids.is_a?(ActiveRecord::Relation)
+
+      archive_model.where(primary_key => Array(ids))
+    end
+
+    def read(batch)
+      sql = batch.select(archive_model.arel_table[Arel.star]).to_sql
+      rows = ArchiveRecord.with_archive_connection do |connection|
+        select_rows_as_hashes(connection, sql, "#{model} Restore Load")
+      end
+
+      stamp = ColdStorage.config.archived_at_column
+      rows.each { |row| row.delete(stamp.to_s) } if stamp
+      rows
+    end
+
+    def write(rows)
+      return rows.size if @dry_run
+
+      # unscoped: a default scope would otherwise force its own values (a
+      # soft-delete scope would resurrect rows as "not deleted").
+      model.unscoped.upsert_all(rows, unique_by: primary_key.to_sym, record_timestamps: false, returning: false)
+      rows.size
+    end
+
+    def delete(ids)
+      return unless @delete_from_archive && !@dry_run
+
+      archive_model.where(primary_key => ids).delete_all
+    end
+
+    # ------------------------------------------------------------ associations
+
+    # Checked before anything is written: a typo in `with:` must not leave a
+    # half restored graph behind.
+    def validate_associations!
+      associations.each_key do |name|
+        reflection = reflection_for(name)
+        assert_archive_table!(reflection.klass, name)
+      end
+    end
+
+    def restore_associations(parent_ids)
+      return 0 if associations.empty?
+
+      associations.sum do |name, nested|
+        reflection = reflection_for(name)
+        child = reflection.klass
+
+        self.class.new(
+          child,
+          with: nested,
+          delete_from_archive: @delete_from_archive,
+          batch_size: @batch_size,
+          dry_run: @dry_run,
+          visited: @visited + [model.table_name]
+        ).call(child_relation(reflection, parent_ids))
+      end
+    end
+
+    def child_relation(reflection, parent_ids)
+      scope = ArchiveModel.for(reflection.klass).where(reflection.foreign_key => parent_ids)
+      scope = scope.where(reflection.type => model.polymorphic_name) if reflection.type
+      scope
+    end
+
+    def reflection_for(name)
+      reflection = model.reflect_on_association(name)
+      raise ArgumentError, "#{model} has no association #{name.inspect} to restore" if reflection.nil?
+
+      if reflection.macro == :belongs_to
+        raise ArgumentError,
+              "#{model}##{name} is a belongs_to; restore the owner first, then its children"
+      end
+
+      reflection
+    end
+
+    def assert_archive_table!(child, name)
+      exists = ArchiveRecord.with_archive_connection { |c| c.table_exists?(child.table_name) }
+      return if exists
+
+      raise SchemaMissingError,
+            "The archive database has no #{child.table_name} table, so #{model}##{name} cannot be restored."
+    end
+
+    # `with:` accepts a symbol, an array, a nested hash, or :all.
+    def expand(value)
+      return every_child_association if value == true || value == :all
+
+      AssociationTree.normalize(value)
+    rescue ArgumentError
+      raise ArgumentError, "with: expects a symbol, array, hash or :all, got #{value.inspect}"
+    end
+
+    # Every child association that actually has something to restore. Cycles
+    # are cut with the tables already visited on the way down.
+    def every_child_association
+      model.reflect_on_all_associations.each_with_object({}) do |reflection, result|
+        next unless %i[has_many has_one].include?(reflection.macro)
+        next if reflection.options[:through] || reflection.options[:polymorphic]
+
+        target = child_table(reflection)
+        next if target.nil? || @visited.include?(target)
+
+        result[reflection.name] = :all
+      end
+    end
+
+    def child_table(reflection)
+      table = reflection.klass.table_name
+      return nil if table.blank? || table == model.table_name
+
+      exists = ArchiveRecord.with_archive_connection { |c| c.table_exists?(table) }
+      exists ? table : nil
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/cold_storage/row_reader.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # Reads rows as plain hashes, cast by the *database* column types rather than
+  # by the model's attribute types.
+  #
+  # This is what makes a copy faithful: model level serializers, enums and
+  # default scopes are bypassed on both sides, while jsonb stays a Hash, arrays
+  # stay Arrays and numerics stay BigDecimals - so the same values are written
+  # back into identically typed columns on the other side.
+  module RowReader
+    private
+
+    # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter]
+    # @param sql [String]
+    # @return [Array<Hash{String => Object}>]
+    def select_rows_as_hashes(connection, sql, name)
+      result = connection.select_all(sql, name)
+      return [] if result.empty?
+
+      columns = result.columns
+      values = result.cast_values
+
+      if columns.size == 1
+        values.map { |value| { columns.first => value } }
+      else
+        values.map { |row| columns.zip(row).to_h }
+      end
+    end
+  end
+end

data/lib/cold_storage/run.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module ColdStorage
+  # One archiving run of one model. Lives in the archive database, and is what
+  # makes the `every:` option work across processes and deploys.
+  class Run < ArchiveRecord
+    self.table_name = InternalSchema::RUNS_TABLE
+
+    STATUSES = %w[running succeeded failed].freeze
+
+    scope :succeeded, -> { where(status: 'succeeded') }
+    scope :for_model, ->(model) { where(archived_model: model.to_s) }
+
+    class << self
+      # @return [Time, nil] when the model was last archived successfully
+      def last_success_at(model)
+        InternalSchema.ensure!
+        for_model(model).succeeded.maximum(:finished_at)
+      end
+
+      def start!(model)
+        InternalSchema.ensure!
+        create!(
+          archived_model: model.to_s,
+          archived_table: (model.table_name if model.respond_to?(:table_name)),
+          status: 'running',
+          started_at: Time.current
+        )
+      end
+    end
+
+    def succeed!(archived_count:, deleted_count:)
+      update!(
+        status: 'succeeded',
+        archived_count: archived_count,
+        deleted_count: deleted_count,
+        finished_at: Time.current
+      )
+    end
+
+    def fail!(error, archived_count: 0, deleted_count: 0)
+      update!(
+        status: 'failed',
+        archived_count: archived_count,
+        deleted_count: deleted_count,
+        finished_at: Time.current,
+        error_message: "#{error.class}: #{error.message}"
+      )
+    end
+  end
+end

data/lib/cold_storage/schema_mirror.rb

@@ -0,0 +1,337 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module ColdStorage
+  # Copies the table definitions of the archivable models (and of the tables
+  # they cascade into) from the primary database to the archive database, and
+  # keeps them in sync when migrations change them.
+  #
+  # Deliberate differences from the source schema:
+  #   * no foreign keys       - an archive holds partial object graphs
+  #   * no NOT NULL, no default values (configurable) - a schema that got
+  #     stricter later must never reject rows archived before that
+  #   * columns are added, never dropped (configurable) - archived rows keep
+  #     the columns they were archived with
+  #   * one extra column, `archived_at`
+  class SchemaMirror
+    include Logging
+
+    # A single difference between the two schemas.
+    Change = Struct.new(:type, :table, :name, :detail, keyword_init: true) do
+      def to_s
+        [type, table, name, detail].compact.join(' ')
+      end
+    end
+
+    attr_reader :changes
+
+    # @param models [Array<Class>, nil] defaults to every archivable model
+    # @param dry_run [Boolean] plan only, never touch the archive database
+    def initialize(models: nil, dry_run: ColdStorage.config.dry_run, logger_prefix: nil)
+      @models  = models
+      @dry_run = dry_run
+      @logger_prefix = logger_prefix
+      @changes = []
+    end
+
+    # Applies the changes.
+    # @return [Array<Change>] what was applied
+    def sync!
+      run(apply: !@dry_run)
+    end
+
+    # @return [Array<Change>] what sync! would apply; empty means "in sync"
+    def plan
+      run(apply: false)
+    end
+    alias check plan
+
+    # Tables the archive database is expected to hold, derived from the models.
+    # @return [Hash{String => Class}] table name => source model
+    def tables
+      @tables ||= begin
+        map = {}
+        source_models.each { |model| collect_tables(model, map) }
+        map
+      end
+    end
+
+    def source_models
+      @source_models ||= (@models || ColdStorage.models).map { |m| m.is_a?(String) ? m.constantize : m }
+    end
+
+    private
+
+    def run(apply:)
+      @changes = []
+      ArchiveRecord.connect!
+      InternalSchema.ensure! if apply
+
+      ArchiveRecord.with_archive_connection do |target|
+        tables.each { |table, model| mirror_table(model, table, target, apply) }
+      end
+
+      record_schema_version if apply
+      log_summary(apply)
+      changes
+    end
+
+    # ------------------------------------------------------------------ tables
+
+    def collect_tables(model, map, cascade = nil, seen = [])
+      return if seen.include?(model.table_name)
+
+      seen += [model.table_name]
+      map[model.table_name] ||= model
+
+      tree = cascade || policy_cascade(model)
+      tree.each do |association, nested|
+        reflection = Policy.cascade_reflection!(model, association)
+        child_tree = nested.nil? ? policy_cascade(reflection.klass) : AssociationTree.normalize(nested)
+        collect_tables(reflection.klass, map, child_tree, seen)
+      end
+    end
+
+    def policy_cascade(model)
+      return {} unless model.respond_to?(:archiving_policy) && model.archiving_policy
+
+      model.archiving_policy.cascade
+    end
+
+    def mirror_table(model, table, target, apply)
+      source_columns = columns_for(model)
+      mirror_enum_types(model, source_columns, target, apply)
+
+      if target.table_exists?(table)
+        sync_columns(table, source_columns, target, apply)
+      else
+        create_table(model, table, source_columns, target, apply)
+      end
+
+      ensure_archived_at(table, source_columns, target, apply)
+      sync_indexes(model, table, target, apply) if ColdStorage.config.mirror_indexes
+    end
+
+    def create_table(model, table, source_columns, target, apply)
+      record(:create_table, table, nil, "#{source_columns.size} columns")
+      return unless apply
+
+      target.create_table(table, id: false) do |t|
+        source_columns.each do |column|
+          t.column(column.name, sql_type_for(column), **column_options(column))
+        end
+      end
+
+      primary_key = model.primary_key
+      return if primary_key.blank?
+
+      quoted = Array(primary_key).map { |key| target.quote_column_name(key) }.join(', ')
+      target.execute("ALTER TABLE #{target.quote_table_name(table)} ADD PRIMARY KEY (#{quoted})")
+    end
+
+    def sync_columns(table, source_columns, target, apply)
+      archive_columns = target.columns(table).index_by(&:name)
+
+      source_columns.each do |column|
+        existing = archive_columns[column.name]
+
+        if existing.nil?
+          record(:add_column, table, column.name, sql_type_for(column))
+          target.add_column(table, column.name, sql_type_for(column), **column_options(column)) if apply
+        elsif normalize_type(sql_type_for(existing)) != normalize_type(sql_type_for(column))
+          handle_type_mismatch(table, column, existing, target, apply)
+        end
+      end
+
+      removed = archive_columns.keys - source_columns.map(&:name) - [archived_at_column.to_s]
+      removed.each { |name| handle_removed_column(table, name, target, apply) }
+    end
+
+    def handle_type_mismatch(table, column, existing, target, apply)
+      from = sql_type_for(existing)
+      to = sql_type_for(column)
+      message = "#{table}.#{column.name} is #{from} in the archive but #{to} in the primary database"
+
+      case ColdStorage.config.on_type_mismatch
+      when :raise
+        raise Error, message
+      when :change
+        record(:change_column, table, column.name, "#{from} -> #{to}")
+        target.change_column(table, column.name, to, **column_options(column)) if apply
+      when :warn
+        record(:type_mismatch, table, column.name, "#{from} != #{to}")
+        warn_log(message)
+      end
+    end
+
+    def handle_removed_column(table, name, target, apply)
+      if ColdStorage.config.drop_removed_columns
+        record(:remove_column, table, name, nil)
+        target.remove_column(table, name) if apply
+      else
+        record(:extra_column, table, name, 'kept (drop_removed_columns is false)')
+      end
+    end
+
+    def ensure_archived_at(table, source_columns, target, apply)
+      column = archived_at_column
+      return if column.nil?
+
+      if source_columns.any? { |c| c.name == column.to_s }
+        warn_log("#{table} already has a #{column} column; ColdStorage will not stamp it")
+        return
+      end
+      return if target.table_exists?(table) && target.column_exists?(table, column)
+
+      record(:add_column, table, column.to_s, 'datetime (archived_at)')
+      return unless apply
+
+      target.add_column(table, column, :datetime)
+      target.add_index(table, column, name: index_name(table, [column.to_s], 'archived_at'))
+    end
+
+    # ----------------------------------------------------------------- indexes
+
+    def sync_indexes(model, table, target, apply)
+      return if !apply && !target.table_exists?(table)
+
+      source_indexes = with_source_connection(model) { |source| source.indexes(table) }
+      existing = target.indexes(table).map(&:name)
+
+      source_indexes.each do |index|
+        next if existing.include?(index.name)
+
+        detail = Array(index.columns).join(', ')
+        detail += ' (unique dropped)' if index.unique
+        record(:add_index, table, index.name, detail)
+        next unless apply
+
+        add_index(target, table, index)
+      end
+    end
+
+    # Unique indexes are mirrored as plain ones. An archive accumulates history:
+    # a natural key that is unique in the primary database at any one moment
+    # (say working_date + colleague_id) is not unique across everything that
+    # table ever held. The primary key is the only uniqueness the archive
+    # enforces, and it is what keeps re-archiving a row idempotent.
+    def add_index(target, table, index)
+      options = {
+        name: index.name,
+        unique: false,
+        where: index.where,
+        using: index.using,
+        order: index.orders.presence,
+        opclass: index.opclasses.presence
+      }.compact
+
+      target.add_index(table, index.columns, **options)
+    rescue StandardError => e
+      # An index is never worth failing a sync for: the data still fits.
+      warn_log("could not mirror index #{index.name} on #{table}: #{e.class}: #{e.message}")
+    end
+
+    # ------------------------------------------------------------- enum types
+
+    def mirror_enum_types(model, source_columns, target, apply)
+      return unless target.respond_to?(:enum_types)
+
+      source_enums = enum_types_for(model)
+      return if source_enums.empty?
+
+      existing = target.enum_types.to_h.keys
+      used = source_columns.map { |column| base_type(column.sql_type) }.uniq
+
+      used.each do |type|
+        values = source_enums[type]
+        next if values.nil? || existing.include?(type)
+
+        record(:create_enum, nil, type, Array(values).join(', '))
+        target.create_enum(type, Array(values)) if apply
+      end
+    end
+
+    def enum_types_for(model)
+      @enum_types ||= {}
+      @enum_types[model.connection_pool.db_config.name] ||=
+        with_source_connection(model) do |source|
+          source.respond_to?(:enum_types) ? source.enum_types.to_h : {}
+        end
+    end
+
+    def base_type(sql_type)
+      sql_type.to_s.sub(/\[\]\z/, '').sub(/\(.*\)\z/, '').strip
+    end
+
+    # ------------------------------------------------------------------ shared
+
+    def columns_for(model)
+      with_source_connection(model) { |source| source.columns(model.table_name) }
+    end
+
+    def with_source_connection(model, &block)
+      model.connection_pool.with_connection(&block)
+    end
+
+    def column_options(column)
+      options = {}
+      options[:null] = column.null if ColdStorage.config.mirror_null_constraints
+      if ColdStorage.config.mirror_defaults && column.default_function.nil?
+        options[:default] = column.default
+      end
+      options
+    end
+
+    # PostgreSQL reports an array column as its element type plus a separate
+    # array flag, so the "[]" has to be put back before the type can be used in
+    # DDL again.
+    def sql_type_for(column)
+      type = column.sql_type
+      type = "#{type}[]" if column.respond_to?(:array?) && column.array?
+      type
+    end
+
+    def normalize_type(sql_type)
+      sql_type.to_s.downcase.gsub(/\s+/, ' ').strip
+    end
+
+    def archived_at_column
+      ColdStorage.config.archived_at_column
+    end
+
+    def index_name(table, columns, suffix)
+      name = "index_#{table}_on_#{columns.join('_and_')}"
+      name = "idx_ra_#{Digest::MD5.hexdigest(name)[0, 20]}_#{suffix}" if name.length > 63
+      name
+    end
+
+    def record(type, table, name, detail)
+      change = Change.new(type: type, table: table, name: name, detail: detail)
+      changes << change
+      log(change.to_s)
+      change
+    end
+
+    def record_schema_version
+      version = source_schema_version
+      Metadata.schema_version = version if version
+    end
+
+    def source_schema_version
+      model = source_models.first || ActiveRecord::Base
+      model.connection_pool.migration_context.current_version
+    rescue StandardError
+      nil
+    end
+
+    def log_summary(apply)
+      verb = apply ? 'applied' : 'pending'
+      if changes.empty?
+        log("archive schema is up to date (#{tables.size} tables)")
+      else
+        log("#{changes.size} schema change(s) #{verb} across #{tables.size} table(s)")
+      end
+    end
+  end
+end