hekenga 0.2.11 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 308f22bb8f112b42b0a9b954e30ac2e3bc206bfa31b426034d3bf520efeb782c
-  data.tar.gz: 0acb44d1ada95aa071b25d54e1e93561e08f0f477b589b4ec214e1c268a0a9e5
+  metadata.gz: 4befa0d3716a0609575aa42f35fe126b854365f5e348ecb6b8babff4912e2522
+  data.tar.gz: 674e20c142d8b8ebc0f51e75a4a372912359c196247471170c5ddae2c317c294
 SHA512:
-  metadata.gz: 422d0ac9c8e68b1a59fcf0e3942c1888b4f1bdac341e95e61593a34570b7cb87d3980bf4291a187d1c7413a1195e6f81cf9b9ed686cc1bb65271aaff6a70cb17
-  data.tar.gz: 539647a7efca0c2e35036b12dd07dcac115dd2815a5034e0b19fae22dabebb5e7a361f2bf1797507dc03c4c2521463990341f85955835692b3140b159b163230
+  metadata.gz: 6625430dc0dc24bf821e7c584fa1141362de50effdf5606ed53583b77d20b8e79b7bd17680128b25e736c24a8fb41b58169f9a0b106bd693165abfec339c49b2
+  data.tar.gz: 84a0af87ac9b21acc5253206d4de311adc6b5edbe47303894aeb1eb169ac89365f6338f79f67257478cc744ad98d8a23fe8002ee67c64b276f0a5e33be214685

data/.gitignore

@@ -7,3 +7,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.gem
+todo.txt

data/CHANGELOG.md

@@ -0,0 +1,29 @@
+# Changelog
+
+## v1.0.0
+
+- `hekenga run!` now has a `--clear` option to clear the migration prior to running it
+- (breaking) `per_document` tasks now generate `Hekenga::DocumentTaskRecord`, used for
+    recovery + progress tracking. `Hekenga::Failure::Cancelled` and
+    `Hekenga::Failure::Validation` are no longer generated.
+- (breaking) Migrations will now always continue when encountering invalid records.
+    Recovering the migration later will reprocess the invalid records. Support
+    for the prompt/cancel/stop when_invalid strategies has been dropped
+- `rollback`, `errors` CLI stubs have been removed as they were never
+    implemented
+- `--edit` now no longer eats an argument when generating a migration
+- Mutexes have been added to the registry to help with thread-safety. You will
+    still need to make sure that your application is eager loaded on workers
+- (breaking) The default write strategy has been changed to use replace bulk operations
+    instead of a batch delete followed by a batch insert. You can swap back to
+    delete then insert on a per-task basis, but you probably shouldn't
+- An experimental option to wrap each document batch in a transaction has been
+    added.
+- (breaking) If a migration doesn't change a document, it will now skip writing
+    it by default. To always write the document, call `always_write!`
+- `batch_size` can now be configured per document task
+- Rather than doing a single `pluck` on the `per_document` scope, the scope will
+    now be iterated in subqueries of 100k IDs when retrieving IDs to queue jobs
+    for.
+- (breaking) `Hekenga::ParallelJob` takes different arguments now.
+- (breaking) some internal methods have been renamed.

data/docker-compose.yml

@@ -0,0 +1,30 @@
+version: "3.8"
+
+volumes:
+  mongo:
+
+networks:
+  hekenga-net:
+
+services:
+  mongo:
+    image: mongo:5
+    command: ["--replSet", "rs0", "--bind_ip", "localhost,mongo"]
+    volumes:
+      - mongo:/data/db
+    ports:
+      - 27017:27017
+    networks:
+      - hekenga-net
+
+  mongosetup:
+    image: mongo:5
+    depends_on:
+      - mongo
+    restart: "no"
+    entrypoint:
+      - bash
+      - "-c"
+      - "sleep 3 && mongo --host mongo:27017 --eval 'rs.initiate({_id: \"rs0\", members: [{_id: 0, host: \"localhost:27017\"}]})'"
+    networks:
+      - hekenga-net

data/exe/hekenga

@@ -2,7 +2,7 @@
 
 require "bundler/setup"
 
-if File.exists?(File.expand_path("config/environment.rb"))
+if File.exist?(File.expand_path("config/environment.rb"))
   require File.expand_path("config/environment.rb")
 end
 
@@ -11,6 +11,10 @@ require "thor"
 require "pathname"
 
 class HekengaCLI < Thor
+  def self.exit_on_failure?
+    true
+  end
+
   desc "status", "Show which migrations have run and their status."
   def status
     Hekenga.load_all!
@@ -33,31 +37,37 @@ class HekengaCLI < Thor
 
   desc "run_all!", "Run all migrations that have not yet run, in date order."
   def run_all!
-    bail_if_errors
     Hekenga.load_all!
     Hekenga.registry.sort_by {|x| x.stamp}.each do |migration|
       if Hekenga.status(migration) == :naught
         migration.perform!
-        bail_if_errors
       end
     end
   end
 
   desc "run! PATH_OR_PKEY --test", "Run a migration (optionally in test mode)."
   option :test, default: false, type: :boolean
+  option :clear, default: false, type: :boolean
   def run!(path_or_pkey)
-    bail_if_errors
+    clear!(path_or_pkey) if options[:clear]
     migration = load_migration(path_or_pkey)
     migration.test_mode! if options[:test]
     migration.perform!
     if options[:test]
-      if Hekenga::Failure.where(pkey: migration.to_key).any?
-        puts "Logs have been preserved for debugging. To reset migration state run:"
-        puts "  hekenga clear! #{path_or_pkey}"
-      else
-        puts "Migration test run completed successfully."
-        clear!(path_or_pkey)
-      end
+      puts "Logs have been preserved for debugging. To reset migration state run:"
+      puts "  hekenga clear! #{path_or_pkey}"
+    end
+  end
+
+  desc "recover! PATH_OR_PKEY", "Recover a migration"
+  option :test, default: false, type: :boolean
+  def recover!(path_or_pkey)
+    migration = load_migration(path_or_pkey)
+    migration.test_mode! if options[:test]
+    migration.recover!
+    if options[:test]
+      puts "Logs have been preserved for debugging. To reset migration state run:"
+      puts "  hekenga clear! #{path_or_pkey}"
     end
   end
 
@@ -79,24 +89,10 @@ class HekengaCLI < Thor
     puts "Clearing #{migration.to_key}.."
     Hekenga::Log.where(pkey: migration.to_key).delete_all
     Hekenga::Failure.where(pkey: migration.to_key).delete_all
+    Hekenga::DocumentTaskRecord.where(migration_key: migration.to_key).delete_all
     puts "Done!"
   end
 
-  desc "rollback", "Rollback a migration."
-  def rollback
-    todo "rollback"
-  end
-
-  desc "recover", "Attempt to resume a failed migration."
-  def recover
-    todo "recover"
-  end
-
-  desc "errors", "Print the errors associated with a failed migration."
-  def errors
-    todo "errors"
-  end
-
   desc "skip PATH_OR_PKEY", "Skip a migration so that it won't run."
   def skip(path_or_pkey)
     migration = load_migration(path_or_pkey)
@@ -114,7 +110,7 @@ class HekengaCLI < Thor
   end
 
   desc "generate <description> --edit", "Generate a migration scaffold (and optionally edit in your editor)."
-  option :edit
+  option :edit, type: :boolean
   def generate(*description)
     description = description.join(" ")
     scaffold = Hekenga::Scaffold.new(description)
@@ -130,20 +126,11 @@ class HekengaCLI < Thor
       exec "$EDITOR #{path}"
     end
   end
+
   private
 
-  def todo(op)
-    puts "#{op.capitalize} has not yet been implemented."
-    exit(99)
-  end
-  def bail_if_errors
-    if Hekenga.any_fatal?
-      puts "Refusing to run migrations while there is an existing cancelled migration."
-      exit(1)
-    end
-  end
   def load_migration(path_or_pkey)
-    if File.exists?(File.expand_path(path_or_pkey))
+    if File.exist?(File.expand_path(path_or_pkey))
       require File.expand_path(path_or_pkey)
       migration = Hekenga.registry.last
     else

data/hekenga.gemspec

@@ -8,6 +8,7 @@ Gem::Specification.new do |spec|
   spec.version       = Hekenga::VERSION
   spec.authors       = ["Tapio Saarinen"]
   spec.email         = ["admin@bitlong.org"]
+  spec.licenses      = ["MIT"]
 
   spec.summary       = %q{Sophisticated migration framework for mongoid, with the ability to parallelise via ActiveJob.}
   spec.homepage      = "https://github.com/tzar/hekenga"
@@ -19,14 +20,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", "~> 2.4.6"
+  spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "database_cleaner"
+  spec.add_development_dependency "database_cleaner-mongoid", "~> 2.0"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "pry-byebug"
 
-  spec.add_runtime_dependency "mongoid", ">= 5"
-  spec.add_runtime_dependency "activejob", ">= 4"
+  spec.add_runtime_dependency "mongoid", ">= 6"
+  spec.add_runtime_dependency "activejob", ">= 5"
   spec.add_runtime_dependency "thor"
 end

data/lib/hekenga/context.rb

@@ -1,14 +1,15 @@
 module Hekenga
   class Context
-    def initialize(test_run)
-      @__test_run = test_run
+    def initialize(test_mode: false)
+      @__test_mode = test_mode
     end
 
-    def test?
-      !!@__test_run
-    end
     def actual?
-      !@__test_run
+      !@__test_mode
+    end
+
+    def test?
+      !actual?
     end
   end
 end

data/lib/hekenga/document_task.rb

@@ -2,15 +2,21 @@ require 'hekenga/irreversible'
 module Hekenga
   class DocumentTask
     attr_reader :ups, :downs, :setups, :filters
-    attr_accessor :parallel, :scope, :timeless
-    attr_accessor :description, :invalid_strategy, :skip_prepare
+    attr_accessor :parallel, :scope, :timeless, :batch_size
+    attr_accessor :description, :invalid_strategy, :skip_prepare, :write_strategy
+    attr_accessor :always_write, :use_transaction
+
     def initialize
       @ups              = []
       @downs            = []
       @setups           = []
       @filters          = []
-      @invalid_strategy = :prompt
+      @invalid_strategy = :continue
+      @write_strategy   = :update
       @skip_prepare     = false
+      @batch_size       = nil
+      @always_write     = false
+      @use_transaction  = false
     end
 
     def validate!

data/lib/hekenga/document_task_executor.rb

@@ -0,0 +1,264 @@
+module Hekenga
+  class DocumentTaskExecutor
+    attr_reader :task_record
+    attr_reader :context, :session
+
+    def initialize(task_record, records: nil)
+      @task_record       = task_record
+      @records           = records
+      @migrated_records  = []
+      @invalid_records   = []
+      @records_to_write  = []
+      @filtered_records  = []
+      @skipped_records   = []
+      @failed_records    = []
+      @backed_up_records = {}
+    end
+
+    def run!
+      with_setup do |context|
+        with_transaction do |session|
+          filter_records
+          run_migration
+          validate_records
+          write_records
+          write_result unless task_record.test_mode
+        end
+        # In test mode, the transaction will be aborted - so we need to write
+        # the result outside of the run! block
+        write_result if task_record.test_mode
+      end
+    end
+
+    def check_for_completion!
+      if migration_complete?
+        migration.log(task_idx).set_without_session(
+          done: true,
+          finished: Time.now,
+          error: migration.task_records(task_idx).failed.any?
+        )
+      end
+    end
+
+    def migration_cancelled?
+      migration.log(task_idx).cancel
+    end
+
+    private
+
+    delegate :task_idx, to: :task_record
+
+    attr_reader :migrated_records, :records_to_write, :filtered_records, :invalid_records, :skipped_records, :failed_records, :backed_up_records
+
+    def migration_complete?
+      migration.task_records(task_idx).incomplete.none?
+    end
+
+    def record_scope
+      task.scope.klass.unscoped.in(_id: task_record.ids)
+    end
+
+    def records
+      @records ||= record_scope.to_a
+    end
+
+    def write_result
+      task_record.update_attributes(
+        complete: true,
+        finished: Time.now,
+        failed_ids: failed_records.map(&:_id),
+        invalid_ids: invalid_records.map(&:_id),
+        written_ids: records_to_write.map(&:_id),
+        stats: {
+          failed: failed_records.length,
+          invalid: invalid_records.length,
+          written: records_to_write.length
+        }
+      )
+    end
+
+    def filter_records
+      records.each do |record|
+        if task.filters.all? {|block| context.instance_exec(record, &block)}
+          filtered_records << record
+        else
+          skipped_records << record
+        end
+      rescue => _e
+        failed_records << record
+      end
+    end
+
+    def run_migration
+      filtered_records.each do |record|
+        backup_record(record)
+        task.up!(@context, record)
+        migrated_records << record
+      rescue => _e
+        failed_records << record
+      end
+    end
+
+    def backup_record(record)
+      return unless task.write_strategy == :delete_then_insert
+
+      backed_up_records[record._id] = record.as_document.deep_dup
+    end
+
+    def validate_records
+      migrated_records.each do |record|
+        if record.valid?
+          records_to_write << record
+        else
+          invalid_records << record
+        end
+      end
+    end
+
+    def write_records
+      records_to_write.keep_if(&:changed?) unless task.always_write
+      return if records_to_write.empty?
+      return if task_record.test_mode
+
+      records_to_write.each {|record| record.send(:prepare_update) {}}
+
+      case task.write_strategy
+      when :delete_then_insert
+        delete_then_insert_records
+      else
+        replace_records
+      end
+    rescue Mongo::Error::BulkWriteError => e
+      # If we're in a transaction, we can retry; so just re-raise
+      raise if @session
+      # Otherwise, we need to log the failure
+      write_failure!(e)
+    end
+
+    def write_failure!(error)
+      log = migration.log(task_idx)
+      backups = records_to_write.map do |record|
+        failed_records << record
+        backed_up_records[record._id]
+      end.compact
+      @records_to_write = []
+      log.add_failure({
+        message:   error.to_s,
+        backtrace: error.backtrace,
+        documents: backups,
+        document_ids: records_to_write.map(&:_id),
+        task_record_id: task_record.id
+      }, Hekenga::Failure::Write)
+      log.set_without_session({error: true})
+    end
+
+    def delete_then_insert_records
+      operations = []
+      operations << {
+        delete_many: {
+          filter: {
+            _id: {
+              '$in': records_to_write.map(&:_id)
+            }
+          }
+        }
+      }
+      records_to_write.each do |record|
+        operations << { insert_one: record.as_document }
+      end
+      bulk_write(operations, ordered: true)
+    end
+
+    def bulk_write(operations, **options)
+      task.scope.klass.collection.bulk_write(operations, session: session, **options)
+    end
+
+    def replace_records
+      operations = records_to_write.map do |record|
+        {
+          replace_one: {
+            filter: { _id: record._id },
+            replacement: record.as_document
+          }
+        }
+      end
+      bulk_write(operations)
+    end
+
+    def migration
+      @migration ||= Hekenga.find_migration(task_record.migration_key)
+    end
+
+    def with_setup(&block)
+      @context = Hekenga::Context.new(test_mode: task_record.test_mode)
+      begin
+        task.setups&.each do |setup|
+          @context.instance_exec(&setup)
+        end
+      rescue => e
+        fail_and_cancel!(e)
+        return
+      end
+      yield
+    ensure
+      @context = nil
+    end
+
+    def fail_and_cancel!(error)
+      log = migration.log(task_idx)
+      log.add_failure({
+        message:   error.to_s,
+        backtrace: error.backtrace
+      }, Hekenga::Failure::Error)
+      log.set_without_session({cancel: true, error: true, done: true, finished: Time.now})
+      task_record.update_attributes(
+        complete: true,
+        finished: Time.now,
+        failed_ids: task_record.ids,
+        invalid_ids: [],
+        written_ids: [],
+        stats: {
+          failed: task_record.ids.length,
+          invalid: 0,
+          written: 0,
+        }
+      )
+    end
+
+    def with_transaction(&block)
+      return yield unless task.use_transaction
+
+      ensure_replicaset!
+      klass = task.scope.klass
+      # NOTE: Dummy session to work around threading bug
+      klass.persistence_context.client.start_session({})
+      klass.with_session do |session|
+        @session = session
+        @session.start_transaction
+        yield
+        if task_record.test_mode
+          @session.abort_transaction
+        else
+          @session.commit_transaction
+        end
+      rescue
+        @session.abort_transaction
+        raise
+      ensure
+        @session = nil
+      end
+    end
+
+    def ensure_replicaset!
+      return unless task.use_transaction
+
+      unless task.scope.klass.collection.client.cluster.replica_set?
+        raise "MongoDB must be in a replica set to use transactions"
+      end
+    end
+
+    def task
+      @task ||= migration.tasks[task_idx]
+    end
+  end
+end

data/lib/hekenga/document_task_record.rb

@@ -0,0 +1,29 @@
+module Hekenga
+  class DocumentTaskRecord
+    include Mongoid::Document
+
+    field :migration_key, type: String
+    field :task_idx, type: Integer
+    field :executor_key, type: BSON::ObjectId
+    field :finished, type: Time
+    field :complete, type: Boolean, default: false
+    field :ids, type: Array, default: []
+    field :id_count, type: Integer
+    field :test_mode, type: Boolean
+
+    field :stats,       type: Hash,  default: {}
+    field :failed_ids,  type: Array, default: []
+    field :invalid_ids, type: Array, default: []
+    field :written_ids, type: Array, default: []
+
+    index(migration_key: 1, task_idx: 1, complete: 1)
+    index(migration_key: 1, task_idx: 1, 'stats.failed': 1)
+    index(migration_key: 1, task_idx: 1, ids: 1)
+
+    scope :incomplete, proc { where(complete: false) }
+    scope :complete,   proc { where(complete: true) }
+    scope :failed,     proc { gt('stats.failed': 1) }
+
+    before_create { self.id_count = ids.count }
+  end
+end

data/lib/hekenga/dsl/document_task.rb

@@ -4,7 +4,15 @@ module Hekenga
     class DocumentTask < Hekenga::DSL
       configures Hekenga::DocumentTask
 
-      INVALID_BEHAVIOR_STRATEGIES = [:prompt, :cancel, :stop, :continue]
+      INVALID_BEHAVIOR_STRATEGIES = %i[continue]
+      VALID_WRITE_STRATEGIES = %i[delete_then_insert update]
+
+      def batch_size(size)
+        unless size.is_a?(Integer) && size > 0
+          raise "Invalid batch size #{size.inspect}"
+        end
+        @object.batch_size = size
+      end
 
       def when_invalid(val)
         unless INVALID_BEHAVIOR_STRATEGIES.include?(val)
@@ -13,18 +21,37 @@ module Hekenga
         @object.invalid_strategy = val
       end
 
+      def write_strategy(strategy)
+        unless VALID_WRITE_STRATEGIES.include?(strategy)
+          raise "Invalid value #{strategy}. Valid values for write_strategy are: #{VALID_WRITE_STRATEGIES.join(", ")}."
+        end
+        @object.write_strategy = strategy
+      end
+
       def scope(scope)
         @object.scope = scope
       end
+
+      def always_write!
+        @object.always_write = true
+      end
+
+      def use_transaction!
+        @object.use_transaction = true
+      end
+
       def parallel!
         @object.parallel = true
       end
+
       def timeless!
         @object.timeless = true
       end
+
       def skip_prepare!
         @object.skip_prepare = true
       end
+
       def setup(&block)
         @object.setups.push block
       end

data/lib/hekenga/dsl/migration.rb

@@ -11,12 +11,15 @@ module Hekenga
         end
         @object.batch_size = size
       end
+
       def created(stamp = nil)
         @object.stamp = Time.parse(stamp)
       end
+
       def task(description = nil, &block)
         @object.tasks.push Hekenga::DSL::SimpleTask.new(description, &block).object
       end
+
       def per_document(description = nil, &block)
         @object.tasks.push Hekenga::DSL::DocumentTask.new(description, &block).object
       end

data/lib/hekenga/failure/write.rb

@@ -6,6 +6,7 @@ module Hekenga
       field :documents
       field :document_ids, type: Array
       field :batch_start
+      field :task_record_id
     end
   end
 end

data/lib/hekenga/iterator.rb

@@ -0,0 +1,26 @@
+module Hekenga
+  class Iterator
+    include Enumerable
+
+    SMALLEST_ID = BSON::ObjectId.from_string('0'*24)
+
+    attr_reader :scope, :size
+
+    def initialize(scope, size:)
+      @scope = scope
+      @size = size
+    end
+
+    def each(&block)
+      current_id = SMALLEST_ID
+      base_scope = scope.asc(:_id).limit(size)
+
+      loop do
+        ids = base_scope.and(_id: {'$gt': current_id}).pluck(:_id)
+        break if ids.empty?
+        yield ids
+        current_id = ids.sort.last
+      end
+    end
+  end
+end