hekenga 1.0.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55037feb4349d723dd34938f3eb9aab8431bad7f287081b685705c6a6d40b046
-  data.tar.gz: 896bdc203b3f30c543a4f7265bdea16e128d64ed59456e32cecb683a789f684c
+  metadata.gz: f21c3c1cb0e45c3b9eb2627fc960d032ff981e01b239dc69c02d2aebc1f7b539
+  data.tar.gz: 62fbbd8a65bae8bacc537bcc40cc4a3960b795ebb19a13c34b8ebd539eb102d0
 SHA512:
-  metadata.gz: 69a634184500d3c5cca3298a3793c0e5f496d31f935af6265840e1ab7e576de9bc06623cff8c03738fed9f2028146d460fd7ed04cd71e3f684c890a792d6986e
-  data.tar.gz: 2b916adfaa97a95fa645cd6442aefacd5125f78c2b62ff1b13bfa237e373998ecd44f0c88bdc0b7a239afa9e8f25e2bfb2cc05a4de11f142898c42ce963f246c
+  metadata.gz: 6317b298a05085564cfaaee1beef6b5749e83f8bc877538b448db66b50c3dde50f0380a5fa7f178dff7feca6840564c486f263891a838d4ed51617b7ff4f8698
+  data.tar.gz: 12ba1c564acca2f7d43c80a384a0767b00592abf5126d0016a0bf7cadf610e6d342890cd54597a602654e8449491135a3dda488bb7c1c8aaeb72d035bdc7c5d6

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## v2.0.0
+
+- `Hekenga::Iterator` has been replaced by `Hekenga::IdIterator`. If any
+    selector or sort is set on a document task migration scope, it no longer forces an
+    ascending ID sort. This should help to prevent index misses, though there is a
+    tradeoff that documents being concurrently updated may be skipped or
+    processed multiple times. Hekenga tries to guard against processing multiple
+    times. Manually specifying an `asc(:_id)` on your scope will continue to
+    process documents in ID order.
+- Document tasks now support a new option, `cursor_timeout`. This is the maximum
+    time a document task's `scope` can be iterated and queue jobs within. The
+    default is one day.
+
+## v1.1.0
+
+- `setup` is now passed the current batch of documents so it can be used to
+    preload effectively
+- `after` has been added - this is a block of code that will be passed the
+    current batch of written documents after a write has been completed. If
+    no documents were written, this block will not be called
+
 ## v1.0.4
 
 - Log errors during document tasks to stdout

data/README.md

@@ -3,9 +3,6 @@
 An attempt at a migration framework for MongoDB that supports parallel document
 processing via ActiveJob, chained jobs and error recovery.
 
-**Note that this gem is currently in pre-alpha - assume most things have a high
-chance of being broken.**
-
 ## Installation
 
 Add this line to your application's Gemfile:

data/docker-compose.yml

@@ -8,7 +8,7 @@ networks:
 
 services:
   mongo:
-    image: mongo:5
+    image: mongo:6
     command: ["--replSet", "rs0", "--bind_ip", "localhost,mongo"]
     volumes:
       - mongo:/data/db
@@ -18,7 +18,7 @@ services:
       - hekenga-net
 
   mongosetup:
-    image: mongo:5
+    image: mongo:6
     depends_on:
       - mongo
     restart: "no"

data/lib/hekenga/base_iterator.rb

@@ -0,0 +1,24 @@
+module Hekenga
+  class BaseIterator
+    include Enumerable
+    DEFAULT_TIMEOUT = 86_400 # 1 day in seconds
+
+    attr_reader :cursor_timeout
+
+    def initialize(scope:, cursor_timeout: DEFAULT_TIMEOUT)
+      @scope = scope
+      @cursor_timeout = cursor_timeout
+    end
+
+    private
+
+    def iteration_scope
+      if @scope.selector.blank? && @scope.options.blank?
+        # Apply a default _id sort, it works the best
+        @scope.asc(:_id)
+      else
+        @scope
+      end.max_time_ms(cursor_timeout * 1000) # convert to ms
+    end
+  end
+end

data/lib/hekenga/document_task.rb

@@ -1,8 +1,9 @@
 require 'hekenga/irreversible'
+require 'hekenga/base_iterator'
 module Hekenga
   class DocumentTask
-    attr_reader :ups, :downs, :setups, :filters
-    attr_accessor :parallel, :scope, :timeless, :batch_size
+    attr_reader :ups, :downs, :setups, :filters, :after_callbacks
+    attr_accessor :parallel, :scope, :timeless, :batch_size, :cursor_timeout
     attr_accessor :description, :invalid_strategy, :skip_prepare, :write_strategy
     attr_accessor :always_write, :use_transaction
 
@@ -11,12 +12,14 @@ module Hekenga
       @downs            = []
       @setups           = []
       @filters          = []
+      @after_callbacks  = []
       @invalid_strategy = :continue
       @write_strategy   = :update
       @skip_prepare     = false
       @batch_size       = nil
       @always_write     = false
       @use_transaction  = false
+      @cursor_timeout   = Hekenga::BaseIterator::DEFAULT_TIMEOUT
     end
 
     def validate!

data/lib/hekenga/document_task_executor.rb

@@ -28,6 +28,7 @@ module Hekenga
         # In test mode, the transaction will be aborted - so we need to write
         # the result outside of the run! block
         write_result if task_record.test_mode
+        after_callback
       end
     end
 
@@ -206,7 +207,7 @@ module Hekenga
       @context = Hekenga::Context.new(test_mode: task_record.test_mode)
       begin
         task.setups&.each do |setup|
-          @context.instance_exec(&setup)
+          @context.instance_exec(records, &setup)
         end
       rescue => e
         fail_and_cancel!(e)
@@ -217,6 +218,18 @@ module Hekenga
       @context = nil
     end
 
+    def after_callback
+      return if records_to_write.empty?
+
+      task.after_callbacks&.each do |callback|
+        @context.instance_exec(records_to_write, &callback)
+      end
+    rescue => e
+      # Error is just printed for now as we've already migrated, don't
+      # want to overwrite the task record state
+      print_error(e)
+    end
+
     def fail_and_cancel!(error)
       log = migration.log(task_idx)
       log.add_failure({

data/lib/hekenga/dsl/document_task.rb

@@ -28,6 +28,10 @@ module Hekenga
         @object.write_strategy = strategy
       end
 
+      def cursor_timeout(timeout)
+        @object.cursor_timeout = timeout.to_i
+      end
+
       def scope(scope)
         @object.scope = scope
       end
@@ -67,6 +71,10 @@ module Hekenga
       def down(&block)
         @object.downs.push block
       end
+
+      def after(&block)
+        @object.after_callbacks.push block
+      end
     end
   end
 end

data/lib/hekenga/id_iterator.rb

@@ -0,0 +1,34 @@
+require "hekenga/base_iterator"
+module Hekenga
+  class IdIterator < BaseIterator
+    DEFAULT_ID = "_id".freeze
+
+    attr_reader :id_property
+
+    def initialize(id_property: DEFAULT_ID, **kwargs)
+      super(**kwargs)
+      @id_property = id_property
+    end
+
+    def each
+      with_view do |view|
+        view.each do |doc|
+          yield doc[id_property]
+        end
+      end
+    end
+
+    private
+
+    def with_view
+      view = iteration_scope.view
+      yield view
+    ensure
+      view.close_query
+    end
+
+    def iteration_scope
+      super.only(id_property)
+    end
+  end
+end

data/lib/hekenga/migration.rb

@@ -2,6 +2,7 @@ require 'hekenga/invalid'
 require 'hekenga/context'
 require 'hekenga/parallel_job'
 require 'hekenga/parallel_task'
+require 'hekenga/mongoid_iterator'
 require 'hekenga/master_process'
 require 'hekenga/document_task_record'
 require 'hekenga/document_task_executor'
@@ -132,18 +133,18 @@ module Hekenga
       records = []
       task_records(task_idx).delete_all unless recover
       executor_key = BSON::ObjectId.new
-      task.scope.asc(:_id).no_timeout.each do |record|
+      Hekenga::MongoidIterator.new(scope: task.scope, cursor_timeout: task.cursor_timeout).each do |record|
         records.push(record)
         next unless records.length == (task.batch_size || batch_size)
 
-        records = filter_out_processed(task, task_idx, records) if recover
+        records = filter_out_processed(task, task_idx, records)
         next unless records.length == (task.batch_size || batch_size)
 
         execute_document_task(task_idx, executor_key, records)
         records = []
         return if log.cancel
       end
-      records = filter_out_processed(task, task_idx, records) if recover
+      records = filter_out_processed(task, task_idx, records)
       execute_document_task(task_idx, executor_key, records) if records.any?
       return if log.cancel
       log_done!

data/lib/hekenga/mongoid_iterator.rb

@@ -0,0 +1,8 @@
+require "hekenga/base_iterator"
+module Hekenga
+  class MongoidIterator < BaseIterator
+    def each(&block)
+      iteration_scope.each(&block)
+    end
+  end
+end

data/lib/hekenga/parallel_task.rb

@@ -1,4 +1,4 @@
-require 'hekenga/iterator'
+require 'hekenga/id_iterator'
 require 'hekenga/document_task_executor'
 require 'hekenga/task_splitter'
 
@@ -15,13 +15,13 @@ module Hekenga
 
     def start!
       clear_task_records!
-      @executor_key = BSON::ObjectId.new
+      regenerate_executor_key
       generate_for_scope(task.scope)
       check_for_completion!
     end
 
     def resume!
-      @executor_key = BSON::ObjectId.new
+      regenerate_executor_key
       task_records.set(executor_key: @executor_key)
       queue_jobs!(task_records.incomplete)
       generate_new_records!
@@ -41,16 +41,43 @@ module Hekenga
 
     private
 
+    def regenerate_executor_key
+      @executor_key = BSON::ObjectId.new
+    end
+
     def generate_for_scope(scope)
-      Hekenga::Iterator.new(scope, size: 100_000).each do |id_block|
-        task_records = id_block.each_slice(batch_size).map do |id_slice|
-          generate_task_records!(id_slice)
-        end
+      Hekenga::IdIterator.new(
+        scope: scope,
+        cursor_timeout: task.cursor_timeout
+        # Batch Batches of IDs
+      ).each_slice(batch_size).each_slice(enqueue_size) do |id_block|
+        sanitize_id_block!(id_block)
+        task_records = id_block.reject(&:empty?).map(&method(:generate_task_record!))
         write_task_records!(task_records)
         queue_jobs!(task_records)
       end
     end
 
+    def enqueue_size
+      500 # task records written + enqueued at a time
+    end
+
+    def sanitize_id_block!(id_block)
+      return if task.scope.options.blank? && task.scope.selector.blank?
+
+      # Custom ordering on cursor with parallel updates may result in the same
+      # ID getting yielded into the migration multiple times. Detect this +
+      # remove
+      doubleups = task_records.in(ids: id_block.flatten).pluck(:ids).flatten.to_set
+      return if doubleups.empty?
+
+      id_block.each do |id_slice|
+        id_slice.reject! do |id|
+          doubleups.include?(id)
+        end
+      end
+    end
+
     def generate_new_records!
       last_record = task_records.desc(:_id).first
       last_id = last_record&.ids&.last
@@ -83,7 +110,7 @@ module Hekenga
       migration.task_records(task_idx)
     end
 
-    def generate_task_records!(id_slice)
+    def generate_task_record!(id_slice)
       Hekenga::DocumentTaskRecord.new(
         migration_key: migration.to_key,
         task_idx:      task_idx,

data/lib/hekenga/scaffold.rb

@@ -48,10 +48,11 @@ module Hekenga
         #  #skip_prepare!
         #  #batch_size 25
         #  #write_strategy :update # :delete_then_insert
+        #  #cursor_timeout 86_400 # max allowed time for the cursor to survive, in seconds
         #
         #  # Called once per batch, instance variables will be accessible
-        #  # in the filter & up blocks
-        #  #setup do
+        #  # in the filter, up and after blocks
+        #  #setup do |docs|
         #  #end
         #
         #  #filter do |doc|
@@ -59,6 +60,10 @@ module Hekenga
         #
         #  up do |doc|
         #  end
+        #
+        #  # Called once per batch passing successfully written records
+        #  #after do |docs|
+        #  #end
         #end
       end
       EOF

data/lib/hekenga/version.rb

@@ -1,3 +1,3 @@
 module Hekenga
-  VERSION = "1.0.4"
+  VERSION = "2.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hekenga
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 2.0.0
 platform: ruby
 authors:
 - Tapio Saarinen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-06 00:00:00.000000000 Z
+date: 2024-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -159,6 +159,7 @@ files:
 - hekenga.gemspec
 - lib/hekenga.rb
 - lib/hekenga/base_error.rb
+- lib/hekenga/base_iterator.rb
 - lib/hekenga/config.rb
 - lib/hekenga/context.rb
 - lib/hekenga/document_task.rb
@@ -173,12 +174,13 @@ files:
 - lib/hekenga/failure/error.rb
 - lib/hekenga/failure/validation.rb
 - lib/hekenga/failure/write.rb
+- lib/hekenga/id_iterator.rb
 - lib/hekenga/invalid.rb
 - lib/hekenga/irreversible.rb
-- lib/hekenga/iterator.rb
 - lib/hekenga/log.rb
 - lib/hekenga/master_process.rb
 - lib/hekenga/migration.rb
+- lib/hekenga/mongoid_iterator.rb
 - lib/hekenga/parallel_job.rb
 - lib/hekenga/parallel_task.rb
 - lib/hekenga/scaffold.rb

data/lib/hekenga/iterator.rb

@@ -1,26 +0,0 @@
-module Hekenga
-  class Iterator
-    include Enumerable
-
-    SMALLEST_ID = BSON::ObjectId.from_string('0'*24)
-
-    attr_reader :scope, :size
-
-    def initialize(scope, size:)
-      @scope = scope
-      @size = size
-    end
-
-    def each(&block)
-      current_id = SMALLEST_ID
-      base_scope = scope.asc(:_id).limit(size)
-
-      loop do
-        ids = base_scope.and(_id: {'$gt': current_id}).pluck(:_id)
-        break if ids.empty?
-        yield ids
-        current_id = ids.sort.last
-      end
-    end
-  end
-end