hekenga 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5a372f23cc2fe1751eb08e07e40705b0361fc609b072e14226b72819cfe9647
-  data.tar.gz: 99228c0f076660abe23c92f37c55509b156810fc0a741764b7872d4a0c597263
+  metadata.gz: d81f0def52813a113005c985703355b9d207a9fb91acae8e4948d01dc43f7b33
+  data.tar.gz: f462545e892ababf6a251dd5d366ef59cfc0f799b1ba3456102ee0483cab652d
 SHA512:
-  metadata.gz: e2dfea01ce0c1c17fc51ab550e1805653a255d7c65a8b8424c7a1d6e646d59b9bc70fc45cfcf558d1fd7a6df81e5a2a8230a240e8c75d578736cdf19f4e2e290
-  data.tar.gz: 0af8e27a9f1b4c7ae490788f64457a67fae9a3fb9459821b8066bca61671ae30463971e9ec277d01ba358817992260281e5dceaaab676f9093dc05a941a520ce
+  metadata.gz: 62e018a5ad375a280f641847c3c85c18b190ff7ef83d558a53210b062a80eecb183168af77baac2d2a7f2b7d5d16438b2f6057faaaa7a52e410f952a46673d69
+  data.tar.gz: 10b34cb83cf1f8d643c3cd11f06389ae6911d84c6afa7ff7901a681a735a83bb24108178a162ba0005ef7ed1992c54448ac3eff4479498533505a7ab65a0a71e

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## v2.2.0
+
+- Added a `hekenga watch <path_or_pkey>` CLI command to attach to a running
+    migration and report its status periodically. Accepts an `--interval`
+    option (defaults to `Hekenga.config.report_sleep`).
+- Added a `hekenga failures <path_or_pkey>` CLI command to print all failed and
+    invalid document IDs across a migration's document tasks.
+- Document tasks now support a `skip_validation!` option to write documents
+    without running `.valid?`.
+
 ## v2.1.0
 
 - `per_document` task `scope` will no longer let you specify `.only` or

data/README.md

@@ -43,6 +43,8 @@ $ hekenga run! <path_or_pkey>           # Run a specific migration
 $ hekenga run! <path_or_pkey> --test    # Dry run (no writes persisted)
 $ hekenga run! <path_or_pkey> --clear   # Clear logs before running
 $ hekenga recover! <path_or_pkey>       # Re-process failed/invalid records
+$ hekenga watch <path_or_pkey>          # Attach to a running migration, report status periodically
+$ hekenga failures <path_or_pkey>       # Print all failed/invalid document IDs for a migration
 $ hekenga cancel                        # Cancel all active migrations
 $ hekenga skip <path_or_pkey>           # Mark a migration as skipped
 $ hekenga clear! <path_or_pkey>         # Remove all logs/failures for a migration
@@ -118,6 +120,7 @@ per_document "Process records" do
   timeless!                           # Don't update Mongoid timestamps
   always_write!                       # Write even if the document didn't change
   skip_prepare!                       # Skip Mongoid callbacks on load
+  skip_validation!                    # Write documents without running Mongoid validations
   use_transaction!                    # Wrap each batch in a MongoDB transaction
   batch_size 50                       # Override migration-level batch size
   write_strategy :update              # :update (default) or :delete_then_insert
@@ -143,12 +146,25 @@ Or via the CLI:
 
     $ hekenga run! <path_or_pkey> --test
 
+### Monitoring
+
+Attach to a migration that's already running (for example one launched in another process or via a background job) and print its status on a fixed interval:
+
+    $ hekenga watch <path_or_pkey>
+    $ hekenga watch <path_or_pkey> --interval 5   # report every 5 seconds
+
+The reporting interval defaults to `Hekenga.config.report_sleep`.
+
 ### Recovery
 
 When a migration fails (due to errors, invalid records, or write failures), Hekenga logs the failures and marks the migration as failed. You can re-process only the failed records:
 
     $ hekenga recover! <path_or_pkey>
 
+To inspect exactly which documents failed or were invalid across all document tasks in a migration:
+
+    $ hekenga failures <path_or_pkey>
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/docker-compose.yml

@@ -9,22 +9,41 @@ networks:
 services:
   mongo:
     image: mongo:6
-    command: ["--replSet", "rs0", "--bind_ip", "localhost,mongo"]
+    command: ["--replSet", "rs0", "--bind_ip_all"]
     volumes:
       - mongo:/data/db
     ports:
       - 27017:27017
     networks:
       - hekenga-net
+    healthcheck:
+      # mongo:6 ships mongosh (the legacy `mongo` shell was removed in 6.0).
+      test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping')"]
+      interval: 5s
+      timeout: 5s
+      retries: 12
 
   mongosetup:
     image: mongo:6
     depends_on:
-      - mongo
+      mongo:
+        condition: service_healthy
     restart: "no"
     entrypoint:
-      - bash
-      - "-c"
-      - "sleep 3 && mongo --host mongo:27017 --eval 'rs.initiate({_id: \"rs0\", members: [{_id: 0, host: \"localhost:27017\"}]})'"
+      - mongosh
+      - "--quiet"
+      - "--host"
+      - "mongo:27017"
+      - "--eval"
+      # Idempotent: initiate the single-member replica set only if it hasn't
+      # been configured yet, so re-running compose doesn't error. The member
+      # advertises as localhost:27017 so the driver on the host can reach it
+      # after replica-set topology discovery (required for transactions).
+      - >
+        try { rs.status(); print('replica set already initialised'); }
+        catch (e) {
+          rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'localhost:27017'}]});
+          print('replica set initiated');
+        }
     networks:
       - hekenga-net

data/exe/hekenga

@@ -71,6 +71,20 @@ class HekengaCLI < Thor
     end
   end
 
+  desc "watch PATH_OR_PKEY", "Attach to a running migration and report its status periodically."
+  option :interval, type: :numeric, desc: "Seconds between status reports (default: Hekenga.config.report_sleep)."
+  def watch(path_or_pkey)
+    migration = load_migration(path_or_pkey)
+    interval  = options[:interval] || Hekenga.config.report_sleep
+    Hekenga::MasterProcess.new(migration).watch!(interval: interval)
+  end
+
+  desc "failures PATH_OR_PKEY", "Print all failed and invalid document IDs for a migration."
+  def failures(path_or_pkey)
+    migration = load_migration(path_or_pkey)
+    Hekenga::FailureReport.new(migration).print!
+  end
+
   desc "cancel", "Cancel all active migrations."
   def cancel
     Hekenga::Log.where(done: false).set(cancel: true)

data/lib/hekenga/document_task.rb

@@ -5,7 +5,7 @@ module Hekenga
     attr_reader :ups, :downs, :setups, :filters, :after_callbacks
     attr_accessor :parallel, :scope, :timeless, :batch_size, :cursor_timeout
     attr_accessor :description, :invalid_strategy, :skip_prepare, :write_strategy
-    attr_accessor :always_write, :use_transaction
+    attr_accessor :always_write, :use_transaction, :skip_validation
 
     def initialize
       @ups              = []
@@ -16,6 +16,7 @@ module Hekenga
       @invalid_strategy = :continue
       @write_strategy   = :update
       @skip_prepare     = false
+      @skip_validation  = false
       @batch_size       = nil
       @always_write     = false
       @use_transaction  = false

data/lib/hekenga/document_task_executor.rb

@@ -116,6 +116,8 @@ module Hekenga
     end
 
     def validate_records
+      return records_to_write.concat(migrated_records) if task.skip_validation
+
       migrated_records.each do |record|
         if record.valid?
           records_to_write << record

data/lib/hekenga/dsl/document_task.rb

@@ -56,6 +56,10 @@ module Hekenga
         @object.skip_prepare = true
       end
 
+      def skip_validation!
+        @object.skip_validation = true
+      end
+
       def setup(&block)
         @object.setups.push block
       end

data/lib/hekenga/failure_report.rb

@@ -0,0 +1,60 @@
+module Hekenga
+  class FailureReport
+    def initialize(migration)
+      @migration = migration
+    end
+
+    # Per-task summaries for document tasks that have any failed or invalid
+    # document IDs.
+    # => [{ idx:, description:, failed_ids: [...], invalid_ids: [...] }, ...]
+    def tasks
+      @migration.tasks.each.with_index.filter_map do |task, idx|
+        next unless task.is_a?(Hekenga::DocumentTask)
+
+        records = @migration.task_records(idx).any_of(
+          { :failed_ids.ne  => [] },
+          { :invalid_ids.ne => [] }
+        )
+
+        failed_ids  = []
+        invalid_ids = []
+        records.pluck(:failed_ids, :invalid_ids).each do |failed, invalid|
+          failed_ids.concat(failed)   if failed
+          invalid_ids.concat(invalid) if invalid
+        end
+        next if failed_ids.empty? && invalid_ids.empty?
+
+        {
+          idx:         idx,
+          description: task.description,
+          failed_ids:  failed_ids,
+          invalid_ids: invalid_ids
+        }
+      end
+    end
+
+    def print!
+      Hekenga.log("Failures for #{@migration.to_key}")
+
+      summaries = tasks
+      if summaries.empty?
+        Hekenga.log("  No failed or invalid documents.")
+        return
+      end
+
+      failed_ids  = summaries.flat_map { |summary| summary[:failed_ids] }
+      invalid_ids = summaries.flat_map { |summary| summary[:invalid_ids] }
+
+      print_ids("Failed",  failed_ids)
+      print_ids("Invalid", invalid_ids)
+    end
+
+    private
+
+    def print_ids(label, ids)
+      return if ids.empty?
+
+      Hekenga.log("    #{label} (#{ids.length}): #{ids.map { |id| "\"#{id}\"" }.join(", ")}")
+    end
+  end
+end

data/lib/hekenga/master_process.rb

@@ -33,18 +33,35 @@ module Hekenga
       true
     end
 
+    def watch!(interval: Hekenga.config.report_sleep)
+      if Hekenga.status(@migration) == :naught
+        Hekenga.log "Migration #{@migration.to_key} has not started yet."
+        return false
+      end
+      Hekenga.log "Watching migration #{@migration.to_key}: #{@migration.description}"
+      @migration.tasks.each.with_index do |task, idx|
+        report_while_active(task, idx, interval: interval)
+      rescue Hekenga::TaskFailedError
+        return false
+      end
+      true
+    end
+
     private
 
-    def report_while_active(task, idx)
-      # Wait for the log to be generated
+    def report_while_active(task, idx, interval: Hekenga.config.report_sleep)
+      # Wait for the log to be generated. When watching an out-of-process
+      # migration there is no @active_thread guaranteeing it, so bail out if
+      # the migration finished without ever reaching this task.
       until (@migration.log(idx) rescue nil)
+        return if %i[complete skipped].include?(Hekenga.status(@migration))
         sleep 1
       end
-      # Periodically report on thread progress
+      # Periodically report on progress
       until @migration.log(idx).reload.done
-        @active_thread.join
+        @active_thread&.join
         report_status(task, idx)
-        sleep Hekenga.config.report_sleep
+        sleep interval
       end
       report_status(task, idx) if task.is_a?(Hekenga::DocumentTask)
       report_result(task, idx)

data/lib/hekenga/version.rb

@@ -1,3 +1,3 @@
 module Hekenga
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

data/lib/hekenga.rb

@@ -6,6 +6,7 @@ require "hekenga/config"
 require "hekenga/irreversible"
 require "hekenga/virtual_method"
 require "hekenga/scaffold"
+require "hekenga/failure_report"
 
 module Hekenga
   @@load_all_mutex = Mutex.new

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hekenga
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Tapio Saarinen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-23 00:00:00.000000000 Z
+date: 2026-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -175,6 +175,7 @@ files:
 - lib/hekenga/failure/error.rb
 - lib/hekenga/failure/validation.rb
 - lib/hekenga/failure/write.rb
+- lib/hekenga/failure_report.rb
 - lib/hekenga/id_iterator.rb
 - lib/hekenga/invalid.rb
 - lib/hekenga/irreversible.rb