crono_trigger 0.8.0 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 734da2ad0480e79b440d8ef73f9b83fb88f13e6f6510513340c62b70c3750c39
-  data.tar.gz: b338b9c86a9fdf01d0ba739fa9f53c9536393cec05d7fe5c8943fe05782ad7b4
+  metadata.gz: 94ee913ba2838c73020154bb6f85f49ff93a4d2d3ba39bed1d0be26255b7d06c
+  data.tar.gz: 4222b9af986177a6190c954b8dc9c639a06f5775568422929e3e33699443e5c5
 SHA512:
-  metadata.gz: 6e73b97cece87a4b887e4cec1c5cbd153cc9ad51c3de1956c38ef2a5000fa74c0d3e52c07401c0aa033f0921852a2f6be85533aea050d425dc220c591681a210
-  data.tar.gz: dfbec13afaa581a544b50481fa54509ace24de39f8ecb6d99788e508bd680da1e5afd89abeb0cde7be8ad8db3c7c4b56ae67dbdf0bd087775eaeed62b92d8cac
+  metadata.gz: 2f0afb27ab496e72e5754c7adb777526aefee00b7c9e76e31775395c1d40dadf5889dc072308193becc43fa2b2e2e87a92f5ef5a93942319869065aa590aeb4b
+  data.tar.gz: 21a0f3570da6a32302bd84ed72630afdcdf7c70f0b6e691fcb0bd21d6e3cdde87cf91129a74a09a2fd692f0a33830d7e904df6fc253ba9a63eb4709c225f2e3c

data/.github/workflows/rspec.yml

@@ -11,17 +11,21 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby-version: ['2.7', '3.0', '3.1']
+        ruby-version: ['3.1', '3.2', '3.3']
         gemfile: ["Gemfile", "gemfiles/activerecord-61.gemfile", "gemfiles/activerecord-70.gemfile", "gemfiles/activerecord-71.gemfile"]
+        exclude:
+          - ruby-version: '3.2'
+            gemfile: "gemfiles/activerecord-61.gemfile"
+          - ruby-version: '3.3'
+            gemfile: "gemfiles/activerecord-61.gemfile"
     env:
       BUNDLE_GEMFILE: ${{ matrix.gemfile }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Ruby
     # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
     # change this to (see https://github.com/ruby/setup-ruby#versioning):
-    # uses: ruby/setup-ruby@v1
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby-version }}

data/README.md

@@ -178,6 +178,57 @@ Usage: crono_trigger [options] MODEL [MODEL..]
     -h, --help                       Prints this help
 ```
 
+#### Handle errors
+
+This gem provides the following options to handle errors:
+
+```ruby
+CronoTrigger.configure do |config|
+  # These handlers are called when the execute method fails even after retries.
+  config.error_handlers << proc do |exception, record|
+    ActiveRecord::Base.logger.error("Failed to process #{record.class}##{record.id}: #{exception} (#{exception.class})")
+  end
+
+  # These handlers are called when an exception occurs outside of the execute method.
+  config.global_error_handlers << proc do |exception|
+    ActiveRecord::Base.logger.error("#{exception} (#{exception.class})\n#{exception.backtrace.join("\n")}")
+  end
+
+  # db_error_retriable_options are passed to `Retriable.retriable` used in the idempotent code that accesses the database
+  # except for the code processing each record.
+  # Here is the default value.
+  config.db_error_retriable_options = {
+    on: {
+      ActiveRecord::ConnectionNotEstablished => nil,
+    },
+  }
+end
+```
+
+For example, if you would like to reconnect to the database before retry for some reason, you can do so using `on_retry` option as follows:
+
+```ruby
+CronoTrigger.configure do |config|
+  config.db_error_retriable_options = {
+    on: {
+      ActiveRecord::ConnectionNotEstablished => nil,
+      Mysql2::Error::ConnectionError => nil,
+      ActiveRecord::StatementInvalid => /MySQL server is running with the --read-only option/,
+    },
+    on_retry: proc do |exception, try, elapsed_time, interval|
+      ActiveRecord::Base.logger.info("#{try}th: Retry on #{exception.class}: #{exception}")
+      next unless exception.is_a?(ActiveRecord::StatementInvalid)
+
+      ActiveRecord::Base.logger.info("#{try}th: Reconnect to MySQL on #{exception.class}: #{exception}")
+      # NOTE: The connection acquired here might be different from the one used in the code that raised the error,
+      #   but we don't have a way to get the latter connection.
+      ActiveRecord::Base.connection_pool.with_connection(&:reconnect!)
+    end,
+  }
+end
+```
+
+
 ## Specification
 
 ### Columns

data/crono_trigger.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rack-contrib"
   spec.add_dependency "oj"
   spec.add_dependency "activerecord", ">= 4.2"
+  spec.add_dependency "retriable"
 
   spec.add_development_dependency "sqlite3", "~> 1.3"
   spec.add_development_dependency "mysql2"
@@ -38,5 +39,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_development_dependency "codecov"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "simplecov-cobertura"
 end

data/lib/crono_trigger/polling_thread.rb

@@ -58,8 +58,10 @@ module CronoTrigger
 
       maybe_has_next = true
       while maybe_has_next && !@stop_flag.set?
-        records, maybe_has_next = model.connection_pool.with_connection do
-          model.executables_with_lock(limit: CronoTrigger.config.fetch_records || CronoTrigger.config.executor_thread * 3, worker_count: @worker_count)
+        records, maybe_has_next = CronoTrigger.retry_on_db_errors do
+          model.connection_pool.with_connection do
+            model.executables_with_lock(limit: CronoTrigger.config.fetch_records || CronoTrigger.config.executor_thread * 3, worker_count: @worker_count)
+          end
         end
 
         records.each do |record|

data/lib/crono_trigger/schedulable.rb

@@ -81,6 +81,12 @@ module CronoTrigger
 
           return [records, maybe_has_next] if records.size == limit
         end
+
+        [records, maybe_has_next]
+      rescue => e
+        raise if records.empty?
+
+        logger&.warn("Failed to fetching some records but continue processing records: #{e} (#{e.class})")
         [records, maybe_has_next]
       end
 

data/lib/crono_trigger/version.rb

@@ -1,3 +1,3 @@
 module CronoTrigger
-  VERSION = "0.8.0"
+  VERSION = "0.8.2"
 end

data/lib/crono_trigger/worker.rb

@@ -38,7 +38,6 @@ module CronoTrigger
       @heartbeat_thread = run_heartbeat_thread
       @signal_fetcn_thread = run_signal_fetch_thread
       @monitor_thread = run_monitor_thread
-      @worker_count_updater_thread = run_worker_count_updater_thread
 
       polling_thread_count = CronoTrigger.config.polling_thread || [@model_names.size, Concurrent.processor_count].min
       # Assign local variable for Signal handling
@@ -46,6 +45,8 @@ module CronoTrigger
       @polling_threads = polling_threads
       @polling_threads.each(&:run)
 
+      @worker_count_updater_thread = run_worker_count_updater_thread
+
       ServerEngine::SignalThread.new do |st|
         st.trap(:TSTP) do
           @logger.info("[worker_id:#{@crono_trigger_worker_id}] Transit to quiet mode")
@@ -117,8 +118,8 @@ module CronoTrigger
     end
 
     def heartbeat
-      CronoTrigger::Models::Worker.connection_pool.with_connection do
-        begin
+      CronoTrigger.retry_on_db_errors do
+        CronoTrigger::Models::Worker.connection_pool.with_connection do
           worker_record = CronoTrigger::Models::Worker.find_or_initialize_by(worker_id: @crono_trigger_worker_id)
           worker_record.max_thread_size = @executor.max_length
           worker_record.current_executing_size = @execution_counter.value
@@ -128,11 +129,11 @@ module CronoTrigger
           worker_record.last_heartbeated_at = Time.current
           @logger.info("[worker_id:#{@crono_trigger_worker_id}] Send heartbeat to database")
           worker_record.save!
-        rescue => ex
-          CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
-          stop
         end
       end
+    rescue => ex
+      CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
+      stop
     end
 
     def executor_status
@@ -152,18 +153,22 @@ module CronoTrigger
 
     def unregister
       @logger.info("[worker_id:#{@crono_trigger_worker_id}] Unregister worker from database")
-      CronoTrigger::Models::Worker.connection_pool.with_connection do
-        CronoTrigger::Models::Worker.find_by(worker_id: @crono_trigger_worker_id)&.destroy
+      CronoTrigger.retry_on_db_errors do
+        CronoTrigger::Models::Worker.connection_pool.with_connection do
+          CronoTrigger::Models::Worker.find_by(worker_id: @crono_trigger_worker_id)&.destroy
+        end
       end
     rescue => ex
       CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
     end
 
     def handle_signal_from_rdb
-      CronoTrigger::Models::Signal.connection_pool.with_connection do
-        CronoTrigger::Models::Signal.sent_to_me.take(1)[0]&.tap do |s|
-          @logger.info("[worker_id:#{@crono_trigger_worker_id}] Receive Signal #{s.signal} from database")
-          s.kill_me(to_supervisor: s.signal != "TSTP")
+      CronoTrigger.retry_on_db_errors do
+        CronoTrigger::Models::Signal.connection_pool.with_connection do
+          CronoTrigger::Models::Signal.sent_to_me.take(1)[0]&.tap do |s|
+            @logger.info("[worker_id:#{@crono_trigger_worker_id}] Receive Signal #{s.signal} from database")
+            s.kill_me(to_supervisor: s.signal != "TSTP")
+          end
         end
       end
     rescue => ex
@@ -173,29 +178,31 @@ module CronoTrigger
     def monitor
       return unless ActiveSupport::Notifications.notifier.listening?(CronoTrigger::Events::MONITOR)
 
-      CronoTrigger::Models::Worker.connection_pool.with_connection do
-        if workers_processing_same_models.order(:worker_id).limit(1).pluck(:worker_id).first != @crono_trigger_worker_id
-          # Return immediately to avoid redundant instruments
-          return
-        end
-
-        @model_names.each do |model_name|
-          model = model_name.classify.constantize
-          executable_count = model.executables.limit(nil).count
-
-          execute_lock_column = model.crono_trigger_column_name(:execute_lock)
-          oldest_execute_lock = model.executables(including_locked: true).where.not(execute_lock_column => 0).order(execute_lock_column).limit(1).pluck(execute_lock_column).first
-
-          next_execute_at_column = model.crono_trigger_column_name(:next_execute_at)
-          oldest_next_execute_at = model.executables.order(next_execute_at_column).limit(1).pluck(next_execute_at_column).first
-
-          now = Time.now
-          ActiveSupport::Notifications.instrument(CronoTrigger::Events::MONITOR, {
-            model_name: model_name,
-            executable_count: executable_count,
-            max_lock_duration_sec: oldest_execute_lock.nil? ? 0 : now.to_i - oldest_execute_lock,
-            max_latency_sec: oldest_next_execute_at.nil? ? 0 : now - oldest_next_execute_at,
-          })
+      CronoTrigger.retry_on_db_errors do
+        CronoTrigger::Models::Worker.connection_pool.with_connection do
+          if workers_processing_same_models.order(:worker_id).limit(1).pluck(:worker_id).first != @crono_trigger_worker_id
+            # Return immediately to avoid redundant instruments
+            return
+          end
+
+          @model_names.each do |model_name|
+            model = model_name.classify.constantize
+            executable_count = model.executables.limit(nil).count
+
+            execute_lock_column = model.crono_trigger_column_name(:execute_lock)
+            oldest_execute_lock = model.executables(including_locked: true).where.not(execute_lock_column => 0).order(execute_lock_column).limit(1).pluck(execute_lock_column).first
+
+            next_execute_at_column = model.crono_trigger_column_name(:next_execute_at)
+            oldest_next_execute_at = model.executables.order(next_execute_at_column).limit(1).pluck(next_execute_at_column).first
+
+            now = Time.now
+            ActiveSupport::Notifications.instrument(CronoTrigger::Events::MONITOR, {
+              model_name: model_name,
+              executable_count: executable_count,
+              max_lock_duration_sec: oldest_execute_lock.nil? ? 0 : now.to_i - oldest_execute_lock,
+              max_latency_sec: oldest_next_execute_at.nil? ? 0 : now - oldest_next_execute_at,
+            })
+          end
         end
       end
     rescue => ex
@@ -203,10 +210,12 @@ module CronoTrigger
     end
 
     def update_worker_count
-      CronoTrigger::Models::Worker.connection_pool.with_connection do
-        worker_count = workers_processing_same_models.count
-        return if worker_count.zero?
-        @polling_threads.each { |th| th.worker_count = worker_count }
+      CronoTrigger.retry_on_db_errors do
+        CronoTrigger::Models::Worker.connection_pool.with_connection do
+          worker_count = workers_processing_same_models.count
+          return if worker_count.zero?
+          @polling_threads.each { |th| th.worker_count = worker_count }
+        end
       end
     rescue => ex
       CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)

data/lib/crono_trigger.rb

@@ -4,6 +4,7 @@ require "ostruct"
 require "socket"
 require "active_record"
 require "concurrent"
+require "retriable"
 require "crono_trigger/events"
 require "crono_trigger/models/worker"
 require "crono_trigger/models/signal"
@@ -22,6 +23,11 @@ module CronoTrigger
     model_names: nil,
     error_handlers: [],
     global_error_handlers: [],
+    db_error_retriable_options: {
+      on: {
+        ActiveRecord::ConnectionNotEstablished => nil,
+      },
+    }
   )
 
   def self.config
@@ -52,6 +58,12 @@ module CronoTrigger
   def self.workers
     CronoTrigger::Models::Worker.alive_workers
   end
+
+  def self.retry_on_db_errors
+    Retriable.retriable(CronoTrigger.config.db_error_retriable_options) do
+      yield
+    end
+  end
 end
 
 if defined?(Rails)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crono_trigger
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.2
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-21 00:00:00.000000000 Z
+date: 2024-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chrono
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: retriable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -221,7 +235,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: codecov
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-cobertura
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -340,7 +368,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.5.9
 signing_key:
 specification_version: 4
 summary: In Service Asynchronous Job Scheduler for Rails