crono_trigger 0.8.1 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fa5ce10c898d430d072b1a8617b35388228cb76db4a988ab8de6aa53f8315985
4
- data.tar.gz: 94fd599e1a3771fe4e04395a4ebffcf35c8d1df67d8c1d40762403a2203089ae
3
+ metadata.gz: 94ee913ba2838c73020154bb6f85f49ff93a4d2d3ba39bed1d0be26255b7d06c
4
+ data.tar.gz: 4222b9af986177a6190c954b8dc9c639a06f5775568422929e3e33699443e5c5
5
5
  SHA512:
6
- metadata.gz: d8f22e3e9a869943f18fa8309fe66a886713092026fe6655076a6962e039216c3142572f444a84e0f3fb504dce1e9921e2299508d803eaf4cd1ff447092b831b
7
- data.tar.gz: 2ebbdd8c5da440e162a81780cae9adad698871f7b79c6028b6083a55e64bd0df2ecb36317afd88b741166591c99828e1bc23ee1a80cae47021f527edf3b7d029
6
+ metadata.gz: 2f0afb27ab496e72e5754c7adb777526aefee00b7c9e76e31775395c1d40dadf5889dc072308193becc43fa2b2e2e87a92f5ef5a93942319869065aa590aeb4b
7
+ data.tar.gz: 21a0f3570da6a32302bd84ed72630afdcdf7c70f0b6e691fcb0bd21d6e3cdde87cf91129a74a09a2fd692f0a33830d7e904df6fc253ba9a63eb4709c225f2e3c
@@ -11,17 +11,21 @@ jobs:
11
11
  runs-on: ubuntu-latest
12
12
  strategy:
13
13
  matrix:
14
- ruby-version: ['2.7', '3.0', '3.1']
14
+ ruby-version: ['3.1', '3.2', '3.3']
15
15
  gemfile: ["Gemfile", "gemfiles/activerecord-61.gemfile", "gemfiles/activerecord-70.gemfile", "gemfiles/activerecord-71.gemfile"]
16
+ exclude:
17
+ - ruby-version: '3.2'
18
+ gemfile: "gemfiles/activerecord-61.gemfile"
19
+ - ruby-version: '3.3'
20
+ gemfile: "gemfiles/activerecord-61.gemfile"
16
21
  env:
17
22
  BUNDLE_GEMFILE: ${{ matrix.gemfile }}
18
23
 
19
24
  steps:
20
- - uses: actions/checkout@v2
25
+ - uses: actions/checkout@v4
21
26
  - name: Set up Ruby
22
27
  # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
23
28
  # change this to (see https://github.com/ruby/setup-ruby#versioning):
24
- # uses: ruby/setup-ruby@v1
25
29
  uses: ruby/setup-ruby@v1
26
30
  with:
27
31
  ruby-version: ${{ matrix.ruby-version }}
data/README.md CHANGED
@@ -178,6 +178,57 @@ Usage: crono_trigger [options] MODEL [MODEL..]
178
178
  -h, --help Prints this help
179
179
  ```
180
180
 
181
+ #### Handle errors
182
+
183
+ This gem provides the following options to handle errors:
184
+
185
+ ```ruby
186
+ CronoTrigger.configure do |config|
187
+ # These handlers are called when the execute method fails even after retries.
188
+ config.error_handlers << proc do |exception, record|
189
+ ActiveRecord::Base.logger.error("Failed to process #{record.class}##{record.id}: #{exception} (#{exception.class})")
190
+ end
191
+
192
+ # These handlers are called when an exception occurs outside of the execute method.
193
+ config.global_error_handlers << proc do |exception|
194
+ ActiveRecord::Base.logger.error("#{exception} (#{exception.class})\n#{exception.backtrace.join("\n")}")
195
+ end
196
+
197
+ # db_error_retriable_options are passed to `Retriable.retriable` used in the idempotent code that accesses the database
198
+ # except for the code processing each record.
199
+ # Here is the default value.
200
+ config.db_error_retriable_options = {
201
+ on: {
202
+ ActiveRecord::ConnectionNotEstablished => nil,
203
+ },
204
+ }
205
+ end
206
+ ```
207
+
208
+ For example, if you would like to reconnect to the database before retry for some reason, you can do so using `on_retry` option as follows:
209
+
210
+ ```ruby
211
+ CronoTrigger.configure do |config|
212
+ config.db_error_retriable_options = {
213
+ on: {
214
+ ActiveRecord::ConnectionNotEstablished => nil,
215
+ Mysql2::Error::ConnectionError => nil,
216
+ ActiveRecord::StatementInvalid => /MySQL server is running with the --read-only option/,
217
+ },
218
+ on_retry: proc do |exception, try, elapsed_time, interval|
219
+ ActiveRecord::Base.logger.info("#{try}th: Retry on #{exception.class}: #{exception}")
220
+ next unless exception.is_a?(ActiveRecord::StatementInvalid)
221
+
222
+ ActiveRecord::Base.logger.info("#{try}th: Reconnect to MySQL on #{exception.class}: #{exception}")
223
+ # NOTE: The connection acquired here might be different from the one used in the code that raised the error,
224
+ # but we don't have a way to get the latter connection.
225
+ ActiveRecord::Base.connection_pool.with_connection(&:reconnect!)
226
+ end,
227
+ }
228
+ end
229
+ ```
230
+
231
+
181
232
  ## Specification
182
233
 
183
234
  ### Columns
@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
30
30
  spec.add_dependency "rack-contrib"
31
31
  spec.add_dependency "oj"
32
32
  spec.add_dependency "activerecord", ">= 4.2"
33
+ spec.add_dependency "retriable"
33
34
 
34
35
  spec.add_development_dependency "sqlite3", "~> 1.3"
35
36
  spec.add_development_dependency "mysql2"
@@ -38,5 +39,6 @@ Gem::Specification.new do |spec|
38
39
  spec.add_development_dependency "bundler", "~> 2.0"
39
40
  spec.add_development_dependency "rake"
40
41
  spec.add_development_dependency "rspec"
41
- spec.add_development_dependency "codecov"
42
+ spec.add_development_dependency "simplecov"
43
+ spec.add_development_dependency "simplecov-cobertura"
42
44
  end
@@ -58,8 +58,10 @@ module CronoTrigger
58
58
 
59
59
  maybe_has_next = true
60
60
  while maybe_has_next && !@stop_flag.set?
61
- records, maybe_has_next = model.connection_pool.with_connection do
62
- model.executables_with_lock(limit: CronoTrigger.config.fetch_records || CronoTrigger.config.executor_thread * 3, worker_count: @worker_count)
61
+ records, maybe_has_next = CronoTrigger.retry_on_db_errors do
62
+ model.connection_pool.with_connection do
63
+ model.executables_with_lock(limit: CronoTrigger.config.fetch_records || CronoTrigger.config.executor_thread * 3, worker_count: @worker_count)
64
+ end
63
65
  end
64
66
 
65
67
  records.each do |record|
@@ -81,6 +81,12 @@ module CronoTrigger
81
81
 
82
82
  return [records, maybe_has_next] if records.size == limit
83
83
  end
84
+
85
+ [records, maybe_has_next]
86
+ rescue => e
87
+ raise if records.empty?
88
+
89
+ logger&.warn("Failed to fetching some records but continue processing records: #{e} (#{e.class})")
84
90
  [records, maybe_has_next]
85
91
  end
86
92
 
@@ -1,3 +1,3 @@
1
1
  module CronoTrigger
2
- VERSION = "0.8.1"
2
+ VERSION = "0.8.2"
3
3
  end
@@ -118,8 +118,8 @@ module CronoTrigger
118
118
  end
119
119
 
120
120
  def heartbeat
121
- CronoTrigger::Models::Worker.connection_pool.with_connection do
122
- begin
121
+ CronoTrigger.retry_on_db_errors do
122
+ CronoTrigger::Models::Worker.connection_pool.with_connection do
123
123
  worker_record = CronoTrigger::Models::Worker.find_or_initialize_by(worker_id: @crono_trigger_worker_id)
124
124
  worker_record.max_thread_size = @executor.max_length
125
125
  worker_record.current_executing_size = @execution_counter.value
@@ -129,11 +129,11 @@ module CronoTrigger
129
129
  worker_record.last_heartbeated_at = Time.current
130
130
  @logger.info("[worker_id:#{@crono_trigger_worker_id}] Send heartbeat to database")
131
131
  worker_record.save!
132
- rescue => ex
133
- CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
134
- stop
135
132
  end
136
133
  end
134
+ rescue => ex
135
+ CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
136
+ stop
137
137
  end
138
138
 
139
139
  def executor_status
@@ -153,18 +153,22 @@ module CronoTrigger
153
153
 
154
154
  def unregister
155
155
  @logger.info("[worker_id:#{@crono_trigger_worker_id}] Unregister worker from database")
156
- CronoTrigger::Models::Worker.connection_pool.with_connection do
157
- CronoTrigger::Models::Worker.find_by(worker_id: @crono_trigger_worker_id)&.destroy
156
+ CronoTrigger.retry_on_db_errors do
157
+ CronoTrigger::Models::Worker.connection_pool.with_connection do
158
+ CronoTrigger::Models::Worker.find_by(worker_id: @crono_trigger_worker_id)&.destroy
159
+ end
158
160
  end
159
161
  rescue => ex
160
162
  CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
161
163
  end
162
164
 
163
165
  def handle_signal_from_rdb
164
- CronoTrigger::Models::Signal.connection_pool.with_connection do
165
- CronoTrigger::Models::Signal.sent_to_me.take(1)[0]&.tap do |s|
166
- @logger.info("[worker_id:#{@crono_trigger_worker_id}] Receive Signal #{s.signal} from database")
167
- s.kill_me(to_supervisor: s.signal != "TSTP")
166
+ CronoTrigger.retry_on_db_errors do
167
+ CronoTrigger::Models::Signal.connection_pool.with_connection do
168
+ CronoTrigger::Models::Signal.sent_to_me.take(1)[0]&.tap do |s|
169
+ @logger.info("[worker_id:#{@crono_trigger_worker_id}] Receive Signal #{s.signal} from database")
170
+ s.kill_me(to_supervisor: s.signal != "TSTP")
171
+ end
168
172
  end
169
173
  end
170
174
  rescue => ex
@@ -174,29 +178,31 @@ module CronoTrigger
174
178
  def monitor
175
179
  return unless ActiveSupport::Notifications.notifier.listening?(CronoTrigger::Events::MONITOR)
176
180
 
177
- CronoTrigger::Models::Worker.connection_pool.with_connection do
178
- if workers_processing_same_models.order(:worker_id).limit(1).pluck(:worker_id).first != @crono_trigger_worker_id
179
- # Return immediately to avoid redundant instruments
180
- return
181
- end
182
-
183
- @model_names.each do |model_name|
184
- model = model_name.classify.constantize
185
- executable_count = model.executables.limit(nil).count
186
-
187
- execute_lock_column = model.crono_trigger_column_name(:execute_lock)
188
- oldest_execute_lock = model.executables(including_locked: true).where.not(execute_lock_column => 0).order(execute_lock_column).limit(1).pluck(execute_lock_column).first
189
-
190
- next_execute_at_column = model.crono_trigger_column_name(:next_execute_at)
191
- oldest_next_execute_at = model.executables.order(next_execute_at_column).limit(1).pluck(next_execute_at_column).first
192
-
193
- now = Time.now
194
- ActiveSupport::Notifications.instrument(CronoTrigger::Events::MONITOR, {
195
- model_name: model_name,
196
- executable_count: executable_count,
197
- max_lock_duration_sec: oldest_execute_lock.nil? ? 0 : now.to_i - oldest_execute_lock,
198
- max_latency_sec: oldest_next_execute_at.nil? ? 0 : now - oldest_next_execute_at,
199
- })
181
+ CronoTrigger.retry_on_db_errors do
182
+ CronoTrigger::Models::Worker.connection_pool.with_connection do
183
+ if workers_processing_same_models.order(:worker_id).limit(1).pluck(:worker_id).first != @crono_trigger_worker_id
184
+ # Return immediately to avoid redundant instruments
185
+ return
186
+ end
187
+
188
+ @model_names.each do |model_name|
189
+ model = model_name.classify.constantize
190
+ executable_count = model.executables.limit(nil).count
191
+
192
+ execute_lock_column = model.crono_trigger_column_name(:execute_lock)
193
+ oldest_execute_lock = model.executables(including_locked: true).where.not(execute_lock_column => 0).order(execute_lock_column).limit(1).pluck(execute_lock_column).first
194
+
195
+ next_execute_at_column = model.crono_trigger_column_name(:next_execute_at)
196
+ oldest_next_execute_at = model.executables.order(next_execute_at_column).limit(1).pluck(next_execute_at_column).first
197
+
198
+ now = Time.now
199
+ ActiveSupport::Notifications.instrument(CronoTrigger::Events::MONITOR, {
200
+ model_name: model_name,
201
+ executable_count: executable_count,
202
+ max_lock_duration_sec: oldest_execute_lock.nil? ? 0 : now.to_i - oldest_execute_lock,
203
+ max_latency_sec: oldest_next_execute_at.nil? ? 0 : now - oldest_next_execute_at,
204
+ })
205
+ end
200
206
  end
201
207
  end
202
208
  rescue => ex
@@ -204,10 +210,12 @@ module CronoTrigger
204
210
  end
205
211
 
206
212
  def update_worker_count
207
- CronoTrigger::Models::Worker.connection_pool.with_connection do
208
- worker_count = workers_processing_same_models.count
209
- return if worker_count.zero?
210
- @polling_threads.each { |th| th.worker_count = worker_count }
213
+ CronoTrigger.retry_on_db_errors do
214
+ CronoTrigger::Models::Worker.connection_pool.with_connection do
215
+ worker_count = workers_processing_same_models.count
216
+ return if worker_count.zero?
217
+ @polling_threads.each { |th| th.worker_count = worker_count }
218
+ end
211
219
  end
212
220
  rescue => ex
213
221
  CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
data/lib/crono_trigger.rb CHANGED
@@ -4,6 +4,7 @@ require "ostruct"
4
4
  require "socket"
5
5
  require "active_record"
6
6
  require "concurrent"
7
+ require "retriable"
7
8
  require "crono_trigger/events"
8
9
  require "crono_trigger/models/worker"
9
10
  require "crono_trigger/models/signal"
@@ -22,6 +23,11 @@ module CronoTrigger
22
23
  model_names: nil,
23
24
  error_handlers: [],
24
25
  global_error_handlers: [],
26
+ db_error_retriable_options: {
27
+ on: {
28
+ ActiveRecord::ConnectionNotEstablished => nil,
29
+ },
30
+ }
25
31
  )
26
32
 
27
33
  def self.config
@@ -52,6 +58,12 @@ module CronoTrigger
52
58
  def self.workers
53
59
  CronoTrigger::Models::Worker.alive_workers
54
60
  end
61
+
62
+ def self.retry_on_db_errors
63
+ Retriable.retriable(CronoTrigger.config.db_error_retriable_options) do
64
+ yield
65
+ end
66
+ end
55
67
  end
56
68
 
57
69
  if defined?(Rails)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crono_trigger
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.1
4
+ version: 0.8.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - joker1007
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-03-25 00:00:00.000000000 Z
11
+ date: 2024-05-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: chrono
@@ -122,6 +122,20 @@ dependencies:
122
122
  - - ">="
123
123
  - !ruby/object:Gem::Version
124
124
  version: '4.2'
125
+ - !ruby/object:Gem::Dependency
126
+ name: retriable
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
125
139
  - !ruby/object:Gem::Dependency
126
140
  name: sqlite3
127
141
  requirement: !ruby/object:Gem::Requirement
@@ -221,7 +235,21 @@ dependencies:
221
235
  - !ruby/object:Gem::Version
222
236
  version: '0'
223
237
  - !ruby/object:Gem::Dependency
224
- name: codecov
238
+ name: simplecov
239
+ requirement: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - ">="
242
+ - !ruby/object:Gem::Version
243
+ version: '0'
244
+ type: :development
245
+ prerelease: false
246
+ version_requirements: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - ">="
249
+ - !ruby/object:Gem::Version
250
+ version: '0'
251
+ - !ruby/object:Gem::Dependency
252
+ name: simplecov-cobertura
225
253
  requirement: !ruby/object:Gem::Requirement
226
254
  requirements:
227
255
  - - ">="
@@ -340,7 +368,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
340
368
  - !ruby/object:Gem::Version
341
369
  version: '0'
342
370
  requirements: []
343
- rubygems_version: 3.5.3
371
+ rubygems_version: 3.5.9
344
372
  signing_key:
345
373
  specification_version: 4
346
374
  summary: In Service Asynchronous Job Scheduler for Rails