crono_trigger 0.8.1 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/rspec.yml +7 -3
- data/README.md +51 -0
- data/crono_trigger.gemspec +3 -1
- data/lib/crono_trigger/polling_thread.rb +4 -2
- data/lib/crono_trigger/schedulable.rb +6 -0
- data/lib/crono_trigger/version.rb +1 -1
- data/lib/crono_trigger/worker.rb +46 -38
- data/lib/crono_trigger.rb +12 -0
- metadata +32 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 94ee913ba2838c73020154bb6f85f49ff93a4d2d3ba39bed1d0be26255b7d06c
|
4
|
+
data.tar.gz: 4222b9af986177a6190c954b8dc9c639a06f5775568422929e3e33699443e5c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2f0afb27ab496e72e5754c7adb777526aefee00b7c9e76e31775395c1d40dadf5889dc072308193becc43fa2b2e2e87a92f5ef5a93942319869065aa590aeb4b
|
7
|
+
data.tar.gz: 21a0f3570da6a32302bd84ed72630afdcdf7c70f0b6e691fcb0bd21d6e3cdde87cf91129a74a09a2fd692f0a33830d7e904df6fc253ba9a63eb4709c225f2e3c
|
data/.github/workflows/rspec.yml
CHANGED
@@ -11,17 +11,21 @@ jobs:
|
|
11
11
|
runs-on: ubuntu-latest
|
12
12
|
strategy:
|
13
13
|
matrix:
|
14
|
-
ruby-version: ['
|
14
|
+
ruby-version: ['3.1', '3.2', '3.3']
|
15
15
|
gemfile: ["Gemfile", "gemfiles/activerecord-61.gemfile", "gemfiles/activerecord-70.gemfile", "gemfiles/activerecord-71.gemfile"]
|
16
|
+
exclude:
|
17
|
+
- ruby-version: '3.2'
|
18
|
+
gemfile: "gemfiles/activerecord-61.gemfile"
|
19
|
+
- ruby-version: '3.3'
|
20
|
+
gemfile: "gemfiles/activerecord-61.gemfile"
|
16
21
|
env:
|
17
22
|
BUNDLE_GEMFILE: ${{ matrix.gemfile }}
|
18
23
|
|
19
24
|
steps:
|
20
|
-
- uses: actions/checkout@
|
25
|
+
- uses: actions/checkout@v4
|
21
26
|
- name: Set up Ruby
|
22
27
|
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
23
28
|
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
24
|
-
# uses: ruby/setup-ruby@v1
|
25
29
|
uses: ruby/setup-ruby@v1
|
26
30
|
with:
|
27
31
|
ruby-version: ${{ matrix.ruby-version }}
|
data/README.md
CHANGED
@@ -178,6 +178,57 @@ Usage: crono_trigger [options] MODEL [MODEL..]
|
|
178
178
|
-h, --help Prints this help
|
179
179
|
```
|
180
180
|
|
181
|
+
#### Handle errors
|
182
|
+
|
183
|
+
This gem provides the following options to handle errors:
|
184
|
+
|
185
|
+
```ruby
|
186
|
+
CronoTrigger.configure do |config|
|
187
|
+
# These handlers are called when the execute method fails even after retries.
|
188
|
+
config.error_handlers << proc do |exception, record|
|
189
|
+
ActiveRecord::Base.logger.error("Failed to process #{record.class}##{record.id}: #{exception} (#{exception.class})")
|
190
|
+
end
|
191
|
+
|
192
|
+
# These handlers are called when an exception occurs outside of the execute method.
|
193
|
+
config.global_error_handlers << proc do |exception|
|
194
|
+
ActiveRecord::Base.logger.error("#{exception} (#{exception.class})\n#{exception.backtrace.join("\n")}")
|
195
|
+
end
|
196
|
+
|
197
|
+
# db_error_retriable_options are passed to `Retriable.retriable` used in the idempotent code that accesses the database
|
198
|
+
# except for the code processing each record.
|
199
|
+
# Here is the default value.
|
200
|
+
config.db_error_retriable_options = {
|
201
|
+
on: {
|
202
|
+
ActiveRecord::ConnectionNotEstablished => nil,
|
203
|
+
},
|
204
|
+
}
|
205
|
+
end
|
206
|
+
```
|
207
|
+
|
208
|
+
For example, if you would like to reconnect to the database before retry for some reason, you can do so using `on_retry` option as follows:
|
209
|
+
|
210
|
+
```ruby
|
211
|
+
CronoTrigger.configure do |config|
|
212
|
+
config.db_error_retriable_options = {
|
213
|
+
on: {
|
214
|
+
ActiveRecord::ConnectionNotEstablished => nil,
|
215
|
+
Mysql2::Error::ConnectionError => nil,
|
216
|
+
ActiveRecord::StatementInvalid => /MySQL server is running with the --read-only option/,
|
217
|
+
},
|
218
|
+
on_retry: proc do |exception, try, elapsed_time, interval|
|
219
|
+
ActiveRecord::Base.logger.info("#{try}th: Retry on #{exception.class}: #{exception}")
|
220
|
+
next unless exception.is_a?(ActiveRecord::StatementInvalid)
|
221
|
+
|
222
|
+
ActiveRecord::Base.logger.info("#{try}th: Reconnect to MySQL on #{exception.class}: #{exception}")
|
223
|
+
# NOTE: The connection acquired here might be different from the one used in the code that raised the error,
|
224
|
+
# but we don't have a way to get the latter connection.
|
225
|
+
ActiveRecord::Base.connection_pool.with_connection(&:reconnect!)
|
226
|
+
end,
|
227
|
+
}
|
228
|
+
end
|
229
|
+
```
|
230
|
+
|
231
|
+
|
181
232
|
## Specification
|
182
233
|
|
183
234
|
### Columns
|
data/crono_trigger.gemspec
CHANGED
@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.add_dependency "rack-contrib"
|
31
31
|
spec.add_dependency "oj"
|
32
32
|
spec.add_dependency "activerecord", ">= 4.2"
|
33
|
+
spec.add_dependency "retriable"
|
33
34
|
|
34
35
|
spec.add_development_dependency "sqlite3", "~> 1.3"
|
35
36
|
spec.add_development_dependency "mysql2"
|
@@ -38,5 +39,6 @@ Gem::Specification.new do |spec|
|
|
38
39
|
spec.add_development_dependency "bundler", "~> 2.0"
|
39
40
|
spec.add_development_dependency "rake"
|
40
41
|
spec.add_development_dependency "rspec"
|
41
|
-
spec.add_development_dependency "
|
42
|
+
spec.add_development_dependency "simplecov"
|
43
|
+
spec.add_development_dependency "simplecov-cobertura"
|
42
44
|
end
|
@@ -58,8 +58,10 @@ module CronoTrigger
|
|
58
58
|
|
59
59
|
maybe_has_next = true
|
60
60
|
while maybe_has_next && !@stop_flag.set?
|
61
|
-
records, maybe_has_next =
|
62
|
-
model.
|
61
|
+
records, maybe_has_next = CronoTrigger.retry_on_db_errors do
|
62
|
+
model.connection_pool.with_connection do
|
63
|
+
model.executables_with_lock(limit: CronoTrigger.config.fetch_records || CronoTrigger.config.executor_thread * 3, worker_count: @worker_count)
|
64
|
+
end
|
63
65
|
end
|
64
66
|
|
65
67
|
records.each do |record|
|
@@ -81,6 +81,12 @@ module CronoTrigger
|
|
81
81
|
|
82
82
|
return [records, maybe_has_next] if records.size == limit
|
83
83
|
end
|
84
|
+
|
85
|
+
[records, maybe_has_next]
|
86
|
+
rescue => e
|
87
|
+
raise if records.empty?
|
88
|
+
|
89
|
+
logger&.warn("Failed to fetching some records but continue processing records: #{e} (#{e.class})")
|
84
90
|
[records, maybe_has_next]
|
85
91
|
end
|
86
92
|
|
data/lib/crono_trigger/worker.rb
CHANGED
@@ -118,8 +118,8 @@ module CronoTrigger
|
|
118
118
|
end
|
119
119
|
|
120
120
|
def heartbeat
|
121
|
-
CronoTrigger
|
122
|
-
|
121
|
+
CronoTrigger.retry_on_db_errors do
|
122
|
+
CronoTrigger::Models::Worker.connection_pool.with_connection do
|
123
123
|
worker_record = CronoTrigger::Models::Worker.find_or_initialize_by(worker_id: @crono_trigger_worker_id)
|
124
124
|
worker_record.max_thread_size = @executor.max_length
|
125
125
|
worker_record.current_executing_size = @execution_counter.value
|
@@ -129,11 +129,11 @@ module CronoTrigger
|
|
129
129
|
worker_record.last_heartbeated_at = Time.current
|
130
130
|
@logger.info("[worker_id:#{@crono_trigger_worker_id}] Send heartbeat to database")
|
131
131
|
worker_record.save!
|
132
|
-
rescue => ex
|
133
|
-
CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
|
134
|
-
stop
|
135
132
|
end
|
136
133
|
end
|
134
|
+
rescue => ex
|
135
|
+
CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
|
136
|
+
stop
|
137
137
|
end
|
138
138
|
|
139
139
|
def executor_status
|
@@ -153,18 +153,22 @@ module CronoTrigger
|
|
153
153
|
|
154
154
|
def unregister
|
155
155
|
@logger.info("[worker_id:#{@crono_trigger_worker_id}] Unregister worker from database")
|
156
|
-
CronoTrigger
|
157
|
-
CronoTrigger::Models::Worker.
|
156
|
+
CronoTrigger.retry_on_db_errors do
|
157
|
+
CronoTrigger::Models::Worker.connection_pool.with_connection do
|
158
|
+
CronoTrigger::Models::Worker.find_by(worker_id: @crono_trigger_worker_id)&.destroy
|
159
|
+
end
|
158
160
|
end
|
159
161
|
rescue => ex
|
160
162
|
CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
|
161
163
|
end
|
162
164
|
|
163
165
|
def handle_signal_from_rdb
|
164
|
-
CronoTrigger
|
165
|
-
CronoTrigger::Models::Signal.
|
166
|
-
|
167
|
-
|
166
|
+
CronoTrigger.retry_on_db_errors do
|
167
|
+
CronoTrigger::Models::Signal.connection_pool.with_connection do
|
168
|
+
CronoTrigger::Models::Signal.sent_to_me.take(1)[0]&.tap do |s|
|
169
|
+
@logger.info("[worker_id:#{@crono_trigger_worker_id}] Receive Signal #{s.signal} from database")
|
170
|
+
s.kill_me(to_supervisor: s.signal != "TSTP")
|
171
|
+
end
|
168
172
|
end
|
169
173
|
end
|
170
174
|
rescue => ex
|
@@ -174,29 +178,31 @@ module CronoTrigger
|
|
174
178
|
def monitor
|
175
179
|
return unless ActiveSupport::Notifications.notifier.listening?(CronoTrigger::Events::MONITOR)
|
176
180
|
|
177
|
-
CronoTrigger
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
181
|
+
CronoTrigger.retry_on_db_errors do
|
182
|
+
CronoTrigger::Models::Worker.connection_pool.with_connection do
|
183
|
+
if workers_processing_same_models.order(:worker_id).limit(1).pluck(:worker_id).first != @crono_trigger_worker_id
|
184
|
+
# Return immediately to avoid redundant instruments
|
185
|
+
return
|
186
|
+
end
|
187
|
+
|
188
|
+
@model_names.each do |model_name|
|
189
|
+
model = model_name.classify.constantize
|
190
|
+
executable_count = model.executables.limit(nil).count
|
191
|
+
|
192
|
+
execute_lock_column = model.crono_trigger_column_name(:execute_lock)
|
193
|
+
oldest_execute_lock = model.executables(including_locked: true).where.not(execute_lock_column => 0).order(execute_lock_column).limit(1).pluck(execute_lock_column).first
|
194
|
+
|
195
|
+
next_execute_at_column = model.crono_trigger_column_name(:next_execute_at)
|
196
|
+
oldest_next_execute_at = model.executables.order(next_execute_at_column).limit(1).pluck(next_execute_at_column).first
|
197
|
+
|
198
|
+
now = Time.now
|
199
|
+
ActiveSupport::Notifications.instrument(CronoTrigger::Events::MONITOR, {
|
200
|
+
model_name: model_name,
|
201
|
+
executable_count: executable_count,
|
202
|
+
max_lock_duration_sec: oldest_execute_lock.nil? ? 0 : now.to_i - oldest_execute_lock,
|
203
|
+
max_latency_sec: oldest_next_execute_at.nil? ? 0 : now - oldest_next_execute_at,
|
204
|
+
})
|
205
|
+
end
|
200
206
|
end
|
201
207
|
end
|
202
208
|
rescue => ex
|
@@ -204,10 +210,12 @@ module CronoTrigger
|
|
204
210
|
end
|
205
211
|
|
206
212
|
def update_worker_count
|
207
|
-
CronoTrigger
|
208
|
-
|
209
|
-
|
210
|
-
|
213
|
+
CronoTrigger.retry_on_db_errors do
|
214
|
+
CronoTrigger::Models::Worker.connection_pool.with_connection do
|
215
|
+
worker_count = workers_processing_same_models.count
|
216
|
+
return if worker_count.zero?
|
217
|
+
@polling_threads.each { |th| th.worker_count = worker_count }
|
218
|
+
end
|
211
219
|
end
|
212
220
|
rescue => ex
|
213
221
|
CronoTrigger::GlobalExceptionHandler.handle_global_exception(ex)
|
data/lib/crono_trigger.rb
CHANGED
@@ -4,6 +4,7 @@ require "ostruct"
|
|
4
4
|
require "socket"
|
5
5
|
require "active_record"
|
6
6
|
require "concurrent"
|
7
|
+
require "retriable"
|
7
8
|
require "crono_trigger/events"
|
8
9
|
require "crono_trigger/models/worker"
|
9
10
|
require "crono_trigger/models/signal"
|
@@ -22,6 +23,11 @@ module CronoTrigger
|
|
22
23
|
model_names: nil,
|
23
24
|
error_handlers: [],
|
24
25
|
global_error_handlers: [],
|
26
|
+
db_error_retriable_options: {
|
27
|
+
on: {
|
28
|
+
ActiveRecord::ConnectionNotEstablished => nil,
|
29
|
+
},
|
30
|
+
}
|
25
31
|
)
|
26
32
|
|
27
33
|
def self.config
|
@@ -52,6 +58,12 @@ module CronoTrigger
|
|
52
58
|
def self.workers
|
53
59
|
CronoTrigger::Models::Worker.alive_workers
|
54
60
|
end
|
61
|
+
|
62
|
+
def self.retry_on_db_errors
|
63
|
+
Retriable.retriable(CronoTrigger.config.db_error_retriable_options) do
|
64
|
+
yield
|
65
|
+
end
|
66
|
+
end
|
55
67
|
end
|
56
68
|
|
57
69
|
if defined?(Rails)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crono_trigger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- joker1007
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-05-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: chrono
|
@@ -122,6 +122,20 @@ dependencies:
|
|
122
122
|
- - ">="
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '4.2'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: retriable
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
125
139
|
- !ruby/object:Gem::Dependency
|
126
140
|
name: sqlite3
|
127
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -221,7 +235,21 @@ dependencies:
|
|
221
235
|
- !ruby/object:Gem::Version
|
222
236
|
version: '0'
|
223
237
|
- !ruby/object:Gem::Dependency
|
224
|
-
name:
|
238
|
+
name: simplecov
|
239
|
+
requirement: !ruby/object:Gem::Requirement
|
240
|
+
requirements:
|
241
|
+
- - ">="
|
242
|
+
- !ruby/object:Gem::Version
|
243
|
+
version: '0'
|
244
|
+
type: :development
|
245
|
+
prerelease: false
|
246
|
+
version_requirements: !ruby/object:Gem::Requirement
|
247
|
+
requirements:
|
248
|
+
- - ">="
|
249
|
+
- !ruby/object:Gem::Version
|
250
|
+
version: '0'
|
251
|
+
- !ruby/object:Gem::Dependency
|
252
|
+
name: simplecov-cobertura
|
225
253
|
requirement: !ruby/object:Gem::Requirement
|
226
254
|
requirements:
|
227
255
|
- - ">="
|
@@ -340,7 +368,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
340
368
|
- !ruby/object:Gem::Version
|
341
369
|
version: '0'
|
342
370
|
requirements: []
|
343
|
-
rubygems_version: 3.5.
|
371
|
+
rubygems_version: 3.5.9
|
344
372
|
signing_key:
|
345
373
|
specification_version: 4
|
346
374
|
summary: In Service Asynchronous Job Scheduler for Rails
|