async-background 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/lib/async/background/job.rb +47 -18
- data/lib/async/background/queue/client.rb +18 -12
- data/lib/async/background/queue/store.rb +122 -51
- data/lib/async/background/runner.rb +51 -38
- data/lib/async/background/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a8f8593628f08094cc53b6c69eff34e3a4be9e8c12de5bd72f1447857aab2682
|
|
4
|
+
data.tar.gz: e9e441eb2b6f775cd52d2704c8a2b24d9ac771a24beb07d33f78d81285d41f1c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 194c82e280d24821e756849cc9f3f74db8cbadc424d4984fb3a58d10d21864cbe13b1adfa84aa3bf1bd6b9996d59e1d423d5c865c2d1ec33846679ed2ab936f0
|
|
7
|
+
data.tar.gz: 518bc2a692d0ade6ea02ff18ebca2a3078e02365ffd77fe9850d8bccc64b7243ea871efeec2913f33fe7ecdaeb8fa6a56067e57f36d5852adf59302cc656416e
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.1
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
- **Tunable `Store` options via `StoreOptions`** — three knobs exposed for SQLite tuning, validated at construction time so misconfigurations fail fast at boot:
|
|
7
|
+
- `mmap` (`true`/`false`, default `true`) — toggle memory-mapped I/O
|
|
8
|
+
- `synchronous` (`:normal`/`:full`/`:extra`, default `:normal`) — durability vs throughput
|
|
9
|
+
- `wal_autocheckpoint` (`Integer` in `100..10_000`, default `1_000`) — WAL checkpoint frequency in pages
|
|
10
|
+
|
|
11
|
+
Range and enum validation prevent foot-guns (e.g. `wal_autocheckpoint: 100_000` would bloat WAL beyond `journal_size_limit`). See [Get Started → Store tuning](docs/GET_STARTED.md) for trade-offs of each knob
|
|
12
|
+
|
|
13
|
+
### Breaking changes
|
|
14
|
+
- `Store.new(path:, mmap:)` → `Store.new(path:, options: { mmap: ... })`. Direct `mmap:` keyword argument removed in favor of the unified `options:` hash. Users who construct `Store` manually (e.g. for web-worker enqueue) need to update the call site
|
|
15
|
+
|
|
3
16
|
## 0.6.2
|
|
4
17
|
|
|
5
18
|
### Features
|
data/lib/async/background/job.rb
CHANGED
|
@@ -4,9 +4,48 @@ module Async
|
|
|
4
4
|
module Background
|
|
5
5
|
module Job
|
|
6
6
|
DEFAULT_TIMEOUT = 120
|
|
7
|
+
BACKOFFS = %i[fixed linear exponential].freeze
|
|
8
|
+
DEFAULT_JITTER_FOR = { fixed: 0.0, linear: 0.0, exponential: 0.5 }.freeze
|
|
7
9
|
|
|
8
|
-
Options = Data.define(:timeout) do
|
|
9
|
-
def initialize(timeout: DEFAULT_TIMEOUT
|
|
10
|
+
Options = Data.define(:timeout, :retry, :retry_delay, :backoff, :attempt, :jitter) do
|
|
11
|
+
def initialize(timeout: DEFAULT_TIMEOUT, retry: 0, retry_delay: nil, backoff: :fixed, attempt: nil, jitter: nil)
|
|
12
|
+
timeout = Integer(timeout)
|
|
13
|
+
retries = Integer(binding.local_variable_get(:retry) || 0)
|
|
14
|
+
retry_delay = Float(retry_delay) unless retry_delay.nil?
|
|
15
|
+
backoff = backoff.to_sym
|
|
16
|
+
attempt = Integer(attempt) unless attempt.nil?
|
|
17
|
+
jitter = Float(jitter) unless jitter.nil?
|
|
18
|
+
|
|
19
|
+
raise ArgumentError, "timeout must be > 0" unless timeout.positive?
|
|
20
|
+
raise ArgumentError, "retry must be >= 0" if retries.negative?
|
|
21
|
+
raise ArgumentError, "attempt must be >= 0" if attempt && attempt.negative?
|
|
22
|
+
raise ArgumentError, "backoff must be one of #{BACKOFFS.inspect}" unless BACKOFFS.include?(backoff)
|
|
23
|
+
if retries.positive?
|
|
24
|
+
raise ArgumentError, "retry_delay is required when retry > 0" if retry_delay.nil?
|
|
25
|
+
raise ArgumentError, "retry_delay must be > 0" unless retry_delay.positive?
|
|
26
|
+
end
|
|
27
|
+
raise ArgumentError, "jitter must be in [0, 1]" if jitter && !(0.0..1.0).cover?(jitter)
|
|
28
|
+
|
|
29
|
+
super(timeout:, retry: retries, retry_delay:, backoff:, attempt:, jitter:)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def retry? = self.retry.positive?
|
|
33
|
+
def attempts_made = attempt || 0
|
|
34
|
+
def next_attempt = attempts_made + 1
|
|
35
|
+
def with_attempt(n) = with(attempt: Integer(n))
|
|
36
|
+
def next_retry_delay(for_attempt = next_attempt, rng: Random)
|
|
37
|
+
raise ArgumentError, "attempt must be > 0" unless for_attempt.positive?
|
|
38
|
+
raise ArgumentError, "retry_delay must be configured" if retry_delay.nil?
|
|
39
|
+
|
|
40
|
+
base = case backoff
|
|
41
|
+
when :fixed then retry_delay
|
|
42
|
+
when :linear then retry_delay * for_attempt
|
|
43
|
+
when :exponential then retry_delay * (2**(for_attempt - 1))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
factor = jitter || DEFAULT_JITTER_FOR[backoff]
|
|
47
|
+
factor.zero? ? base : base * (1 + rng.rand * factor)
|
|
48
|
+
end
|
|
10
49
|
end
|
|
11
50
|
|
|
12
51
|
def self.included(base)
|
|
@@ -14,30 +53,20 @@ module Async
|
|
|
14
53
|
end
|
|
15
54
|
|
|
16
55
|
module ClassMethods
|
|
17
|
-
def perform_now(*args)
|
|
18
|
-
new.perform(*args)
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
def perform_async(*args, options: {})
|
|
22
|
-
Async::Background::Queue.enqueue(self, *args, options: options)
|
|
23
|
-
end
|
|
56
|
+
def perform_now(*args) = new.perform(*args)
|
|
24
57
|
|
|
25
|
-
def
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def perform_at(time, *args, options: {})
|
|
30
|
-
Async::Background::Queue.enqueue_at(time, self, *args, options: options)
|
|
31
|
-
end
|
|
58
|
+
def perform_async(*args, options: {}) = Async::Background::Queue.enqueue(self, *args, options: options)
|
|
59
|
+
def perform_in(delay, *args, options: {}) = Async::Background::Queue.enqueue_in(delay, self, *args, options: options)
|
|
60
|
+
def perform_at(time, *args, options: {}) = Async::Background::Queue.enqueue_at(time, self, *args, options: options)
|
|
32
61
|
|
|
33
62
|
def options(**values)
|
|
34
|
-
@options = Options.new(**values).to_h
|
|
63
|
+
@options = Options.new(**values).to_h.compact
|
|
35
64
|
end
|
|
36
65
|
|
|
37
66
|
def resolve_options = @options || {}
|
|
38
67
|
end
|
|
39
68
|
|
|
40
|
-
def perform(*
|
|
69
|
+
def perform(*)
|
|
41
70
|
raise NotImplementedError, "#{self.class} must implement #perform"
|
|
42
71
|
end
|
|
43
72
|
end
|
|
@@ -22,8 +22,7 @@ module Async
|
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
def push_in(delay, class_name, args = [], options: {})
|
|
25
|
-
|
|
26
|
-
push(class_name, args, run_at, options: options)
|
|
25
|
+
push(class_name, args, realtime_now + delay.to_f, options: options)
|
|
27
26
|
end
|
|
28
27
|
|
|
29
28
|
def push_at(time, class_name, args = [], options: {})
|
|
@@ -37,29 +36,36 @@ module Async
|
|
|
37
36
|
|
|
38
37
|
def enqueue(job_class, *args, options: {})
|
|
39
38
|
ensure_configured!
|
|
40
|
-
|
|
41
|
-
default_client.push(resolve_class_name(job_class), args, nil, options: merged)
|
|
39
|
+
default_client.push(resolve_class_name(job_class), args, nil, options: build_options(job_class, options))
|
|
42
40
|
end
|
|
43
41
|
|
|
44
42
|
def enqueue_in(delay, job_class, *args, options: {})
|
|
45
43
|
ensure_configured!
|
|
46
|
-
|
|
47
|
-
default_client.push_in(delay, resolve_class_name(job_class), args, options: merged)
|
|
44
|
+
default_client.push_in(delay, resolve_class_name(job_class), args, options: build_options(job_class, options))
|
|
48
45
|
end
|
|
49
46
|
|
|
50
47
|
def enqueue_at(time, job_class, *args, options: {})
|
|
51
48
|
ensure_configured!
|
|
52
|
-
|
|
53
|
-
default_client.push_at(time, resolve_class_name(job_class), args, options: merged)
|
|
49
|
+
default_client.push_at(time, resolve_class_name(job_class), args, options: build_options(job_class, options))
|
|
54
50
|
end
|
|
55
51
|
|
|
56
52
|
private
|
|
57
53
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
54
|
+
RETRY_KEYS = %i[retry retry_delay backoff jitter].freeze
|
|
55
|
+
private_constant :RETRY_KEYS
|
|
56
|
+
|
|
57
|
+
def build_options(job_class, call_site)
|
|
58
|
+
call_site ||= {}
|
|
59
|
+
merged = resolve_options(job_class).merge(call_site.compact)
|
|
60
|
+
apply_retry_overrides!(merged, call_site)
|
|
61
|
+
|
|
62
|
+
merged.empty? ? EMPTY_OPTIONS : Job::Options.new(**merged).to_h.compact
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def apply_retry_overrides!(merged, call_site)
|
|
66
|
+
return unless RETRY_KEYS.any? { |k| call_site.key?(k) && !call_site[k].nil? }
|
|
61
67
|
|
|
62
|
-
|
|
68
|
+
RETRY_KEYS.each { |k| merged[k] = call_site[k] if call_site.key?(k) }
|
|
63
69
|
end
|
|
64
70
|
|
|
65
71
|
def ensure_configured!
|
|
@@ -6,6 +6,49 @@ require_relative '../clock'
|
|
|
6
6
|
module Async
|
|
7
7
|
module Background
|
|
8
8
|
module Queue
|
|
9
|
+
SYNCHRONOUS_LEVELS = { normal: 'NORMAL', full: 'FULL', extra: 'EXTRA' }.freeze
|
|
10
|
+
WAL_AUTOCHECKPOINT_RANGE = 100..10_000
|
|
11
|
+
DEFAULTS = { mmap: true, synchronous: :normal, wal_autocheckpoint: 1_000 }.freeze
|
|
12
|
+
MMAP_SIZE = 268_435_456
|
|
13
|
+
|
|
14
|
+
StoreOptions = Data.define(:mmap, :synchronous, :wal_autocheckpoint) do
|
|
15
|
+
def self.build(value = {}) = value.is_a?(self) ? value : new(**DEFAULTS, **value)
|
|
16
|
+
|
|
17
|
+
def initialize(mmap:, synchronous:, wal_autocheckpoint:)
|
|
18
|
+
unless mmap == true || mmap == false
|
|
19
|
+
raise ArgumentError, "mmap must be true or false, got #{mmap.inspect}"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
unless SYNCHRONOUS_LEVELS.key?(synchronous)
|
|
23
|
+
raise ArgumentError,
|
|
24
|
+
"synchronous must be one of #{SYNCHRONOUS_LEVELS.keys.inspect}, got #{synchronous.inspect}"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
unless wal_autocheckpoint.is_a?(Integer) && WAL_AUTOCHECKPOINT_RANGE.cover?(wal_autocheckpoint)
|
|
28
|
+
raise ArgumentError,
|
|
29
|
+
"wal_autocheckpoint must be an Integer in #{WAL_AUTOCHECKPOINT_RANGE}, " \
|
|
30
|
+
"got #{wal_autocheckpoint.inspect}"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
super
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def synchronous_pragma = SYNCHRONOUS_LEVELS.fetch(synchronous)
|
|
37
|
+
def mmap_size = mmap ? MMAP_SIZE : 0
|
|
38
|
+
|
|
39
|
+
def pragma_sql
|
|
40
|
+
<<~SQL
|
|
41
|
+
PRAGMA journal_mode = WAL;
|
|
42
|
+
PRAGMA synchronous = #{synchronous_pragma};
|
|
43
|
+
PRAGMA mmap_size = #{mmap_size};
|
|
44
|
+
PRAGMA cache_size = -16000;
|
|
45
|
+
PRAGMA temp_store = MEMORY;
|
|
46
|
+
PRAGMA journal_size_limit = 67108864;
|
|
47
|
+
PRAGMA wal_autocheckpoint = #{wal_autocheckpoint};
|
|
48
|
+
SQL
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
9
52
|
class Store
|
|
10
53
|
include Clock
|
|
11
54
|
|
|
@@ -25,29 +68,16 @@ module Async
|
|
|
25
68
|
CREATE INDEX IF NOT EXISTS idx_jobs_pending ON jobs(run_at, id) WHERE status = 'pending';
|
|
26
69
|
SQL
|
|
27
70
|
|
|
28
|
-
MMAP_SIZE = 268_435_456
|
|
29
|
-
PRAGMAS = ->(mmap_size) {
|
|
30
|
-
<<~SQL
|
|
31
|
-
PRAGMA journal_mode = WAL;
|
|
32
|
-
PRAGMA synchronous = NORMAL;
|
|
33
|
-
PRAGMA mmap_size = #{mmap_size};
|
|
34
|
-
PRAGMA cache_size = -16000;
|
|
35
|
-
PRAGMA temp_store = MEMORY;
|
|
36
|
-
PRAGMA busy_timeout = 5000;
|
|
37
|
-
PRAGMA journal_size_limit = 67108864;
|
|
38
|
-
SQL
|
|
39
|
-
}.freeze
|
|
40
|
-
|
|
41
71
|
CLEANUP_INTERVAL = 300
|
|
42
72
|
CLEANUP_AGE = 3600
|
|
43
73
|
|
|
44
|
-
attr_reader :path
|
|
74
|
+
attr_reader :path, :options
|
|
45
75
|
|
|
46
|
-
def initialize(path: self.class.default_path,
|
|
47
|
-
@path
|
|
48
|
-
@
|
|
49
|
-
@pragma_sql
|
|
50
|
-
@db
|
|
76
|
+
def initialize(path: self.class.default_path, options: {})
|
|
77
|
+
@path = path
|
|
78
|
+
@options = StoreOptions.build(options)
|
|
79
|
+
@pragma_sql = @options.pragma_sql.freeze
|
|
80
|
+
@db = nil
|
|
51
81
|
@schema_checked = false
|
|
52
82
|
@last_cleanup_at = nil
|
|
53
83
|
end
|
|
@@ -55,8 +85,7 @@ module Async
|
|
|
55
85
|
def ensure_database!
|
|
56
86
|
require_sqlite3
|
|
57
87
|
db = SQLite3::Database.new(@path)
|
|
58
|
-
db
|
|
59
|
-
db.execute_batch(@pragma_sql)
|
|
88
|
+
configure_database(db)
|
|
60
89
|
db.execute_batch(SCHEMA)
|
|
61
90
|
db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
62
91
|
db.close
|
|
@@ -65,34 +94,22 @@ module Async
|
|
|
65
94
|
|
|
66
95
|
def enqueue(class_name, args = [], run_at = nil, options: {})
|
|
67
96
|
ensure_connection
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
@enqueue_stmt.execute(class_name, JSON.generate(args), options_json, realtime_now, run_at)
|
|
97
|
+
now = realtime_now
|
|
98
|
+
@enqueue_stmt.execute(class_name, JSON.generate(args), dump_options(options), now, run_at || now)
|
|
71
99
|
@db.last_insert_row_id
|
|
72
100
|
end
|
|
73
101
|
|
|
74
102
|
def fetch(worker_id)
|
|
75
103
|
ensure_connection
|
|
76
104
|
now = realtime_now
|
|
77
|
-
@db.execute("BEGIN IMMEDIATE")
|
|
78
|
-
|
|
79
|
-
begin
|
|
80
|
-
results = @fetch_stmt.execute(worker_id, now, now)
|
|
81
|
-
row = results.first
|
|
82
|
-
ensure
|
|
83
|
-
@fetch_stmt.reset! rescue nil
|
|
84
|
-
end
|
|
85
105
|
|
|
86
|
-
@
|
|
106
|
+
row = transaction { with_stmt(@fetch_stmt) { |s| s.execute(worker_id, now, now).first } }
|
|
87
107
|
return unless row
|
|
88
108
|
|
|
89
109
|
maybe_cleanup
|
|
90
|
-
|
|
91
|
-
{ id: row[0], class_name: row[1], args: JSON.parse(row[2]), options: options }
|
|
92
|
-
rescue
|
|
93
|
-
@db.execute("ROLLBACK") rescue nil
|
|
94
|
-
raise
|
|
110
|
+
{ id: row[0], class_name: row[1], args: JSON.parse(row[2]), options: load_options(row[3]) }
|
|
95
111
|
end
|
|
112
|
+
|
|
96
113
|
def complete(job_id)
|
|
97
114
|
ensure_connection
|
|
98
115
|
@complete_stmt.execute(job_id)
|
|
@@ -103,6 +120,24 @@ module Async
|
|
|
103
120
|
@fail_stmt.execute(job_id)
|
|
104
121
|
end
|
|
105
122
|
|
|
123
|
+
def retry_or_fail(job_id, fallback_options: nil)
|
|
124
|
+
ensure_connection
|
|
125
|
+
|
|
126
|
+
transaction do
|
|
127
|
+
stored = with_stmt(@retry_state_stmt) { |s| load_options(s.execute(job_id).first&.first) }
|
|
128
|
+
policy = stored.empty? ? normalize_options(fallback_options) : Job::Options.new(**stored)
|
|
129
|
+
|
|
130
|
+
if policy&.retry? && policy.next_attempt <= policy.retry
|
|
131
|
+
advanced = policy.with_attempt(policy.next_attempt)
|
|
132
|
+
@retry_stmt.execute(realtime_now + advanced.next_retry_delay(advanced.attempt), dump_options(advanced.to_h.compact), job_id)
|
|
133
|
+
:retried
|
|
134
|
+
else
|
|
135
|
+
@fail_stmt.execute(job_id)
|
|
136
|
+
:failed
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
|
106
141
|
def recover(worker_id)
|
|
107
142
|
ensure_connection
|
|
108
143
|
@requeue_stmt.execute(worker_id)
|
|
@@ -138,8 +173,7 @@ module Async
|
|
|
138
173
|
require_sqlite3
|
|
139
174
|
finalize_statements
|
|
140
175
|
@db = SQLite3::Database.new(@path)
|
|
141
|
-
@db
|
|
142
|
-
@db.execute_batch(@pragma_sql)
|
|
176
|
+
configure_database(@db)
|
|
143
177
|
|
|
144
178
|
unless @schema_checked
|
|
145
179
|
@db.execute_batch(SCHEMA)
|
|
@@ -151,6 +185,42 @@ module Async
|
|
|
151
185
|
@last_cleanup_at = monotonic_now
|
|
152
186
|
end
|
|
153
187
|
|
|
188
|
+
def configure_database(db)
|
|
189
|
+
db.execute("PRAGMA busy_timeout = 5000")
|
|
190
|
+
db.execute_batch(@pragma_sql)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def transaction
|
|
194
|
+
@db.execute("BEGIN IMMEDIATE")
|
|
195
|
+
result = yield
|
|
196
|
+
@db.execute("COMMIT")
|
|
197
|
+
result
|
|
198
|
+
rescue
|
|
199
|
+
@db.execute("ROLLBACK") rescue nil
|
|
200
|
+
raise
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def with_stmt(stmt)
|
|
204
|
+
yield stmt
|
|
205
|
+
ensure
|
|
206
|
+
stmt.reset! rescue nil
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def dump_options(options)
|
|
210
|
+
options.empty? ? nil : JSON.generate(options)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def load_options(json)
|
|
214
|
+
json ? JSON.parse(json, symbolize_names: true) : {}
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def normalize_options(options)
|
|
218
|
+
return if options.nil?
|
|
219
|
+
return options if options.is_a?(Job::Options)
|
|
220
|
+
|
|
221
|
+
Job::Options.new(**options)
|
|
222
|
+
end
|
|
223
|
+
|
|
154
224
|
def prepare_statements
|
|
155
225
|
@enqueue_stmt = @db.prepare(
|
|
156
226
|
"INSERT INTO jobs (class_name, args, options, created_at, run_at) VALUES (?, ?, ?, ?, ?)"
|
|
@@ -168,23 +238,27 @@ module Async
|
|
|
168
238
|
RETURNING id, class_name, args, options
|
|
169
239
|
SQL
|
|
170
240
|
|
|
171
|
-
@complete_stmt
|
|
172
|
-
@fail_stmt
|
|
173
|
-
|
|
241
|
+
@complete_stmt = @db.prepare("UPDATE jobs SET status = 'done', locked_by = NULL, locked_at = NULL WHERE id = ?")
|
|
242
|
+
@fail_stmt = @db.prepare("UPDATE jobs SET status = 'failed', locked_by = NULL, locked_at = NULL WHERE id = ?")
|
|
243
|
+
@retry_state_stmt = @db.prepare("SELECT options FROM jobs WHERE id = ?")
|
|
244
|
+
@retry_stmt = @db.prepare(
|
|
245
|
+
"UPDATE jobs SET status = 'pending', locked_by = NULL, locked_at = NULL, run_at = ?, options = ? WHERE id = ?"
|
|
246
|
+
)
|
|
174
247
|
@requeue_stmt = @db.prepare(
|
|
175
248
|
"UPDATE jobs SET status = 'pending', locked_by = NULL, locked_at = NULL " \
|
|
176
249
|
"WHERE status = 'running' AND locked_by = ?"
|
|
177
250
|
)
|
|
178
|
-
|
|
179
251
|
@cleanup_stmt = @db.prepare("DELETE FROM jobs WHERE status = 'done' AND created_at < ?")
|
|
180
252
|
end
|
|
181
253
|
|
|
182
254
|
def finalize_statements
|
|
183
|
-
[@enqueue_stmt, @fetch_stmt, @complete_stmt, @fail_stmt,
|
|
184
|
-
|
|
255
|
+
[@enqueue_stmt, @fetch_stmt, @complete_stmt, @fail_stmt,
|
|
256
|
+
@retry_state_stmt, @retry_stmt, @requeue_stmt, @cleanup_stmt].each do |stmt|
|
|
257
|
+
stmt&.close rescue next
|
|
185
258
|
end
|
|
186
259
|
|
|
187
|
-
@enqueue_stmt = @fetch_stmt = @complete_stmt = @fail_stmt =
|
|
260
|
+
@enqueue_stmt = @fetch_stmt = @complete_stmt = @fail_stmt = nil
|
|
261
|
+
@retry_state_stmt = @retry_stmt = @requeue_stmt = @cleanup_stmt = nil
|
|
188
262
|
end
|
|
189
263
|
|
|
190
264
|
def maybe_cleanup
|
|
@@ -193,10 +267,7 @@ module Async
|
|
|
193
267
|
|
|
194
268
|
@last_cleanup_at = now
|
|
195
269
|
@cleanup_stmt.execute(realtime_now - CLEANUP_AGE)
|
|
196
|
-
|
|
197
|
-
if @db.changes > 100
|
|
198
|
-
@db.execute("PRAGMA incremental_vacuum")
|
|
199
|
-
end
|
|
270
|
+
@db.execute("PRAGMA incremental_vacuum") if @db.changes > 100
|
|
200
271
|
end
|
|
201
272
|
end
|
|
202
273
|
end
|
|
@@ -69,18 +69,17 @@ module Async
|
|
|
69
69
|
@listen_queue = false
|
|
70
70
|
return unless queue_socket_dir
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
isolated = ENV.fetch("ISOLATION_FORKS", "").split(",").map(&:to_i)
|
|
73
|
+
return if isolated.include?(worker_index)
|
|
74
|
+
|
|
73
75
|
require_relative 'queue/store'
|
|
74
76
|
require_relative 'queue/socket_waker'
|
|
75
77
|
require_relative 'queue/client'
|
|
76
78
|
|
|
77
|
-
isolated = ENV.fetch("ISOLATION_FORKS", "").split(",").map(&:to_i)
|
|
78
|
-
return if isolated.include?(worker_index)
|
|
79
|
-
|
|
80
79
|
@listen_queue = true
|
|
81
80
|
@queue_store = Queue::Store.new(
|
|
82
|
-
path:
|
|
83
|
-
mmap: queue_mmap
|
|
81
|
+
path: queue_db_path || Queue::Store.default_path,
|
|
82
|
+
options: { mmap: queue_mmap }
|
|
84
83
|
)
|
|
85
84
|
|
|
86
85
|
socket_path = File.join(queue_socket_dir, "async_bg_worker_#{worker_index}.sock")
|
|
@@ -112,40 +111,55 @@ module Async
|
|
|
112
111
|
|
|
113
112
|
def run_queue_job(job_task, job)
|
|
114
113
|
class_name = job[:class_name]
|
|
115
|
-
args = job[:args]
|
|
116
114
|
klass = resolve_job_class(class_name)
|
|
117
|
-
|
|
118
|
-
timeout = opts.timeout
|
|
115
|
+
options = parse_job_options(job[:options])
|
|
119
116
|
|
|
120
117
|
metrics.job_started(nil)
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
118
|
+
started = monotonic_now
|
|
119
|
+
job_task.with_timeout(options.timeout) { klass.perform_now(*job[:args]) }
|
|
120
|
+
duration = monotonic_now - started
|
|
124
121
|
|
|
125
|
-
duration = monotonic_now - t
|
|
126
122
|
metrics.job_finished(nil, duration)
|
|
127
123
|
@queue_store.complete(job[:id])
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
124
|
+
logger.info('Async::Background') { "queue(#{class_name}): completed in #{duration.round(2)}s" }
|
|
125
|
+
rescue ConfigError => e
|
|
126
|
+
metrics.job_failed(nil, e) if options
|
|
127
|
+
@queue_store.fail(job[:id])
|
|
128
|
+
logger.error('Async::Background') { "queue(#{class_name}): #{e.class} #{e.message}" }
|
|
132
129
|
rescue ::Async::TimeoutError
|
|
133
130
|
metrics.job_timed_out(nil)
|
|
134
|
-
|
|
135
|
-
logger.error('Async::Background') { "queue(#{class_name}): timed out after #{timeout}s" }
|
|
131
|
+
handle_queue_failure(job, options, "timed out after #{options.timeout}s", backtrace: nil)
|
|
136
132
|
rescue => e
|
|
137
133
|
metrics.job_failed(nil, e)
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
134
|
+
handle_queue_failure(job, options, "#{e.class} #{e.message}", backtrace: e.backtrace)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def parse_job_options(raw)
|
|
138
|
+
Job::Options.new(**(raw || {}))
|
|
139
|
+
rescue ArgumentError, TypeError => e
|
|
140
|
+
raise ConfigError, "invalid queue options: #{e.message}"
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def handle_queue_failure(job, options, message, backtrace:)
|
|
144
|
+
result = @queue_store.retry_or_fail(job[:id], fallback_options: options)
|
|
145
|
+
class_name = job[:class_name]
|
|
146
|
+
|
|
147
|
+
if result == :retried
|
|
148
|
+
@queue_waker&.signal
|
|
149
|
+
attempt = options.next_attempt
|
|
150
|
+
logger.warn('Async::Background') do
|
|
151
|
+
"queue(#{class_name}): #{message}; retry #{attempt}/#{options.retry}"
|
|
152
|
+
end
|
|
153
|
+
else
|
|
154
|
+
tail = backtrace ? "\n#{backtrace.join("\n")}" : ''
|
|
155
|
+
logger.error('Async::Background') { "queue(#{class_name}): #{message}#{tail}" }
|
|
156
|
+
end
|
|
142
157
|
end
|
|
143
158
|
|
|
144
159
|
def resolve_job_class(class_name)
|
|
145
|
-
raise ConfigError, "empty class name in queue job" if class_name.nil? || class_name.strip.empty?
|
|
160
|
+
raise ConfigError, "empty class name in queue job" if class_name.nil? || class_name.to_s.strip.empty?
|
|
146
161
|
|
|
147
|
-
|
|
148
|
-
klass = names.reduce(Object) do |mod, name|
|
|
162
|
+
klass = class_name.split("::").reduce(Object) do |mod, name|
|
|
149
163
|
raise ConfigError, "unknown class: #{class_name}" unless mod.const_defined?(name, false)
|
|
150
164
|
mod.const_get(name, false)
|
|
151
165
|
end
|
|
@@ -259,14 +273,12 @@ module Async
|
|
|
259
273
|
class_name = config&.dig('class').to_s.strip
|
|
260
274
|
raise ConfigError, "[#{name}] missing class" if class_name.empty?
|
|
261
275
|
|
|
262
|
-
begin
|
|
263
|
-
|
|
264
|
-
rescue
|
|
265
|
-
raise ConfigError, "[#{name}]
|
|
276
|
+
job_class = begin
|
|
277
|
+
resolve_job_class(class_name)
|
|
278
|
+
rescue ConfigError => e
|
|
279
|
+
raise ConfigError, "[#{name}] #{e.message}"
|
|
266
280
|
end
|
|
267
281
|
|
|
268
|
-
raise ConfigError, "[#{name}] #{class_name} must include Async::Background::Job" unless job_class.respond_to?(:perform_now)
|
|
269
|
-
|
|
270
282
|
interval = config['every']&.then { |v|
|
|
271
283
|
int = v.to_i
|
|
272
284
|
raise ConfigError, "[#{name}] 'every' must be > 0" unless int.positive?
|
|
@@ -279,12 +291,13 @@ module Async
|
|
|
279
291
|
|
|
280
292
|
raise ConfigError, "[#{name}] specify 'every' or 'cron'" unless interval || cron
|
|
281
293
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
294
|
+
timeout = begin
|
|
295
|
+
Job::Options.new(timeout: config.fetch('timeout', DEFAULT_TIMEOUT)).timeout
|
|
296
|
+
rescue ArgumentError, TypeError => e
|
|
297
|
+
raise ConfigError, "[#{name}] #{e.message}"
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
{ job_class: job_class, interval: interval, cron: cron, timeout: timeout }
|
|
288
301
|
end
|
|
289
302
|
|
|
290
303
|
def run_job(job_task, entry)
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: async-background
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.7.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Roman Hajdarov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-05-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: async
|