async-background 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/README.md +48 -7
- data/async-background.gemspec +2 -1
- data/lib/async/background/job.rb +5 -3
- data/lib/async/background/metrics.rb +157 -86
- data/lib/async/background/queue/client.rb +33 -15
- data/lib/async/background/queue/options.rb +70 -0
- data/lib/async/background/queue/schema.rb +160 -0
- data/lib/async/background/queue/sql.rb +205 -0
- data/lib/async/background/queue/store.rb +275 -122
- data/lib/async/background/runner/queue_execution.rb +199 -0
- data/lib/async/background/runner/schedule.rb +127 -0
- data/lib/async/background/runner.rb +99 -231
- data/lib/async/background/version.rb +1 -1
- metadata +31 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '095ad7028f5492c5f116a86adbbacaabe39167586405f696910e6ff799900831'
|
|
4
|
+
data.tar.gz: 2eb448ca15863b2ae6177a8ab2f3c4d348b9ecc9453f5603ec50363879c9f50d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9533784a209703347f448b7b5a497c92415a339f4a0c3d2f808f569068f5db64a833d2ae7098b3c64d4dbab497d68da22ca3cd2381d746d552fe1bd98f2f555b
|
|
7
|
+
data.tar.gz: 4091871c888e1b80832cd764c6c44bfb6e6ca1335f21456855e303d0469ca0127ab1e463d4e23318aabc5e171010d6b3b55764c88f7fa179e650b3d934bf8b7d
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.2
|
|
4
|
+
|
|
5
|
+
- Harden queue execution, retries, shutdown, and metrics.
|
|
6
|
+
- Add schema v1, optional dashboard indexes, and a faster enqueue path.
|
|
7
|
+
|
|
8
|
+
## 0.7.1
|
|
9
|
+
|
|
10
|
+
### Features
|
|
11
|
+
- **Tunable `Store` options via `StoreOptions`** — three knobs exposed for SQLite tuning, validated at construction time so misconfigurations fail fast at boot:
|
|
12
|
+
- `mmap` (`true`/`false`, default `true`) — toggle memory-mapped I/O
|
|
13
|
+
- `synchronous` (`:normal`/`:full`/`:extra`, default `:normal`) — durability vs throughput
|
|
14
|
+
- `wal_autocheckpoint` (`Integer` in `100..10_000`, default `1_000`) — WAL checkpoint frequency in pages
|
|
15
|
+
|
|
16
|
+
Range and enum validation prevent foot-guns (e.g. `wal_autocheckpoint: 100_000` would bloat WAL beyond `journal_size_limit`). See [Get Started → Store tuning](docs/GET_STARTED.md) for trade-offs of each knob
|
|
17
|
+
|
|
18
|
+
### Breaking changes
|
|
19
|
+
- `Store.new(path:, mmap:)` → `Store.new(path:, options: { mmap: ... })`. Direct `mmap:` keyword argument removed in favor of the unified `options:` hash. Users who construct `Store` manually (e.g. for web-worker enqueue) need to update the call site
|
|
20
|
+
|
|
3
21
|
## 0.6.2
|
|
4
22
|
|
|
5
23
|
### Features
|
data/README.md
CHANGED
|
@@ -13,7 +13,7 @@ A lightweight cron, interval, and job-queue scheduler for Ruby's [Async](https:/
|
|
|
13
13
|
- Ruby >= 3.3
|
|
14
14
|
- `async ~> 2.0`, `fugit ~> 1.0`
|
|
15
15
|
- `sqlite3 ~> 2.0` (optional, for the job queue)
|
|
16
|
-
- `async-utilization
|
|
16
|
+
- `async-utilization >= 0.3, < 0.5` (optional, for metrics)
|
|
17
17
|
|
|
18
18
|
## Install
|
|
19
19
|
|
|
@@ -21,7 +21,7 @@ A lightweight cron, interval, and job-queue scheduler for Ruby's [Async](https:/
|
|
|
21
21
|
# Gemfile
|
|
22
22
|
gem "async-background"
|
|
23
23
|
gem "sqlite3", "~> 2.0" # optional
|
|
24
|
-
gem "async-utilization", "
|
|
24
|
+
gem "async-utilization", ">= 0.3", "< 0.5" # optional
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
## ➡️ [Get Started](docs/GET_STARTED.md)
|
|
@@ -92,9 +92,8 @@ Without this, you will get database crashes in multi-process mode. See [Get Star
|
|
|
92
92
|
**Don't share SQLite connections across `fork()`.** The gem opens connections lazily after fork, but if you create a `Queue::Store` manually for schema setup, close it before forking:
|
|
93
93
|
|
|
94
94
|
```ruby
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
store.close # ← before fork
|
|
95
|
+
Async::Background::Queue.migrate!(path: db_path) # ← once, before fork
|
|
96
|
+
# Every process opens its own Store lazily after fork.
|
|
98
97
|
```
|
|
99
98
|
|
|
100
99
|
**Two clocks, on purpose.** Interval jobs use `CLOCK_MONOTONIC` (immune to NTP drift). Cron jobs use wall-clock time, because "every day at 3am" needs to mean 3am.
|
|
@@ -127,19 +126,61 @@ The dynamic queue runs alongside it:
|
|
|
127
126
|
|
|
128
127
|
Jobs are persisted in SQLite, so a missed wake-up is never a lost job — workers also poll every 5 seconds as a safety net.
|
|
129
128
|
|
|
129
|
+
### Schema migration during deploy
|
|
130
|
+
|
|
131
|
+
Run queue migrations once in the release/pre-deploy step, before starting new web or worker
|
|
132
|
+
processes. This serializes the schema upgrade with `BEGIN IMMEDIATE`, records the version in
|
|
133
|
+
SQLite, and avoids a first producer doing DDL under live queue traffic:
|
|
134
|
+
|
|
135
|
+
```ruby
|
|
136
|
+
Async::Background::Queue.migrate!(path: ENV.fetch("QUEUE_DB_PATH"))
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
A fresh database still self-initializes on first use for local development, but explicit
|
|
140
|
+
migration is the production path. For an existing queue, finish or stop 0.7.1 producers/workers,
|
|
141
|
+
run the migration once, then start 0.7.2 processes.
|
|
142
|
+
|
|
143
|
+
### Future dashboard indexes
|
|
144
|
+
|
|
145
|
+
The queue does **not** install dashboard indexes by default. They slow every enqueue even though
|
|
146
|
+
pending rows never enter terminal or in-flight read-model indexes. When the 1.0 dashboard module
|
|
147
|
+
is enabled, its installer will run this once in the same release step:
|
|
148
|
+
|
|
149
|
+
```ruby
|
|
150
|
+
Async::Background::Queue.prepare_dashboard!(path: ENV.fetch("QUEUE_DB_PATH"))
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
It adds three compact indexes: one each for cursor-sorted done and failed jobs, plus one for
|
|
154
|
+
the bounded in-flight list. It does not change queue behavior or rerun the core migration.
|
|
155
|
+
|
|
130
156
|
## Metrics
|
|
131
157
|
|
|
132
|
-
|
|
158
|
+
Metrics are an optional integration with `async-utilization` (`>= 0.3`, `< 0.5`). The
|
|
159
|
+
background worker remains fully functional when that gem is absent. With it installed, each
|
|
160
|
+
worker publishes counters to a shared-memory segment.
|
|
133
161
|
|
|
134
162
|
```ruby
|
|
163
|
+
runner.metrics.enabled?
|
|
135
164
|
runner.metrics.values
|
|
136
165
|
# => { total_runs: 142, total_successes: 140, total_failures: 2,
|
|
137
166
|
# total_timeouts: 0, total_skips: 5, active_jobs: 1, ... }
|
|
138
167
|
|
|
139
168
|
Async::Background::Metrics.read_all(total_workers: 2)
|
|
169
|
+
# => [{ worker: 1, ... }, { worker: 2, ... }]
|
|
140
170
|
```
|
|
141
171
|
|
|
142
|
-
|
|
172
|
+
`Metrics.read_all` returns `[]` until the optional gem is installed and a worker has created
|
|
173
|
+
the file, so an observer can render an unavailable state without rescuing `LoadError`. Its
|
|
174
|
+
snapshot is lock-free best effort: cumulative fields (`total_runs`, `total_successes`,
|
|
175
|
+
`total_failures`, `total_timeouts`, `total_skips`) are counters; `active_jobs`,
|
|
176
|
+
`last_run_at`, and `last_duration_ms` are gauges. Fields can describe adjacent moments in time
|
|
177
|
+
rather than one globally atomic instant.
|
|
178
|
+
|
|
179
|
+
By default the file is `/tmp/async-background.shm`. Set `ASYNC_BACKGROUND_METRICS_PATH`
|
|
180
|
+
or pass `metrics_shm_path:` to `Runner.new` when another observer runs in a separate process
|
|
181
|
+
or container; both sides must see the same mounted file.
|
|
182
|
+
|
|
183
|
+
|
|
143
184
|
|
|
144
185
|
## License
|
|
145
186
|
|
data/async-background.gemspec
CHANGED
|
@@ -36,8 +36,9 @@ Gem::Specification.new do |spec|
|
|
|
36
36
|
|
|
37
37
|
# Optional: add to your own Gemfile if you need these features
|
|
38
38
|
# gem 'sqlite3', '~> 2.0' # dynamic job queue
|
|
39
|
-
# gem 'async-utilization', '
|
|
39
|
+
# gem 'async-utilization', '>= 0.3', '< 0.5' # shared-memory worker metrics
|
|
40
40
|
|
|
41
41
|
spec.add_development_dependency 'rake', '~> 13.0'
|
|
42
42
|
spec.add_development_dependency 'rspec', '~> 3.12'
|
|
43
|
+
spec.add_development_dependency 'async-utilization', '>= 0.3', '< 0.5'
|
|
43
44
|
end
|
data/lib/async/background/job.rb
CHANGED
|
@@ -4,6 +4,7 @@ module Async
|
|
|
4
4
|
module Background
|
|
5
5
|
module Job
|
|
6
6
|
DEFAULT_TIMEOUT = 120
|
|
7
|
+
EMPTY_OPTIONS = {}.freeze
|
|
7
8
|
BACKOFFS = %i[fixed linear exponential].freeze
|
|
8
9
|
DEFAULT_JITTER_FOR = { fixed: 0.0, linear: 0.0, exponential: 0.5 }.freeze
|
|
9
10
|
|
|
@@ -55,15 +56,16 @@ module Async
|
|
|
55
56
|
module ClassMethods
|
|
56
57
|
def perform_now(*args) = new.perform(*args)
|
|
57
58
|
|
|
58
|
-
def perform_async(*args, options:
|
|
59
|
-
def perform_in(delay, *args, options:
|
|
60
|
-
def perform_at(time, *args, options:
|
|
59
|
+
def perform_async(*args, options: EMPTY_OPTIONS) = Async::Background::Queue.enqueue(self, *args, options: options)
|
|
60
|
+
def perform_in(delay, *args, options: EMPTY_OPTIONS) = Async::Background::Queue.enqueue_in(delay, self, *args, options: options)
|
|
61
|
+
def perform_at(time, *args, options: EMPTY_OPTIONS) = Async::Background::Queue.enqueue_at(time, self, *args, options: options)
|
|
61
62
|
|
|
62
63
|
def options(**values)
|
|
63
64
|
@options = Options.new(**values).to_h.compact
|
|
64
65
|
end
|
|
65
66
|
|
|
66
67
|
def resolve_options = @options || {}
|
|
68
|
+
def queue_options = @options || EMPTY_OPTIONS
|
|
67
69
|
end
|
|
68
70
|
|
|
69
71
|
def perform(*)
|
|
@@ -1,143 +1,214 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'tmpdir'
|
|
4
|
+
|
|
3
5
|
module Async
|
|
4
6
|
module Background
|
|
5
7
|
class Metrics
|
|
6
8
|
SCHEMA_FIELDS = {
|
|
7
|
-
total_runs:
|
|
8
|
-
total_successes:
|
|
9
|
-
total_failures:
|
|
10
|
-
total_timeouts:
|
|
11
|
-
total_skips:
|
|
12
|
-
active_jobs:
|
|
13
|
-
last_run_at:
|
|
9
|
+
total_runs: :u64,
|
|
10
|
+
total_successes: :u64,
|
|
11
|
+
total_failures: :u64,
|
|
12
|
+
total_timeouts: :u64,
|
|
13
|
+
total_skips: :u64,
|
|
14
|
+
active_jobs: :u32,
|
|
15
|
+
last_run_at: :u64,
|
|
14
16
|
last_duration_ms: :u32
|
|
15
17
|
}.freeze
|
|
16
18
|
|
|
17
|
-
|
|
19
|
+
EMPTY_HANDLES = {}.freeze
|
|
18
20
|
|
|
19
|
-
|
|
20
|
-
require 'async/utilization'
|
|
21
|
+
attr_reader :registry, :shm_path, :unavailable_reason
|
|
21
22
|
|
|
23
|
+
def initialize(worker_index:, total_workers:, shm_path: self.class.default_shm_path)
|
|
24
|
+
@enabled = false
|
|
22
25
|
@registry = nil
|
|
23
|
-
@
|
|
24
|
-
@
|
|
25
|
-
@
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
@metric_handles = EMPTY_HANDLES
|
|
27
|
+
@shm_path = shm_path
|
|
28
|
+
@unavailable_reason = nil
|
|
29
|
+
|
|
30
|
+
validate_worker!(worker_index, total_workers)
|
|
31
|
+
initialize_registry!(worker_index, total_workers, shm_path)
|
|
32
|
+
rescue LoadError => error
|
|
33
|
+
mark_unavailable!(error)
|
|
31
34
|
end
|
|
32
35
|
|
|
33
|
-
def enabled?
|
|
34
|
-
|
|
36
|
+
def enabled? = @enabled
|
|
37
|
+
|
|
38
|
+
def job_started(_entry)
|
|
39
|
+
return unless enabled?
|
|
40
|
+
|
|
41
|
+
increment(:total_runs)
|
|
42
|
+
increment(:active_jobs)
|
|
43
|
+
set(:last_run_at, Process.clock_gettime(Process::CLOCK_REALTIME).to_i)
|
|
35
44
|
end
|
|
36
45
|
|
|
37
|
-
def
|
|
38
|
-
return unless
|
|
46
|
+
def job_succeeded(_entry, duration)
|
|
47
|
+
return unless enabled?
|
|
39
48
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
@registry.set(:last_run_at, Process.clock_gettime(Process::CLOCK_REALTIME).to_i)
|
|
49
|
+
increment(:total_successes)
|
|
50
|
+
set(:last_duration_ms, duration_to_milliseconds(duration))
|
|
43
51
|
end
|
|
44
52
|
|
|
45
53
|
def job_finished(entry, duration)
|
|
46
|
-
|
|
54
|
+
job_succeeded(entry, duration)
|
|
55
|
+
job_stopped(entry)
|
|
56
|
+
end
|
|
47
57
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
@registry.set(:last_duration_ms, (duration * 1000).to_i)
|
|
58
|
+
def job_failed(_entry, _error)
|
|
59
|
+
increment(:total_failures) if enabled?
|
|
51
60
|
end
|
|
52
61
|
|
|
53
|
-
def
|
|
54
|
-
|
|
62
|
+
def job_timed_out(_entry)
|
|
63
|
+
increment(:total_timeouts) if enabled?
|
|
64
|
+
end
|
|
55
65
|
|
|
56
|
-
|
|
57
|
-
|
|
66
|
+
def job_stopped(_entry)
|
|
67
|
+
decrement(:active_jobs) if enabled?
|
|
58
68
|
end
|
|
59
69
|
|
|
60
|
-
def
|
|
61
|
-
|
|
70
|
+
def job_skipped(_entry)
|
|
71
|
+
increment(:total_skips) if enabled?
|
|
72
|
+
end
|
|
62
73
|
|
|
63
|
-
|
|
64
|
-
|
|
74
|
+
def values
|
|
75
|
+
enabled? ? registry.values : {}
|
|
65
76
|
end
|
|
66
77
|
|
|
67
|
-
|
|
68
|
-
|
|
78
|
+
class << self
|
|
79
|
+
def available?
|
|
80
|
+
load_utilization!
|
|
81
|
+
true
|
|
82
|
+
rescue LoadError
|
|
83
|
+
false
|
|
84
|
+
end
|
|
69
85
|
|
|
70
|
-
|
|
71
|
-
|
|
86
|
+
def load_utilization!
|
|
87
|
+
require 'async/utilization'
|
|
88
|
+
end
|
|
72
89
|
|
|
73
|
-
|
|
74
|
-
|
|
90
|
+
def schema
|
|
91
|
+
load_utilization!
|
|
92
|
+
::Async::Utilization::Schema.build(SCHEMA_FIELDS)
|
|
93
|
+
end
|
|
75
94
|
|
|
76
|
-
|
|
77
|
-
|
|
95
|
+
def read_all(total_workers:, path: default_shm_path)
|
|
96
|
+
validate_total_workers!(total_workers)
|
|
97
|
+
return [] unless available? && File.file?(path)
|
|
78
98
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
99
|
+
layout = schema
|
|
100
|
+
segment = segment_size
|
|
101
|
+
required_size = segment * total_workers
|
|
102
|
+
|
|
103
|
+
File.open(path, 'rb') do |file|
|
|
104
|
+
return [] if file.size < required_size
|
|
83
105
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
# # { worker: 2, total_runs: 98, active_jobs: 0, ... }
|
|
91
|
-
# # ]
|
|
92
|
-
#
|
|
93
|
-
def self.read_all(total_workers:, path: default_shm_path)
|
|
94
|
-
require 'async/utilization'
|
|
106
|
+
buffer = IO::Buffer.map(file, required_size, 0, IO::Buffer::READONLY)
|
|
107
|
+
decode_workers(buffer, layout, segment, total_workers)
|
|
108
|
+
end
|
|
109
|
+
rescue Errno::ENOENT
|
|
110
|
+
[]
|
|
111
|
+
end
|
|
95
112
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
113
|
+
def default_shm_path
|
|
114
|
+
ENV.fetch('ASYNC_BACKGROUND_METRICS_PATH') { File.join(Dir.tmpdir, 'async-background.shm') }
|
|
115
|
+
end
|
|
99
116
|
|
|
100
|
-
|
|
101
|
-
IO::Buffer.
|
|
117
|
+
def segment_size
|
|
118
|
+
SCHEMA_FIELDS.sum { |_, type| IO::Buffer.size_of(type) }
|
|
102
119
|
end
|
|
103
120
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
121
|
+
private
|
|
122
|
+
|
|
123
|
+
def validate_total_workers!(total_workers)
|
|
124
|
+
return if total_workers.is_a?(Integer) && total_workers.positive?
|
|
125
|
+
|
|
126
|
+
raise ArgumentError, 'total_workers must be a positive Integer'
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def decode_workers(buffer, schema, segment, total_workers)
|
|
130
|
+
(1..total_workers).map do |worker|
|
|
131
|
+
decode_worker(buffer, schema, segment, worker)
|
|
132
|
+
end.freeze
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def decode_worker(buffer, schema, segment, worker)
|
|
136
|
+
offset = (worker - 1) * segment
|
|
137
|
+
schema.fields.each_with_object(worker: worker) do |field, values|
|
|
138
|
+
values[field.name] = buffer.get_value(field.type, offset + field.offset)
|
|
139
|
+
end.freeze
|
|
111
140
|
end
|
|
112
141
|
end
|
|
113
142
|
|
|
114
|
-
|
|
115
|
-
|
|
143
|
+
private
|
|
144
|
+
|
|
145
|
+
def initialize_registry!(worker_index, total_workers, path)
|
|
146
|
+
self.class.load_utilization!
|
|
147
|
+
ensure_shm!(total_workers, path)
|
|
148
|
+
|
|
149
|
+
@registry = ::Async::Utilization::Registry.new
|
|
150
|
+
unless @registry.respond_to?(:metric)
|
|
151
|
+
raise LoadError, 'async-utilization >= 0.3 is required for metrics'
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
attach_observer!(worker_index, path)
|
|
155
|
+
@metric_handles = SCHEMA_FIELDS.keys.to_h { |name| [name, @registry.metric(name)] }.freeze
|
|
156
|
+
@enabled = true
|
|
116
157
|
end
|
|
117
158
|
|
|
118
|
-
def
|
|
119
|
-
|
|
159
|
+
def mark_unavailable!(error)
|
|
160
|
+
@registry = nil
|
|
161
|
+
@metric_handles = EMPTY_HANDLES
|
|
162
|
+
@unavailable_reason = error.message
|
|
120
163
|
end
|
|
121
164
|
|
|
122
|
-
|
|
165
|
+
def increment(name)
|
|
166
|
+
metric(name).increment
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def decrement(name)
|
|
170
|
+
metric(name).decrement
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def set(name, value)
|
|
174
|
+
metric(name).set(value)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def metric(name)
|
|
178
|
+
@metric_handles.fetch(name)
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def duration_to_milliseconds(duration)
|
|
182
|
+
(duration * 1000).to_i
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def validate_worker!(worker_index, total_workers)
|
|
186
|
+
self.class.send(:validate_total_workers!, total_workers)
|
|
187
|
+
return if worker_index.is_a?(Integer) && worker_index.between?(1, total_workers)
|
|
188
|
+
|
|
189
|
+
raise ArgumentError, 'worker_index must be an Integer between 1 and total_workers'
|
|
190
|
+
end
|
|
123
191
|
|
|
124
192
|
def ensure_shm!(total_workers, path)
|
|
125
|
-
|
|
193
|
+
required_size = self.class.segment_size * total_workers
|
|
126
194
|
|
|
127
|
-
File.open(path, File::CREAT | File::RDWR,
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
195
|
+
File.open(path, File::CREAT | File::RDWR, 0o644) do |file|
|
|
196
|
+
file.flock(File::LOCK_EX)
|
|
197
|
+
file.truncate(required_size) if file.size < required_size
|
|
198
|
+
ensure
|
|
199
|
+
file.flock(File::LOCK_UN) rescue nil
|
|
131
200
|
end
|
|
132
201
|
end
|
|
133
202
|
|
|
134
|
-
def attach_observer!(worker_index,
|
|
203
|
+
def attach_observer!(worker_index, path)
|
|
135
204
|
segment = self.class.segment_size
|
|
136
|
-
offset = (worker_index - 1) * segment
|
|
137
205
|
observer = ::Async::Utilization::Observer.open(
|
|
138
|
-
self.class.schema,
|
|
206
|
+
self.class.schema,
|
|
207
|
+
path,
|
|
208
|
+
segment,
|
|
209
|
+
(worker_index - 1) * segment
|
|
139
210
|
)
|
|
140
|
-
|
|
211
|
+
registry.observer = observer
|
|
141
212
|
end
|
|
142
213
|
end
|
|
143
214
|
end
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative '../clock'
|
|
4
|
+
require_relative 'store'
|
|
4
5
|
|
|
5
6
|
module Async
|
|
6
7
|
module Background
|
|
7
8
|
module Queue
|
|
8
|
-
EMPTY_OPTIONS = {}.freeze
|
|
9
|
-
|
|
10
9
|
class Client
|
|
11
10
|
include Clock
|
|
12
11
|
|
|
@@ -15,17 +14,17 @@ module Async
|
|
|
15
14
|
@notifier = notifier
|
|
16
15
|
end
|
|
17
16
|
|
|
18
|
-
def push(class_name, args =
|
|
17
|
+
def push(class_name, args = EMPTY_ARGS, run_at = nil, options: EMPTY_OPTIONS)
|
|
19
18
|
id = @store.enqueue(class_name, args, run_at, options: options)
|
|
20
19
|
@notifier&.notify_all
|
|
21
20
|
id
|
|
22
21
|
end
|
|
23
22
|
|
|
24
|
-
def push_in(delay, class_name, args =
|
|
23
|
+
def push_in(delay, class_name, args = EMPTY_ARGS, options: EMPTY_OPTIONS)
|
|
25
24
|
push(class_name, args, realtime_now + delay.to_f, options: options)
|
|
26
25
|
end
|
|
27
26
|
|
|
28
|
-
def push_at(time, class_name, args =
|
|
27
|
+
def push_at(time, class_name, args = EMPTY_ARGS, options: EMPTY_OPTIONS)
|
|
29
28
|
run_at = time.respond_to?(:to_f) ? time.to_f : time
|
|
30
29
|
push(class_name, args, run_at, options: options)
|
|
31
30
|
end
|
|
@@ -34,19 +33,27 @@ module Async
|
|
|
34
33
|
class << self
|
|
35
34
|
attr_accessor :default_client
|
|
36
35
|
|
|
37
|
-
def
|
|
36
|
+
def migrate!(path: Store.default_path, options: {})
|
|
37
|
+
Store.migrate!(path: path, options: options)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def prepare_dashboard!(path: Store.default_path, options: {})
|
|
41
|
+
Store.prepare_dashboard!(path: path, options: options)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def enqueue(job_class, *args, options: EMPTY_OPTIONS)
|
|
38
45
|
ensure_configured!
|
|
39
|
-
default_client.push(resolve_class_name(job_class), args, nil, options: build_options(job_class, options))
|
|
46
|
+
default_client.push(resolve_class_name(job_class), normalized_args(args), nil, options: build_options(job_class, options))
|
|
40
47
|
end
|
|
41
48
|
|
|
42
|
-
def enqueue_in(delay, job_class, *args, options:
|
|
49
|
+
def enqueue_in(delay, job_class, *args, options: EMPTY_OPTIONS)
|
|
43
50
|
ensure_configured!
|
|
44
|
-
default_client.push_in(delay, resolve_class_name(job_class), args, options: build_options(job_class, options))
|
|
51
|
+
default_client.push_in(delay, resolve_class_name(job_class), normalized_args(args), options: build_options(job_class, options))
|
|
45
52
|
end
|
|
46
53
|
|
|
47
|
-
def enqueue_at(time, job_class, *args, options:
|
|
54
|
+
def enqueue_at(time, job_class, *args, options: EMPTY_OPTIONS)
|
|
48
55
|
ensure_configured!
|
|
49
|
-
default_client.push_at(time, resolve_class_name(job_class), args, options: build_options(job_class, options))
|
|
56
|
+
default_client.push_at(time, resolve_class_name(job_class), normalized_args(args), options: build_options(job_class, options))
|
|
50
57
|
end
|
|
51
58
|
|
|
52
59
|
private
|
|
@@ -55,8 +62,11 @@ module Async
|
|
|
55
62
|
private_constant :RETRY_KEYS
|
|
56
63
|
|
|
57
64
|
def build_options(job_class, call_site)
|
|
58
|
-
call_site ||=
|
|
59
|
-
|
|
65
|
+
call_site ||= EMPTY_OPTIONS
|
|
66
|
+
class_options = resolve_options(job_class)
|
|
67
|
+
return EMPTY_OPTIONS if class_options.empty? && call_site.empty?
|
|
68
|
+
|
|
69
|
+
merged = class_options.merge(call_site.compact)
|
|
60
70
|
apply_retry_overrides!(merged, call_site)
|
|
61
71
|
|
|
62
72
|
merged.empty? ? EMPTY_OPTIONS : Job::Options.new(**merged).to_h.compact
|
|
@@ -72,6 +82,8 @@ module Async
|
|
|
72
82
|
raise "Async::Background::Queue not configured" unless default_client
|
|
73
83
|
end
|
|
74
84
|
|
|
85
|
+
def normalized_args(args) = args.empty? ? EMPTY_ARGS : args
|
|
86
|
+
|
|
75
87
|
def resolve_class_name(job_class)
|
|
76
88
|
return job_class if job_class.is_a?(String)
|
|
77
89
|
return job_class.name if job_class.respond_to?(:perform_now)
|
|
@@ -80,9 +92,15 @@ module Async
|
|
|
80
92
|
end
|
|
81
93
|
|
|
82
94
|
def resolve_options(job_class)
|
|
83
|
-
return
|
|
95
|
+
return EMPTY_OPTIONS unless job_class.respond_to?(:resolve_options)
|
|
96
|
+
|
|
97
|
+
options = if job_class.respond_to?(:queue_options)
|
|
98
|
+
job_class.queue_options
|
|
99
|
+
else
|
|
100
|
+
job_class.resolve_options
|
|
101
|
+
end
|
|
84
102
|
|
|
85
|
-
|
|
103
|
+
options.empty? ? EMPTY_OPTIONS : options
|
|
86
104
|
end
|
|
87
105
|
end
|
|
88
106
|
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Async
|
|
4
|
+
module Background
|
|
5
|
+
module Queue
|
|
6
|
+
EMPTY_ARGS = [].freeze
|
|
7
|
+
EMPTY_OPTIONS = {}.freeze
|
|
8
|
+
|
|
9
|
+
SYNCHRONOUS_LEVELS = {normal: 'NORMAL', full: 'FULL', extra: 'EXTRA'}.freeze
|
|
10
|
+
WAL_AUTOCHECKPOINT_RANGE = 100..10_000
|
|
11
|
+
DEFAULT_STORE_OPTIONS = {mmap: true, synchronous: :normal, wal_autocheckpoint: 1_000}.freeze
|
|
12
|
+
DEFAULTS = DEFAULT_STORE_OPTIONS
|
|
13
|
+
MMAP_SIZE = 268_435_456
|
|
14
|
+
|
|
15
|
+
StoreOptions = Data.define(:mmap, :synchronous, :wal_autocheckpoint) do
|
|
16
|
+
def self.build(value = {})
|
|
17
|
+
return value if value.is_a?(self)
|
|
18
|
+
|
|
19
|
+
new(**DEFAULT_STORE_OPTIONS, **value)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def initialize(mmap:, synchronous:, wal_autocheckpoint:)
|
|
23
|
+
validate_mmap!(mmap)
|
|
24
|
+
validate_synchronous!(synchronous)
|
|
25
|
+
validate_wal_autocheckpoint!(wal_autocheckpoint)
|
|
26
|
+
|
|
27
|
+
super
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def synchronous_pragma = SYNCHRONOUS_LEVELS.fetch(synchronous)
|
|
31
|
+
def mmap_size = mmap ? MMAP_SIZE : 0
|
|
32
|
+
|
|
33
|
+
def pragma_sql
|
|
34
|
+
<<~SQL
|
|
35
|
+
PRAGMA journal_mode = WAL;
|
|
36
|
+
PRAGMA synchronous = #{synchronous_pragma};
|
|
37
|
+
PRAGMA mmap_size = #{mmap_size};
|
|
38
|
+
PRAGMA cache_size = -16000;
|
|
39
|
+
PRAGMA temp_store = MEMORY;
|
|
40
|
+
PRAGMA journal_size_limit = 67108864;
|
|
41
|
+
PRAGMA wal_autocheckpoint = #{wal_autocheckpoint};
|
|
42
|
+
SQL
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def validate_mmap!(value)
|
|
48
|
+
return if value == true || value == false
|
|
49
|
+
|
|
50
|
+
raise ArgumentError, "mmap must be true or false, got #{value.inspect}"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def validate_synchronous!(value)
|
|
54
|
+
return if SYNCHRONOUS_LEVELS.key?(value)
|
|
55
|
+
|
|
56
|
+
raise ArgumentError,
|
|
57
|
+
"synchronous must be one of #{SYNCHRONOUS_LEVELS.keys.inspect}, got #{value.inspect}"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def validate_wal_autocheckpoint!(value)
|
|
61
|
+
return if value.is_a?(Integer) && WAL_AUTOCHECKPOINT_RANGE.cover?(value)
|
|
62
|
+
|
|
63
|
+
raise ArgumentError,
|
|
64
|
+
"wal_autocheckpoint must be an Integer in #{WAL_AUTOCHECKPOINT_RANGE}, " \
|
|
65
|
+
"got #{value.inspect}"
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|