async-background 0.7.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +113 -1
- data/README.md +56 -9
- data/async-background.gemspec +5 -2
- data/lib/async/background/job.rb +5 -3
- data/lib/async/background/metrics.rb +160 -87
- data/lib/async/background/queue/client.rb +33 -15
- data/lib/async/background/queue/options.rb +70 -0
- data/lib/async/background/queue/schema.rb +165 -0
- data/lib/async/background/queue/sql.rb +216 -0
- data/lib/async/background/queue/store.rb +270 -148
- data/lib/async/background/runner/queue_execution.rb +199 -0
- data/lib/async/background/runner/schedule.rb +129 -0
- data/lib/async/background/runner.rb +112 -229
- data/lib/async/background/version.rb +1 -1
- data/lib/async/background/web/app.rb +138 -0
- data/lib/async/background/web/assets.rb +726 -0
- data/lib/async/background/web/auth.rb +19 -0
- data/lib/async/background/web/configuration.rb +158 -0
- data/lib/async/background/web/cursor.rb +58 -0
- data/lib/async/background/web/errors.rb +14 -0
- data/lib/async/background/web/event_hub.rb +194 -0
- data/lib/async/background/web/metrics_reader.rb +96 -0
- data/lib/async/background/web/request.rb +36 -0
- data/lib/async/background/web/response.rb +85 -0
- data/lib/async/background/web/router.rb +30 -0
- data/lib/async/background/web/serializer.rb +154 -0
- data/lib/async/background/web/snapshot.rb +247 -0
- data/lib/async/background/web/sql.rb +88 -0
- data/lib/async/background/web/stream.rb +43 -0
- data/lib/async/background/web.rb +52 -0
- metadata +71 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3d7244ccfb7156c423229ed59f36df9354d293711bab53e6a87d943743bc7f44
|
|
4
|
+
data.tar.gz: 84752c52033b89c5b957d39976f5d025a56215b378138eda8813c05635c5d58e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6e90dd0f6be9ff7361aba7d84e7a7cdfdd712b351938c50dc8b84358d10a20f65514880a3eac3ab2ad8c0adf5164c06f9668ead03df5161fc4482b4e27aa03ec
|
|
7
|
+
data.tar.gz: 1867b17975c63966aeaf437414795b3b5d9083c9871a9f6c44dc6b7280651bbf46a8c929b4751349f969ffbdf72c32aeb8c6965634d149e74a0a352e4ef958df
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,117 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
### Dashboard SSE hardening
|
|
6
|
+
|
|
7
|
+
- Make SSE the default dashboard transport. `:polling` remains an explicit compatibility fallback.
|
|
8
|
+
- Replace one `PRAGMA data_version` loop **per browser connection** with one shared `EventHub` watcher per Rack app process while at least one dashboard tab is connected. It fans complete overview snapshots to every SSE body through latest-value mailboxes, so slow tabs do not accumulate unbounded event queues.
|
|
9
|
+
- Use complete snapshots on connect and after a change rather than a replay log: reconnecting EventSource clients cannot miss the current queue state.
|
|
10
|
+
- Coalesce client-side list refreshes after a burst of changes, cancel stale list fetches on tab switches, and retain previous pages when `Load more` is used.
|
|
11
|
+
- Add SSE retry control, 25-second heartbeat frames, `no-cache, no-transform`, and `X-Accel-Buffering: no`. Remove the hop-by-hop `Connection` response header.
|
|
12
|
+
- Fingerprint asset URLs and cache immutable assets by digest, so a dashboard deploy cannot leave a browser on incompatible HTML/JS/CSS.
|
|
13
|
+
- Add coverage for event fan-out, latest-value coalescing, clean stream shutdown, forced overview reads, and the SSE configuration constraints.
|
|
14
|
+
|
|
15
|
+
## 1.1.0
|
|
16
|
+
|
|
17
|
+
Server-Sent Events transport for the dashboard. Replaces HTTP polling as the recommended transport.
|
|
18
|
+
|
|
19
|
+
### Added
|
|
20
|
+
|
|
21
|
+
- **SSE transport for the dashboard.** Set `c.transport = :sse` in `Async::Background::Web.configure` and the dashboard now uses a single long-lived `text/event-stream` connection per browser tab instead of polling `/api/overview` every 2 seconds. The browser opens `EventSource(mount_path + '/api/stream')` once; the server pushes an `overview` event whenever `PRAGMA data_version` changes, and a `:keepalive` comment frame every 30 seconds. Result: 1 HTTP connection per dashboard tab regardless of how long it stays open, instead of 30 req/min per tab.
|
|
22
|
+
|
|
23
|
+
- New module `Async::Background::Web::Stream` implements the event loop as a Rack streaming body (responds to `#each`, yields SSE frames). Holds no state across requests.
|
|
24
|
+
- New route `GET /api/stream` returns `200 text/event-stream` when `transport == :sse`, `404` otherwise. Subject to the same auth gate as every other endpoint.
|
|
25
|
+
- New `Response.sse(body)` helper sets the correct headers including `x-accel-buffering: no` (disables nginx buffering for the streaming response).
|
|
26
|
+
- JS client (`assets.rb`) detects `state.config.transport === 'sse'` at boot and chooses `EventSource` over `setInterval(tick, ...)`. Both transports share the same `applyOverview()` and `refreshActiveList()` handlers, so the UI behaves identically.
|
|
27
|
+
|
|
28
|
+
- **`Configuration#transport`** with default `:polling` (backward compatible) and accepted values `:polling | :sse`. Validation rejects anything else with `ConfigurationError`. The chosen transport is exposed at `/api/config` so the client knows which path to take.
|
|
29
|
+
|
|
30
|
+
### Migration
|
|
31
|
+
|
|
32
|
+
Existing deployments keep working unchanged. To opt into SSE:
|
|
33
|
+
|
|
34
|
+
```ruby
|
|
35
|
+
Async::Background::Web.configure do |c|
|
|
36
|
+
c.queue_path = ...
|
|
37
|
+
c.auth = ->(env) { ... }
|
|
38
|
+
c.transport = :sse
|
|
39
|
+
end
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### Server compatibility note
|
|
43
|
+
|
|
44
|
+
SSE holds the request thread/fiber open for the lifetime of the dashboard tab. **Recommended for Falcon**, which handles long-lived connections natively via fibers. **Puma works** but each open dashboard tab holds one worker thread for its lifetime — fine for an admin dashboard with a handful of operators, problematic if many concurrent operators would starve the worker pool. **Unicorn does not work** for SSE since its blocking worker model can't hold long-lived connections without timeouts; stay on `:polling` there.
|
|
45
|
+
|
|
46
|
+
### Backend-side polling
|
|
47
|
+
|
|
48
|
+
The server still polls `PRAGMA data_version` every 500ms inside the snapshot connection to detect changes. This is a connection-local PRAGMA call, microseconds, never hits a rate limiter. Client-facing transport is push.
|
|
49
|
+
|
|
50
|
+
### Tests
|
|
51
|
+
|
|
52
|
+
- New `spec/async/background/web/stream_spec.rb` — covers overview event on data_version change, heartbeat after idle, graceful exit on `EPIPE`/`IOError`, error frame on `ClosedError`/`UnavailableError`.
|
|
53
|
+
- Extended `spec/async/background/web/app_spec.rb` — `/api/stream` returns 404 on polling default, 200 text/event-stream on `:sse`, 401 without auth.
|
|
54
|
+
- Extended `spec/async/background/web/configuration_spec.rb` — accepts `:sse`, rejects unknown transports.
|
|
55
|
+
|
|
56
|
+
## 1.0.0
|
|
57
|
+
|
|
58
|
+
First stable release. The queue execution contract from 0.7.2 (claim-token CAS, lifecycle columns, barrier-based shutdown drain, per-status partial indexes, versioned migrations) is now considered the public API.
|
|
59
|
+
|
|
60
|
+
### Features
|
|
61
|
+
|
|
62
|
+
- **Web dashboard.** Rack-mountable read-only UI under `require 'async/background/web'`. Vanilla HTML/CSS/JS, no JS framework, no npm.
|
|
63
|
+
- Endpoints: `GET /`, `GET /assets/app.css`, `GET /assets/app.js`, `GET /api/overview`, `GET /api/executing`, `GET /api/claimed`, `GET /api/pending`, `GET /api/done`, `GET /api/failed`, `GET /api/metrics`, `GET /api/config`.
|
|
64
|
+
- Default transport is JSON polling (`poll_interval_ms`, default 2000). SSE adapter for Falcon is intentionally deferred to a later release; the dashboard already coalesces work via a shared overview cache, so adding SSE later is a backward-compatible change.
|
|
65
|
+
- Read path runs through `Async::Background::Web::Snapshot`, which opens SQLite with `file:?mode=ro`, wraps a `Mutex` around a single shared connection, and uses one read transaction per endpoint and caches each overview as one consistent snapshot.
|
|
66
|
+
- Distinguishes `Executing` (`status='running' AND started_at IS NOT NULL`) from `Claimed` (`status='running' AND started_at IS NULL`).
|
|
67
|
+
- Overview snapshot cache for `counts_cache_ttl` seconds (default 3.0) so a busy queue does not turn the dashboard into a hot reader.
|
|
68
|
+
- Cursor pagination for `done`/`failed`/`pending` using `(finished_at, id)` / `(run_at, id)` tuples. Stable on ties.
|
|
69
|
+
- Args hidden by default (`expose_args: false`); when enabled, content runs through `redact_args`. All user content rendered through `textContent`, never `innerHTML`.
|
|
70
|
+
- Auth hook is **mandatory**. `Configuration#validate!` rejects an unconfigured `auth`. There is no permissive default.
|
|
71
|
+
|
|
72
|
+
- Add the optional Rack dashboard for the SQLite queue.
|
|
73
|
+
- Make sqlite3 an explicit runtime dependency for queue/dashboard installs.
|
|
74
|
+
|
|
75
|
+
### Configuration
|
|
76
|
+
|
|
77
|
+
```ruby
|
|
78
|
+
require 'async/background/web'
|
|
79
|
+
|
|
80
|
+
Async::Background::Queue::Store.prepare_dashboard!(path: '/var/lib/app/queue.db')
|
|
81
|
+
|
|
82
|
+
Async::Background::Web.configure do |c|
|
|
83
|
+
c.queue_path = '/var/lib/app/queue.db'
|
|
84
|
+
c.auth = ->(env) { env['warden'].user&.admin? }
|
|
85
|
+
c.expose_args = false
|
|
86
|
+
c.metrics_path = '/run/app/async-background.shm'
|
|
87
|
+
c.total_workers = 4
|
|
88
|
+
c.counts_cache_ttl = 3.0
|
|
89
|
+
c.poll_interval_ms = 2000
|
|
90
|
+
c.list_limit = 50
|
|
91
|
+
c.mount_path = '/admin/background'
|
|
92
|
+
c.title = 'My App background jobs'
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
run Async::Background::Web.app
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### Dependencies
|
|
99
|
+
|
|
100
|
+
- `rack` is an optional dependency. Required only when `require 'async/background/web'` is loaded. Core gem and worker processes do not require it.
|
|
101
|
+
|
|
102
|
+
### Breaking changes from 0.7.x
|
|
103
|
+
|
|
104
|
+
None beyond what 0.7.2 already shipped. The 1.0 line locks the existing contract:
|
|
105
|
+
|
|
106
|
+
- `Queue::Store#fetch` returns `claim_token` in the result hash.
|
|
107
|
+
- All terminal `Queue::Store` methods (`complete`, `fail`, `retry_or_fail`) require the `claim_token:` kwarg and return CAS success boolean / `:retried` / `:failed` / `nil`.
|
|
108
|
+
- Schema is versioned via `PRAGMA user_version`. Use `Queue::Store.migrate!(path:)` to upgrade. Use `Queue::Store.prepare_dashboard!(path:)` from the dashboard process to lazily create dashboard-only indexes (per-status partial indexes for `done` / `failed`, plus separate `executing` and `claimed` indexes).
|
|
109
|
+
|
|
110
|
+
## 0.7.2
|
|
111
|
+
|
|
112
|
+
- Harden queue execution, retries, shutdown, and metrics.
|
|
113
|
+
- Add schema v1, optional dashboard indexes, and a faster enqueue path.
|
|
114
|
+
|
|
3
115
|
## 0.7.1
|
|
4
116
|
|
|
5
117
|
### Features
|
|
@@ -97,7 +209,7 @@
|
|
|
97
209
|
- Proper job distribution validation across worker pool
|
|
98
210
|
- **Test fixtures** — dedicated `ci/fixtures/jobs.rb` and `ci/fixtures/schedule.yml` for scenario testing
|
|
99
211
|
|
|
100
|
-
### Bug Fixes
|
|
212
|
+
### Bug Fixes
|
|
101
213
|
- **SQLite busy timeout** — added `PRAGMA busy_timeout = 5000` to `Queue::Store` to prevent `SQLITE_BUSY` errors under concurrent multi-process database access
|
|
102
214
|
- **Enhanced Queue::Notifier error handling** — restructured IO error handling with clearer categorization:
|
|
103
215
|
- `WRITE_DROPPED` for write failures (`IO::WaitWritable`, `Errno::EAGAIN`, `IOError`, `Errno::EPIPE`) — all non-fatal as job is already in store
|
data/README.md
CHANGED
|
@@ -12,16 +12,16 @@ A lightweight cron, interval, and job-queue scheduler for Ruby's [Async](https:/
|
|
|
12
12
|
|
|
13
13
|
- Ruby >= 3.3
|
|
14
14
|
- `async ~> 2.0`, `fugit ~> 1.0`
|
|
15
|
-
- `sqlite3 ~> 2.0` (optional,
|
|
16
|
-
- `async-utilization
|
|
15
|
+
- `sqlite3 ~> 2.0` (optional, storage)
|
|
16
|
+
- `async-utilization >= 0.3, < 0.5` (optional, for metrics)
|
|
17
17
|
|
|
18
18
|
## Install
|
|
19
19
|
|
|
20
20
|
```ruby
|
|
21
21
|
# Gemfile
|
|
22
22
|
gem "async-background"
|
|
23
|
-
gem "sqlite3", "~> 2.0"
|
|
24
|
-
gem "async-utilization", "
|
|
23
|
+
gem "sqlite3", "~> 2.0" # optional
|
|
24
|
+
gem "async-utilization", ">= 0.3", "< 0.5" # optional
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
## ➡️ [Get Started](docs/GET_STARTED.md)
|
|
@@ -92,9 +92,8 @@ Without this, you will get database crashes in multi-process mode. See [Get Star
|
|
|
92
92
|
**Don't share SQLite connections across `fork()`.** The gem opens connections lazily after fork, but if you create a `Queue::Store` manually for schema setup, close it before forking:
|
|
93
93
|
|
|
94
94
|
```ruby
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
store.close # ← before fork
|
|
95
|
+
Async::Background::Queue.migrate!(path: db_path) # ← once, before fork
|
|
96
|
+
# Every process opens its own Store lazily after fork.
|
|
98
97
|
```
|
|
99
98
|
|
|
100
99
|
**Two clocks, on purpose.** Interval jobs use `CLOCK_MONOTONIC` (immune to NTP drift). Cron jobs use wall-clock time, because "every day at 3am" needs to mean 3am.
|
|
@@ -127,19 +126,67 @@ The dynamic queue runs alongside it:
|
|
|
127
126
|
|
|
128
127
|
Jobs are persisted in SQLite, so a missed wake-up is never a lost job — workers also poll every 5 seconds as a safety net.
|
|
129
128
|
|
|
129
|
+
### Queue-only workers
|
|
130
|
+
|
|
131
|
+
Recurring schedules are optional. A worker that serves only dynamic jobs starts with
|
|
132
|
+
`config_path: nil` and a `queue_socket_dir`; it does not need a placeholder schedule file.
|
|
133
|
+
A supplied schedule path stays strict and raises when the file is missing or empty.
|
|
134
|
+
|
|
135
|
+
### Schema migration during deploy
|
|
136
|
+
|
|
137
|
+
Run queue migrations once in the release/pre-deploy step, before starting new web or worker
|
|
138
|
+
processes. This serializes the schema upgrade with `BEGIN IMMEDIATE`, records the version in
|
|
139
|
+
SQLite, and avoids a first producer doing DDL under live queue traffic:
|
|
140
|
+
|
|
141
|
+
```ruby
|
|
142
|
+
Async::Background::Queue.migrate!(path: ENV.fetch("QUEUE_DB_PATH"))
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
A fresh database still self-initializes on first use for local development, but explicit
|
|
146
|
+
migration is the production path. For an existing queue, finish or stop 0.7.1 producers/workers,
|
|
147
|
+
run the migration once, then start 0.7.2 processes.
|
|
148
|
+
|
|
149
|
+
### Dashboard indexes
|
|
150
|
+
|
|
151
|
+
The queue does **not** install dashboard indexes by default. They slow every enqueue even though
|
|
152
|
+
pending rows never enter terminal or in-flight read-model indexes. Enable them once before
|
|
153
|
+
mounting the dashboard:
|
|
154
|
+
|
|
155
|
+
```ruby
|
|
156
|
+
Async::Background::Queue.prepare_dashboard!(path: ENV.fetch("QUEUE_DB_PATH"))
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
It adds four compact indexes: cursor-sorted `done` / `failed` history plus separate
|
|
160
|
+
`executing` / `claimed` in-flight lists. It does not change queue behavior or rerun the core migration.
|
|
161
|
+
|
|
130
162
|
## Metrics
|
|
131
163
|
|
|
132
|
-
|
|
164
|
+
Metrics are an optional integration with `async-utilization` (`>= 0.3`, `< 0.5`). The
|
|
165
|
+
background worker remains fully functional when that gem is absent. With it installed, each
|
|
166
|
+
worker publishes counters to a shared-memory segment.
|
|
133
167
|
|
|
134
168
|
```ruby
|
|
169
|
+
runner.metrics.enabled?
|
|
135
170
|
runner.metrics.values
|
|
136
171
|
# => { total_runs: 142, total_successes: 140, total_failures: 2,
|
|
137
172
|
# total_timeouts: 0, total_skips: 5, active_jobs: 1, ... }
|
|
138
173
|
|
|
139
174
|
Async::Background::Metrics.read_all(total_workers: 2)
|
|
175
|
+
# => [{ worker: 1, ... }, { worker: 2, ... }]
|
|
140
176
|
```
|
|
141
177
|
|
|
142
|
-
|
|
178
|
+
`Metrics.read_all` returns `[]` until the optional gem is installed and a worker has created
|
|
179
|
+
the file, so an observer can render an unavailable state without rescuing `LoadError`. Its
|
|
180
|
+
snapshot is lock-free best effort: cumulative fields (`total_runs`, `total_successes`,
|
|
181
|
+
`total_failures`, `total_timeouts`, `total_skips`) are counters; `active_jobs`,
|
|
182
|
+
`last_run_at`, and `last_duration_ms` are gauges. Fields can describe adjacent moments in time
|
|
183
|
+
rather than one globally atomic instant.
|
|
184
|
+
|
|
185
|
+
By default the file is `/tmp/async-background.shm`. Set `ASYNC_BACKGROUND_METRICS_PATH`
|
|
186
|
+
or pass `metrics_shm_path:` to `Runner.new` when another observer runs in a separate process
|
|
187
|
+
or container; both sides must see the same mounted file.
|
|
188
|
+
|
|
189
|
+
|
|
143
190
|
|
|
144
191
|
## License
|
|
145
192
|
|
data/async-background.gemspec
CHANGED
|
@@ -33,11 +33,14 @@ Gem::Specification.new do |spec|
|
|
|
33
33
|
spec.add_dependency 'async', '~> 2.0'
|
|
34
34
|
spec.add_dependency 'console', '~> 1.0'
|
|
35
35
|
spec.add_dependency 'fugit', '~> 1.0'
|
|
36
|
+
spec.add_dependency 'base64', '~> 0.2'
|
|
36
37
|
|
|
37
38
|
# Optional: add to your own Gemfile if you need these features
|
|
38
|
-
# gem 'sqlite3',
|
|
39
|
-
# gem 'async-utilization', '
|
|
39
|
+
# gem 'sqlite3', '~> 2.0'
|
|
40
|
+
# gem 'async-utilization', '>= 0.3', '< 0.5' # shared-memory worker metrics
|
|
40
41
|
|
|
41
42
|
spec.add_development_dependency 'rake', '~> 13.0'
|
|
42
43
|
spec.add_development_dependency 'rspec', '~> 3.12'
|
|
44
|
+
spec.add_development_dependency 'rack', '~> 3.0'
|
|
45
|
+
spec.add_development_dependency 'async-utilization', '>= 0.3', '< 0.5'
|
|
43
46
|
end
|
data/lib/async/background/job.rb
CHANGED
|
@@ -4,6 +4,7 @@ module Async
|
|
|
4
4
|
module Background
|
|
5
5
|
module Job
|
|
6
6
|
DEFAULT_TIMEOUT = 120
|
|
7
|
+
EMPTY_OPTIONS = {}.freeze
|
|
7
8
|
BACKOFFS = %i[fixed linear exponential].freeze
|
|
8
9
|
DEFAULT_JITTER_FOR = { fixed: 0.0, linear: 0.0, exponential: 0.5 }.freeze
|
|
9
10
|
|
|
@@ -55,15 +56,16 @@ module Async
|
|
|
55
56
|
module ClassMethods
|
|
56
57
|
def perform_now(*args) = new.perform(*args)
|
|
57
58
|
|
|
58
|
-
def perform_async(*args, options:
|
|
59
|
-
def perform_in(delay, *args, options:
|
|
60
|
-
def perform_at(time, *args, options:
|
|
59
|
+
def perform_async(*args, options: EMPTY_OPTIONS) = Async::Background::Queue.enqueue(self, *args, options: options)
|
|
60
|
+
def perform_in(delay, *args, options: EMPTY_OPTIONS) = Async::Background::Queue.enqueue_in(delay, self, *args, options: options)
|
|
61
|
+
def perform_at(time, *args, options: EMPTY_OPTIONS) = Async::Background::Queue.enqueue_at(time, self, *args, options: options)
|
|
61
62
|
|
|
62
63
|
def options(**values)
|
|
63
64
|
@options = Options.new(**values).to_h.compact
|
|
64
65
|
end
|
|
65
66
|
|
|
66
67
|
def resolve_options = @options || {}
|
|
68
|
+
def queue_options = @options || EMPTY_OPTIONS
|
|
67
69
|
end
|
|
68
70
|
|
|
69
71
|
def perform(*)
|
|
@@ -1,143 +1,216 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'tmpdir'
|
|
4
|
+
|
|
3
5
|
module Async
|
|
4
6
|
module Background
|
|
5
7
|
class Metrics
|
|
6
8
|
SCHEMA_FIELDS = {
|
|
7
|
-
total_runs:
|
|
8
|
-
total_successes:
|
|
9
|
-
total_failures:
|
|
10
|
-
total_timeouts:
|
|
11
|
-
total_skips:
|
|
12
|
-
active_jobs:
|
|
13
|
-
last_run_at:
|
|
9
|
+
total_runs: :u64,
|
|
10
|
+
total_successes: :u64,
|
|
11
|
+
total_failures: :u64,
|
|
12
|
+
total_timeouts: :u64,
|
|
13
|
+
total_skips: :u64,
|
|
14
|
+
active_jobs: :u32,
|
|
15
|
+
last_run_at: :u64,
|
|
14
16
|
last_duration_ms: :u32
|
|
15
17
|
}.freeze
|
|
16
18
|
|
|
17
|
-
|
|
19
|
+
EMPTY_HANDLES = {}.freeze
|
|
18
20
|
|
|
19
|
-
|
|
20
|
-
require 'async/utilization'
|
|
21
|
+
attr_reader :registry, :shm_path, :unavailable_reason
|
|
21
22
|
|
|
23
|
+
def initialize(worker_index:, total_workers:, shm_path: self.class.default_shm_path)
|
|
24
|
+
@enabled = false
|
|
22
25
|
@registry = nil
|
|
23
|
-
@
|
|
24
|
-
@
|
|
25
|
-
@
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
@metric_handles = EMPTY_HANDLES
|
|
27
|
+
@shm_path = shm_path
|
|
28
|
+
@unavailable_reason = nil
|
|
29
|
+
|
|
30
|
+
validate_worker!(worker_index, total_workers)
|
|
31
|
+
initialize_registry!(worker_index, total_workers, shm_path)
|
|
32
|
+
rescue LoadError => error
|
|
33
|
+
mark_unavailable!(error)
|
|
31
34
|
end
|
|
32
35
|
|
|
33
|
-
def enabled?
|
|
34
|
-
|
|
36
|
+
def enabled? = @enabled
|
|
37
|
+
|
|
38
|
+
def job_started(_entry)
|
|
39
|
+
return unless enabled?
|
|
40
|
+
|
|
41
|
+
increment(:total_runs)
|
|
42
|
+
increment(:active_jobs)
|
|
43
|
+
set(:last_run_at, Process.clock_gettime(Process::CLOCK_REALTIME).to_i)
|
|
35
44
|
end
|
|
36
45
|
|
|
37
|
-
def
|
|
38
|
-
return unless
|
|
46
|
+
def job_succeeded(_entry, duration)
|
|
47
|
+
return unless enabled?
|
|
39
48
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
@registry.set(:last_run_at, Process.clock_gettime(Process::CLOCK_REALTIME).to_i)
|
|
49
|
+
increment(:total_successes)
|
|
50
|
+
set(:last_duration_ms, duration_to_milliseconds(duration))
|
|
43
51
|
end
|
|
44
52
|
|
|
45
53
|
def job_finished(entry, duration)
|
|
46
|
-
|
|
54
|
+
job_succeeded(entry, duration)
|
|
55
|
+
job_stopped(entry)
|
|
56
|
+
end
|
|
47
57
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
@registry.set(:last_duration_ms, (duration * 1000).to_i)
|
|
58
|
+
def job_failed(_entry, _error)
|
|
59
|
+
increment(:total_failures) if enabled?
|
|
51
60
|
end
|
|
52
61
|
|
|
53
|
-
def
|
|
54
|
-
|
|
62
|
+
def job_timed_out(_entry)
|
|
63
|
+
increment(:total_timeouts) if enabled?
|
|
64
|
+
end
|
|
55
65
|
|
|
56
|
-
|
|
57
|
-
|
|
66
|
+
def job_stopped(_entry)
|
|
67
|
+
decrement(:active_jobs) if enabled?
|
|
58
68
|
end
|
|
59
69
|
|
|
60
|
-
def
|
|
61
|
-
|
|
70
|
+
def job_skipped(_entry)
|
|
71
|
+
increment(:total_skips) if enabled?
|
|
72
|
+
end
|
|
62
73
|
|
|
63
|
-
|
|
64
|
-
|
|
74
|
+
def values
|
|
75
|
+
enabled? ? registry.values : {}
|
|
65
76
|
end
|
|
66
77
|
|
|
67
|
-
|
|
68
|
-
|
|
78
|
+
class << self
|
|
79
|
+
def available?
|
|
80
|
+
load_utilization!
|
|
81
|
+
true
|
|
82
|
+
rescue LoadError
|
|
83
|
+
false
|
|
84
|
+
end
|
|
69
85
|
|
|
70
|
-
|
|
71
|
-
|
|
86
|
+
def load_utilization!
|
|
87
|
+
require 'async/utilization'
|
|
88
|
+
end
|
|
72
89
|
|
|
73
|
-
|
|
74
|
-
|
|
90
|
+
def schema
|
|
91
|
+
load_utilization!
|
|
92
|
+
::Async::Utilization::Schema.build(SCHEMA_FIELDS)
|
|
93
|
+
end
|
|
75
94
|
|
|
76
|
-
|
|
77
|
-
|
|
95
|
+
def read_all(total_workers:, path: default_shm_path)
|
|
96
|
+
validate_total_workers!(total_workers)
|
|
97
|
+
return [] unless available? && File.file?(path)
|
|
78
98
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
99
|
+
layout = schema
|
|
100
|
+
segment = segment_size
|
|
101
|
+
required_size = segment * total_workers
|
|
102
|
+
|
|
103
|
+
File.open(path, 'rb') do |file|
|
|
104
|
+
return [] if file.size < required_size
|
|
83
105
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
# # { worker: 2, total_runs: 98, active_jobs: 0, ... }
|
|
91
|
-
# # ]
|
|
92
|
-
#
|
|
93
|
-
def self.read_all(total_workers:, path: default_shm_path)
|
|
94
|
-
require 'async/utilization'
|
|
106
|
+
buffer = IO::Buffer.map(file, required_size, 0, IO::Buffer::READONLY)
|
|
107
|
+
decode_workers(buffer, layout, segment, total_workers)
|
|
108
|
+
end
|
|
109
|
+
rescue Errno::ENOENT
|
|
110
|
+
[]
|
|
111
|
+
end
|
|
95
112
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
113
|
+
def default_shm_path
|
|
114
|
+
ENV.fetch('ASYNC_BACKGROUND_METRICS_PATH') { File.join(Dir.tmpdir, 'async-background.shm') }
|
|
115
|
+
end
|
|
99
116
|
|
|
100
|
-
|
|
101
|
-
IO::Buffer.
|
|
117
|
+
def segment_size
|
|
118
|
+
SCHEMA_FIELDS.sum { |_, type| IO::Buffer.size_of(type) }
|
|
102
119
|
end
|
|
103
120
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
121
|
+
private
|
|
122
|
+
|
|
123
|
+
def validate_total_workers!(total_workers)
|
|
124
|
+
return if total_workers.is_a?(Integer) && total_workers.positive?
|
|
125
|
+
|
|
126
|
+
raise ArgumentError, 'total_workers must be a positive Integer'
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def decode_workers(buffer, schema, segment, total_workers)
|
|
130
|
+
(1..total_workers).map do |worker|
|
|
131
|
+
decode_worker(buffer, schema, segment, worker)
|
|
132
|
+
end.freeze
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def decode_worker(buffer, schema, segment, worker)
|
|
136
|
+
offset = (worker - 1) * segment
|
|
137
|
+
schema.fields.each_with_object(worker: worker) do |field, values|
|
|
138
|
+
values[field.name] = buffer.get_value(field.type, offset + field.offset)
|
|
139
|
+
end.freeze
|
|
111
140
|
end
|
|
112
141
|
end
|
|
113
142
|
|
|
114
|
-
|
|
115
|
-
|
|
143
|
+
private
|
|
144
|
+
|
|
145
|
+
def initialize_registry!(worker_index, total_workers, path)
|
|
146
|
+
self.class.load_utilization!
|
|
147
|
+
ensure_shm!(total_workers, path)
|
|
148
|
+
|
|
149
|
+
@registry = ::Async::Utilization::Registry.new
|
|
150
|
+
unless @registry.respond_to?(:metric)
|
|
151
|
+
raise LoadError, 'async-utilization >= 0.3 is required for metrics'
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
attach_observer!(worker_index, path)
|
|
155
|
+
@metric_handles = SCHEMA_FIELDS.keys.to_h { |name| [name, @registry.metric(name)] }.freeze
|
|
156
|
+
@enabled = true
|
|
116
157
|
end
|
|
117
158
|
|
|
118
|
-
def
|
|
119
|
-
|
|
159
|
+
def mark_unavailable!(error)
|
|
160
|
+
@registry = nil
|
|
161
|
+
@metric_handles = EMPTY_HANDLES
|
|
162
|
+
@unavailable_reason = error.message
|
|
120
163
|
end
|
|
121
164
|
|
|
122
|
-
|
|
165
|
+
def increment(name)
|
|
166
|
+
metric(name).increment
|
|
167
|
+
end
|
|
123
168
|
|
|
124
|
-
def
|
|
125
|
-
|
|
169
|
+
def decrement(name)
|
|
170
|
+
metric(name).decrement
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def set(name, value)
|
|
174
|
+
metric(name).set(value)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def metric(name)
|
|
178
|
+
@metric_handles.fetch(name)
|
|
179
|
+
end
|
|
126
180
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
181
|
+
def duration_to_milliseconds(duration)
|
|
182
|
+
(duration * 1000).to_i
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def validate_worker!(worker_index, total_workers)
|
|
186
|
+
self.class.send(:validate_total_workers!, total_workers)
|
|
187
|
+
return if worker_index.is_a?(Integer) && worker_index.between?(1, total_workers)
|
|
188
|
+
|
|
189
|
+
raise ArgumentError, 'worker_index must be an Integer between 1 and total_workers'
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def ensure_shm!(total_workers, path)
|
|
193
|
+
required_size = self.class.segment_size * total_workers
|
|
194
|
+
page_size = IO::Buffer::PAGE_SIZE
|
|
195
|
+
mapped_size = ((required_size + page_size - 1) / page_size) * page_size
|
|
196
|
+
|
|
197
|
+
File.open(path, File::CREAT | File::RDWR, 0o644) do |file|
|
|
198
|
+
file.flock(File::LOCK_EX)
|
|
199
|
+
file.truncate(mapped_size) if file.size < mapped_size
|
|
200
|
+
ensure
|
|
201
|
+
file.flock(File::LOCK_UN) rescue nil
|
|
131
202
|
end
|
|
132
203
|
end
|
|
133
204
|
|
|
134
|
-
def attach_observer!(worker_index,
|
|
205
|
+
def attach_observer!(worker_index, path)
|
|
135
206
|
segment = self.class.segment_size
|
|
136
|
-
offset = (worker_index - 1) * segment
|
|
137
207
|
observer = ::Async::Utilization::Observer.open(
|
|
138
|
-
self.class.schema,
|
|
208
|
+
self.class.schema,
|
|
209
|
+
path,
|
|
210
|
+
segment,
|
|
211
|
+
(worker_index - 1) * segment
|
|
139
212
|
)
|
|
140
|
-
|
|
213
|
+
registry.observer = observer
|
|
141
214
|
end
|
|
142
215
|
end
|
|
143
216
|
end
|