wurk 0.0.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +16 -2
- data/app/controllers/wurk/api/serializers.rb +48 -2
- data/app/controllers/wurk/api_controller.rb +216 -1
- data/app/controllers/wurk/dashboard_controller.rb +20 -2
- data/app/controllers/wurk/extensions_controller.rb +56 -0
- data/app/controllers/wurk/profiles_controller.rb +68 -0
- data/config/routes.rb +54 -1
- data/exe/sidekiqswarm +8 -0
- data/exe/wurkswarm +23 -0
- data/lib/active_job/queue_adapters/wurk_adapter.rb +35 -0
- data/lib/generators/wurk/install/templates/wurk.rb +14 -3
- data/lib/sidekiq/api.rb +4 -0
- data/lib/sidekiq/cli.rb +9 -0
- data/lib/sidekiq/client.rb +4 -0
- data/lib/sidekiq/job.rb +4 -0
- data/lib/sidekiq/launcher.rb +4 -0
- data/lib/sidekiq/middleware/chain.rb +4 -0
- data/lib/sidekiq/middleware/server/statsd.rb +12 -0
- data/lib/sidekiq/rails.rb +10 -0
- data/lib/sidekiq/redis_connection.rb +4 -0
- data/lib/sidekiq/scheduled.rb +4 -0
- data/lib/sidekiq/testing.rb +4 -0
- data/lib/sidekiq/version.rb +4 -0
- data/lib/sidekiq/web.rb +4 -0
- data/lib/sidekiq/worker.rb +4 -0
- data/lib/sidekiq.rb +16 -0
- data/lib/wurk/batch/callbacks.rb +103 -13
- data/lib/wurk/batch/death_handler.rb +5 -2
- data/lib/wurk/batch/server_middleware.rb +35 -3
- data/lib/wurk/batch/status.rb +9 -0
- data/lib/wurk/batch.rb +23 -1
- data/lib/wurk/capsule.rb +20 -1
- data/lib/wurk/cli.rb +84 -1
- data/lib/wurk/client.rb +20 -17
- data/lib/wurk/compat.rb +44 -2
- data/lib/wurk/component.rb +5 -4
- data/lib/wurk/configuration.rb +120 -3
- data/lib/wurk/cron.rb +51 -9
- data/lib/wurk/dead_set.rb +8 -3
- data/lib/wurk/deploy.rb +8 -4
- data/lib/wurk/encryption.rb +6 -1
- data/lib/wurk/fetcher/reaper.rb +78 -11
- data/lib/wurk/fetcher/reliable.rb +14 -4
- data/lib/wurk/heartbeat.rb +45 -0
- data/lib/wurk/history.rb +174 -0
- data/lib/wurk/iterable_job/active_record_enumerator.rb +71 -0
- data/lib/wurk/iterable_job/csv_enumerator.rb +51 -0
- data/lib/wurk/iterable_job.rb +41 -0
- data/lib/wurk/iterable_job_query.rb +75 -0
- data/lib/wurk/job.rb +8 -0
- data/lib/wurk/job_record.rb +16 -1
- data/lib/wurk/job_set.rb +4 -4
- data/lib/wurk/job_util.rb +15 -6
- data/lib/wurk/keys.rb +10 -0
- data/lib/wurk/launcher.rb +35 -1
- data/lib/wurk/leader.rb +15 -6
- data/lib/wurk/limiter/bucket.rb +14 -3
- data/lib/wurk/limiter/concurrent.rb +1 -1
- data/lib/wurk/limiter/window.rb +2 -1
- data/lib/wurk/limiter.rb +12 -0
- data/lib/wurk/lua/loader.rb +10 -0
- data/lib/wurk/lua.rb +106 -14
- data/lib/wurk/metrics/history.rb +5 -0
- data/lib/wurk/metrics/query.rb +39 -0
- data/lib/wurk/metrics/queue_rollup.rb +151 -0
- data/lib/wurk/metrics/statsd.rb +11 -0
- data/lib/wurk/middleware/current_attributes.rb +29 -6
- data/lib/wurk/middleware/interrupt_handler.rb +5 -0
- data/lib/wurk/middleware/poison_pill.rb +35 -5
- data/lib/wurk/processor.rb +17 -8
- data/lib/wurk/profile_set.rb +65 -0
- data/lib/wurk/profiler.rb +127 -0
- data/lib/wurk/railtie.rb +19 -5
- data/lib/wurk/redis_client_adapter.rb +72 -0
- data/lib/wurk/redis_connection.rb +30 -0
- data/lib/wurk/redis_pool.rb +5 -1
- data/lib/wurk/scheduled.rb +42 -0
- data/lib/wurk/sorted_entry.rb +13 -11
- data/lib/wurk/stats.rb +11 -4
- data/lib/wurk/swarm/child_boot.rb +26 -4
- data/lib/wurk/swarm.rb +1 -1
- data/lib/wurk/transaction_aware_client.rb +69 -0
- data/lib/wurk/unique.rb +49 -7
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/batch_status.rb +42 -0
- data/lib/wurk/web/config.rb +219 -17
- data/lib/wurk/web/enterprise.rb +14 -0
- data/lib/wurk/web/extension.rb +348 -0
- data/lib/wurk/web/rack_app.rb +77 -0
- data/lib/wurk/web.rb +2 -0
- data/lib/wurk/worker/setter.rb +5 -1
- data/lib/wurk/worker.rb +17 -6
- data/lib/wurk.rb +44 -0
- data/vendor/assets/dashboard/assets/fa-brands-400-BP5tdqmh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-nyy7hhHF.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-DRAAbZTg.woff2 +0 -0
- data/vendor/assets/dashboard/assets/index-9CFRWpfG.js +77 -0
- data/vendor/assets/dashboard/assets/index-CW8AFQIv.css +2 -0
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/favicon.png +0 -0
- data/vendor/assets/dashboard/index.html +10 -3
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +42 -6
- data/CHANGELOG.md +0 -67
- data/CONTRIBUTING.md +0 -73
- data/SECURITY.md +0 -39
- data/vendor/assets/dashboard/assets/index-D2XR0iGw.js +0 -60
- data/vendor/assets/dashboard/assets/index-DlPr4YXw.css +0 -1
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'extension'
|
|
4
|
+
|
|
5
|
+
module Wurk
|
|
6
|
+
class Web
|
|
7
|
+
# Upstream-compatible class-level Rack surface (#204): Sidekiq apps do
|
|
8
|
+
# `run Sidekiq::Web`, `mount Sidekiq::Web => "/sidekiq"`, and rack-test
|
|
9
|
+
# ecosystem suites call `Sidekiq::Web.call(env)` directly. Wurk's full
|
|
10
|
+
# dashboard is the engine-mounted SPA; this standalone entry serves the
|
|
11
|
+
# registered third-party Web extensions (#187's renderer) under their own
|
|
12
|
+
# route paths — the surface ecosystem gems exercise. Non-extension paths
|
|
13
|
+
# 404 rather than half-serving the SPA without its engine.
|
|
14
|
+
#
|
|
15
|
+
# Same CSRF model as upstream Sidekiq 8 and ExtensionsController (spec
|
|
16
|
+
# §25.1): unsafe methods must carry `Sec-Fetch-Site: same-origin`.
|
|
17
|
+
SAFE_METHODS = %w[GET HEAD OPTIONS TRACE].freeze
|
|
18
|
+
NOT_FOUND_HEADERS = { 'Content-Type' => 'text/plain' }.freeze
|
|
19
|
+
|
|
20
|
+
class << self
|
|
21
|
+
# Class-level Rack entry — wraps host-registered middleware (`Wurk::Web.use`)
|
|
22
|
+
# around the extensions dispatcher. INTENTIONALLY bypasses
|
|
23
|
+
# `Wurk::Web::Authorization` (`config.authorized?` + `config.read_only?`):
|
|
24
|
+
# the full dashboard with its auth/read-only enforcement is the
|
|
25
|
+
# engine-mounted SPA wired in `lib/wurk/engine.rb`, and the engine's
|
|
26
|
+
# middleware stack already includes `Authorization`. This standalone
|
|
27
|
+
# surface exists for ecosystem rack-test consumers (`Sidekiq::Web.call`)
|
|
28
|
+
# whose mounted-app expectations cover extension routes only — wrapping
|
|
29
|
+
# auth here would break `run Sidekiq::Web` / rack-test parity with
|
|
30
|
+
# upstream Sidekiq, which also doesn't auth-gate `Sidekiq::Web.call`.
|
|
31
|
+
def call(env)
|
|
32
|
+
config.rack_app(method(:dispatch)).call(env)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def dispatch(env)
|
|
38
|
+
return [403, NOT_FOUND_HEADERS.dup, ['Forbidden']] unless safe_request?(env)
|
|
39
|
+
|
|
40
|
+
dispatch_to_extensions(env)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def safe_request?(env)
|
|
44
|
+
SAFE_METHODS.include?(env['REQUEST_METHOD']) ||
|
|
45
|
+
env['HTTP_SEC_FETCH_SITE'] == 'same-origin'
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# First extension whose routes answer the path wins; a per-extension
|
|
49
|
+
# "no such route" 404 keeps scanning so extensions can't shadow each
|
|
50
|
+
# other, and is returned only when nothing else matched.
|
|
51
|
+
def dispatch_to_extensions(env)
|
|
52
|
+
not_found = nil
|
|
53
|
+
config.extensions.each do |ext|
|
|
54
|
+
result = extension_result(ext, env)
|
|
55
|
+
next unless result
|
|
56
|
+
return rackify(result) unless result[0] == 404
|
|
57
|
+
|
|
58
|
+
not_found ||= result
|
|
59
|
+
end
|
|
60
|
+
rackify(not_found || [404, NOT_FOUND_HEADERS.dup, 'Not Found'])
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def extension_result(ext, env)
|
|
64
|
+
Extension::Renderer.call(
|
|
65
|
+
name: ext[:name], method: env['REQUEST_METHOD'],
|
|
66
|
+
subpath: env['PATH_INFO'].to_s, env: env, mount: env['SCRIPT_NAME'].to_s,
|
|
67
|
+
embed: false
|
|
68
|
+
)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Renderer returns [status, headers, String]; Rack wants an each-able body.
|
|
72
|
+
def rackify((status, headers, body))
|
|
73
|
+
[status, headers, body.is_a?(::Array) ? body : [body.to_s]]
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
data/lib/wurk/web.rb
CHANGED
data/lib/wurk/worker/setter.rb
CHANGED
|
@@ -49,7 +49,11 @@ module Wurk
|
|
|
49
49
|
'class' => @klass,
|
|
50
50
|
'args' => args
|
|
51
51
|
)
|
|
52
|
-
|
|
52
|
+
# Mirror client_push: a per-call `set(pool:)` selects the Redis pool and
|
|
53
|
+
# is removed so it never persists (normalize_item strips the class-level
|
|
54
|
+
# pool re-merged into each payload).
|
|
55
|
+
pool = merged.delete('pool') || @klass.get_sidekiq_options['pool']
|
|
56
|
+
@klass.build_client(pool).push_bulk(merged)
|
|
53
57
|
end
|
|
54
58
|
|
|
55
59
|
private
|
data/lib/wurk/worker.rb
CHANGED
|
@@ -117,12 +117,23 @@ module Wurk
|
|
|
117
117
|
def client_push(item)
|
|
118
118
|
raise ArgumentError, "Job arguments to #{name || self} must have string keys" if symbol_keyed?(item)
|
|
119
119
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
pool
|
|
125
|
-
|
|
120
|
+
# `pool` is a transient enqueue-time attribute: a per-call `set(pool:)`
|
|
121
|
+
# overrides the class-level option, then it's deleted so it never reaches
|
|
122
|
+
# the wire (normalize_item strips any class-level pool re-merged below).
|
|
123
|
+
pool = item.delete('pool') || get_sidekiq_options['pool']
|
|
124
|
+
build_client(pool, client_class: item.delete('client_class')).push(item)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# `client_class` swaps the enqueue client (e.g. TransactionAwareClient via
|
|
128
|
+
# Wurk.transactional_push!). Resolution order: per-call `set(client_class:)`,
|
|
129
|
+
# then the class option, then the live process default, then Wurk::Client.
|
|
130
|
+
# The default_job_options fallback keeps a global `transactional_push!`
|
|
131
|
+
# order-independent: a class whose options memoized before the opt-in (its
|
|
132
|
+
# inherited copy is a stale dup) still routes through the new client.
|
|
133
|
+
def build_client(pool = get_sidekiq_options['pool'], client_class: nil)
|
|
134
|
+
klass = client_class || get_sidekiq_options['client_class'] ||
|
|
135
|
+
Wurk.default_job_options['client_class'] || Wurk::Client
|
|
136
|
+
klass.new(pool: pool)
|
|
126
137
|
end
|
|
127
138
|
|
|
128
139
|
# --- Sidekiq::Testing class-level helpers (spec §24.3) --------------
|
data/lib/wurk.rb
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
require_relative 'wurk/version'
|
|
8
8
|
require_relative 'wurk/keys'
|
|
9
9
|
require_relative 'wurk/redis_pool'
|
|
10
|
+
require_relative 'wurk/redis_connection'
|
|
10
11
|
require_relative 'wurk/middleware'
|
|
11
12
|
require_relative 'wurk/middleware/chain'
|
|
12
13
|
require_relative 'wurk/component'
|
|
@@ -18,6 +19,7 @@ require_relative 'wurk/configuration'
|
|
|
18
19
|
require_relative 'wurk/job_util'
|
|
19
20
|
require_relative 'wurk/client'
|
|
20
21
|
require_relative 'wurk/client/buffered'
|
|
22
|
+
require_relative 'wurk/transaction_aware_client'
|
|
21
23
|
require_relative 'wurk/worker'
|
|
22
24
|
require_relative 'wurk/worker/setter'
|
|
23
25
|
require_relative 'wurk/job'
|
|
@@ -25,6 +27,7 @@ require_relative 'wurk/job/options'
|
|
|
25
27
|
require_relative 'wurk/queues'
|
|
26
28
|
require_relative 'wurk/testing'
|
|
27
29
|
require_relative 'wurk/iterable_job'
|
|
30
|
+
require_relative 'wurk/iterable_job_query'
|
|
28
31
|
require_relative 'wurk/job_retry'
|
|
29
32
|
require_relative 'wurk/job_record'
|
|
30
33
|
require_relative 'wurk/queue'
|
|
@@ -36,6 +39,8 @@ require_relative 'wurk/dead_set'
|
|
|
36
39
|
require_relative 'wurk/stats'
|
|
37
40
|
require_relative 'wurk/process_set'
|
|
38
41
|
require_relative 'wurk/work_set'
|
|
42
|
+
require_relative 'wurk/profiler'
|
|
43
|
+
require_relative 'wurk/profile_set'
|
|
39
44
|
require_relative 'wurk/heartbeat'
|
|
40
45
|
require_relative 'wurk/fetcher'
|
|
41
46
|
require_relative 'wurk/fetcher/reliable'
|
|
@@ -54,6 +59,7 @@ require_relative 'wurk/batch_set'
|
|
|
54
59
|
require_relative 'wurk/limiter'
|
|
55
60
|
require_relative 'wurk/cron'
|
|
56
61
|
require_relative 'wurk/leader'
|
|
62
|
+
require_relative 'wurk/history'
|
|
57
63
|
require_relative 'wurk/unique'
|
|
58
64
|
require_relative 'wurk/encryption'
|
|
59
65
|
require_relative 'wurk/metrics'
|
|
@@ -122,6 +128,10 @@ module Wurk
|
|
|
122
128
|
configuration.logger
|
|
123
129
|
end
|
|
124
130
|
|
|
131
|
+
def logger=(logger)
|
|
132
|
+
configuration.logger = logger
|
|
133
|
+
end
|
|
134
|
+
|
|
125
135
|
# --- JSON ---------------------------------------------------------
|
|
126
136
|
|
|
127
137
|
def load_json(string)
|
|
@@ -144,6 +154,13 @@ module Wurk
|
|
|
144
154
|
@default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
|
|
145
155
|
end
|
|
146
156
|
|
|
157
|
+
# Opt in to enqueue-after-commit globally: every `perform_async` builds a
|
|
158
|
+
# Wurk::TransactionAwareClient that defers its push to the surrounding
|
|
159
|
+
# ActiveRecord transaction's commit. Idempotent. Spec: sidekiq-free.md §3.
|
|
160
|
+
def transactional_push!
|
|
161
|
+
default_job_options['client_class'] = Wurk::TransactionAwareClient
|
|
162
|
+
end
|
|
163
|
+
|
|
147
164
|
# --- strict args -------------------------------------------------
|
|
148
165
|
|
|
149
166
|
# Sets the global mode used by Wurk::JobUtil#verify_json.
|
|
@@ -174,6 +191,17 @@ module Wurk
|
|
|
174
191
|
|
|
175
192
|
attr_writer :server
|
|
176
193
|
|
|
194
|
+
# Enter server mode: set the module flag (read by third-party gems via
|
|
195
|
+
# `Sidekiq.server?`) AND the per-config `server?` predicate that gates
|
|
196
|
+
# `configure_server`. Both must be set before the app's initializers run,
|
|
197
|
+
# or `configure_server` blocks (server middleware, error handlers,
|
|
198
|
+
# lifecycle hooks) are silently skipped. Defaults to the global config; the
|
|
199
|
+
# CLI passes its own (possibly test-injected) Configuration instance.
|
|
200
|
+
def enter_server_mode(config = configuration)
|
|
201
|
+
@server = true
|
|
202
|
+
config[:server] = true
|
|
203
|
+
end
|
|
204
|
+
|
|
177
205
|
# Wurk ships Pro+Ent features in the free gem; these flags exist solely
|
|
178
206
|
# for third-party gems that branch on Sidekiq.pro? / Sidekiq.ent?.
|
|
179
207
|
def pro?
|
|
@@ -211,6 +239,15 @@ require_relative 'wurk/health'
|
|
|
211
239
|
# Spec: docs/target/sidekiq-pro.md §3.2.
|
|
212
240
|
require_relative 'wurk/middleware/poison_pill'
|
|
213
241
|
|
|
242
|
+
# InterruptHandler self-prepends to the head of the server chain so a
|
|
243
|
+
# cooperatively-cancelled job (IterableJob) is re-pushed + cleanly skipped
|
|
244
|
+
# instead of surfacing as a raw error. Sidekiq registers it from
|
|
245
|
+
# `require "sidekiq/cli"`; we require it here at load so every server boot
|
|
246
|
+
# path picks it up — standalone CLI, embedded, and swarm-forked children
|
|
247
|
+
# (the swarm and embedded paths never run through Wurk::CLI#run).
|
|
248
|
+
# Spec: docs/target/sidekiq-free.md §10.3.
|
|
249
|
+
require_relative 'wurk/middleware/interrupt_handler'
|
|
250
|
+
|
|
214
251
|
# Limiter server middleware: catches OverLimit, reschedules onto the same
|
|
215
252
|
# queue with `Time.now + backoff` until `overrated` hits the reschedule cap.
|
|
216
253
|
# Registered AFTER Batch::ServerMiddleware so a rescheduled OverLimit
|
|
@@ -226,6 +263,13 @@ Wurk.configuration.server_middleware.add(Wurk::Limiter::ServerMiddleware)
|
|
|
226
263
|
# Spec: docs/target/sidekiq-pro.md §9.
|
|
227
264
|
Wurk.configuration.server_middleware.add(Wurk::Metrics::Statsd)
|
|
228
265
|
|
|
266
|
+
# Historical metrics middleware: records per-class processed/failed/ms into
|
|
267
|
+
# Redis time-buckets so the dashboard history pane has data on a default boot.
|
|
268
|
+
# Sidekiq auto-installs `Sidekiq::Metrics::Middleware` on the server for
|
|
269
|
+
# embedded/CLI; we mirror that here at load. Runs server-side only (the chain
|
|
270
|
+
# never executes in a client-only process). Spec: docs/target/sidekiq-free.md §10.3.
|
|
271
|
+
Wurk.configuration.server_middleware.add(Wurk::Metrics::History)
|
|
272
|
+
|
|
229
273
|
# Pro Fast API: Lua-backed Queue#delete_job / #delete_by_class plus
|
|
230
274
|
# SortedSet#scan { |JobRecord| … }. Mixed in via include/prepend on the
|
|
231
275
|
# existing data API classes so the surface is wire-compat with Sidekiq Pro.
|
|
Binary file
|
|
Binary file
|
|
Binary file
|