wurk 0.0.4 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +16 -2
  3. data/app/controllers/wurk/api/serializers.rb +48 -2
  4. data/app/controllers/wurk/api_controller.rb +216 -1
  5. data/app/controllers/wurk/dashboard_controller.rb +20 -2
  6. data/app/controllers/wurk/extensions_controller.rb +56 -0
  7. data/app/controllers/wurk/profiles_controller.rb +68 -0
  8. data/config/routes.rb +54 -1
  9. data/exe/sidekiqswarm +8 -0
  10. data/exe/wurkswarm +23 -0
  11. data/lib/active_job/queue_adapters/wurk_adapter.rb +35 -0
  12. data/lib/generators/wurk/install/templates/wurk.rb +14 -3
  13. data/lib/sidekiq/api.rb +4 -0
  14. data/lib/sidekiq/cli.rb +9 -0
  15. data/lib/sidekiq/client.rb +4 -0
  16. data/lib/sidekiq/job.rb +4 -0
  17. data/lib/sidekiq/launcher.rb +4 -0
  18. data/lib/sidekiq/middleware/chain.rb +4 -0
  19. data/lib/sidekiq/middleware/server/statsd.rb +12 -0
  20. data/lib/sidekiq/rails.rb +10 -0
  21. data/lib/sidekiq/redis_connection.rb +4 -0
  22. data/lib/sidekiq/scheduled.rb +4 -0
  23. data/lib/sidekiq/testing.rb +4 -0
  24. data/lib/sidekiq/version.rb +4 -0
  25. data/lib/sidekiq/web.rb +4 -0
  26. data/lib/sidekiq/worker.rb +4 -0
  27. data/lib/sidekiq.rb +16 -0
  28. data/lib/wurk/batch/callbacks.rb +103 -13
  29. data/lib/wurk/batch/death_handler.rb +5 -2
  30. data/lib/wurk/batch/server_middleware.rb +35 -3
  31. data/lib/wurk/batch/status.rb +9 -0
  32. data/lib/wurk/batch.rb +23 -1
  33. data/lib/wurk/capsule.rb +20 -1
  34. data/lib/wurk/cli.rb +84 -1
  35. data/lib/wurk/client.rb +20 -17
  36. data/lib/wurk/compat.rb +44 -2
  37. data/lib/wurk/component.rb +5 -4
  38. data/lib/wurk/configuration.rb +120 -3
  39. data/lib/wurk/cron.rb +51 -9
  40. data/lib/wurk/dead_set.rb +8 -3
  41. data/lib/wurk/deploy.rb +8 -4
  42. data/lib/wurk/encryption.rb +6 -1
  43. data/lib/wurk/fetcher/reaper.rb +78 -11
  44. data/lib/wurk/fetcher/reliable.rb +14 -4
  45. data/lib/wurk/heartbeat.rb +45 -0
  46. data/lib/wurk/history.rb +174 -0
  47. data/lib/wurk/iterable_job/active_record_enumerator.rb +71 -0
  48. data/lib/wurk/iterable_job/csv_enumerator.rb +51 -0
  49. data/lib/wurk/iterable_job.rb +41 -0
  50. data/lib/wurk/iterable_job_query.rb +75 -0
  51. data/lib/wurk/job.rb +8 -0
  52. data/lib/wurk/job_record.rb +16 -1
  53. data/lib/wurk/job_set.rb +4 -4
  54. data/lib/wurk/job_util.rb +15 -6
  55. data/lib/wurk/keys.rb +10 -0
  56. data/lib/wurk/launcher.rb +35 -1
  57. data/lib/wurk/leader.rb +15 -6
  58. data/lib/wurk/limiter/bucket.rb +14 -3
  59. data/lib/wurk/limiter/concurrent.rb +1 -1
  60. data/lib/wurk/limiter/window.rb +2 -1
  61. data/lib/wurk/limiter.rb +12 -0
  62. data/lib/wurk/lua/loader.rb +10 -0
  63. data/lib/wurk/lua.rb +106 -14
  64. data/lib/wurk/metrics/history.rb +5 -0
  65. data/lib/wurk/metrics/query.rb +39 -0
  66. data/lib/wurk/metrics/queue_rollup.rb +151 -0
  67. data/lib/wurk/metrics/statsd.rb +11 -0
  68. data/lib/wurk/middleware/current_attributes.rb +29 -6
  69. data/lib/wurk/middleware/interrupt_handler.rb +5 -0
  70. data/lib/wurk/middleware/poison_pill.rb +35 -5
  71. data/lib/wurk/processor.rb +17 -8
  72. data/lib/wurk/profile_set.rb +65 -0
  73. data/lib/wurk/profiler.rb +127 -0
  74. data/lib/wurk/railtie.rb +19 -5
  75. data/lib/wurk/redis_client_adapter.rb +72 -0
  76. data/lib/wurk/redis_connection.rb +30 -0
  77. data/lib/wurk/redis_pool.rb +5 -1
  78. data/lib/wurk/scheduled.rb +42 -0
  79. data/lib/wurk/sorted_entry.rb +13 -11
  80. data/lib/wurk/stats.rb +11 -4
  81. data/lib/wurk/swarm/child_boot.rb +26 -4
  82. data/lib/wurk/swarm.rb +1 -1
  83. data/lib/wurk/transaction_aware_client.rb +69 -0
  84. data/lib/wurk/unique.rb +49 -7
  85. data/lib/wurk/version.rb +1 -1
  86. data/lib/wurk/web/batch_status.rb +42 -0
  87. data/lib/wurk/web/config.rb +219 -17
  88. data/lib/wurk/web/enterprise.rb +14 -0
  89. data/lib/wurk/web/extension.rb +348 -0
  90. data/lib/wurk/web/rack_app.rb +77 -0
  91. data/lib/wurk/web.rb +2 -0
  92. data/lib/wurk/worker/setter.rb +5 -1
  93. data/lib/wurk/worker.rb +17 -6
  94. data/lib/wurk.rb +44 -0
  95. data/vendor/assets/dashboard/assets/fa-brands-400-BP5tdqmh.woff2 +0 -0
  96. data/vendor/assets/dashboard/assets/fa-regular-400-nyy7hhHF.woff2 +0 -0
  97. data/vendor/assets/dashboard/assets/fa-solid-900-DRAAbZTg.woff2 +0 -0
  98. data/vendor/assets/dashboard/assets/index-9CFRWpfG.js +77 -0
  99. data/vendor/assets/dashboard/assets/index-CW8AFQIv.css +2 -0
  100. data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
  101. data/vendor/assets/dashboard/favicon.png +0 -0
  102. data/vendor/assets/dashboard/index.html +10 -3
  103. data/vendor/assets/dashboard/wurk-manifest.json +2 -2
  104. metadata +42 -6
  105. data/CHANGELOG.md +0 -67
  106. data/CONTRIBUTING.md +0 -73
  107. data/SECURITY.md +0 -39
  108. data/vendor/assets/dashboard/assets/index-D2XR0iGw.js +0 -60
  109. data/vendor/assets/dashboard/assets/index-DlPr4YXw.css +0 -1
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'extension'
4
+
5
+ module Wurk
6
+ class Web
7
+ # Upstream-compatible class-level Rack surface (#204): Sidekiq apps do
8
+ # `run Sidekiq::Web`, `mount Sidekiq::Web => "/sidekiq"`, and rack-test
9
+ # ecosystem suites call `Sidekiq::Web.call(env)` directly. Wurk's full
10
+ # dashboard is the engine-mounted SPA; this standalone entry serves the
11
+ # registered third-party Web extensions (#187's renderer) under their own
12
+ # route paths — the surface ecosystem gems exercise. Non-extension paths
13
+ # 404 rather than half-serving the SPA without its engine.
14
+ #
15
+ # Same CSRF model as upstream Sidekiq 8 and ExtensionsController (spec
16
+ # §25.1): unsafe methods must carry `Sec-Fetch-Site: same-origin`.
17
+ SAFE_METHODS = %w[GET HEAD OPTIONS TRACE].freeze
18
+ NOT_FOUND_HEADERS = { 'Content-Type' => 'text/plain' }.freeze
19
+
20
+ class << self
21
+ # Class-level Rack entry — wraps host-registered middleware (`Wurk::Web.use`)
22
+ # around the extensions dispatcher. INTENTIONALLY bypasses
23
+ # `Wurk::Web::Authorization` (`config.authorized?` + `config.read_only?`):
24
+ # the full dashboard with its auth/read-only enforcement is the
25
+ # engine-mounted SPA wired in `lib/wurk/engine.rb`, and the engine's
26
+ # middleware stack already includes `Authorization`. This standalone
27
+ # surface exists for ecosystem rack-test consumers (`Sidekiq::Web.call`)
28
+ # whose mounted-app expectations cover extension routes only — wrapping
29
+ # auth here would break `run Sidekiq::Web` / rack-test parity with
30
+ # upstream Sidekiq, which also doesn't auth-gate `Sidekiq::Web.call`.
31
+ def call(env)
32
+ config.rack_app(method(:dispatch)).call(env)
33
+ end
34
+
35
+ private
36
+
37
+ def dispatch(env)
38
+ return [403, NOT_FOUND_HEADERS.dup, ['Forbidden']] unless safe_request?(env)
39
+
40
+ dispatch_to_extensions(env)
41
+ end
42
+
43
+ def safe_request?(env)
44
+ SAFE_METHODS.include?(env['REQUEST_METHOD']) ||
45
+ env['HTTP_SEC_FETCH_SITE'] == 'same-origin'
46
+ end
47
+
48
+ # First extension whose routes answer the path wins; a per-extension
49
+ # "no such route" 404 keeps scanning so extensions can't shadow each
50
+ # other, and is returned only when nothing else matched.
51
+ def dispatch_to_extensions(env)
52
+ not_found = nil
53
+ config.extensions.each do |ext|
54
+ result = extension_result(ext, env)
55
+ next unless result
56
+ return rackify(result) unless result[0] == 404
57
+
58
+ not_found ||= result
59
+ end
60
+ rackify(not_found || [404, NOT_FOUND_HEADERS.dup, 'Not Found'])
61
+ end
62
+
63
+ def extension_result(ext, env)
64
+ Extension::Renderer.call(
65
+ name: ext[:name], method: env['REQUEST_METHOD'],
66
+ subpath: env['PATH_INFO'].to_s, env: env, mount: env['SCRIPT_NAME'].to_s,
67
+ embed: false
68
+ )
69
+ end
70
+
71
+ # Renderer returns [status, headers, String]; Rack wants an each-able body.
72
+ def rackify((status, headers, body))
73
+ [status, headers, body.is_a?(::Array) ? body : [body.to_s]]
74
+ end
75
+ end
76
+ end
77
+ end
data/lib/wurk/web.rb CHANGED
@@ -3,6 +3,8 @@
3
3
  require_relative 'web/config'
4
4
  require_relative 'web/search'
5
5
  require_relative 'web/enterprise'
6
+ require_relative 'web/batch_status'
7
+ require_relative 'web/rack_app'
6
8
 
7
9
  module Wurk
8
10
  # Web UI namespace. Holds three sibling concerns:
@@ -49,7 +49,11 @@ module Wurk
49
49
  'class' => @klass,
50
50
  'args' => args
51
51
  )
52
- @klass.build_client.push_bulk(merged)
52
+ # Mirror client_push: a per-call `set(pool:)` selects the Redis pool and
53
+ # is removed so it never persists (normalize_item strips the class-level
54
+ # pool re-merged into each payload).
55
+ pool = merged.delete('pool') || @klass.get_sidekiq_options['pool']
56
+ @klass.build_client(pool).push_bulk(merged)
53
57
  end
54
58
 
55
59
  private
data/lib/wurk/worker.rb CHANGED
@@ -117,12 +117,23 @@ module Wurk
117
117
  def client_push(item)
118
118
  raise ArgumentError, "Job arguments to #{name || self} must have string keys" if symbol_keyed?(item)
119
119
 
120
- build_client.push(item)
121
- end
122
-
123
- def build_client
124
- pool = get_sidekiq_options['pool']
125
- Wurk::Client.new(pool: pool)
120
+ # `pool` is a transient enqueue-time attribute: a per-call `set(pool:)`
121
+ # overrides the class-level option, then it's deleted so it never reaches
122
+ # the wire (normalize_item strips any class-level pool re-merged below).
123
+ pool = item.delete('pool') || get_sidekiq_options['pool']
124
+ build_client(pool, client_class: item.delete('client_class')).push(item)
125
+ end
126
+
127
+ # `client_class` swaps the enqueue client (e.g. TransactionAwareClient via
128
+ # Wurk.transactional_push!). Resolution order: per-call `set(client_class:)`,
129
+ # then the class option, then the live process default, then Wurk::Client.
130
+ # The default_job_options fallback keeps a global `transactional_push!`
131
+ # order-independent: a class whose options memoized before the opt-in (its
132
+ # inherited copy is a stale dup) still routes through the new client.
133
+ def build_client(pool = get_sidekiq_options['pool'], client_class: nil)
134
+ klass = client_class || get_sidekiq_options['client_class'] ||
135
+ Wurk.default_job_options['client_class'] || Wurk::Client
136
+ klass.new(pool: pool)
126
137
  end
127
138
 
128
139
  # --- Sidekiq::Testing class-level helpers (spec §24.3) --------------
data/lib/wurk.rb CHANGED
@@ -7,6 +7,7 @@
7
7
  require_relative 'wurk/version'
8
8
  require_relative 'wurk/keys'
9
9
  require_relative 'wurk/redis_pool'
10
+ require_relative 'wurk/redis_connection'
10
11
  require_relative 'wurk/middleware'
11
12
  require_relative 'wurk/middleware/chain'
12
13
  require_relative 'wurk/component'
@@ -18,6 +19,7 @@ require_relative 'wurk/configuration'
18
19
  require_relative 'wurk/job_util'
19
20
  require_relative 'wurk/client'
20
21
  require_relative 'wurk/client/buffered'
22
+ require_relative 'wurk/transaction_aware_client'
21
23
  require_relative 'wurk/worker'
22
24
  require_relative 'wurk/worker/setter'
23
25
  require_relative 'wurk/job'
@@ -25,6 +27,7 @@ require_relative 'wurk/job/options'
25
27
  require_relative 'wurk/queues'
26
28
  require_relative 'wurk/testing'
27
29
  require_relative 'wurk/iterable_job'
30
+ require_relative 'wurk/iterable_job_query'
28
31
  require_relative 'wurk/job_retry'
29
32
  require_relative 'wurk/job_record'
30
33
  require_relative 'wurk/queue'
@@ -36,6 +39,8 @@ require_relative 'wurk/dead_set'
36
39
  require_relative 'wurk/stats'
37
40
  require_relative 'wurk/process_set'
38
41
  require_relative 'wurk/work_set'
42
+ require_relative 'wurk/profiler'
43
+ require_relative 'wurk/profile_set'
39
44
  require_relative 'wurk/heartbeat'
40
45
  require_relative 'wurk/fetcher'
41
46
  require_relative 'wurk/fetcher/reliable'
@@ -54,6 +59,7 @@ require_relative 'wurk/batch_set'
54
59
  require_relative 'wurk/limiter'
55
60
  require_relative 'wurk/cron'
56
61
  require_relative 'wurk/leader'
62
+ require_relative 'wurk/history'
57
63
  require_relative 'wurk/unique'
58
64
  require_relative 'wurk/encryption'
59
65
  require_relative 'wurk/metrics'
@@ -122,6 +128,10 @@ module Wurk
122
128
  configuration.logger
123
129
  end
124
130
 
131
+ def logger=(logger)
132
+ configuration.logger = logger
133
+ end
134
+
125
135
  # --- JSON ---------------------------------------------------------
126
136
 
127
137
  def load_json(string)
@@ -144,6 +154,13 @@ module Wurk
144
154
  @default_job_options = default_job_options.merge(hash.transform_keys(&:to_s))
145
155
  end
146
156
 
157
+ # Opt in to enqueue-after-commit globally: every `perform_async` builds a
158
+ # Wurk::TransactionAwareClient that defers its push to the surrounding
159
+ # ActiveRecord transaction's commit. Idempotent. Spec: sidekiq-free.md §3.
160
+ def transactional_push!
161
+ default_job_options['client_class'] = Wurk::TransactionAwareClient
162
+ end
163
+
147
164
  # --- strict args -------------------------------------------------
148
165
 
149
166
  # Sets the global mode used by Wurk::JobUtil#verify_json.
@@ -174,6 +191,17 @@ module Wurk
174
191
 
175
192
  attr_writer :server
176
193
 
194
+ # Enter server mode: set the module flag (read by third-party gems via
195
+ # `Sidekiq.server?`) AND the per-config `server?` predicate that gates
196
+ # `configure_server`. Both must be set before the app's initializers run,
197
+ # or `configure_server` blocks (server middleware, error handlers,
198
+ # lifecycle hooks) are silently skipped. Defaults to the global config; the
199
+ # CLI passes its own (possibly test-injected) Configuration instance.
200
+ def enter_server_mode(config = configuration)
201
+ @server = true
202
+ config[:server] = true
203
+ end
204
+
177
205
  # Wurk ships Pro+Ent features in the free gem; these flags exist solely
178
206
  # for third-party gems that branch on Sidekiq.pro? / Sidekiq.ent?.
179
207
  def pro?
@@ -211,6 +239,15 @@ require_relative 'wurk/health'
211
239
  # Spec: docs/target/sidekiq-pro.md §3.2.
212
240
  require_relative 'wurk/middleware/poison_pill'
213
241
 
242
+ # InterruptHandler self-prepends to the head of the server chain so a
243
+ # cooperatively-cancelled job (IterableJob) is re-pushed + cleanly skipped
244
+ # instead of surfacing as a raw error. Sidekiq registers it from
245
+ # `require "sidekiq/cli"`; we require it here at load so every server boot
246
+ # path picks it up — standalone CLI, embedded, and swarm-forked children
247
+ # (the swarm and embedded paths never run through Wurk::CLI#run).
248
+ # Spec: docs/target/sidekiq-free.md §10.3.
249
+ require_relative 'wurk/middleware/interrupt_handler'
250
+
214
251
  # Limiter server middleware: catches OverLimit, reschedules onto the same
215
252
  # queue with `Time.now + backoff` until `overrated` hits the reschedule cap.
216
253
  # Registered AFTER Batch::ServerMiddleware so a rescheduled OverLimit
@@ -226,6 +263,13 @@ Wurk.configuration.server_middleware.add(Wurk::Limiter::ServerMiddleware)
226
263
  # Spec: docs/target/sidekiq-pro.md §9.
227
264
  Wurk.configuration.server_middleware.add(Wurk::Metrics::Statsd)
228
265
 
266
+ # Historical metrics middleware: records per-class processed/failed/ms into
267
+ # Redis time-buckets so the dashboard history pane has data on a default boot.
268
+ # Sidekiq auto-installs `Sidekiq::Metrics::Middleware` on the server for
269
+ # embedded/CLI; we mirror that here at load. Runs server-side only (the chain
270
+ # never executes in a client-only process). Spec: docs/target/sidekiq-free.md §10.3.
271
+ Wurk.configuration.server_middleware.add(Wurk::Metrics::History)
272
+
229
273
  # Pro Fast API: Lua-backed Queue#delete_job / #delete_by_class plus
230
274
  # SortedSet#scan { |JobRecord| … }. Mixed in via include/prepend on the
231
275
  # existing data API classes so the surface is wire-compat with Sidekiq Pro.