wurk 1.13.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/concerns/wurk/sse_streaming.rb +36 -1
- data/app/controllers/wurk/api/pagination.rb +9 -7
- data/app/controllers/wurk/api/serializers.rb +8 -2
- data/app/controllers/wurk/api_controller.rb +57 -24
- data/app/controllers/wurk/application_controller.rb +0 -2
- data/app/controllers/wurk/dashboard_controller.rb +0 -1
- data/config/routes.rb +5 -1
- data/lib/sidekiq/cli.rb +4 -3
- data/lib/sidekiq/deploy.rb +2 -1
- data/lib/sidekiq/embedded.rb +2 -1
- data/lib/sidekiq/iterable_job.rb +2 -1
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable/csv_enumerator.rb +2 -1
- data/lib/sidekiq/job/iterable.rb +2 -1
- data/lib/sidekiq/launcher.rb +2 -1
- data/lib/sidekiq/manager.rb +2 -1
- data/lib/sidekiq/metrics/query.rb +2 -1
- data/lib/sidekiq/pro/web.rb +2 -1
- data/lib/sidekiq/scheduled.rb +2 -1
- data/lib/sidekiq/web.rb +2 -1
- data/lib/sidekiq.rb +3 -2
- data/lib/wurk/api/fast.rb +4 -18
- data/lib/wurk/api/queues.rb +12 -10
- data/lib/wurk/api/serializers.rb +6 -7
- data/lib/wurk/api/swarm.rb +0 -1
- data/lib/wurk/batch/server_middleware.rb +4 -0
- data/lib/wurk/batch/status.rb +45 -22
- data/lib/wurk/batch_set.rb +34 -0
- data/lib/wurk/cli.rb +17 -19
- data/lib/wurk/client/buffered.rb +97 -26
- data/lib/wurk/client.rb +21 -1
- data/lib/wurk/compat/deploy.rb +7 -0
- data/lib/wurk/compat/encryption.rb +7 -0
- data/lib/wurk/compat/flow.rb +8 -0
- data/lib/wurk/compat/iterable_job.rb +8 -0
- data/lib/wurk/compat/profile_set.rb +8 -0
- data/lib/wurk/compat/server.rb +15 -0
- data/lib/wurk/compat/web.rb +20 -0
- data/lib/wurk/compat.rb +34 -40
- data/lib/wurk/configuration.rb +46 -9
- data/lib/wurk/cron.rb +146 -47
- data/lib/wurk/dead_set.rb +10 -6
- data/lib/wurk/embedded.rb +0 -2
- data/lib/wurk/engine.rb +46 -4
- data/lib/wurk/error_coalescer.rb +21 -1
- data/lib/wurk/fetcher/many_queues.rb +8 -6
- data/lib/wurk/fetcher/poison_pill.rb +1 -1
- data/lib/wurk/fetcher/reaper.rb +0 -5
- data/lib/wurk/fetcher/reliable.rb +46 -27
- data/lib/wurk/fetcher/self_reconcile.rb +0 -1
- data/lib/wurk/fetcher/unit_of_work.rb +16 -1
- data/lib/wurk/flow/status.rb +10 -4
- data/lib/wurk/flow_set.rb +24 -2
- data/lib/wurk/health.rb +0 -1
- data/lib/wurk/heartbeat.rb +13 -7
- data/lib/wurk/import/sidekiq_cron.rb +0 -1
- data/lib/wurk/job_record.rb +2 -1
- data/lib/wurk/job_retry.rb +1 -1
- data/lib/wurk/job_set.rb +132 -6
- data/lib/wurk/launcher.rb +0 -12
- data/lib/wurk/lazy_loader.rb +60 -0
- data/lib/wurk/limiter/bucket.rb +0 -2
- data/lib/wurk/limiter/concurrent.rb +19 -36
- data/lib/wurk/limiter/leaky.rb +0 -2
- data/lib/wurk/limiter/points.rb +0 -2
- data/lib/wurk/limiter/server_middleware.rb +4 -1
- data/lib/wurk/limiter/unlimited.rb +0 -2
- data/lib/wurk/limiter/window.rb +0 -2
- data/lib/wurk/limiter.rb +0 -7
- data/lib/wurk/logger.rb +44 -5
- data/lib/wurk/lua/fast_delete.rb +58 -0
- data/lib/wurk/lua/fast_delete_window.lua +46 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +10 -3
- data/lib/wurk/lua/limiter_concurrent_release.lua +13 -4
- data/lib/wurk/lua.rb +1 -41
- data/lib/wurk/metrics/dashboard_series.rb +38 -18
- data/lib/wurk/metrics/history.rb +24 -11
- data/lib/wurk/metrics/prometheus.rb +0 -1
- data/lib/wurk/metrics/query.rb +0 -2
- data/lib/wurk/metrics/queue_rollup.rb +33 -11
- data/lib/wurk/metrics/statsd.rb +4 -0
- data/lib/wurk/middleware/chain.rb +25 -3
- data/lib/wurk/middleware/expiry.rb +4 -0
- data/lib/wurk/middleware/interrupt_handler.rb +4 -1
- data/lib/wurk/middleware/status.rb +4 -0
- data/lib/wurk/middleware/timeout.rb +4 -0
- data/lib/wurk/preflight/owners.rb +0 -1
- data/lib/wurk/preflight/payload_sample.rb +0 -1
- data/lib/wurk/preflight/verify.rb +0 -2
- data/lib/wurk/process_set.rb +6 -1
- data/lib/wurk/profiler.rb +2 -1
- data/lib/wurk/queue.rb +33 -6
- data/lib/wurk/queue_consumers.rb +10 -1
- data/lib/wurk/rails_boot.rb +2 -2
- data/lib/wurk/railtie.rb +3 -4
- data/lib/wurk/sentry/error_handler.rb +5 -0
- data/lib/wurk/sentry/middleware.rb +4 -0
- data/lib/wurk/sorted_entry.rb +16 -12
- data/lib/wurk/stats.rb +85 -1
- data/lib/wurk/swarm/child_boot.rb +0 -2
- data/lib/wurk/swarm/liveness.rb +0 -2
- data/lib/wurk/swarm/memory_watch.rb +116 -0
- data/lib/wurk/swarm/prefork.rb +54 -0
- data/lib/wurk/swarm.rb +33 -42
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/web/enterprise.rb +63 -7
- data/lib/wurk.rb +33 -29
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js +1 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js +1 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +2 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowState-yb7virqR.js → FlowState-ClzInw0-.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js +2 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js +1 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js +1 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js +1 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +1 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js +1 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +1 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +1 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +1 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js +1 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js +141 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +1 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +1 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +1 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{utils-CnWZzW2L.js → utils-69XSoGbQ.js} +1 -1
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/wurk-logo-ydeSB9B8.png +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +1 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
- data/vendor/assets/dashboard/index.html +3 -23
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +122 -37
- data/vendor/assets/dashboard/assets/ArgsValue-BJol9S0s.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-D_zGQ9Sd.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Dt_xC660.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-CWRthsmc.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-qeGr2YAm.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DQWaAwPA.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-D9gOPHtK.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DONXVLdA.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-BbB6gXHp.js +0 -2
- data/vendor/assets/dashboard/assets/FlowDetail-CrmnVRp0.js +0 -1
- data/vendor/assets/dashboard/assets/Flows-g4GC8KmE.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-C86k_1wj.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-zyO-iMlK.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-Ckq5fNBv.js +0 -1
- data/vendor/assets/dashboard/assets/NotFound-C8NnyjlY.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-vWeBbSPF.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-DRl-FXPA.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-DkXWrihA.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-CmlojYIy.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-Cbbpp3xw.js +0 -1
- data/vendor/assets/dashboard/assets/Search-LaEpj2fM.js +0 -1
- data/vendor/assets/dashboard/assets/charts-D_PQLoyc.js +0 -1
- data/vendor/assets/dashboard/assets/fa-brands-400-Bs6tcqqs.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-regular-400-DRN8N0d1.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-solid-900-IAB4Droh.woff2 +0 -0
- data/vendor/assets/dashboard/assets/fa-v4compatibility-CErXDOsT.woff2 +0 -0
- data/vendor/assets/dashboard/assets/i18n-CBewVzQ_.js +0 -1
- data/vendor/assets/dashboard/assets/index--xvvNlYx.css +0 -1
- data/vendor/assets/dashboard/assets/index-FBKGtysy.js +0 -141
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-BiwhHqgh.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-jBYVwXvg.js +0 -1
- data/vendor/assets/dashboard/assets/wurk-logo-Vy3xW4K0.png +0 -0
- data/vendor/assets/dashboard/og-image.png +0 -0
- data/vendor/assets/dashboard/wurk-logo.webp +0 -0
data/lib/wurk/batch/status.rb
CHANGED
|
@@ -18,11 +18,19 @@ module Wurk
|
|
|
18
18
|
|
|
19
19
|
attr_reader :bid
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
# `data:` is the `b-<bid>` hash already read, by a listing that fetched a
|
|
22
|
+
# page of them in one pipeline (BatchSet#snapshot); without it the hash
|
|
23
|
+
# is read here.
|
|
24
|
+
def initialize(bid, data: nil)
|
|
22
25
|
raise ArgumentError, 'bid required' if bid.nil? || bid.to_s.empty?
|
|
23
26
|
|
|
24
27
|
@bid = bid.to_s
|
|
25
|
-
reload!
|
|
28
|
+
data ? self.data_hash = data : reload!
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# The four set sizes #summary takes, as one pipeline's commands.
|
|
32
|
+
def self.summary_commands(bid)
|
|
33
|
+
%w[failed died kids jids].map { |set| ['SCARD', "b-#{bid}-#{set}"] }
|
|
26
34
|
end
|
|
27
35
|
|
|
28
36
|
# False when no `b-<bid>` hash exists — a well-formed bid that was never
|
|
@@ -87,24 +95,18 @@ module Wurk
|
|
|
87
95
|
# set reads it needs ride one pipeline.
|
|
88
96
|
def data
|
|
89
97
|
failed, dead, kids, live = set_snapshot
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
'parent_bid' => parent_bid,
|
|
103
|
-
'tags' => tags,
|
|
104
|
-
'failed_jids' => failed,
|
|
105
|
-
'dead_jids' => dead,
|
|
106
|
-
'child_count' => kids.to_i
|
|
107
|
-
}
|
|
98
|
+
header(live).merge('failed_jids' => failed, 'dead_jids' => dead, 'child_count' => kids.to_i)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# #data for a listing row: the failed and dead jid sets by size rather
|
|
102
|
+
# than by member, so a page of batches with ten thousand failures each
|
|
103
|
+
# is 25 rows and not 250,000 jids. `sizes` are the replies to
|
|
104
|
+
# .summary_commands when the caller pipelined them for a whole page.
|
|
105
|
+
def summary(sizes = nil)
|
|
106
|
+
failed, dead, kids, live = sizes || Wurk.redis do |conn|
|
|
107
|
+
conn.pipelined { |pipe| Status.summary_commands(@bid).each { |cmd| pipe.call(*cmd) } }
|
|
108
|
+
end
|
|
109
|
+
header(live).merge('failed_count' => failed.to_i, 'dead_count' => dead.to_i, 'child_count' => kids.to_i)
|
|
108
110
|
end
|
|
109
111
|
|
|
110
112
|
# Blocks the current thread until `complete?` is true. Test/util only —
|
|
@@ -139,13 +141,34 @@ module Wurk
|
|
|
139
141
|
end
|
|
140
142
|
|
|
141
143
|
def reload!
|
|
142
|
-
|
|
143
|
-
@data = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
|
|
144
|
+
self.data_hash = Wurk.redis { |conn| conn.call('HGETALL', "b-#{@bid}") }
|
|
144
145
|
self
|
|
145
146
|
end
|
|
146
147
|
|
|
147
148
|
private
|
|
148
149
|
|
|
150
|
+
def header(live)
|
|
151
|
+
{
|
|
152
|
+
'bid' => @bid,
|
|
153
|
+
'total' => total,
|
|
154
|
+
'pending' => pending,
|
|
155
|
+
'failures' => failures,
|
|
156
|
+
'created_at' => created_at,
|
|
157
|
+
'complete_at' => complete_at,
|
|
158
|
+
'success_at' => success_at,
|
|
159
|
+
'death_at' => death_at,
|
|
160
|
+
'complete' => complete_with?(live.to_i),
|
|
161
|
+
'invalidated' => invalidated?,
|
|
162
|
+
'description' => description,
|
|
163
|
+
'parent_bid' => parent_bid,
|
|
164
|
+
'tags' => tags
|
|
165
|
+
}
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def data_hash=(raw)
|
|
169
|
+
@data = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
|
|
170
|
+
end
|
|
171
|
+
|
|
149
172
|
# [failed jids, dead jids, child count, live count] in one round trip.
|
|
150
173
|
def set_snapshot
|
|
151
174
|
Wurk.redis do |conn|
|
data/lib/wurk/batch_set.rb
CHANGED
|
@@ -38,6 +38,20 @@ module Wurk
|
|
|
38
38
|
end
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
+
# The set's size and the #each-ordered page of `count` batches from
|
|
42
|
+
# `offset`, each as its Status#summary: the page's ids in one round trip
|
|
43
|
+
# and every row's hash and set sizes in a second, where #each pays a round
|
|
44
|
+
# trip per batch from the top of the set down. Internal reader for the
|
|
45
|
+
# dashboard.
|
|
46
|
+
#
|
|
47
|
+
# @return [Array(Integer, Array<Hash>)]
|
|
48
|
+
def snapshot(offset, count)
|
|
49
|
+
Wurk.redis do |conn|
|
|
50
|
+
size, bids = page_ids(conn, offset, count)
|
|
51
|
+
[size.to_i, summaries(conn, bids)]
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
41
55
|
# Yields each BID indexed under the given tag (set at `tags:<tag>`).
|
|
42
56
|
# No JSON parsing; cheap iteration. Caller wraps in `Status.new(bid)`
|
|
43
57
|
# if it needs full state.
|
|
@@ -53,6 +67,26 @@ module Wurk
|
|
|
53
67
|
end
|
|
54
68
|
end
|
|
55
69
|
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# ZCARD and one newest-first page of ids, in one round trip.
|
|
74
|
+
def page_ids(conn, offset, count)
|
|
75
|
+
conn.pipelined do |pipe|
|
|
76
|
+
pipe.call('ZCARD', @key)
|
|
77
|
+
pipe.call('ZRANGE', @key, offset, offset + count - 1, 'REV')
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def summaries(conn, bids)
|
|
82
|
+
return [] if bids.empty?
|
|
83
|
+
|
|
84
|
+
commands = bids.map { |bid| [['HGETALL', "b-#{bid}"], *Wurk::Batch::Status.summary_commands(bid)] }
|
|
85
|
+
replies = conn.pipelined { |pipe| commands.flatten(1).each { |cmd| pipe.call(*cmd) } }
|
|
86
|
+
bids.zip(replies.each_slice(commands.first.size)).map do |bid, (hash, *sizes)|
|
|
87
|
+
Wurk::Batch::Status.new(bid, data: hash).summary(sizes)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
56
90
|
end
|
|
57
91
|
|
|
58
92
|
class Batch
|
data/lib/wurk/cli.rb
CHANGED
|
@@ -11,8 +11,6 @@ require_relative 'version'
|
|
|
11
11
|
require_relative 'configuration'
|
|
12
12
|
require_relative 'component'
|
|
13
13
|
require_relative 'capsule'
|
|
14
|
-
require_relative 'launcher'
|
|
15
|
-
require_relative 'rails_boot'
|
|
16
14
|
require_relative 'boot_checks'
|
|
17
15
|
|
|
18
16
|
module Wurk
|
|
@@ -204,19 +202,17 @@ module Wurk
|
|
|
204
202
|
# Mark server mode BEFORE the app loads so `configure_server` blocks in
|
|
205
203
|
# the required initializer actually fire (they gate on `config.server?`).
|
|
206
204
|
# Matches Sidekiq, which sets `Sidekiq.server = true` before requiring
|
|
207
|
-
# the app in its CLI.
|
|
208
|
-
#
|
|
209
|
-
# skip logs one WARN naming the gate (`configure_server` / `server? is
|
|
210
|
-
# false`).
|
|
205
|
+
# the app in its CLI. Each configure_server block the app's load reaches
|
|
206
|
+
# before it would be kept and run only here, after the app's own code.
|
|
211
207
|
enter_server_mode
|
|
212
208
|
boot_application if boot_app
|
|
213
209
|
Swarm::TestModeGuard.enforce!(@config.logger)
|
|
214
210
|
self_read, self_write = ::IO.pipe
|
|
215
211
|
begin
|
|
216
212
|
trap_signals(self_write)
|
|
217
|
-
validate_redis!
|
|
213
|
+
info = validate_redis!
|
|
218
214
|
validate_pool_sizes!
|
|
219
|
-
BootChecks.run(@config,
|
|
215
|
+
BootChecks.run(@config, info)
|
|
220
216
|
BootChecks.warn_on_ignored_topology(@config, 'wurk')
|
|
221
217
|
BootChecks.log_process_role(@config, 'wurk')
|
|
222
218
|
@config[:identity] = identity
|
|
@@ -243,9 +239,9 @@ module Wurk
|
|
|
243
239
|
# This is the only way to get fork-based parallelism without Rails — the
|
|
244
240
|
# railtie auto-boot path is Rails-only. Honors the swarm preload knobs
|
|
245
241
|
# (`WURK_PRELOAD`/`SIDEKIQ_PRELOAD` Bundler groups, `WURK_PRELOAD_APP`/
|
|
246
|
-
# `SIDEKIQ_PRELOAD_APP` whole-app eager-load)
|
|
247
|
-
# before the fork so children share
|
|
248
|
-
def run_swarm(boot_app: true
|
|
242
|
+
# `SIDEKIQ_PRELOAD_APP` whole-app eager-load); Swarm#boot warms the heap
|
|
243
|
+
# before the fork so children share its pages (Swarm::Prefork). Spec §7.
|
|
244
|
+
def run_swarm(boot_app: true) # rubocop:disable Metrics/AbcSize
|
|
249
245
|
raise StartupError, "the swarm runner takes no subcommand; run `wurk #{@command}` instead" if @command
|
|
250
246
|
|
|
251
247
|
# Server mode before the app loads — see #run. The flag rides through the
|
|
@@ -256,12 +252,11 @@ module Wurk
|
|
|
256
252
|
# crash-looping the fleet until the crash-loop exit.
|
|
257
253
|
Health.options_from_env
|
|
258
254
|
boot_swarm_application if boot_app
|
|
259
|
-
validate_redis!
|
|
255
|
+
info = validate_redis!
|
|
260
256
|
validate_pool_sizes!
|
|
261
|
-
BootChecks.run(@config,
|
|
257
|
+
BootChecks.run(@config, info)
|
|
262
258
|
log_swarm_role
|
|
263
259
|
@config[:identity] = identity
|
|
264
|
-
warm_up_process if warmup
|
|
265
260
|
@swarm = Wurk::Swarm.new(topology: @config.topology, config: @config, exit_on_crash_loop: true,
|
|
266
261
|
shutdown_timeout: @config[:timeout] || Swarm::DEFAULT_SHUTDOWN_TIMEOUT)
|
|
267
262
|
begin
|
|
@@ -352,9 +347,9 @@ module Wurk
|
|
|
352
347
|
Wurk.enter_server_mode(@config)
|
|
353
348
|
end
|
|
354
349
|
|
|
355
|
-
# Pre-fault and compact the heap before workers start
|
|
356
|
-
#
|
|
357
|
-
#
|
|
350
|
+
# Pre-fault and compact the heap before workers start (the swarm does its
|
|
351
|
+
# own, before the fork: Swarm::Prefork). RUBY_DISABLE_WARMUP=1 is the escape
|
|
352
|
+
# hatch for hosts that already warmed up.
|
|
358
353
|
def warm_up_process
|
|
359
354
|
return unless ::Process.respond_to?(:warmup) && ENV['RUBY_DISABLE_WARMUP'] != '1'
|
|
360
355
|
|
|
@@ -436,6 +431,8 @@ module Wurk
|
|
|
436
431
|
warn "wurk: USR2 log reopen failed, still writing to the previous log: #{e.class}: #{e.message}"
|
|
437
432
|
end
|
|
438
433
|
|
|
434
|
+
# Returns the INFO it read, so a boot pays one round trip for the version
|
|
435
|
+
# check and BootChecks together.
|
|
439
436
|
def validate_redis!
|
|
440
437
|
info = redis_info
|
|
441
438
|
ver = ::Gem::Version.new(info['redis_version'])
|
|
@@ -444,7 +441,7 @@ module Wurk
|
|
|
444
441
|
end
|
|
445
442
|
|
|
446
443
|
maxmemory_policy = info['maxmemory_policy']
|
|
447
|
-
return if maxmemory_policy.nil? || maxmemory_policy.empty? || maxmemory_policy == 'noeviction'
|
|
444
|
+
return info if maxmemory_policy.nil? || maxmemory_policy.empty? || maxmemory_policy == 'noeviction'
|
|
448
445
|
|
|
449
446
|
logger.warn { <<~WARN }
|
|
450
447
|
|
|
@@ -453,6 +450,7 @@ module Wurk
|
|
|
453
450
|
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
|
454
451
|
|
|
455
452
|
WARN
|
|
453
|
+
info
|
|
456
454
|
end
|
|
457
455
|
|
|
458
456
|
def redis_info
|
|
@@ -496,6 +494,7 @@ module Wurk
|
|
|
496
494
|
|
|
497
495
|
# What a worker loads only later: every constant, and the routes.
|
|
498
496
|
def boot_check_reach
|
|
497
|
+
Wurk.eager_load!
|
|
499
498
|
eager_load_rails
|
|
500
499
|
draw_routes
|
|
501
500
|
end
|
|
@@ -504,7 +503,6 @@ module Wurk
|
|
|
504
503
|
return unless @config[:boot_check_redis]
|
|
505
504
|
|
|
506
505
|
validate_redis!
|
|
507
|
-
redis_info
|
|
508
506
|
end
|
|
509
507
|
|
|
510
508
|
def log_swarm_role
|
data/lib/wurk/client/buffered.rb
CHANGED
|
@@ -58,7 +58,9 @@ module Wurk
|
|
|
58
58
|
# Returned by the append helpers when everything fit.
|
|
59
59
|
NOTHING_UNDELIVERED = [].freeze
|
|
60
60
|
|
|
61
|
-
|
|
61
|
+
# `bytes` is the payload's JSON size, what `reliable_push_buffer_bytes`
|
|
62
|
+
# counts.
|
|
63
|
+
Entry = Struct.new(:payload, :origin, :bytes)
|
|
62
64
|
|
|
63
65
|
# Where a buffered payload has to be replayed. What must NOT be captured
|
|
64
66
|
# is a pool object the config hands out: `reset_redis_pools!` — every
|
|
@@ -159,6 +161,7 @@ module Wurk
|
|
|
159
161
|
@install_mutex = Mutex.new
|
|
160
162
|
@buffer_mutex = Mutex.new
|
|
161
163
|
@replaying = 0
|
|
164
|
+
@buffer_bytes = 0
|
|
162
165
|
@circuit = Circuit.new
|
|
163
166
|
|
|
164
167
|
# Process that owns the state above; a mismatch means we're running in a
|
|
@@ -167,7 +170,7 @@ module Wurk
|
|
|
167
170
|
|
|
168
171
|
class << self
|
|
169
172
|
# Idempotent. Prepends the wrapper module into Wurk::Client so push /
|
|
170
|
-
# push_bulk drain the buffer before each call and raw_push catches
|
|
173
|
+
# push_bulk / push_prepared drain the buffer before each call and raw_push catches
|
|
171
174
|
# connection errors. Safe to call from multiple threads.
|
|
172
175
|
def install!
|
|
173
176
|
install_mutex.synchronize do
|
|
@@ -195,10 +198,30 @@ module Wurk
|
|
|
195
198
|
@buffer_cap = value
|
|
196
199
|
end
|
|
197
200
|
|
|
201
|
+
# Optional second cap, in bytes of payload JSON (Wurk-only; nil, the
|
|
202
|
+
# default, leaves only the entry cap). The entry cap alone lets a burst
|
|
203
|
+
# of large payloads hold `cap x payload size` in a web process for the
|
|
204
|
+
# whole outage. Over either cap the overflow mode applies; a payload
|
|
205
|
+
# larger than this cap on its own never fits, so it is the one dropped
|
|
206
|
+
# (or raised).
|
|
207
|
+
attr_reader :buffer_bytes_cap
|
|
208
|
+
|
|
209
|
+
def buffer_bytes_cap=(value)
|
|
210
|
+
unless value.nil? || (value.is_a?(Integer) && value.positive?)
|
|
211
|
+
raise ArgumentError, 'reliable_push_buffer_bytes must be nil or a positive Integer'
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
@buffer_bytes_cap = value
|
|
215
|
+
end
|
|
216
|
+
|
|
198
217
|
def buffer_size
|
|
199
218
|
buffer_mutex.synchronize { buffer.size }
|
|
200
219
|
end
|
|
201
220
|
|
|
221
|
+
def buffer_bytes
|
|
222
|
+
buffer_mutex.synchronize { @buffer_bytes }
|
|
223
|
+
end
|
|
224
|
+
|
|
202
225
|
def overflow_mode
|
|
203
226
|
@overflow_mode ||= DEFAULT_OVERFLOW_MODE
|
|
204
227
|
end
|
|
@@ -220,7 +243,9 @@ module Wurk
|
|
|
220
243
|
def reset!
|
|
221
244
|
buffer_mutex.synchronize do
|
|
222
245
|
@buffer = []
|
|
246
|
+
@buffer_bytes = 0
|
|
223
247
|
@buffer_cap = nil
|
|
248
|
+
@buffer_bytes_cap = nil
|
|
224
249
|
@overflow_mode = nil
|
|
225
250
|
@replaying = 0
|
|
226
251
|
end
|
|
@@ -263,6 +288,7 @@ module Wurk
|
|
|
263
288
|
@install_mutex = Mutex.new
|
|
264
289
|
@buffer_mutex = Mutex.new
|
|
265
290
|
@buffer = []
|
|
291
|
+
@buffer_bytes = 0
|
|
266
292
|
@replaying = 0
|
|
267
293
|
@circuit = Circuit.new
|
|
268
294
|
@drops = DropLog.new
|
|
@@ -283,29 +309,34 @@ module Wurk
|
|
|
283
309
|
# `client` is the one whose push failed; every payload is tagged with
|
|
284
310
|
# the Redis that client was writing to (see Origin).
|
|
285
311
|
def enbuffer(payloads, client:)
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
cap = buffer_cap
|
|
312
|
+
entries = entries_for(payloads, client)
|
|
313
|
+
caps = [buffer_cap, buffer_bytes_cap]
|
|
290
314
|
mode = overflow_mode
|
|
291
315
|
undelivered = buffer_mutex.synchronize do
|
|
292
|
-
mode == :raise ? append_within_capacity(entries,
|
|
316
|
+
mode == :raise ? append_within_capacity(entries, *caps) : append_dropping_oldest(entries, *caps)
|
|
293
317
|
end
|
|
294
318
|
|
|
295
319
|
raise Overflow, undelivered.map(&:payload) unless undelivered.empty?
|
|
296
320
|
ensure
|
|
297
|
-
drops.report(
|
|
321
|
+
drops.report(*caps)
|
|
298
322
|
end
|
|
299
323
|
|
|
300
324
|
private
|
|
301
325
|
|
|
326
|
+
def entries_for(payloads, client)
|
|
327
|
+
origin = Origin.for(client)
|
|
328
|
+
payloads.map { |payload| Entry.new(payload, origin, Wurk.dump_json(payload).bytesize) }
|
|
329
|
+
end
|
|
330
|
+
|
|
302
331
|
# Both append helpers run with buffer_mutex held and return the entries
|
|
303
332
|
# they could not take.
|
|
304
333
|
|
|
305
|
-
def append_dropping_oldest(entries, cap)
|
|
334
|
+
def append_dropping_oldest(entries, cap, bytes_cap)
|
|
306
335
|
entries.each do |entry|
|
|
307
|
-
drops.record(
|
|
308
|
-
|
|
336
|
+
next drops.record(entry.payload) if bytes_cap && entry.bytes > bytes_cap
|
|
337
|
+
|
|
338
|
+
drops.record(take_oldest.payload) until fits?(entry, cap, bytes_cap)
|
|
339
|
+
push_entry(entry)
|
|
309
340
|
end
|
|
310
341
|
NOTHING_UNDELIVERED
|
|
311
342
|
end
|
|
@@ -316,15 +347,38 @@ module Wurk
|
|
|
316
347
|
# negative when the cap was lowered after the buffer filled — clamped,
|
|
317
348
|
# so an over-full buffer rejects the whole call instead of raising on
|
|
318
349
|
# `first`/`drop`.
|
|
319
|
-
def append_within_capacity(entries, cap)
|
|
350
|
+
def append_within_capacity(entries, cap, bytes_cap)
|
|
320
351
|
room = (cap - buffer.size).clamp(0, entries.size)
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
352
|
+
room = bytes_room(entries.first(room), bytes_cap) if bytes_cap
|
|
353
|
+
entries.first(room).each { |entry| push_entry(entry) }
|
|
354
|
+
room == entries.size ? NOTHING_UNDELIVERED : entries.drop(room)
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# How many of `entries`, in order, fit under the byte cap.
|
|
358
|
+
def bytes_room(entries, bytes_cap)
|
|
359
|
+
total = @buffer_bytes
|
|
360
|
+
entries.take_while { |entry| (total += entry.bytes) <= bytes_cap }.size
|
|
361
|
+
end
|
|
325
362
|
|
|
326
|
-
|
|
327
|
-
|
|
363
|
+
def fits?(entry, cap, bytes_cap)
|
|
364
|
+
buffer.size < cap && (bytes_cap.nil? || @buffer_bytes + entry.bytes <= bytes_cap)
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
# The three buffer mutations, each under buffer_mutex, each keeping the
|
|
368
|
+
# byte total (`@buffer_bytes`) in step.
|
|
369
|
+
def push_entry(entry)
|
|
370
|
+
@buffer_bytes += entry.bytes
|
|
371
|
+
buffer << entry
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
def take_oldest
|
|
375
|
+
remove_at(0)
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
def remove_at(index)
|
|
379
|
+
entry = buffer.delete_at(index)
|
|
380
|
+
@buffer_bytes -= entry.bytes
|
|
381
|
+
entry
|
|
328
382
|
end
|
|
329
383
|
|
|
330
384
|
public
|
|
@@ -508,7 +562,7 @@ module Wurk
|
|
|
508
562
|
next unless index
|
|
509
563
|
|
|
510
564
|
@replaying += 1
|
|
511
|
-
|
|
565
|
+
remove_at(index)
|
|
512
566
|
end
|
|
513
567
|
end
|
|
514
568
|
|
|
@@ -550,7 +604,10 @@ module Wurk
|
|
|
550
604
|
end
|
|
551
605
|
|
|
552
606
|
def restore(entry)
|
|
553
|
-
buffer_mutex.synchronize
|
|
607
|
+
buffer_mutex.synchronize do
|
|
608
|
+
@buffer_bytes += entry.bytes
|
|
609
|
+
buffer.unshift(entry)
|
|
610
|
+
end
|
|
554
611
|
end
|
|
555
612
|
|
|
556
613
|
# Drain marks the thread so our prepended raw_push re-raises the
|
|
@@ -587,7 +644,7 @@ module Wurk
|
|
|
587
644
|
@lock.synchronize { @pending << payload }
|
|
588
645
|
end
|
|
589
646
|
|
|
590
|
-
def report(cap)
|
|
647
|
+
def report(cap, bytes_cap = nil)
|
|
591
648
|
dropped, announce = @lock.synchronize do
|
|
592
649
|
next [nil, false] if @pending.empty?
|
|
593
650
|
|
|
@@ -599,7 +656,7 @@ module Wurk
|
|
|
599
656
|
end
|
|
600
657
|
return unless dropped
|
|
601
658
|
|
|
602
|
-
log(dropped.first, cap) if announce
|
|
659
|
+
log(dropped.first, cap, bytes_cap) if announce
|
|
603
660
|
dropped.size.times { count_drop }
|
|
604
661
|
end
|
|
605
662
|
|
|
@@ -620,13 +677,14 @@ module Wurk
|
|
|
620
677
|
nil
|
|
621
678
|
end
|
|
622
679
|
|
|
623
|
-
def log(first, cap)
|
|
680
|
+
def log(first, cap, bytes_cap)
|
|
624
681
|
Wurk.configuration.logger.error do
|
|
625
|
-
"reliable_push buffer full (cap=#{cap}
|
|
682
|
+
"reliable_push buffer full (cap=#{cap}#{", bytes=#{bytes_cap}" if bytes_cap}): " \
|
|
683
|
+
'dropping the oldest buffered jobs to make room — ' \
|
|
626
684
|
"#{first['class']} jid=#{first['jid']} is the first of this burst and will never run. " \
|
|
627
685
|
'Further drops are counted in statsd jobs.dropped.push until Redis takes a replay. Raise ' \
|
|
628
|
-
'`Wurk::Client.reliable_push_buffer
|
|
629
|
-
'overflow yourself.'
|
|
686
|
+
'`Wurk::Client.reliable_push_buffer` (or `reliable_push_buffer_bytes`), or set ' \
|
|
687
|
+
'`reliable_push_overflow = :raise` to handle overflow yourself.'
|
|
630
688
|
end
|
|
631
689
|
rescue StandardError
|
|
632
690
|
nil
|
|
@@ -742,6 +800,11 @@ module Wurk
|
|
|
742
800
|
super
|
|
743
801
|
end
|
|
744
802
|
|
|
803
|
+
def push_prepared(payloads)
|
|
804
|
+
Buffered.drain!(self) unless Buffered.unbuffered?
|
|
805
|
+
super
|
|
806
|
+
end
|
|
807
|
+
|
|
745
808
|
private
|
|
746
809
|
|
|
747
810
|
# Opens the delivery ledger Client writes into. It lives here rather
|
|
@@ -851,6 +914,14 @@ module Wurk
|
|
|
851
914
|
Buffered.buffer_cap = value
|
|
852
915
|
end
|
|
853
916
|
|
|
917
|
+
def reliable_push_buffer_bytes
|
|
918
|
+
Buffered.buffer_bytes_cap
|
|
919
|
+
end
|
|
920
|
+
|
|
921
|
+
def reliable_push_buffer_bytes=(value)
|
|
922
|
+
Buffered.buffer_bytes_cap = value
|
|
923
|
+
end
|
|
924
|
+
|
|
854
925
|
def reliable_push_overflow
|
|
855
926
|
Buffered.overflow_mode
|
|
856
927
|
end
|
data/lib/wurk/client.rb
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative 'iterable_job'
|
|
4
3
|
require_relative 'job_util'
|
|
5
4
|
require_relative 'lua'
|
|
6
5
|
require_relative 'status'
|
|
@@ -90,6 +89,27 @@ module Wurk
|
|
|
90
89
|
payload['jid']
|
|
91
90
|
end
|
|
92
91
|
|
|
92
|
+
# #push in two halves, for a caller that pushes many jobs in one write
|
|
93
|
+
# (JobSet#retry_all): #prepare_push runs one item through normalize →
|
|
94
|
+
# middleware → verify, #push_prepared writes what came back. Internal.
|
|
95
|
+
#
|
|
96
|
+
# @return [Hash, nil] the payload to write; nil when middleware halted it.
|
|
97
|
+
def prepare_push(item)
|
|
98
|
+
payload = invoke_chain(normalize_item(item))
|
|
99
|
+
verify_json(payload) if payload
|
|
100
|
+
payload
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# One #raw_push for every payload, so they should share a queue, a batch
|
|
104
|
+
# or none, and a schedule or none: anything else is several writes, and one
|
|
105
|
+
# can land before another fails.
|
|
106
|
+
# @return [Array<String>] jids
|
|
107
|
+
def push_prepared(payloads)
|
|
108
|
+
buffered = raw_push(payloads)
|
|
109
|
+
emit_enqueued(payloads, buffered)
|
|
110
|
+
payloads.map { |payload| payload['jid'] }
|
|
111
|
+
end
|
|
112
|
+
|
|
93
113
|
# @param items [Hash] String keys only: class, args (Array<Array>), at?, spread_interval?, batch_size?, jid?
|
|
94
114
|
# @return [Array<String, nil>] jids in submission order; nil entries mark middleware-halted jobs.
|
|
95
115
|
def push_bulk(items)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Autoload target of the server tier's Sidekiq aliases (compat.rb). The Wurk
|
|
4
|
+
# files define only their Wurk names; this assigns the Sidekiq ones.
|
|
5
|
+
|
|
6
|
+
module Sidekiq
|
|
7
|
+
CLI = Wurk::CLI
|
|
8
|
+
Embedded = Wurk::Embedded
|
|
9
|
+
Launcher = Wurk::Launcher
|
|
10
|
+
Manager = Wurk::Manager
|
|
11
|
+
Scheduled = Wurk::Scheduled
|
|
12
|
+
History = Wurk::History
|
|
13
|
+
Periodic = Wurk::Cron
|
|
14
|
+
CronParser = Wurk::Cron::Parser
|
|
15
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Autoload target of the dashboard's Sidekiq aliases (compat.rb).
|
|
4
|
+
|
|
5
|
+
module Sidekiq
|
|
6
|
+
Web = Wurk::Web
|
|
7
|
+
WebHelpers = Wurk::Web::Extension::Helpers
|
|
8
|
+
Paginator = Wurk::Web::Paginator
|
|
9
|
+
|
|
10
|
+
module Pro
|
|
11
|
+
# Sidekiq Pro mounts the dashboard at `Sidekiq::Pro::Web`; it is the same
|
|
12
|
+
# board as `Sidekiq::Web` here, so a Pro app's `mount Sidekiq::Pro::Web`
|
|
13
|
+
# drops in unchanged.
|
|
14
|
+
Web = Wurk::Web
|
|
15
|
+
|
|
16
|
+
# `use Sidekiq::Pro::BatchStatus` — the polling Rack middleware that serves
|
|
17
|
+
# GET /batch_status/<bid>.json. Spec: docs/target/sidekiq-pro.md §10.3.
|
|
18
|
+
BatchStatus = Wurk::Web::BatchStatus
|
|
19
|
+
end
|
|
20
|
+
end
|