wurk 1.14.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/concerns/wurk/same_origin_guard.rb +30 -8
- data/app/controllers/wurk/api/pagination.rb +70 -53
- data/app/controllers/wurk/api_controller.rb +96 -27
- data/app/controllers/wurk/dashboard_controller.rb +26 -2
- data/app/controllers/wurk/profiles_controller.rb +28 -23
- data/config/routes.rb +8 -6
- data/lib/wurk/api/admin.rb +84 -0
- data/lib/wurk/api/app.rb +7 -2
- data/lib/wurk/api/jobs.rb +11 -3
- data/lib/wurk/api/queues.rb +10 -0
- data/lib/wurk/api/router.rb +6 -0
- data/lib/wurk/api/validation.rb +17 -0
- data/lib/wurk/batch/callbacks.rb +9 -2
- data/lib/wurk/batch.rb +15 -4
- data/lib/wurk/batch_set.rb +22 -0
- data/lib/wurk/cli.rb +4 -2
- data/lib/wurk/client/buffered.rb +104 -48
- data/lib/wurk/configuration.rb +28 -0
- data/lib/wurk/fetcher/many_queues.rb +18 -8
- data/lib/wurk/fetcher/poison_pill.rb +2 -0
- data/lib/wurk/fetcher/reaper.rb +3 -0
- data/lib/wurk/fetcher/reliable.rb +26 -12
- data/lib/wurk/health.rb +6 -1
- data/lib/wurk/health_probe.rb +45 -7
- data/lib/wurk/heartbeat.rb +29 -6
- data/lib/wurk/job_set.rb +25 -3
- data/lib/wurk/leader.rb +2 -0
- data/lib/wurk/metrics/accumulator.rb +27 -2
- data/lib/wurk/metrics/exposition.rb +45 -0
- data/lib/wurk/metrics/history.rb +10 -5
- data/lib/wurk/metrics/pool_registry.rb +27 -0
- data/lib/wurk/metrics/process_counters.rb +16 -0
- data/lib/wurk/metrics/process_families.rb +115 -0
- data/lib/wurk/metrics/prometheus.rb +84 -51
- data/lib/wurk/redis_pool.rb +6 -1
- data/lib/wurk/scheduled.rb +7 -0
- data/lib/wurk/swarm/child_boot.rb +6 -1
- data/lib/wurk/swarm/crash_watch.rb +5 -0
- data/lib/wurk/swarm/memory_watch.rb +1 -7
- data/lib/wurk/swarm/restart.rb +14 -5
- data/lib/wurk/swarm/restart_counters.rb +79 -0
- data/lib/wurk/swarm.rb +14 -6
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk.rb +2 -1
- data/vendor/assets/dashboard/assets/{ArgsValue-BSTICTw4.js → ArgsValue-D10BT0xi.js} +1 -1
- data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-D10BT0xi.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{BatchDetail-BBMiNTok.js → BatchDetail-ByPRHv86.js} +1 -1
- data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-ByPRHv86.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-CVVOJFhz.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{Busy-CcSvIQFY.js → Busy-Dp26yz1A.js} +1 -1
- data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-Dp26yz1A.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{Cron-ByRxtItH.js → Cron-BiRV6eoU.js} +1 -1
- data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-BiRV6eoU.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CnpCQ8Of.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-ykYbsK8n.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{Extension--fRIfdcW.js → Extension-BUsciXhN.js} +1 -1
- data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension-BUsciXhN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js +1 -0
- data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-DBIZXNtY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowDetail-CxmWxNpF.js → FlowDetail-CIGu7pRb.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CIGu7pRb.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{FlowState-ClzInw0-.js → FlowState-B04ZLDe-.js} +1 -1
- data/vendor/assets/dashboard/assets/FlowState-B04ZLDe-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-B04ZLDe-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{Flows-CCnTVfkN.js → Flows-Ce9Qoq0c.js} +1 -1
- data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-Ce9Qoq0c.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{JobDetailModal-Btzt9HOQ.js → JobDetailModal-DWp0OMoV.js} +1 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-DWp0OMoV.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{Limiters-DYxSN0sy.js → Limiters-DiWCiVay.js} +1 -1
- data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DiWCiVay.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{Metrics-BfSRkQ2U.js → Metrics-DmMS2wo_.js} +1 -1
- data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-DmMS2wo_.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{NotFound-DrAE8U92.js → NotFound-CM1j8-Fv.js} +1 -1
- data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-CM1j8-Fv.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{PageHeader-w3hA_hMm.js → PageHeader-BcHVxRbX.js} +1 -1
- data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-BcHVxRbX.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-B70MlVYQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-D3JZtWBv.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-BTWoV6so.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-D3PyiFqr.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{Search-D42Va7lB.js → Search-bLwIhsaR.js} +1 -1
- data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-bLwIhsaR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-CugpXGea.js +1 -0
- data/vendor/assets/dashboard/assets/ar-CugpXGea.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-CugpXGea.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{charts-C1xMUYFp.js → charts-DZyiygHX.js} +1 -1
- data/vendor/assets/dashboard/assets/charts-DZyiygHX.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-DZyiygHX.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js +1 -0
- data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-Bl5Jn_0l.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-lqQrmjYI.js +1 -0
- data/vendor/assets/dashboard/assets/es-lqQrmjYI.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-lqQrmjYI.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-D70VBoh3.js +1 -0
- data/vendor/assets/dashboard/assets/fr-D70VBoh3.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-D70VBoh3.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{i18n-BXp3XejR.js → i18n-DdPClaCG.js} +1 -1
- data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-DdPClaCG.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css +1 -0
- data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-CtK2Dy_G.css.gz +0 -0
- data/vendor/assets/dashboard/assets/{index-CiLU1zz8.js → index-ygQ-XC-L.js} +3 -3
- data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-ygQ-XC-L.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js +1 -0
- data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-BYMqnHjg.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js +1 -0
- data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-4k5M2MYH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js +2 -0
- data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js.br +0 -0
- data/vendor/assets/dashboard/assets/useJobSetList-D-uafkUL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{useResetPageOnEmpty-Cuv4BHkC.js → useResetPageOnEmpty-DUA0JTgs.js} +1 -1
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DUA0JTgs.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{useSort-V9yUrByY.js → useSort-BwoC7g__.js} +1 -1
- data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-BwoC7g__.js.gz +0 -0
- data/vendor/assets/dashboard/assets/{utils-69XSoGbQ.js → utils-CQj4rqng.js} +1 -1
- data/vendor/assets/dashboard/assets/utils-CQj4rqng.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-CQj4rqng.js.gz +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js +1 -0
- data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B9E1MvsM.js.gz +0 -0
- data/vendor/assets/dashboard/index.html +8 -8
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +117 -109
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.br +0 -0
- data/vendor/assets/dashboard/assets/ArgsValue-BSTICTw4.js.gz +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.br +0 -0
- data/vendor/assets/dashboard/assets/BatchDetail-BBMiNTok.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Batches-D8I3hCXy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.br +0 -0
- data/vendor/assets/dashboard/assets/Busy-CcSvIQFY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.br +0 -0
- data/vendor/assets/dashboard/assets/Cron-ByRxtItH.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dashboard-CO7_6OUM.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.br +0 -0
- data/vendor/assets/dashboard/assets/Dead-C8mDGQAF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.br +0 -0
- data/vendor/assets/dashboard/assets/Extension--fRIfdcW.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js +0 -2
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.br +0 -0
- data/vendor/assets/dashboard/assets/FilterBox-BS1_G5S5.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowDetail-CxmWxNpF.js.gz +0 -0
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.br +0 -0
- data/vendor/assets/dashboard/assets/FlowState-ClzInw0-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.br +0 -0
- data/vendor/assets/dashboard/assets/Flows-CCnTVfkN.js.gz +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-Btzt9HOQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.br +0 -0
- data/vendor/assets/dashboard/assets/Limiters-DYxSN0sy.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.br +0 -0
- data/vendor/assets/dashboard/assets/Metrics-BfSRkQ2U.js.gz +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.br +0 -0
- data/vendor/assets/dashboard/assets/NotFound-DrAE8U92.js.gz +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.br +0 -0
- data/vendor/assets/dashboard/assets/PageHeader-w3hA_hMm.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.br +0 -0
- data/vendor/assets/dashboard/assets/Profiles-191Gq7P-.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.br +0 -0
- data/vendor/assets/dashboard/assets/Queues-CIvl1fjf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.br +0 -0
- data/vendor/assets/dashboard/assets/Retries-sdaiHACO.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.br +0 -0
- data/vendor/assets/dashboard/assets/Scheduled-CcildKBh.js.gz +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.br +0 -0
- data/vendor/assets/dashboard/assets/Search-D42Va7lB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js +0 -1
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.br +0 -0
- data/vendor/assets/dashboard/assets/ar-Br6DtdyK.js.gz +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.br +0 -0
- data/vendor/assets/dashboard/assets/charts-C1xMUYFp.js.gz +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js +0 -1
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.br +0 -0
- data/vendor/assets/dashboard/assets/de-B2NCIuzo.js.gz +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js +0 -1
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.br +0 -0
- data/vendor/assets/dashboard/assets/es-BGeu_JNB.js.gz +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js +0 -1
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.br +0 -0
- data/vendor/assets/dashboard/assets/fr-DQVBWFId.js.gz +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.br +0 -0
- data/vendor/assets/dashboard/assets/i18n-BXp3XejR.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.br +0 -0
- data/vendor/assets/dashboard/assets/index-CiLU1zz8.js.gz +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css +0 -1
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.br +0 -0
- data/vendor/assets/dashboard/assets/index-r3T9Ugl4.css.gz +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js +0 -1
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.br +0 -0
- data/vendor/assets/dashboard/assets/ja-D06IBqzL.js.gz +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js +0 -1
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.br +0 -0
- data/vendor/assets/dashboard/assets/pt-BR-Dn_okdxf.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.br +0 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Cuv4BHkC.js.gz +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.br +0 -0
- data/vendor/assets/dashboard/assets/useSort-V9yUrByY.js.gz +0 -0
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.br +0 -0
- data/vendor/assets/dashboard/assets/utils-69XSoGbQ.js.gz +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js +0 -1
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.br +0 -0
- data/vendor/assets/dashboard/assets/zh-CN-B1ObO-uZ.js.gz +0 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'idempotency'
|
|
4
|
+
require_relative 'jobs'
|
|
5
|
+
require_relative 'page'
|
|
6
|
+
require_relative 'problem'
|
|
7
|
+
require_relative 'response'
|
|
8
|
+
require_relative 'validation'
|
|
9
|
+
|
|
10
|
+
module Wurk
|
|
11
|
+
module API
|
|
12
|
+
# The operator actions the dashboard already offers, for a client that has
|
|
13
|
+
# no screen: put one retrying or dead job back on its queue, empty a queue.
|
|
14
|
+
#
|
|
15
|
+
# Each goes through the object the dashboard's action calls —
|
|
16
|
+
# SortedEntry#retry, Queue#clear — so the two surfaces cannot disagree about
|
|
17
|
+
# what "retry" does to `retry_count` or what "clear" removes.
|
|
18
|
+
#
|
|
19
|
+
# Neither is safe to send twice blind. A second retry after a lost reply
|
|
20
|
+
# finds nothing and reads as a 404 the job did not earn, and a second clear
|
|
21
|
+
# empties whatever was enqueued in between. So every route here takes
|
|
22
|
+
# `Idempotency-Key` exactly as the produce plane does.
|
|
23
|
+
module Admin
|
|
24
|
+
module_function
|
|
25
|
+
|
|
26
|
+
TABLE = [
|
|
27
|
+
[:post, '/retries/:jid/retry', :admin, :retry_retrying],
|
|
28
|
+
[:post, '/dead/:jid/retry', :admin, :retry_dead],
|
|
29
|
+
[:post, '/queues/:name/clear', :admin, :clear]
|
|
30
|
+
].freeze
|
|
31
|
+
|
|
32
|
+
def draw(router)
|
|
33
|
+
TABLE.each do |verb, pattern, scope, handler|
|
|
34
|
+
router.public_send(verb, pattern, scope: scope) do |request|
|
|
35
|
+
claimed(request) { public_send(handler, request) }
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def retry_retrying(request) = retry_entry(request, ::Wurk::RetrySet.new)
|
|
41
|
+
def retry_dead(request) = retry_entry(request, ::Wurk::DeadSet.new)
|
|
42
|
+
|
|
43
|
+
# A job the poller promoted, or another client retried, between the
|
|
44
|
+
# lookup and the removal is not retried twice: SortedEntry#retry answers
|
|
45
|
+
# nil when its ZREM finds nothing, and that is the same 404 a jid that
|
|
46
|
+
# was never there gets.
|
|
47
|
+
def retry_entry(request, set)
|
|
48
|
+
jid = Validation.jid!(request.path_params[:jid].to_s)
|
|
49
|
+
score = Validation.score!(Page.query!(request))
|
|
50
|
+
entry = Jobs.find_entry(set, jid, score)
|
|
51
|
+
return job_not_found(request, set, jid) unless entry&.retry
|
|
52
|
+
|
|
53
|
+
Response.json(200, jid: jid, set: set.name, queue: entry.queue)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def clear(request)
|
|
57
|
+
queue = ::Wurk::Queue.new(Validation.queue_name!(request.path_params[:name]))
|
|
58
|
+
queue.clear
|
|
59
|
+
Response.json(200, name: queue.name, cleared: true)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The body cap and the shape of the key are checked before the claim, the
|
|
63
|
+
# rest inside it, in the order Jobs#produce uses and for its reason: a
|
|
64
|
+
# refusal releases the key, a success is what a resend replays.
|
|
65
|
+
def claimed(request, &action)
|
|
66
|
+
config = request.config
|
|
67
|
+
raw = Validation.body!(request, config.api_max_body_bytes)
|
|
68
|
+
Idempotency.around(request, raw, config) { Jobs.rejectable(request, &action) }
|
|
69
|
+
rescue Validation::Invalid => e
|
|
70
|
+
Problem.from(e, instance: request.path)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def job_not_found(request, set, jid)
|
|
74
|
+
Problem.render(
|
|
75
|
+
Problem::JOB_NOT_FOUND,
|
|
76
|
+
status: 404,
|
|
77
|
+
detail: "No job in #{set.name} has jid #{jid}.",
|
|
78
|
+
instance: request.path,
|
|
79
|
+
jid: jid
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
data/lib/wurk/api/app.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'rack'
|
|
4
4
|
require_relative '../api'
|
|
5
5
|
require_relative '../version'
|
|
6
|
+
require_relative 'admin'
|
|
6
7
|
require_relative 'auth'
|
|
7
8
|
require_relative 'flows'
|
|
8
9
|
require_relative 'jobs'
|
|
@@ -36,6 +37,10 @@ module Wurk
|
|
|
36
37
|
draw(@router)
|
|
37
38
|
end
|
|
38
39
|
|
|
40
|
+
# Every route this app answers, for test/unit/api_openapi_parity_test.rb
|
|
41
|
+
# to hold docs/api/openapi.yaml to.
|
|
42
|
+
def routes = @router.routes
|
|
43
|
+
|
|
39
44
|
def call(env)
|
|
40
45
|
request = Request.new(env)
|
|
41
46
|
# Stamped once, before anything reads it, so every handler sees the
|
|
@@ -52,14 +57,14 @@ module Wurk
|
|
|
52
57
|
@config || ::Wurk.configuration
|
|
53
58
|
end
|
|
54
59
|
|
|
55
|
-
# One table per plane, each owning its own routes and handlers.
|
|
56
|
-
# Flows and Swarm join Jobs here.
|
|
60
|
+
# One table per plane, each owning its own routes and handlers.
|
|
57
61
|
def draw(router)
|
|
58
62
|
router.get('/', scope: Auth::ANY) { |request| root(request) }
|
|
59
63
|
Jobs.draw(router)
|
|
60
64
|
Queues.draw(router)
|
|
61
65
|
Flows.draw(router)
|
|
62
66
|
Swarm.draw(router)
|
|
67
|
+
Admin.draw(router)
|
|
63
68
|
end
|
|
64
69
|
|
|
65
70
|
# Authentication runs before the version gate and before routing, so an
|
data/lib/wurk/api/jobs.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative 'idempotency'
|
|
4
|
+
require_relative 'page'
|
|
4
5
|
require_relative 'problem'
|
|
5
6
|
require_relative 'response'
|
|
6
7
|
require_relative 'validation'
|
|
@@ -123,7 +124,8 @@ module Wurk
|
|
|
123
124
|
# action against a different set.
|
|
124
125
|
def destroy(request)
|
|
125
126
|
jid = Validation.jid!(request.path_params[:jid].to_s)
|
|
126
|
-
|
|
127
|
+
score = Validation.score!(Page.query!(request))
|
|
128
|
+
set = cancellable_sets.find { |candidate| remove(candidate, jid, score) }
|
|
127
129
|
return job_not_found(request, jid) unless set
|
|
128
130
|
|
|
129
131
|
Response.json(200, jid: jid, set: set.name)
|
|
@@ -142,11 +144,17 @@ module Wurk
|
|
|
142
144
|
# them to disagree. A job promoted out of the set between the two steps
|
|
143
145
|
# removes nothing and reads as "already gone", the honest answer to a
|
|
144
146
|
# cancel that lost the race.
|
|
145
|
-
def remove(set, jid)
|
|
146
|
-
entry = set
|
|
147
|
+
def remove(set, jid, score)
|
|
148
|
+
entry = find_entry(set, jid, score)
|
|
147
149
|
entry ? entry.delete : false
|
|
148
150
|
end
|
|
149
151
|
|
|
152
|
+
# Without a score this is JobSet#find_job's ZSCAN, which walks the whole
|
|
153
|
+
# set on a miss; with one it is a single ZRANGEBYSCORE.
|
|
154
|
+
def find_entry(set, jid, score)
|
|
155
|
+
score ? set.fetch(score, jid).first : set.find_job(jid)
|
|
156
|
+
end
|
|
157
|
+
|
|
150
158
|
# Validation decided which problem this is; rendering it is all that is
|
|
151
159
|
# left, so a new rejection never needs a new arm here.
|
|
152
160
|
def problem(request, error) = Problem.from(error, instance: request.path)
|
data/lib/wurk/api/queues.rb
CHANGED
|
@@ -39,6 +39,7 @@ module Wurk
|
|
|
39
39
|
[:get, '/retries', :read, :retries],
|
|
40
40
|
[:get, '/scheduled', :read, :scheduled],
|
|
41
41
|
[:get, '/dead', :read, :dead],
|
|
42
|
+
[:get, '/batches', :read, :batches],
|
|
42
43
|
[:get, '/batches/:bid', :read, :batch],
|
|
43
44
|
[:post, '/queues/:name/pause', :admin, :pause],
|
|
44
45
|
[:post, '/queues/:name/unpause', :admin, :unpause]
|
|
@@ -106,6 +107,15 @@ module Wurk
|
|
|
106
107
|
Response.json(200, name: set.name, total: total, page: window.page, count: window.count, jobs: jobs)
|
|
107
108
|
end
|
|
108
109
|
|
|
110
|
+
# Newest first, each row Batch::Status#summary: the failed and dead jid
|
|
111
|
+
# sets counted rather than listed, so a page of 200 batches with 10k
|
|
112
|
+
# failures each is not two million jids. `/batches/:bid` carries them.
|
|
113
|
+
def batches(request)
|
|
114
|
+
window = Page.window!(request)
|
|
115
|
+
total, rows = ::Wurk::BatchSet.new.snapshot(window.offset, window.count)
|
|
116
|
+
Response.json(200, total: total, page: window.page, count: window.count, batches: rows)
|
|
117
|
+
end
|
|
118
|
+
|
|
109
119
|
def batch(request)
|
|
110
120
|
bid = Validation.bid!(request.path_params[:bid])
|
|
111
121
|
status = ::Wurk::Batch::Status.new(bid)
|
data/lib/wurk/api/router.rb
CHANGED
|
@@ -31,6 +31,12 @@ module Wurk
|
|
|
31
31
|
@routes = []
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
# Pattern strings rebuilt from segments, so '/jobs/:jid' reads back as
|
|
35
|
+
# it was drawn.
|
|
36
|
+
def routes
|
|
37
|
+
@routes.map { |route| [route.verb, "/#{route.segments.join('/')}", route.scope] }
|
|
38
|
+
end
|
|
39
|
+
|
|
34
40
|
def get(pattern, scope:, &handler) = add('GET', pattern, scope, handler)
|
|
35
41
|
def post(pattern, scope:, &handler) = add('POST', pattern, scope, handler)
|
|
36
42
|
def delete(pattern, scope:, &handler) = add('DELETE', pattern, scope, handler)
|
data/lib/wurk/api/validation.rb
CHANGED
|
@@ -176,6 +176,23 @@ module Wurk
|
|
|
176
176
|
# @return [String] the fid.
|
|
177
177
|
def fid!(fid) = url_safe!(fid, 'flow id')
|
|
178
178
|
|
|
179
|
+
# `?score=` is a member's ZSET score, the `at` a listing row carries.
|
|
180
|
+
# With it a lookup is one ZRANGEBYSCORE at that score instead of a ZSCAN
|
|
181
|
+
# over the whole set; nil when absent, so the caller falls back to the scan.
|
|
182
|
+
# @return [Float, nil]
|
|
183
|
+
def score!(query)
|
|
184
|
+
raw = query['score']
|
|
185
|
+
return nil if raw.nil? || raw == ''
|
|
186
|
+
|
|
187
|
+
score = Float(raw)
|
|
188
|
+
# Float accepts "nan" and "inf"; Redis refuses NaN as a range bound.
|
|
189
|
+
raise Invalid, "The 'score' parameter must be a number." unless score.finite?
|
|
190
|
+
|
|
191
|
+
score
|
|
192
|
+
rescue ::ArgumentError, ::TypeError
|
|
193
|
+
raise Invalid, "The 'score' parameter must be a number."
|
|
194
|
+
end
|
|
195
|
+
|
|
179
196
|
def url_safe!(value, label)
|
|
180
197
|
return value if value.is_a?(::String) && JID_FORMAT.match?(value)
|
|
181
198
|
|
data/lib/wurk/batch/callbacks.rb
CHANGED
|
@@ -283,9 +283,16 @@ module Wurk
|
|
|
283
283
|
Wurk.redis { |conn| conn.call('SCARD', "b-#{bid}-died") }.to_i.positive?
|
|
284
284
|
end
|
|
285
285
|
|
|
286
|
+
# Two round trips however many kids: the death flags are read in one
|
|
287
|
+
# pipeline rather than one HGET per kid.
|
|
286
288
|
def any_child_dead?(bid)
|
|
287
|
-
|
|
288
|
-
|
|
289
|
+
Wurk.redis do |conn|
|
|
290
|
+
kids = conn.call('SMEMBERS', "b-#{bid}-kids")
|
|
291
|
+
next false if kids.empty?
|
|
292
|
+
|
|
293
|
+
flags = conn.pipelined { |pipe| kids.each { |kid| pipe.call('HGET', "b-#{kid}", 'death') } }
|
|
294
|
+
flags.include?('1')
|
|
295
|
+
end
|
|
289
296
|
end
|
|
290
297
|
|
|
291
298
|
# Per-callback rescue: one bad spec or a transient enqueue failure must
|
data/lib/wurk/batch.rb
CHANGED
|
@@ -546,11 +546,22 @@ module Wurk
|
|
|
546
546
|
end
|
|
547
547
|
end
|
|
548
548
|
|
|
549
|
+
# One pipeline per generation of the batch tree rather than two round
|
|
550
|
+
# trips per batch: each flags its batches and reads their kids, the next
|
|
551
|
+
# flags those. Replay-safe for pipelined_eval: the flag converges and the
|
|
552
|
+
# reads are reads.
|
|
549
553
|
def cascade_invalidate(bid)
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
+
generation = [bid]
|
|
555
|
+
until generation.empty?
|
|
556
|
+
replies = Wurk.redis do |conn|
|
|
557
|
+
Wurk::Lua::Loader.pipelined_eval(conn) do |pipe, eval_method|
|
|
558
|
+
generation.each do |member|
|
|
559
|
+
Wurk::Lua::Loader.public_send(eval_method, pipe, :batch_invalidate, keys: ["b-#{member}"], argv: [])
|
|
560
|
+
pipe.call('SMEMBERS', "b-#{member}-kids")
|
|
561
|
+
end
|
|
562
|
+
end
|
|
563
|
+
end
|
|
564
|
+
generation = replies.each_slice(2).flat_map { |_, kids| kids || [] }
|
|
554
565
|
end
|
|
555
566
|
end
|
|
556
567
|
|
data/lib/wurk/batch_set.rb
CHANGED
|
@@ -11,6 +11,7 @@ module Wurk
|
|
|
11
11
|
include Enumerable
|
|
12
12
|
|
|
13
13
|
PAGE_SIZE = 100
|
|
14
|
+
TAG_SCAN_LIMIT = 20_000
|
|
14
15
|
|
|
15
16
|
def initialize(key: 'batches')
|
|
16
17
|
@key = key
|
|
@@ -52,6 +53,21 @@ module Wurk
|
|
|
52
53
|
end
|
|
53
54
|
end
|
|
54
55
|
|
|
56
|
+
# #snapshot narrowed to the batches tagged `tag` and still indexed in
|
|
57
|
+
# this set: [how many, the page of summaries]. The tag index is a SET, so
|
|
58
|
+
# ordering it newest-first reads every member's score (one ZMSCORE) —
|
|
59
|
+
# bounded at TAG_SCAN_LIMIT members, past which the rest are not listed.
|
|
60
|
+
# Internal reader for the dashboard.
|
|
61
|
+
#
|
|
62
|
+
# @return [Array(Integer, Array<Hash>)]
|
|
63
|
+
def tagged_snapshot(tag, offset, count)
|
|
64
|
+
bids = scan_tags(tag).first(TAG_SCAN_LIMIT)
|
|
65
|
+
Wurk.redis do |conn|
|
|
66
|
+
ranked = newest_first(bids, bids.empty? ? [] : conn.call('ZMSCORE', @key, *bids))
|
|
67
|
+
[ranked.size, summaries(conn, ranked[offset, count] || [])]
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
55
71
|
# Yields each BID indexed under the given tag (set at `tags:<tag>`).
|
|
56
72
|
# No JSON parsing; cheap iteration. Caller wraps in `Status.new(bid)`
|
|
57
73
|
# if it needs full state.
|
|
@@ -78,6 +94,12 @@ module Wurk
|
|
|
78
94
|
end
|
|
79
95
|
end
|
|
80
96
|
|
|
97
|
+
# The bids `scores` ranks, in ZRANGE REV order (score, then member,
|
|
98
|
+
# descending); an unscored bid is not in this set.
|
|
99
|
+
def newest_first(bids, scores)
|
|
100
|
+
bids.zip(scores).select(&:last).sort { |(a, sa), (b, sb)| [sb.to_f, b] <=> [sa.to_f, a] }.map(&:first)
|
|
101
|
+
end
|
|
102
|
+
|
|
81
103
|
def summaries(conn, bids)
|
|
82
104
|
return [] if bids.empty?
|
|
83
105
|
|
data/lib/wurk/cli.rb
CHANGED
|
@@ -119,7 +119,8 @@ module Wurk
|
|
|
119
119
|
['-w', '--window SECONDS', :health_window, 'A heartbeat older than this does not count (default 60)', :to_i],
|
|
120
120
|
['-m', '--min COUNT', :health_min, 'Processes on this host that must be beating (default 1)', :to_i],
|
|
121
121
|
['-u', '--redis-url URL', :health_redis_url, 'Redis to read (default REDIS_URL)'],
|
|
122
|
-
['-s', '--strict', :health_strict, 'Fail when Redis cannot be read, rather than passing']
|
|
122
|
+
['-s', '--strict', :health_strict, 'Fail when Redis cannot be read, rather than passing'],
|
|
123
|
+
['-a', '--max-job-age SECONDS', :health_max_job_age, 'Also fail when a job here has run this long (off)', :to_i]
|
|
123
124
|
].freeze
|
|
124
125
|
|
|
125
126
|
HELP_COMMANDS = "\nCommands:\n " \
|
|
@@ -298,7 +299,8 @@ module Wurk
|
|
|
298
299
|
opts = @health_options
|
|
299
300
|
@config.redis = { url: opts[:health_redis_url] } if opts[:health_redis_url]
|
|
300
301
|
HealthProbe.new(@config, window: opts[:health_window] || HealthProbe::DEFAULT_WINDOW,
|
|
301
|
-
min: opts[:health_min] || 1, strict: opts[:health_strict] == true
|
|
302
|
+
min: opts[:health_min] || 1, strict: opts[:health_strict] == true,
|
|
303
|
+
max_job_age: opts[:health_max_job_age]).call
|
|
302
304
|
end
|
|
303
305
|
|
|
304
306
|
# `wurk api` — serve the machine-facing HTTP API and nothing else: no
|
data/lib/wurk/client/buffered.rb
CHANGED
|
@@ -24,6 +24,12 @@ module Wurk
|
|
|
24
24
|
DEFAULT_BUFFER_CAP = 1_000
|
|
25
25
|
DRAINING_KEY = :wurk_reliable_push_draining
|
|
26
26
|
UNBUFFERED_KEY = :wurk_reliable_push_unbuffered
|
|
27
|
+
# What a failed replay's raw_push had already landed, handed from the
|
|
28
|
+
# prepended raw_push to Buffered.attempt_replay.
|
|
29
|
+
REPLAY_ACKED_KEY = :wurk_reliable_push_replay_acked
|
|
30
|
+
# Payloads per replayed raw_push. A backlog reaches Redis in about
|
|
31
|
+
# N / REPLAY_SLICE calls, each one pipeline per queue.
|
|
32
|
+
REPLAY_SLICE = 100
|
|
27
33
|
|
|
28
34
|
# A clean exit tries to deliver what is still in memory, but never holds the
|
|
29
35
|
# process up for longer than this: a hung Redis would otherwise stall shutdown.
|
|
@@ -528,15 +534,17 @@ module Wurk
|
|
|
528
534
|
end
|
|
529
535
|
|
|
530
536
|
# Leaves the circuit open on the first failed replay and closed once the
|
|
531
|
-
# backlog for this Redis has run out.
|
|
537
|
+
# backlog for this Redis has run out. Jobs buffered while a pass is in
|
|
538
|
+
# flight are taken by the next pass of the loop.
|
|
532
539
|
def replay_all(client, target)
|
|
533
540
|
drained = 0
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
541
|
+
until (entries = take_all(target)).empty?
|
|
542
|
+
landed = replay_entries(client, entries)
|
|
543
|
+
drained += landed
|
|
544
|
+
next if landed == entries.size
|
|
545
|
+
|
|
546
|
+
trip!(target)
|
|
547
|
+
return drained
|
|
540
548
|
end
|
|
541
549
|
circuit.close(target)
|
|
542
550
|
drained
|
|
@@ -555,72 +563,113 @@ module Wurk
|
|
|
555
563
|
end
|
|
556
564
|
end
|
|
557
565
|
|
|
558
|
-
#
|
|
559
|
-
|
|
566
|
+
# Every entry bound for `target`, oldest first, removed from the buffer
|
|
567
|
+
# in one pass.
|
|
568
|
+
def take_all(target)
|
|
560
569
|
buffer_mutex.synchronize do
|
|
561
|
-
|
|
562
|
-
next
|
|
570
|
+
taken, kept = buffer.partition { |entry| entry.origin.resolve.equal?(target) }
|
|
571
|
+
next taken if taken.empty?
|
|
563
572
|
|
|
564
|
-
|
|
565
|
-
|
|
573
|
+
buffer.replace(kept)
|
|
574
|
+
@buffer_bytes -= taken.sum(&:bytes)
|
|
575
|
+
@replaying += taken.size
|
|
576
|
+
taken
|
|
566
577
|
end
|
|
567
578
|
end
|
|
568
579
|
|
|
569
|
-
def replay_settled
|
|
570
|
-
buffer_mutex.synchronize { @replaying = [@replaying -
|
|
580
|
+
def replay_settled(count)
|
|
581
|
+
buffer_mutex.synchronize { @replaying = [@replaying - count, 0].max }
|
|
571
582
|
end
|
|
572
583
|
|
|
573
584
|
def buffered_pools
|
|
574
585
|
buffer_mutex.synchronize { buffer.map { |entry| entry.origin.resolve } }.uniq(&:object_id)
|
|
575
586
|
end
|
|
576
587
|
|
|
577
|
-
#
|
|
578
|
-
#
|
|
579
|
-
#
|
|
580
|
-
#
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
restore(entry)
|
|
596
|
-
return
|
|
588
|
+
# Replays `entries` slice by slice and returns how many reached Redis.
|
|
589
|
+
# On the first slice that does not fully land, what did not land and
|
|
590
|
+
# every later slice go back at the absolute head of the buffer, in
|
|
591
|
+
# order: same-origin order is all that matters (other entries go to
|
|
592
|
+
# another server).
|
|
593
|
+
#
|
|
594
|
+
# A slice is a run of immediate or of scheduled payloads, because
|
|
595
|
+
# Client#atomic_push routes a whole call by its first payload.
|
|
596
|
+
def replay_entries(client, entries)
|
|
597
|
+
landed = 0
|
|
598
|
+
remaining = entries
|
|
599
|
+
replay_slices(entries).each do |slice|
|
|
600
|
+
remaining = remaining.drop(slice.size)
|
|
601
|
+
undelivered, error = replay(client, slice)
|
|
602
|
+
landed += slice.size - undelivered.size
|
|
603
|
+
next if undelivered.empty? && error.nil?
|
|
604
|
+
|
|
605
|
+
return give_back(client, undelivered + remaining, error, landed)
|
|
597
606
|
end
|
|
598
|
-
|
|
599
|
-
client.send(:emit_enqueued, [entry.payload])
|
|
600
|
-
Wurk::Metrics::Statsd.increment('jobs.recovered.push')
|
|
601
|
-
entry
|
|
607
|
+
landed
|
|
602
608
|
ensure
|
|
603
|
-
replay_settled
|
|
609
|
+
replay_settled(entries.size)
|
|
610
|
+
end
|
|
611
|
+
|
|
612
|
+
def replay_slices(entries)
|
|
613
|
+
entries.chunk_while { |a, b| a.payload.key?('at') == b.payload.key?('at') }
|
|
614
|
+
.flat_map { |run| run.each_slice(REPLAY_SLICE).to_a }
|
|
604
615
|
end
|
|
605
616
|
|
|
606
|
-
|
|
617
|
+
# Puts `entries` back and returns `landed`. Non-connection failures
|
|
618
|
+
# (OOM, LOADING, READONLY…) must not drop the taken payloads either, or
|
|
619
|
+
# a recovering-but-not-ready Redis silently eats buffered jobs; they
|
|
620
|
+
# are restored before the error is raised. Redis answered,
|
|
621
|
+
# so the circuit closes and pushes go back to surfacing the refusal to
|
|
622
|
+
# their callers.
|
|
623
|
+
def give_back(client, entries, error, landed)
|
|
624
|
+
restore(entries)
|
|
625
|
+
return landed unless error
|
|
626
|
+
|
|
627
|
+
circuit.close(client.send(:pool))
|
|
628
|
+
raise error
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
# [entries that did not reach Redis, the non-connection error or nil].
|
|
632
|
+
def replay(client, slice)
|
|
633
|
+
acknowledged, error = attempt_replay(client, slice.map(&:payload))
|
|
634
|
+
landed, undelivered = acknowledged ? split_acknowledged(slice, acknowledged) : [slice, []]
|
|
635
|
+
payloads = landed.map(&:payload)
|
|
636
|
+
client.send(:emit_enqueued, payloads)
|
|
637
|
+
payloads.each { Wurk::Metrics::Statsd.increment('jobs.recovered.push') }
|
|
638
|
+
[undelivered, error]
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
def split_acknowledged(slice, acknowledged)
|
|
642
|
+
seen = {}.compare_by_identity
|
|
643
|
+
acknowledged.each { |payload| seen[payload] = true }
|
|
644
|
+
slice.partition { |entry| seen.key?(entry.payload) }
|
|
645
|
+
end
|
|
646
|
+
|
|
647
|
+
def restore(entries)
|
|
648
|
+
return if entries.empty?
|
|
649
|
+
|
|
607
650
|
buffer_mutex.synchronize do
|
|
608
|
-
@buffer_bytes +=
|
|
609
|
-
buffer.unshift(
|
|
651
|
+
@buffer_bytes += entries.sum(&:bytes)
|
|
652
|
+
buffer.unshift(*entries)
|
|
610
653
|
end
|
|
611
654
|
end
|
|
612
655
|
|
|
613
656
|
# Drain marks the thread so our prepended raw_push re-raises the
|
|
614
657
|
# transient error back here instead of swallowing it into the buffer
|
|
615
|
-
# (which would spin forever).
|
|
616
|
-
|
|
658
|
+
# (which would spin forever). Returns [nil, nil] when the call landed,
|
|
659
|
+
# else [the payloads Redis acknowledged, the non-connection error or
|
|
660
|
+
# nil]: one call is a pipeline per queue, so it can fail after some
|
|
661
|
+
# of them landed.
|
|
662
|
+
def attempt_replay(client, payloads)
|
|
617
663
|
Thread.current[DRAINING_KEY] = true
|
|
618
|
-
client.send(:raw_push,
|
|
619
|
-
|
|
664
|
+
client.send(:raw_push, payloads)
|
|
665
|
+
[nil, nil]
|
|
620
666
|
rescue RedisClient::ConnectionError, ConnectionPool::TimeoutError
|
|
621
|
-
|
|
667
|
+
[Thread.current[REPLAY_ACKED_KEY] || [], nil]
|
|
668
|
+
rescue StandardError => e
|
|
669
|
+
[Thread.current[REPLAY_ACKED_KEY] || [], e]
|
|
622
670
|
ensure
|
|
623
671
|
Thread.current[DRAINING_KEY] = false
|
|
672
|
+
Thread.current[REPLAY_ACKED_KEY] = nil
|
|
624
673
|
end
|
|
625
674
|
end
|
|
626
675
|
|
|
@@ -672,6 +721,7 @@ module Wurk
|
|
|
672
721
|
# replace the push's own result (its return, or the Overflow it is
|
|
673
722
|
# raising), so the metric is strictly best-effort.
|
|
674
723
|
def count_drop
|
|
724
|
+
Wurk::Metrics::ProcessCounters.increment(:client_buffer_drops)
|
|
675
725
|
Wurk::Metrics::Statsd.increment('jobs.dropped.push')
|
|
676
726
|
rescue StandardError
|
|
677
727
|
nil
|
|
@@ -820,6 +870,12 @@ module Wurk
|
|
|
820
870
|
|
|
821
871
|
buffer_undelivered(payloads)
|
|
822
872
|
ensure
|
|
873
|
+
close_ledger
|
|
874
|
+
end
|
|
875
|
+
|
|
876
|
+
def close_ledger
|
|
877
|
+
ledger = Thread.current[DELIVERED_KEY]
|
|
878
|
+
Thread.current[Buffered::REPLAY_ACKED_KEY] = ledger if Thread.current[Buffered::DRAINING_KEY]
|
|
823
879
|
Thread.current[DELIVERED_KEY] = nil
|
|
824
880
|
end
|
|
825
881
|
|
data/lib/wurk/configuration.rb
CHANGED
|
@@ -60,6 +60,9 @@ module Wurk
|
|
|
60
60
|
# own behavior, where a job that succeeds leaves nothing behind.
|
|
61
61
|
status_ttl: Keys::STATUS_TTL,
|
|
62
62
|
status_retention: nil,
|
|
63
|
+
# TTL in seconds of the 10-minute `j|` metrics buckets (Metrics::History);
|
|
64
|
+
# nil keeps MID_TERM, 30 days. See #metrics_retention=.
|
|
65
|
+
metrics_retention: nil,
|
|
63
66
|
# Distributed tracing (Wurk::Telemetry). Seeded here, not written on
|
|
64
67
|
# demand, so the read side answers on a config that `freeze!` has already
|
|
65
68
|
# closed — and so a gem inspecting @options sees the same key set whether
|
|
@@ -186,6 +189,7 @@ module Wurk
|
|
|
186
189
|
def []=(key, val)
|
|
187
190
|
guard_frozen!
|
|
188
191
|
RedisPool.validate_idle_timeout(val) if key == :redis_idle_timeout
|
|
192
|
+
val = parse_metrics_retention(val) if key == :metrics_retention && !val.nil?
|
|
189
193
|
@options[key] = val
|
|
190
194
|
end
|
|
191
195
|
|
|
@@ -195,6 +199,7 @@ module Wurk
|
|
|
195
199
|
def merge!(other)
|
|
196
200
|
guard_frozen!
|
|
197
201
|
RedisPool.validate_idle_timeout(other[:redis_idle_timeout]) if other.key?(:redis_idle_timeout)
|
|
202
|
+
parse_metrics_retention(other[:metrics_retention]) unless other[:metrics_retention].nil?
|
|
198
203
|
@options.merge!(other)
|
|
199
204
|
end
|
|
200
205
|
|
|
@@ -973,6 +978,28 @@ module Wurk
|
|
|
973
978
|
# WURK_MAXMEM_METRIC sets it, an explicit value wins.
|
|
974
979
|
MEMORY_LIMIT_METRICS = %i[rss pss uss].freeze
|
|
975
980
|
|
|
981
|
+
# Upstream's own TTL for the 10-minute buckets; `hours:` windows read up to
|
|
982
|
+
# 72 hours of them, so nothing shorter is accepted.
|
|
983
|
+
METRICS_RETENTION_FLOOR = 3 * 24 * 60 * 60
|
|
984
|
+
|
|
985
|
+
# How long the 10-minute `j|` metrics buckets live. A TTL only: the keys,
|
|
986
|
+
# fields and scores stay Sidekiq's. Each live job class costs 3 fields per
|
|
987
|
+
# bucket, one bucket per 10 minutes, so retention scales those keys
|
|
988
|
+
# linearly (docs/metrics.md, "Redis footprint and retention"). nil restores
|
|
989
|
+
# the 30-day default.
|
|
990
|
+
def metrics_retention=(seconds)
|
|
991
|
+
guard_frozen!
|
|
992
|
+
@options[:metrics_retention] = seconds.nil? ? nil : parse_metrics_retention(seconds)
|
|
993
|
+
end
|
|
994
|
+
|
|
995
|
+
def parse_metrics_retention(seconds)
|
|
996
|
+
value = seconds.respond_to?(:to_i) ? seconds.to_i : 0
|
|
997
|
+
return value if value >= METRICS_RETENTION_FLOOR
|
|
998
|
+
|
|
999
|
+
raise ArgumentError, "metrics_retention must be >= #{METRICS_RETENTION_FLOOR}s (3 days), got #{seconds.inspect}"
|
|
1000
|
+
end
|
|
1001
|
+
private :parse_metrics_retention
|
|
1002
|
+
|
|
976
1003
|
def memory_limit_metric
|
|
977
1004
|
@memory_limit_metric || env_memory_limit_metric || :rss
|
|
978
1005
|
end
|
|
@@ -1101,6 +1128,7 @@ module Wurk
|
|
|
1101
1128
|
def validate_constructor_options!
|
|
1102
1129
|
@options[:global_concurrency] = normalize_global_concurrency(@options[:global_concurrency])
|
|
1103
1130
|
RedisPool.validate_idle_timeout(@options[:redis_idle_timeout])
|
|
1131
|
+
parse_metrics_retention(@options[:metrics_retention]) unless @options[:metrics_retention].nil?
|
|
1104
1132
|
end
|
|
1105
1133
|
|
|
1106
1134
|
# One flat fork running the default capsule's queues + concurrency. The
|
|
@@ -62,22 +62,32 @@ module Wurk
|
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
# `[public, private, public, private, ...]` for queues 2..n, fetch_first's
|
|
65
|
-
# KEYS
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
#
|
|
65
|
+
# KEYS. A list #queues_cmd reuses across passes comes frozen (strict, with
|
|
66
|
+
# or without a pause), and its KEYS are built once and kept, keyed on the
|
|
67
|
+
# list's identity and the pid (for the reason #queue_keys is).
|
|
68
|
+
# Copy-on-write: processor threads share this fetcher. A shuffled list is
|
|
69
|
+
# new every pass, so its KEYS go into this thread's buffer instead: the
|
|
70
|
+
# script call copies them out before the buffer is written again.
|
|
70
71
|
def claim_keys(queues)
|
|
71
72
|
pid = PidCache.pid
|
|
72
73
|
cached = @claim_keys
|
|
73
74
|
return cached[2] if cached && cached[0].equal?(queues) && cached[1] == pid
|
|
75
|
+
return fill_claim_keys(claim_keys_buffer, queues) unless queues.frozen?
|
|
74
76
|
|
|
75
|
-
keys = []
|
|
76
|
-
queues.each_with_index { |public_q, i| keys << public_q << queue_keys(public_q).first unless i.zero? }
|
|
77
|
-
keys.freeze
|
|
77
|
+
keys = fill_claim_keys([], queues).freeze
|
|
78
78
|
@claim_keys = [queues, pid, keys].freeze
|
|
79
79
|
keys
|
|
80
80
|
end
|
|
81
|
+
|
|
82
|
+
def fill_claim_keys(keys, queues)
|
|
83
|
+
keys.clear
|
|
84
|
+
queues.each_with_index { |public_q, i| keys << public_q << queue_keys(public_q).first unless i.zero? }
|
|
85
|
+
keys
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def claim_keys_buffer
|
|
89
|
+
::Thread.current[:wurk_claim_keys] ||= []
|
|
90
|
+
end
|
|
81
91
|
end
|
|
82
92
|
end
|
|
83
93
|
end
|