wurk 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +40 -2
- data/app/controllers/wurk/api/serializers.rb +51 -1
- data/app/controllers/wurk/api_controller.rb +42 -6
- data/app/controllers/wurk/dashboard_controller.rb +67 -10
- data/config/routes.rb +30 -3
- data/exe/wurk +6 -2
- data/lib/generators/wurk/install/install_generator.rb +3 -3
- data/lib/sidekiq/job_retry.rb +4 -0
- data/lib/sidekiq/manager.rb +4 -0
- data/lib/sidekiq/processor.rb +4 -0
- data/lib/wurk/api/app.rb +186 -0
- data/lib/wurk/api/auth.rb +167 -0
- data/lib/wurk/api/flows.rb +66 -0
- data/lib/wurk/api/idempotency.rb +168 -0
- data/lib/wurk/api/jobs.rb +180 -0
- data/lib/wurk/api/page.rb +98 -0
- data/lib/wurk/api/problem.rb +121 -0
- data/lib/wurk/api/queues.rb +126 -0
- data/lib/wurk/api/read_only.rb +68 -0
- data/lib/wurk/api/request.rb +46 -0
- data/lib/wurk/api/response.rb +28 -0
- data/lib/wurk/api/roll_up.rb +126 -0
- data/lib/wurk/api/router.rb +88 -0
- data/lib/wurk/api/serializers.rb +232 -0
- data/lib/wurk/api/swarm.rb +297 -0
- data/lib/wurk/api/throttle.rb +109 -0
- data/lib/wurk/api/validation.rb +306 -0
- data/lib/wurk/api.rb +96 -0
- data/lib/wurk/batch/server_middleware.rb +2 -2
- data/lib/wurk/batch.rb +12 -3
- data/lib/wurk/capsule.rb +8 -1
- data/lib/wurk/cli.rb +110 -8
- data/lib/wurk/client/buffered.rb +1 -1
- data/lib/wurk/client.rb +42 -3
- data/lib/wurk/collapse.rb +383 -0
- data/lib/wurk/command_builder.rb +49 -0
- data/lib/wurk/compat.rb +19 -0
- data/lib/wurk/component.rb +16 -3
- data/lib/wurk/configuration.rb +303 -4
- data/lib/wurk/debounce.rb +125 -0
- data/lib/wurk/encryption.rb +6 -1
- data/lib/wurk/fetcher/capped.rb +218 -0
- data/lib/wurk/fetcher/reliable.rb +22 -66
- data/lib/wurk/fetcher/unit_of_work.rb +107 -0
- data/lib/wurk/flow/builder.rb +271 -0
- data/lib/wurk/flow/chain.rb +42 -0
- data/lib/wurk/flow/completion.rb +114 -0
- data/lib/wurk/flow/creation.rb +254 -0
- data/lib/wurk/flow/node.rb +124 -0
- data/lib/wurk/flow/status.rb +206 -0
- data/lib/wurk/flow.rb +255 -0
- data/lib/wurk/flow_set.rb +53 -0
- data/lib/wurk/health.rb +8 -7
- data/lib/wurk/heartbeat.rb +21 -3
- data/lib/wurk/job/options.rb +11 -1
- data/lib/wurk/job.rb +19 -0
- data/lib/wurk/job_logger.rb +6 -2
- data/lib/wurk/job_retry.rb +6 -1
- data/lib/wurk/job_util.rb +110 -9
- data/lib/wurk/keys.rb +125 -0
- data/lib/wurk/launcher.rb +1 -1
- data/lib/wurk/leader.rb +1 -1
- data/lib/wurk/limiter/bucket.rb +1 -1
- data/lib/wurk/limiter/points.rb +1 -1
- data/lib/wurk/lua/debounce.lua +75 -0
- data/lib/wurk/lua/fetch_slot.lua +83 -0
- data/lib/wurk/lua/flow_abandon.lua +65 -0
- data/lib/wurk/lua/flow_advance.lua +177 -0
- data/lib/wurk/lua/flow_create.lua +141 -0
- data/lib/wurk/lua/flow_fail.lua +52 -0
- data/lib/wurk/lua/limiter_bucket_acquire.lua +26 -0
- data/lib/wurk/lua/limiter_concurrent_acquire.lua +33 -0
- data/lib/wurk/lua/limiter_concurrent_release.lua +7 -0
- data/lib/wurk/lua/limiter_leaky_acquire.lua +31 -0
- data/lib/wurk/lua/limiter_list_sweep.lua +30 -0
- data/lib/wurk/lua/limiter_points_acquire.lua +34 -0
- data/lib/wurk/lua/limiter_points_refund.lua +18 -0
- data/lib/wurk/lua/limiter_register.lua +27 -0
- data/lib/wurk/lua/limiter_window_acquire.lua +35 -0
- data/lib/wurk/lua/limiter_window_status.lua +22 -0
- data/lib/wurk/lua/loader.rb +23 -0
- data/lib/wurk/lua/queue_slot.lua +83 -0
- data/lib/wurk/lua/refresh_slots.lua +38 -0
- data/lib/wurk/lua/status_write.lua +29 -0
- data/lib/wurk/lua/throttle_slot.lua +71 -0
- data/lib/wurk/lua.rb +3 -3
- data/lib/wurk/manager.rb +12 -0
- data/lib/wurk/metrics/history.rb +34 -2
- data/lib/wurk/metrics/query.rb +1 -1
- data/lib/wurk/metrics/statsd.rb +14 -0
- data/lib/wurk/middleware/expiry.rb +71 -10
- data/lib/wurk/middleware/status.rb +274 -0
- data/lib/wurk/middleware/timeout.rb +137 -0
- data/lib/wurk/pid_cache.rb +43 -0
- data/lib/wurk/processor.rb +66 -22
- data/lib/wurk/profiler.rb +5 -3
- data/lib/wurk/queue_slot.rb +285 -0
- data/lib/wurk/rails.rb +3 -3
- data/lib/wurk/redis_pool.rb +5 -2
- data/lib/wurk/status/progress.rb +103 -0
- data/lib/wurk/status/record.rb +81 -0
- data/lib/wurk/status.rb +155 -0
- data/lib/wurk/telemetry/client_middleware.rb +59 -0
- data/lib/wurk/telemetry/server_middleware.rb +152 -0
- data/lib/wurk/telemetry.rb +137 -0
- data/lib/wurk/throttle.rb +142 -0
- data/lib/wurk/unique.rb +10 -2
- data/lib/wurk/version.rb +1 -1
- data/lib/wurk/watchdog.rb +188 -0
- data/lib/wurk/web/config.rb +77 -10
- data/lib/wurk/web/extension.rb +12 -4
- data/lib/wurk/web/locale_negotiator.rb +67 -0
- data/lib/wurk/web.rb +1 -0
- data/lib/wurk/worker.rb +39 -4
- data/lib/wurk.rb +42 -1
- data/vendor/assets/dashboard/assets/ArgsValue-D-x_ifLY.js +1 -0
- data/vendor/assets/dashboard/assets/BatchDetail-C39NJuew.js +1 -0
- data/vendor/assets/dashboard/assets/Batches-CSwo7Asa.js +1 -0
- data/vendor/assets/dashboard/assets/Busy-BOFMu-sq.js +1 -0
- data/vendor/assets/dashboard/assets/Cron-Dy8RQzDI.js +1 -0
- data/vendor/assets/dashboard/assets/Dashboard-BuTHI-O1.js +1 -0
- data/vendor/assets/dashboard/assets/Dead-B9KRvQ0N.js +1 -0
- data/vendor/assets/dashboard/assets/Extension-BnBVHfux.js +1 -0
- data/vendor/assets/dashboard/assets/FilterBox-DC24zite.js +1 -0
- data/vendor/assets/dashboard/assets/FlowDetail-DyLuzUvt.js +1 -0
- data/vendor/assets/dashboard/assets/FlowState-DAPKUahm.js +1 -0
- data/vendor/assets/dashboard/assets/Flows-Fr3rjZM_.js +1 -0
- data/vendor/assets/dashboard/assets/JobDetailModal-N6kiJXq3.js +2 -0
- data/vendor/assets/dashboard/assets/Limiters-kbFA7uS1.js +1 -0
- data/vendor/assets/dashboard/assets/Metrics-Dj2uoZ3o.js +1 -0
- data/vendor/assets/dashboard/assets/PageHeader-B_F94azl.js +1 -0
- data/vendor/assets/dashboard/assets/Profiles-D_DjEezN.js +1 -0
- data/vendor/assets/dashboard/assets/Queues-CO4V9hAz.js +1 -0
- data/vendor/assets/dashboard/assets/Retries-DCWnzeLa.js +1 -0
- data/vendor/assets/dashboard/assets/Scheduled-BebDUjLU.js +1 -0
- data/vendor/assets/dashboard/assets/Search-Cvr5fy4Y.js +1 -0
- data/vendor/assets/dashboard/assets/Skeleton-Bu3Ke6rV.js +1 -0
- data/vendor/assets/dashboard/assets/charts-BCs9bQKz.js +1 -0
- data/vendor/assets/dashboard/assets/index-BIwyOC5Q.js +141 -0
- data/vendor/assets/dashboard/assets/index-DBQN6Jk8.css +1 -0
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-Bzh-BJyL.js +1 -0
- data/vendor/assets/dashboard/assets/useSort-COA3fVJ5.js +1 -0
- data/vendor/assets/dashboard/assets/utils-BIrvZ1hi.js +1 -0
- data/vendor/assets/dashboard/index.html +42 -11
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +102 -33
- data/vendor/assets/dashboard/assets/ArgsValue-CcR2ya6e.js +0 -1
- data/vendor/assets/dashboard/assets/BatchDetail-CUXJUQ3Q.js +0 -1
- data/vendor/assets/dashboard/assets/Batches-Cxan6Ngw.js +0 -1
- data/vendor/assets/dashboard/assets/Busy-DC5EGM0g.js +0 -1
- data/vendor/assets/dashboard/assets/Cron-Dlt8tXJA.js +0 -1
- data/vendor/assets/dashboard/assets/Dashboard-DNLu_WCg.js +0 -1
- data/vendor/assets/dashboard/assets/Dead-dZ7VGlKS.js +0 -1
- data/vendor/assets/dashboard/assets/Extension-DaFpEIJf.js +0 -1
- data/vendor/assets/dashboard/assets/FilterBox-CO3aYWIq.js +0 -1
- data/vendor/assets/dashboard/assets/JobDetailModal-DSWbT6G0.js +0 -2
- data/vendor/assets/dashboard/assets/Limiters-Cb4PKXNR.js +0 -1
- data/vendor/assets/dashboard/assets/Metrics-CCGzgCsT.js +0 -1
- data/vendor/assets/dashboard/assets/Modal-B86q6ruL.js +0 -1
- data/vendor/assets/dashboard/assets/PageHeader-fPrCcp_-.js +0 -1
- data/vendor/assets/dashboard/assets/Profiles-BnS82nR_.js +0 -1
- data/vendor/assets/dashboard/assets/Queues-CIyPevOy.js +0 -1
- data/vendor/assets/dashboard/assets/Retries-DopwXkXl.js +0 -1
- data/vendor/assets/dashboard/assets/Scheduled-1-Z7i1zE.js +0 -1
- data/vendor/assets/dashboard/assets/Search-ByA6eTma.js +0 -1
- data/vendor/assets/dashboard/assets/Skeleton-bC7HfQ9r.js +0 -1
- data/vendor/assets/dashboard/assets/charts-CLLzJ7vK.js +0 -1
- data/vendor/assets/dashboard/assets/index-B1N8hQUh.js +0 -141
- data/vendor/assets/dashboard/assets/index-BdiUEDXX.css +0 -1
- data/vendor/assets/dashboard/assets/useResetPageOnEmpty-DpBjkf6_.js +0 -1
- data/vendor/assets/dashboard/assets/useSort-DvpwuNQE.js +0 -1
- data/vendor/assets/dashboard/assets/utils-DDJC7tJV.js +0 -1
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require_relative 'problem'
|
|
5
|
+
|
|
6
|
+
module Wurk
|
|
7
|
+
module API
|
|
8
|
+
# Everything the API refuses at its boundary, before a Wurk object sees it.
|
|
9
|
+
#
|
|
10
|
+
# Client validates a job hash written by Ruby running in this process. This
|
|
11
|
+
# validates one typed by a stranger, and the two are different jobs.
|
|
12
|
+
# `JobUtil::TRANSIENT_ATTRIBUTES` (job_util.rb) is a *strip*-list: it names
|
|
13
|
+
# the two keys that must never reach the wire and passes everything else
|
|
14
|
+
# through, which is the right default for a producer that already runs your
|
|
15
|
+
# code and the wrong one for a producer that doesn't. So the allow-list
|
|
16
|
+
# JobUtil has no reason to own lives here, at the only door a stranger can
|
|
17
|
+
# knock on.
|
|
18
|
+
#
|
|
19
|
+
# Nothing in this file changes what a Ruby `perform_async` may push. The
|
|
20
|
+
# boundary is the HTTP request, not the payload format — wire-compat is
|
|
21
|
+
# untouched.
|
|
22
|
+
module Validation
|
|
23
|
+
# Carries the problem it should be rendered as, so a route rescues one
|
|
24
|
+
# class and renders whatever it was handed instead of mapping message
|
|
25
|
+
# text back onto a status code.
|
|
26
|
+
class Invalid < StandardError
|
|
27
|
+
attr_reader :type, :status, :extra
|
|
28
|
+
|
|
29
|
+
def initialize(detail, type: Problem::INVALID_REQUEST, status: 400, **extra)
|
|
30
|
+
super(detail)
|
|
31
|
+
@type = type
|
|
32
|
+
@status = status
|
|
33
|
+
@extra = extra
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# A Ruby constant path and nothing else. `class` reaches the consumer as
|
|
38
|
+
# a constantize plus `new.perform`, so the string is a code selector, not
|
|
39
|
+
# data: any shape that is not a constant path is either an attack or a
|
|
40
|
+
# bug, and both are better answered here than by a NameError in a worker
|
|
41
|
+
# process two hosts away.
|
|
42
|
+
#
|
|
43
|
+
# Linear, not backtracking: `:` is outside the character class, so each
|
|
44
|
+
# `::` boundary is unambiguous.
|
|
45
|
+
CLASS_FORMAT = /\A[A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*\z/
|
|
46
|
+
|
|
47
|
+
# Length is checked separately from shape so a megabyte of `A`s is
|
|
48
|
+
# refused on a fact about the string rather than matched, stored in a
|
|
49
|
+
# payload and echoed back in an error.
|
|
50
|
+
MAX_CLASS_LENGTH = 255
|
|
51
|
+
|
|
52
|
+
# A jid reaches Redis as a ZSCAN glob (JobSet#find_job wraps it in `*`),
|
|
53
|
+
# so an unbounded or metacharacter-carrying jid off the network is a scan
|
|
54
|
+
# this API runs on the client's behalf. Sidekiq's own jids are
|
|
55
|
+
# `SecureRandom.hex(12)`; admit any URL-safe token of a sane length and
|
|
56
|
+
# refuse the rest at the door.
|
|
57
|
+
JID_FORMAT = /\A[A-Za-z0-9_-]{1,255}\z/
|
|
58
|
+
|
|
59
|
+
# A queue name arrives as one path segment (percent-decoded by the
|
|
60
|
+
# router, so a name containing '/' is reachable) and reaches Redis as
|
|
61
|
+
# `queue:<name>`. The Sidekiq schema bounds it nowhere, so this door
|
|
62
|
+
# does: whitespace and control characters are out because a name
|
|
63
|
+
# carrying them is one no operator can type back into the dashboard or a
|
|
64
|
+
# CLI flag, and the length cap exists for the same reason a jid's does —
|
|
65
|
+
# it is echoed back on every listing.
|
|
66
|
+
QUEUE_NAME_FORMAT = /\A[^[:space:][:cntrl:]]{1,255}\z/
|
|
67
|
+
|
|
68
|
+
# A process identity is `<hostname>:<pid>:<nonce>` and is itself a Redis
|
|
69
|
+
# key — the heartbeat HASH, `<identity>:work` and `<identity>-signals`
|
|
70
|
+
# are all built by interpolating it. The same shape as a queue name for
|
|
71
|
+
# the same reason, spelled separately because the two doors are
|
|
72
|
+
# independent: bounding what may be addressed as a queue says nothing
|
|
73
|
+
# about what may be addressed as a process.
|
|
74
|
+
IDENTITY_FORMAT = /\A[^[:space:][:cntrl:]]{1,255}\z/
|
|
75
|
+
|
|
76
|
+
# Same word the router uses for "any scope": here it turns the class
|
|
77
|
+
# allow-list off.
|
|
78
|
+
ANY = :any
|
|
79
|
+
|
|
80
|
+
# The top-level keys a job hash may carry in over HTTP.
|
|
81
|
+
#
|
|
82
|
+
# Anything absent is refused rather than dropped: a silently ignored
|
|
83
|
+
# `deadlin:` typo is a job that runs unbounded and a producer that never
|
|
84
|
+
# finds out. The server-written fields are absent on purpose —
|
|
85
|
+
# `created_at`/`enqueued_at` would let a client backdate its own queue
|
|
86
|
+
# latency, `retry_count`/`failed_at` would let it lie to the retrier,
|
|
87
|
+
# `expiry`/`deadline_at` are stamps derived from `expires_in`/`deadline`,
|
|
88
|
+
# `bid` would attach a job to a batch whose `pending` counter was never
|
|
89
|
+
# incremented for it, and `pool`/`client_class` name Ruby objects a JSON
|
|
90
|
+
# body cannot hold at all.
|
|
91
|
+
#
|
|
92
|
+
# Mutable for the same reason `TRANSIENT_ATTRIBUTES` is: a gem that adds
|
|
93
|
+
# a job option (sidekiq-unique-jobs' `lock:`) has to be able to append
|
|
94
|
+
# one without reopening this file. Baked into the literal rather than
|
|
95
|
+
# appended at load, so the parallel test runner never observes a
|
|
96
|
+
# half-built list.
|
|
97
|
+
# rubocop:disable Style/MutableConstant
|
|
98
|
+
JOB_KEYS = %w[
|
|
99
|
+
class args queue at retry jid backtrace tags dead
|
|
100
|
+
retry_for retry_queue expires_in track timeout deadline
|
|
101
|
+
encrypt unique_for unique_until collapse log_level locale wrapped
|
|
102
|
+
]
|
|
103
|
+
# rubocop:enable Style/MutableConstant
|
|
104
|
+
|
|
105
|
+
# `push_bulk`'s envelope keys, on top of the job keys it shares.
|
|
106
|
+
BULK_KEYS = %w[batch_size spread_interval].freeze
|
|
107
|
+
|
|
108
|
+
# Echoing a rejected key back is the only way a client finds its typo,
|
|
109
|
+
# but the keys came off the network — bound how many and how long, for
|
|
110
|
+
# the same reason the JSON parser's own message is never forwarded.
|
|
111
|
+
MAX_REPORTED_KEYS = 10
|
|
112
|
+
MAX_REPORTED_KEY_LENGTH = 40
|
|
113
|
+
|
|
114
|
+
module_function
|
|
115
|
+
|
|
116
|
+
# Reads at most `limit + 1` bytes, so an oversized body is refused on the
|
|
117
|
+
# strength of what a cap-sized read already proved rather than by
|
|
118
|
+
# buffering the whole thing to measure it.
|
|
119
|
+
def body!(request, limit)
|
|
120
|
+
raw = request.read_body(limit + 1)
|
|
121
|
+
return raw unless raw.bytesize > limit
|
|
122
|
+
|
|
123
|
+
raise Invalid.new(
|
|
124
|
+
"The request body may not exceed #{limit} bytes.",
|
|
125
|
+
type: Problem::PAYLOAD_TOO_LARGE, status: 413, max_bytes: limit
|
|
126
|
+
)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# The body is the job hash with no envelope around it, so this only
|
|
130
|
+
# insists it is a JSON object at all — everything downstream indexes it
|
|
131
|
+
# by key.
|
|
132
|
+
def object!(raw)
|
|
133
|
+
parsed = ::JSON.parse(raw)
|
|
134
|
+
raise Invalid, 'The request body must be a JSON object.' unless parsed.is_a?(::Hash)
|
|
135
|
+
|
|
136
|
+
parsed
|
|
137
|
+
rescue ::JSON::ParserError
|
|
138
|
+
# Deliberately not the parser's own message: it quotes the unparsed
|
|
139
|
+
# remainder of the document, so a large malformed body would come back
|
|
140
|
+
# as a large error.
|
|
141
|
+
raise Invalid, 'The request body is not valid JSON.'
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# One job hash, for `POST /jobs`.
|
|
145
|
+
def job!(payload, principal:, config:)
|
|
146
|
+
known_keys!(payload, JOB_KEYS)
|
|
147
|
+
jid!(payload['jid']) if payload.key?('jid')
|
|
148
|
+
allowed!(class!(payload), principal, config) if payload.key?('class')
|
|
149
|
+
args_size!(payload['args'], config.api_max_args_bytes)
|
|
150
|
+
payload
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# The `push_bulk` envelope, for `POST /jobs/bulk`: the same job keys plus
|
|
154
|
+
# the two that describe the batch, and an `args` that is one array per
|
|
155
|
+
# job rather than one array of arguments.
|
|
156
|
+
def bulk!(payload, principal:, config:)
|
|
157
|
+
known_keys!(payload, JOB_KEYS + BULK_KEYS)
|
|
158
|
+
jid!(payload['jid']) if payload.key?('jid')
|
|
159
|
+
allowed!(class!(payload), principal, config) if payload.key?('class')
|
|
160
|
+
bulk_args_size!(payload['args'], config.api_max_args_bytes)
|
|
161
|
+
payload
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# @raise [Invalid] unless `jid` is a URL-safe token of a sane length.
|
|
165
|
+
# @return [String] the jid.
|
|
166
|
+
def jid!(jid) = url_safe!(jid, 'jid')
|
|
167
|
+
|
|
168
|
+
# A bid indexes `b-<bid>` and its satellite keys, and Batch::Status
|
|
169
|
+
# builds every one of them by interpolation — the same key-fragment
|
|
170
|
+
# exposure a jid has, so the same door.
|
|
171
|
+
# @return [String] the bid.
|
|
172
|
+
def bid!(bid) = url_safe!(bid, 'batch id')
|
|
173
|
+
|
|
174
|
+
# A fid interpolates into `flow:<fid>` and, one segment further, into
|
|
175
|
+
# every node record under it — the same key-fragment exposure a bid has,
|
|
176
|
+
# so the same door.
|
|
177
|
+
# @return [String] the fid.
|
|
178
|
+
def fid!(fid) = url_safe!(fid, 'flow id')
|
|
179
|
+
|
|
180
|
+
def url_safe!(value, label)
|
|
181
|
+
return value if value.is_a?(::String) && JID_FORMAT.match?(value)
|
|
182
|
+
|
|
183
|
+
raise Invalid, "A #{label} is a URL-safe token of up to 255 characters."
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# @raise [Invalid] unless `name` fits {QUEUE_NAME_FORMAT}.
|
|
187
|
+
# @return [String] the queue name.
|
|
188
|
+
def queue_name!(name)
|
|
189
|
+
return name if name.is_a?(::String) && QUEUE_NAME_FORMAT.match?(name)
|
|
190
|
+
|
|
191
|
+
raise Invalid, 'A queue name is up to 255 characters with no whitespace or control characters.'
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# @raise [Invalid] unless `identity` fits {IDENTITY_FORMAT}.
|
|
195
|
+
# @return [String] the process identity.
|
|
196
|
+
def identity!(identity)
|
|
197
|
+
return identity if identity.is_a?(::String) && IDENTITY_FORMAT.match?(identity)
|
|
198
|
+
|
|
199
|
+
raise Invalid, 'A process identity is up to 255 characters with no whitespace or control characters.'
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Validates the allow-list where it is declared, so a typo'd class name
|
|
203
|
+
# raises in the initializer that wrote it instead of 403ing a producer in
|
|
204
|
+
# production. Accepts Classes as well as names — a host listing the
|
|
205
|
+
# constant it already has in scope is spelling the same thing.
|
|
206
|
+
#
|
|
207
|
+
# @return [Array<String>, :any, nil] normalized allow-list.
|
|
208
|
+
def enqueueable_classes!(classes)
|
|
209
|
+
return nil if classes.nil?
|
|
210
|
+
return ANY if classes == ANY
|
|
211
|
+
|
|
212
|
+
list = Array(classes).map(&:to_s)
|
|
213
|
+
raise ArgumentError, 'api_enqueue_classes must name at least one class, :any, or nil' if list.empty?
|
|
214
|
+
|
|
215
|
+
unknown = list.reject { |name| class_name?(name) }
|
|
216
|
+
raise ArgumentError, "api_enqueue_classes entries must be Ruby class names: #{unknown.inspect}" if unknown.any?
|
|
217
|
+
|
|
218
|
+
list.freeze
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def class_name?(value)
|
|
222
|
+
value.is_a?(::String) && value.length <= MAX_CLASS_LENGTH && CLASS_FORMAT.match?(value)
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def known_keys!(payload, allowed)
|
|
226
|
+
unknown = payload.keys - allowed
|
|
227
|
+
return if unknown.empty?
|
|
228
|
+
|
|
229
|
+
reported = reportable(unknown)
|
|
230
|
+
raise Invalid.new("Unknown job keys: #{reported.join(', ')}.", unknown_keys: reported)
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def reportable(keys)
|
|
234
|
+
keys.sort.first(MAX_REPORTED_KEYS).map { |key| key.to_s[0, MAX_REPORTED_KEY_LENGTH] }
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def class!(payload)
|
|
238
|
+
job_class = payload['class']
|
|
239
|
+
return job_class if class_name?(job_class)
|
|
240
|
+
|
|
241
|
+
raise Invalid, %(Job 'class' must be a Ruby class name, e.g. "HardWorker" or "Billing::Charge".)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# The allow-list JobUtil has no reason to own:
|
|
245
|
+
#
|
|
246
|
+
# nil → only an `admin` token may enqueue, and it may enqueue anything
|
|
247
|
+
# Array → exactly these names, for every token, `admin` included
|
|
248
|
+
# :any → the check is off
|
|
249
|
+
#
|
|
250
|
+
# Unset denies an enqueue-scoped producer everything, because `class`
|
|
251
|
+
# selects which of the host's jobs runs and a producer that can name any
|
|
252
|
+
# constant is choosing. `admin` is exempt only while no list exists: it
|
|
253
|
+
# already holds the destructive routes, so restricting which class it may
|
|
254
|
+
# enqueue would be theatre. Once a host writes a list, it means it.
|
|
255
|
+
def allowed!(job_class, principal, config)
|
|
256
|
+
return if enqueueable?(job_class, principal, config.api_enqueue_classes)
|
|
257
|
+
|
|
258
|
+
raise Invalid.new(
|
|
259
|
+
"#{job_class} may not be enqueued over HTTP; config.api_enqueue_classes names the classes that may.",
|
|
260
|
+
type: Problem::CLASS_NOT_ALLOWED, status: 403, job_class: job_class
|
|
261
|
+
)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def enqueueable?(job_class, principal, classes)
|
|
265
|
+
case classes
|
|
266
|
+
when nil then principal.permits?(:admin)
|
|
267
|
+
when ANY then true
|
|
268
|
+
else classes.include?(job_class)
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Only an Array is sizeable and only an Array is legal; Client names the
|
|
273
|
+
# other shapes, with the offending value in the message.
|
|
274
|
+
def args_size!(args, limit)
|
|
275
|
+
return unless args.is_a?(::Array)
|
|
276
|
+
|
|
277
|
+
size = ::JSON.generate(args).bytesize
|
|
278
|
+
return if size <= limit
|
|
279
|
+
|
|
280
|
+
raise Invalid.new(
|
|
281
|
+
"Job args may not exceed #{limit} bytes; these serialize to #{size}.",
|
|
282
|
+
type: Problem::PAYLOAD_TOO_LARGE, status: 413, max_bytes: limit
|
|
283
|
+
)
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# The body cap bounds the request; this bounds each job the request turns
|
|
287
|
+
# into, which for a bulk push is the only one of the two that bounds what
|
|
288
|
+
# lands in Redis.
|
|
289
|
+
def bulk_args_size!(args, limit)
|
|
290
|
+
return unless args.is_a?(::Array)
|
|
291
|
+
|
|
292
|
+
args.each_with_index do |job_args, index|
|
|
293
|
+
next unless job_args.is_a?(::Array)
|
|
294
|
+
|
|
295
|
+
size = ::JSON.generate(job_args).bytesize
|
|
296
|
+
next if size <= limit
|
|
297
|
+
|
|
298
|
+
raise Invalid.new(
|
|
299
|
+
"Job #{index}'s args may not exceed #{limit} bytes; they serialize to #{size}.",
|
|
300
|
+
type: Problem::PAYLOAD_TOO_LARGE, status: 413, max_bytes: limit, job_index: index
|
|
301
|
+
)
|
|
302
|
+
end
|
|
303
|
+
end
|
|
304
|
+
end
|
|
305
|
+
end
|
|
306
|
+
end
|
data/lib/wurk/api.rb
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Wurk
|
|
4
|
+
# Namespace for two unrelated things that both answer to "the API": the Pro
|
|
5
|
+
# data-API Lua extensions (API::Fast, required by lib/wurk.rb at its
|
|
6
|
+
# load-order-sensitive point) and the machine-facing HTTP API (API::App).
|
|
7
|
+
module API
|
|
8
|
+
# The one prefix the machine plane answers under, and the version it pins.
|
|
9
|
+
# They live on the namespace rather than on App because the engine's mount
|
|
10
|
+
# constraint has to answer "is this path mine?" on every routing pass, and
|
|
11
|
+
# loading App to ask would undo the lazy load below.
|
|
12
|
+
API_VERSION = 'v1'
|
|
13
|
+
VERSION_PREFIX = "/#{API_VERSION}".freeze
|
|
14
|
+
NESTED_PREFIX = "#{VERSION_PREFIX}/".freeze
|
|
15
|
+
SUPPORTED_VERSIONS = [API_VERSION].freeze
|
|
16
|
+
|
|
17
|
+
# A path that names *some* version of this plane — `/v1`, `/v2/jobs`, not
|
|
18
|
+
# `/v1x` and not the dashboard's own `/stats`. Version-shaped rather than
|
|
19
|
+
# `v1`-only because the mount has to claim a version it does not serve in
|
|
20
|
+
# order for App to answer it `unsupported_api_version`, which is what the
|
|
21
|
+
# standalone and separately-mounted modes already do. Claiming only `/v1`
|
|
22
|
+
# would leave mode 1 alone in falling through to the host's router, and a
|
|
23
|
+
# client would learn "wrong version" from Rails' 404 in one deployment
|
|
24
|
+
# shape and from a problem document naming the supported versions in the
|
|
25
|
+
# other two.
|
|
26
|
+
VERSION_PATH = %r{\A/v\d+(?:/|\z)}
|
|
27
|
+
|
|
28
|
+
# Where mount mode 1 puts this plane inside the engine (config/routes.rb).
|
|
29
|
+
# Named here because two callers have to agree on it: the mount itself, and
|
|
30
|
+
# `Wurk::Web::Authorization`, which runs before routing and so sees the
|
|
31
|
+
# engine-relative path with this prefix still on it.
|
|
32
|
+
ENGINE_MOUNT = '/api'
|
|
33
|
+
|
|
34
|
+
# Rack env key the engine's Authorization middleware stamps when the
|
|
35
|
+
# dashboard is read-only. It is how mount mode 1 — and only mode 1 —
|
|
36
|
+
# inherits `WURK_WEB_READ_ONLY`: a separately mounted or standalone API is
|
|
37
|
+
# a different deployment and opts in on its own (`config.api_read_only`).
|
|
38
|
+
READ_ONLY_ENV = 'wurk.web.read_only'
|
|
39
|
+
|
|
40
|
+
class << self
|
|
41
|
+
# Class-level Rack entry, the same shape as Wurk::Web.call, so
|
|
42
|
+
# `mount Wurk::API => '/wurk-api'` and `run Wurk::API` both work.
|
|
43
|
+
#
|
|
44
|
+
# App and its dependencies (Rack::Request among them) load on the first
|
|
45
|
+
# request rather than at `require "wurk"`. The HTTP API is off unless a
|
|
46
|
+
# host mounts it, and eager-loading it cost every swarm child ~1.2 MB of
|
|
47
|
+
# pre-fork heap and measurably slower boot for a surface it never serves.
|
|
48
|
+
def call(env)
|
|
49
|
+
app.call(env)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Whether the API should answer for `path` — the request path relative to
|
|
53
|
+
# wherever it was mounted. The engine's conditional mount (config/
|
|
54
|
+
# routes.rb) is a constraint over this, which settles two things a bare
|
|
55
|
+
# `mount` could not:
|
|
56
|
+
#
|
|
57
|
+
# * Off means absent. With no token registered the constraint fails,
|
|
58
|
+
# Rails falls through to the next route, and the surface does not
|
|
59
|
+
# exist — not a 401 advertising one that does (07 plan, step 1).
|
|
60
|
+
# Asked per request, not at draw time: routes load once at boot, and
|
|
61
|
+
# a host is free to register its token after that (a Puma-cluster web
|
|
62
|
+
# process never enters server mode, so its `configure_server` block
|
|
63
|
+
# has not run by then).
|
|
64
|
+
# * Nested in the engine, this plane shares the /api prefix with the
|
|
65
|
+
# dashboard's own JSON API. Without the version check a mistyped
|
|
66
|
+
# dashboard path would fall through to the machine plane and draw a
|
|
67
|
+
# bearer challenge for a route that was never part of this contract.
|
|
68
|
+
# A version-shaped path is never one of the dashboard's, so this
|
|
69
|
+
# claims every version and lets App refuse the ones it cannot serve.
|
|
70
|
+
def serves?(path, config = ::Wurk.configuration)
|
|
71
|
+
return false unless config.api_enabled?
|
|
72
|
+
|
|
73
|
+
VERSION_PATH.match?(path)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# The same question asked one prefix out, for callers that run before the
|
|
77
|
+
# engine's mount has stripped it — `Wurk::Web::Authorization` is the only
|
|
78
|
+
# one. It has to tell a machine-plane path from a dashboard one (both live
|
|
79
|
+
# under /api) without loading App, which is why this is here and not there.
|
|
80
|
+
def engine_serves?(path, config = ::Wurk.configuration)
|
|
81
|
+
path.start_with?(ENGINE_MOUNT) && serves?(path.delete_prefix(ENGINE_MOUNT), config)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
# App is stateless, so every mount — engine-nested, separately mounted,
|
|
87
|
+
# standalone — can share one instance.
|
|
88
|
+
def app
|
|
89
|
+
@app ||= begin
|
|
90
|
+
require_relative 'api/app'
|
|
91
|
+
App.new
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -30,7 +30,7 @@ module Wurk
|
|
|
30
30
|
class ServerMiddleware
|
|
31
31
|
include Wurk::Middleware::ServerMiddleware
|
|
32
32
|
|
|
33
|
-
def call(_worker, job, _queue)
|
|
33
|
+
def call(_worker, job, _queue, &)
|
|
34
34
|
bid = job['bid']
|
|
35
35
|
return yield unless bid
|
|
36
36
|
|
|
@@ -39,7 +39,7 @@ module Wurk
|
|
|
39
39
|
return
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
run_and_ack(bid, job['jid'])
|
|
42
|
+
run_and_ack(bid, job['jid'], &)
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
private
|
data/lib/wurk/batch.rb
CHANGED
|
@@ -36,7 +36,7 @@ module Wurk
|
|
|
36
36
|
# Nested batches: a job opening its OWN batch (`batch.jobs { ... }`)
|
|
37
37
|
# increments live counters on the existing batch. A callback opening its
|
|
38
38
|
# PARENT batch links via parent_bid and adds child BID to b-<bid>-kids.
|
|
39
|
-
class Batch
|
|
39
|
+
class Batch # rubocop:disable Metrics/ClassLength
|
|
40
40
|
DEFAULT_EXPIRY_SECONDS = 30 * 24 * 60 * 60
|
|
41
41
|
POST_SUCCESS_EXPIRY_SECONDS = 24 * 60 * 60
|
|
42
42
|
CALLBACK_NOTIFY_TTL = 30 * 24 * 60 * 60
|
|
@@ -117,9 +117,18 @@ module Wurk
|
|
|
117
117
|
# can drive the trim on isolated limits without mutating the process-global
|
|
118
118
|
# `Wurk.configuration`.
|
|
119
119
|
def self.trim_index(pipe, key, max: nil, timeout: nil)
|
|
120
|
+
score, rank = trim_bounds(max: max, timeout: timeout)
|
|
121
|
+
pipe.call('ZREMRANGEBYSCORE', key, '-inf', score)
|
|
122
|
+
pipe.call('ZREMRANGEBYRANK', key, 0, rank)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# The same two bounds as arguments, for a writer that cannot append to a
|
|
126
|
+
# pipeline: `Wurk::Flow`'s creation script has to trim from inside its own
|
|
127
|
+
# atomic write, and one policy read out of here beats a second copy of the
|
|
128
|
+
# cutoff arithmetic that drifts the day this one changes.
|
|
129
|
+
def self.trim_bounds(max: nil, timeout: nil)
|
|
120
130
|
cutoff = ::Process.clock_gettime(::Process::CLOCK_REALTIME) - (timeout || DEFAULT_EXPIRY_SECONDS)
|
|
121
|
-
|
|
122
|
-
pipe.call('ZREMRANGEBYRANK', key, 0, -(max || INDEX_MAX))
|
|
131
|
+
["(#{cutoff}", -(max || INDEX_MAX)]
|
|
123
132
|
end
|
|
124
133
|
|
|
125
134
|
def initialize(bid = nil)
|
data/lib/wurk/capsule.rb
CHANGED
|
@@ -4,6 +4,7 @@ require_relative 'redis_pool'
|
|
|
4
4
|
require_relative 'pool_checkout'
|
|
5
5
|
require_relative 'middleware/chain'
|
|
6
6
|
require_relative 'fetcher/reliable'
|
|
7
|
+
require_relative 'watchdog'
|
|
7
8
|
|
|
8
9
|
module Wurk
|
|
9
10
|
# One processing unit: a set of threads + queues sharing a fetcher and a
|
|
@@ -14,7 +15,7 @@ module Wurk
|
|
|
14
15
|
class Capsule
|
|
15
16
|
MODES = %i[strict weighted random].freeze
|
|
16
17
|
|
|
17
|
-
attr_reader :name, :queues, :mode, :weights, :config
|
|
18
|
+
attr_reader :name, :queues, :mode, :weights, :config, :watchdog
|
|
18
19
|
attr_accessor :concurrency, :fetcher
|
|
19
20
|
|
|
20
21
|
# Capsule-hosted components (Manager, Processor, Fetcher) hand their capsule
|
|
@@ -41,6 +42,7 @@ module Wurk
|
|
|
41
42
|
@pools = {}
|
|
42
43
|
@client_chain = nil
|
|
43
44
|
@server_chain = nil
|
|
45
|
+
@watchdog = nil
|
|
44
46
|
end
|
|
45
47
|
|
|
46
48
|
def to_h
|
|
@@ -78,6 +80,11 @@ module Wurk
|
|
|
78
80
|
def prepare!
|
|
79
81
|
prepare_shared!
|
|
80
82
|
@fetcher ||= build_fetcher
|
|
83
|
+
# Here rather than in the constructor so a client-only process — a web
|
|
84
|
+
# dyno building the same Configuration — allocates nothing, and so the
|
|
85
|
+
# scanner thread a bounded job spawns belongs to the child that runs it.
|
|
86
|
+
# The object itself holds no thread until something is bounded.
|
|
87
|
+
@watchdog ||= Watchdog.new(self)
|
|
81
88
|
redis_pool
|
|
82
89
|
fetch_redis_pool
|
|
83
90
|
self
|
data/lib/wurk/cli.rb
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
require
|
|
6
|
-
|
|
3
|
+
# yaml, optparse and erb are required inside the two methods that use them
|
|
4
|
+
# (#parse_config and #option_parser) rather than here. Together they cost ~45ms
|
|
5
|
+
# of the ~290ms `require "wurk"` spends, and only `exe/wurk` ever reaches them —
|
|
6
|
+
# a Rails app that enqueues and processes jobs pays that on every boot for a
|
|
7
|
+
# command-line parser it never runs. `require` is idempotent, so the lazy call
|
|
8
|
+
# is a $LOADED_FEATURES hash lookup on every invocation after the first.
|
|
7
9
|
require_relative 'version'
|
|
8
10
|
require_relative 'configuration'
|
|
9
11
|
require_relative 'component'
|
|
@@ -24,6 +26,27 @@ module Wurk
|
|
|
24
26
|
# format and Lua scripts rely on commands introduced in Redis 7.
|
|
25
27
|
MIN_REDIS_VERSION = '7.0.0'
|
|
26
28
|
|
|
29
|
+
# Subcommands. Bare `wurk` runs the worker — the historical shape, and the
|
|
30
|
+
# only one the swarm binaries accept — so a subcommand is never anything
|
|
31
|
+
# but the first argument, and every argument after it is that command's.
|
|
32
|
+
COMMANDS = %w[api].freeze
|
|
33
|
+
|
|
34
|
+
# `wurk api` defaults. 0.0.0.0 because the point of the machine API is for
|
|
35
|
+
# another service to reach it; it is bearer-gated, and answers 404 to
|
|
36
|
+
# everything until a token exists. `--bind 127.0.0.1` narrows it.
|
|
37
|
+
DEFAULT_API_BIND = '0.0.0.0'
|
|
38
|
+
DEFAULT_API_PORT = 7433
|
|
39
|
+
|
|
40
|
+
# Serving the API with nothing registered would bind a port and 404 every
|
|
41
|
+
# request — the surface is off until a token exists, so say so at boot
|
|
42
|
+
# rather than leaving an operator to diagnose it over HTTP.
|
|
43
|
+
NO_API_TOKEN_MESSAGE = <<~MSG
|
|
44
|
+
No API token is registered, so every request would answer 404.
|
|
45
|
+
Register one where the app loaded by -r configures wurk:
|
|
46
|
+
|
|
47
|
+
Wurk.configuration.api_token(ENV.fetch('WURK_API_TOKEN'), scopes: %i[enqueue read])
|
|
48
|
+
MSG
|
|
49
|
+
|
|
27
50
|
# Thread-backtrace dumper used by both TTIN and INFO. Same body — INFO is
|
|
28
51
|
# the modern name, TTIN is kept for parity with older Sidekiq users.
|
|
29
52
|
BACKTRACE_DUMPER = lambda do |cli|
|
|
@@ -68,8 +91,20 @@ module Wurk
|
|
|
68
91
|
['-C', '--config PATH', :config_file, 'path to YAML config file']
|
|
69
92
|
].freeze
|
|
70
93
|
|
|
94
|
+
# Same table shape, offered only when `api` is the subcommand: none of it
|
|
95
|
+
# means anything to a worker process, and `wurk --help` stays the worker's.
|
|
96
|
+
API_OPTION_FLAGS = [
|
|
97
|
+
['-b', '--bind ADDRESS', :api_bind, "Address to bind (default #{DEFAULT_API_BIND})"],
|
|
98
|
+
['-p', '--port PORT', :api_port, "Port to listen on (default #{DEFAULT_API_PORT})", :to_i],
|
|
99
|
+
['-s', '--server NAME', :api_server, 'Rack handler to serve with (default: the first installed)']
|
|
100
|
+
].freeze
|
|
101
|
+
|
|
71
102
|
attr_accessor :launcher, :environment, :config
|
|
72
103
|
|
|
104
|
+
# The subcommand `parse` pulled off argv, or nil for the worker runner.
|
|
105
|
+
# The binary picks the mode from it — `exe/wurk` runs the API for `api`.
|
|
106
|
+
attr_reader :command
|
|
107
|
+
|
|
73
108
|
def self.instance
|
|
74
109
|
@instance ||= new
|
|
75
110
|
end
|
|
@@ -84,12 +119,14 @@ module Wurk
|
|
|
84
119
|
@launcher = nil
|
|
85
120
|
@environment = nil
|
|
86
121
|
@parser = nil
|
|
122
|
+
@command = nil
|
|
87
123
|
end
|
|
88
124
|
|
|
89
125
|
# `parse` is split from `run` so tests can drive option parsing without
|
|
90
126
|
# touching Redis or booting the host app.
|
|
91
127
|
def parse(args = ARGV.dup)
|
|
92
128
|
@config ||= Wurk.default_configuration
|
|
129
|
+
@command = extract_command!(args)
|
|
93
130
|
setup_options(args)
|
|
94
131
|
initialize_logger
|
|
95
132
|
validate!
|
|
@@ -137,6 +174,8 @@ module Wurk
|
|
|
137
174
|
# `SIDEKIQ_PRELOAD_APP` whole-app eager-load) and boots `Process.warmup`
|
|
138
175
|
# before the fork so children share warmed pages (copy-on-write). Spec §7.
|
|
139
176
|
def run_swarm(boot_app: true, warmup: true)
|
|
177
|
+
raise ArgumentError, "the swarm runner takes no subcommand; run `wurk #{@command}` instead" if @command
|
|
178
|
+
|
|
140
179
|
# Server mode before the app loads — see #run. The flag rides through the
|
|
141
180
|
# fork into every child (the config object is copied), so configure_server
|
|
142
181
|
# blocks registered in the parent take effect in the workers.
|
|
@@ -163,6 +202,32 @@ module Wurk
|
|
|
163
202
|
end
|
|
164
203
|
end
|
|
165
204
|
|
|
205
|
+
# `wurk api` — serve the machine-facing HTTP API and nothing else: no
|
|
206
|
+
# fetcher, no heartbeat, no job ever runs here. This is mount mode 3, the
|
|
207
|
+
# standalone one: the same Rack app the engine nests and a host can mount
|
|
208
|
+
# itself, served without Rails.
|
|
209
|
+
#
|
|
210
|
+
# Server mode is entered for the reason #run does it — a host that
|
|
211
|
+
# registers its token inside a `configure_server` block would otherwise
|
|
212
|
+
# boot an API with no credential and 404 every request. Nothing starts as a
|
|
213
|
+
# side effect: no lifecycle event fires and no Launcher is ever built.
|
|
214
|
+
#
|
|
215
|
+
# `handler:` is a test seam; production resolves one from the installed
|
|
216
|
+
# gems.
|
|
217
|
+
def run_api(boot_app: true, handler: nil)
|
|
218
|
+
enter_server_mode
|
|
219
|
+
boot_application if boot_app
|
|
220
|
+
validate_redis!
|
|
221
|
+
# Load the app here, not on the first request: a broken load should fail
|
|
222
|
+
# the boot an operator is watching, not the first client to call.
|
|
223
|
+
require_relative 'api/app'
|
|
224
|
+
raise ArgumentError, NO_API_TOKEN_MESSAGE unless @config.api_enabled?
|
|
225
|
+
|
|
226
|
+
handler ||= api_handler(@config[:api_server])
|
|
227
|
+
logger.info { "Wurk API listening on http://#{api_bind}:#{api_port}#{Wurk::API::VERSION_PREFIX}" }
|
|
228
|
+
handler.run(Wurk::API, Host: api_bind, Port: api_port)
|
|
229
|
+
end
|
|
230
|
+
|
|
166
231
|
def handle_signal(sig)
|
|
167
232
|
logger.debug { "Got #{sig} signal" }
|
|
168
233
|
handler = SIGNAL_HANDLERS[sig]
|
|
@@ -188,6 +253,29 @@ module Wurk
|
|
|
188
253
|
::Process.warmup
|
|
189
254
|
end
|
|
190
255
|
|
|
256
|
+
def api_bind = @config[:api_bind] || DEFAULT_API_BIND
|
|
257
|
+
def api_port = @config[:api_port] || DEFAULT_API_PORT
|
|
258
|
+
|
|
259
|
+
# Rack 3 moved the server handlers out to the `rackup` gem; rack 2 still
|
|
260
|
+
# ships them itself. Neither is a wurk dependency — the web server is the
|
|
261
|
+
# host's choice here exactly as it is for the app this sits next to — so
|
|
262
|
+
# resolve one at run time and name the fix when there is none.
|
|
263
|
+
def api_handler(name)
|
|
264
|
+
registry = handler_registry
|
|
265
|
+
name ? registry.get(name) : registry.default
|
|
266
|
+
rescue ::LoadError, ::NameError, ::ArgumentError => e
|
|
267
|
+
raise ArgumentError, "wurk api could not start a web server (#{e.message}). Add one to your Gemfile " \
|
|
268
|
+
'— puma, falcon, or rackup + webrick — or name an installed one with --server.'
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def handler_registry
|
|
272
|
+
require 'rackup/handler'
|
|
273
|
+
::Rackup::Handler
|
|
274
|
+
rescue ::LoadError
|
|
275
|
+
require 'rack/handler'
|
|
276
|
+
::Rack::Handler
|
|
277
|
+
end
|
|
278
|
+
|
|
191
279
|
def launch(self_read)
|
|
192
280
|
@launcher = Wurk::Launcher.new(@config)
|
|
193
281
|
begin
|
|
@@ -270,6 +358,14 @@ module Wurk
|
|
|
270
358
|
|
|
271
359
|
# --- options + config-file --------------------------------------------
|
|
272
360
|
|
|
361
|
+
# Pulled off before OptionParser sees argv, so the banner and the flag
|
|
362
|
+
# table can depend on which command was named. A bare word OptionParser
|
|
363
|
+
# doesn't recognize would otherwise survive `parse!` and be silently
|
|
364
|
+
# ignored, which is how `wurk api` would look like it worked.
|
|
365
|
+
def extract_command!(args)
|
|
366
|
+
COMMANDS.include?(args.first) ? args.shift : nil
|
|
367
|
+
end
|
|
368
|
+
|
|
273
369
|
def setup_options(args)
|
|
274
370
|
opts = parse_options(args)
|
|
275
371
|
set_environment(opts[:environment])
|
|
@@ -316,6 +412,9 @@ module Wurk
|
|
|
316
412
|
def parse_config(path)
|
|
317
413
|
raise ArgumentError, "No such file #{path}" unless ::File.exist?(path)
|
|
318
414
|
|
|
415
|
+
require 'erb'
|
|
416
|
+
require 'yaml'
|
|
417
|
+
|
|
319
418
|
erb = ::ERB.new(::File.read(path), trim_mode: '-')
|
|
320
419
|
erb.filename = ::File.expand_path(path)
|
|
321
420
|
opts = ::YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
|
@@ -350,15 +449,18 @@ module Wurk
|
|
|
350
449
|
end
|
|
351
450
|
|
|
352
451
|
def option_parser(opts)
|
|
452
|
+
require 'optparse'
|
|
453
|
+
|
|
353
454
|
::OptionParser.new do |o|
|
|
354
|
-
o.banner = 'wurk [options]'
|
|
355
|
-
define_value_flags(o, opts)
|
|
455
|
+
o.banner = @command ? "wurk #{@command} [options]" : 'wurk [options]'
|
|
456
|
+
define_value_flags(o, opts, OPTION_FLAGS)
|
|
457
|
+
define_value_flags(o, opts, API_OPTION_FLAGS) if @command == 'api'
|
|
356
458
|
define_meta_flags(o, opts)
|
|
357
459
|
end
|
|
358
460
|
end
|
|
359
461
|
|
|
360
|
-
def define_value_flags(parser, opts)
|
|
361
|
-
|
|
462
|
+
def define_value_flags(parser, opts, flags)
|
|
463
|
+
flags.each do |short, long, key, desc, transform|
|
|
362
464
|
parser.on(short, long, desc) { |arg| assign_flag(opts, key, arg, transform) }
|
|
363
465
|
end
|
|
364
466
|
end
|