raptor 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.buildkite/pipeline.yml +100 -0
- data/CHANGELOG.md +11 -0
- data/README.md +47 -31
- data/docs/raptor-vs-puma.md +81 -63
- data/ext/raptor_bpf/reuseport_select.bpf.c +8 -3
- data/ext/raptor_native/raptor_native.c +16 -2
- data/lib/raptor/binder.rb +37 -87
- data/lib/raptor/cli.rb +18 -25
- data/lib/raptor/cluster.rb +385 -82
- data/lib/raptor/http.rb +18 -0
- data/lib/raptor/http1.rb +244 -233
- data/lib/raptor/http2.rb +42 -63
- data/lib/raptor/reactor.rb +21 -52
- data/lib/raptor/reuseport_bpf.rb +27 -5
- data/lib/raptor/server.rb +24 -42
- data/lib/raptor/stats.rb +3 -8
- data/lib/raptor/systemd.rb +1 -1
- data/lib/raptor/version.rb +1 -1
- data/sig/generated/raptor/binder.rbs +34 -80
- data/sig/generated/raptor/cli.rbs +9 -25
- data/sig/generated/raptor/cluster.rbs +183 -85
- data/sig/generated/raptor/http.rbs +10 -0
- data/sig/generated/raptor/http1.rbs +129 -113
- data/sig/generated/raptor/http2.rbs +31 -48
- data/sig/generated/raptor/reactor.rbs +21 -52
- data/sig/generated/raptor/reuseport_bpf.rbs +20 -2
- data/sig/generated/raptor/server.rbs +19 -39
- data/sig/generated/raptor/stats.rbs +3 -8
- metadata +1 -1
data/lib/raptor/cluster.rb
CHANGED
|
@@ -18,39 +18,15 @@ require_relative "stats"
|
|
|
18
18
|
require_relative "systemd"
|
|
19
19
|
|
|
20
20
|
module Raptor
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
# application processing, plus dedicated reactor and server threads. It
|
|
26
|
-
# handles process forking, signal management, graceful shutdown, and
|
|
27
|
-
# automatic worker restart when a worker process unexpectedly exits.
|
|
28
|
-
#
|
|
29
|
-
# The architecture provides horizontal scaling through processes while
|
|
30
|
-
# maintaining efficient I/O and CPU utilization within each process
|
|
31
|
-
# through the combination of ractor-based parsing and thread pools on
|
|
32
|
-
# top of NIO reactors.
|
|
33
|
-
#
|
|
34
|
-
# Flow per worker process:
|
|
35
|
-
# 1. Server continuously accepts connections but skips acceptance when backlog is high
|
|
36
|
-
# 2. Reactor manages I/O multiplexing and provides backlog metrics for load control
|
|
37
|
-
# 3. Ractor pool handles CPU-intensive HTTP parsing in parallel
|
|
38
|
-
# 4. Thread pool processes Rack applications and handles response writing
|
|
39
|
-
# 5. Natural load balancing occurs through backpressure-based acceptance control
|
|
40
|
-
#
|
|
41
|
-
# @example Basic usage
|
|
42
|
-
# options = {
|
|
43
|
-
# workers: 4, ractors: 2, threads: 8,
|
|
44
|
-
# binds: ["tcp://0.0.0.0:3000"],
|
|
45
|
-
# rackup: "config.ru",
|
|
46
|
-
# connection: { first_data_timeout: 30, chunk_data_timeout: 10 }
|
|
47
|
-
# }
|
|
48
|
-
# Cluster.run(options)
|
|
21
|
+
# Forks and supervises worker processes. Handles graceful shutdown on
|
|
22
|
+
# `INT` and `TERM`, phased restart on `USR1`, hot restart on `USR2`,
|
|
23
|
+
# and (on Linux) refork on `URG`. Restarts workers that exit
|
|
24
|
+
# unexpectedly or stop checking in.
|
|
49
25
|
#
|
|
50
26
|
class Cluster
|
|
51
27
|
INHERITED_FDS_ENV = "RAPTOR_INHERITED_FDS"
|
|
52
28
|
|
|
53
|
-
#
|
|
29
|
+
# Creates and runs a cluster with the given options.
|
|
54
30
|
#
|
|
55
31
|
# @param options [Hash] cluster configuration options
|
|
56
32
|
# @return [void]
|
|
@@ -72,6 +48,10 @@ module Raptor
|
|
|
72
48
|
# @rbs @worker_timeout: Integer
|
|
73
49
|
# @rbs @worker_drain_timeout: Integer
|
|
74
50
|
# @rbs @worker_shutdown_timeout: Integer
|
|
51
|
+
# @rbs @before_fork: Array[Proc]
|
|
52
|
+
# @rbs @before_worker_boot: Array[Proc]
|
|
53
|
+
# @rbs @before_worker_shutdown: Array[Proc]
|
|
54
|
+
# @rbs @before_refork: Array[Proc]
|
|
75
55
|
# @rbs @stats_file: String?
|
|
76
56
|
# @rbs @pid_file: String?
|
|
77
57
|
# @rbs @stdout_file: String?
|
|
@@ -92,13 +72,19 @@ module Raptor
|
|
|
92
72
|
# @rbs @phased_restart_requested: bool
|
|
93
73
|
# @rbs @phased_restarting: bool
|
|
94
74
|
# @rbs @hot_restart_requested: bool
|
|
75
|
+
# @rbs @refork_thresholds: Array[Integer]
|
|
76
|
+
# @rbs @refork_requested: bool
|
|
77
|
+
# @rbs @refork_threshold_idx: Integer
|
|
78
|
+
# @rbs @seed_pid: Integer?
|
|
79
|
+
# @rbs @seed_ready: bool
|
|
80
|
+
# @rbs @seed_vacated_index: Integer?
|
|
81
|
+
# @rbs @fork_r: IO?
|
|
82
|
+
# @rbs @fork_w: IO?
|
|
83
|
+
# @rbs @resp_r: IO?
|
|
84
|
+
# @rbs @resp_w: IO?
|
|
95
85
|
# @rbs @bpf_active: bool
|
|
96
86
|
|
|
97
|
-
# Creates a new Cluster with the
|
|
98
|
-
#
|
|
99
|
-
# Initializes the cluster with worker, ractor, and thread counts,
|
|
100
|
-
# sets up network binding, loads the Rack application, and prepares
|
|
101
|
-
# for multi-process operation.
|
|
87
|
+
# Creates a new Cluster with the given options.
|
|
102
88
|
#
|
|
103
89
|
# @param options [Hash] cluster configuration options
|
|
104
90
|
# @option options [Array<String>] :binds array of bind URIs
|
|
@@ -118,6 +104,11 @@ module Raptor
|
|
|
118
104
|
# @option options [Integer] :worker_timeout seconds to wait for a booted worker to check in before killing it
|
|
119
105
|
# @option options [Integer] :worker_drain_timeout seconds a worker waits for in-flight requests during shutdown before force-killing app threads
|
|
120
106
|
# @option options [Integer] :worker_shutdown_timeout seconds to wait for graceful worker exit before force-killing
|
|
107
|
+
# @option options [Integer, Array<Integer>, nil] :refork_after request-count threshold(s) at which a warm worker is promoted to a fork source for phased refork; nil or 0 disables. Requires `PR_SET_CHILD_SUBREAPER` (Linux)
|
|
108
|
+
# @option options [Array<Proc>] :before_fork procs called in the master before every worker fork
|
|
109
|
+
# @option options [Array<Proc>] :before_worker_boot procs called in each worker before it begins serving
|
|
110
|
+
# @option options [Array<Proc>] :before_worker_shutdown procs called in each worker before its graceful shutdown
|
|
111
|
+
# @option options [Array<Proc>] :before_refork procs called in a worker before it transitions to a seed
|
|
121
112
|
# @option options [String, nil] :stats_file path to write per-worker stats JSON, or nil to disable
|
|
122
113
|
# @option options [String, nil] :pid_file path to write the master PID to, or nil to disable
|
|
123
114
|
# @option options [String, nil] :stdout_file path to redirect stdout to, reopened on SIGHUP, or nil to disable
|
|
@@ -142,6 +133,10 @@ module Raptor
|
|
|
142
133
|
@worker_timeout = options[:worker_timeout]
|
|
143
134
|
@worker_drain_timeout = options[:worker_drain_timeout]
|
|
144
135
|
@worker_shutdown_timeout = options[:worker_shutdown_timeout]
|
|
136
|
+
@before_fork = Array(options[:before_fork])
|
|
137
|
+
@before_worker_boot = Array(options[:before_worker_boot])
|
|
138
|
+
@before_worker_shutdown = Array(options[:before_worker_shutdown])
|
|
139
|
+
@before_refork = Array(options[:before_refork])
|
|
145
140
|
@stats_file = options[:stats_file]
|
|
146
141
|
@pid_file = options[:pid_file]
|
|
147
142
|
@stdout_file = options[:stdout_file]
|
|
@@ -175,27 +170,24 @@ module Raptor
|
|
|
175
170
|
@phased_restart_requested = false
|
|
176
171
|
@phased_restarting = false
|
|
177
172
|
@hot_restart_requested = false
|
|
173
|
+
|
|
174
|
+
@refork_thresholds = normalize_refork_thresholds(options[:refork_after])
|
|
175
|
+
@refork_requested = false
|
|
176
|
+
@refork_threshold_idx = 0
|
|
177
|
+
@seed_pid = nil
|
|
178
|
+
@seed_ready = false
|
|
179
|
+
@seed_vacated_index = nil
|
|
178
180
|
end
|
|
179
181
|
|
|
180
|
-
#
|
|
181
|
-
#
|
|
182
|
-
# Forks the configured number of worker processes and monitors them,
|
|
183
|
-
# restarting any that exit unexpectedly or stop checking in. Handles
|
|
184
|
-
# graceful shutdown via INT or TERM signals, phased restart via USR1,
|
|
185
|
-
# and hot restart via USR2.
|
|
186
|
-
#
|
|
187
|
-
# Each worker process includes:
|
|
188
|
-
# - 1 server thread (continuously accepts connections with backpressure control)
|
|
189
|
-
# - 1 reactor thread (I/O multiplexing, timeout handling, backlog monitoring)
|
|
190
|
-
# - N pipeline ractors (parallel HTTP parsing)
|
|
191
|
-
# - 1 pipeline collector thread (coordinates parsing results)
|
|
192
|
-
# - M worker threads (Rack application processing and response writing)
|
|
193
|
-
# - 1 stats thread (writes per-worker metrics to shared memory every second)
|
|
182
|
+
# Runs the cluster until a graceful shutdown is signalled.
|
|
194
183
|
#
|
|
195
184
|
# @return [void]
|
|
196
185
|
#
|
|
197
186
|
# @rbs () -> void
|
|
198
187
|
def run
|
|
188
|
+
$stdout.sync = true
|
|
189
|
+
$stderr.sync = true
|
|
190
|
+
|
|
199
191
|
reopen_logs
|
|
200
192
|
|
|
201
193
|
trap("INT") { shutdown }
|
|
@@ -204,6 +196,17 @@ module Raptor
|
|
|
204
196
|
trap("USR1") { @phased_restart_requested = true }
|
|
205
197
|
trap("USR2") { @hot_restart_requested = true }
|
|
206
198
|
|
|
199
|
+
if @refork_thresholds.any?
|
|
200
|
+
if Subreaper.enable
|
|
201
|
+
@fork_r, @fork_w = IO.pipe
|
|
202
|
+
@resp_r, @resp_w = IO.pipe
|
|
203
|
+
trap("URG") { @refork_requested = true }
|
|
204
|
+
else
|
|
205
|
+
Log.warn "Ignoring refork_after: PR_SET_CHILD_SUBREAPER not supported on this platform"
|
|
206
|
+
@refork_thresholds = [].freeze
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
207
210
|
File.open(@pid_file, File::CREAT | File::EXCL | File::WRONLY) { |file| file.write(Process.pid.to_s) } if @pid_file
|
|
208
211
|
|
|
209
212
|
@bpf_active = ReuseportBPF.setup(@worker_count)
|
|
@@ -224,7 +227,9 @@ module Raptor
|
|
|
224
227
|
break if reap_workers == :no_children
|
|
225
228
|
|
|
226
229
|
perform_hot_restart if @hot_restart_requested
|
|
230
|
+
poll_seed_ready if @seed_pid && !@seed_ready
|
|
227
231
|
perform_phased_restart if @phased_restart_requested && !@phased_restarting
|
|
232
|
+
check_refork_trigger if @refork_thresholds.any? && !@phased_restarting
|
|
228
233
|
timeout_hung_workers
|
|
229
234
|
|
|
230
235
|
sleep 0.1
|
|
@@ -242,7 +247,8 @@ module Raptor
|
|
|
242
247
|
# Returns stats for all worker processes.
|
|
243
248
|
#
|
|
244
249
|
# @return [Array<Hash>] array of per-worker stat hashes, each containing
|
|
245
|
-
# :pid, :
|
|
250
|
+
# :pid, :index, :phase, :requests, :backlog, :busy_threads,
|
|
251
|
+
# :thread_capacity, :started_at, :last_checkin, and :booted
|
|
246
252
|
#
|
|
247
253
|
# @rbs () -> Array[Hash[Symbol, untyped]]
|
|
248
254
|
def stats
|
|
@@ -253,8 +259,7 @@ module Raptor
|
|
|
253
259
|
|
|
254
260
|
# Returns the inherited-FDs hash for a systemd socket-activation handoff,
|
|
255
261
|
# pairing each bind URI with the FD systemd passed at the same index.
|
|
256
|
-
#
|
|
257
|
-
# match the number of bind URIs.
|
|
262
|
+
# Returns an empty hash when the FD count doesn't match the bind count.
|
|
258
263
|
#
|
|
259
264
|
# @param bind_uris [Array<String>] the configured bind URIs
|
|
260
265
|
# @param filenos [Array<Integer>] file descriptors passed by systemd
|
|
@@ -270,61 +275,256 @@ module Raptor
|
|
|
270
275
|
bind_uris.zip(filenos).to_h { |bind_uri, fileno| [bind_uri, [fileno]] }
|
|
271
276
|
end
|
|
272
277
|
|
|
273
|
-
#
|
|
274
|
-
#
|
|
278
|
+
# Normalises the `refork_after` option into a sorted array of positive
|
|
279
|
+
# thresholds. Accepts nil, 0, an Integer, or an Array<Integer>; anything
|
|
280
|
+
# else falls back to an empty array (feature disabled).
|
|
281
|
+
#
|
|
282
|
+
# @param value [Integer, Array<Integer>, nil] the raw option value
|
|
283
|
+
# @return [Array<Integer>]
|
|
284
|
+
#
|
|
285
|
+
# @rbs (untyped value) -> Array[Integer]
|
|
286
|
+
def normalize_refork_thresholds(value)
|
|
287
|
+
case value
|
|
288
|
+
when Integer
|
|
289
|
+
value.positive? ? [value].freeze : [].freeze
|
|
290
|
+
when Array
|
|
291
|
+
value.select { |threshold| threshold.is_a?(Integer) && threshold.positive? }.sort.freeze
|
|
292
|
+
else
|
|
293
|
+
[].freeze
|
|
294
|
+
end
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# Forks a new worker process and registers it at the given index,
|
|
298
|
+
# forking from the seed when one is active and off the master otherwise.
|
|
275
299
|
#
|
|
276
300
|
# @param index [Integer] slot index for this worker in the stats region
|
|
277
301
|
# @return [void]
|
|
278
302
|
#
|
|
279
303
|
# @rbs (Integer index) -> void
|
|
280
304
|
def spawn_worker(index)
|
|
305
|
+
if @seed_pid && pid_alive?(@seed_pid)
|
|
306
|
+
pid = spawn_worker_via_seed(index)
|
|
307
|
+
if pid
|
|
308
|
+
@workers[index] = pid
|
|
309
|
+
return
|
|
310
|
+
end
|
|
311
|
+
Log.warn "Seed (#{@seed_pid}) failed to fork worker #{index}, falling back to direct fork"
|
|
312
|
+
@seed_pid = nil
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
@before_fork.each(&:call)
|
|
281
316
|
pid = fork { run_worker(index, @phase) }
|
|
282
317
|
@workers[index] = pid
|
|
283
318
|
end
|
|
284
319
|
|
|
320
|
+
# Asks the seed to fork a new worker at the given index, returning the
|
|
321
|
+
# child pid, or nil when the seed doesn't respond in time.
|
|
322
|
+
#
|
|
323
|
+
# @param index [Integer] slot index for the new worker
|
|
324
|
+
# @return [Integer, nil] the forked worker's pid, or nil on failure
|
|
325
|
+
#
|
|
326
|
+
# @rbs (Integer index) -> Integer?
|
|
327
|
+
def spawn_worker_via_seed(index)
|
|
328
|
+
@fork_w.write([index, @phase].pack("LL"))
|
|
329
|
+
return nil unless @resp_r.wait_readable(5)
|
|
330
|
+
|
|
331
|
+
bytes = @resp_r.read_nonblock(4, exception: false)
|
|
332
|
+
return nil unless bytes.is_a?(String) && bytes.bytesize == 4
|
|
333
|
+
|
|
334
|
+
bytes.unpack1("L")
|
|
335
|
+
rescue Errno::EPIPE, IOError
|
|
336
|
+
nil
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
# Checks whether a process with the given pid is currently alive.
|
|
340
|
+
#
|
|
341
|
+
# @param pid [Integer] the pid to probe
|
|
342
|
+
# @return [Boolean] true if the process exists
|
|
343
|
+
#
|
|
344
|
+
# @rbs (Integer pid) -> bool
|
|
345
|
+
def pid_alive?(pid)
|
|
346
|
+
Process.kill(0, pid)
|
|
347
|
+
true
|
|
348
|
+
rescue Errno::ESRCH, Errno::ECHILD
|
|
349
|
+
false
|
|
350
|
+
rescue Errno::EPERM
|
|
351
|
+
true
|
|
352
|
+
end
|
|
353
|
+
|
|
285
354
|
# Reaps any worker processes that have exited, respawning each one
|
|
286
355
|
# unless the cluster is shutting down.
|
|
287
356
|
#
|
|
288
|
-
# @return [Symbol] :no_children when
|
|
357
|
+
# @return [Symbol] :no_children when nothing is left to supervise, otherwise :reaped
|
|
289
358
|
#
|
|
290
359
|
# @rbs () -> Symbol
|
|
291
360
|
def reap_workers
|
|
292
361
|
loop do
|
|
293
362
|
pid, status = Process.wait2(-1, Process::WNOHANG)
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
index = @workers.key(pid)
|
|
297
|
-
@workers.delete(index)
|
|
298
|
-
@timed_out.delete(pid)
|
|
363
|
+
break unless pid
|
|
299
364
|
|
|
300
|
-
|
|
301
|
-
Log.warn "Restarting worker #{index} (#{pid}), #{exit_description(status)}"
|
|
302
|
-
spawn_worker(index)
|
|
303
|
-
end
|
|
365
|
+
reap_pid(pid, status)
|
|
304
366
|
end
|
|
367
|
+
|
|
368
|
+
@workers.empty? && !@seed_pid ? :no_children : :reaped
|
|
305
369
|
rescue Errno::ECHILD
|
|
306
|
-
:no_children
|
|
370
|
+
@workers.empty? && !@seed_pid ? :no_children : :reaped
|
|
307
371
|
end
|
|
308
372
|
|
|
309
|
-
#
|
|
310
|
-
#
|
|
373
|
+
# Records a reaped pid, respawning its worker slot unless the cluster
|
|
374
|
+
# is shutting down. Clears the seed reference when the seed exits.
|
|
375
|
+
#
|
|
376
|
+
# @param pid [Integer] the reaped pid
|
|
377
|
+
# @param status [Process::Status] the exit status
|
|
378
|
+
# @return [void]
|
|
379
|
+
#
|
|
380
|
+
# @rbs (Integer pid, Process::Status status) -> void
|
|
381
|
+
def reap_pid(pid, status)
|
|
382
|
+
if pid == @seed_pid
|
|
383
|
+
Log.info "Seed (#{pid}) exited, #{exit_description(status)}"
|
|
384
|
+
@seed_pid = nil
|
|
385
|
+
return
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
index = @workers.key(pid)
|
|
389
|
+
return unless index
|
|
390
|
+
|
|
391
|
+
@workers.delete(index)
|
|
392
|
+
@timed_out.delete(pid)
|
|
393
|
+
|
|
394
|
+
unless @shutdown
|
|
395
|
+
Log.warn "Restarting worker #{index} (#{pid}), #{exit_description(status)}"
|
|
396
|
+
spawn_worker(index)
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
# Promotes the most-experienced worker to a seed and starts a phased
|
|
401
|
+
# refork when the next `refork_after` threshold is crossed or a manual
|
|
402
|
+
# refork was requested via `SIGURG`.
|
|
403
|
+
#
|
|
404
|
+
# @return [void]
|
|
405
|
+
#
|
|
406
|
+
# @rbs () -> void
|
|
407
|
+
def check_refork_trigger
|
|
408
|
+
candidate = pick_refork_candidate
|
|
409
|
+
return unless candidate
|
|
410
|
+
|
|
411
|
+
candidate_index, candidate_requests = candidate
|
|
412
|
+
threshold = @refork_thresholds[@refork_threshold_idx]
|
|
413
|
+
|
|
414
|
+
if @refork_requested
|
|
415
|
+
@refork_requested = false
|
|
416
|
+
elsif !threshold || candidate_requests < threshold
|
|
417
|
+
return
|
|
418
|
+
else
|
|
419
|
+
@refork_threshold_idx += 1
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
promote_worker_to_seed(candidate_index)
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
# Picks the most-experienced booted worker in the current phase,
|
|
426
|
+
# returning its slot index and its request count. Returns nil when
|
|
427
|
+
# no worker qualifies.
|
|
428
|
+
#
|
|
429
|
+
# @return [Array<Integer>, nil]
|
|
430
|
+
#
|
|
431
|
+
# @rbs () -> Array[Integer]?
|
|
432
|
+
def pick_refork_candidate
|
|
433
|
+
best_index = nil
|
|
434
|
+
best_requests = -1
|
|
435
|
+
@stats.all.each_with_index do |stat, index|
|
|
436
|
+
next unless @workers[index] == stat[:pid]
|
|
437
|
+
next unless stat[:booted]
|
|
438
|
+
next unless stat[:phase] == @phase
|
|
439
|
+
next unless stat[:requests] > best_requests
|
|
440
|
+
|
|
441
|
+
best_index = index
|
|
442
|
+
best_requests = stat[:requests]
|
|
443
|
+
end
|
|
444
|
+
[best_index, best_requests] if best_index
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
# Retires the current seed and promotes the given worker into its place,
|
|
448
|
+
# queueing a phased refork for the remaining workers.
|
|
449
|
+
#
|
|
450
|
+
# @param index [Integer] slot index of the worker to promote
|
|
451
|
+
# @return [void]
|
|
452
|
+
#
|
|
453
|
+
# @rbs (Integer index) -> void
|
|
454
|
+
def promote_worker_to_seed(index)
|
|
455
|
+
pid = @workers[index]
|
|
456
|
+
return unless pid
|
|
457
|
+
|
|
458
|
+
retire_current_seed
|
|
459
|
+
Log.info "Promoting worker #{index} to seed for phased refork"
|
|
460
|
+
ReuseportBPF.mark_unavailable(index) if @bpf_active
|
|
461
|
+
Process.kill("URG", pid) rescue nil
|
|
462
|
+
@workers.delete(index)
|
|
463
|
+
@seed_pid = pid
|
|
464
|
+
@seed_ready = false
|
|
465
|
+
@seed_vacated_index = index
|
|
466
|
+
@phased_restart_requested = true
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
# Terminates the currently-active seed process, if any, and waits for
|
|
470
|
+
# it to exit. Its seed-forked workers stay attached to the master and
|
|
471
|
+
# keep serving.
|
|
472
|
+
#
|
|
473
|
+
# @return [void]
|
|
474
|
+
#
|
|
475
|
+
# @rbs () -> void
|
|
476
|
+
def retire_current_seed
|
|
477
|
+
return unless @seed_pid && pid_alive?(@seed_pid)
|
|
478
|
+
|
|
479
|
+
Log.info "Retiring seed (#{@seed_pid})"
|
|
480
|
+
Process.kill("TERM", @seed_pid) rescue nil
|
|
481
|
+
|
|
482
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @worker_shutdown_timeout
|
|
483
|
+
while @seed_pid && pid_alive?(@seed_pid)
|
|
484
|
+
reap_workers
|
|
485
|
+
break if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
486
|
+
|
|
487
|
+
sleep 0.05
|
|
488
|
+
end
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
# Records the seed's readiness when its ready marker has arrived.
|
|
492
|
+
# Non-blocking.
|
|
493
|
+
#
|
|
494
|
+
# @return [void]
|
|
495
|
+
#
|
|
496
|
+
# @rbs () -> void
|
|
497
|
+
def poll_seed_ready
|
|
498
|
+
return unless @resp_r && @seed_pid && !@seed_ready
|
|
499
|
+
return unless @resp_r.wait_readable(0)
|
|
500
|
+
|
|
501
|
+
bytes = @resp_r.read_nonblock(4, exception: false)
|
|
502
|
+
return unless bytes.is_a?(String) && bytes.bytesize == 4
|
|
503
|
+
|
|
504
|
+
@seed_ready = true if bytes.unpack1("L") == 0
|
|
505
|
+
end
|
|
506
|
+
|
|
507
|
+
# Stops every worker (and the seed if one is active), escalating
|
|
508
|
+
# from TERM to KILL if any fail to exit within `worker_shutdown_timeout`.
|
|
311
509
|
#
|
|
312
510
|
# @return [void]
|
|
313
511
|
#
|
|
314
512
|
# @rbs () -> void
|
|
315
513
|
def stop_workers
|
|
316
514
|
@workers.values.each { |pid| Process.kill("TERM", pid) rescue nil }
|
|
515
|
+
Process.kill("TERM", @seed_pid) rescue nil if @seed_pid
|
|
317
516
|
|
|
318
517
|
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @worker_shutdown_timeout
|
|
319
|
-
until @workers.empty? || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
518
|
+
until (@workers.empty? && !@seed_pid) || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
320
519
|
reap_workers
|
|
321
520
|
sleep 0.05
|
|
322
521
|
end
|
|
323
|
-
return if @workers.empty?
|
|
522
|
+
return if @workers.empty? && !@seed_pid
|
|
324
523
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
524
|
+
pids = @workers.values + [@seed_pid].compact
|
|
525
|
+
Log.warn "Force-killing #{pids.size} process(es) after #{@worker_shutdown_timeout}s"
|
|
526
|
+
pids.each { |pid| Process.kill("KILL", pid) rescue nil }
|
|
527
|
+
pids.each { |pid| Process.wait(pid) rescue nil }
|
|
328
528
|
end
|
|
329
529
|
|
|
330
530
|
# Kills workers that have stopped checking in. A booted worker that
|
|
@@ -370,8 +570,12 @@ module Raptor
|
|
|
370
570
|
Log.info "Phased restart starting"
|
|
371
571
|
|
|
372
572
|
begin
|
|
573
|
+
wait_for_seed_ready
|
|
574
|
+
filled_index = fill_vacated_seed_slot
|
|
575
|
+
|
|
373
576
|
@workers.keys.sort.each do |index|
|
|
374
577
|
return if @shutdown
|
|
578
|
+
next if index == filled_index
|
|
375
579
|
|
|
376
580
|
target_pid = @workers[index]
|
|
377
581
|
next unless target_pid
|
|
@@ -382,7 +586,8 @@ module Raptor
|
|
|
382
586
|
until @shutdown
|
|
383
587
|
reap_workers
|
|
384
588
|
current = @workers[index]
|
|
385
|
-
|
|
589
|
+
stat = @stats.all[index]
|
|
590
|
+
break if current && current != target_pid && stat[:pid] == current && stat[:booted]
|
|
386
591
|
break if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
387
592
|
|
|
388
593
|
sleep 0.1
|
|
@@ -395,6 +600,48 @@ module Raptor
|
|
|
395
600
|
end
|
|
396
601
|
end
|
|
397
602
|
|
|
603
|
+
# Blocks until the freshly-promoted seed has signalled readiness,
|
|
604
|
+
# or times out and clears the seed reference. No-op when no seed is
|
|
605
|
+
# being promoted.
|
|
606
|
+
#
|
|
607
|
+
# @return [void]
|
|
608
|
+
#
|
|
609
|
+
# @rbs () -> void
|
|
610
|
+
def wait_for_seed_ready
|
|
611
|
+
return unless @seed_pid && !@seed_ready
|
|
612
|
+
|
|
613
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @worker_drain_timeout + 5
|
|
614
|
+
until @seed_ready || @shutdown
|
|
615
|
+
break unless pid_alive?(@seed_pid)
|
|
616
|
+
break if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
617
|
+
|
|
618
|
+
poll_seed_ready
|
|
619
|
+
sleep 0.1
|
|
620
|
+
end
|
|
621
|
+
|
|
622
|
+
return if @seed_ready
|
|
623
|
+
|
|
624
|
+
Log.warn "Seed (#{@seed_pid}) didn't signal ready in time, falling back to direct forks"
|
|
625
|
+
@seed_pid = nil
|
|
626
|
+
end
|
|
627
|
+
|
|
628
|
+
# Spawns a replacement worker for the slot the seed vacated when
|
|
629
|
+
# it was promoted, returning the slot index it filled.
|
|
630
|
+
#
|
|
631
|
+
# @return [Integer, nil] the filled slot index, or nil if none was vacated
|
|
632
|
+
#
|
|
633
|
+
# @rbs () -> Integer?
|
|
634
|
+
def fill_vacated_seed_slot
|
|
635
|
+
index = @seed_vacated_index
|
|
636
|
+
return unless index
|
|
637
|
+
|
|
638
|
+
@seed_vacated_index = nil
|
|
639
|
+
return if @shutdown
|
|
640
|
+
|
|
641
|
+
spawn_worker(index)
|
|
642
|
+
index
|
|
643
|
+
end
|
|
644
|
+
|
|
398
645
|
# Re-execs the master process with a fresh boot of the same Raptor
|
|
399
646
|
# invocation, handing the new master its listening sockets so accepted
|
|
400
647
|
# connections continue to be served across the swap.
|
|
@@ -425,11 +672,10 @@ module Raptor
|
|
|
425
672
|
exec(@launch_command, *@launch_argv)
|
|
426
673
|
end
|
|
427
674
|
|
|
428
|
-
# Runs
|
|
429
|
-
#
|
|
430
|
-
#
|
|
431
|
-
#
|
|
432
|
-
# critical component fails.
|
|
675
|
+
# Runs a worker process's full server stack until a shutdown signal is
|
|
676
|
+
# received or a critical component fails. On `SIGURG` the worker drains
|
|
677
|
+
# and transitions into a seed loop that forks replacement workers on
|
|
678
|
+
# master's request.
|
|
433
679
|
#
|
|
434
680
|
# @param index [Integer] slot index for this worker in the stats region
|
|
435
681
|
# @param phase [Integer] the cluster phase this worker was forked at
|
|
@@ -437,12 +683,17 @@ module Raptor
|
|
|
437
683
|
#
|
|
438
684
|
# @rbs (Integer index, Integer phase) -> void
|
|
439
685
|
def run_worker(index, phase)
|
|
686
|
+
@fork_w.close if @fork_w && !@fork_w.closed?
|
|
687
|
+
@resp_r.close if @resp_r && !@resp_r.closed?
|
|
688
|
+
|
|
440
689
|
shutdown_requested = false
|
|
690
|
+
promote_to_seed = false
|
|
441
691
|
trap("INT") { shutdown_requested = true }
|
|
442
692
|
trap("TERM") { shutdown_requested = true }
|
|
443
693
|
trap("HUP") { reopen_logs }
|
|
444
694
|
trap("USR1", "IGNORE")
|
|
445
695
|
trap("USR2", "IGNORE")
|
|
696
|
+
trap("URG") { promote_to_seed = true } if @fork_r
|
|
446
697
|
|
|
447
698
|
Raptor::CPU.pin(index) if Raptor::CPU.count >= @worker_count
|
|
448
699
|
|
|
@@ -487,7 +738,7 @@ module Raptor
|
|
|
487
738
|
)
|
|
488
739
|
ractor_pool = RactorPool.new(
|
|
489
740
|
size: @ractor_count,
|
|
490
|
-
worker: http1.
|
|
741
|
+
worker: http1.parser_worker
|
|
491
742
|
) do |parsed_result|
|
|
492
743
|
begin
|
|
493
744
|
if parsed_result[:protocol] == :http2
|
|
@@ -510,6 +761,7 @@ module Raptor
|
|
|
510
761
|
|
|
511
762
|
worker_listeners = if @bpf_active
|
|
512
763
|
bpf_listeners = ReuseportBPF.create_worker_listeners(@binder.bind_uris, index, @binder.socket_backlog)
|
|
764
|
+
ReuseportBPF.enable_load_reporting(index)
|
|
513
765
|
non_tcp_listeners = @binder.listeners.reject { |listener| listener.is_a?(TCPServer) }
|
|
514
766
|
bpf_listeners + non_tcp_listeners
|
|
515
767
|
else
|
|
@@ -529,6 +781,8 @@ module Raptor
|
|
|
529
781
|
)
|
|
530
782
|
server_thread = server.run
|
|
531
783
|
|
|
784
|
+
@before_worker_boot.each(&:call)
|
|
785
|
+
|
|
532
786
|
Log.info "Worker #{index} booted"
|
|
533
787
|
|
|
534
788
|
stats_thread = Thread.new do
|
|
@@ -547,19 +801,26 @@ module Raptor
|
|
|
547
801
|
last_checkin: Process.clock_gettime(Process::CLOCK_REALTIME),
|
|
548
802
|
booted: true
|
|
549
803
|
)
|
|
550
|
-
break if shutdown_requested
|
|
804
|
+
break if shutdown_requested || promote_to_seed
|
|
551
805
|
|
|
552
806
|
sleep 1
|
|
553
807
|
end
|
|
554
808
|
end
|
|
555
809
|
|
|
556
|
-
until shutdown_requested
|
|
810
|
+
until shutdown_requested || promote_to_seed
|
|
557
811
|
break unless server_thread.alive? && reactor_thread.alive?
|
|
558
812
|
|
|
559
813
|
sleep 0.5
|
|
560
814
|
end
|
|
561
815
|
|
|
562
|
-
|
|
816
|
+
if promote_to_seed
|
|
817
|
+
@before_refork.each(&:call)
|
|
818
|
+
server.stop_accepting
|
|
819
|
+
else
|
|
820
|
+
@before_worker_shutdown.each(&:call)
|
|
821
|
+
server.shutdown
|
|
822
|
+
end
|
|
823
|
+
|
|
563
824
|
server_thread.join
|
|
564
825
|
reactor.shutdown
|
|
565
826
|
reactor_thread.join
|
|
@@ -567,6 +828,44 @@ module Raptor
|
|
|
567
828
|
http1.shutdown
|
|
568
829
|
drain_thread_pool(thread_pool)
|
|
569
830
|
stats_thread.join
|
|
831
|
+
|
|
832
|
+
run_seed_loop(index) if promote_to_seed
|
|
833
|
+
end
|
|
834
|
+
|
|
835
|
+
# Runs the seed's fork loop, forking a replacement worker for each
|
|
836
|
+
# slot index the master asks for.
|
|
837
|
+
#
|
|
838
|
+
# @param index [Integer] the seed's original slot index
|
|
839
|
+
# @return [void]
|
|
840
|
+
#
|
|
841
|
+
# @rbs (Integer index) -> void
|
|
842
|
+
def run_seed_loop(index)
|
|
843
|
+
Log.info "Worker #{index} promoted to seed"
|
|
844
|
+
|
|
845
|
+
seed_shutdown = false
|
|
846
|
+
trap("INT") { seed_shutdown = true }
|
|
847
|
+
trap("TERM") { seed_shutdown = true }
|
|
848
|
+
trap("URG", "IGNORE")
|
|
849
|
+
|
|
850
|
+
child_pids = []
|
|
851
|
+
trap("CHLD") do
|
|
852
|
+
child_pids.reject! { Process.wait(_1, Process::WNOHANG) rescue true }
|
|
853
|
+
end
|
|
854
|
+
|
|
855
|
+
@resp_w.write([0].pack("L"))
|
|
856
|
+
|
|
857
|
+
until seed_shutdown
|
|
858
|
+
next unless @fork_r.wait_readable(1)
|
|
859
|
+
|
|
860
|
+
bytes = @fork_r.read_nonblock(8, exception: false)
|
|
861
|
+
break unless bytes.is_a?(String) && bytes.bytesize == 8
|
|
862
|
+
|
|
863
|
+
slot_index, child_phase = bytes.unpack("LL")
|
|
864
|
+
pid = fork { run_worker(slot_index, child_phase) }
|
|
865
|
+
child_pids << pid
|
|
866
|
+
@resp_w.write([pid].pack("L")) rescue nil
|
|
867
|
+
end
|
|
868
|
+
rescue Errno::EPIPE, IOError
|
|
570
869
|
end
|
|
571
870
|
|
|
572
871
|
# Shuts down the worker's application thread pool, force-killing the
|
|
@@ -578,7 +877,11 @@ module Raptor
|
|
|
578
877
|
#
|
|
579
878
|
# @rbs (AtomicThreadPool thread_pool) -> void
|
|
580
879
|
def drain_thread_pool(thread_pool)
|
|
581
|
-
drain = Thread.new
|
|
880
|
+
drain = Thread.new do
|
|
881
|
+
Thread.current.name = "Thread Pool Drain"
|
|
882
|
+
|
|
883
|
+
thread_pool.shutdown
|
|
884
|
+
end
|
|
582
885
|
return if drain.join(@worker_drain_timeout)
|
|
583
886
|
|
|
584
887
|
Log.warn "Force-killing in-flight app threads after #{@worker_drain_timeout}s drain timeout"
|