raptor 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,39 +18,15 @@ require_relative "stats"
18
18
  require_relative "systemd"
19
19
 
20
20
  module Raptor
21
- # Multi-process web server cluster with advanced concurrency architecture.
22
- #
23
- # Cluster manages multiple worker processes, each running a complete server
24
- # stack including a ractor pool for HTTP parsing, a thread pool for
25
- # application processing, plus dedicated reactor and server threads. It
26
- # handles process forking, signal management, graceful shutdown, and
27
- # automatic worker restart when a worker process unexpectedly exits.
28
- #
29
- # The architecture provides horizontal scaling through processes while
30
- # maintaining efficient I/O and CPU utilization within each process
31
- # through the combination of ractor-based parsing and thread pools on
32
- # top of NIO reactors.
33
- #
34
- # Flow per worker process:
35
- # 1. Server continuously accepts connections but skips acceptance when backlog is high
36
- # 2. Reactor manages I/O multiplexing and provides backlog metrics for load control
37
- # 3. Ractor pool handles CPU-intensive HTTP parsing in parallel
38
- # 4. Thread pool processes Rack applications and handles response writing
39
- # 5. Natural load balancing occurs through backpressure-based acceptance control
40
- #
41
- # @example Basic usage
42
- # options = {
43
- # workers: 4, ractors: 2, threads: 8,
44
- # binds: ["tcp://0.0.0.0:3000"],
45
- # rackup: "config.ru",
46
- # connection: { first_data_timeout: 30, chunk_data_timeout: 10 }
47
- # }
48
- # Cluster.run(options)
21
+ # Forks and supervises worker processes. Handles graceful shutdown on
22
+ # `INT` and `TERM`, phased restart on `USR1`, hot restart on `USR2`,
23
+ # and (on Linux) refork on `URG`. Restarts workers that exit
24
+ # unexpectedly or stop checking in.
49
25
  #
50
26
  class Cluster
51
27
  INHERITED_FDS_ENV = "RAPTOR_INHERITED_FDS"
52
28
 
53
- # Convenience method to create and run a cluster with the given options.
29
+ # Creates and runs a cluster with the given options.
54
30
  #
55
31
  # @param options [Hash] cluster configuration options
56
32
  # @return [void]
@@ -72,6 +48,10 @@ module Raptor
72
48
  # @rbs @worker_timeout: Integer
73
49
  # @rbs @worker_drain_timeout: Integer
74
50
  # @rbs @worker_shutdown_timeout: Integer
51
+ # @rbs @before_fork: Array[Proc]
52
+ # @rbs @before_worker_boot: Array[Proc]
53
+ # @rbs @before_worker_shutdown: Array[Proc]
54
+ # @rbs @before_refork: Array[Proc]
75
55
  # @rbs @stats_file: String?
76
56
  # @rbs @pid_file: String?
77
57
  # @rbs @stdout_file: String?
@@ -92,13 +72,19 @@ module Raptor
92
72
  # @rbs @phased_restart_requested: bool
93
73
  # @rbs @phased_restarting: bool
94
74
  # @rbs @hot_restart_requested: bool
75
+ # @rbs @refork_thresholds: Array[Integer]
76
+ # @rbs @refork_requested: bool
77
+ # @rbs @refork_threshold_idx: Integer
78
+ # @rbs @seed_pid: Integer?
79
+ # @rbs @seed_ready: bool
80
+ # @rbs @seed_vacated_index: Integer?
81
+ # @rbs @fork_r: IO?
82
+ # @rbs @fork_w: IO?
83
+ # @rbs @resp_r: IO?
84
+ # @rbs @resp_w: IO?
95
85
  # @rbs @bpf_active: bool
96
86
 
97
- # Creates a new Cluster with the specified configuration.
98
- #
99
- # Initializes the cluster with worker, ractor, and thread counts,
100
- # sets up network binding, loads the Rack application, and prepares
101
- # for multi-process operation.
87
+ # Creates a new Cluster with the given options.
102
88
  #
103
89
  # @param options [Hash] cluster configuration options
104
90
  # @option options [Array<String>] :binds array of bind URIs
@@ -118,6 +104,11 @@ module Raptor
118
104
  # @option options [Integer] :worker_timeout seconds to wait for a booted worker to check in before killing it
119
105
  # @option options [Integer] :worker_drain_timeout seconds a worker waits for in-flight requests during shutdown before force-killing app threads
120
106
  # @option options [Integer] :worker_shutdown_timeout seconds to wait for graceful worker exit before force-killing
107
+ # @option options [Integer, Array<Integer>, nil] :refork_after request-count threshold(s) at which a warm worker is promoted to a fork source for phased refork; nil or 0 disables. Requires `PR_SET_CHILD_SUBREAPER` (Linux)
108
+ # @option options [Array<Proc>] :before_fork procs called in the master before every worker fork
109
+ # @option options [Array<Proc>] :before_worker_boot procs called in each worker before it begins serving
110
+ # @option options [Array<Proc>] :before_worker_shutdown procs called in each worker before its graceful shutdown
111
+ # @option options [Array<Proc>] :before_refork procs called in a worker before it transitions to a seed
121
112
  # @option options [String, nil] :stats_file path to write per-worker stats JSON, or nil to disable
122
113
  # @option options [String, nil] :pid_file path to write the master PID to, or nil to disable
123
114
  # @option options [String, nil] :stdout_file path to redirect stdout to, reopened on SIGHUP, or nil to disable
@@ -142,6 +133,10 @@ module Raptor
142
133
  @worker_timeout = options[:worker_timeout]
143
134
  @worker_drain_timeout = options[:worker_drain_timeout]
144
135
  @worker_shutdown_timeout = options[:worker_shutdown_timeout]
136
+ @before_fork = Array(options[:before_fork])
137
+ @before_worker_boot = Array(options[:before_worker_boot])
138
+ @before_worker_shutdown = Array(options[:before_worker_shutdown])
139
+ @before_refork = Array(options[:before_refork])
145
140
  @stats_file = options[:stats_file]
146
141
  @pid_file = options[:pid_file]
147
142
  @stdout_file = options[:stdout_file]
@@ -175,27 +170,24 @@ module Raptor
175
170
  @phased_restart_requested = false
176
171
  @phased_restarting = false
177
172
  @hot_restart_requested = false
173
+
174
+ @refork_thresholds = normalize_refork_thresholds(options[:refork_after])
175
+ @refork_requested = false
176
+ @refork_threshold_idx = 0
177
+ @seed_pid = nil
178
+ @seed_ready = false
179
+ @seed_vacated_index = nil
178
180
  end
179
181
 
180
- # Starts the multi-process cluster and manages worker processes.
181
- #
182
- # Forks the configured number of worker processes and monitors them,
183
- # restarting any that exit unexpectedly or stop checking in. Handles
184
- # graceful shutdown via INT or TERM signals, phased restart via USR1,
185
- # and hot restart via USR2.
186
- #
187
- # Each worker process includes:
188
- # - 1 server thread (continuously accepts connections with backpressure control)
189
- # - 1 reactor thread (I/O multiplexing, timeout handling, backlog monitoring)
190
- # - N pipeline ractors (parallel HTTP parsing)
191
- # - 1 pipeline collector thread (coordinates parsing results)
192
- # - M worker threads (Rack application processing and response writing)
193
- # - 1 stats thread (writes per-worker metrics to shared memory every second)
182
+ # Runs the cluster until a graceful shutdown is signalled.
194
183
  #
195
184
  # @return [void]
196
185
  #
197
186
  # @rbs () -> void
198
187
  def run
188
+ $stdout.sync = true
189
+ $stderr.sync = true
190
+
199
191
  reopen_logs
200
192
 
201
193
  trap("INT") { shutdown }
@@ -204,6 +196,17 @@ module Raptor
204
196
  trap("USR1") { @phased_restart_requested = true }
205
197
  trap("USR2") { @hot_restart_requested = true }
206
198
 
199
+ if @refork_thresholds.any?
200
+ if Subreaper.enable
201
+ @fork_r, @fork_w = IO.pipe
202
+ @resp_r, @resp_w = IO.pipe
203
+ trap("URG") { @refork_requested = true }
204
+ else
205
+ Log.warn "Ignoring refork_after: PR_SET_CHILD_SUBREAPER not supported on this platform"
206
+ @refork_thresholds = [].freeze
207
+ end
208
+ end
209
+
207
210
  File.open(@pid_file, File::CREAT | File::EXCL | File::WRONLY) { |file| file.write(Process.pid.to_s) } if @pid_file
208
211
 
209
212
  @bpf_active = ReuseportBPF.setup(@worker_count)
@@ -224,7 +227,9 @@ module Raptor
224
227
  break if reap_workers == :no_children
225
228
 
226
229
  perform_hot_restart if @hot_restart_requested
230
+ poll_seed_ready if @seed_pid && !@seed_ready
227
231
  perform_phased_restart if @phased_restart_requested && !@phased_restarting
232
+ check_refork_trigger if @refork_thresholds.any? && !@phased_restarting
228
233
  timeout_hung_workers
229
234
 
230
235
  sleep 0.1
@@ -242,7 +247,8 @@ module Raptor
242
247
  # Returns stats for all worker processes.
243
248
  #
244
249
  # @return [Array<Hash>] array of per-worker stat hashes, each containing
245
- # :pid, :requests, :backlog, :started_at, :last_checkin, and :booted
250
+ # :pid, :index, :phase, :requests, :backlog, :busy_threads,
251
+ # :thread_capacity, :started_at, :last_checkin, and :booted
246
252
  #
247
253
  # @rbs () -> Array[Hash[Symbol, untyped]]
248
254
  def stats
@@ -253,8 +259,7 @@ module Raptor
253
259
 
254
260
  # Returns the inherited-FDs hash for a systemd socket-activation handoff,
255
261
  # pairing each bind URI with the FD systemd passed at the same index.
256
- # The activation is skipped (with a warning) when the FD count doesn't
257
- # match the number of bind URIs.
262
+ # Returns an empty hash when the FD count doesn't match the bind count.
258
263
  #
259
264
  # @param bind_uris [Array<String>] the configured bind URIs
260
265
  # @param filenos [Array<Integer>] file descriptors passed by systemd
@@ -270,61 +275,256 @@ module Raptor
270
275
  bind_uris.zip(filenos).to_h { |bind_uri, fileno| [bind_uri, [fileno]] }
271
276
  end
272
277
 
273
- # Forks a new worker process and registers it at the given index.
274
- # The worker inherits the cluster's current phase.
278
+ # Normalises the `refork_after` option into a sorted array of positive
279
+ # thresholds. Accepts nil, 0, an Integer, or an Array<Integer>; anything
280
+ # else falls back to an empty array (feature disabled).
281
+ #
282
+ # @param value [Integer, Array<Integer>, nil] the raw option value
283
+ # @return [Array<Integer>]
284
+ #
285
+ # @rbs (untyped value) -> Array[Integer]
286
+ def normalize_refork_thresholds(value)
287
+ case value
288
+ when Integer
289
+ value.positive? ? [value].freeze : [].freeze
290
+ when Array
291
+ value.select { |threshold| threshold.is_a?(Integer) && threshold.positive? }.sort.freeze
292
+ else
293
+ [].freeze
294
+ end
295
+ end
296
+
297
+ # Forks a new worker process and registers it at the given index,
298
+ # forking from the seed when one is active and off the master otherwise.
275
299
  #
276
300
  # @param index [Integer] slot index for this worker in the stats region
277
301
  # @return [void]
278
302
  #
279
303
  # @rbs (Integer index) -> void
280
304
  def spawn_worker(index)
305
+ if @seed_pid && pid_alive?(@seed_pid)
306
+ pid = spawn_worker_via_seed(index)
307
+ if pid
308
+ @workers[index] = pid
309
+ return
310
+ end
311
+ Log.warn "Seed (#{@seed_pid}) failed to fork worker #{index}, falling back to direct fork"
312
+ @seed_pid = nil
313
+ end
314
+
315
+ @before_fork.each(&:call)
281
316
  pid = fork { run_worker(index, @phase) }
282
317
  @workers[index] = pid
283
318
  end
284
319
 
320
+ # Asks the seed to fork a new worker at the given index, returning the
321
+ # child pid, or nil when the seed doesn't respond in time.
322
+ #
323
+ # @param index [Integer] slot index for the new worker
324
+ # @return [Integer, nil] the forked worker's pid, or nil on failure
325
+ #
326
+ # @rbs (Integer index) -> Integer?
327
+ def spawn_worker_via_seed(index)
328
+ @fork_w.write([index, @phase].pack("LL"))
329
+ return nil unless @resp_r.wait_readable(5)
330
+
331
+ bytes = @resp_r.read_nonblock(4, exception: false)
332
+ return nil unless bytes.is_a?(String) && bytes.bytesize == 4
333
+
334
+ bytes.unpack1("L")
335
+ rescue Errno::EPIPE, IOError
336
+ nil
337
+ end
338
+
339
+ # Checks whether a process with the given pid is currently alive.
340
+ #
341
+ # @param pid [Integer] the pid to probe
342
+ # @return [Boolean] true if the process exists
343
+ #
344
+ # @rbs (Integer pid) -> bool
345
+ def pid_alive?(pid)
346
+ Process.kill(0, pid)
347
+ true
348
+ rescue Errno::ESRCH, Errno::ECHILD
349
+ false
350
+ rescue Errno::EPERM
351
+ true
352
+ end
353
+
285
354
  # Reaps any worker processes that have exited, respawning each one
286
355
  # unless the cluster is shutting down.
287
356
  #
288
- # @return [Symbol] :no_children when there are no remaining children, otherwise :reaped
357
+ # @return [Symbol] :no_children when nothing is left to supervise, otherwise :reaped
289
358
  #
290
359
  # @rbs () -> Symbol
291
360
  def reap_workers
292
361
  loop do
293
362
  pid, status = Process.wait2(-1, Process::WNOHANG)
294
- return :reaped unless pid
295
-
296
- index = @workers.key(pid)
297
- @workers.delete(index)
298
- @timed_out.delete(pid)
363
+ break unless pid
299
364
 
300
- unless @shutdown
301
- Log.warn "Restarting worker #{index} (#{pid}), #{exit_description(status)}"
302
- spawn_worker(index)
303
- end
365
+ reap_pid(pid, status)
304
366
  end
367
+
368
+ @workers.empty? && !@seed_pid ? :no_children : :reaped
305
369
  rescue Errno::ECHILD
306
- :no_children
370
+ @workers.empty? && !@seed_pid ? :no_children : :reaped
307
371
  end
308
372
 
309
- # Stops every worker, escalating from TERM to KILL if any fail to
310
- # exit within `worker_shutdown_timeout`.
373
+ # Records a reaped pid, respawning its worker slot unless the cluster
374
+ # is shutting down. Clears the seed reference when the seed exits.
375
+ #
376
+ # @param pid [Integer] the reaped pid
377
+ # @param status [Process::Status] the exit status
378
+ # @return [void]
379
+ #
380
+ # @rbs (Integer pid, Process::Status status) -> void
381
+ def reap_pid(pid, status)
382
+ if pid == @seed_pid
383
+ Log.info "Seed (#{pid}) exited, #{exit_description(status)}"
384
+ @seed_pid = nil
385
+ return
386
+ end
387
+
388
+ index = @workers.key(pid)
389
+ return unless index
390
+
391
+ @workers.delete(index)
392
+ @timed_out.delete(pid)
393
+
394
+ unless @shutdown
395
+ Log.warn "Restarting worker #{index} (#{pid}), #{exit_description(status)}"
396
+ spawn_worker(index)
397
+ end
398
+ end
399
+
400
+ # Promotes the most-experienced worker to a seed and starts a phased
401
+ # refork when the next `refork_after` threshold is crossed or a manual
402
+ # refork was requested via `SIGURG`.
403
+ #
404
+ # @return [void]
405
+ #
406
+ # @rbs () -> void
407
+ def check_refork_trigger
408
+ candidate = pick_refork_candidate
409
+ return unless candidate
410
+
411
+ candidate_index, candidate_requests = candidate
412
+ threshold = @refork_thresholds[@refork_threshold_idx]
413
+
414
+ if @refork_requested
415
+ @refork_requested = false
416
+ elsif !threshold || candidate_requests < threshold
417
+ return
418
+ else
419
+ @refork_threshold_idx += 1
420
+ end
421
+
422
+ promote_worker_to_seed(candidate_index)
423
+ end
424
+
425
+ # Picks the most-experienced booted worker in the current phase,
426
+ # returning its slot index and its request count. Returns nil when
427
+ # no worker qualifies.
428
+ #
429
+ # @return [Array<Integer>, nil]
430
+ #
431
+ # @rbs () -> Array[Integer]?
432
+ def pick_refork_candidate
433
+ best_index = nil
434
+ best_requests = -1
435
+ @stats.all.each_with_index do |stat, index|
436
+ next unless @workers[index] == stat[:pid]
437
+ next unless stat[:booted]
438
+ next unless stat[:phase] == @phase
439
+ next unless stat[:requests] > best_requests
440
+
441
+ best_index = index
442
+ best_requests = stat[:requests]
443
+ end
444
+ [best_index, best_requests] if best_index
445
+ end
446
+
447
+ # Retires the current seed and promotes the given worker into its place,
448
+ # queueing a phased refork for the remaining workers.
449
+ #
450
+ # @param index [Integer] slot index of the worker to promote
451
+ # @return [void]
452
+ #
453
+ # @rbs (Integer index) -> void
454
+ def promote_worker_to_seed(index)
455
+ pid = @workers[index]
456
+ return unless pid
457
+
458
+ retire_current_seed
459
+ Log.info "Promoting worker #{index} to seed for phased refork"
460
+ ReuseportBPF.mark_unavailable(index) if @bpf_active
461
+ Process.kill("URG", pid) rescue nil
462
+ @workers.delete(index)
463
+ @seed_pid = pid
464
+ @seed_ready = false
465
+ @seed_vacated_index = index
466
+ @phased_restart_requested = true
467
+ end
468
+
469
+ # Terminates the currently-active seed process, if any, and waits for
470
+ # it to exit. Its seed-forked workers stay attached to the master and
471
+ # keep serving.
472
+ #
473
+ # @return [void]
474
+ #
475
+ # @rbs () -> void
476
+ def retire_current_seed
477
+ return unless @seed_pid && pid_alive?(@seed_pid)
478
+
479
+ Log.info "Retiring seed (#{@seed_pid})"
480
+ Process.kill("TERM", @seed_pid) rescue nil
481
+
482
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @worker_shutdown_timeout
483
+ while @seed_pid && pid_alive?(@seed_pid)
484
+ reap_workers
485
+ break if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
486
+
487
+ sleep 0.05
488
+ end
489
+ end
490
+
491
+ # Records the seed's readiness when its ready marker has arrived.
492
+ # Non-blocking.
493
+ #
494
+ # @return [void]
495
+ #
496
+ # @rbs () -> void
497
+ def poll_seed_ready
498
+ return unless @resp_r && @seed_pid && !@seed_ready
499
+ return unless @resp_r.wait_readable(0)
500
+
501
+ bytes = @resp_r.read_nonblock(4, exception: false)
502
+ return unless bytes.is_a?(String) && bytes.bytesize == 4
503
+
504
+ @seed_ready = true if bytes.unpack1("L") == 0
505
+ end
506
+
507
+ # Stops every worker (and the seed if one is active), escalating
508
+ # from TERM to KILL if any fail to exit within `worker_shutdown_timeout`.
311
509
  #
312
510
  # @return [void]
313
511
  #
314
512
  # @rbs () -> void
315
513
  def stop_workers
316
514
  @workers.values.each { |pid| Process.kill("TERM", pid) rescue nil }
515
+ Process.kill("TERM", @seed_pid) rescue nil if @seed_pid
317
516
 
318
517
  deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @worker_shutdown_timeout
319
- until @workers.empty? || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
518
+ until (@workers.empty? && !@seed_pid) || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
320
519
  reap_workers
321
520
  sleep 0.05
322
521
  end
323
- return if @workers.empty?
522
+ return if @workers.empty? && !@seed_pid
324
523
 
325
- Log.warn "Force-killing #{@workers.size} worker(s) after #{@worker_shutdown_timeout}s"
326
- @workers.values.each { |pid| Process.kill("KILL", pid) rescue nil }
327
- @workers.values.each { |pid| Process.wait(pid) rescue nil }
524
+ pids = @workers.values + [@seed_pid].compact
525
+ Log.warn "Force-killing #{pids.size} process(es) after #{@worker_shutdown_timeout}s"
526
+ pids.each { |pid| Process.kill("KILL", pid) rescue nil }
527
+ pids.each { |pid| Process.wait(pid) rescue nil }
328
528
  end
329
529
 
330
530
  # Kills workers that have stopped checking in. A booted worker that
@@ -370,8 +570,12 @@ module Raptor
370
570
  Log.info "Phased restart starting"
371
571
 
372
572
  begin
573
+ wait_for_seed_ready
574
+ filled_index = fill_vacated_seed_slot
575
+
373
576
  @workers.keys.sort.each do |index|
374
577
  return if @shutdown
578
+ next if index == filled_index
375
579
 
376
580
  target_pid = @workers[index]
377
581
  next unless target_pid
@@ -382,7 +586,8 @@ module Raptor
382
586
  until @shutdown
383
587
  reap_workers
384
588
  current = @workers[index]
385
- break if current && current != target_pid && @stats.all[index][:booted]
589
+ stat = @stats.all[index]
590
+ break if current && current != target_pid && stat[:pid] == current && stat[:booted]
386
591
  break if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
387
592
 
388
593
  sleep 0.1
@@ -395,6 +600,48 @@ module Raptor
395
600
  end
396
601
  end
397
602
 
603
+ # Blocks until the freshly-promoted seed has signalled readiness,
604
+ # or times out and clears the seed reference. No-op when no seed is
605
+ # being promoted.
606
+ #
607
+ # @return [void]
608
+ #
609
+ # @rbs () -> void
610
+ def wait_for_seed_ready
611
+ return unless @seed_pid && !@seed_ready
612
+
613
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @worker_drain_timeout + 5
614
+ until @seed_ready || @shutdown
615
+ break unless pid_alive?(@seed_pid)
616
+ break if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
617
+
618
+ poll_seed_ready
619
+ sleep 0.1
620
+ end
621
+
622
+ return if @seed_ready
623
+
624
+ Log.warn "Seed (#{@seed_pid}) didn't signal ready in time, falling back to direct forks"
625
+ @seed_pid = nil
626
+ end
627
+
628
+ # Spawns a replacement worker for the slot the seed vacated when
629
+ # it was promoted, returning the slot index it filled.
630
+ #
631
+ # @return [Integer, nil] the filled slot index, or nil if none was vacated
632
+ #
633
+ # @rbs () -> Integer?
634
+ def fill_vacated_seed_slot
635
+ index = @seed_vacated_index
636
+ return unless index
637
+
638
+ @seed_vacated_index = nil
639
+ return if @shutdown
640
+
641
+ spawn_worker(index)
642
+ index
643
+ end
644
+
398
645
  # Re-execs the master process with a fresh boot of the same Raptor
399
646
  # invocation, handing the new master its listening sockets so accepted
400
647
  # connections continue to be served across the swap.
@@ -425,11 +672,10 @@ module Raptor
425
672
  exec(@launch_command, *@launch_argv)
426
673
  end
427
674
 
428
- # Runs the full server stack inside a worker process.
429
- #
430
- # Sets up and coordinates the reactor, server, ractor pool, thread pool,
431
- # and stats thread, running until a shutdown signal is received or a
432
- # critical component fails.
675
+ # Runs a worker process's full server stack until a shutdown signal is
676
+ # received or a critical component fails. On `SIGURG` the worker drains
677
+ # and transitions into a seed loop that forks replacement workers on
678
+ # master's request.
433
679
  #
434
680
  # @param index [Integer] slot index for this worker in the stats region
435
681
  # @param phase [Integer] the cluster phase this worker was forked at
@@ -437,12 +683,17 @@ module Raptor
437
683
  #
438
684
  # @rbs (Integer index, Integer phase) -> void
439
685
  def run_worker(index, phase)
686
+ @fork_w.close if @fork_w && !@fork_w.closed?
687
+ @resp_r.close if @resp_r && !@resp_r.closed?
688
+
440
689
  shutdown_requested = false
690
+ promote_to_seed = false
441
691
  trap("INT") { shutdown_requested = true }
442
692
  trap("TERM") { shutdown_requested = true }
443
693
  trap("HUP") { reopen_logs }
444
694
  trap("USR1", "IGNORE")
445
695
  trap("USR2", "IGNORE")
696
+ trap("URG") { promote_to_seed = true } if @fork_r
446
697
 
447
698
  Raptor::CPU.pin(index) if Raptor::CPU.count >= @worker_count
448
699
 
@@ -487,7 +738,7 @@ module Raptor
487
738
  )
488
739
  ractor_pool = RactorPool.new(
489
740
  size: @ractor_count,
490
- worker: http1.http_parser_worker
741
+ worker: http1.parser_worker
491
742
  ) do |parsed_result|
492
743
  begin
493
744
  if parsed_result[:protocol] == :http2
@@ -510,6 +761,7 @@ module Raptor
510
761
 
511
762
  worker_listeners = if @bpf_active
512
763
  bpf_listeners = ReuseportBPF.create_worker_listeners(@binder.bind_uris, index, @binder.socket_backlog)
764
+ ReuseportBPF.enable_load_reporting(index)
513
765
  non_tcp_listeners = @binder.listeners.reject { |listener| listener.is_a?(TCPServer) }
514
766
  bpf_listeners + non_tcp_listeners
515
767
  else
@@ -529,6 +781,8 @@ module Raptor
529
781
  )
530
782
  server_thread = server.run
531
783
 
784
+ @before_worker_boot.each(&:call)
785
+
532
786
  Log.info "Worker #{index} booted"
533
787
 
534
788
  stats_thread = Thread.new do
@@ -547,19 +801,26 @@ module Raptor
547
801
  last_checkin: Process.clock_gettime(Process::CLOCK_REALTIME),
548
802
  booted: true
549
803
  )
550
- break if shutdown_requested
804
+ break if shutdown_requested || promote_to_seed
551
805
 
552
806
  sleep 1
553
807
  end
554
808
  end
555
809
 
556
- until shutdown_requested
810
+ until shutdown_requested || promote_to_seed
557
811
  break unless server_thread.alive? && reactor_thread.alive?
558
812
 
559
813
  sleep 0.5
560
814
  end
561
815
 
562
- server.shutdown
816
+ if promote_to_seed
817
+ @before_refork.each(&:call)
818
+ server.stop_accepting
819
+ else
820
+ @before_worker_shutdown.each(&:call)
821
+ server.shutdown
822
+ end
823
+
563
824
  server_thread.join
564
825
  reactor.shutdown
565
826
  reactor_thread.join
@@ -567,6 +828,44 @@ module Raptor
567
828
  http1.shutdown
568
829
  drain_thread_pool(thread_pool)
569
830
  stats_thread.join
831
+
832
+ run_seed_loop(index) if promote_to_seed
833
+ end
834
+
835
+ # Runs the seed's fork loop, forking a replacement worker for each
836
+ # slot index the master asks for.
837
+ #
838
+ # @param index [Integer] the seed's original slot index
839
+ # @return [void]
840
+ #
841
+ # @rbs (Integer index) -> void
842
+ def run_seed_loop(index)
843
+ Log.info "Worker #{index} promoted to seed"
844
+
845
+ seed_shutdown = false
846
+ trap("INT") { seed_shutdown = true }
847
+ trap("TERM") { seed_shutdown = true }
848
+ trap("URG", "IGNORE")
849
+
850
+ child_pids = []
851
+ trap("CHLD") do
852
+ child_pids.reject! { Process.wait(_1, Process::WNOHANG) rescue true }
853
+ end
854
+
855
+ @resp_w.write([0].pack("L"))
856
+
857
+ until seed_shutdown
858
+ next unless @fork_r.wait_readable(1)
859
+
860
+ bytes = @fork_r.read_nonblock(8, exception: false)
861
+ break unless bytes.is_a?(String) && bytes.bytesize == 8
862
+
863
+ slot_index, child_phase = bytes.unpack("LL")
864
+ pid = fork { run_worker(slot_index, child_phase) }
865
+ child_pids << pid
866
+ @resp_w.write([pid].pack("L")) rescue nil
867
+ end
868
+ rescue Errno::EPIPE, IOError
570
869
  end
571
870
 
572
871
  # Shuts down the worker's application thread pool, force-killing the
@@ -578,7 +877,11 @@ module Raptor
578
877
  #
579
878
  # @rbs (AtomicThreadPool thread_pool) -> void
580
879
  def drain_thread_pool(thread_pool)
581
- drain = Thread.new { thread_pool.shutdown }
880
+ drain = Thread.new do
881
+ Thread.current.name = "Thread Pool Drain"
882
+
883
+ thread_pool.shutdown
884
+ end
582
885
  return if drain.join(@worker_drain_timeout)
583
886
 
584
887
  Log.warn "Force-killing in-flight app threads after #{@worker_drain_timeout}s drain timeout"