pitchfork 0.18.2 → 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,7 @@
3
3
 
4
4
  require 'pitchfork/pitchfork_http'
5
5
  require 'pitchfork/listeners'
6
+ require 'pitchfork/select_waiter'
6
7
  require 'pitchfork/flock'
7
8
  require 'pitchfork/soft_timeout'
8
9
  require 'pitchfork/shared_memory'
@@ -42,13 +43,19 @@ module Pitchfork
42
43
  @callback = callback
43
44
  @rack_env = nil
44
45
  @timeout_request = nil
46
+ @timed_out = false
45
47
  end
46
48
 
47
49
  def inspect
48
50
  "#<Pitchfork::HttpServer::TimeoutHandler##{object_id}>"
49
51
  end
50
52
 
53
+ def timed_out? # :nodoc:
54
+ @timed_out
55
+ end
56
+
51
57
  def call(original_thread) # :nodoc:
58
+ @timed_out = true
52
59
  begin
53
60
  @server.logger.error("#{@worker.to_log} timed out, exiting")
54
61
  if @callback
@@ -79,13 +86,13 @@ module Pitchfork
79
86
  # :stopdoc:
80
87
  attr_accessor :app, :timeout, :timeout_signal, :soft_timeout, :cleanup_timeout, :spawn_timeout, :worker_processes,
81
88
  :before_fork, :after_worker_fork, :after_mold_fork, :before_service_worker_ready, :before_service_worker_exit,
82
- :listener_opts, :children,
83
- :orig_app, :config, :ready_pipe, :early_hints, :setpgid
89
+ :listener_opts, :children, :poller,
90
+ :orig_app, :config, :ready_pipe, :early_hints, :setpgid, :restart_command, :restart_command_argv, :working_directory
84
91
  attr_writer :after_worker_exit, :before_worker_exit, :after_worker_ready, :after_request_complete,
85
92
  :refork_condition, :after_worker_timeout, :after_worker_hard_timeout, :after_monitor_ready, :refork_max_unavailable,
86
93
  :max_consecutive_spawn_errors
87
94
 
88
- attr_reader :logger
95
+ attr_reader :logger, :version
89
96
  include Pitchfork::SocketHelper
90
97
  include Pitchfork::HttpResponse
91
98
 
@@ -101,6 +108,7 @@ module Pitchfork
101
108
  # HttpServer.run.join to join the thread that's processing
102
109
  # incoming requests on the socket.
103
110
  def initialize(app, options = {})
111
+ @version = 0
104
112
  @exit_status = 0
105
113
  @app = app
106
114
  @respawn = false
@@ -113,7 +121,6 @@ module Pitchfork
113
121
 
114
122
  options = options.dup
115
123
  @ready_pipe = options.delete(:ready_pipe)
116
- @init_listeners = options[:listeners].dup || []
117
124
  options[:use_defaults] = true
118
125
  self.config = Pitchfork::Configurator.new(options)
119
126
  self.listener_opts = {}
@@ -147,31 +154,76 @@ module Pitchfork
147
154
  config.commit!(self, :skip => [:listeners, :pid])
148
155
  @orig_app = app
149
156
  # list of signals we care about and trap in monitor.
150
- @queue_sigs = [
151
- :QUIT, :INT, :TERM, :USR2, :TTIN, :TTOU ]
152
-
153
- Info.workers_count = worker_processes
154
- SharedMemory.preallocate_pages(worker_processes)
157
+ @queue_sigs = [ :QUIT, :INT, :TERM, :USR1, :USR2, :TTIN, :TTOU ]
155
158
  end
156
159
 
157
160
  # Runs the thing. Returns self so you can run join on it
158
161
  def start(sync = true)
159
162
  Pitchfork.enable_child_subreaper # noop if not supported
160
163
 
164
+ restart_state = ENV.delete("PITCHFORK_RESTART_STATE")
165
+ if restart_state
166
+ restart_state = Marshal.load(restart_state.unpack1("m0"))
167
+ logger.info "monitor v=#{restart_state[:version]} initializing with inherited state"
168
+ else
169
+ restart_state = {}
170
+ end
171
+
172
+ if restart_state[:version]
173
+ @version = restart_state[:version]
174
+ end
175
+
176
+ if working_directory
177
+ real_pwd = File.realpath(Dir.pwd)
178
+ real_working_directory = File.realpath(working_directory)
179
+
180
+ if real_pwd != real_working_directory
181
+ logger.info("monitor v=#{@version} chdir #{File.realpath(working_directory)}")
182
+ Dir.chdir(real_working_directory)
183
+ end
184
+ end
185
+
186
+ if restart_state[:sig_queue]
187
+ @sig_queue = [*restart_state[:sig_queue], *@sig_queue]
188
+ end
189
+
190
+ if restart_state[:shared_memory_fds]
191
+ SharedMemory.reopen(restart_state[:shared_memory_fds])
192
+ end
193
+ SharedMemory.preallocate_pages(worker_processes)
194
+ Info.workers_count = worker_processes
195
+
161
196
  # This socketpair is used to wake us up from select(2) in #join when signals
162
197
  # are trapped. See trap_deferred.
163
198
  # It's also used by newly spawned children to send their soft_signal pipe
164
199
  # to the monitor when they are spawned.
165
- @control_socket.replace(Pitchfork.socketpair)
200
+ if fds = restart_state[:control_socket_fds]
201
+ @control_socket.replace(Pitchfork.socketpair_for_fds(fds))
202
+ else
203
+ @control_socket.replace(Pitchfork.socketpair)
204
+ end
166
205
  Info.keep_ios(@control_socket)
167
206
  @monitor_pid = $$
168
207
 
208
+ if restart_state[:children]
209
+ @children = Children.load(restart_state[:children])
210
+ @children.close_on_exec = true
211
+ end
212
+
169
213
  # setup signal handlers before writing pid file in case people get
170
214
  # trigger happy and send signals as soon as the pid file exists.
171
215
  # Note that signals don't actually get handled until the #join method
172
216
  @queue_sigs.each { |sig| trap(sig) { @sig_queue << sig; awaken_monitor } }
173
217
  trap(:CHLD) { awaken_monitor }
174
218
 
219
+ if restart_state[:listeners]
220
+ @inherited_listeners = Hash.new { |h, k| h[k] = [] }
221
+ Listeners.load(restart_state[:listeners]).each do |socket|
222
+ @inherited_listeners[sock_name(socket)] << socket
223
+ end
224
+ end
225
+ bind_listeners!
226
+
175
227
  if REFORKING_AVAILABLE
176
228
  spawn_initial_mold
177
229
  wait_for_pending_workers
@@ -180,8 +232,8 @@ module Pitchfork
180
232
  end
181
233
  else
182
234
  build_app!
183
- bind_listeners!
184
- after_mold_fork.call(self, Worker.new(nil, pid: $$).promoted!(@spawn_timeout))
235
+ SharedMemory.current_version = @version
236
+ after_mold_fork&.call(self, Worker.new(nil, pid: $$).promoted!(@spawn_timeout))
185
237
  end
186
238
 
187
239
  if sync
@@ -197,25 +249,6 @@ module Pitchfork
197
249
  self
198
250
  end
199
251
 
200
- # replaces current listener set with +listeners+. This will
201
- # close the socket if it will not exist in the new listener set
202
- def listeners=(listeners)
203
- unless LISTENERS.empty?
204
- raise "Listeners can only be initialized once"
205
- end
206
-
207
- cur_names, dead_names = [], []
208
- listener_names.each do |name|
209
- if name.start_with?('/')
210
- # mark unlinked sockets as dead so we can rebind them
211
- (File.socket?(name) ? cur_names : dead_names) << name
212
- else
213
- cur_names << name
214
- end
215
- end
216
- listener_names(listeners).each { |addr| listen(addr) }
217
- end
218
-
219
252
  def logger=(obj)
220
253
  Pitchfork::HttpParser::DEFAULTS["rack.logger"] = @logger = obj
221
254
  end
@@ -240,7 +273,7 @@ module Pitchfork
240
273
  opt[:reuseport] = true if queues > 1
241
274
 
242
275
  begin
243
- io = bind_listen(address, opt)
276
+ io = bind_listen(address, opt, @inherited_listeners)
244
277
  unless TCPServer === io || UNIXServer === io
245
278
  io.autoclose = false
246
279
  io = server_cast(io)
@@ -266,7 +299,7 @@ module Pitchfork
266
299
  ios = [io]
267
300
 
268
301
  (queues - 1).times do
269
- io = bind_listen(address, opt)
302
+ io = bind_listen(address, opt, @inherited_listeners)
270
303
  unless TCPServer === io || UNIXServer === io
271
304
  io.autoclose = false
272
305
  io = server_cast(io)
@@ -295,7 +328,6 @@ module Pitchfork
295
328
 
296
329
  proc_name role: 'monitor', status: ARGV.join(' ')
297
330
 
298
- logger.info "monitor process ready" # test_exec.rb relies on this message
299
331
  if @ready_pipe
300
332
  begin
301
333
  @ready_pipe.syswrite($$.to_s)
@@ -330,6 +362,13 @@ module Pitchfork
330
362
 
331
363
  case message = @sig_queue.shift
332
364
  when nil
365
+ if @respawn
366
+ # Spawn missing workers first so that `murder_lazy_workers` can consider their spawn timeout
367
+ # when computing how long to sleep.
368
+ maintain_worker_count
369
+ restart_outdated_workers
370
+ end
371
+
333
372
  # avoid murdering workers after our monitor process (or the
334
373
  # machine) comes out of suspend/hibernation
335
374
  if (@last_check + @timeout) >= (@last_check = Pitchfork.time_now)
@@ -345,12 +384,15 @@ module Pitchfork
345
384
  sleep_time = @timeout/2.0 + 1
346
385
  @logger.debug("waiting #{sleep_time}s after suspend/hibernation")
347
386
  end
348
- if @respawn
349
- maintain_worker_count
350
- restart_outdated_workers if REFORKING_AVAILABLE
351
- end
352
387
 
353
388
  monitor_sleep(sleep_time) if sleep
389
+ else
390
+ monitor_handle_message(message)
391
+ end
392
+ end
393
+
394
+ def monitor_handle_message(message)
395
+ case message
354
396
  when :QUIT, :TERM # graceful shutdown
355
397
  SharedMemory.shutting_down!
356
398
  logger.info "#{message} received, starting graceful shutdown"
@@ -360,6 +402,8 @@ module Pitchfork
360
402
  logger.info "#{message} received, starting immediate shutdown"
361
403
  stop(false)
362
404
  return StopIteration
405
+ when :USR1 # trigger a hot restart
406
+ trigger_restart
363
407
  when :USR2 # trigger a promotion
364
408
  if @respawn
365
409
  trigger_refork
@@ -371,30 +415,45 @@ module Pitchfork
371
415
  self.worker_processes += 1
372
416
  when :TTOU
373
417
  self.worker_processes -= 1 if self.worker_processes > 0
374
- when Message::WorkerSpawned
375
- worker = @children.update(message)
376
- # TODO: should we send a message to the worker to acknowledge?
377
- logger.info "#{worker.to_log} registered"
378
- when Message::MoldSpawned
379
- new_mold = @children.update(message)
380
- logger.info("#{new_mold.to_log} spawned")
381
- when Message::ServiceSpawned
382
- new_service = @children.update(message)
383
- logger.info("#{new_service.to_log} spawned")
418
+ when Message::ProcessReaped
419
+ unless try_reaping_worker(message.status)
420
+ logger.info("reaped unknown subprocess #{message.status.inspect}")
421
+ end
422
+ when Message::WorkerSpawned, Message::MoldSpawned, Message::ServiceSpawned
423
+ if worker = @children.update(message)
424
+ logger.info("#{worker.to_log} spawned")
425
+ else
426
+ logger.error("worker=#{message.nr} gen=#{message.version}.#{message.generation} pid=#{message.pid} spawned but slot is taken (abandoned worker?), killing")
427
+
428
+ # Send SIGINT for immediate shutdown so that shared state isn't reset on exit.
429
+ MessageSocket.new(message.pipe).try_send_message(Message::SoftKill.new(Signal.list.fetch("INT")))
430
+ message.pipe.close
431
+ end
384
432
  when Message::MoldReady
385
- @consecutive_spawn_errors = 0
386
- old_molds = @children.molds
387
433
  new_mold = @children.update(message)
388
- logger.info("#{new_mold.to_log} ready")
389
- old_molds.each do |old_mold|
390
- logger.info("Terminating old #{old_mold.to_log}")
391
- old_mold.soft_kill(:TERM)
434
+
435
+ # current_mold may not be new_mold if there was a race
436
+ # between refork and restart.
437
+ if new_mold != @children.mold
438
+ logger.info("#{new_mold.to_log} ready but outdated, terminating it")
439
+ new_mold.soft_kill(:TERM)
440
+ else
441
+ @consecutive_spawn_errors = 0
442
+ logger.info("#{new_mold.to_log} ready")
443
+ SharedMemory.current_generation = new_mold.generation
444
+ SharedMemory.current_version = new_mold.version
445
+
446
+ @children.molds.each do |mold|
447
+ if mold != new_mold
448
+ logger.info("Terminating old #{mold.to_log}")
449
+ mold.soft_kill(:TERM)
450
+ end
451
+ end
392
452
  end
393
453
  when Message::WorkerReady, Message::ServiceReady
394
454
  @consecutive_spawn_errors = 0
395
455
  else
396
456
  logger.error("Unexpected message in sig_queue #{message.inspect}")
397
- logger.error(@sig_queue.inspect)
398
457
  end
399
458
  end
400
459
 
@@ -404,7 +463,7 @@ module Pitchfork
404
463
  @respawn = false
405
464
  SharedMemory.shutting_down!
406
465
  wait_for_pending_workers
407
- LISTENERS.each(&:close).clear
466
+ Pitchfork.close_listeners
408
467
 
409
468
  limit = Pitchfork.time_now + timeout
410
469
  until @children.empty? || Pitchfork.time_now > limit
@@ -430,13 +489,7 @@ module Pitchfork
430
489
  logger.info "#{worker.to_log} exiting"
431
490
  proc_name status: "exiting"
432
491
 
433
- if @before_worker_exit
434
- begin
435
- @before_worker_exit.call(self, worker)
436
- rescue => error
437
- Pitchfork.log_error(logger, "before_worker_exit error", error)
438
- end
439
- end
492
+ safe_run_callback(@before_worker_exit, "before_worker_exit", self, worker)
440
493
  Process.exit
441
494
  end
442
495
 
@@ -444,13 +497,7 @@ module Pitchfork
444
497
  logger.info "#{service.to_log} exiting"
445
498
  proc_name status: "exiting"
446
499
 
447
- if @before_service_worker_exit
448
- begin
449
- @before_service_worker_exit.call(self, service)
450
- rescue => error
451
- Pitchfork.log_error(logger, "before_service_worker_exit error", error)
452
- end
453
- end
500
+ safe_run_callback(@before_service_worker_exit, "before_service_worker_exit", self, service)
454
501
  Process.exit
455
502
  end
456
503
 
@@ -506,27 +553,39 @@ module Pitchfork
506
553
  loop do
507
554
  wpid, status = Process.waitpid2(-1, Process::WNOHANG)
508
555
  wpid or return
509
- worker = @children.reap(wpid) and worker.close rescue nil
510
- if worker
511
- unless worker.ready?
512
- @consecutive_spawn_errors += 1
556
+
557
+ unless try_reaping_worker(status)
558
+ # We may not have yet processed the WorkerSpawned message, so let's queue that event.
559
+ message = Message::ProcessReaped.new(status)
560
+ case @control_socket[1].sendmsg_nonblock(message, exception: false)
561
+ when :wait_writable
562
+ @sig_queue << message # fallback
513
563
  end
514
- @after_worker_exit.call(self, worker, status)
515
- else
516
- logger.info("reaped unknown subprocess #{status.inspect}")
517
564
  end
518
565
  rescue Errno::ECHILD
519
566
  break
520
567
  end
521
568
  end
522
569
 
523
- def listener_sockets
524
- listener_fds = {}
525
- LISTENERS.each do |sock|
526
- sock.close_on_exec = false
527
- listener_fds[sock.fileno] = sock
570
+ def try_reaping_worker(status)
571
+ worker = @children.reap(status.pid) and worker.close rescue nil
572
+ if worker
573
+ reap_worker(worker, status)
574
+ true
575
+ end
576
+ end
577
+
578
+ def reap_worker(worker, status)
579
+ unless worker.ready?
580
+ @consecutive_spawn_errors += 1
581
+ end
582
+ worker.reaped!
583
+
584
+ run_callback!(@after_worker_exit, "after_worker_exit", self, worker, status)
585
+
586
+ if worker.mold? && @children.mold != worker && worker.version > (@children.mold&.version || 0)
587
+ logger.error("#{worker.to_log} crashed before being ready, restart cancelled")
528
588
  end
529
- listener_fds
530
589
  end
531
590
 
532
591
  # forcibly terminate all workers that haven't checked in in timeout seconds. The timeout is implemented using an unlinked File
@@ -555,25 +614,66 @@ module Pitchfork
555
614
  end
556
615
 
557
616
  def hard_timeout(child)
617
+ @consecutive_spawn_errors += 1 unless child.ready?
618
+
558
619
  if child.pid.nil? # Not yet registered, likely never spawned
559
- logger.error "worker=#{child.nr} timed out during spawn, abandoning"
560
- @children.abandon(worker)
620
+ logger.error "#{child.to_log} timed out during spawn, abandoning"
621
+ @children.abandon(child)
561
622
  return
562
623
  end
563
624
 
564
- if @after_worker_hard_timeout && !child.mold?
565
- begin
566
- @after_worker_hard_timeout.call(self, child)
567
- rescue => error
568
- Pitchfork.log_error(@logger, "after_worker_hard_timeout callback", error)
569
- end
625
+ unless child.mold?
626
+ safe_run_callback(@after_worker_hard_timeout, "after_worker_hard_timeout", self, child)
570
627
  end
571
628
 
572
629
  logger.error "#{child.to_log} timed out, killing"
573
- @consecutive_spawn_errors += 1 unless child.ready?
574
630
  @children.hard_kill(@timeout_signal.call(child.pid), child) # take no prisoners for hard timeout violations
575
631
  end
576
632
 
633
+ def trigger_restart
634
+ # We must handle all the Message instances in the sig queue, as they could contain sockets or
635
+ # other FD we can't marshal down to the next process.
636
+ # It is OK to handle them out of order as they don't have any dependency with regular signals.
637
+ # Signals will be handled by the new monitor.
638
+ @sig_queue.reject! do |message|
639
+ if message.is_a?(Message)
640
+ monitor_handle_message(message)
641
+ true
642
+ else
643
+ false
644
+ end
645
+ end
646
+
647
+ with_unbundled_env do
648
+ SharedMemory.close_on_exec = false
649
+ @control_socket.map {|s| s.close_on_exec = false }
650
+ @children.close_on_exec = false
651
+ LISTENERS.close_on_exec = false
652
+
653
+ state = {
654
+ shared_memory_fds: SharedMemory.fds,
655
+ control_socket_fds: @control_socket.map(&:fileno),
656
+ children: @children.dump,
657
+ listeners: LISTENERS.dump,
658
+ version: @version + 1,
659
+ sig_queue: @sig_queue,
660
+ }
661
+ env = { "PITCHFORK_RESTART_STATE" => [Marshal.dump(state)].pack("m0") }
662
+ chdir = File.realpath(working_directory)
663
+ cmd = [*restart_command, *restart_command_argv]
664
+ logger.info "monitor v=#{@version} reexecuting pwd=#{chdir} cmd=#{cmd.inspect}"
665
+ Process.exec(env, *cmd, chdir: chdir)
666
+ end
667
+ end
668
+
669
+ def with_unbundled_env(&block)
670
+ if defined?(Bundler)
671
+ Bundler.with_unbundled_env(&block)
672
+ else
673
+ yield
674
+ end
675
+ end
676
+
577
677
  def trigger_refork
578
678
  unless REFORKING_AVAILABLE
579
679
  logger.error("This system doesn't support PR_SET_CHILD_SUBREAPER, can't refork")
@@ -593,14 +693,41 @@ module Pitchfork
593
693
  @control_socket[0].close_write # this is monitor-only, now
594
694
  @ready_pipe.close if @ready_pipe
595
695
  Pitchfork::Configurator::RACKUP.clear
596
- @ready_pipe = @init_listeners = nil
696
+ @ready_pipe = nil
597
697
 
598
698
  # The OpenSSL PRNG is seeded with only the pid, and apps with frequently
599
699
  # dying workers can recycle pids
600
700
  OpenSSL::Random.seed(rand.to_s) if defined?(OpenSSL::Random)
601
701
  end
602
702
 
603
- def spawn_worker(worker, detach:)
703
+ def spawn_workers_from_mold(workers)
704
+ if REFORKING_AVAILABLE
705
+ if mold = @children.mold
706
+ # On larger app, the fork call necessary to spawn a worker can take hundreds of milliseconds
707
+ # so we're giving some extra spawn timeout when spawning more than a single worker.
708
+ workers.each_with_index do |worker, i|
709
+ worker.update_deadline(@spawn_timeout + (i * 0.25))
710
+ end
711
+
712
+ if mold.spawn_workers(workers)
713
+ workers.each { |worker| @children.register(worker) }
714
+ else
715
+ @logger.error("Failed to send a spawn_workers command to #{mold.to_log}")
716
+ workers.each(&:reaped!)
717
+ end
718
+ else
719
+ @logger.error("Failed to send a spawn_workers command: no active mold")
720
+ end
721
+ else
722
+ workers.each do |worker|
723
+ worker.update_deadline(@spawn_timeout)
724
+ spawn_worker(worker)
725
+ @children.register(worker)
726
+ end
727
+ end
728
+ end
729
+
730
+ def spawn_worker(worker)
604
731
  logger.info("#{worker.to_log} spawning...")
605
732
 
606
733
  # We set the deadline before spawning the child so that if for some
@@ -626,14 +753,7 @@ module Pitchfork
626
753
 
627
754
  ready = readers.dup
628
755
 
629
- if @before_service_worker_ready
630
- begin
631
- @before_service_worker_ready.call(self, service)
632
- rescue => error
633
- Pitchfork.log_error(logger, "before_service_worker_ready", error)
634
- Process.exit(1)
635
- end
636
- end
756
+ run_callback!(@before_service_worker_ready, "before_service_worker_ready", self, service)
637
757
 
638
758
  service.notify_ready(@control_socket[1])
639
759
  proc_name status: "ready"
@@ -665,7 +785,23 @@ module Pitchfork
665
785
  end
666
786
  end
667
787
 
668
- def spawn_service(service, detach:)
788
+ def spawn_service_from_mold(service)
789
+ service.update_deadline(@spawn_timeout)
790
+
791
+ if REFORKING_AVAILABLE
792
+ if @children.mold&.spawn_service(service)
793
+ @children.register_service(service)
794
+ else
795
+ @logger.error("Failed to send a spawn_service command")
796
+ service.reaped!
797
+ end
798
+ else
799
+ spawn_service(service)
800
+ @children.register_service(service)
801
+ end
802
+ end
803
+
804
+ def spawn_service(service)
669
805
  logger.info("#{service.to_log} spawning...")
670
806
 
671
807
  # We set the deadline before spawning the child so that if for some
@@ -685,14 +821,15 @@ module Pitchfork
685
821
  end
686
822
 
687
823
  def spawn_initial_mold
688
- mold = Worker.new(nil)
824
+ mold = Worker.new(nil, version: @version)
689
825
  mold.create_socketpair!
690
826
  mold.pid = Pitchfork.clean_fork(setpgid: setpgid) do
827
+ proc_name role: "(#{mold.version}.#{mold.generation}) mold", status: "init"
691
828
  mold.pid = Process.pid
692
829
  @promotion_lock.try_lock
693
830
  mold.after_fork_in_child
694
831
  build_app!
695
- bind_listeners!
832
+ mold.start_promotion(@control_socket[1])
696
833
  mold_loop(mold)
697
834
  end
698
835
  @promotion_lock.at_fork
@@ -700,42 +837,35 @@ module Pitchfork
700
837
  end
701
838
 
702
839
  def spawn_missing_workers
703
- if @before_service_worker_ready && !@children.service
704
- service = Pitchfork::Service.new
705
- if REFORKING_AVAILABLE
706
- service.generation = @children.mold&.generation || 0
840
+ if REFORKING_AVAILABLE
841
+ return unless @children.mold
707
842
 
708
- unless @children.mold&.spawn_service(service)
709
- @logger.error("Failed to send a spawn_service command")
710
- end
711
- else
712
- spawn_service(service, detach: false)
713
- end
843
+ version = @children.mold&.version || 0
844
+ generation = @children.mold&.generation || 0
845
+ else
846
+ version = @version
847
+ generation = 0
848
+ end
714
849
 
715
- @children.register_service(service)
850
+ if @before_service_worker_ready && !@children.service
851
+ service = Worker.new(nil, service: true, version: version, generation: generation)
852
+ spawn_service_from_mold(service)
716
853
  end
717
854
 
718
855
  worker_nr = -1
856
+ missing_workers = []
857
+
858
+
719
859
  until (worker_nr += 1) == @worker_processes
720
860
  if @children.nr_alive?(worker_nr)
721
861
  next
722
862
  end
723
- worker = Pitchfork::Worker.new(worker_nr)
724
863
 
725
- if REFORKING_AVAILABLE
726
- worker.generation = @children.mold&.generation || 0
864
+ missing_workers << Worker.new(worker_nr, version: version, generation: generation)
865
+ end
727
866
 
728
- unless @children.mold&.spawn_worker(worker)
729
- @logger.error("Failed to send a spawn_worker command")
730
- end
731
- else
732
- spawn_worker(worker, detach: false)
733
- end
734
- # We could directly register workers when we spawn from the
735
- # monitor, like pitchfork does. However it is preferable to
736
- # always go through the asynchronous registering process for
737
- # consistency.
738
- @children.register(worker)
867
+ unless missing_workers.empty?
868
+ spawn_workers_from_mold(missing_workers)
739
869
  end
740
870
  rescue => e
741
871
  @logger.error(e) rescue nil
@@ -753,14 +883,23 @@ module Pitchfork
753
883
 
754
884
  def maintain_worker_count
755
885
  off = @children.workers_count - worker_processes
756
- off -= 1 if @before_service_worker_ready && !@children.service
757
886
 
758
887
  if off < 0
759
888
  spawn_missing_workers
760
- elsif off > 0
761
- @children.each_worker do |worker|
762
- if worker.nr >= worker_processes
763
- worker.soft_kill(:TERM)
889
+ else
890
+ if @before_service_worker_ready && !@children.service
891
+ spawn_missing_workers
892
+ end
893
+
894
+ if off > 0
895
+ @children.each_worker do |worker|
896
+ if worker.nr >= worker_processes
897
+ if worker.soft_kill(:TERM)
898
+ logger.info("Sent SIGTERM to #{worker.to_log}")
899
+ else
900
+ logger.info("Failed to send SIGTERM to #{worker.to_log}")
901
+ end
902
+ end
764
903
  end
765
904
  end
766
905
  end
@@ -768,7 +907,7 @@ module Pitchfork
768
907
 
769
908
  def restart_outdated_workers
770
909
  # If we're already in the middle of forking a new generation, we just continue
771
- return unless @children.mold
910
+ return if REFORKING_AVAILABLE && !@children.mold
772
911
 
773
912
  # We don't shutdown any outdated worker if any worker is already being
774
913
  # spawned or a worker is exiting. Only 10% of workers can be reforked at
@@ -786,7 +925,16 @@ module Pitchfork
786
925
  end
787
926
 
788
927
  if workers_to_restart > 0
789
- outdated_workers = @children.workers.select { |w| !w.exiting? && w.generation < @children.mold.generation }
928
+ outdated_workers = @children.workers.select { |w| !w.exiting? && w.outdated? }
929
+
930
+ # Restart the most outdated workers first, can happen if a
931
+ # restart or refork was triggered before the previous one was
932
+ # done fully rolling out.
933
+ # NB: Array#sort_by isn't guaranteed stable so we use the original order as the
934
+ # last ordering criteria.
935
+ indexes = outdated_workers.each_with_index.to_h
936
+ outdated_workers = outdated_workers.sort_by { |w| [w.version, w.generation, indexes[w]] }
937
+
790
938
  outdated_workers.each do |worker|
791
939
  if worker.soft_kill(:TERM)
792
940
  logger.info("Sent SIGTERM to #{worker.to_log}")
@@ -842,6 +990,7 @@ module Pitchfork
842
990
  # once a client is accepted, it is processed in its entirety here
843
991
  # in 3 easy steps: read request, call app, write app response
844
992
  def process_client(client, worker, timeout_handler)
993
+ accepted_at = Process.clock_gettime(Process::CLOCK_REALTIME)
845
994
  response_written = false
846
995
  env = nil
847
996
  @request = Pitchfork::HttpParser.new
@@ -854,6 +1003,7 @@ module Pitchfork
854
1003
  proc_name status: status
855
1004
 
856
1005
  env["pitchfork.worker"] = worker
1006
+ env["pitchfork.accepted_at"] = accepted_at
857
1007
  timeout_handler.rack_env = env
858
1008
  env["pitchfork.timeout"] = timeout_handler
859
1009
 
@@ -924,7 +1074,7 @@ module Pitchfork
924
1074
  # traps for USR2, and HUP may be set in the after_worker_fork/after_mold_fork Procs
925
1075
  # by the user.
926
1076
  def init_worker_process(worker)
927
- proc_name role: "(gen:#{worker.generation}) worker[#{worker.nr}]", status: "init"
1077
+ proc_name role: "(#{worker.version}.#{worker.generation}) worker[#{worker.nr}]", status: "init"
928
1078
  worker.reset
929
1079
  worker.register_to_monitor(@control_socket[1])
930
1080
  # we'll re-trap :QUIT and :TERM later for graceful shutdown iff we accept clients
@@ -936,7 +1086,7 @@ module Pitchfork
936
1086
  @sig_queue.clear
937
1087
  @children = nil
938
1088
 
939
- after_worker_fork.call(self, worker) # can drop perms and create listeners
1089
+ run_callback!(after_worker_fork, "after_worker_fork", self, worker) # can drop perms and create listeners
940
1090
  LISTENERS.each { |sock| sock.close_on_exec = true }
941
1091
 
942
1092
  @config = nil
@@ -950,8 +1100,8 @@ module Pitchfork
950
1100
  end
951
1101
 
952
1102
  def init_service_process(service)
953
- proc_name role: "(gen:#{service.generation}) service", status: "init"
954
- LISTENERS.each(&:close).clear # Don't appear as listening to incoming requests
1103
+ proc_name role: "(#{service.version}.#{service.generation}) service", status: "init"
1104
+ Pitchfork.close_listeners # Don't appear as listening to incoming requests
955
1105
  service.register_to_monitor(@control_socket[1])
956
1106
  readers = [service]
957
1107
  trap(:QUIT) { nuke_listeners!(readers) }
@@ -961,8 +1111,8 @@ module Pitchfork
961
1111
  end
962
1112
 
963
1113
  def init_mold_process(mold)
964
- proc_name role: "(gen:#{mold.generation}) mold", status: "init"
965
- after_mold_fork.call(self, mold)
1114
+ proc_name role: "(#{mold.version}.#{mold.generation}) mold", status: "init"
1115
+ run_callback!(after_mold_fork, "after_mold_fork", self, mold)
966
1116
  readers = [mold]
967
1117
  trap(:QUIT) { nuke_listeners!(readers) }
968
1118
  trap(:TERM) { nuke_listeners!(readers) }
@@ -972,10 +1122,16 @@ module Pitchfork
972
1122
 
973
1123
  if Pitchfork.const_defined?(:Waiter)
974
1124
  def prep_readers(readers)
975
- Pitchfork::Info.keep_io(Pitchfork::Waiter.prep_readers(readers))
1125
+ case @poller
1126
+ when :epoll_exclusive
1127
+ Info.keep_io(Waiter.prep_readers(readers, true))
1128
+ when :epoll
1129
+ Info.keep_io(Waiter.prep_readers(readers, false))
1130
+ else
1131
+ Pitchfork::SelectWaiter.new
1132
+ end
976
1133
  end
977
1134
  else
978
- require_relative 'select_waiter'
979
1135
  def prep_readers(_readers)
980
1136
  Pitchfork::SelectWaiter.new
981
1137
  end
@@ -989,7 +1145,7 @@ module Pitchfork
989
1145
  waiter = prep_readers(readers)
990
1146
 
991
1147
  ready = readers.dup
992
- @after_worker_ready.call(self, worker)
1148
+ run_callback!(@after_worker_ready, "after_worker_ready", self, worker)
993
1149
 
994
1150
  worker.notify_ready(@control_socket[1])
995
1151
  proc_name status: "ready"
@@ -1014,12 +1170,22 @@ module Pitchfork
1014
1170
  when Message
1015
1171
  worker.update(client)
1016
1172
  else
1017
- if (request_env = process_client(client, worker, prepare_timeout(worker)))
1173
+ worker.busy = true
1174
+ timeout_handler = prepare_timeout(worker)
1175
+ request_env = process_client(client, worker, timeout_handler)
1176
+ if timeout_handler.timed_out?
1177
+ # before_worker_exit already ran in the timeout thread.
1178
+ logger.error("#{worker.to_log} SystemExit was swallowed by the application, exiting")
1179
+ Process.exit
1180
+ end
1181
+
1182
+ if request_env
1018
1183
  worker.increment_requests_count
1019
- @after_request_complete&.call(self, worker, request_env)
1184
+ safe_run_callback(@after_request_complete, "after_request_complete", self, worker, request_env)
1020
1185
  end
1021
1186
  end
1022
1187
 
1188
+ worker.busy = false
1023
1189
  worker.update_deadline(@timeout)
1024
1190
  end
1025
1191
 
@@ -1033,6 +1199,8 @@ module Pitchfork
1033
1199
  logger.info("#{worker.to_log} refork condition met, promoting ourselves")
1034
1200
  end
1035
1201
  @refork_condition.backoff!
1202
+ # spawn_mold may have shortened our deadline to spawn_timeout, restore it before waiting for requests
1203
+ worker.update_deadline(@timeout)
1036
1204
  end
1037
1205
  end
1038
1206
 
@@ -1053,7 +1221,7 @@ module Pitchfork
1053
1221
 
1054
1222
  begin
1055
1223
  fork_sibling("spawn_mold") do
1056
- mold = Worker.new(nil, pid: Process.pid, generation: worker.generation)
1224
+ mold = Worker.new(nil, pid: Process.pid, generation: worker.generation, version: worker.version)
1057
1225
  mold.promote!(@spawn_timeout)
1058
1226
  mold.start_promotion(@control_socket[1])
1059
1227
  mold_loop(mold)
@@ -1085,45 +1253,7 @@ module Pitchfork
1085
1253
  while sock = ready.shift
1086
1254
  # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
1087
1255
  # but that will return false
1088
- message = sock.accept_nonblock(exception: false)
1089
- case message
1090
- when false
1091
- # no message, keep looping
1092
- when Message::SpawnWorker
1093
- retries = 1
1094
- begin
1095
- spawn_worker(Worker.new(message.nr, generation: mold.generation), detach: true)
1096
- rescue ForkFailure
1097
- if retries > 0
1098
- @logger.fatal("#{mold.to_log} failed to spawn a worker, retrying")
1099
- retries -= 1
1100
- retry
1101
- else
1102
- @logger.fatal("#{mold.to_log} failed to spawn a worker twice in a row - corrupted mold process?")
1103
- Process.exit(1)
1104
- end
1105
- rescue => error
1106
- raise BootFailure, error.message
1107
- end
1108
- when Message::SpawnService
1109
- retries = 1
1110
- begin
1111
- spawn_service(Service.new(generation: mold.generation), detach: true)
1112
- rescue ForkFailure
1113
- if retries > 0
1114
- @logger.fatal("#{mold.to_log} failed to spawn a service, retrying")
1115
- retries -= 1
1116
- retry
1117
- else
1118
- @logger.fatal("#{mold.to_log} failed to spawn a service twice in a row - corrupted mold process?")
1119
- Process.exit(1)
1120
- end
1121
- rescue => error
1122
- raise BootFailure, error.message
1123
- end
1124
- else
1125
- logger.error("Unexpected mold message #{message.inspect}")
1126
- end
1256
+ mold_handle_message(mold, sock.accept_nonblock(exception: false))
1127
1257
  end
1128
1258
 
1129
1259
  # timeout so we can .tick and keep parent from SIGKILL-ing us
@@ -1135,6 +1265,51 @@ module Pitchfork
1135
1265
  end
1136
1266
  end
1137
1267
 
1268
+ def mold_handle_message(mold, message)
1269
+ case message
1270
+ when false
1271
+ # no message, keep looping
1272
+ when Message::SpawnWorkers
1273
+ message.nrs.each do |nr|
1274
+ retries = 1
1275
+ mold.update_deadline(@timeout)
1276
+ worker = Worker.new(nr, generation: mold.generation, version: mold.version)
1277
+ begin
1278
+ spawn_worker(worker)
1279
+ rescue ForkFailure
1280
+ if retries > 0
1281
+ @logger.fatal("#{mold.to_log} failed to spawn #{worker.to_log}, retrying")
1282
+ retries -= 1
1283
+ retry
1284
+ else
1285
+ @logger.fatal("#{mold.to_log} failed to spawn #{worker.to_log} twice in a row - corrupted mold process?")
1286
+ Process.exit(1)
1287
+ end
1288
+ rescue => error
1289
+ raise BootFailure, error.message
1290
+ end
1291
+ end
1292
+ when Message::SpawnService
1293
+ retries = 1
1294
+ begin
1295
+ spawn_service(Pitchfork::Worker.new(nil, generation: mold.generation, version: mold.version, service: true))
1296
+ rescue ForkFailure
1297
+ if retries > 0
1298
+ @logger.fatal("#{mold.to_log} failed to spawn a service, retrying")
1299
+ retries -= 1
1300
+ retry
1301
+ else
1302
+ @logger.fatal("#{mold.to_log} failed to spawn a service twice in a row - corrupted mold process?")
1303
+ Process.exit(1)
1304
+ end
1305
+ rescue => error
1306
+ raise BootFailure, error.message
1307
+ end
1308
+ else
1309
+ logger.error("Unexpected mold message #{message.inspect}")
1310
+ end
1311
+ end
1312
+
1138
1313
  # delivers a signal to a worker and fails gracefully if the worker
1139
1314
  # is no longer running.
1140
1315
  def kill_worker(signal, wpid)
@@ -1143,6 +1318,20 @@ module Pitchfork
1143
1318
  worker = @children.reap(wpid) and worker.close rescue nil
1144
1319
  end
1145
1320
 
1321
+ def safe_run_callback(callback, name, *args)
1322
+ callback&.call(*args)
1323
+ true
1324
+ rescue => error
1325
+ Pitchfork.log_error(logger, "#{name} error", error)
1326
+ false
1327
+ end
1328
+
1329
+ def run_callback!(callback, name, *args)
1330
+ unless safe_run_callback(callback, name, *args)
1331
+ Process.exit(1)
1332
+ end
1333
+ end
1334
+
1146
1335
  # returns an array of string names for the given listener array
1147
1336
  def listener_names(listeners = LISTENERS)
1148
1337
  listeners.map { |io| sock_name(io) }
@@ -1174,9 +1363,15 @@ module Pitchfork
1174
1363
  listeners = config[:listeners].dup
1175
1364
  if listeners.empty?
1176
1365
  listeners << Pitchfork::Const::DEFAULT_LISTEN
1177
- @init_listeners << Pitchfork::Const::DEFAULT_LISTEN
1178
1366
  end
1179
1367
  listeners.each { |addr| listen(addr) }
1368
+
1369
+ if @inherited_listeners
1370
+ @inherited_listeners.each_value { |sockets| sockets.each(&:close) }
1371
+ @inherited_listeners.clear
1372
+ @inherited_listeners = nil
1373
+ end
1374
+
1180
1375
  raise ArgumentError, "no listeners" if LISTENERS.empty?
1181
1376
  end
1182
1377