pitchfork 0.18.2 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/actions.lock +40 -0
- data/.github/workflows/ci.yml +27 -10
- data/.github/workflows/release.yml +3 -3
- data/.ruby-version +1 -1
- data/CHANGELOG.md +9 -1
- data/Gemfile +1 -0
- data/Rakefile +15 -0
- data/benchmark/request_benchmark.rb +86 -0
- data/docs/CONFIGURATION.md +51 -14
- data/docs/MIGRATING_FROM_UNICORN.md +6 -0
- data/docs/SIGNALS.md +6 -0
- data/examples/pitchfork.conf.reload.rb +41 -0
- data/exe/pitchfork +3 -2
- data/ext/pitchfork_http/c_util.h +3 -8
- data/ext/pitchfork_http/child_subreaper.h +7 -1
- data/ext/pitchfork_http/epollexclusive.h +6 -3
- data/ext/pitchfork_http/extconf.rb +1 -0
- data/ext/pitchfork_http/global_variables.h +2 -0
- data/ext/pitchfork_http/memory_page.c +188 -25
- data/ext/pitchfork_http/pitchfork_http.c +425 -394
- data/ext/pitchfork_http/pitchfork_http.rl +54 -23
- data/ext/pitchfork_http/pitchfork_http_common.rl +1 -1
- data/lib/pitchfork/children.rb +74 -20
- data/lib/pitchfork/chunked.rb +9 -0
- data/lib/pitchfork/configurator.rb +25 -6
- data/lib/pitchfork/http_response.rb +35 -12
- data/lib/pitchfork/http_server.rb +384 -189
- data/lib/pitchfork/info.rb +8 -0
- data/lib/pitchfork/listeners.rb +52 -0
- data/lib/pitchfork/message.rb +58 -8
- data/lib/pitchfork/shared_memory.rb +64 -13
- data/lib/pitchfork/socket_helper.rb +6 -2
- data/lib/pitchfork/stream_input.rb +12 -9
- data/lib/pitchfork/version.rb +1 -1
- data/lib/pitchfork/worker.rb +124 -64
- data/lib/pitchfork.rb +13 -1
- metadata +5 -2
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require 'pitchfork/pitchfork_http'
|
|
5
5
|
require 'pitchfork/listeners'
|
|
6
|
+
require 'pitchfork/select_waiter'
|
|
6
7
|
require 'pitchfork/flock'
|
|
7
8
|
require 'pitchfork/soft_timeout'
|
|
8
9
|
require 'pitchfork/shared_memory'
|
|
@@ -42,13 +43,19 @@ module Pitchfork
|
|
|
42
43
|
@callback = callback
|
|
43
44
|
@rack_env = nil
|
|
44
45
|
@timeout_request = nil
|
|
46
|
+
@timed_out = false
|
|
45
47
|
end
|
|
46
48
|
|
|
47
49
|
def inspect
|
|
48
50
|
"#<Pitchfork::HttpServer::TimeoutHandler##{object_id}>"
|
|
49
51
|
end
|
|
50
52
|
|
|
53
|
+
def timed_out? # :nodoc:
|
|
54
|
+
@timed_out
|
|
55
|
+
end
|
|
56
|
+
|
|
51
57
|
def call(original_thread) # :nodoc:
|
|
58
|
+
@timed_out = true
|
|
52
59
|
begin
|
|
53
60
|
@server.logger.error("#{@worker.to_log} timed out, exiting")
|
|
54
61
|
if @callback
|
|
@@ -79,13 +86,13 @@ module Pitchfork
|
|
|
79
86
|
# :stopdoc:
|
|
80
87
|
attr_accessor :app, :timeout, :timeout_signal, :soft_timeout, :cleanup_timeout, :spawn_timeout, :worker_processes,
|
|
81
88
|
:before_fork, :after_worker_fork, :after_mold_fork, :before_service_worker_ready, :before_service_worker_exit,
|
|
82
|
-
:listener_opts, :children,
|
|
83
|
-
:orig_app, :config, :ready_pipe, :early_hints, :setpgid
|
|
89
|
+
:listener_opts, :children, :poller,
|
|
90
|
+
:orig_app, :config, :ready_pipe, :early_hints, :setpgid, :restart_command, :restart_command_argv, :working_directory
|
|
84
91
|
attr_writer :after_worker_exit, :before_worker_exit, :after_worker_ready, :after_request_complete,
|
|
85
92
|
:refork_condition, :after_worker_timeout, :after_worker_hard_timeout, :after_monitor_ready, :refork_max_unavailable,
|
|
86
93
|
:max_consecutive_spawn_errors
|
|
87
94
|
|
|
88
|
-
attr_reader :logger
|
|
95
|
+
attr_reader :logger, :version
|
|
89
96
|
include Pitchfork::SocketHelper
|
|
90
97
|
include Pitchfork::HttpResponse
|
|
91
98
|
|
|
@@ -101,6 +108,7 @@ module Pitchfork
|
|
|
101
108
|
# HttpServer.run.join to join the thread that's processing
|
|
102
109
|
# incoming requests on the socket.
|
|
103
110
|
def initialize(app, options = {})
|
|
111
|
+
@version = 0
|
|
104
112
|
@exit_status = 0
|
|
105
113
|
@app = app
|
|
106
114
|
@respawn = false
|
|
@@ -113,7 +121,6 @@ module Pitchfork
|
|
|
113
121
|
|
|
114
122
|
options = options.dup
|
|
115
123
|
@ready_pipe = options.delete(:ready_pipe)
|
|
116
|
-
@init_listeners = options[:listeners].dup || []
|
|
117
124
|
options[:use_defaults] = true
|
|
118
125
|
self.config = Pitchfork::Configurator.new(options)
|
|
119
126
|
self.listener_opts = {}
|
|
@@ -147,31 +154,76 @@ module Pitchfork
|
|
|
147
154
|
config.commit!(self, :skip => [:listeners, :pid])
|
|
148
155
|
@orig_app = app
|
|
149
156
|
# list of signals we care about and trap in monitor.
|
|
150
|
-
@queue_sigs = [
|
|
151
|
-
:QUIT, :INT, :TERM, :USR2, :TTIN, :TTOU ]
|
|
152
|
-
|
|
153
|
-
Info.workers_count = worker_processes
|
|
154
|
-
SharedMemory.preallocate_pages(worker_processes)
|
|
157
|
+
@queue_sigs = [ :QUIT, :INT, :TERM, :USR1, :USR2, :TTIN, :TTOU ]
|
|
155
158
|
end
|
|
156
159
|
|
|
157
160
|
# Runs the thing. Returns self so you can run join on it
|
|
158
161
|
def start(sync = true)
|
|
159
162
|
Pitchfork.enable_child_subreaper # noop if not supported
|
|
160
163
|
|
|
164
|
+
restart_state = ENV.delete("PITCHFORK_RESTART_STATE")
|
|
165
|
+
if restart_state
|
|
166
|
+
restart_state = Marshal.load(restart_state.unpack1("m0"))
|
|
167
|
+
logger.info "monitor v=#{restart_state[:version]} initializing with inherited state"
|
|
168
|
+
else
|
|
169
|
+
restart_state = {}
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
if restart_state[:version]
|
|
173
|
+
@version = restart_state[:version]
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
if working_directory
|
|
177
|
+
real_pwd = File.realpath(Dir.pwd)
|
|
178
|
+
real_working_directory = File.realpath(working_directory)
|
|
179
|
+
|
|
180
|
+
if real_pwd != real_working_directory
|
|
181
|
+
logger.info("monitor v=#{@version} chdir #{File.realpath(working_directory)}")
|
|
182
|
+
Dir.chdir(real_working_directory)
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
if restart_state[:sig_queue]
|
|
187
|
+
@sig_queue = [*restart_state[:sig_queue], *@sig_queue]
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
if restart_state[:shared_memory_fds]
|
|
191
|
+
SharedMemory.reopen(restart_state[:shared_memory_fds])
|
|
192
|
+
end
|
|
193
|
+
SharedMemory.preallocate_pages(worker_processes)
|
|
194
|
+
Info.workers_count = worker_processes
|
|
195
|
+
|
|
161
196
|
# This socketpair is used to wake us up from select(2) in #join when signals
|
|
162
197
|
# are trapped. See trap_deferred.
|
|
163
198
|
# It's also used by newly spawned children to send their soft_signal pipe
|
|
164
199
|
# to the monitor when they are spawned.
|
|
165
|
-
|
|
200
|
+
if fds = restart_state[:control_socket_fds]
|
|
201
|
+
@control_socket.replace(Pitchfork.socketpair_for_fds(fds))
|
|
202
|
+
else
|
|
203
|
+
@control_socket.replace(Pitchfork.socketpair)
|
|
204
|
+
end
|
|
166
205
|
Info.keep_ios(@control_socket)
|
|
167
206
|
@monitor_pid = $$
|
|
168
207
|
|
|
208
|
+
if restart_state[:children]
|
|
209
|
+
@children = Children.load(restart_state[:children])
|
|
210
|
+
@children.close_on_exec = true
|
|
211
|
+
end
|
|
212
|
+
|
|
169
213
|
# setup signal handlers before writing pid file in case people get
|
|
170
214
|
# trigger happy and send signals as soon as the pid file exists.
|
|
171
215
|
# Note that signals don't actually get handled until the #join method
|
|
172
216
|
@queue_sigs.each { |sig| trap(sig) { @sig_queue << sig; awaken_monitor } }
|
|
173
217
|
trap(:CHLD) { awaken_monitor }
|
|
174
218
|
|
|
219
|
+
if restart_state[:listeners]
|
|
220
|
+
@inherited_listeners = Hash.new { |h, k| h[k] = [] }
|
|
221
|
+
Listeners.load(restart_state[:listeners]).each do |socket|
|
|
222
|
+
@inherited_listeners[sock_name(socket)] << socket
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
bind_listeners!
|
|
226
|
+
|
|
175
227
|
if REFORKING_AVAILABLE
|
|
176
228
|
spawn_initial_mold
|
|
177
229
|
wait_for_pending_workers
|
|
@@ -180,8 +232,8 @@ module Pitchfork
|
|
|
180
232
|
end
|
|
181
233
|
else
|
|
182
234
|
build_app!
|
|
183
|
-
|
|
184
|
-
after_mold_fork
|
|
235
|
+
SharedMemory.current_version = @version
|
|
236
|
+
after_mold_fork&.call(self, Worker.new(nil, pid: $$).promoted!(@spawn_timeout))
|
|
185
237
|
end
|
|
186
238
|
|
|
187
239
|
if sync
|
|
@@ -197,25 +249,6 @@ module Pitchfork
|
|
|
197
249
|
self
|
|
198
250
|
end
|
|
199
251
|
|
|
200
|
-
# replaces current listener set with +listeners+. This will
|
|
201
|
-
# close the socket if it will not exist in the new listener set
|
|
202
|
-
def listeners=(listeners)
|
|
203
|
-
unless LISTENERS.empty?
|
|
204
|
-
raise "Listeners can only be initialized once"
|
|
205
|
-
end
|
|
206
|
-
|
|
207
|
-
cur_names, dead_names = [], []
|
|
208
|
-
listener_names.each do |name|
|
|
209
|
-
if name.start_with?('/')
|
|
210
|
-
# mark unlinked sockets as dead so we can rebind them
|
|
211
|
-
(File.socket?(name) ? cur_names : dead_names) << name
|
|
212
|
-
else
|
|
213
|
-
cur_names << name
|
|
214
|
-
end
|
|
215
|
-
end
|
|
216
|
-
listener_names(listeners).each { |addr| listen(addr) }
|
|
217
|
-
end
|
|
218
|
-
|
|
219
252
|
def logger=(obj)
|
|
220
253
|
Pitchfork::HttpParser::DEFAULTS["rack.logger"] = @logger = obj
|
|
221
254
|
end
|
|
@@ -240,7 +273,7 @@ module Pitchfork
|
|
|
240
273
|
opt[:reuseport] = true if queues > 1
|
|
241
274
|
|
|
242
275
|
begin
|
|
243
|
-
io = bind_listen(address, opt)
|
|
276
|
+
io = bind_listen(address, opt, @inherited_listeners)
|
|
244
277
|
unless TCPServer === io || UNIXServer === io
|
|
245
278
|
io.autoclose = false
|
|
246
279
|
io = server_cast(io)
|
|
@@ -266,7 +299,7 @@ module Pitchfork
|
|
|
266
299
|
ios = [io]
|
|
267
300
|
|
|
268
301
|
(queues - 1).times do
|
|
269
|
-
io = bind_listen(address, opt)
|
|
302
|
+
io = bind_listen(address, opt, @inherited_listeners)
|
|
270
303
|
unless TCPServer === io || UNIXServer === io
|
|
271
304
|
io.autoclose = false
|
|
272
305
|
io = server_cast(io)
|
|
@@ -295,7 +328,6 @@ module Pitchfork
|
|
|
295
328
|
|
|
296
329
|
proc_name role: 'monitor', status: ARGV.join(' ')
|
|
297
330
|
|
|
298
|
-
logger.info "monitor process ready" # test_exec.rb relies on this message
|
|
299
331
|
if @ready_pipe
|
|
300
332
|
begin
|
|
301
333
|
@ready_pipe.syswrite($$.to_s)
|
|
@@ -330,6 +362,13 @@ module Pitchfork
|
|
|
330
362
|
|
|
331
363
|
case message = @sig_queue.shift
|
|
332
364
|
when nil
|
|
365
|
+
if @respawn
|
|
366
|
+
# Spawn missing workers first so that `murder_lazy_workers` can consider their spawn timeout
|
|
367
|
+
# when computing how long to sleep.
|
|
368
|
+
maintain_worker_count
|
|
369
|
+
restart_outdated_workers
|
|
370
|
+
end
|
|
371
|
+
|
|
333
372
|
# avoid murdering workers after our monitor process (or the
|
|
334
373
|
# machine) comes out of suspend/hibernation
|
|
335
374
|
if (@last_check + @timeout) >= (@last_check = Pitchfork.time_now)
|
|
@@ -345,12 +384,15 @@ module Pitchfork
|
|
|
345
384
|
sleep_time = @timeout/2.0 + 1
|
|
346
385
|
@logger.debug("waiting #{sleep_time}s after suspend/hibernation")
|
|
347
386
|
end
|
|
348
|
-
if @respawn
|
|
349
|
-
maintain_worker_count
|
|
350
|
-
restart_outdated_workers if REFORKING_AVAILABLE
|
|
351
|
-
end
|
|
352
387
|
|
|
353
388
|
monitor_sleep(sleep_time) if sleep
|
|
389
|
+
else
|
|
390
|
+
monitor_handle_message(message)
|
|
391
|
+
end
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
def monitor_handle_message(message)
|
|
395
|
+
case message
|
|
354
396
|
when :QUIT, :TERM # graceful shutdown
|
|
355
397
|
SharedMemory.shutting_down!
|
|
356
398
|
logger.info "#{message} received, starting graceful shutdown"
|
|
@@ -360,6 +402,8 @@ module Pitchfork
|
|
|
360
402
|
logger.info "#{message} received, starting immediate shutdown"
|
|
361
403
|
stop(false)
|
|
362
404
|
return StopIteration
|
|
405
|
+
when :USR1 # trigger a hot restart
|
|
406
|
+
trigger_restart
|
|
363
407
|
when :USR2 # trigger a promotion
|
|
364
408
|
if @respawn
|
|
365
409
|
trigger_refork
|
|
@@ -371,30 +415,45 @@ module Pitchfork
|
|
|
371
415
|
self.worker_processes += 1
|
|
372
416
|
when :TTOU
|
|
373
417
|
self.worker_processes -= 1 if self.worker_processes > 0
|
|
374
|
-
when Message::
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
when Message::MoldSpawned
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
418
|
+
when Message::ProcessReaped
|
|
419
|
+
unless try_reaping_worker(message.status)
|
|
420
|
+
logger.info("reaped unknown subprocess #{message.status.inspect}")
|
|
421
|
+
end
|
|
422
|
+
when Message::WorkerSpawned, Message::MoldSpawned, Message::ServiceSpawned
|
|
423
|
+
if worker = @children.update(message)
|
|
424
|
+
logger.info("#{worker.to_log} spawned")
|
|
425
|
+
else
|
|
426
|
+
logger.error("worker=#{message.nr} gen=#{message.version}.#{message.generation} pid=#{message.pid} spawned but slot is taken (abandoned worker?), killing")
|
|
427
|
+
|
|
428
|
+
# Send SIGINT for immediate shutdown so that shared state isn't reset on exit.
|
|
429
|
+
MessageSocket.new(message.pipe).try_send_message(Message::SoftKill.new(Signal.list.fetch("INT")))
|
|
430
|
+
message.pipe.close
|
|
431
|
+
end
|
|
384
432
|
when Message::MoldReady
|
|
385
|
-
@consecutive_spawn_errors = 0
|
|
386
|
-
old_molds = @children.molds
|
|
387
433
|
new_mold = @children.update(message)
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
434
|
+
|
|
435
|
+
# current_mold may not be new_mold if there was a race
|
|
436
|
+
# between refork and restart.
|
|
437
|
+
if new_mold != @children.mold
|
|
438
|
+
logger.info("#{new_mold.to_log} ready but outdated, terminating it")
|
|
439
|
+
new_mold.soft_kill(:TERM)
|
|
440
|
+
else
|
|
441
|
+
@consecutive_spawn_errors = 0
|
|
442
|
+
logger.info("#{new_mold.to_log} ready")
|
|
443
|
+
SharedMemory.current_generation = new_mold.generation
|
|
444
|
+
SharedMemory.current_version = new_mold.version
|
|
445
|
+
|
|
446
|
+
@children.molds.each do |mold|
|
|
447
|
+
if mold != new_mold
|
|
448
|
+
logger.info("Terminating old #{mold.to_log}")
|
|
449
|
+
mold.soft_kill(:TERM)
|
|
450
|
+
end
|
|
451
|
+
end
|
|
392
452
|
end
|
|
393
453
|
when Message::WorkerReady, Message::ServiceReady
|
|
394
454
|
@consecutive_spawn_errors = 0
|
|
395
455
|
else
|
|
396
456
|
logger.error("Unexpected message in sig_queue #{message.inspect}")
|
|
397
|
-
logger.error(@sig_queue.inspect)
|
|
398
457
|
end
|
|
399
458
|
end
|
|
400
459
|
|
|
@@ -404,7 +463,7 @@ module Pitchfork
|
|
|
404
463
|
@respawn = false
|
|
405
464
|
SharedMemory.shutting_down!
|
|
406
465
|
wait_for_pending_workers
|
|
407
|
-
|
|
466
|
+
Pitchfork.close_listeners
|
|
408
467
|
|
|
409
468
|
limit = Pitchfork.time_now + timeout
|
|
410
469
|
until @children.empty? || Pitchfork.time_now > limit
|
|
@@ -430,13 +489,7 @@ module Pitchfork
|
|
|
430
489
|
logger.info "#{worker.to_log} exiting"
|
|
431
490
|
proc_name status: "exiting"
|
|
432
491
|
|
|
433
|
-
|
|
434
|
-
begin
|
|
435
|
-
@before_worker_exit.call(self, worker)
|
|
436
|
-
rescue => error
|
|
437
|
-
Pitchfork.log_error(logger, "before_worker_exit error", error)
|
|
438
|
-
end
|
|
439
|
-
end
|
|
492
|
+
safe_run_callback(@before_worker_exit, "before_worker_exit", self, worker)
|
|
440
493
|
Process.exit
|
|
441
494
|
end
|
|
442
495
|
|
|
@@ -444,13 +497,7 @@ module Pitchfork
|
|
|
444
497
|
logger.info "#{service.to_log} exiting"
|
|
445
498
|
proc_name status: "exiting"
|
|
446
499
|
|
|
447
|
-
|
|
448
|
-
begin
|
|
449
|
-
@before_service_worker_exit.call(self, service)
|
|
450
|
-
rescue => error
|
|
451
|
-
Pitchfork.log_error(logger, "before_service_worker_exit error", error)
|
|
452
|
-
end
|
|
453
|
-
end
|
|
500
|
+
safe_run_callback(@before_service_worker_exit, "before_service_worker_exit", self, service)
|
|
454
501
|
Process.exit
|
|
455
502
|
end
|
|
456
503
|
|
|
@@ -506,27 +553,39 @@ module Pitchfork
|
|
|
506
553
|
loop do
|
|
507
554
|
wpid, status = Process.waitpid2(-1, Process::WNOHANG)
|
|
508
555
|
wpid or return
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
556
|
+
|
|
557
|
+
unless try_reaping_worker(status)
|
|
558
|
+
# We may not have yet processed the WorkerSpawned message, so let's queue that event.
|
|
559
|
+
message = Message::ProcessReaped.new(status)
|
|
560
|
+
case @control_socket[1].sendmsg_nonblock(message, exception: false)
|
|
561
|
+
when :wait_writable
|
|
562
|
+
@sig_queue << message # fallback
|
|
513
563
|
end
|
|
514
|
-
@after_worker_exit.call(self, worker, status)
|
|
515
|
-
else
|
|
516
|
-
logger.info("reaped unknown subprocess #{status.inspect}")
|
|
517
564
|
end
|
|
518
565
|
rescue Errno::ECHILD
|
|
519
566
|
break
|
|
520
567
|
end
|
|
521
568
|
end
|
|
522
569
|
|
|
523
|
-
def
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
570
|
+
def try_reaping_worker(status)
|
|
571
|
+
worker = @children.reap(status.pid) and worker.close rescue nil
|
|
572
|
+
if worker
|
|
573
|
+
reap_worker(worker, status)
|
|
574
|
+
true
|
|
575
|
+
end
|
|
576
|
+
end
|
|
577
|
+
|
|
578
|
+
def reap_worker(worker, status)
|
|
579
|
+
unless worker.ready?
|
|
580
|
+
@consecutive_spawn_errors += 1
|
|
581
|
+
end
|
|
582
|
+
worker.reaped!
|
|
583
|
+
|
|
584
|
+
run_callback!(@after_worker_exit, "after_worker_exit", self, worker, status)
|
|
585
|
+
|
|
586
|
+
if worker.mold? && @children.mold != worker && worker.version > (@children.mold&.version || 0)
|
|
587
|
+
logger.error("#{worker.to_log} crashed before being ready, restart cancelled")
|
|
528
588
|
end
|
|
529
|
-
listener_fds
|
|
530
589
|
end
|
|
531
590
|
|
|
532
591
|
# forcibly terminate all workers that haven't checked in in timeout seconds. The timeout is implemented using an unlinked File
|
|
@@ -555,25 +614,66 @@ module Pitchfork
|
|
|
555
614
|
end
|
|
556
615
|
|
|
557
616
|
def hard_timeout(child)
|
|
617
|
+
@consecutive_spawn_errors += 1 unless child.ready?
|
|
618
|
+
|
|
558
619
|
if child.pid.nil? # Not yet registered, likely never spawned
|
|
559
|
-
logger.error "
|
|
560
|
-
@children.abandon(
|
|
620
|
+
logger.error "#{child.to_log} timed out during spawn, abandoning"
|
|
621
|
+
@children.abandon(child)
|
|
561
622
|
return
|
|
562
623
|
end
|
|
563
624
|
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
@after_worker_hard_timeout.call(self, child)
|
|
567
|
-
rescue => error
|
|
568
|
-
Pitchfork.log_error(@logger, "after_worker_hard_timeout callback", error)
|
|
569
|
-
end
|
|
625
|
+
unless child.mold?
|
|
626
|
+
safe_run_callback(@after_worker_hard_timeout, "after_worker_hard_timeout", self, child)
|
|
570
627
|
end
|
|
571
628
|
|
|
572
629
|
logger.error "#{child.to_log} timed out, killing"
|
|
573
|
-
@consecutive_spawn_errors += 1 unless child.ready?
|
|
574
630
|
@children.hard_kill(@timeout_signal.call(child.pid), child) # take no prisoners for hard timeout violations
|
|
575
631
|
end
|
|
576
632
|
|
|
633
|
+
def trigger_restart
|
|
634
|
+
# We must handle all the Message instances in the sig queue, as they could contain sockets or
|
|
635
|
+
# other FD we can't marshal down to the next process.
|
|
636
|
+
# It is OK to handle them out of order as they don't have any dependency with regular signals.
|
|
637
|
+
# Signals will be handled by the new monitor.
|
|
638
|
+
@sig_queue.reject! do |message|
|
|
639
|
+
if message.is_a?(Message)
|
|
640
|
+
monitor_handle_message(message)
|
|
641
|
+
true
|
|
642
|
+
else
|
|
643
|
+
false
|
|
644
|
+
end
|
|
645
|
+
end
|
|
646
|
+
|
|
647
|
+
with_unbundled_env do
|
|
648
|
+
SharedMemory.close_on_exec = false
|
|
649
|
+
@control_socket.map {|s| s.close_on_exec = false }
|
|
650
|
+
@children.close_on_exec = false
|
|
651
|
+
LISTENERS.close_on_exec = false
|
|
652
|
+
|
|
653
|
+
state = {
|
|
654
|
+
shared_memory_fds: SharedMemory.fds,
|
|
655
|
+
control_socket_fds: @control_socket.map(&:fileno),
|
|
656
|
+
children: @children.dump,
|
|
657
|
+
listeners: LISTENERS.dump,
|
|
658
|
+
version: @version + 1,
|
|
659
|
+
sig_queue: @sig_queue,
|
|
660
|
+
}
|
|
661
|
+
env = { "PITCHFORK_RESTART_STATE" => [Marshal.dump(state)].pack("m0") }
|
|
662
|
+
chdir = File.realpath(working_directory)
|
|
663
|
+
cmd = [*restart_command, *restart_command_argv]
|
|
664
|
+
logger.info "monitor v=#{@version} reexecuting pwd=#{chdir} cmd=#{cmd.inspect}"
|
|
665
|
+
Process.exec(env, *cmd, chdir: chdir)
|
|
666
|
+
end
|
|
667
|
+
end
|
|
668
|
+
|
|
669
|
+
def with_unbundled_env(&block)
|
|
670
|
+
if defined?(Bundler)
|
|
671
|
+
Bundler.with_unbundled_env(&block)
|
|
672
|
+
else
|
|
673
|
+
yield
|
|
674
|
+
end
|
|
675
|
+
end
|
|
676
|
+
|
|
577
677
|
def trigger_refork
|
|
578
678
|
unless REFORKING_AVAILABLE
|
|
579
679
|
logger.error("This system doesn't support PR_SET_CHILD_SUBREAPER, can't refork")
|
|
@@ -593,14 +693,41 @@ module Pitchfork
|
|
|
593
693
|
@control_socket[0].close_write # this is monitor-only, now
|
|
594
694
|
@ready_pipe.close if @ready_pipe
|
|
595
695
|
Pitchfork::Configurator::RACKUP.clear
|
|
596
|
-
@ready_pipe =
|
|
696
|
+
@ready_pipe = nil
|
|
597
697
|
|
|
598
698
|
# The OpenSSL PRNG is seeded with only the pid, and apps with frequently
|
|
599
699
|
# dying workers can recycle pids
|
|
600
700
|
OpenSSL::Random.seed(rand.to_s) if defined?(OpenSSL::Random)
|
|
601
701
|
end
|
|
602
702
|
|
|
603
|
-
def
|
|
703
|
+
def spawn_workers_from_mold(workers)
|
|
704
|
+
if REFORKING_AVAILABLE
|
|
705
|
+
if mold = @children.mold
|
|
706
|
+
# On larger app, the fork call necessary to spawn a worker can take hundreds of milliseconds
|
|
707
|
+
# so we're giving some extra spawn timeout when spawning more than a single worker.
|
|
708
|
+
workers.each_with_index do |worker, i|
|
|
709
|
+
worker.update_deadline(@spawn_timeout + (i * 0.25))
|
|
710
|
+
end
|
|
711
|
+
|
|
712
|
+
if mold.spawn_workers(workers)
|
|
713
|
+
workers.each { |worker| @children.register(worker) }
|
|
714
|
+
else
|
|
715
|
+
@logger.error("Failed to send a spawn_workers command to #{mold.to_log}")
|
|
716
|
+
workers.each(&:reaped!)
|
|
717
|
+
end
|
|
718
|
+
else
|
|
719
|
+
@logger.error("Failed to send a spawn_workers command: no active mold")
|
|
720
|
+
end
|
|
721
|
+
else
|
|
722
|
+
workers.each do |worker|
|
|
723
|
+
worker.update_deadline(@spawn_timeout)
|
|
724
|
+
spawn_worker(worker)
|
|
725
|
+
@children.register(worker)
|
|
726
|
+
end
|
|
727
|
+
end
|
|
728
|
+
end
|
|
729
|
+
|
|
730
|
+
def spawn_worker(worker)
|
|
604
731
|
logger.info("#{worker.to_log} spawning...")
|
|
605
732
|
|
|
606
733
|
# We set the deadline before spawning the child so that if for some
|
|
@@ -626,14 +753,7 @@ module Pitchfork
|
|
|
626
753
|
|
|
627
754
|
ready = readers.dup
|
|
628
755
|
|
|
629
|
-
|
|
630
|
-
begin
|
|
631
|
-
@before_service_worker_ready.call(self, service)
|
|
632
|
-
rescue => error
|
|
633
|
-
Pitchfork.log_error(logger, "before_service_worker_ready", error)
|
|
634
|
-
Process.exit(1)
|
|
635
|
-
end
|
|
636
|
-
end
|
|
756
|
+
run_callback!(@before_service_worker_ready, "before_service_worker_ready", self, service)
|
|
637
757
|
|
|
638
758
|
service.notify_ready(@control_socket[1])
|
|
639
759
|
proc_name status: "ready"
|
|
@@ -665,7 +785,23 @@ module Pitchfork
|
|
|
665
785
|
end
|
|
666
786
|
end
|
|
667
787
|
|
|
668
|
-
def
|
|
788
|
+
def spawn_service_from_mold(service)
|
|
789
|
+
service.update_deadline(@spawn_timeout)
|
|
790
|
+
|
|
791
|
+
if REFORKING_AVAILABLE
|
|
792
|
+
if @children.mold&.spawn_service(service)
|
|
793
|
+
@children.register_service(service)
|
|
794
|
+
else
|
|
795
|
+
@logger.error("Failed to send a spawn_service command")
|
|
796
|
+
service.reaped!
|
|
797
|
+
end
|
|
798
|
+
else
|
|
799
|
+
spawn_service(service)
|
|
800
|
+
@children.register_service(service)
|
|
801
|
+
end
|
|
802
|
+
end
|
|
803
|
+
|
|
804
|
+
def spawn_service(service)
|
|
669
805
|
logger.info("#{service.to_log} spawning...")
|
|
670
806
|
|
|
671
807
|
# We set the deadline before spawning the child so that if for some
|
|
@@ -685,14 +821,15 @@ module Pitchfork
|
|
|
685
821
|
end
|
|
686
822
|
|
|
687
823
|
def spawn_initial_mold
|
|
688
|
-
mold = Worker.new(nil)
|
|
824
|
+
mold = Worker.new(nil, version: @version)
|
|
689
825
|
mold.create_socketpair!
|
|
690
826
|
mold.pid = Pitchfork.clean_fork(setpgid: setpgid) do
|
|
827
|
+
proc_name role: "(#{mold.version}.#{mold.generation}) mold", status: "init"
|
|
691
828
|
mold.pid = Process.pid
|
|
692
829
|
@promotion_lock.try_lock
|
|
693
830
|
mold.after_fork_in_child
|
|
694
831
|
build_app!
|
|
695
|
-
|
|
832
|
+
mold.start_promotion(@control_socket[1])
|
|
696
833
|
mold_loop(mold)
|
|
697
834
|
end
|
|
698
835
|
@promotion_lock.at_fork
|
|
@@ -700,42 +837,35 @@ module Pitchfork
|
|
|
700
837
|
end
|
|
701
838
|
|
|
702
839
|
def spawn_missing_workers
|
|
703
|
-
if
|
|
704
|
-
|
|
705
|
-
if REFORKING_AVAILABLE
|
|
706
|
-
service.generation = @children.mold&.generation || 0
|
|
840
|
+
if REFORKING_AVAILABLE
|
|
841
|
+
return unless @children.mold
|
|
707
842
|
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
843
|
+
version = @children.mold&.version || 0
|
|
844
|
+
generation = @children.mold&.generation || 0
|
|
845
|
+
else
|
|
846
|
+
version = @version
|
|
847
|
+
generation = 0
|
|
848
|
+
end
|
|
714
849
|
|
|
715
|
-
|
|
850
|
+
if @before_service_worker_ready && !@children.service
|
|
851
|
+
service = Worker.new(nil, service: true, version: version, generation: generation)
|
|
852
|
+
spawn_service_from_mold(service)
|
|
716
853
|
end
|
|
717
854
|
|
|
718
855
|
worker_nr = -1
|
|
856
|
+
missing_workers = []
|
|
857
|
+
|
|
858
|
+
|
|
719
859
|
until (worker_nr += 1) == @worker_processes
|
|
720
860
|
if @children.nr_alive?(worker_nr)
|
|
721
861
|
next
|
|
722
862
|
end
|
|
723
|
-
worker = Pitchfork::Worker.new(worker_nr)
|
|
724
863
|
|
|
725
|
-
|
|
726
|
-
|
|
864
|
+
missing_workers << Worker.new(worker_nr, version: version, generation: generation)
|
|
865
|
+
end
|
|
727
866
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
end
|
|
731
|
-
else
|
|
732
|
-
spawn_worker(worker, detach: false)
|
|
733
|
-
end
|
|
734
|
-
# We could directly register workers when we spawn from the
|
|
735
|
-
# monitor, like pitchfork does. However it is preferable to
|
|
736
|
-
# always go through the asynchronous registering process for
|
|
737
|
-
# consistency.
|
|
738
|
-
@children.register(worker)
|
|
867
|
+
unless missing_workers.empty?
|
|
868
|
+
spawn_workers_from_mold(missing_workers)
|
|
739
869
|
end
|
|
740
870
|
rescue => e
|
|
741
871
|
@logger.error(e) rescue nil
|
|
@@ -753,14 +883,23 @@ module Pitchfork
|
|
|
753
883
|
|
|
754
884
|
def maintain_worker_count
|
|
755
885
|
off = @children.workers_count - worker_processes
|
|
756
|
-
off -= 1 if @before_service_worker_ready && !@children.service
|
|
757
886
|
|
|
758
887
|
if off < 0
|
|
759
888
|
spawn_missing_workers
|
|
760
|
-
|
|
761
|
-
@children.
|
|
762
|
-
|
|
763
|
-
|
|
889
|
+
else
|
|
890
|
+
if @before_service_worker_ready && !@children.service
|
|
891
|
+
spawn_missing_workers
|
|
892
|
+
end
|
|
893
|
+
|
|
894
|
+
if off > 0
|
|
895
|
+
@children.each_worker do |worker|
|
|
896
|
+
if worker.nr >= worker_processes
|
|
897
|
+
if worker.soft_kill(:TERM)
|
|
898
|
+
logger.info("Sent SIGTERM to #{worker.to_log}")
|
|
899
|
+
else
|
|
900
|
+
logger.info("Failed to send SIGTERM to #{worker.to_log}")
|
|
901
|
+
end
|
|
902
|
+
end
|
|
764
903
|
end
|
|
765
904
|
end
|
|
766
905
|
end
|
|
@@ -768,7 +907,7 @@ module Pitchfork
|
|
|
768
907
|
|
|
769
908
|
def restart_outdated_workers
|
|
770
909
|
# If we're already in the middle of forking a new generation, we just continue
|
|
771
|
-
return
|
|
910
|
+
return if REFORKING_AVAILABLE && !@children.mold
|
|
772
911
|
|
|
773
912
|
# We don't shutdown any outdated worker if any worker is already being
|
|
774
913
|
# spawned or a worker is exiting. Only 10% of workers can be reforked at
|
|
@@ -786,7 +925,16 @@ module Pitchfork
|
|
|
786
925
|
end
|
|
787
926
|
|
|
788
927
|
if workers_to_restart > 0
|
|
789
|
-
outdated_workers = @children.workers.select { |w| !w.exiting? && w.
|
|
928
|
+
outdated_workers = @children.workers.select { |w| !w.exiting? && w.outdated? }
|
|
929
|
+
|
|
930
|
+
# Restart the most outdated workers first, can happen if a
|
|
931
|
+
# restart or refork was triggered before the previous one was
|
|
932
|
+
# done fully rolling out.
|
|
933
|
+
# NB: Array#sort_by isn't guaranteed stable so we use the original order as the
|
|
934
|
+
# last ordering criteria.
|
|
935
|
+
indexes = outdated_workers.each_with_index.to_h
|
|
936
|
+
outdated_workers = outdated_workers.sort_by { |w| [w.version, w.generation, indexes[w]] }
|
|
937
|
+
|
|
790
938
|
outdated_workers.each do |worker|
|
|
791
939
|
if worker.soft_kill(:TERM)
|
|
792
940
|
logger.info("Sent SIGTERM to #{worker.to_log}")
|
|
@@ -842,6 +990,7 @@ module Pitchfork
|
|
|
842
990
|
# once a client is accepted, it is processed in its entirety here
|
|
843
991
|
# in 3 easy steps: read request, call app, write app response
|
|
844
992
|
def process_client(client, worker, timeout_handler)
|
|
993
|
+
accepted_at = Process.clock_gettime(Process::CLOCK_REALTIME)
|
|
845
994
|
response_written = false
|
|
846
995
|
env = nil
|
|
847
996
|
@request = Pitchfork::HttpParser.new
|
|
@@ -854,6 +1003,7 @@ module Pitchfork
|
|
|
854
1003
|
proc_name status: status
|
|
855
1004
|
|
|
856
1005
|
env["pitchfork.worker"] = worker
|
|
1006
|
+
env["pitchfork.accepted_at"] = accepted_at
|
|
857
1007
|
timeout_handler.rack_env = env
|
|
858
1008
|
env["pitchfork.timeout"] = timeout_handler
|
|
859
1009
|
|
|
@@ -924,7 +1074,7 @@ module Pitchfork
|
|
|
924
1074
|
# traps for USR2, and HUP may be set in the after_worker_fork/after_mold_fork Procs
|
|
925
1075
|
# by the user.
|
|
926
1076
|
def init_worker_process(worker)
|
|
927
|
-
proc_name role: "(
|
|
1077
|
+
proc_name role: "(#{worker.version}.#{worker.generation}) worker[#{worker.nr}]", status: "init"
|
|
928
1078
|
worker.reset
|
|
929
1079
|
worker.register_to_monitor(@control_socket[1])
|
|
930
1080
|
# we'll re-trap :QUIT and :TERM later for graceful shutdown iff we accept clients
|
|
@@ -936,7 +1086,7 @@ module Pitchfork
|
|
|
936
1086
|
@sig_queue.clear
|
|
937
1087
|
@children = nil
|
|
938
1088
|
|
|
939
|
-
after_worker_fork
|
|
1089
|
+
run_callback!(after_worker_fork, "after_worker_fork", self, worker) # can drop perms and create listeners
|
|
940
1090
|
LISTENERS.each { |sock| sock.close_on_exec = true }
|
|
941
1091
|
|
|
942
1092
|
@config = nil
|
|
@@ -950,8 +1100,8 @@ module Pitchfork
|
|
|
950
1100
|
end
|
|
951
1101
|
|
|
952
1102
|
def init_service_process(service)
|
|
953
|
-
proc_name role: "(
|
|
954
|
-
|
|
1103
|
+
proc_name role: "(#{service.version}.#{service.generation}) service", status: "init"
|
|
1104
|
+
Pitchfork.close_listeners # Don't appear as listening to incoming requests
|
|
955
1105
|
service.register_to_monitor(@control_socket[1])
|
|
956
1106
|
readers = [service]
|
|
957
1107
|
trap(:QUIT) { nuke_listeners!(readers) }
|
|
@@ -961,8 +1111,8 @@ module Pitchfork
|
|
|
961
1111
|
end
|
|
962
1112
|
|
|
963
1113
|
def init_mold_process(mold)
|
|
964
|
-
proc_name role: "(
|
|
965
|
-
after_mold_fork
|
|
1114
|
+
proc_name role: "(#{mold.version}.#{mold.generation}) mold", status: "init"
|
|
1115
|
+
run_callback!(after_mold_fork, "after_mold_fork", self, mold)
|
|
966
1116
|
readers = [mold]
|
|
967
1117
|
trap(:QUIT) { nuke_listeners!(readers) }
|
|
968
1118
|
trap(:TERM) { nuke_listeners!(readers) }
|
|
@@ -972,10 +1122,16 @@ module Pitchfork
|
|
|
972
1122
|
|
|
973
1123
|
if Pitchfork.const_defined?(:Waiter)
|
|
974
1124
|
def prep_readers(readers)
|
|
975
|
-
|
|
1125
|
+
case @poller
|
|
1126
|
+
when :epoll_exclusive
|
|
1127
|
+
Info.keep_io(Waiter.prep_readers(readers, true))
|
|
1128
|
+
when :epoll
|
|
1129
|
+
Info.keep_io(Waiter.prep_readers(readers, false))
|
|
1130
|
+
else
|
|
1131
|
+
Pitchfork::SelectWaiter.new
|
|
1132
|
+
end
|
|
976
1133
|
end
|
|
977
1134
|
else
|
|
978
|
-
require_relative 'select_waiter'
|
|
979
1135
|
def prep_readers(_readers)
|
|
980
1136
|
Pitchfork::SelectWaiter.new
|
|
981
1137
|
end
|
|
@@ -989,7 +1145,7 @@ module Pitchfork
|
|
|
989
1145
|
waiter = prep_readers(readers)
|
|
990
1146
|
|
|
991
1147
|
ready = readers.dup
|
|
992
|
-
@after_worker_ready
|
|
1148
|
+
run_callback!(@after_worker_ready, "after_worker_ready", self, worker)
|
|
993
1149
|
|
|
994
1150
|
worker.notify_ready(@control_socket[1])
|
|
995
1151
|
proc_name status: "ready"
|
|
@@ -1014,12 +1170,22 @@ module Pitchfork
|
|
|
1014
1170
|
when Message
|
|
1015
1171
|
worker.update(client)
|
|
1016
1172
|
else
|
|
1017
|
-
|
|
1173
|
+
worker.busy = true
|
|
1174
|
+
timeout_handler = prepare_timeout(worker)
|
|
1175
|
+
request_env = process_client(client, worker, timeout_handler)
|
|
1176
|
+
if timeout_handler.timed_out?
|
|
1177
|
+
# before_worker_exit already ran in the timeout thread.
|
|
1178
|
+
logger.error("#{worker.to_log} SystemExit was swallowed by the application, exiting")
|
|
1179
|
+
Process.exit
|
|
1180
|
+
end
|
|
1181
|
+
|
|
1182
|
+
if request_env
|
|
1018
1183
|
worker.increment_requests_count
|
|
1019
|
-
@after_request_complete
|
|
1184
|
+
safe_run_callback(@after_request_complete, "after_request_complete", self, worker, request_env)
|
|
1020
1185
|
end
|
|
1021
1186
|
end
|
|
1022
1187
|
|
|
1188
|
+
worker.busy = false
|
|
1023
1189
|
worker.update_deadline(@timeout)
|
|
1024
1190
|
end
|
|
1025
1191
|
|
|
@@ -1033,6 +1199,8 @@ module Pitchfork
|
|
|
1033
1199
|
logger.info("#{worker.to_log} refork condition met, promoting ourselves")
|
|
1034
1200
|
end
|
|
1035
1201
|
@refork_condition.backoff!
|
|
1202
|
+
# spawn_mold may have shortened our deadline to spawn_timeout, restore it before waiting for requests
|
|
1203
|
+
worker.update_deadline(@timeout)
|
|
1036
1204
|
end
|
|
1037
1205
|
end
|
|
1038
1206
|
|
|
@@ -1053,7 +1221,7 @@ module Pitchfork
|
|
|
1053
1221
|
|
|
1054
1222
|
begin
|
|
1055
1223
|
fork_sibling("spawn_mold") do
|
|
1056
|
-
mold = Worker.new(nil, pid: Process.pid, generation: worker.generation)
|
|
1224
|
+
mold = Worker.new(nil, pid: Process.pid, generation: worker.generation, version: worker.version)
|
|
1057
1225
|
mold.promote!(@spawn_timeout)
|
|
1058
1226
|
mold.start_promotion(@control_socket[1])
|
|
1059
1227
|
mold_loop(mold)
|
|
@@ -1085,45 +1253,7 @@ module Pitchfork
|
|
|
1085
1253
|
while sock = ready.shift
|
|
1086
1254
|
# Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
|
|
1087
1255
|
# but that will return false
|
|
1088
|
-
|
|
1089
|
-
case message
|
|
1090
|
-
when false
|
|
1091
|
-
# no message, keep looping
|
|
1092
|
-
when Message::SpawnWorker
|
|
1093
|
-
retries = 1
|
|
1094
|
-
begin
|
|
1095
|
-
spawn_worker(Worker.new(message.nr, generation: mold.generation), detach: true)
|
|
1096
|
-
rescue ForkFailure
|
|
1097
|
-
if retries > 0
|
|
1098
|
-
@logger.fatal("#{mold.to_log} failed to spawn a worker, retrying")
|
|
1099
|
-
retries -= 1
|
|
1100
|
-
retry
|
|
1101
|
-
else
|
|
1102
|
-
@logger.fatal("#{mold.to_log} failed to spawn a worker twice in a row - corrupted mold process?")
|
|
1103
|
-
Process.exit(1)
|
|
1104
|
-
end
|
|
1105
|
-
rescue => error
|
|
1106
|
-
raise BootFailure, error.message
|
|
1107
|
-
end
|
|
1108
|
-
when Message::SpawnService
|
|
1109
|
-
retries = 1
|
|
1110
|
-
begin
|
|
1111
|
-
spawn_service(Service.new(generation: mold.generation), detach: true)
|
|
1112
|
-
rescue ForkFailure
|
|
1113
|
-
if retries > 0
|
|
1114
|
-
@logger.fatal("#{mold.to_log} failed to spawn a service, retrying")
|
|
1115
|
-
retries -= 1
|
|
1116
|
-
retry
|
|
1117
|
-
else
|
|
1118
|
-
@logger.fatal("#{mold.to_log} failed to spawn a service twice in a row - corrupted mold process?")
|
|
1119
|
-
Process.exit(1)
|
|
1120
|
-
end
|
|
1121
|
-
rescue => error
|
|
1122
|
-
raise BootFailure, error.message
|
|
1123
|
-
end
|
|
1124
|
-
else
|
|
1125
|
-
logger.error("Unexpected mold message #{message.inspect}")
|
|
1126
|
-
end
|
|
1256
|
+
mold_handle_message(mold, sock.accept_nonblock(exception: false))
|
|
1127
1257
|
end
|
|
1128
1258
|
|
|
1129
1259
|
# timeout so we can .tick and keep parent from SIGKILL-ing us
|
|
@@ -1135,6 +1265,51 @@ module Pitchfork
|
|
|
1135
1265
|
end
|
|
1136
1266
|
end
|
|
1137
1267
|
|
|
1268
|
+
def mold_handle_message(mold, message)
|
|
1269
|
+
case message
|
|
1270
|
+
when false
|
|
1271
|
+
# no message, keep looping
|
|
1272
|
+
when Message::SpawnWorkers
|
|
1273
|
+
message.nrs.each do |nr|
|
|
1274
|
+
retries = 1
|
|
1275
|
+
mold.update_deadline(@timeout)
|
|
1276
|
+
worker = Worker.new(nr, generation: mold.generation, version: mold.version)
|
|
1277
|
+
begin
|
|
1278
|
+
spawn_worker(worker)
|
|
1279
|
+
rescue ForkFailure
|
|
1280
|
+
if retries > 0
|
|
1281
|
+
@logger.fatal("#{mold.to_log} failed to spawn #{worker.to_log}, retrying")
|
|
1282
|
+
retries -= 1
|
|
1283
|
+
retry
|
|
1284
|
+
else
|
|
1285
|
+
@logger.fatal("#{mold.to_log} failed to spawn #{worker.to_log} twice in a row - corrupted mold process?")
|
|
1286
|
+
Process.exit(1)
|
|
1287
|
+
end
|
|
1288
|
+
rescue => error
|
|
1289
|
+
raise BootFailure, error.message
|
|
1290
|
+
end
|
|
1291
|
+
end
|
|
1292
|
+
when Message::SpawnService
|
|
1293
|
+
retries = 1
|
|
1294
|
+
begin
|
|
1295
|
+
spawn_service(Pitchfork::Worker.new(nil, generation: mold.generation, version: mold.version, service: true))
|
|
1296
|
+
rescue ForkFailure
|
|
1297
|
+
if retries > 0
|
|
1298
|
+
@logger.fatal("#{mold.to_log} failed to spawn a service, retrying")
|
|
1299
|
+
retries -= 1
|
|
1300
|
+
retry
|
|
1301
|
+
else
|
|
1302
|
+
@logger.fatal("#{mold.to_log} failed to spawn a service twice in a row - corrupted mold process?")
|
|
1303
|
+
Process.exit(1)
|
|
1304
|
+
end
|
|
1305
|
+
rescue => error
|
|
1306
|
+
raise BootFailure, error.message
|
|
1307
|
+
end
|
|
1308
|
+
else
|
|
1309
|
+
logger.error("Unexpected mold message #{message.inspect}")
|
|
1310
|
+
end
|
|
1311
|
+
end
|
|
1312
|
+
|
|
1138
1313
|
# delivers a signal to a worker and fails gracefully if the worker
|
|
1139
1314
|
# is no longer running.
|
|
1140
1315
|
def kill_worker(signal, wpid)
|
|
@@ -1143,6 +1318,20 @@ module Pitchfork
|
|
|
1143
1318
|
worker = @children.reap(wpid) and worker.close rescue nil
|
|
1144
1319
|
end
|
|
1145
1320
|
|
|
1321
|
+
def safe_run_callback(callback, name, *args)
|
|
1322
|
+
callback&.call(*args)
|
|
1323
|
+
true
|
|
1324
|
+
rescue => error
|
|
1325
|
+
Pitchfork.log_error(logger, "#{name} error", error)
|
|
1326
|
+
false
|
|
1327
|
+
end
|
|
1328
|
+
|
|
1329
|
+
def run_callback!(callback, name, *args)
|
|
1330
|
+
unless safe_run_callback(callback, name, *args)
|
|
1331
|
+
Process.exit(1)
|
|
1332
|
+
end
|
|
1333
|
+
end
|
|
1334
|
+
|
|
1146
1335
|
# returns an array of string names for the given listener array
|
|
1147
1336
|
def listener_names(listeners = LISTENERS)
|
|
1148
1337
|
listeners.map { |io| sock_name(io) }
|
|
@@ -1174,9 +1363,15 @@ module Pitchfork
|
|
|
1174
1363
|
listeners = config[:listeners].dup
|
|
1175
1364
|
if listeners.empty?
|
|
1176
1365
|
listeners << Pitchfork::Const::DEFAULT_LISTEN
|
|
1177
|
-
@init_listeners << Pitchfork::Const::DEFAULT_LISTEN
|
|
1178
1366
|
end
|
|
1179
1367
|
listeners.each { |addr| listen(addr) }
|
|
1368
|
+
|
|
1369
|
+
if @inherited_listeners
|
|
1370
|
+
@inherited_listeners.each_value { |sockets| sockets.each(&:close) }
|
|
1371
|
+
@inherited_listeners.clear
|
|
1372
|
+
@inherited_listeners = nil
|
|
1373
|
+
end
|
|
1374
|
+
|
|
1180
1375
|
raise ArgumentError, "no listeners" if LISTENERS.empty?
|
|
1181
1376
|
end
|
|
1182
1377
|
|