pitchfork 0.18.1 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/actions.lock +40 -0
- data/.github/workflows/ci.yml +30 -11
- data/.github/workflows/release.yml +29 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +16 -2
- data/Gemfile +1 -0
- data/Rakefile +15 -0
- data/benchmark/request_benchmark.rb +86 -0
- data/docs/CONFIGURATION.md +51 -14
- data/docs/MIGRATING_FROM_UNICORN.md +6 -0
- data/docs/SIGNALS.md +6 -0
- data/examples/pitchfork.conf.reload.rb +41 -0
- data/exe/pitchfork +3 -2
- data/ext/pitchfork_http/c_util.h +3 -8
- data/ext/pitchfork_http/child_subreaper.h +7 -1
- data/ext/pitchfork_http/epollexclusive.h +6 -3
- data/ext/pitchfork_http/extconf.rb +1 -0
- data/ext/pitchfork_http/global_variables.h +2 -0
- data/ext/pitchfork_http/memory_page.c +188 -25
- data/ext/pitchfork_http/pitchfork_http.c +578 -471
- data/ext/pitchfork_http/pitchfork_http.rl +63 -33
- data/ext/pitchfork_http/pitchfork_http_common.rl +1 -1
- data/lib/pitchfork/children.rb +74 -20
- data/lib/pitchfork/chunked.rb +9 -0
- data/lib/pitchfork/configurator.rb +25 -6
- data/lib/pitchfork/http_parser.rb +4 -0
- data/lib/pitchfork/http_response.rb +35 -12
- data/lib/pitchfork/http_server.rb +391 -193
- data/lib/pitchfork/info.rb +8 -0
- data/lib/pitchfork/listeners.rb +52 -0
- data/lib/pitchfork/message.rb +58 -8
- data/lib/pitchfork/shared_memory.rb +64 -13
- data/lib/pitchfork/socket_helper.rb +6 -2
- data/lib/pitchfork/stream_input.rb +12 -9
- data/lib/pitchfork/version.rb +1 -1
- data/lib/pitchfork/worker.rb +124 -64
- data/lib/pitchfork.rb +15 -7
- metadata +7 -6
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require 'pitchfork/pitchfork_http'
|
|
5
5
|
require 'pitchfork/listeners'
|
|
6
|
+
require 'pitchfork/select_waiter'
|
|
6
7
|
require 'pitchfork/flock'
|
|
7
8
|
require 'pitchfork/soft_timeout'
|
|
8
9
|
require 'pitchfork/shared_memory'
|
|
@@ -42,13 +43,19 @@ module Pitchfork
|
|
|
42
43
|
@callback = callback
|
|
43
44
|
@rack_env = nil
|
|
44
45
|
@timeout_request = nil
|
|
46
|
+
@timed_out = false
|
|
45
47
|
end
|
|
46
48
|
|
|
47
49
|
def inspect
|
|
48
50
|
"#<Pitchfork::HttpServer::TimeoutHandler##{object_id}>"
|
|
49
51
|
end
|
|
50
52
|
|
|
53
|
+
def timed_out? # :nodoc:
|
|
54
|
+
@timed_out
|
|
55
|
+
end
|
|
56
|
+
|
|
51
57
|
def call(original_thread) # :nodoc:
|
|
58
|
+
@timed_out = true
|
|
52
59
|
begin
|
|
53
60
|
@server.logger.error("#{@worker.to_log} timed out, exiting")
|
|
54
61
|
if @callback
|
|
@@ -79,13 +86,13 @@ module Pitchfork
|
|
|
79
86
|
# :stopdoc:
|
|
80
87
|
attr_accessor :app, :timeout, :timeout_signal, :soft_timeout, :cleanup_timeout, :spawn_timeout, :worker_processes,
|
|
81
88
|
:before_fork, :after_worker_fork, :after_mold_fork, :before_service_worker_ready, :before_service_worker_exit,
|
|
82
|
-
:listener_opts, :children,
|
|
83
|
-
:orig_app, :config, :ready_pipe, :early_hints, :setpgid
|
|
89
|
+
:listener_opts, :children, :poller,
|
|
90
|
+
:orig_app, :config, :ready_pipe, :early_hints, :setpgid, :restart_command, :restart_command_argv, :working_directory
|
|
84
91
|
attr_writer :after_worker_exit, :before_worker_exit, :after_worker_ready, :after_request_complete,
|
|
85
92
|
:refork_condition, :after_worker_timeout, :after_worker_hard_timeout, :after_monitor_ready, :refork_max_unavailable,
|
|
86
93
|
:max_consecutive_spawn_errors
|
|
87
94
|
|
|
88
|
-
attr_reader :logger
|
|
95
|
+
attr_reader :logger, :version
|
|
89
96
|
include Pitchfork::SocketHelper
|
|
90
97
|
include Pitchfork::HttpResponse
|
|
91
98
|
|
|
@@ -101,6 +108,7 @@ module Pitchfork
|
|
|
101
108
|
# HttpServer.run.join to join the thread that's processing
|
|
102
109
|
# incoming requests on the socket.
|
|
103
110
|
def initialize(app, options = {})
|
|
111
|
+
@version = 0
|
|
104
112
|
@exit_status = 0
|
|
105
113
|
@app = app
|
|
106
114
|
@respawn = false
|
|
@@ -113,7 +121,6 @@ module Pitchfork
|
|
|
113
121
|
|
|
114
122
|
options = options.dup
|
|
115
123
|
@ready_pipe = options.delete(:ready_pipe)
|
|
116
|
-
@init_listeners = options[:listeners].dup || []
|
|
117
124
|
options[:use_defaults] = true
|
|
118
125
|
self.config = Pitchfork::Configurator.new(options)
|
|
119
126
|
self.listener_opts = {}
|
|
@@ -147,31 +154,76 @@ module Pitchfork
|
|
|
147
154
|
config.commit!(self, :skip => [:listeners, :pid])
|
|
148
155
|
@orig_app = app
|
|
149
156
|
# list of signals we care about and trap in monitor.
|
|
150
|
-
@queue_sigs = [
|
|
151
|
-
:QUIT, :INT, :TERM, :USR2, :TTIN, :TTOU ]
|
|
152
|
-
|
|
153
|
-
Info.workers_count = worker_processes
|
|
154
|
-
SharedMemory.preallocate_pages(worker_processes)
|
|
157
|
+
@queue_sigs = [ :QUIT, :INT, :TERM, :USR1, :USR2, :TTIN, :TTOU ]
|
|
155
158
|
end
|
|
156
159
|
|
|
157
160
|
# Runs the thing. Returns self so you can run join on it
|
|
158
161
|
def start(sync = true)
|
|
159
162
|
Pitchfork.enable_child_subreaper # noop if not supported
|
|
160
163
|
|
|
164
|
+
restart_state = ENV.delete("PITCHFORK_RESTART_STATE")
|
|
165
|
+
if restart_state
|
|
166
|
+
restart_state = Marshal.load(restart_state.unpack1("m0"))
|
|
167
|
+
logger.info "monitor v=#{restart_state[:version]} initializing with inherited state"
|
|
168
|
+
else
|
|
169
|
+
restart_state = {}
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
if restart_state[:version]
|
|
173
|
+
@version = restart_state[:version]
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
if working_directory
|
|
177
|
+
real_pwd = File.realpath(Dir.pwd)
|
|
178
|
+
real_working_directory = File.realpath(working_directory)
|
|
179
|
+
|
|
180
|
+
if real_pwd != real_working_directory
|
|
181
|
+
logger.info("monitor v=#{@version} chdir #{File.realpath(working_directory)}")
|
|
182
|
+
Dir.chdir(real_working_directory)
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
if restart_state[:sig_queue]
|
|
187
|
+
@sig_queue = [*restart_state[:sig_queue], *@sig_queue]
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
if restart_state[:shared_memory_fds]
|
|
191
|
+
SharedMemory.reopen(restart_state[:shared_memory_fds])
|
|
192
|
+
end
|
|
193
|
+
SharedMemory.preallocate_pages(worker_processes)
|
|
194
|
+
Info.workers_count = worker_processes
|
|
195
|
+
|
|
161
196
|
# This socketpair is used to wake us up from select(2) in #join when signals
|
|
162
197
|
# are trapped. See trap_deferred.
|
|
163
198
|
# It's also used by newly spawned children to send their soft_signal pipe
|
|
164
199
|
# to the monitor when they are spawned.
|
|
165
|
-
|
|
200
|
+
if fds = restart_state[:control_socket_fds]
|
|
201
|
+
@control_socket.replace(Pitchfork.socketpair_for_fds(fds))
|
|
202
|
+
else
|
|
203
|
+
@control_socket.replace(Pitchfork.socketpair)
|
|
204
|
+
end
|
|
166
205
|
Info.keep_ios(@control_socket)
|
|
167
206
|
@monitor_pid = $$
|
|
168
207
|
|
|
208
|
+
if restart_state[:children]
|
|
209
|
+
@children = Children.load(restart_state[:children])
|
|
210
|
+
@children.close_on_exec = true
|
|
211
|
+
end
|
|
212
|
+
|
|
169
213
|
# setup signal handlers before writing pid file in case people get
|
|
170
214
|
# trigger happy and send signals as soon as the pid file exists.
|
|
171
215
|
# Note that signals don't actually get handled until the #join method
|
|
172
216
|
@queue_sigs.each { |sig| trap(sig) { @sig_queue << sig; awaken_monitor } }
|
|
173
217
|
trap(:CHLD) { awaken_monitor }
|
|
174
218
|
|
|
219
|
+
if restart_state[:listeners]
|
|
220
|
+
@inherited_listeners = Hash.new { |h, k| h[k] = [] }
|
|
221
|
+
Listeners.load(restart_state[:listeners]).each do |socket|
|
|
222
|
+
@inherited_listeners[sock_name(socket)] << socket
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
bind_listeners!
|
|
226
|
+
|
|
175
227
|
if REFORKING_AVAILABLE
|
|
176
228
|
spawn_initial_mold
|
|
177
229
|
wait_for_pending_workers
|
|
@@ -180,8 +232,8 @@ module Pitchfork
|
|
|
180
232
|
end
|
|
181
233
|
else
|
|
182
234
|
build_app!
|
|
183
|
-
|
|
184
|
-
after_mold_fork
|
|
235
|
+
SharedMemory.current_version = @version
|
|
236
|
+
after_mold_fork&.call(self, Worker.new(nil, pid: $$).promoted!(@spawn_timeout))
|
|
185
237
|
end
|
|
186
238
|
|
|
187
239
|
if sync
|
|
@@ -197,25 +249,6 @@ module Pitchfork
|
|
|
197
249
|
self
|
|
198
250
|
end
|
|
199
251
|
|
|
200
|
-
# replaces current listener set with +listeners+. This will
|
|
201
|
-
# close the socket if it will not exist in the new listener set
|
|
202
|
-
def listeners=(listeners)
|
|
203
|
-
unless LISTENERS.empty?
|
|
204
|
-
raise "Listeners can only be initialized once"
|
|
205
|
-
end
|
|
206
|
-
|
|
207
|
-
cur_names, dead_names = [], []
|
|
208
|
-
listener_names.each do |name|
|
|
209
|
-
if name.start_with?('/')
|
|
210
|
-
# mark unlinked sockets as dead so we can rebind them
|
|
211
|
-
(File.socket?(name) ? cur_names : dead_names) << name
|
|
212
|
-
else
|
|
213
|
-
cur_names << name
|
|
214
|
-
end
|
|
215
|
-
end
|
|
216
|
-
listener_names(listeners).each { |addr| listen(addr) }
|
|
217
|
-
end
|
|
218
|
-
|
|
219
252
|
def logger=(obj)
|
|
220
253
|
Pitchfork::HttpParser::DEFAULTS["rack.logger"] = @logger = obj
|
|
221
254
|
end
|
|
@@ -240,7 +273,7 @@ module Pitchfork
|
|
|
240
273
|
opt[:reuseport] = true if queues > 1
|
|
241
274
|
|
|
242
275
|
begin
|
|
243
|
-
io = bind_listen(address, opt)
|
|
276
|
+
io = bind_listen(address, opt, @inherited_listeners)
|
|
244
277
|
unless TCPServer === io || UNIXServer === io
|
|
245
278
|
io.autoclose = false
|
|
246
279
|
io = server_cast(io)
|
|
@@ -266,7 +299,7 @@ module Pitchfork
|
|
|
266
299
|
ios = [io]
|
|
267
300
|
|
|
268
301
|
(queues - 1).times do
|
|
269
|
-
io = bind_listen(address, opt)
|
|
302
|
+
io = bind_listen(address, opt, @inherited_listeners)
|
|
270
303
|
unless TCPServer === io || UNIXServer === io
|
|
271
304
|
io.autoclose = false
|
|
272
305
|
io = server_cast(io)
|
|
@@ -295,7 +328,6 @@ module Pitchfork
|
|
|
295
328
|
|
|
296
329
|
proc_name role: 'monitor', status: ARGV.join(' ')
|
|
297
330
|
|
|
298
|
-
logger.info "monitor process ready" # test_exec.rb relies on this message
|
|
299
331
|
if @ready_pipe
|
|
300
332
|
begin
|
|
301
333
|
@ready_pipe.syswrite($$.to_s)
|
|
@@ -330,6 +362,13 @@ module Pitchfork
|
|
|
330
362
|
|
|
331
363
|
case message = @sig_queue.shift
|
|
332
364
|
when nil
|
|
365
|
+
if @respawn
|
|
366
|
+
# Spawn missing workers first so that `murder_lazy_workers` can consider their spawn timeout
|
|
367
|
+
# when computing how long to sleep.
|
|
368
|
+
maintain_worker_count
|
|
369
|
+
restart_outdated_workers
|
|
370
|
+
end
|
|
371
|
+
|
|
333
372
|
# avoid murdering workers after our monitor process (or the
|
|
334
373
|
# machine) comes out of suspend/hibernation
|
|
335
374
|
if (@last_check + @timeout) >= (@last_check = Pitchfork.time_now)
|
|
@@ -345,12 +384,15 @@ module Pitchfork
|
|
|
345
384
|
sleep_time = @timeout/2.0 + 1
|
|
346
385
|
@logger.debug("waiting #{sleep_time}s after suspend/hibernation")
|
|
347
386
|
end
|
|
348
|
-
if @respawn
|
|
349
|
-
maintain_worker_count
|
|
350
|
-
restart_outdated_workers if REFORKING_AVAILABLE
|
|
351
|
-
end
|
|
352
387
|
|
|
353
388
|
monitor_sleep(sleep_time) if sleep
|
|
389
|
+
else
|
|
390
|
+
monitor_handle_message(message)
|
|
391
|
+
end
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
def monitor_handle_message(message)
|
|
395
|
+
case message
|
|
354
396
|
when :QUIT, :TERM # graceful shutdown
|
|
355
397
|
SharedMemory.shutting_down!
|
|
356
398
|
logger.info "#{message} received, starting graceful shutdown"
|
|
@@ -360,6 +402,8 @@ module Pitchfork
|
|
|
360
402
|
logger.info "#{message} received, starting immediate shutdown"
|
|
361
403
|
stop(false)
|
|
362
404
|
return StopIteration
|
|
405
|
+
when :USR1 # trigger a hot restart
|
|
406
|
+
trigger_restart
|
|
363
407
|
when :USR2 # trigger a promotion
|
|
364
408
|
if @respawn
|
|
365
409
|
trigger_refork
|
|
@@ -371,30 +415,45 @@ module Pitchfork
|
|
|
371
415
|
self.worker_processes += 1
|
|
372
416
|
when :TTOU
|
|
373
417
|
self.worker_processes -= 1 if self.worker_processes > 0
|
|
374
|
-
when Message::
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
when Message::MoldSpawned
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
418
|
+
when Message::ProcessReaped
|
|
419
|
+
unless try_reaping_worker(message.status)
|
|
420
|
+
logger.info("reaped unknown subprocess #{message.status.inspect}")
|
|
421
|
+
end
|
|
422
|
+
when Message::WorkerSpawned, Message::MoldSpawned, Message::ServiceSpawned
|
|
423
|
+
if worker = @children.update(message)
|
|
424
|
+
logger.info("#{worker.to_log} spawned")
|
|
425
|
+
else
|
|
426
|
+
logger.error("worker=#{message.nr} gen=#{message.version}.#{message.generation} pid=#{message.pid} spawned but slot is taken (abandoned worker?), killing")
|
|
427
|
+
|
|
428
|
+
# Send SIGINT for immediate shutdown so that shared state isn't reset on exit.
|
|
429
|
+
MessageSocket.new(message.pipe).try_send_message(Message::SoftKill.new(Signal.list.fetch("INT")))
|
|
430
|
+
message.pipe.close
|
|
431
|
+
end
|
|
384
432
|
when Message::MoldReady
|
|
385
|
-
@consecutive_spawn_errors = 0
|
|
386
|
-
old_molds = @children.molds
|
|
387
433
|
new_mold = @children.update(message)
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
434
|
+
|
|
435
|
+
# current_mold may not be new_mold if there was a race
|
|
436
|
+
# between refork and restart.
|
|
437
|
+
if new_mold != @children.mold
|
|
438
|
+
logger.info("#{new_mold.to_log} ready but outdated, terminating it")
|
|
439
|
+
new_mold.soft_kill(:TERM)
|
|
440
|
+
else
|
|
441
|
+
@consecutive_spawn_errors = 0
|
|
442
|
+
logger.info("#{new_mold.to_log} ready")
|
|
443
|
+
SharedMemory.current_generation = new_mold.generation
|
|
444
|
+
SharedMemory.current_version = new_mold.version
|
|
445
|
+
|
|
446
|
+
@children.molds.each do |mold|
|
|
447
|
+
if mold != new_mold
|
|
448
|
+
logger.info("Terminating old #{mold.to_log}")
|
|
449
|
+
mold.soft_kill(:TERM)
|
|
450
|
+
end
|
|
451
|
+
end
|
|
392
452
|
end
|
|
393
453
|
when Message::WorkerReady, Message::ServiceReady
|
|
394
454
|
@consecutive_spawn_errors = 0
|
|
395
455
|
else
|
|
396
456
|
logger.error("Unexpected message in sig_queue #{message.inspect}")
|
|
397
|
-
logger.error(@sig_queue.inspect)
|
|
398
457
|
end
|
|
399
458
|
end
|
|
400
459
|
|
|
@@ -404,7 +463,7 @@ module Pitchfork
|
|
|
404
463
|
@respawn = false
|
|
405
464
|
SharedMemory.shutting_down!
|
|
406
465
|
wait_for_pending_workers
|
|
407
|
-
|
|
466
|
+
Pitchfork.close_listeners
|
|
408
467
|
|
|
409
468
|
limit = Pitchfork.time_now + timeout
|
|
410
469
|
until @children.empty? || Pitchfork.time_now > limit
|
|
@@ -430,13 +489,7 @@ module Pitchfork
|
|
|
430
489
|
logger.info "#{worker.to_log} exiting"
|
|
431
490
|
proc_name status: "exiting"
|
|
432
491
|
|
|
433
|
-
|
|
434
|
-
begin
|
|
435
|
-
@before_worker_exit.call(self, worker)
|
|
436
|
-
rescue => error
|
|
437
|
-
Pitchfork.log_error(logger, "before_worker_exit error", error)
|
|
438
|
-
end
|
|
439
|
-
end
|
|
492
|
+
safe_run_callback(@before_worker_exit, "before_worker_exit", self, worker)
|
|
440
493
|
Process.exit
|
|
441
494
|
end
|
|
442
495
|
|
|
@@ -444,13 +497,7 @@ module Pitchfork
|
|
|
444
497
|
logger.info "#{service.to_log} exiting"
|
|
445
498
|
proc_name status: "exiting"
|
|
446
499
|
|
|
447
|
-
|
|
448
|
-
begin
|
|
449
|
-
@before_service_worker_exit.call(self, service)
|
|
450
|
-
rescue => error
|
|
451
|
-
Pitchfork.log_error(logger, "before_service_worker_exit error", error)
|
|
452
|
-
end
|
|
453
|
-
end
|
|
500
|
+
safe_run_callback(@before_service_worker_exit, "before_service_worker_exit", self, service)
|
|
454
501
|
Process.exit
|
|
455
502
|
end
|
|
456
503
|
|
|
@@ -506,27 +553,39 @@ module Pitchfork
|
|
|
506
553
|
loop do
|
|
507
554
|
wpid, status = Process.waitpid2(-1, Process::WNOHANG)
|
|
508
555
|
wpid or return
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
556
|
+
|
|
557
|
+
unless try_reaping_worker(status)
|
|
558
|
+
# We may not have yet processed the WorkerSpawned message, so let's queue that event.
|
|
559
|
+
message = Message::ProcessReaped.new(status)
|
|
560
|
+
case @control_socket[1].sendmsg_nonblock(message, exception: false)
|
|
561
|
+
when :wait_writable
|
|
562
|
+
@sig_queue << message # fallback
|
|
513
563
|
end
|
|
514
|
-
@after_worker_exit.call(self, worker, status)
|
|
515
|
-
else
|
|
516
|
-
logger.info("reaped unknown subprocess #{status.inspect}")
|
|
517
564
|
end
|
|
518
565
|
rescue Errno::ECHILD
|
|
519
566
|
break
|
|
520
567
|
end
|
|
521
568
|
end
|
|
522
569
|
|
|
523
|
-
def
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
570
|
+
def try_reaping_worker(status)
|
|
571
|
+
worker = @children.reap(status.pid) and worker.close rescue nil
|
|
572
|
+
if worker
|
|
573
|
+
reap_worker(worker, status)
|
|
574
|
+
true
|
|
575
|
+
end
|
|
576
|
+
end
|
|
577
|
+
|
|
578
|
+
def reap_worker(worker, status)
|
|
579
|
+
unless worker.ready?
|
|
580
|
+
@consecutive_spawn_errors += 1
|
|
581
|
+
end
|
|
582
|
+
worker.reaped!
|
|
583
|
+
|
|
584
|
+
run_callback!(@after_worker_exit, "after_worker_exit", self, worker, status)
|
|
585
|
+
|
|
586
|
+
if worker.mold? && @children.mold != worker && worker.version > (@children.mold&.version || 0)
|
|
587
|
+
logger.error("#{worker.to_log} crashed before being ready, restart cancelled")
|
|
528
588
|
end
|
|
529
|
-
listener_fds
|
|
530
589
|
end
|
|
531
590
|
|
|
532
591
|
# forcibly terminate all workers that haven't checked in in timeout seconds. The timeout is implemented using an unlinked File
|
|
@@ -555,25 +614,66 @@ module Pitchfork
|
|
|
555
614
|
end
|
|
556
615
|
|
|
557
616
|
def hard_timeout(child)
|
|
617
|
+
@consecutive_spawn_errors += 1 unless child.ready?
|
|
618
|
+
|
|
558
619
|
if child.pid.nil? # Not yet registered, likely never spawned
|
|
559
|
-
logger.error "
|
|
560
|
-
@children.abandon(
|
|
620
|
+
logger.error "#{child.to_log} timed out during spawn, abandoning"
|
|
621
|
+
@children.abandon(child)
|
|
561
622
|
return
|
|
562
623
|
end
|
|
563
624
|
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
@after_worker_hard_timeout.call(self, child)
|
|
567
|
-
rescue => error
|
|
568
|
-
Pitchfork.log_error(@logger, "after_worker_hard_timeout callback", error)
|
|
569
|
-
end
|
|
625
|
+
unless child.mold?
|
|
626
|
+
safe_run_callback(@after_worker_hard_timeout, "after_worker_hard_timeout", self, child)
|
|
570
627
|
end
|
|
571
628
|
|
|
572
629
|
logger.error "#{child.to_log} timed out, killing"
|
|
573
|
-
@consecutive_spawn_errors += 1 unless child.ready?
|
|
574
630
|
@children.hard_kill(@timeout_signal.call(child.pid), child) # take no prisoners for hard timeout violations
|
|
575
631
|
end
|
|
576
632
|
|
|
633
|
+
def trigger_restart
|
|
634
|
+
# We must handle all the Message instances in the sig queue, as they could contain sockets or
|
|
635
|
+
# other FD we can't marshal down to the next process.
|
|
636
|
+
# It is OK to handle them out of order as they don't have any dependency with regular signals.
|
|
637
|
+
# Signals will be handled by the new monitor.
|
|
638
|
+
@sig_queue.reject! do |message|
|
|
639
|
+
if message.is_a?(Message)
|
|
640
|
+
monitor_handle_message(message)
|
|
641
|
+
true
|
|
642
|
+
else
|
|
643
|
+
false
|
|
644
|
+
end
|
|
645
|
+
end
|
|
646
|
+
|
|
647
|
+
with_unbundled_env do
|
|
648
|
+
SharedMemory.close_on_exec = false
|
|
649
|
+
@control_socket.map {|s| s.close_on_exec = false }
|
|
650
|
+
@children.close_on_exec = false
|
|
651
|
+
LISTENERS.close_on_exec = false
|
|
652
|
+
|
|
653
|
+
state = {
|
|
654
|
+
shared_memory_fds: SharedMemory.fds,
|
|
655
|
+
control_socket_fds: @control_socket.map(&:fileno),
|
|
656
|
+
children: @children.dump,
|
|
657
|
+
listeners: LISTENERS.dump,
|
|
658
|
+
version: @version + 1,
|
|
659
|
+
sig_queue: @sig_queue,
|
|
660
|
+
}
|
|
661
|
+
env = { "PITCHFORK_RESTART_STATE" => [Marshal.dump(state)].pack("m0") }
|
|
662
|
+
chdir = File.realpath(working_directory)
|
|
663
|
+
cmd = [*restart_command, *restart_command_argv]
|
|
664
|
+
logger.info "monitor v=#{@version} reexecuting pwd=#{chdir} cmd=#{cmd.inspect}"
|
|
665
|
+
Process.exec(env, *cmd, chdir: chdir)
|
|
666
|
+
end
|
|
667
|
+
end
|
|
668
|
+
|
|
669
|
+
def with_unbundled_env(&block)
|
|
670
|
+
if defined?(Bundler)
|
|
671
|
+
Bundler.with_unbundled_env(&block)
|
|
672
|
+
else
|
|
673
|
+
yield
|
|
674
|
+
end
|
|
675
|
+
end
|
|
676
|
+
|
|
577
677
|
def trigger_refork
|
|
578
678
|
unless REFORKING_AVAILABLE
|
|
579
679
|
logger.error("This system doesn't support PR_SET_CHILD_SUBREAPER, can't refork")
|
|
@@ -593,14 +693,41 @@ module Pitchfork
|
|
|
593
693
|
@control_socket[0].close_write # this is monitor-only, now
|
|
594
694
|
@ready_pipe.close if @ready_pipe
|
|
595
695
|
Pitchfork::Configurator::RACKUP.clear
|
|
596
|
-
@ready_pipe =
|
|
696
|
+
@ready_pipe = nil
|
|
597
697
|
|
|
598
698
|
# The OpenSSL PRNG is seeded with only the pid, and apps with frequently
|
|
599
699
|
# dying workers can recycle pids
|
|
600
700
|
OpenSSL::Random.seed(rand.to_s) if defined?(OpenSSL::Random)
|
|
601
701
|
end
|
|
602
702
|
|
|
603
|
-
def
|
|
703
|
+
def spawn_workers_from_mold(workers)
|
|
704
|
+
if REFORKING_AVAILABLE
|
|
705
|
+
if mold = @children.mold
|
|
706
|
+
# On larger app, the fork call necessary to spawn a worker can take hundreds of milliseconds
|
|
707
|
+
# so we're giving some extra spawn timeout when spawning more than a single worker.
|
|
708
|
+
workers.each_with_index do |worker, i|
|
|
709
|
+
worker.update_deadline(@spawn_timeout + (i * 0.25))
|
|
710
|
+
end
|
|
711
|
+
|
|
712
|
+
if mold.spawn_workers(workers)
|
|
713
|
+
workers.each { |worker| @children.register(worker) }
|
|
714
|
+
else
|
|
715
|
+
@logger.error("Failed to send a spawn_workers command to #{mold.to_log}")
|
|
716
|
+
workers.each(&:reaped!)
|
|
717
|
+
end
|
|
718
|
+
else
|
|
719
|
+
@logger.error("Failed to send a spawn_workers command: no active mold")
|
|
720
|
+
end
|
|
721
|
+
else
|
|
722
|
+
workers.each do |worker|
|
|
723
|
+
worker.update_deadline(@spawn_timeout)
|
|
724
|
+
spawn_worker(worker)
|
|
725
|
+
@children.register(worker)
|
|
726
|
+
end
|
|
727
|
+
end
|
|
728
|
+
end
|
|
729
|
+
|
|
730
|
+
def spawn_worker(worker)
|
|
604
731
|
logger.info("#{worker.to_log} spawning...")
|
|
605
732
|
|
|
606
733
|
# We set the deadline before spawning the child so that if for some
|
|
@@ -626,14 +753,7 @@ module Pitchfork
|
|
|
626
753
|
|
|
627
754
|
ready = readers.dup
|
|
628
755
|
|
|
629
|
-
|
|
630
|
-
begin
|
|
631
|
-
@before_service_worker_ready.call(self, service)
|
|
632
|
-
rescue => error
|
|
633
|
-
Pitchfork.log_error(logger, "before_service_worker_ready", error)
|
|
634
|
-
Process.exit(1)
|
|
635
|
-
end
|
|
636
|
-
end
|
|
756
|
+
run_callback!(@before_service_worker_ready, "before_service_worker_ready", self, service)
|
|
637
757
|
|
|
638
758
|
service.notify_ready(@control_socket[1])
|
|
639
759
|
proc_name status: "ready"
|
|
@@ -665,7 +785,23 @@ module Pitchfork
|
|
|
665
785
|
end
|
|
666
786
|
end
|
|
667
787
|
|
|
668
|
-
def
|
|
788
|
+
def spawn_service_from_mold(service)
|
|
789
|
+
service.update_deadline(@spawn_timeout)
|
|
790
|
+
|
|
791
|
+
if REFORKING_AVAILABLE
|
|
792
|
+
if @children.mold&.spawn_service(service)
|
|
793
|
+
@children.register_service(service)
|
|
794
|
+
else
|
|
795
|
+
@logger.error("Failed to send a spawn_service command")
|
|
796
|
+
service.reaped!
|
|
797
|
+
end
|
|
798
|
+
else
|
|
799
|
+
spawn_service(service)
|
|
800
|
+
@children.register_service(service)
|
|
801
|
+
end
|
|
802
|
+
end
|
|
803
|
+
|
|
804
|
+
def spawn_service(service)
|
|
669
805
|
logger.info("#{service.to_log} spawning...")
|
|
670
806
|
|
|
671
807
|
# We set the deadline before spawning the child so that if for some
|
|
@@ -685,14 +821,15 @@ module Pitchfork
|
|
|
685
821
|
end
|
|
686
822
|
|
|
687
823
|
def spawn_initial_mold
|
|
688
|
-
mold = Worker.new(nil)
|
|
824
|
+
mold = Worker.new(nil, version: @version)
|
|
689
825
|
mold.create_socketpair!
|
|
690
826
|
mold.pid = Pitchfork.clean_fork(setpgid: setpgid) do
|
|
827
|
+
proc_name role: "(#{mold.version}.#{mold.generation}) mold", status: "init"
|
|
691
828
|
mold.pid = Process.pid
|
|
692
829
|
@promotion_lock.try_lock
|
|
693
830
|
mold.after_fork_in_child
|
|
694
831
|
build_app!
|
|
695
|
-
|
|
832
|
+
mold.start_promotion(@control_socket[1])
|
|
696
833
|
mold_loop(mold)
|
|
697
834
|
end
|
|
698
835
|
@promotion_lock.at_fork
|
|
@@ -700,42 +837,35 @@ module Pitchfork
|
|
|
700
837
|
end
|
|
701
838
|
|
|
702
839
|
def spawn_missing_workers
|
|
703
|
-
if
|
|
704
|
-
|
|
705
|
-
if REFORKING_AVAILABLE
|
|
706
|
-
service.generation = @children.mold&.generation || 0
|
|
840
|
+
if REFORKING_AVAILABLE
|
|
841
|
+
return unless @children.mold
|
|
707
842
|
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
843
|
+
version = @children.mold&.version || 0
|
|
844
|
+
generation = @children.mold&.generation || 0
|
|
845
|
+
else
|
|
846
|
+
version = @version
|
|
847
|
+
generation = 0
|
|
848
|
+
end
|
|
714
849
|
|
|
715
|
-
|
|
850
|
+
if @before_service_worker_ready && !@children.service
|
|
851
|
+
service = Worker.new(nil, service: true, version: version, generation: generation)
|
|
852
|
+
spawn_service_from_mold(service)
|
|
716
853
|
end
|
|
717
854
|
|
|
718
855
|
worker_nr = -1
|
|
856
|
+
missing_workers = []
|
|
857
|
+
|
|
858
|
+
|
|
719
859
|
until (worker_nr += 1) == @worker_processes
|
|
720
860
|
if @children.nr_alive?(worker_nr)
|
|
721
861
|
next
|
|
722
862
|
end
|
|
723
|
-
worker = Pitchfork::Worker.new(worker_nr)
|
|
724
863
|
|
|
725
|
-
|
|
726
|
-
|
|
864
|
+
missing_workers << Worker.new(worker_nr, version: version, generation: generation)
|
|
865
|
+
end
|
|
727
866
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
end
|
|
731
|
-
else
|
|
732
|
-
spawn_worker(worker, detach: false)
|
|
733
|
-
end
|
|
734
|
-
# We could directly register workers when we spawn from the
|
|
735
|
-
# monitor, like pitchfork does. However it is preferable to
|
|
736
|
-
# always go through the asynchronous registering process for
|
|
737
|
-
# consistency.
|
|
738
|
-
@children.register(worker)
|
|
867
|
+
unless missing_workers.empty?
|
|
868
|
+
spawn_workers_from_mold(missing_workers)
|
|
739
869
|
end
|
|
740
870
|
rescue => e
|
|
741
871
|
@logger.error(e) rescue nil
|
|
@@ -753,14 +883,23 @@ module Pitchfork
|
|
|
753
883
|
|
|
754
884
|
def maintain_worker_count
|
|
755
885
|
off = @children.workers_count - worker_processes
|
|
756
|
-
off -= 1 if @before_service_worker_ready && !@children.service
|
|
757
886
|
|
|
758
887
|
if off < 0
|
|
759
888
|
spawn_missing_workers
|
|
760
|
-
|
|
761
|
-
@children.
|
|
762
|
-
|
|
763
|
-
|
|
889
|
+
else
|
|
890
|
+
if @before_service_worker_ready && !@children.service
|
|
891
|
+
spawn_missing_workers
|
|
892
|
+
end
|
|
893
|
+
|
|
894
|
+
if off > 0
|
|
895
|
+
@children.each_worker do |worker|
|
|
896
|
+
if worker.nr >= worker_processes
|
|
897
|
+
if worker.soft_kill(:TERM)
|
|
898
|
+
logger.info("Sent SIGTERM to #{worker.to_log}")
|
|
899
|
+
else
|
|
900
|
+
logger.info("Failed to send SIGTERM to #{worker.to_log}")
|
|
901
|
+
end
|
|
902
|
+
end
|
|
764
903
|
end
|
|
765
904
|
end
|
|
766
905
|
end
|
|
@@ -768,7 +907,7 @@ module Pitchfork
|
|
|
768
907
|
|
|
769
908
|
def restart_outdated_workers
|
|
770
909
|
# If we're already in the middle of forking a new generation, we just continue
|
|
771
|
-
return
|
|
910
|
+
return if REFORKING_AVAILABLE && !@children.mold
|
|
772
911
|
|
|
773
912
|
# We don't shutdown any outdated worker if any worker is already being
|
|
774
913
|
# spawned or a worker is exiting. Only 10% of workers can be reforked at
|
|
@@ -786,7 +925,16 @@ module Pitchfork
|
|
|
786
925
|
end
|
|
787
926
|
|
|
788
927
|
if workers_to_restart > 0
|
|
789
|
-
outdated_workers = @children.workers.select { |w| !w.exiting? && w.
|
|
928
|
+
outdated_workers = @children.workers.select { |w| !w.exiting? && w.outdated? }
|
|
929
|
+
|
|
930
|
+
# Restart the most outdated workers first, can happen if a
|
|
931
|
+
# restart or refork was triggered before the previous one was
|
|
932
|
+
# done fully rolling out.
|
|
933
|
+
# NB: Array#sort_by isn't guaranteed stable so we use the original order as the
|
|
934
|
+
# last ordering criteria.
|
|
935
|
+
indexes = outdated_workers.each_with_index.to_h
|
|
936
|
+
outdated_workers = outdated_workers.sort_by { |w| [w.version, w.generation, indexes[w]] }
|
|
937
|
+
|
|
790
938
|
outdated_workers.each do |worker|
|
|
791
939
|
if worker.soft_kill(:TERM)
|
|
792
940
|
logger.info("Sent SIGTERM to #{worker.to_log}")
|
|
@@ -803,7 +951,7 @@ module Pitchfork
|
|
|
803
951
|
# assuming we haven't closed the socket, but don't get hung up
|
|
804
952
|
# if the socket is already closed or broken. We'll always ensure
|
|
805
953
|
# the socket is closed at the end of this function
|
|
806
|
-
def handle_error(client, e)
|
|
954
|
+
def handle_error(client, e, response_written)
|
|
807
955
|
code = case e
|
|
808
956
|
when EOFError,Errno::ECONNRESET,Errno::EPIPE,Errno::ENOTCONN
|
|
809
957
|
# client disconnected on us and there's nothing we can do
|
|
@@ -817,7 +965,7 @@ module Pitchfork
|
|
|
817
965
|
Pitchfork.log_error(@logger, "app error", e)
|
|
818
966
|
500
|
|
819
967
|
end
|
|
820
|
-
if code
|
|
968
|
+
if code && !response_written
|
|
821
969
|
client.write_nonblock(err_response(code, @request.response_start_sent), exception: false)
|
|
822
970
|
end
|
|
823
971
|
client.close
|
|
@@ -842,6 +990,8 @@ module Pitchfork
|
|
|
842
990
|
# once a client is accepted, it is processed in its entirety here
|
|
843
991
|
# in 3 easy steps: read request, call app, write app response
|
|
844
992
|
def process_client(client, worker, timeout_handler)
|
|
993
|
+
accepted_at = Process.clock_gettime(Process::CLOCK_REALTIME)
|
|
994
|
+
response_written = false
|
|
845
995
|
env = nil
|
|
846
996
|
@request = Pitchfork::HttpParser.new
|
|
847
997
|
env = @request.read(client)
|
|
@@ -853,6 +1003,7 @@ module Pitchfork
|
|
|
853
1003
|
proc_name status: status
|
|
854
1004
|
|
|
855
1005
|
env["pitchfork.worker"] = worker
|
|
1006
|
+
env["pitchfork.accepted_at"] = accepted_at
|
|
856
1007
|
timeout_handler.rack_env = env
|
|
857
1008
|
env["pitchfork.timeout"] = timeout_handler
|
|
858
1009
|
|
|
@@ -876,6 +1027,7 @@ module Pitchfork
|
|
|
876
1027
|
end
|
|
877
1028
|
@request.headers? or headers = nil
|
|
878
1029
|
http_response_write(client, status, headers, body, @request)
|
|
1030
|
+
response_written = true
|
|
879
1031
|
ensure
|
|
880
1032
|
body.respond_to?(:close) and body.close
|
|
881
1033
|
end
|
|
@@ -889,7 +1041,7 @@ module Pitchfork
|
|
|
889
1041
|
end
|
|
890
1042
|
env
|
|
891
1043
|
rescue => application_error
|
|
892
|
-
handle_error(client, application_error)
|
|
1044
|
+
handle_error(client, application_error, response_written)
|
|
893
1045
|
env
|
|
894
1046
|
ensure
|
|
895
1047
|
if env
|
|
@@ -922,7 +1074,7 @@ module Pitchfork
|
|
|
922
1074
|
# traps for USR2, and HUP may be set in the after_worker_fork/after_mold_fork Procs
|
|
923
1075
|
# by the user.
|
|
924
1076
|
def init_worker_process(worker)
|
|
925
|
-
proc_name role: "(
|
|
1077
|
+
proc_name role: "(#{worker.version}.#{worker.generation}) worker[#{worker.nr}]", status: "init"
|
|
926
1078
|
worker.reset
|
|
927
1079
|
worker.register_to_monitor(@control_socket[1])
|
|
928
1080
|
# we'll re-trap :QUIT and :TERM later for graceful shutdown iff we accept clients
|
|
@@ -934,7 +1086,7 @@ module Pitchfork
|
|
|
934
1086
|
@sig_queue.clear
|
|
935
1087
|
@children = nil
|
|
936
1088
|
|
|
937
|
-
after_worker_fork
|
|
1089
|
+
run_callback!(after_worker_fork, "after_worker_fork", self, worker) # can drop perms and create listeners
|
|
938
1090
|
LISTENERS.each { |sock| sock.close_on_exec = true }
|
|
939
1091
|
|
|
940
1092
|
@config = nil
|
|
@@ -948,8 +1100,8 @@ module Pitchfork
|
|
|
948
1100
|
end
|
|
949
1101
|
|
|
950
1102
|
def init_service_process(service)
|
|
951
|
-
proc_name role: "(
|
|
952
|
-
|
|
1103
|
+
proc_name role: "(#{service.version}.#{service.generation}) service", status: "init"
|
|
1104
|
+
Pitchfork.close_listeners # Don't appear as listening to incoming requests
|
|
953
1105
|
service.register_to_monitor(@control_socket[1])
|
|
954
1106
|
readers = [service]
|
|
955
1107
|
trap(:QUIT) { nuke_listeners!(readers) }
|
|
@@ -959,8 +1111,8 @@ module Pitchfork
|
|
|
959
1111
|
end
|
|
960
1112
|
|
|
961
1113
|
def init_mold_process(mold)
|
|
962
|
-
proc_name role: "(
|
|
963
|
-
after_mold_fork
|
|
1114
|
+
proc_name role: "(#{mold.version}.#{mold.generation}) mold", status: "init"
|
|
1115
|
+
run_callback!(after_mold_fork, "after_mold_fork", self, mold)
|
|
964
1116
|
readers = [mold]
|
|
965
1117
|
trap(:QUIT) { nuke_listeners!(readers) }
|
|
966
1118
|
trap(:TERM) { nuke_listeners!(readers) }
|
|
@@ -970,10 +1122,16 @@ module Pitchfork
|
|
|
970
1122
|
|
|
971
1123
|
if Pitchfork.const_defined?(:Waiter)
|
|
972
1124
|
def prep_readers(readers)
|
|
973
|
-
|
|
1125
|
+
case @poller
|
|
1126
|
+
when :epoll_exclusive
|
|
1127
|
+
Info.keep_io(Waiter.prep_readers(readers, true))
|
|
1128
|
+
when :epoll
|
|
1129
|
+
Info.keep_io(Waiter.prep_readers(readers, false))
|
|
1130
|
+
else
|
|
1131
|
+
Pitchfork::SelectWaiter.new
|
|
1132
|
+
end
|
|
974
1133
|
end
|
|
975
1134
|
else
|
|
976
|
-
require_relative 'select_waiter'
|
|
977
1135
|
def prep_readers(_readers)
|
|
978
1136
|
Pitchfork::SelectWaiter.new
|
|
979
1137
|
end
|
|
@@ -987,7 +1145,7 @@ module Pitchfork
|
|
|
987
1145
|
waiter = prep_readers(readers)
|
|
988
1146
|
|
|
989
1147
|
ready = readers.dup
|
|
990
|
-
@after_worker_ready
|
|
1148
|
+
run_callback!(@after_worker_ready, "after_worker_ready", self, worker)
|
|
991
1149
|
|
|
992
1150
|
worker.notify_ready(@control_socket[1])
|
|
993
1151
|
proc_name status: "ready"
|
|
@@ -1012,11 +1170,22 @@ module Pitchfork
|
|
|
1012
1170
|
when Message
|
|
1013
1171
|
worker.update(client)
|
|
1014
1172
|
else
|
|
1015
|
-
|
|
1016
|
-
worker
|
|
1017
|
-
|
|
1173
|
+
worker.busy = true
|
|
1174
|
+
timeout_handler = prepare_timeout(worker)
|
|
1175
|
+
request_env = process_client(client, worker, timeout_handler)
|
|
1176
|
+
if timeout_handler.timed_out?
|
|
1177
|
+
# before_worker_exit already ran in the timeout thread.
|
|
1178
|
+
logger.error("#{worker.to_log} SystemExit was swallowed by the application, exiting")
|
|
1179
|
+
Process.exit
|
|
1180
|
+
end
|
|
1181
|
+
|
|
1182
|
+
if request_env
|
|
1183
|
+
worker.increment_requests_count
|
|
1184
|
+
safe_run_callback(@after_request_complete, "after_request_complete", self, worker, request_env)
|
|
1185
|
+
end
|
|
1018
1186
|
end
|
|
1019
1187
|
|
|
1188
|
+
worker.busy = false
|
|
1020
1189
|
worker.update_deadline(@timeout)
|
|
1021
1190
|
end
|
|
1022
1191
|
|
|
@@ -1030,6 +1199,8 @@ module Pitchfork
|
|
|
1030
1199
|
logger.info("#{worker.to_log} refork condition met, promoting ourselves")
|
|
1031
1200
|
end
|
|
1032
1201
|
@refork_condition.backoff!
|
|
1202
|
+
# spawn_mold may have shortened our deadline to spawn_timeout, restore it before waiting for requests
|
|
1203
|
+
worker.update_deadline(@timeout)
|
|
1033
1204
|
end
|
|
1034
1205
|
end
|
|
1035
1206
|
|
|
@@ -1050,7 +1221,7 @@ module Pitchfork
|
|
|
1050
1221
|
|
|
1051
1222
|
begin
|
|
1052
1223
|
fork_sibling("spawn_mold") do
|
|
1053
|
-
mold = Worker.new(nil, pid: Process.pid, generation: worker.generation)
|
|
1224
|
+
mold = Worker.new(nil, pid: Process.pid, generation: worker.generation, version: worker.version)
|
|
1054
1225
|
mold.promote!(@spawn_timeout)
|
|
1055
1226
|
mold.start_promotion(@control_socket[1])
|
|
1056
1227
|
mold_loop(mold)
|
|
@@ -1082,45 +1253,7 @@ module Pitchfork
|
|
|
1082
1253
|
while sock = ready.shift
|
|
1083
1254
|
# Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
|
|
1084
1255
|
# but that will return false
|
|
1085
|
-
|
|
1086
|
-
case message
|
|
1087
|
-
when false
|
|
1088
|
-
# no message, keep looping
|
|
1089
|
-
when Message::SpawnWorker
|
|
1090
|
-
retries = 1
|
|
1091
|
-
begin
|
|
1092
|
-
spawn_worker(Worker.new(message.nr, generation: mold.generation), detach: true)
|
|
1093
|
-
rescue ForkFailure
|
|
1094
|
-
if retries > 0
|
|
1095
|
-
@logger.fatal("#{mold.to_log} failed to spawn a worker, retrying")
|
|
1096
|
-
retries -= 1
|
|
1097
|
-
retry
|
|
1098
|
-
else
|
|
1099
|
-
@logger.fatal("#{mold.to_log} failed to spawn a worker twice in a row - corrupted mold process?")
|
|
1100
|
-
Process.exit(1)
|
|
1101
|
-
end
|
|
1102
|
-
rescue => error
|
|
1103
|
-
raise BootFailure, error.message
|
|
1104
|
-
end
|
|
1105
|
-
when Message::SpawnService
|
|
1106
|
-
retries = 1
|
|
1107
|
-
begin
|
|
1108
|
-
spawn_service(Service.new(generation: mold.generation), detach: true)
|
|
1109
|
-
rescue ForkFailure
|
|
1110
|
-
if retries > 0
|
|
1111
|
-
@logger.fatal("#{mold.to_log} failed to spawn a service, retrying")
|
|
1112
|
-
retries -= 1
|
|
1113
|
-
retry
|
|
1114
|
-
else
|
|
1115
|
-
@logger.fatal("#{mold.to_log} failed to spawn a service twice in a row - corrupted mold process?")
|
|
1116
|
-
Process.exit(1)
|
|
1117
|
-
end
|
|
1118
|
-
rescue => error
|
|
1119
|
-
raise BootFailure, error.message
|
|
1120
|
-
end
|
|
1121
|
-
else
|
|
1122
|
-
logger.error("Unexpected mold message #{message.inspect}")
|
|
1123
|
-
end
|
|
1256
|
+
mold_handle_message(mold, sock.accept_nonblock(exception: false))
|
|
1124
1257
|
end
|
|
1125
1258
|
|
|
1126
1259
|
# timeout so we can .tick and keep parent from SIGKILL-ing us
|
|
@@ -1132,6 +1265,51 @@ module Pitchfork
|
|
|
1132
1265
|
end
|
|
1133
1266
|
end
|
|
1134
1267
|
|
|
1268
|
+
def mold_handle_message(mold, message)
|
|
1269
|
+
case message
|
|
1270
|
+
when false
|
|
1271
|
+
# no message, keep looping
|
|
1272
|
+
when Message::SpawnWorkers
|
|
1273
|
+
message.nrs.each do |nr|
|
|
1274
|
+
retries = 1
|
|
1275
|
+
mold.update_deadline(@timeout)
|
|
1276
|
+
worker = Worker.new(nr, generation: mold.generation, version: mold.version)
|
|
1277
|
+
begin
|
|
1278
|
+
spawn_worker(worker)
|
|
1279
|
+
rescue ForkFailure
|
|
1280
|
+
if retries > 0
|
|
1281
|
+
@logger.fatal("#{mold.to_log} failed to spawn #{worker.to_log}, retrying")
|
|
1282
|
+
retries -= 1
|
|
1283
|
+
retry
|
|
1284
|
+
else
|
|
1285
|
+
@logger.fatal("#{mold.to_log} failed to spawn #{worker.to_log} twice in a row - corrupted mold process?")
|
|
1286
|
+
Process.exit(1)
|
|
1287
|
+
end
|
|
1288
|
+
rescue => error
|
|
1289
|
+
raise BootFailure, error.message
|
|
1290
|
+
end
|
|
1291
|
+
end
|
|
1292
|
+
when Message::SpawnService
|
|
1293
|
+
retries = 1
|
|
1294
|
+
begin
|
|
1295
|
+
spawn_service(Pitchfork::Worker.new(nil, generation: mold.generation, version: mold.version, service: true))
|
|
1296
|
+
rescue ForkFailure
|
|
1297
|
+
if retries > 0
|
|
1298
|
+
@logger.fatal("#{mold.to_log} failed to spawn a service, retrying")
|
|
1299
|
+
retries -= 1
|
|
1300
|
+
retry
|
|
1301
|
+
else
|
|
1302
|
+
@logger.fatal("#{mold.to_log} failed to spawn a service twice in a row - corrupted mold process?")
|
|
1303
|
+
Process.exit(1)
|
|
1304
|
+
end
|
|
1305
|
+
rescue => error
|
|
1306
|
+
raise BootFailure, error.message
|
|
1307
|
+
end
|
|
1308
|
+
else
|
|
1309
|
+
logger.error("Unexpected mold message #{message.inspect}")
|
|
1310
|
+
end
|
|
1311
|
+
end
|
|
1312
|
+
|
|
1135
1313
|
# delivers a signal to a worker and fails gracefully if the worker
|
|
1136
1314
|
# is no longer running.
|
|
1137
1315
|
def kill_worker(signal, wpid)
|
|
@@ -1140,6 +1318,20 @@ module Pitchfork
|
|
|
1140
1318
|
worker = @children.reap(wpid) and worker.close rescue nil
|
|
1141
1319
|
end
|
|
1142
1320
|
|
|
1321
|
+
def safe_run_callback(callback, name, *args)
|
|
1322
|
+
callback&.call(*args)
|
|
1323
|
+
true
|
|
1324
|
+
rescue => error
|
|
1325
|
+
Pitchfork.log_error(logger, "#{name} error", error)
|
|
1326
|
+
false
|
|
1327
|
+
end
|
|
1328
|
+
|
|
1329
|
+
def run_callback!(callback, name, *args)
|
|
1330
|
+
unless safe_run_callback(callback, name, *args)
|
|
1331
|
+
Process.exit(1)
|
|
1332
|
+
end
|
|
1333
|
+
end
|
|
1334
|
+
|
|
1143
1335
|
# returns an array of string names for the given listener array
|
|
1144
1336
|
def listener_names(listeners = LISTENERS)
|
|
1145
1337
|
listeners.map { |io| sock_name(io) }
|
|
@@ -1171,9 +1363,15 @@ module Pitchfork
|
|
|
1171
1363
|
listeners = config[:listeners].dup
|
|
1172
1364
|
if listeners.empty?
|
|
1173
1365
|
listeners << Pitchfork::Const::DEFAULT_LISTEN
|
|
1174
|
-
@init_listeners << Pitchfork::Const::DEFAULT_LISTEN
|
|
1175
1366
|
end
|
|
1176
1367
|
listeners.each { |addr| listen(addr) }
|
|
1368
|
+
|
|
1369
|
+
if @inherited_listeners
|
|
1370
|
+
@inherited_listeners.each_value { |sockets| sockets.each(&:close) }
|
|
1371
|
+
@inherited_listeners.clear
|
|
1372
|
+
@inherited_listeners = nil
|
|
1373
|
+
end
|
|
1374
|
+
|
|
1177
1375
|
raise ArgumentError, "no listeners" if LISTENERS.empty?
|
|
1178
1376
|
end
|
|
1179
1377
|
|