pitchfork 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pitchfork might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/Gemfile.lock +2 -2
- data/docs/Application_Timeouts.md +1 -1
- data/docs/CONFIGURATION.md +77 -11
- data/ext/pitchfork_http/pitchfork_http.c +166 -166
- data/lib/pitchfork/chunked.rb +1 -1
- data/lib/pitchfork/configurator.rb +17 -6
- data/lib/pitchfork/http_server.rb +104 -25
- data/lib/pitchfork/soft_timeout.rb +113 -0
- data/lib/pitchfork/version.rb +1 -1
- data/lib/pitchfork/worker.rb +12 -8
- metadata +3 -2
data/lib/pitchfork/chunked.rb
CHANGED
@@ -27,7 +27,9 @@ module Pitchfork
|
|
27
27
|
|
28
28
|
# Default settings for Pitchfork
|
29
29
|
DEFAULTS = {
|
30
|
-
:
|
30
|
+
:soft_timeout => 20,
|
31
|
+
:cleanup_timeout => 2,
|
32
|
+
:timeout => 22,
|
31
33
|
:logger => Logger.new($stderr),
|
32
34
|
:worker_processes => 1,
|
33
35
|
:after_worker_fork => lambda { |server, worker|
|
@@ -53,6 +55,8 @@ module Pitchfork
|
|
53
55
|
:after_worker_ready => lambda { |server, worker|
|
54
56
|
server.logger.info("worker=#{worker.nr} gen=#{worker.generation} ready")
|
55
57
|
},
|
58
|
+
:after_worker_timeout => nil,
|
59
|
+
:after_request_complete => nil,
|
56
60
|
:early_hints => false,
|
57
61
|
:refork_condition => nil,
|
58
62
|
:check_client_connection => false,
|
@@ -131,15 +135,22 @@ module Pitchfork
|
|
131
135
|
set_hook(:after_worker_ready, block_given? ? block : args[0])
|
132
136
|
end
|
133
137
|
|
138
|
+
def after_worker_timeout(*args, &block)
|
139
|
+
set_hook(:after_worker_timeout, block_given? ? block : args[0], 3)
|
140
|
+
end
|
141
|
+
|
134
142
|
def after_worker_exit(*args, &block)
|
135
143
|
set_hook(:after_worker_exit, block_given? ? block : args[0], 3)
|
136
144
|
end
|
137
145
|
|
138
|
-
def
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
146
|
+
def after_request_complete(*args, &block)
|
147
|
+
set_hook(:after_request_complete, block_given? ? block : args[0])
|
148
|
+
end
|
149
|
+
|
150
|
+
def timeout(seconds, cleanup: 2)
|
151
|
+
soft_timeout = set_int(:soft_timeout, seconds, 3)
|
152
|
+
cleanup_timeout = set_int(:cleanup_timeout, cleanup, 2)
|
153
|
+
set_int(:timeout, soft_timeout + cleanup_timeout, 5)
|
143
154
|
end
|
144
155
|
|
145
156
|
def worker_processes(nr)
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
2
|
require 'pitchfork/pitchfork_http'
|
3
3
|
require 'pitchfork/flock'
|
4
|
+
require 'pitchfork/soft_timeout'
|
4
5
|
|
5
6
|
module Pitchfork
|
6
7
|
# This is the process manager of Pitchfork. This manages worker
|
@@ -8,13 +9,75 @@ module Pitchfork
|
|
8
9
|
# Listener sockets are started in the master process and shared with
|
9
10
|
# forked worker children.
|
10
11
|
class HttpServer
|
12
|
+
class TimeoutHandler
|
13
|
+
class Info
|
14
|
+
attr_reader :thread, :rack_env
|
15
|
+
|
16
|
+
def initialize(thread, rack_env)
|
17
|
+
@thread = thread
|
18
|
+
@rack_env = rack_env
|
19
|
+
end
|
20
|
+
|
21
|
+
def copy_thread_variables!
|
22
|
+
current_thread = Thread.current
|
23
|
+
@thread.keys.each do |key|
|
24
|
+
current_thread[key] = @thread[key]
|
25
|
+
end
|
26
|
+
@thread.thread_variables.each do |variable|
|
27
|
+
current_thread.thread_variable_set(variable, @thread.thread_variable_get(variable))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
attr_writer :rack_env, :timeout_request # :nodoc:
|
33
|
+
|
34
|
+
def initialize(server, worker, callback) # :nodoc:
|
35
|
+
@server = server
|
36
|
+
@worker = worker
|
37
|
+
@callback = callback
|
38
|
+
@rack_env = nil
|
39
|
+
@timeout_request = nil
|
40
|
+
end
|
41
|
+
|
42
|
+
def inspect
|
43
|
+
"#<Pitchfork::HttpServer::TimeoutHandler##{object_id}>"
|
44
|
+
end
|
45
|
+
|
46
|
+
def call(original_thread) # :nodoc:
|
47
|
+
@server.logger.error("worker=#{@worker.nr} pid=#{@worker.pid} timed out, exiting")
|
48
|
+
if @callback
|
49
|
+
@callback.call(@server, @worker, Info.new(original_thread, @rack_env))
|
50
|
+
end
|
51
|
+
rescue => error
|
52
|
+
Pitchfork.log_error(@server.logger, "after_worker_timeout error", error)
|
53
|
+
ensure
|
54
|
+
exit
|
55
|
+
end
|
56
|
+
|
57
|
+
def finished # :nodoc:
|
58
|
+
@timeout_request.finished
|
59
|
+
end
|
60
|
+
|
61
|
+
def deadline
|
62
|
+
@timeout_request.deadline
|
63
|
+
end
|
64
|
+
|
65
|
+
def extend_deadline(extra_time)
|
66
|
+
extra_time = Integer(extra_time)
|
67
|
+
@worker.deadline += extra_time
|
68
|
+
@timeout_request.extend_deadline(extra_time)
|
69
|
+
self
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
11
73
|
# :stopdoc:
|
12
|
-
attr_accessor :app, :timeout, :worker_processes,
|
74
|
+
attr_accessor :app, :timeout, :soft_timeout, :cleanup_timeout, :worker_processes,
|
13
75
|
:after_worker_fork, :after_mold_fork,
|
14
76
|
:listener_opts, :children,
|
15
77
|
:orig_app, :config, :ready_pipe,
|
16
78
|
:default_middleware, :early_hints
|
17
|
-
attr_writer :after_worker_exit, :after_worker_ready, :refork_condition
|
79
|
+
attr_writer :after_worker_exit, :after_worker_ready, :after_request_complete, :refork_condition,
|
80
|
+
:after_worker_timeout
|
18
81
|
|
19
82
|
attr_reader :logger
|
20
83
|
include Pitchfork::SocketHelper
|
@@ -376,7 +439,7 @@ module Pitchfork
|
|
376
439
|
if worker
|
377
440
|
@after_worker_exit.call(self, worker, status)
|
378
441
|
else
|
379
|
-
logger.
|
442
|
+
logger.info("reaped unknown subprocess #{status.inspect}")
|
380
443
|
end
|
381
444
|
rescue Errno::ECHILD
|
382
445
|
break
|
@@ -394,25 +457,30 @@ module Pitchfork
|
|
394
457
|
|
395
458
|
# forcibly terminate all workers that haven't checked in in timeout seconds. The timeout is implemented using an unlinked File
|
396
459
|
def murder_lazy_workers
|
397
|
-
next_sleep = @timeout - 1
|
398
460
|
now = Pitchfork.time_now(true)
|
461
|
+
next_sleep = @timeout - 1
|
462
|
+
|
399
463
|
@children.workers.each do |worker|
|
400
|
-
|
401
|
-
0 ==
|
402
|
-
diff = now - tick
|
403
|
-
tmp = @timeout - diff
|
404
|
-
if tmp >= 0
|
405
|
-
next_sleep > tmp and next_sleep = tmp
|
464
|
+
deadline = worker.deadline
|
465
|
+
if 0 == deadline # worker is idle
|
406
466
|
next
|
467
|
+
elsif deadline > now # worker still has time
|
468
|
+
time_left = deadline - now
|
469
|
+
if time_left < next_sleep
|
470
|
+
next_sleep = time_left
|
471
|
+
end
|
472
|
+
next
|
473
|
+
else # worker is out of time
|
474
|
+
next_sleep = 0
|
475
|
+
if worker.mold?
|
476
|
+
logger.error "mold pid=#{worker.pid} deadline=#{deadline} timed out, killing"
|
477
|
+
else
|
478
|
+
logger.error "worker=#{worker.nr} pid=#{worker.pid} deadline=#{deadline} timed out, killing"
|
479
|
+
end
|
480
|
+
kill_worker(:KILL, worker.pid) # take no prisoners for hard timeout violations
|
407
481
|
end
|
408
|
-
next_sleep = 0
|
409
|
-
if worker.mold?
|
410
|
-
logger.error "mold pid=#{worker.pid} timeout (#{diff}s > #{@timeout}s), killing"
|
411
|
-
else
|
412
|
-
logger.error "worker=#{worker.nr} pid=#{worker.pid} timeout (#{diff}s > #{@timeout}s), killing"
|
413
|
-
end
|
414
|
-
kill_worker(:KILL, worker.pid) # take no prisoners for timeout violations
|
415
482
|
end
|
483
|
+
|
416
484
|
next_sleep <= 0 ? 1 : next_sleep
|
417
485
|
end
|
418
486
|
|
@@ -577,9 +645,12 @@ module Pitchfork
|
|
577
645
|
|
578
646
|
# once a client is accepted, it is processed in its entirety here
|
579
647
|
# in 3 easy steps: read request, call app, write app response
|
580
|
-
def process_client(client)
|
648
|
+
def process_client(client, timeout_handler)
|
649
|
+
env = nil
|
581
650
|
@request = Pitchfork::HttpParser.new
|
582
651
|
env = @request.read(client)
|
652
|
+
timeout_handler.rack_env = env
|
653
|
+
env["pitchfork.timeout"] = timeout_handler
|
583
654
|
|
584
655
|
if early_hints
|
585
656
|
env["rack.early_hints"] = lambda do |headers|
|
@@ -616,6 +687,7 @@ module Pitchfork
|
|
616
687
|
handle_error(client, e)
|
617
688
|
ensure
|
618
689
|
env["rack.after_reply"].each(&:call) if env
|
690
|
+
timeout_handler.finished
|
619
691
|
end
|
620
692
|
|
621
693
|
def nuke_listeners!(readers)
|
@@ -684,7 +756,7 @@ module Pitchfork
|
|
684
756
|
|
685
757
|
while readers[0]
|
686
758
|
begin
|
687
|
-
worker.
|
759
|
+
worker.update_deadline(@timeout)
|
688
760
|
while sock = ready.shift
|
689
761
|
# Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
|
690
762
|
# but that will return false
|
@@ -697,15 +769,16 @@ module Pitchfork
|
|
697
769
|
when Message
|
698
770
|
worker.update(client)
|
699
771
|
else
|
700
|
-
process_client(client)
|
772
|
+
process_client(client, prepare_timeout(worker))
|
773
|
+
@after_request_complete&.call(self, worker)
|
701
774
|
worker.increment_requests_count
|
702
775
|
end
|
703
|
-
worker.
|
776
|
+
worker.update_deadline(@timeout)
|
704
777
|
end
|
705
778
|
end
|
706
779
|
|
707
|
-
# timeout so we can .
|
708
|
-
worker.
|
780
|
+
# timeout so we can update .deadline and keep parent from SIGKILL-ing us
|
781
|
+
worker.update_deadline(@timeout)
|
709
782
|
|
710
783
|
if @refork_condition && !worker.outdated?
|
711
784
|
if @refork_condition.met?(worker, logger)
|
@@ -754,7 +827,7 @@ module Pitchfork
|
|
754
827
|
|
755
828
|
while readers[0]
|
756
829
|
begin
|
757
|
-
mold.
|
830
|
+
mold.update_deadline(@timeout)
|
758
831
|
while sock = ready.shift
|
759
832
|
# Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
|
760
833
|
# but that will return false
|
@@ -774,7 +847,7 @@ module Pitchfork
|
|
774
847
|
end
|
775
848
|
|
776
849
|
# timeout so we can .tick and keep parent from SIGKILL-ing us
|
777
|
-
mold.
|
850
|
+
mold.update_deadline(@timeout)
|
778
851
|
waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
|
779
852
|
rescue => e
|
780
853
|
Pitchfork.log_error(@logger, "mold loop error", e) if readers[0]
|
@@ -832,5 +905,11 @@ module Pitchfork
|
|
832
905
|
listeners.each { |addr| listen(addr) }
|
833
906
|
raise ArgumentError, "no listeners" if LISTENERS.empty?
|
834
907
|
end
|
908
|
+
|
909
|
+
def prepare_timeout(worker)
|
910
|
+
handler = TimeoutHandler.new(self, worker, @after_worker_timeout)
|
911
|
+
handler.timeout_request = SoftTimeout.request(@soft_timeout, handler)
|
912
|
+
handler
|
913
|
+
end
|
835
914
|
end
|
836
915
|
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Pitchfork
|
4
|
+
# :stopdoc:
|
5
|
+
module SoftTimeout
|
6
|
+
extend self
|
7
|
+
|
8
|
+
CONDVAR = ConditionVariable.new
|
9
|
+
QUEUE = Queue.new
|
10
|
+
QUEUE_MUTEX = Mutex.new
|
11
|
+
TIMEOUT_THREAD_MUTEX = Mutex.new
|
12
|
+
@timeout_thread = nil
|
13
|
+
private_constant :CONDVAR, :QUEUE, :QUEUE_MUTEX, :TIMEOUT_THREAD_MUTEX
|
14
|
+
|
15
|
+
class Request
|
16
|
+
attr_reader :deadline, :thread
|
17
|
+
|
18
|
+
def initialize(thread, timeout, block)
|
19
|
+
@thread = thread
|
20
|
+
@deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
|
21
|
+
@block = block
|
22
|
+
|
23
|
+
@mutex = Mutex.new
|
24
|
+
@done = false # protected by @mutex
|
25
|
+
end
|
26
|
+
|
27
|
+
def extend_deadline(timeout)
|
28
|
+
@deadline += timeout
|
29
|
+
QUEUE_MUTEX.synchronize do
|
30
|
+
CONDVAR.signal
|
31
|
+
end
|
32
|
+
self
|
33
|
+
end
|
34
|
+
|
35
|
+
def done?
|
36
|
+
@mutex.synchronize do
|
37
|
+
@done
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def expired?(now)
|
42
|
+
now >= @deadline
|
43
|
+
end
|
44
|
+
|
45
|
+
def interrupt
|
46
|
+
@mutex.synchronize do
|
47
|
+
unless @done
|
48
|
+
begin
|
49
|
+
@block.call(@thread)
|
50
|
+
ensure
|
51
|
+
@done = true
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def finished
|
58
|
+
@mutex.synchronize do
|
59
|
+
@done = true
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def request(sec, callback)
|
65
|
+
ensure_timeout_thread_created
|
66
|
+
request = Request.new(Thread.current, sec, callback)
|
67
|
+
QUEUE_MUTEX.synchronize do
|
68
|
+
QUEUE << request
|
69
|
+
CONDVAR.signal
|
70
|
+
end
|
71
|
+
request
|
72
|
+
end
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def create_timeout_thread
|
77
|
+
watcher = Thread.new do
|
78
|
+
requests = []
|
79
|
+
while true
|
80
|
+
until QUEUE.empty? and !requests.empty? # wait to have at least one request
|
81
|
+
req = QUEUE.pop
|
82
|
+
requests << req unless req.done?
|
83
|
+
end
|
84
|
+
closest_deadline = requests.min_by(&:deadline).deadline
|
85
|
+
|
86
|
+
now = 0.0
|
87
|
+
QUEUE_MUTEX.synchronize do
|
88
|
+
while (now = Process.clock_gettime(Process::CLOCK_MONOTONIC)) < closest_deadline and QUEUE.empty?
|
89
|
+
CONDVAR.wait(QUEUE_MUTEX, closest_deadline - now)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
requests.each do |req|
|
94
|
+
req.interrupt if req.expired?(now)
|
95
|
+
end
|
96
|
+
requests.reject!(&:done?)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
watcher.name = "Pitchfork::Timeout"
|
100
|
+
watcher
|
101
|
+
end
|
102
|
+
|
103
|
+
def ensure_timeout_thread_created
|
104
|
+
unless @timeout_thread and @timeout_thread.alive?
|
105
|
+
TIMEOUT_THREAD_MUTEX.synchronize do
|
106
|
+
unless @timeout_thread and @timeout_thread.alive?
|
107
|
+
@timeout_thread = create_timeout_thread
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
data/lib/pitchfork/version.rb
CHANGED
data/lib/pitchfork/worker.rb
CHANGED
@@ -93,7 +93,7 @@ module Pitchfork
|
|
93
93
|
@mold = true
|
94
94
|
@nr = nil
|
95
95
|
@drop_offset = 0
|
96
|
-
@
|
96
|
+
@deadline_drop = MOLD_DROP
|
97
97
|
self
|
98
98
|
end
|
99
99
|
|
@@ -167,21 +167,25 @@ module Pitchfork
|
|
167
167
|
super || (!@nr.nil? && @nr == other)
|
168
168
|
end
|
169
169
|
|
170
|
+
def update_deadline(timeout)
|
171
|
+
self.deadline = Pitchfork.time_now(true) + timeout
|
172
|
+
end
|
173
|
+
|
170
174
|
# called in the worker process
|
171
|
-
def
|
175
|
+
def deadline=(value) # :nodoc:
|
172
176
|
if mold?
|
173
177
|
MOLD_DROP[0] = value
|
174
178
|
else
|
175
|
-
@
|
179
|
+
@deadline_drop[@drop_offset] = value
|
176
180
|
end
|
177
181
|
end
|
178
182
|
|
179
183
|
# called in the master process
|
180
|
-
def
|
184
|
+
def deadline # :nodoc:
|
181
185
|
if mold?
|
182
186
|
MOLD_DROP[0]
|
183
187
|
else
|
184
|
-
@
|
188
|
+
@deadline_drop[@drop_offset]
|
185
189
|
end
|
186
190
|
end
|
187
191
|
|
@@ -221,7 +225,7 @@ module Pitchfork
|
|
221
225
|
else
|
222
226
|
success = true
|
223
227
|
end
|
224
|
-
rescue Errno::EPIPE, Errno::ECONNRESET
|
228
|
+
rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNREFUSED
|
225
229
|
# worker will be reaped soon
|
226
230
|
end
|
227
231
|
success
|
@@ -248,8 +252,8 @@ module Pitchfork
|
|
248
252
|
def build_raindrops(drop_nr)
|
249
253
|
drop_index = drop_nr / PER_DROP
|
250
254
|
@drop_offset = drop_nr % PER_DROP
|
251
|
-
@
|
252
|
-
@
|
255
|
+
@deadline_drop = TICK_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
|
256
|
+
@deadline_drop[@drop_offset] = 0
|
253
257
|
end
|
254
258
|
end
|
255
259
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pitchfork
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jean Boussier
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-06-
|
11
|
+
date: 2023-06-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: raindrops
|
@@ -108,6 +108,7 @@ files:
|
|
108
108
|
- lib/pitchfork/refork_condition.rb
|
109
109
|
- lib/pitchfork/select_waiter.rb
|
110
110
|
- lib/pitchfork/socket_helper.rb
|
111
|
+
- lib/pitchfork/soft_timeout.rb
|
111
112
|
- lib/pitchfork/stream_input.rb
|
112
113
|
- lib/pitchfork/tee_input.rb
|
113
114
|
- lib/pitchfork/tmpio.rb
|