pitchfork 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pitchfork might be problematic. Click here for more details.

@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "rack"
4
- if Rack::VERSION.first >= 3
4
+ if defined?(Rack::RELEASE) && Rack::RELEASE > "3"
5
5
  require "rack/constants"
6
6
  require "rack/utils"
7
7
  end
@@ -27,7 +27,9 @@ module Pitchfork
27
27
 
28
28
  # Default settings for Pitchfork
29
29
  DEFAULTS = {
30
- :timeout => 20,
30
+ :soft_timeout => 20,
31
+ :cleanup_timeout => 2,
32
+ :timeout => 22,
31
33
  :logger => Logger.new($stderr),
32
34
  :worker_processes => 1,
33
35
  :after_worker_fork => lambda { |server, worker|
@@ -53,6 +55,8 @@ module Pitchfork
53
55
  :after_worker_ready => lambda { |server, worker|
54
56
  server.logger.info("worker=#{worker.nr} gen=#{worker.generation} ready")
55
57
  },
58
+ :after_worker_timeout => nil,
59
+ :after_request_complete => nil,
56
60
  :early_hints => false,
57
61
  :refork_condition => nil,
58
62
  :check_client_connection => false,
@@ -131,15 +135,22 @@ module Pitchfork
131
135
  set_hook(:after_worker_ready, block_given? ? block : args[0])
132
136
  end
133
137
 
138
+ def after_worker_timeout(*args, &block)
139
+ set_hook(:after_worker_timeout, block_given? ? block : args[0], 3)
140
+ end
141
+
134
142
  def after_worker_exit(*args, &block)
135
143
  set_hook(:after_worker_exit, block_given? ? block : args[0], 3)
136
144
  end
137
145
 
138
- def timeout(seconds)
139
- set_int(:timeout, seconds, 3)
140
- # POSIX says 31 days is the smallest allowed maximum timeout for select()
141
- max = 30 * 60 * 60 * 24
142
- set[:timeout] = seconds > max ? max : seconds
146
+ def after_request_complete(*args, &block)
147
+ set_hook(:after_request_complete, block_given? ? block : args[0])
148
+ end
149
+
150
+ def timeout(seconds, cleanup: 2)
151
+ soft_timeout = set_int(:soft_timeout, seconds, 3)
152
+ cleanup_timeout = set_int(:cleanup_timeout, cleanup, 2)
153
+ set_int(:timeout, soft_timeout + cleanup_timeout, 5)
143
154
  end
144
155
 
145
156
  def worker_processes(nr)
@@ -1,6 +1,7 @@
1
1
  # -*- encoding: binary -*-
2
2
  require 'pitchfork/pitchfork_http'
3
3
  require 'pitchfork/flock'
4
+ require 'pitchfork/soft_timeout'
4
5
 
5
6
  module Pitchfork
6
7
  # This is the process manager of Pitchfork. This manages worker
@@ -8,13 +9,75 @@ module Pitchfork
8
9
  # Listener sockets are started in the master process and shared with
9
10
  # forked worker children.
10
11
  class HttpServer
12
+ class TimeoutHandler
13
+ class Info
14
+ attr_reader :thread, :rack_env
15
+
16
+ def initialize(thread, rack_env)
17
+ @thread = thread
18
+ @rack_env = rack_env
19
+ end
20
+
21
+ def copy_thread_variables!
22
+ current_thread = Thread.current
23
+ @thread.keys.each do |key|
24
+ current_thread[key] = @thread[key]
25
+ end
26
+ @thread.thread_variables.each do |variable|
27
+ current_thread.thread_variable_set(variable, @thread.thread_variable_get(variable))
28
+ end
29
+ end
30
+ end
31
+
32
+ attr_writer :rack_env, :timeout_request # :nodoc:
33
+
34
+ def initialize(server, worker, callback) # :nodoc:
35
+ @server = server
36
+ @worker = worker
37
+ @callback = callback
38
+ @rack_env = nil
39
+ @timeout_request = nil
40
+ end
41
+
42
+ def inspect
43
+ "#<Pitchfork::HttpServer::TimeoutHandler##{object_id}>"
44
+ end
45
+
46
+ def call(original_thread) # :nodoc:
47
+ @server.logger.error("worker=#{@worker.nr} pid=#{@worker.pid} timed out, exiting")
48
+ if @callback
49
+ @callback.call(@server, @worker, Info.new(original_thread, @rack_env))
50
+ end
51
+ rescue => error
52
+ Pitchfork.log_error(@server.logger, "after_worker_timeout error", error)
53
+ ensure
54
+ exit
55
+ end
56
+
57
+ def finished # :nodoc:
58
+ @timeout_request.finished
59
+ end
60
+
61
+ def deadline
62
+ @timeout_request.deadline
63
+ end
64
+
65
+ def extend_deadline(extra_time)
66
+ extra_time = Integer(extra_time)
67
+ @worker.deadline += extra_time
68
+ @timeout_request.extend_deadline(extra_time)
69
+ self
70
+ end
71
+ end
72
+
11
73
  # :stopdoc:
12
- attr_accessor :app, :timeout, :worker_processes,
74
+ attr_accessor :app, :timeout, :soft_timeout, :cleanup_timeout, :worker_processes,
13
75
  :after_worker_fork, :after_mold_fork,
14
76
  :listener_opts, :children,
15
77
  :orig_app, :config, :ready_pipe,
16
78
  :default_middleware, :early_hints
17
- attr_writer :after_worker_exit, :after_worker_ready, :refork_condition
79
+ attr_writer :after_worker_exit, :after_worker_ready, :after_request_complete, :refork_condition,
80
+ :after_worker_timeout
18
81
 
19
82
  attr_reader :logger
20
83
  include Pitchfork::SocketHelper
@@ -376,7 +439,7 @@ module Pitchfork
376
439
  if worker
377
440
  @after_worker_exit.call(self, worker, status)
378
441
  else
379
- logger.error("reaped unknown subprocess #{status.inspect}")
442
+ logger.info("reaped unknown subprocess #{status.inspect}")
380
443
  end
381
444
  rescue Errno::ECHILD
382
445
  break
@@ -394,25 +457,30 @@ module Pitchfork
394
457
 
395
458
  # forcibly terminate all workers that haven't checked in in timeout seconds. The timeout is implemented using an unlinked File
396
459
  def murder_lazy_workers
397
- next_sleep = @timeout - 1
398
460
  now = Pitchfork.time_now(true)
461
+ next_sleep = @timeout - 1
462
+
399
463
  @children.workers.each do |worker|
400
- tick = worker.tick
401
- 0 == tick and next # skip workers that haven't processed any clients
402
- diff = now - tick
403
- tmp = @timeout - diff
404
- if tmp >= 0
405
- next_sleep > tmp and next_sleep = tmp
464
+ deadline = worker.deadline
465
+ if 0 == deadline # worker is idle
406
466
  next
467
+ elsif deadline > now # worker still has time
468
+ time_left = deadline - now
469
+ if time_left < next_sleep
470
+ next_sleep = time_left
471
+ end
472
+ next
473
+ else # worker is out of time
474
+ next_sleep = 0
475
+ if worker.mold?
476
+ logger.error "mold pid=#{worker.pid} deadline=#{deadline} timed out, killing"
477
+ else
478
+ logger.error "worker=#{worker.nr} pid=#{worker.pid} deadline=#{deadline} timed out, killing"
479
+ end
480
+ kill_worker(:KILL, worker.pid) # take no prisoners for hard timeout violations
407
481
  end
408
- next_sleep = 0
409
- if worker.mold?
410
- logger.error "mold pid=#{worker.pid} timeout (#{diff}s > #{@timeout}s), killing"
411
- else
412
- logger.error "worker=#{worker.nr} pid=#{worker.pid} timeout (#{diff}s > #{@timeout}s), killing"
413
- end
414
- kill_worker(:KILL, worker.pid) # take no prisoners for timeout violations
415
482
  end
483
+
416
484
  next_sleep <= 0 ? 1 : next_sleep
417
485
  end
418
486
 
@@ -577,9 +645,12 @@ module Pitchfork
577
645
 
578
646
  # once a client is accepted, it is processed in its entirety here
579
647
  # in 3 easy steps: read request, call app, write app response
580
- def process_client(client)
648
+ def process_client(client, timeout_handler)
649
+ env = nil
581
650
  @request = Pitchfork::HttpParser.new
582
651
  env = @request.read(client)
652
+ timeout_handler.rack_env = env
653
+ env["pitchfork.timeout"] = timeout_handler
583
654
 
584
655
  if early_hints
585
656
  env["rack.early_hints"] = lambda do |headers|
@@ -616,6 +687,7 @@ module Pitchfork
616
687
  handle_error(client, e)
617
688
  ensure
618
689
  env["rack.after_reply"].each(&:call) if env
690
+ timeout_handler.finished
619
691
  end
620
692
 
621
693
  def nuke_listeners!(readers)
@@ -684,7 +756,7 @@ module Pitchfork
684
756
 
685
757
  while readers[0]
686
758
  begin
687
- worker.tick = Pitchfork.time_now(true)
759
+ worker.update_deadline(@timeout)
688
760
  while sock = ready.shift
689
761
  # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
690
762
  # but that will return false
@@ -697,15 +769,16 @@ module Pitchfork
697
769
  when Message
698
770
  worker.update(client)
699
771
  else
700
- process_client(client)
772
+ process_client(client, prepare_timeout(worker))
773
+ @after_request_complete&.call(self, worker)
701
774
  worker.increment_requests_count
702
775
  end
703
- worker.tick = Pitchfork.time_now(true)
776
+ worker.update_deadline(@timeout)
704
777
  end
705
778
  end
706
779
 
707
- # timeout so we can .tick and keep parent from SIGKILL-ing us
708
- worker.tick = Pitchfork.time_now(true)
780
+ # timeout so we can update .deadline and keep parent from SIGKILL-ing us
781
+ worker.update_deadline(@timeout)
709
782
 
710
783
  if @refork_condition && !worker.outdated?
711
784
  if @refork_condition.met?(worker, logger)
@@ -754,7 +827,7 @@ module Pitchfork
754
827
 
755
828
  while readers[0]
756
829
  begin
757
- mold.tick = Pitchfork.time_now(true)
830
+ mold.update_deadline(@timeout)
758
831
  while sock = ready.shift
759
832
  # Pitchfork::Worker#accept_nonblock is not like accept(2) at all,
760
833
  # but that will return false
@@ -774,7 +847,7 @@ module Pitchfork
774
847
  end
775
848
 
776
849
  # timeout so we can .tick and keep parent from SIGKILL-ing us
777
- mold.tick = Pitchfork.time_now(true)
850
+ mold.update_deadline(@timeout)
778
851
  waiter.get_readers(ready, readers, @timeout * 500) # to milliseconds, but halved
779
852
  rescue => e
780
853
  Pitchfork.log_error(@logger, "mold loop error", e) if readers[0]
@@ -832,5 +905,11 @@ module Pitchfork
832
905
  listeners.each { |addr| listen(addr) }
833
906
  raise ArgumentError, "no listeners" if LISTENERS.empty?
834
907
  end
908
+
909
+ def prepare_timeout(worker)
910
+ handler = TimeoutHandler.new(self, worker, @after_worker_timeout)
911
+ handler.timeout_request = SoftTimeout.request(@soft_timeout, handler)
912
+ handler
913
+ end
835
914
  end
836
915
  end
@@ -0,0 +1,113 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pitchfork
4
+ # :stopdoc:
5
+ module SoftTimeout
6
+ extend self
7
+
8
+ CONDVAR = ConditionVariable.new
9
+ QUEUE = Queue.new
10
+ QUEUE_MUTEX = Mutex.new
11
+ TIMEOUT_THREAD_MUTEX = Mutex.new
12
+ @timeout_thread = nil
13
+ private_constant :CONDVAR, :QUEUE, :QUEUE_MUTEX, :TIMEOUT_THREAD_MUTEX
14
+
15
+ class Request
16
+ attr_reader :deadline, :thread
17
+
18
+ def initialize(thread, timeout, block)
19
+ @thread = thread
20
+ @deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
21
+ @block = block
22
+
23
+ @mutex = Mutex.new
24
+ @done = false # protected by @mutex
25
+ end
26
+
27
+ def extend_deadline(timeout)
28
+ @deadline += timeout
29
+ QUEUE_MUTEX.synchronize do
30
+ CONDVAR.signal
31
+ end
32
+ self
33
+ end
34
+
35
+ def done?
36
+ @mutex.synchronize do
37
+ @done
38
+ end
39
+ end
40
+
41
+ def expired?(now)
42
+ now >= @deadline
43
+ end
44
+
45
+ def interrupt
46
+ @mutex.synchronize do
47
+ unless @done
48
+ begin
49
+ @block.call(@thread)
50
+ ensure
51
+ @done = true
52
+ end
53
+ end
54
+ end
55
+ end
56
+
57
+ def finished
58
+ @mutex.synchronize do
59
+ @done = true
60
+ end
61
+ end
62
+ end
63
+
64
+ def request(sec, callback)
65
+ ensure_timeout_thread_created
66
+ request = Request.new(Thread.current, sec, callback)
67
+ QUEUE_MUTEX.synchronize do
68
+ QUEUE << request
69
+ CONDVAR.signal
70
+ end
71
+ request
72
+ end
73
+
74
+ private
75
+
76
+ def create_timeout_thread
77
+ watcher = Thread.new do
78
+ requests = []
79
+ while true
80
+ until QUEUE.empty? and !requests.empty? # wait to have at least one request
81
+ req = QUEUE.pop
82
+ requests << req unless req.done?
83
+ end
84
+ closest_deadline = requests.min_by(&:deadline).deadline
85
+
86
+ now = 0.0
87
+ QUEUE_MUTEX.synchronize do
88
+ while (now = Process.clock_gettime(Process::CLOCK_MONOTONIC)) < closest_deadline and QUEUE.empty?
89
+ CONDVAR.wait(QUEUE_MUTEX, closest_deadline - now)
90
+ end
91
+ end
92
+
93
+ requests.each do |req|
94
+ req.interrupt if req.expired?(now)
95
+ end
96
+ requests.reject!(&:done?)
97
+ end
98
+ end
99
+ watcher.name = "Pitchfork::Timeout"
100
+ watcher
101
+ end
102
+
103
+ def ensure_timeout_thread_created
104
+ unless @timeout_thread and @timeout_thread.alive?
105
+ TIMEOUT_THREAD_MUTEX.synchronize do
106
+ unless @timeout_thread and @timeout_thread.alive?
107
+ @timeout_thread = create_timeout_thread
108
+ end
109
+ end
110
+ end
111
+ end
112
+ end
113
+ end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pitchfork
4
- VERSION = "0.4.0"
4
+ VERSION = "0.5.0"
5
5
  module Const
6
6
  UNICORN_VERSION = '6.1.0'
7
7
  end
@@ -93,7 +93,7 @@ module Pitchfork
93
93
  @mold = true
94
94
  @nr = nil
95
95
  @drop_offset = 0
96
- @tick_drop = MOLD_DROP
96
+ @deadline_drop = MOLD_DROP
97
97
  self
98
98
  end
99
99
 
@@ -167,21 +167,25 @@ module Pitchfork
167
167
  super || (!@nr.nil? && @nr == other)
168
168
  end
169
169
 
170
+ def update_deadline(timeout)
171
+ self.deadline = Pitchfork.time_now(true) + timeout
172
+ end
173
+
170
174
  # called in the worker process
171
- def tick=(value) # :nodoc:
175
+ def deadline=(value) # :nodoc:
172
176
  if mold?
173
177
  MOLD_DROP[0] = value
174
178
  else
175
- @tick_drop[@drop_offset] = value
179
+ @deadline_drop[@drop_offset] = value
176
180
  end
177
181
  end
178
182
 
179
183
  # called in the master process
180
- def tick # :nodoc:
184
+ def deadline # :nodoc:
181
185
  if mold?
182
186
  MOLD_DROP[0]
183
187
  else
184
- @tick_drop[@drop_offset]
188
+ @deadline_drop[@drop_offset]
185
189
  end
186
190
  end
187
191
 
@@ -221,7 +225,7 @@ module Pitchfork
221
225
  else
222
226
  success = true
223
227
  end
224
- rescue Errno::EPIPE, Errno::ECONNRESET
228
+ rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNREFUSED
225
229
  # worker will be reaped soon
226
230
  end
227
231
  success
@@ -248,8 +252,8 @@ module Pitchfork
248
252
  def build_raindrops(drop_nr)
249
253
  drop_index = drop_nr / PER_DROP
250
254
  @drop_offset = drop_nr % PER_DROP
251
- @tick_drop = TICK_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
252
- @tick_drop[@drop_offset] = 0
255
+ @deadline_drop = TICK_DROPS[drop_index] ||= Raindrops.new(PER_DROP)
256
+ @deadline_drop[@drop_offset] = 0
253
257
  end
254
258
  end
255
259
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pitchfork
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jean Boussier
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-06-13 00:00:00.000000000 Z
11
+ date: 2023-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: raindrops
@@ -108,6 +108,7 @@ files:
108
108
  - lib/pitchfork/refork_condition.rb
109
109
  - lib/pitchfork/select_waiter.rb
110
110
  - lib/pitchfork/socket_helper.rb
111
+ - lib/pitchfork/soft_timeout.rb
111
112
  - lib/pitchfork/stream_input.rb
112
113
  - lib/pitchfork/tee_input.rb
113
114
  - lib/pitchfork/tmpio.rb