pitchfork 0.18.2 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/actions.lock +40 -0
- data/.github/workflows/ci.yml +27 -10
- data/.github/workflows/release.yml +3 -3
- data/.ruby-version +1 -1
- data/CHANGELOG.md +9 -1
- data/Gemfile +1 -0
- data/Rakefile +15 -0
- data/benchmark/request_benchmark.rb +86 -0
- data/docs/CONFIGURATION.md +51 -14
- data/docs/MIGRATING_FROM_UNICORN.md +6 -0
- data/docs/SIGNALS.md +6 -0
- data/examples/pitchfork.conf.reload.rb +41 -0
- data/exe/pitchfork +3 -2
- data/ext/pitchfork_http/c_util.h +3 -8
- data/ext/pitchfork_http/child_subreaper.h +7 -1
- data/ext/pitchfork_http/epollexclusive.h +6 -3
- data/ext/pitchfork_http/extconf.rb +1 -0
- data/ext/pitchfork_http/global_variables.h +2 -0
- data/ext/pitchfork_http/memory_page.c +188 -25
- data/ext/pitchfork_http/pitchfork_http.c +425 -394
- data/ext/pitchfork_http/pitchfork_http.rl +54 -23
- data/ext/pitchfork_http/pitchfork_http_common.rl +1 -1
- data/lib/pitchfork/children.rb +74 -20
- data/lib/pitchfork/chunked.rb +9 -0
- data/lib/pitchfork/configurator.rb +25 -6
- data/lib/pitchfork/http_response.rb +35 -12
- data/lib/pitchfork/http_server.rb +384 -189
- data/lib/pitchfork/info.rb +8 -0
- data/lib/pitchfork/listeners.rb +52 -0
- data/lib/pitchfork/message.rb +58 -8
- data/lib/pitchfork/shared_memory.rb +64 -13
- data/lib/pitchfork/socket_helper.rb +6 -2
- data/lib/pitchfork/stream_input.rb +12 -9
- data/lib/pitchfork/version.rb +1 -1
- data/lib/pitchfork/worker.rb +124 -64
- data/lib/pitchfork.rb +13 -1
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1c65128a941c7398384d6c3cf7e9351b79a1877d872bead7a9586334479f0862
|
|
4
|
+
data.tar.gz: d7db35a780e04e2d9178958b0f3626db0790316b947c8b249796fc253bd21661
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b77a9bedb4a632ea0f3462a99b4b890f24d06e89fe14c6e501e39b6d52965027de778745a01b1f9a282be86b7af10d27abe5c23f95c5eeca035fbe43bae5e9d8
|
|
7
|
+
data.tar.gz: 37aa9507774d6eda9e53ff68a050eecb939e3f9d45206fe40a5ce2ca8d47e521405d2a8cb21da8d83ebf42e39d4a845f4ea0d16426ebacbe8f97bf4714ac12f8
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# This file is machine-generated by `gh actions-lock`.
|
|
2
|
+
# Do not edit by hand; run `gh actions-lock` to update.
|
|
3
|
+
# Docs: https://gh.io/actions-lockfile
|
|
4
|
+
version: 'v0.0.3'
|
|
5
|
+
workflows:
|
|
6
|
+
'.github/workflows/ci.yml':
|
|
7
|
+
- 'actions/checkout@v4.4.0'
|
|
8
|
+
- 'ruby/setup-ruby@v1.336.0'
|
|
9
|
+
'.github/workflows/release.yml':
|
|
10
|
+
- 'actions/checkout@v6.1.0'
|
|
11
|
+
- 'ruby/setup-ruby@v1.336.0'
|
|
12
|
+
- 'rubygems/release-gem@v1.4.1'
|
|
13
|
+
dependencies:
|
|
14
|
+
'actions/checkout@v4.4.0':
|
|
15
|
+
ref: 'v4.4.0'
|
|
16
|
+
commit: 'sha1-11d5960a326750d5838078e36cf38b85af677262'
|
|
17
|
+
owner_id: 44036562
|
|
18
|
+
repo_id: 197814629
|
|
19
|
+
'actions/checkout@v6.1.0':
|
|
20
|
+
ref: 'v6.1.0'
|
|
21
|
+
commit: 'sha1-d23441a48e516b6c34aea4fa41551a30e30af803'
|
|
22
|
+
owner_id: 44036562
|
|
23
|
+
repo_id: 197814629
|
|
24
|
+
'ruby/setup-ruby@v1.336.0':
|
|
25
|
+
ref: 'v1.336.0'
|
|
26
|
+
commit: 'sha1-de492cf36cbab201753d720ce4f4e336cd688c94'
|
|
27
|
+
owner_id: 210414
|
|
28
|
+
repo_id: 231208785
|
|
29
|
+
'rubygems/configure-rubygems-credentials@dc5a8d8553e6ee01fc26761a49e99e733d17954a':
|
|
30
|
+
ref: 'v2.1.0'
|
|
31
|
+
commit: 'sha1-dc5a8d8553e6ee01fc26761a49e99e733d17954a'
|
|
32
|
+
owner_id: 208761
|
|
33
|
+
repo_id: 631633510
|
|
34
|
+
'rubygems/release-gem@v1.4.1':
|
|
35
|
+
ref: 'v1.4.1'
|
|
36
|
+
commit: 'sha1-7f9650160c1a4e7989fdc9855807bdbd421d8b6b'
|
|
37
|
+
owner_id: 208761
|
|
38
|
+
repo_id: 731435765
|
|
39
|
+
uses:
|
|
40
|
+
- 'rubygems/configure-rubygems-credentials@dc5a8d8553e6ee01fc26761a49e99e733d17954a'
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -10,41 +10,58 @@ on:
|
|
|
10
10
|
|
|
11
11
|
jobs:
|
|
12
12
|
ruby:
|
|
13
|
-
name: Ruby ${{ matrix.ruby }} / Rack ${{ matrix.rack }} ${{ matrix.rubyopt }}
|
|
13
|
+
name: Ruby ${{ matrix.ruby }} / Rack ${{ matrix.rack }} ${{ matrix.rubyopt }} / ${{ matrix.os }}
|
|
14
14
|
timeout-minutes: 15
|
|
15
15
|
strategy:
|
|
16
16
|
fail-fast: false
|
|
17
17
|
matrix:
|
|
18
18
|
os: ["ubuntu-latest"]
|
|
19
|
-
ruby: ["ruby-head", "
|
|
19
|
+
ruby: ["ruby-head", "4.0", "3.4", "3.3", "3.2"]
|
|
20
20
|
rubyopt: [""]
|
|
21
21
|
rack: ["~> 3.2"]
|
|
22
22
|
include:
|
|
23
|
-
-
|
|
23
|
+
- os: "ubuntu-latest"
|
|
24
|
+
ruby: "4.0"
|
|
24
25
|
rubyopt: "--enable-frozen-string-literal"
|
|
25
26
|
rack: "~> 3.2"
|
|
26
|
-
-
|
|
27
|
+
- os: "ubuntu-latest"
|
|
28
|
+
ruby: "4.0"
|
|
27
29
|
rack: "~> 3.1.0"
|
|
28
|
-
-
|
|
30
|
+
- os: "ubuntu-latest"
|
|
31
|
+
ruby: "4.0"
|
|
29
32
|
rack: "~> 3.0.0"
|
|
30
|
-
-
|
|
33
|
+
- os: "ubuntu-latest"
|
|
34
|
+
ruby: "3.4"
|
|
31
35
|
rack: "~> 2.0"
|
|
36
|
+
# setup-ruby only has prebuilt binaries of these up to ubuntu-24.04
|
|
37
|
+
- os: "ubuntu-24.04"
|
|
38
|
+
ruby: "3.1"
|
|
39
|
+
rack: "~> 3.2"
|
|
40
|
+
- os: "ubuntu-24.04"
|
|
41
|
+
ruby: "3.0"
|
|
42
|
+
rack: "~> 3.2"
|
|
43
|
+
- os: "ubuntu-24.04"
|
|
44
|
+
ruby: "2.7"
|
|
45
|
+
rack: "~> 3.2"
|
|
46
|
+
- os: "ubuntu-24.04"
|
|
47
|
+
ruby: "2.6"
|
|
48
|
+
rack: "~> 3.2"
|
|
32
49
|
env:
|
|
33
50
|
RACK_VERSION: "${{ matrix.rack }}"
|
|
34
51
|
RUBYOPT: "${{ matrix.rubyopt }}"
|
|
35
|
-
runs-on:
|
|
52
|
+
runs-on: ${{ matrix.os }}
|
|
36
53
|
steps:
|
|
37
54
|
- name: Check out code
|
|
38
|
-
uses: actions/checkout@v4
|
|
55
|
+
uses: actions/checkout@v4.4.0
|
|
39
56
|
|
|
40
57
|
- name: Set up Ruby ${{ matrix.ruby }}
|
|
41
|
-
uses: ruby/setup-ruby@v1
|
|
58
|
+
uses: ruby/setup-ruby@v1.336.0
|
|
42
59
|
with:
|
|
43
60
|
ruby-version: ${{ matrix.ruby }}
|
|
44
61
|
bundler-cache: true
|
|
45
62
|
|
|
46
63
|
- name: Install packages
|
|
47
|
-
run: sudo apt-get install -y ragel socat netcat
|
|
64
|
+
run: sudo apt-get install -y ragel socat netcat-openbsd
|
|
48
65
|
|
|
49
66
|
- name: Tests Rack ${{ matrix.rack }} ${{ matrix.rubyopt }}
|
|
50
67
|
run: bundle exec rake
|
|
@@ -18,12 +18,12 @@ jobs:
|
|
|
18
18
|
environment: release
|
|
19
19
|
|
|
20
20
|
steps:
|
|
21
|
-
- uses: actions/checkout@v6
|
|
21
|
+
- uses: actions/checkout@v6.1.0
|
|
22
22
|
with:
|
|
23
23
|
persist-credentials: false
|
|
24
24
|
|
|
25
|
-
- uses: ruby/setup-ruby@v1
|
|
25
|
+
- uses: ruby/setup-ruby@v1.336.0
|
|
26
26
|
with:
|
|
27
27
|
bundler-cache: true
|
|
28
28
|
|
|
29
|
-
- uses: rubygems/release-gem@v1
|
|
29
|
+
- uses: rubygems/release-gem@v1.4.1
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
4.0.7
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Unreleased
|
|
2
2
|
|
|
3
|
+
- [GHSA-gc8c-7rmr-hp64](https://github.com/Shopify/pitchfork/security/advisories/GHSA-gc8c-7rmr-hp64) Prevent header confusion: reject `Content_Length` and `Transfer_Encoding`, and ignore other headers with underscores in their names.
|
|
4
|
+
- [GHSA-pw9g-874p-x237](https://github.com/Shopify/pitchfork/security/advisories/GHSA-pw9g-874p-x237) Store request trailers separately in `pitchfork.request_trailers`.
|
|
5
|
+
- [GHSA-mr52-m5rh-frmq](https://github.com/Shopify/pitchfork/security/advisories/GHSA-mr52-m5rh-frmq) Fix CRLF injection in response trailers.
|
|
6
|
+
- Fix response headers with non-ASCII bytes raising `ArgumentError` (invalid UTF-8) or `Encoding::CompatibilityError` (values in different encodings). Header values are now checked and written as bytes.
|
|
7
|
+
- [GHSA-p9r3-wjcg-rwv5](https://github.com/Shopify/pitchfork/security/advisories/GHSA-p9r3-wjcg-rwv5) Fix chunked body parse errors that could disclose unread data.
|
|
8
|
+
- `RACK_ENV` now default to `"production"` instead of `"development"`.
|
|
9
|
+
- Fix workers surviving a soft timeout when the application swallows the `SystemExit` (e.g. Rails' `ActionDispatch::ShowExceptions`). The worker now exits once the response is written, and `after_request_complete` isn't called for that request.
|
|
10
|
+
|
|
3
11
|
# 0.18.2
|
|
4
12
|
|
|
5
13
|
- Fix after_request_complete getting nil env on parsing exceptions.
|
|
@@ -29,7 +37,7 @@
|
|
|
29
37
|
|
|
30
38
|
# 0.16.0
|
|
31
39
|
|
|
32
|
-
- Use `exit!` for exiting the middle process when forking a new worker or mold.
|
|
40
|
+
- Use `exit!` for exiting the middle process when forking a new worker or mold.
|
|
33
41
|
This skips `at_exit` and finalizer hooks, which shouldn't be needed.
|
|
34
42
|
|
|
35
43
|
# 0.15.0
|
data/Gemfile
CHANGED
data/Rakefile
CHANGED
|
@@ -8,11 +8,13 @@ namespace :test do
|
|
|
8
8
|
Megatest::TestTask.create(:unit) do |t|
|
|
9
9
|
t.tests = FileList["test/unit/**/test_*.rb"]
|
|
10
10
|
t.deps << :compile
|
|
11
|
+
t.extra_args = ['--jobs', '2'] if ENV["CI"]
|
|
11
12
|
end
|
|
12
13
|
|
|
13
14
|
Megatest::TestTask.create(:integration) do |t|
|
|
14
15
|
t.tests = FileList["test/integration/**/test_*.rb"]
|
|
15
16
|
t.deps << :compile
|
|
17
|
+
t.extra_args = ['--jobs', '2'] if ENV["CI"]
|
|
16
18
|
end
|
|
17
19
|
|
|
18
20
|
# It's not so much that these tests are slow, but they tend to fork
|
|
@@ -23,6 +25,7 @@ namespace :test do
|
|
|
23
25
|
Megatest::TestTask.create(:slow) do |t|
|
|
24
26
|
t.tests = FileList["test/slow/**/test_*.rb"]
|
|
25
27
|
t.deps << :compile
|
|
28
|
+
t.extra_args = ['--jobs', '2'] if ENV["CI"]
|
|
26
29
|
end
|
|
27
30
|
|
|
28
31
|
# Unicorn had its own POSIX-shell based testing framework.
|
|
@@ -60,6 +63,18 @@ task :ragel do
|
|
|
60
63
|
puts "* compiling pitchfork_http.rl"
|
|
61
64
|
cmd = ["ragel", "-G2", "pitchfork_http.rl", "-o", "pitchfork_http.c"]
|
|
62
65
|
system(*cmd) or raise "== #{cmd.join(' ')} failed =="
|
|
66
|
+
|
|
67
|
+
# Ragel 6.10 counts the lines it emits by overriding std::filebuf#xsputn,
|
|
68
|
+
# but libstdc++ 12+ writes single characters with sputc, so the `#line`
|
|
69
|
+
# directives pointing back into the generated file drift depending on
|
|
70
|
+
# which toolchain ragel was built with.
|
|
71
|
+
lines = File.readlines("pitchfork_http.c")
|
|
72
|
+
lines.each_with_index do |line, index|
|
|
73
|
+
if line.start_with?("#line ") && line.end_with?(%("pitchfork_http.c"\n))
|
|
74
|
+
lines[index] = %(#line #{index + 2} "pitchfork_http.c"\n)
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
File.write("pitchfork_http.c", lines.join)
|
|
63
78
|
end
|
|
64
79
|
end
|
|
65
80
|
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# Benchmarks the per-request Ruby-level hot path: HTTP parsing
|
|
5
|
+
# (Pitchfork::HttpParser#read) and response writing (http_response_write).
|
|
6
|
+
# This isolates the part of a request pitchfork controls in Ruby, without
|
|
7
|
+
# the noise of a real TCP stack, listener accept loop, or a Rack app.
|
|
8
|
+
|
|
9
|
+
require "socket"
|
|
10
|
+
require "fileutils"
|
|
11
|
+
require "benchmark/ips"
|
|
12
|
+
require "pitchfork"
|
|
13
|
+
|
|
14
|
+
#Headers emitted by Firefox when querying https://example.com
|
|
15
|
+
REQUEST = (<<~HTTP.split("\n").join("\r\n") << "\r\n\r\n").b
|
|
16
|
+
GET /hello?foo=bar HTTP/1.1
|
|
17
|
+
Host: example.com
|
|
18
|
+
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:157.0) Gecko/20100101 Firefox/157.0
|
|
19
|
+
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
|
|
20
|
+
Accept-Language: en-US,en;q=0.9
|
|
21
|
+
Accept-Encoding: gzip, deflate, br, zstd
|
|
22
|
+
Upgrade-Insecure-Requests: 1
|
|
23
|
+
Connection: keep-alive
|
|
24
|
+
Sec-Fetch-Dest: document
|
|
25
|
+
Sec-Fetch-Mode: navigate
|
|
26
|
+
Sec-Fetch-Site: none
|
|
27
|
+
Sec-Fetch-User: ?1
|
|
28
|
+
Priority: u=0, i
|
|
29
|
+
HTTP
|
|
30
|
+
|
|
31
|
+
BRANCH ||= ENV["BRANCH"] || begin
|
|
32
|
+
branch = `git rev-parse --abbrev-ref HEAD`.strip
|
|
33
|
+
branch = `git rev-parse HEAD`.strip if branch == "HEAD"
|
|
34
|
+
branch
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
DATA_DIR = "/tmp/pitchfork-bench"
|
|
38
|
+
|
|
39
|
+
FileUtils.mkdir_p(DATA_DIR)
|
|
40
|
+
|
|
41
|
+
class RequestBench
|
|
42
|
+
include Pitchfork::HttpResponse
|
|
43
|
+
|
|
44
|
+
def initialize
|
|
45
|
+
@client, @server = Socket.pair(:UNIX, :STREAM, 0)
|
|
46
|
+
@body = ("Hello World!\n" * 1000).freeze
|
|
47
|
+
# Headers somewhat representative of a vanilla Rails app
|
|
48
|
+
@headers = {
|
|
49
|
+
"content-type" => "text/plain",
|
|
50
|
+
"content-length" => @body.bytesize.to_s,
|
|
51
|
+
"x-frame-options" => "SAMEORIGIN",
|
|
52
|
+
"X-Xss-Protection" => "0",
|
|
53
|
+
"X-Content-Type-Options" => "nosniff",
|
|
54
|
+
"X-Permitted-Cross-Domain-Policies" => "none",
|
|
55
|
+
"Referrer-Policy" => "strict-origin-when-cross-origin",
|
|
56
|
+
"Link" => "</assets/ansi_stream.self-92d4ee1a07d3e06eb3145a4809f336213d91ae61be15427ab27b6015a10f0375.css?body=1>; rel=preload; as=style; nopush,</assets/clusterize.self-e800b47f82f06468feec0f3d7582faad615cbc3bd97086ce25fc6430347a314f.css?body=1>; rel=preload; as=style; nopush,</assets/shipit.self-9435813317a8e23afe88c6733ae24f084ec0a975b665ab91ba7a9885e6a16b8f.css?body=1>; rel=preload; as=style; nopush,</assets/jquery.self-bd7ddd393353a8d2480a622e80342adf488fb6006d667e8b42e4c0073393abee.js?body=1>; rel=preload; as=script; nopush,</assets/jquery_ujs.self-a6375ab2b275439c922ac52b02a475d288976458aa11a6e8103c925ef2961f66.js?body=1>; rel=preload; as=script; nopush,</assets/jquery.timeago.self-93eb9343105e184ba97229398b7379fd3b3a776e75caed52b1822caa85c7dc1c.js?body=1>; rel=preload; as=script; nopush,</assets/rails-timeago.self-e64fd06c0b47de91c2d4d8add679349965a5fc3d79df086ed5e05f9a453a7682.js?body=1>; rel=preload; as=script; nopush",
|
|
57
|
+
"Vary" => "Accept",
|
|
58
|
+
"Etag" => 'W/"53563805e72c3850a046b02bf8ccd7ef"',
|
|
59
|
+
"Cache-Control" => "max-age=0, private, must-revalidate",
|
|
60
|
+
"X-Request-Id" => "602a0bce-1d08-4730-bc82-7630faa2e848",
|
|
61
|
+
"X-Runtime" => "0.022851",
|
|
62
|
+
"Set-Cookie" => "_dummy_session=2TRV9WNdqO02QYfGpNf5cdIhOBumI3TsndmAabnE3WdjDcqF09XPpcVzTF9FJIU26xhpZW8yeKBIS5qQHpBlSs63YZIv6lWXShZjLe24V%2Bq2xt8JthapJwcuH56ONUMtoWHVdVzqqu%2BgrF96ckyaFqu9JYF0%2FPfwhLbmXSLfUCgbNE3QSTquCTzvxxtHK5dbVLYtvDpG74Z1j9vTO%2FNnuuuVPOu3LXPW6ZZhMk7OnMQ8jVLWEx8wkB3e1yn%2Bz32fvttYSgbs865E9w8voN65NfZLXgzgXw%3D%3D--R7jc4YqCjlFYV6pD--%2Bten7S%2FmnotUFK6mHStxpQ%3D%3D; path=/; HttpOnly; SameSite=Lax",
|
|
63
|
+
}.freeze
|
|
64
|
+
@server.setsockopt(:SOL_SOCKET, :SO_SNDBUF, @body.bytesize * 10)
|
|
65
|
+
@buf = "".b
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def run_once
|
|
69
|
+
@client.write(REQUEST)
|
|
70
|
+
req = Pitchfork::HttpParser.new
|
|
71
|
+
env = req.read(@server)
|
|
72
|
+
http_response_write(@server, 200, @headers, [@body], req)
|
|
73
|
+
@client.readpartial(@body.bytesize * 2, @buf)
|
|
74
|
+
env
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
bench = RequestBench.new
|
|
79
|
+
env = bench.run_once
|
|
80
|
+
raise "sanity check failed" unless env["PATH_INFO"] == "/hello"
|
|
81
|
+
|
|
82
|
+
Benchmark.ips do |x|
|
|
83
|
+
x.report(BRANCH) { bench.run_once }
|
|
84
|
+
x.compare!(order: :baseline)
|
|
85
|
+
x.save!(File.join(DATA_DIR, "request-response.data"))
|
|
86
|
+
end
|
data/docs/CONFIGURATION.md
CHANGED
|
@@ -169,18 +169,6 @@ The following options may be specified (but are generally not needed):
|
|
|
169
169
|
|
|
170
170
|
Default: `queues - 1`.
|
|
171
171
|
|
|
172
|
-
- `setpgid: true or false`
|
|
173
|
-
|
|
174
|
-
When enabled, child processes will be made into a group leader with a pgid matching
|
|
175
|
-
their pid. The child process will no longer be in the foreground process group of
|
|
176
|
-
its control terminal.
|
|
177
|
-
|
|
178
|
-
This can break debugging sessions and interactive terminals inside the worker
|
|
179
|
-
processes. Set it to `false` if you need to activate a debugging session or read from
|
|
180
|
-
`STDIN` inside a worker process.
|
|
181
|
-
|
|
182
|
-
Default: `true` (enabled)
|
|
183
|
-
|
|
184
172
|
- `umask: mode`
|
|
185
173
|
|
|
186
174
|
Sets the file mode creation mask for UNIX sockets.
|
|
@@ -215,6 +203,37 @@ The following options may be specified (but are generally not needed):
|
|
|
215
203
|
|
|
216
204
|
Default: `1`
|
|
217
205
|
|
|
206
|
+
|
|
207
|
+
### `poller`
|
|
208
|
+
|
|
209
|
+
Pitchfork can use different methods to poll for incomming requests.
|
|
210
|
+
|
|
211
|
+
When available the default is to use `epoll` with the `EPOLLEXCLUSIVE` flags.
|
|
212
|
+
`EPOLLEXCLUSIVE` solves the thundering herd problem, reducing CPU usage a litte bit,
|
|
213
|
+
however it has the downside of behaving like a LIFO queue (see https://blog.cloudflare.com/the-sad-state-of-linux-socket-balancing/),
|
|
214
|
+
resulting in most requests being handled by older workers, which is somewhat desirable when reforking is enabled,
|
|
215
|
+
but can be problematic when not using reforking and using hot restarts as traffic can
|
|
216
|
+
very suddenly shift from very warm workers to new very cold workers, causing latency spikes.
|
|
217
|
+
|
|
218
|
+
In such environment it may be more desirable to use either `select` or raw `epoll`, which have a
|
|
219
|
+
bit more overhead but produce a more even distribution of requests between workers allowing them
|
|
220
|
+
to warm up more progressively. We've found that the better polling in such setup is `select`.
|
|
221
|
+
|
|
222
|
+
Valid values: `:epoll_exclusive`, `:epoll`, `:select`
|
|
223
|
+
Default: `:epoll_exclusive`.
|
|
224
|
+
|
|
225
|
+
### `setpgid`
|
|
226
|
+
|
|
227
|
+
When enabled, child processes will be made into a group leader with a pgid matching
|
|
228
|
+
their pid. The child process will no longer be in the foreground process group of
|
|
229
|
+
its control terminal.
|
|
230
|
+
|
|
231
|
+
This can break debugging sessions and interactive terminals inside the worker
|
|
232
|
+
processes. Set it to `false` if you need to activate a debugging session or read from
|
|
233
|
+
`STDIN` inside a worker process.
|
|
234
|
+
|
|
235
|
+
Default: `true` (enabled)
|
|
236
|
+
|
|
218
237
|
### `timeout`
|
|
219
238
|
|
|
220
239
|
```ruby
|
|
@@ -229,6 +248,13 @@ A first "soft" timeout will invoke the `after_worker_timeout` from
|
|
|
229
248
|
within the worker (but from a background thread) and then call `exit`
|
|
230
249
|
to terminate the worker cleanly.
|
|
231
250
|
|
|
251
|
+
Ruby delivers the resulting `SystemExit` to the thread serving the
|
|
252
|
+
request. If the application rescues it and returns a response anyway, as
|
|
253
|
+
Rails does in production, where `ActionDispatch::ShowExceptions` renders
|
|
254
|
+
an error page, the response is written, then the worker logs that the
|
|
255
|
+
`SystemExit` was swallowed and exits without calling
|
|
256
|
+
`after_request_complete`.
|
|
257
|
+
|
|
232
258
|
The second "hard" timeout, is the sum of `timeout` and `cleanup`.
|
|
233
259
|
Workers taking longer than this time period to be ready to handle a new
|
|
234
260
|
request will be forcibly killed (via `SIGKILL`).
|
|
@@ -418,11 +444,16 @@ thread local state could contain thread unsafe objects.
|
|
|
418
444
|
Also note that at this stage, the thread is still alive, if your callback does
|
|
419
445
|
substantial work, you may want to kill the thread.
|
|
420
446
|
|
|
421
|
-
After the callback is executed
|
|
447
|
+
After the callback is executed, `before_worker_exit` is called from the
|
|
448
|
+
same thread, and the worker exits with status `0`. See [`timeout`](#timeout)
|
|
449
|
+
for what happens when the application rescues the resulting `SystemExit`.
|
|
422
450
|
|
|
423
451
|
It is recommended not to do slow operations in this callback, but if you
|
|
424
452
|
really have to, make sure to configure the `cleanup` timeout so that the
|
|
425
453
|
callback has time to complete before the "hard" timeout triggers.
|
|
454
|
+
`cleanup` also has to cover `before_worker_exit`, the rest of the request
|
|
455
|
+
and any `at_exit` hooks, since a worker still running at the "hard"
|
|
456
|
+
timeout is killed with `SIGKILL`.
|
|
426
457
|
By default the cleanup timeout is 2 seconds.
|
|
427
458
|
|
|
428
459
|
### `after_worker_hard_timeout`
|
|
@@ -444,9 +475,12 @@ soft timeout from working.
|
|
|
444
475
|
|
|
445
476
|
Called in the worker process before it is shut down.
|
|
446
477
|
|
|
478
|
+
On a soft timeout it is called from the timeout thread, right after
|
|
479
|
+
`after_worker_timeout`, while the request may still be running.
|
|
480
|
+
|
|
447
481
|
```ruby
|
|
448
482
|
before_worker_exit do |server, worker|
|
|
449
|
-
server.logger.info("
|
|
483
|
+
server.logger.info("#{worker.to_log} shuts down after #{worker.requests_count} requests")
|
|
450
484
|
end
|
|
451
485
|
```
|
|
452
486
|
|
|
@@ -469,6 +503,9 @@ Called in the worker processes after a request has completed.
|
|
|
469
503
|
|
|
470
504
|
Can be used for out of band work, or to exit unhealthy workers.
|
|
471
505
|
|
|
506
|
+
It isn't called for a request that hit the soft timeout, since the worker
|
|
507
|
+
exits instead.
|
|
508
|
+
|
|
472
509
|
```ruby
|
|
473
510
|
after_request_complete do |server, worker, env|
|
|
474
511
|
if something_wrong?
|
|
@@ -23,6 +23,12 @@ file, if it contains it. Reloading is not a supported feature of Pitchfork.
|
|
|
23
23
|
|
|
24
24
|
* The configuration `after_fork` has been split between `after_worker_fork` and `after_mold_fork`.
|
|
25
25
|
|
|
26
|
+
* The `before_fork` callback is only passed the server: `before_fork do |server|`. Unicorn's is also passed
|
|
27
|
+
the worker, and Pitchfork raises an `ArgumentError` at boot for a block that takes two arguments.
|
|
28
|
+
|
|
29
|
+
* `TERM` triggers a graceful shutdown, like `QUIT`, for both the monitor and the workers, where Unicorn treats
|
|
30
|
+
`TERM` as a quick shutdown, like `INT`. Send `INT` for a quick shutdown. See [Signals](SIGNALS.md).
|
|
31
|
+
|
|
26
32
|
* If you use `unicorn-worker-killer` or similar gems, you will need to implement this functionally yourself since
|
|
27
33
|
there is no `pitchfork-worker-killer`. Changes to Pitchfork internals make this a pretty painless
|
|
28
34
|
ordeal, you can check out the following GitHub issue to get started: https://github.com/Shopify/pitchfork/issues/92
|
data/docs/SIGNALS.md
CHANGED
|
@@ -11,6 +11,12 @@ processes are documented here as well.
|
|
|
11
11
|
* `QUIT/TERM` - graceful shutdown, waits for workers to finish their
|
|
12
12
|
current request before finishing.
|
|
13
13
|
|
|
14
|
+
* `USR1` - trigger a host restart. The monitor self-reexec,
|
|
15
|
+
then a new mold is booted and replace the existing one,
|
|
16
|
+
and finally all workers are cycled out and replaced.
|
|
17
|
+
If the config is invalid or for some other reasons the
|
|
18
|
+
monitor fails to boot the process will crash.
|
|
19
|
+
|
|
14
20
|
* `USR2` - trigger a manual refork. A worker is promoted as
|
|
15
21
|
a new mold, and existing workers progressively replaced
|
|
16
22
|
by fresh ones.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Sample verbose configuration file for Pitchfork
|
|
3
|
+
|
|
4
|
+
# Use at least one worker per core if you're on a dedicated server,
|
|
5
|
+
# more will usually help for _short_ waits on databases/caches.
|
|
6
|
+
worker_processes 2
|
|
7
|
+
|
|
8
|
+
restart_command ["bundle", "exec", "pitchfork"]
|
|
9
|
+
|
|
10
|
+
# listen on both a Unix domain socket and a TCP port,
|
|
11
|
+
# we use a shorter backlog for quicker failover when busy
|
|
12
|
+
listen 8080, :tcp_nopush => true
|
|
13
|
+
|
|
14
|
+
# nuke workers after 30 seconds instead of 60 seconds (the default)
|
|
15
|
+
timeout 30
|
|
16
|
+
|
|
17
|
+
# Enable this flag to have pitchfork test client connections by writing the
|
|
18
|
+
# beginning of the HTTP headers before calling the application. This
|
|
19
|
+
# prevents calling the application for connections that have disconnected
|
|
20
|
+
# while queued. This is only guaranteed to detect clients on the same
|
|
21
|
+
# host pitchfork runs on, and unlikely to detect disconnects even on a
|
|
22
|
+
# fast LAN.
|
|
23
|
+
check_client_connection false
|
|
24
|
+
|
|
25
|
+
# local variable to guard against running a hook multiple times
|
|
26
|
+
run_once = true
|
|
27
|
+
|
|
28
|
+
after_mold_fork do |server, mold|
|
|
29
|
+
# Occasionally, it may be necessary to run non-idempotent code in the
|
|
30
|
+
# monitor before forking. Keep in mind the above disconnect! example
|
|
31
|
+
# is idempotent and does not need a guard.
|
|
32
|
+
if run_once
|
|
33
|
+
# do_something_once_here ...
|
|
34
|
+
run_once = false # prevent from firing again
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
after_worker_fork do |server, worker|
|
|
40
|
+
# You may want to check and restart any shared sockets/descriptors
|
|
41
|
+
end
|
data/exe/pitchfork
CHANGED
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
require 'pitchfork/launcher'
|
|
5
5
|
require 'optparse'
|
|
6
6
|
|
|
7
|
-
ENV["RACK_ENV"] ||= "
|
|
7
|
+
ENV["RACK_ENV"] ||= "production"
|
|
8
8
|
rackup_opts = Pitchfork::Configurator::RACKUP
|
|
9
9
|
options = rackup_opts[:options]
|
|
10
|
+
options[:restart_command_argv] = ARGV.dup
|
|
10
11
|
|
|
11
12
|
op = OptionParser.new("", 24, ' ') do |opts|
|
|
12
13
|
cmd = File.basename($0)
|
|
@@ -55,7 +56,7 @@ op = OptionParser.new("", 24, ' ') do |opts|
|
|
|
55
56
|
end
|
|
56
57
|
|
|
57
58
|
opts.on("-E", "--env RACK_ENV",
|
|
58
|
-
"use RACK_ENV for defaults (default:
|
|
59
|
+
"use RACK_ENV for defaults (default: production)") do |e|
|
|
59
60
|
ENV["RACK_ENV"] = e
|
|
60
61
|
end
|
|
61
62
|
|
data/ext/pitchfork_http/c_util.h
CHANGED
|
@@ -34,16 +34,11 @@ static inline void *deconst(const void *in)
|
|
|
34
34
|
return tmp.out;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
/*
|
|
38
|
-
|
|
39
|
-
* to underscores for HTTP headers. Locale-agnostic.
|
|
40
|
-
*/
|
|
41
|
-
static void snake_upcase_char(char *c)
|
|
37
|
+
/* Capitalizes a single ASCII character. Locale-agnostic. */
|
|
38
|
+
static void upcase_char(char *c)
|
|
42
39
|
{
|
|
43
40
|
if (*c >= 'a' && *c <= 'z')
|
|
44
41
|
*c &= ~0x20;
|
|
45
|
-
else if (*c == '-')
|
|
46
|
-
*c = '_';
|
|
47
42
|
}
|
|
48
43
|
|
|
49
44
|
/* Downcases a single ASCII character. Locale-agnostic. */
|
|
@@ -81,7 +76,7 @@ static off_t step_incr(off_t i, int xdigit, const int base)
|
|
|
81
76
|
|
|
82
77
|
if (offset > (base - 1))
|
|
83
78
|
return -1;
|
|
84
|
-
if (i > next_max)
|
|
79
|
+
if (i < 0 || i > next_max)
|
|
85
80
|
return -1;
|
|
86
81
|
i *= base;
|
|
87
82
|
|
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
#include <sys/prctl.h>
|
|
4
4
|
|
|
5
5
|
static VALUE enable_child_subreaper(VALUE module) {
|
|
6
|
+
int status = -1;
|
|
7
|
+
if (prctl(PR_GET_CHILD_SUBREAPER, &status) < 0) {
|
|
8
|
+
return Qfalse;
|
|
9
|
+
}
|
|
6
10
|
if (prctl(PR_SET_CHILD_SUBREAPER, 1) < 0) {
|
|
7
11
|
rb_sys_fail("prctl(2) PR_SET_CHILD_SUBREAPER");
|
|
8
12
|
}
|
|
@@ -17,7 +21,9 @@ static VALUE enable_child_subreaper(VALUE module) {
|
|
|
17
21
|
void init_child_subreaper(VALUE mPitchfork)
|
|
18
22
|
{
|
|
19
23
|
#ifdef HAVE_CONST_PR_SET_CHILD_SUBREAPER
|
|
20
|
-
|
|
24
|
+
int status = -1;
|
|
25
|
+
int ret = prctl(PR_GET_CHILD_SUBREAPER, &status);
|
|
26
|
+
rb_define_const(mPitchfork, "CHILD_SUBREAPER_AVAILABLE", ret < 0 ? Qfalse : Qtrue);
|
|
21
27
|
#else
|
|
22
28
|
rb_define_const(mPitchfork, "CHILD_SUBREAPER_AVAILABLE", Qfalse);
|
|
23
29
|
#endif
|
|
@@ -37,7 +37,7 @@ static int rb_io_descriptor(VALUE io)
|
|
|
37
37
|
* :nodoc:
|
|
38
38
|
* returns IO object if EPOLLEXCLUSIVE works and arms readers
|
|
39
39
|
*/
|
|
40
|
-
static VALUE prep_readers(VALUE cls, VALUE readers)
|
|
40
|
+
static VALUE prep_readers(VALUE cls, VALUE readers, VALUE exclusive)
|
|
41
41
|
{
|
|
42
42
|
long i;
|
|
43
43
|
int epfd = epoll_create1(EPOLL_CLOEXEC);
|
|
@@ -62,7 +62,10 @@ static VALUE prep_readers(VALUE cls, VALUE readers)
|
|
|
62
62
|
* code path for doing it). So let the kernel spend
|
|
63
63
|
* cycles on maintaining level-triggering.
|
|
64
64
|
*/
|
|
65
|
-
e.events =
|
|
65
|
+
e.events = EPOLLIN;
|
|
66
|
+
if (RTEST(exclusive)) {
|
|
67
|
+
e.events |= EPOLLEXCLUSIVE;
|
|
68
|
+
}
|
|
66
69
|
io = rb_io_get_io(io);
|
|
67
70
|
rc = epoll_ctl(epfd, EPOLL_CTL_ADD, rb_io_descriptor(io), &e);
|
|
68
71
|
if (rc < 0) rb_sys_fail("epoll_ctl");
|
|
@@ -122,7 +125,7 @@ static void init_epollexclusive(VALUE mPitchfork)
|
|
|
122
125
|
{
|
|
123
126
|
#if USE_EPOLL
|
|
124
127
|
VALUE cWaiter = rb_define_class_under(mPitchfork, "Waiter", rb_cIO);
|
|
125
|
-
rb_define_singleton_method(cWaiter, "prep_readers", prep_readers,
|
|
128
|
+
rb_define_singleton_method(cWaiter, "prep_readers", prep_readers, 2);
|
|
126
129
|
rb_define_method(cWaiter, "get_readers", get_readers, 3);
|
|
127
130
|
#endif
|
|
128
131
|
}
|
|
@@ -10,6 +10,7 @@ have_func("rb_enc_interned_str", "ruby.h") # Ruby 3.0+
|
|
|
10
10
|
have_func("rb_io_descriptor", "ruby.h") # Ruby 3.1+
|
|
11
11
|
have_func("rb_hash_new_capa", "ruby.h") # Ruby 3.2+
|
|
12
12
|
have_func("getpagesize", "unistd.h")
|
|
13
|
+
have_func("memfd_create", "sys/mman.h")
|
|
13
14
|
|
|
14
15
|
if RUBY_VERSION.start_with?('3.0.')
|
|
15
16
|
# https://bugs.ruby-lang.org/issues/18772
|
|
@@ -5,6 +5,7 @@ static VALUE e413;
|
|
|
5
5
|
static VALUE e414;
|
|
6
6
|
|
|
7
7
|
static VALUE g_rack_url_scheme;
|
|
8
|
+
static VALUE g_request_trailers;
|
|
8
9
|
static VALUE g_request_method;
|
|
9
10
|
static VALUE g_request_uri;
|
|
10
11
|
static VALUE g_fragment;
|
|
@@ -70,6 +71,7 @@ DEF_MAX_LENGTH(QUERY_STRING, (1024 * 10));
|
|
|
70
71
|
static void init_globals(void)
|
|
71
72
|
{
|
|
72
73
|
DEF_GLOBAL(rack_url_scheme, "rack.url_scheme");
|
|
74
|
+
DEF_GLOBAL(request_trailers, "pitchfork.request_trailers");
|
|
73
75
|
DEF_GLOBAL(request_method, "REQUEST_METHOD");
|
|
74
76
|
DEF_GLOBAL(request_uri, "REQUEST_URI");
|
|
75
77
|
DEF_GLOBAL(fragment, "FRAGMENT");
|