pitchfork 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CHANGELOG.md +9 -0
- data/Gemfile.lock +1 -1
- data/docs/CONFIGURATION.md +10 -21
- data/docs/DESIGN.md +1 -1
- data/docs/FORK_SAFETY.md +8 -3
- data/docs/REFORKING.md +33 -23
- data/examples/echo.ru +0 -1
- data/examples/pitchfork.conf.rb +3 -41
- data/ext/pitchfork_http/pitchfork_http.c +166 -166
- data/lib/pitchfork/children.rb +15 -5
- data/lib/pitchfork/chunked.rb +123 -0
- data/lib/pitchfork/configurator.rb +7 -7
- data/lib/pitchfork/http_parser.rb +0 -1
- data/lib/pitchfork/http_server.rb +115 -115
- data/lib/pitchfork/message.rb +2 -1
- data/lib/pitchfork/refork_condition.rb +18 -0
- data/lib/pitchfork/version.rb +1 -1
- data/lib/pitchfork/worker.rb +17 -11
- data/lib/pitchfork.rb +51 -17
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3e42e6177d834ad380a59378ccea33b4110b443975d913ca6bc66c84ee5abbdb
|
4
|
+
data.tar.gz: 79ae3e1c674b3fa6b6bd78103d177d6593e8ce9239f4603450f300ad5685a827
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1ff1761d7107bf62e2e7f31c7b1843ffe78eb99d650a8175ef0f9173f871499afb69b13407e9c70d80be4afd799a2c18243942318b92d5bd339bbbf591abb32b
|
7
|
+
data.tar.gz: 7c49a7909dfd4d3405f66859a308700dd047aa360412a752831348a000ecd4be58d493856539e46b56825e74a7fd19ae81731344bef0a1cde16e3529134c94eb
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,14 @@
|
|
1
1
|
# Unreleased
|
2
2
|
|
3
|
+
# 0.3.0
|
4
|
+
|
5
|
+
- Renamed `after_promotion` in `after_mold_fork`.
|
6
|
+
- Renamed `after_fork` in `after_worker_fork`.
|
7
|
+
- Backoff 10s after every mold spawning attempt.
|
8
|
+
- Spawn mold from workers instead of promoting workers (#42).
|
9
|
+
|
10
|
+
# 0.2.0
|
11
|
+
|
3
12
|
- Remove default middlewares.
|
4
13
|
- Refork indefinitely when `refork_after` is set, unless the last element is `false`.
|
5
14
|
- Remove `mold_selector`. The promotion logic has been moved inside workers (#38).
|
data/Gemfile.lock
CHANGED
data/docs/CONFIGURATION.md
CHANGED
@@ -238,17 +238,19 @@ application that are used in hooks.
|
|
238
238
|
`pitchfork` also don't attempt to rescue hook errors. Raising from a worker hook will crash the worker,
|
239
239
|
and raising from a master hook will bring the whole cluster down.
|
240
240
|
|
241
|
-
### `
|
241
|
+
### `after_mold_fork`
|
242
242
|
|
243
243
|
```ruby
|
244
|
-
|
244
|
+
after_mold_fork do |server, mold|
|
245
245
|
Database.disconnect!
|
246
|
+
3.times { GC.start } # promote surviving objects to oldgen
|
247
|
+
GC.compact
|
246
248
|
end
|
247
249
|
```
|
248
250
|
|
249
|
-
Called in the context of the mold
|
251
|
+
Called in the context of the mold after it has been spawned.
|
250
252
|
|
251
|
-
|
253
|
+
Its usage is similar to a `before_fork` callback found on other servers
|
252
254
|
but it is called once on promotion rather than before forking each worker.
|
253
255
|
|
254
256
|
For most protocols connections can be closed after fork, but some
|
@@ -259,31 +261,18 @@ That is the case for instance of many SQL databases protocols.
|
|
259
261
|
This is also the callback in which memory optimizations, such as
|
260
262
|
heap compaction should be done.
|
261
263
|
|
262
|
-
### `
|
264
|
+
### `after_worker_fork`
|
263
265
|
|
264
266
|
```ruby
|
265
|
-
|
266
|
-
NetworkClient.
|
267
|
+
after_worker_fork do |server, worker|
|
268
|
+
NetworkClient.reconnect!
|
269
|
+
BackgroundThread.restart!
|
267
270
|
end
|
268
271
|
```
|
269
272
|
|
270
273
|
Called in the worker after forking. Generally used to close inherited connections
|
271
274
|
or to restart backgrounds threads for libraries that don't do it automatically.
|
272
275
|
|
273
|
-
### `after_promotion`
|
274
|
-
|
275
|
-
```ruby
|
276
|
-
after_promotion do |server, mold|
|
277
|
-
NetworkClient.disconnect!
|
278
|
-
4.times { GC.start } # promote surviving objects to oldgen
|
279
|
-
GC.compact
|
280
|
-
end
|
281
|
-
```
|
282
|
-
|
283
|
-
Called in the worker after it was promoted into a mold. Generally used to shutdown
|
284
|
-
open connections and file descriptors, as well as to perform memory optimiations
|
285
|
-
such as compacting the heap, trimming memory etc.
|
286
|
-
|
287
276
|
### `after_worker_ready`
|
288
277
|
|
289
278
|
Called by a worker process after it has been fully loaded, directly before it
|
data/docs/DESIGN.md
CHANGED
@@ -28,7 +28,7 @@
|
|
28
28
|
|
29
29
|
* Configuration is purely in Ruby. Ruby is less
|
30
30
|
ambiguous than YAML and lets lambdas for
|
31
|
-
|
31
|
+
after_worker_fork/after_mold_fork hooks be defined inline. An
|
32
32
|
optional, separate config_file may be used to modify supported
|
33
33
|
configuration changes.
|
34
34
|
|
data/docs/FORK_SAFETY.md
CHANGED
@@ -12,16 +12,21 @@ end up shared between the parent and child process. This is never what you
|
|
12
12
|
want, so any code keeping persistent connections should close them either
|
13
13
|
before or after the fork happens.
|
14
14
|
|
15
|
-
|
15
|
+
Most modern Ruby libraries automatically handle this, it's the case of
|
16
|
+
Active Record, redis-rb, dalli, net-http-persistent, etc.
|
17
|
+
|
18
|
+
However, for libraries that aren't automatically fork-safe
|
19
|
+
`pitchfork` provide two callbacks in its configuration file to manually
|
20
|
+
reopen connections and restart threads:
|
16
21
|
|
17
22
|
```ruby
|
18
23
|
# pitchfork.conf.rb
|
19
24
|
|
20
|
-
|
25
|
+
after_mold_fork do
|
21
26
|
Sequel::DATABASES.each(&:disconnect)
|
22
27
|
end
|
23
28
|
|
24
|
-
|
29
|
+
after_worker_fork do
|
25
30
|
SomeLibary.connection.close
|
26
31
|
end
|
27
32
|
```
|
data/docs/REFORKING.md
CHANGED
@@ -46,50 +46,60 @@ first time, if you take a warmed up worker out of rotation, and use it to fork n
|
|
46
46
|
be shared again, and most of them won't be invalidated anymore.
|
47
47
|
|
48
48
|
|
49
|
-
When you start `pitchfork` it forks
|
49
|
+
When you start `pitchfork` it forks a `mold` process which loads your application:
|
50
50
|
|
51
51
|
```
|
52
52
|
PID COMMAND
|
53
53
|
100 \_ pitchfork master
|
54
|
-
101 \_ pitchfork
|
55
|
-
102 \_ pitchfork worker[1] (gen:0)
|
56
|
-
103 \_ pitchfork worker[2] (gen:0)
|
57
|
-
104 \_ pitchfork worker[3] (gen:0)
|
54
|
+
101 \_ pitchfork (gen:0) mold
|
58
55
|
```
|
59
56
|
|
60
|
-
|
61
|
-
When promoted, molds no longer process any incoming HTTP requests, they become inactive:
|
57
|
+
Once the `mold` is done loading, the `master` asks it to spawn the desired number of workers:
|
62
58
|
|
63
59
|
```
|
64
60
|
PID COMMAND
|
65
61
|
100 \_ pitchfork master
|
66
|
-
101 \_ pitchfork
|
67
|
-
102 \_ pitchfork
|
68
|
-
103 \_ pitchfork
|
69
|
-
104 \_ pitchfork
|
62
|
+
101 \_ pitchfork (gen:0) mold
|
63
|
+
102 \_ pitchfork (gen:0) worker[0]
|
64
|
+
103 \_ pitchfork (gen:0) worker[1]
|
65
|
+
104 \_ pitchfork (gen:0) worker[2]
|
66
|
+
105 \_ pitchfork (gen:0) worker[3]
|
70
67
|
```
|
71
68
|
|
72
|
-
When a
|
69
|
+
When a reforking is triggered, one of the workers is selected to fork a new `mold`.
|
70
|
+
|
71
|
+
```
|
72
|
+
PID COMMAND
|
73
|
+
100 \_ pitchfork master
|
74
|
+
101 \_ pitchfork (gen:0) mold
|
75
|
+
102 \_ pitchfork (gen:0) worker[0]
|
76
|
+
103 \_ pitchfork (gen:0) worker[1]
|
77
|
+
104 \_ pitchfork (gen:0) worker[2]
|
78
|
+
105 \_ pitchfork (gen:0) worker[3]
|
79
|
+
105 \_ pitchfork (gen:1) mold
|
80
|
+
```
|
81
|
+
|
82
|
+
When that new mold is ready, `pitchfork` terminates the old mold and starts a slow rollout of older workers and replace them with fresh workers
|
73
83
|
forked from the mold:
|
74
84
|
|
75
85
|
```
|
76
86
|
PID COMMAND
|
77
87
|
100 \_ pitchfork master
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
88
|
+
102 \_ pitchfork (gen:0) worker[0]
|
89
|
+
103 \_ pitchfork (gen:0) worker[1]
|
90
|
+
104 \_ pitchfork (gen:0) worker[2]
|
91
|
+
105 \_ pitchfork (gen:0) worker[3]
|
92
|
+
105 \_ pitchfork (gen:1) mold
|
83
93
|
```
|
84
94
|
|
85
95
|
```
|
86
96
|
PID COMMAND
|
87
97
|
100 \_ pitchfork master
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
98
|
+
103 \_ pitchfork (gen:0) worker[1]
|
99
|
+
104 \_ pitchfork (gen:0) worker[2]
|
100
|
+
105 \_ pitchfork (gen:0) worker[3]
|
101
|
+
105 \_ pitchfork (gen:1) mold
|
102
|
+
106 \_ pitchfork (gen:1) worker[0]
|
93
103
|
```
|
94
104
|
|
95
105
|
etc.
|
@@ -103,7 +113,7 @@ a process tree such as:
|
|
103
113
|
PID COMMAND
|
104
114
|
100 \_ pitchfork master
|
105
115
|
101 \_ pitchfork mold (gen:1)
|
106
|
-
105 \_ pitchfork
|
116
|
+
105 \_ pitchfork (gen:1) worker[0]
|
107
117
|
```
|
108
118
|
|
109
119
|
However the `pitchfork` master process registers itself as a "child subreaper" via [`PR_SET_CHILD_SUBREAPER`](https://man7.org/linux/man-pages/man2/prctl.2.html).
|
data/examples/echo.ru
CHANGED
data/examples/pitchfork.conf.rb
CHANGED
@@ -23,12 +23,7 @@ check_client_connection false
|
|
23
23
|
# local variable to guard against running a hook multiple times
|
24
24
|
run_once = true
|
25
25
|
|
26
|
-
|
27
|
-
# the following is highly recommended for Rails
|
28
|
-
# as there's no need for the mold process to hold a connection
|
29
|
-
defined?(ActiveRecord::Base) and
|
30
|
-
ActiveRecord::Base.connection.disconnect!
|
31
|
-
|
26
|
+
after_mold_fork do |server, mold|
|
32
27
|
# Occasionally, it may be necessary to run non-idempotent code in the
|
33
28
|
# master before forking. Keep in mind the above disconnect! example
|
34
29
|
# is idempotent and does not need a guard.
|
@@ -37,41 +32,8 @@ after_promotion do |server, worker|
|
|
37
32
|
run_once = false # prevent from firing again
|
38
33
|
end
|
39
34
|
|
40
|
-
# The following is only recommended for memory/DB-constrained
|
41
|
-
# installations. It is not needed if your system can house
|
42
|
-
# twice as many worker_processes as you have configured.
|
43
|
-
#
|
44
|
-
# # This allows a new master process to incrementally
|
45
|
-
# # phase out the old master process with SIGTTOU to avoid a
|
46
|
-
# # thundering herd
|
47
|
-
# # when doing a transparent upgrade. The last worker spawned
|
48
|
-
# # will then kill off the old master process with a SIGQUIT.
|
49
|
-
# old_pid = "#{server.config[:pid]}.oldbin"
|
50
|
-
# if old_pid != server.pid
|
51
|
-
# begin
|
52
|
-
# sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
|
53
|
-
# Process.kill(sig, File.read(old_pid).to_i)
|
54
|
-
# rescue Errno::ENOENT, Errno::ESRCH
|
55
|
-
# end
|
56
|
-
# end
|
57
|
-
#
|
58
|
-
# Throttle the master from forking too quickly by sleeping. Due
|
59
|
-
# to the implementation of standard Unix signal handlers, this
|
60
|
-
# helps (but does not completely) prevent identical, repeated signals
|
61
|
-
# from being lost when the receiving process is busy.
|
62
|
-
# sleep 1
|
63
35
|
end
|
64
36
|
|
65
|
-
|
66
|
-
#
|
67
|
-
# addr = "127.0.0.1:#{9293 + worker.nr}"
|
68
|
-
# server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
|
69
|
-
|
70
|
-
# the following is *required* for Rails
|
71
|
-
defined?(ActiveRecord::Base) and
|
72
|
-
ActiveRecord::Base.establish_connection
|
73
|
-
|
74
|
-
# You may also want to check and
|
75
|
-
# restart any other shared sockets/descriptors such as Memcached,
|
76
|
-
# and Redis.
|
37
|
+
after_worker_fork do |server, worker|
|
38
|
+
# You may want to check and restart any shared sockets/descriptors
|
77
39
|
end
|