hotcell-server 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/hot_cell/configuration.rb +1 -1
- data/lib/hot_cell/counters.rb +3 -2
- data/lib/hot_cell/operation.rb +2 -2
- data/lib/hot_cell/server/errors.rb +1 -1
- data/lib/hot_cell/server/version.rb +1 -1
- data/lib/hot_cell/supervisor.rb +10 -10
- data/lib/hot_cell/test_operations.rb +2 -2
- data/lib/hot_cell/timing.rb +1 -1
- data/lib/hot_cell/worker.rb +6 -6
- metadata +8 -8
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7f09004467e489b37a062a04d9f4e4f418cfa91de46cab560316de61d624be2b
|
|
4
|
+
data.tar.gz: 06126ba9b14c71d06c8de92639fe25471eb61ab298561811dd059193d0476f98
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 652224e5285628af8183e8e11b9195c776072caff19f6e99bab4072278838bc960ec47c14a6634fc24ef91e83a0e45579037e4b2716fa8da6c79cca07723a581
|
|
7
|
+
data.tar.gz: f01b60a20606eae2706953fb46928801a304c0d8b3b1945018c17425c8189fd2419c11d45ab29a492906c7c164650cbd587778c3d2adb960c728de3f8281ea87
|
data/README.md
CHANGED
|
@@ -83,7 +83,7 @@ module HotCell
|
|
|
83
83
|
# Derived here rather than reassembled in the client: a client adding up `queue_wait + deadline` plus its
|
|
84
84
|
# own guess at the kill-to-answer step cannot follow a change to any of them, and the error points the
|
|
85
85
|
# unsafe way — the client believes its timeout is generous and takes a transport failure instead of the
|
|
86
|
-
# cell's
|
|
86
|
+
# cell's answer. A stage added later that costs a caller time belongs in this sum.
|
|
87
87
|
def answer_within
|
|
88
88
|
queue_wait + limits.deadline + KILL_GRACE
|
|
89
89
|
end
|
data/lib/hot_cell/counters.rb
CHANGED
|
@@ -18,6 +18,7 @@ module HotCell
|
|
|
18
18
|
class Counters
|
|
19
19
|
def initialize
|
|
20
20
|
@started_at = Clock.now
|
|
21
|
+
@start_time = Time.now.to_f
|
|
21
22
|
@requests = Hash.new(0)
|
|
22
23
|
@killed_by = Hash.new(0)
|
|
23
24
|
@cancelled = 0
|
|
@@ -42,8 +43,8 @@ module HotCell
|
|
|
42
43
|
end
|
|
43
44
|
|
|
44
45
|
def to_h(running: 0, queued: 0)
|
|
45
|
-
{ uptime_s: (Clock.now - @started_at).round, running: running,
|
|
46
|
-
queue_high_water: @queue_high_water, requests: @requests, killed_by: @killed_by,
|
|
46
|
+
{ start_time_s: @start_time, uptime_s: (Clock.now - @started_at).round, running: running,
|
|
47
|
+
queued: queued, queue_high_water: @queue_high_water, requests: @requests, killed_by: @killed_by,
|
|
47
48
|
cancelled: @cancelled }
|
|
48
49
|
end
|
|
49
50
|
end
|
data/lib/hot_cell/operation.rb
CHANGED
|
@@ -75,7 +75,7 @@ module HotCell
|
|
|
75
75
|
# does not survive fork and the child waits on a pool with no threads. Not the first worker — every
|
|
76
76
|
# worker. A later change that pre-warms the pool to save the fork cost is exactly what this forbids.
|
|
77
77
|
#
|
|
78
|
-
# Note that a
|
|
78
|
+
# Note that a cell forks per request, continuously, so "before the fork" and "at boot" are not
|
|
79
79
|
# the same moment the way they are in Puma. This runs once.
|
|
80
80
|
def before_fork(&block)
|
|
81
81
|
return collected(:@before_fork) if block.nil?
|
|
@@ -147,7 +147,7 @@ module HotCell
|
|
|
147
147
|
# streams in full and hands them over at exit, so slicing afterwards bounded the Strings this method
|
|
148
148
|
# returns and nothing else: an input that makes a tool print gigabytes of diagnostics had already
|
|
149
149
|
# cost gigabytes of this worker's address space, and took RLIMIT_DATA with it — arriving as a `memory`
|
|
150
|
-
#
|
|
150
|
+
# failure, which is permanent, for a document whose only crime was being noisy.
|
|
151
151
|
# `pass` hands the tool a set of the worker's own descriptors — an input to read, an output to write —
|
|
152
152
|
# at their existing fd numbers, so the tool reaches them at `Descriptor#fd_path` — `/dev/fd/N`, or the
|
|
153
153
|
# file's own path on macOS, where opening `/dev/fd/N` would share the worker's offset — and no
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module HotCell
|
|
4
|
-
# What an operation raises to choose its own
|
|
4
|
+
# What an operation raises to choose its own failure. An operation may also declare the library
|
|
5
5
|
# exceptions that mean the same thing, with `unreadable Vips::Error`.
|
|
6
6
|
class OperationError < StandardError; end
|
|
7
7
|
|
data/lib/hot_cell/supervisor.rb
CHANGED
|
@@ -427,7 +427,7 @@ module HotCell
|
|
|
427
427
|
end
|
|
428
428
|
|
|
429
429
|
# A worker can die between the fork and this write. Answering rather than raising is what keeps one dead
|
|
430
|
-
# worker from taking the whole cell down with it, and the caller gets a transient
|
|
430
|
+
# worker from taking the whole cell down with it, and the caller gets a transient failure either way.
|
|
431
431
|
def dispatch(child, connection, queued_ms)
|
|
432
432
|
child.dispatched connection, configuration.limits.deadline, at: Clock.now
|
|
433
433
|
|
|
@@ -784,7 +784,7 @@ module HotCell
|
|
|
784
784
|
end
|
|
785
785
|
|
|
786
786
|
def finish(child, code, cause = nil)
|
|
787
|
-
counters.record
|
|
787
|
+
counters.record reported_code(code)
|
|
788
788
|
counters.record_kill cause if reported_cause?(code, cause)
|
|
789
789
|
child.finished
|
|
790
790
|
discard child
|
|
@@ -792,7 +792,7 @@ module HotCell
|
|
|
792
792
|
retire child if configuration.retire?(child.served)
|
|
793
793
|
end
|
|
794
794
|
|
|
795
|
-
# The
|
|
795
|
+
# The code rides an untrusted worker report and becomes both a counter key and a Symbol, so it
|
|
796
796
|
# is bounded to a code this cell mints before either happens. Without this a report of `code: []` raised
|
|
797
797
|
# NoMethodError on `to_sym` past `apply_report`'s rescue and took the cell down, and a stream of unique
|
|
798
798
|
# strings grew the counters without bound. An unknown code is recorded, so a misreporting worker is
|
|
@@ -801,7 +801,7 @@ module HotCell
|
|
|
801
801
|
# It is checked against the known causes rather than passed through, because `record_kill` interns it
|
|
802
802
|
# as a symbol and an unchecked one is an unbounded symbol table keyed by whatever a tool decides.
|
|
803
803
|
def reported_cause?(code, cause)
|
|
804
|
-
|
|
804
|
+
reported_code(code) == Codes::KILLED && cause.is_a?(String) &&
|
|
805
805
|
Codes::PERMANENT_BY_CAUSE.key?(cause)
|
|
806
806
|
end
|
|
807
807
|
|
|
@@ -810,7 +810,7 @@ module HotCell
|
|
|
810
810
|
reported if reported.is_a?(String) && Registry.lookup(reported)
|
|
811
811
|
end
|
|
812
812
|
|
|
813
|
-
def
|
|
813
|
+
def reported_code(reported)
|
|
814
814
|
return reported if reported.is_a?(String) && (reported == "ok" || Codes.known?(reported))
|
|
815
815
|
|
|
816
816
|
"unknown"
|
|
@@ -853,7 +853,7 @@ module HotCell
|
|
|
853
853
|
now = Clock.now
|
|
854
854
|
|
|
855
855
|
@children.each_value.select { |child| child.lingering?(now, Configuration::KILL_GRACE) }.each do |child|
|
|
856
|
-
# The latch rather than
|
|
856
|
+
# The latch rather than an answer: a lingering child is never busy, so no caller hears this cause.
|
|
857
857
|
child.killed_for = Codes::CRASHED
|
|
858
858
|
|
|
859
859
|
kill_group child
|
|
@@ -968,7 +968,7 @@ module HotCell
|
|
|
968
968
|
end
|
|
969
969
|
|
|
970
970
|
# A killed worker cannot report its own death, because the deadline KILL is enforced by a signal. So
|
|
971
|
-
# the supervisor holds its copy of every dispatched connection and writes the
|
|
971
|
+
# the supervisor holds its copy of every dispatched connection and writes the answer itself. Without
|
|
972
972
|
# this the cold side sees a bare end of stream and cannot tell a limit
|
|
973
973
|
# breach from a crash.
|
|
974
974
|
# A worker still holding a connection at reap time never answered: it reports itself idle after writing,
|
|
@@ -978,10 +978,10 @@ module HotCell
|
|
|
978
978
|
# supervisor sent, and it is the one thing here that knows why a worker died. Everything else is a
|
|
979
979
|
# wait status, which says how — and workers share a uid, so any signal in one may have come from a
|
|
980
980
|
# sibling rather than from the kernel. Reading XFSZ, SEGV, ABRT or TRAP as this request's file size or
|
|
981
|
-
# memory let one compromised worker write a permanent
|
|
981
|
+
# memory let one compromised worker write a permanent failure against another request's unrelated
|
|
982
982
|
# input, which Active Storage then kept. `Codes` already stated the rule this broke.
|
|
983
983
|
#
|
|
984
|
-
# The
|
|
984
|
+
# The failures themselves are not gone, they moved to where they can be earned: the worker answers
|
|
985
985
|
# `memory` when it catches NoMemoryError and `fsize` when a write of its own returns EFBIG, on the
|
|
986
986
|
# connection it is holding. See Worker#disarm_file_size_signal.
|
|
987
987
|
#
|
|
@@ -1001,7 +1001,7 @@ module HotCell
|
|
|
1001
1001
|
signal: signal_name(status), duration_ms: Clock.ms_since(child.dispatched_at),
|
|
1002
1002
|
**captured
|
|
1003
1003
|
|
|
1004
|
-
# The capture rides the
|
|
1004
|
+
# The capture rides the failure as well as the log line, so an application logs
|
|
1005
1005
|
# `killed: crashed (libgomp: ...)` rather than a bare `crashed`. Additive on the wire: `from_wire`
|
|
1006
1006
|
# reads named keys, so an old client ignores the field and a new one against an old cell meets nil.
|
|
1007
1007
|
answer child.connection,
|
|
@@ -187,7 +187,7 @@ module HotCell
|
|
|
187
187
|
|
|
188
188
|
# A tool spawn that fails because the host is out of memory. fork raises Errno::ENOMEM under host
|
|
189
189
|
# pressure, before the tool has read a byte of the input, so this is the cell having a bad moment
|
|
190
|
-
# rather than a decompression bomb — and it must not be recorded as a permanent memory
|
|
190
|
+
# rather than a decompression bomb — and it must not be recorded as a permanent `memory` failure.
|
|
191
191
|
class StarvedSpawn < HotCell::Operation
|
|
192
192
|
operation "test.starved_spawn"
|
|
193
193
|
|
|
@@ -200,7 +200,7 @@ module HotCell
|
|
|
200
200
|
operation "test.signals_sibling"
|
|
201
201
|
|
|
202
202
|
# Workers share a uid and a pid namespace, so one finds another by looking for a process the
|
|
203
|
-
# supervisor also fathered. This is the reproducer for the forged
|
|
203
|
+
# supervisor also fathered. This is the reproducer for the forged failure: nothing here touches the
|
|
204
204
|
# victim's input, and the victim is holding an unrelated one.
|
|
205
205
|
def perform(_inputs, _outputs, signal:)
|
|
206
206
|
sibling = siblings.first
|
data/lib/hot_cell/timing.rb
CHANGED
|
@@ -11,7 +11,7 @@ module HotCell
|
|
|
11
11
|
# here, when `performing` is called.
|
|
12
12
|
#
|
|
13
13
|
# Phases accumulate as they complete, so a refusal reports the ones that finished before the failure. An
|
|
14
|
-
# `unreadable`
|
|
14
|
+
# `unreadable` failure that arrives with no `operation_ms` says exactly where it got to.
|
|
15
15
|
class Timing
|
|
16
16
|
attr_reader :queued_ms, :started
|
|
17
17
|
|
data/lib/hot_cell/worker.rb
CHANGED
|
@@ -29,7 +29,7 @@ module HotCell
|
|
|
29
29
|
#
|
|
30
30
|
# A non-zero status for anything unexpected, because the supervisor holds the connection and is the only
|
|
31
31
|
# thing that can answer for a worker that died mid-request. Exiting zero here would leave a caller reading
|
|
32
|
-
# a closed socket with no
|
|
32
|
+
# a closed socket with no answer at all.
|
|
33
33
|
#
|
|
34
34
|
# **The one deliberate `rescue Exception` in this repository.** Not for the exit status — Ruby's own handler
|
|
35
35
|
# would also exit non-zero — but for Failure.sanitize. Left to Ruby, a NoMemoryError or a SystemStackError
|
|
@@ -67,8 +67,8 @@ module HotCell
|
|
|
67
67
|
#
|
|
68
68
|
# Catching it makes the kernel fail the offending write with EFBIG rather than killing the process,
|
|
69
69
|
# and that error return is what a signal is not: it is raised by a write this process made, and no
|
|
70
|
-
# signal any sibling sends produces one. So the
|
|
71
|
-
# does nothing at all. A handler that set so much as a flag
|
|
70
|
+
# signal any sibling sends produces one. So the `fsize` failure below keys on Errno::EFBIG and this handler
|
|
71
|
+
# does nothing at all. A handler that set so much as a flag that failure consulted would hand the
|
|
72
72
|
# forgery straight back.
|
|
73
73
|
#
|
|
74
74
|
# EFBIG is evidence of this request's own write and not proof of which limit stopped it. A filesystem
|
|
@@ -137,7 +137,7 @@ module HotCell
|
|
|
137
137
|
# transient. Adding it back would condemn a blob for the cell's own bad moment.
|
|
138
138
|
rescue NoMemoryError, MemoryExhausted => error
|
|
139
139
|
response = refuse(Codes::KILLED, error, timing, cause: Codes::MEMORY)
|
|
140
|
-
# The one place
|
|
140
|
+
# The one place an `fsize` failure can be earned. EFBIG comes back from a write this worker made
|
|
141
141
|
# past its own RLIMIT_FSIZE, so unlike the signal it cannot arrive from anywhere else.
|
|
142
142
|
rescue Errno::EFBIG => error
|
|
143
143
|
response = refuse(Codes::KILLED, error, timing, cause: Codes::FSIZE)
|
|
@@ -229,7 +229,7 @@ module HotCell
|
|
|
229
229
|
# reads as success. This is the side that knows which one is empty, so it is the side that says so.
|
|
230
230
|
#
|
|
231
231
|
# Transient, for the reason the client's own check is: the commonest way to write nothing is a full
|
|
232
|
-
# tmpfs, and a full filesystem must never be recorded as a
|
|
232
|
+
# tmpfs, and a full filesystem must never be recorded as a permanent failure of the document.
|
|
233
233
|
def unwritten(outputs, written, timing)
|
|
234
234
|
empty = written.each_index.select { |index| written[index].zero? }
|
|
235
235
|
|
|
@@ -295,7 +295,7 @@ module HotCell
|
|
|
295
295
|
# that off a wait status; it reads the worker's own report now, and only when there is a cause to send,
|
|
296
296
|
# so an ordinary idle report is the two keys it always was.
|
|
297
297
|
#
|
|
298
|
-
# **This is a metric and not
|
|
298
|
+
# **This is a metric and not an answer.** The answer went to the caller on the work connection before
|
|
299
299
|
# this line runs. A compromised worker can report a cause its request never had, or withhold one it
|
|
300
300
|
# did, so `killed_by` is what workers said rather than what happened — which is why the supervisor
|
|
301
301
|
# checks the value against the known causes before interning it, and why nothing downstream may treat
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: hotcell-server
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mike Dalessio
|
|
@@ -15,16 +15,16 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - '='
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: 1.
|
|
18
|
+
version: 1.2.0
|
|
19
19
|
type: :runtime
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - '='
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version: 1.
|
|
25
|
+
version: 1.2.0
|
|
26
26
|
description: |
|
|
27
|
-
Runs a
|
|
27
|
+
Runs a Hot Cell container. A supervisor listens on two Unix sockets, forks a worker for each request,
|
|
28
28
|
and enforces a wall clock deadline and resource limits on it. Write the work as a subclass of
|
|
29
29
|
HotCell::Operation.
|
|
30
30
|
email:
|
|
@@ -64,8 +64,8 @@ licenses:
|
|
|
64
64
|
- MIT
|
|
65
65
|
metadata:
|
|
66
66
|
homepage_uri: https://github.com/basecamp/hotcell
|
|
67
|
-
source_code_uri: https://github.com/basecamp/hotcell/tree/v1.
|
|
68
|
-
changelog_uri: https://github.com/basecamp/hotcell/blob/v1.
|
|
67
|
+
source_code_uri: https://github.com/basecamp/hotcell/tree/v1.2.0/hotcell-server
|
|
68
|
+
changelog_uri: https://github.com/basecamp/hotcell/blob/v1.2.0/CHANGELOG.md
|
|
69
69
|
documentation_uri: https://basecamp.github.io/hotcell/
|
|
70
70
|
bug_tracker_uri: https://github.com/basecamp/hotcell/issues
|
|
71
71
|
rubygems_mfa_required: 'true'
|
|
@@ -83,7 +83,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
83
83
|
- !ruby/object:Gem::Version
|
|
84
84
|
version: '0'
|
|
85
85
|
requirements: []
|
|
86
|
-
rubygems_version: 4.0.
|
|
86
|
+
rubygems_version: 4.0.6
|
|
87
87
|
specification_version: 4
|
|
88
|
-
summary: 'Run a
|
|
88
|
+
summary: 'Run a Hot Cell: the supervisor, the worker, and the operation API.'
|
|
89
89
|
test_files: []
|