hotcell-core 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/MIT-LICENSE +20 -0
- data/README.md +3 -0
- data/lib/hot_cell/clock.rb +17 -0
- data/lib/hot_cell/codes.rb +118 -0
- data/lib/hot_cell/connection.rb +150 -0
- data/lib/hot_cell/core/version.rb +5 -0
- data/lib/hot_cell/core.rb +16 -0
- data/lib/hot_cell/declarations.rb +26 -0
- data/lib/hot_cell/descriptors.rb +157 -0
- data/lib/hot_cell/errors.rb +37 -0
- data/lib/hot_cell/failure.rb +95 -0
- data/lib/hot_cell/fields.rb +38 -0
- data/lib/hot_cell/naming.rb +32 -0
- data/lib/hot_cell/payload.rb +116 -0
- data/lib/hot_cell/protocol.rb +28 -0
- data/lib/hot_cell/request.rb +92 -0
- data/lib/hot_cell/response.rb +86 -0
- data/lib/hot_cell/test_support.rb +81 -0
- data/lib/hotcell-core.rb +6 -0
- metadata +34 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e60aebb6474f67f6a35f01d9fe1e3a686c422a073e11ebf785af0816ade80e79
|
|
4
|
+
data.tar.gz: eb13e25de57e23ff35d57fe100926cd64c11ecad18af96104dd3864d2246411b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0c0373bb9b2831d65e5fce1b02c7cb0ede24eed5a3d64144d905abbd93d72a7dcbf3d46250acb24caf195e1e1f9e0a530fa07ef2845c1ccba2118aed4f4e1aa2
|
|
7
|
+
data.tar.gz: a087aedd5855c87b413c53085e23d9d579a59176bc9453d1e07e4712592d1c1405f8939d42c6123ae3fa5bcf9b50ad3d0112353d3adba359a7814d55a0e0b10f
|
data/MIT-LICENSE
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Copyright (c) 37signals, LLC
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
4
|
+
a copy of this software and associated documentation files (the
|
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
9
|
+
the following conditions:
|
|
10
|
+
|
|
11
|
+
The above copyright notice and this permission notice shall be
|
|
12
|
+
included in all copies or substantial portions of the Software.
|
|
13
|
+
|
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# Monotonic, because every duration here is reported to a caller that will alarm on it, and a clock
|
|
5
|
+
# that can step backwards produces negative latencies during an NTP correction.
|
|
6
|
+
module Clock
|
|
7
|
+
class << self
|
|
8
|
+
def now
|
|
9
|
+
Process.clock_gettime Process::CLOCK_MONOTONIC
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def ms_since(at)
|
|
13
|
+
((now - at) * 1000).round(2)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# Every failure carries a code and a `permanent` flag, and `permanent` is the only distinction that
|
|
5
|
+
# changes what a caller must do.
|
|
6
|
+
#
|
|
7
|
+
# Terminal means the same request fails the same way until the input or the code changes — not until
|
|
8
|
+
# the load or the deployment changes. A permanent failure may be recorded against a blob and served
|
|
9
|
+
# from a cache. A non-permanent one must be retried and must never be written down.
|
|
10
|
+
#
|
|
11
|
+
# The flag travels on the wire, set by the side that knows, rather than being derived by each caller
|
|
12
|
+
# from the code. That is what makes a code added later safe: an old client will not recognise it but
|
|
13
|
+
# will still dispose of it correctly.
|
|
14
|
+
module Codes
|
|
15
|
+
PERMANENT = {
|
|
16
|
+
"unreadable" => true, # the input could not be decoded — the operation said so explicitly
|
|
17
|
+
"invalid" => true, # malformed request, or a descriptor that failed its access-mode check
|
|
18
|
+
"failed" => false, # the operation raised something nobody classified — see below
|
|
19
|
+
"unsupported" => false, # this cell does not carry that operation — see below
|
|
20
|
+
"protocol" => false, # version mismatch, which heals when the accessory reboots
|
|
21
|
+
"capacity" => false, # the queue is full
|
|
22
|
+
"unavailable" => false, # no connection, or a connection closed with no response
|
|
23
|
+
"timeout" => false, # the client's own deadline fired
|
|
24
|
+
}.freeze
|
|
25
|
+
|
|
26
|
+
# **`failed` is what an unclassified exception becomes, so it cannot be permanent.**
|
|
27
|
+
#
|
|
28
|
+
# A worker rescues StandardError around the whole request and calls it `failed`. Errno::ENOSPC is a
|
|
29
|
+
# StandardError, and so are EMFILE, EIO, ENOENT and ENOMEM. A shared tmpfs filled by concurrent requests,
|
|
30
|
+
# a full disk under the caller's own output, a descriptor table exhausted by load, a fork that cannot get
|
|
31
|
+
# memory under host pressure, a tool missing during a broken deploy — every one of those raises inside
|
|
32
|
+
# staging or writeback and arrives here. ENOMEM is the one that looks like the input's fault and is not:
|
|
33
|
+
# the worker's own out-of-memory, where the input drove it past its limit, is a NoMemoryError, which the
|
|
34
|
+
# worker classifies `killed`/`memory` instead.
|
|
35
|
+
#
|
|
36
|
+
# Terminal meant each of them was written down against a customer's file and served from a cache forever,
|
|
37
|
+
# for a condition that would have succeeded on retry. Permanence has to be claimed, never inferred from
|
|
38
|
+
# not knowing: an operation says `unreadable` for an input it could not decode, and the protocol says
|
|
39
|
+
# `invalid` for a caller that broke its own contract. Everything else is this cell having a bad day.
|
|
40
|
+
#
|
|
41
|
+
# The cost of the other mistake is a genuinely broken operation being retried. That is bounded by the
|
|
42
|
+
# job's attempts, it is visible in the `failed` rate, and it is recoverable.
|
|
43
|
+
|
|
44
|
+
# **`unsupported` is transient, and the design document says otherwise.** Its reasoning was that an
|
|
45
|
+
# unknown *operation* is a caller bug that never heals, where an unknown *version* is a deploy window
|
|
46
|
+
# that does. That holds for a typo and not for the case that actually happens: an accessory is not
|
|
47
|
+
# updated by a deploy, so an application that ships a client for a new operation before anybody reboots
|
|
48
|
+
# the cell gets `unsupported` at one hundred percent for as long as that takes.
|
|
49
|
+
#
|
|
50
|
+
# The two mistakes are not symmetrical. Retrying a caller's typo costs some work and shows up in the
|
|
51
|
+
# `unsupported` rate and in the client's boot-time warning. Recording a deploy window as permanent
|
|
52
|
+
# condemns every blob uploaded during it, and needs a hand-written backfill to undo.
|
|
53
|
+
|
|
54
|
+
# `killed` splits on what the worker hit, because a caller cannot otherwise tell a decompression
|
|
55
|
+
# bomb from a slow afternoon. Size and memory are properties of the input, so the same bytes will
|
|
56
|
+
# do it again on an idle cell. A deadline is as much a property of the load, and treating it as
|
|
57
|
+
# permanent means a busy hour permanently condemns whatever was uploaded during it.
|
|
58
|
+
# `crashed` is the cell's own fault rather than the input's — a worker that died without answering, which
|
|
59
|
+
# a misconfigured cell does on every request. Recording that against a blob would condemn everything
|
|
60
|
+
# uploaded during a broken deploy, so it is transient. An older client that has never heard of it still
|
|
61
|
+
# disposes of it correctly, because `permanent` travels on the wire.
|
|
62
|
+
#
|
|
63
|
+
# **A limit this table has never heard of is not permanent, and that default is the point of the table.**
|
|
64
|
+
# A cell mints these; adding a kill reason to the supervisor without adding a row here used to make it
|
|
65
|
+
# permanent, silently, and permanent is the answer that cannot be taken back. The names below are
|
|
66
|
+
# constants so that the two places that mint them cannot spell one the table does not carry.
|
|
67
|
+
#
|
|
68
|
+
# Raising instead — the way an unknown *code* raises — would be worse than the bug. `Supervisor#answer_for`
|
|
69
|
+
# builds its Failure as an argument to `answer`, so the raise would land before that method's rescue, and
|
|
70
|
+
# neither `reap` nor `drain_signals` nor `run` catches it. A typo would take the whole cell down from the
|
|
71
|
+
# one path whose job is reporting a dead worker.
|
|
72
|
+
FSIZE = "fsize"
|
|
73
|
+
MEMORY = "memory"
|
|
74
|
+
DEADLINE = "deadline"
|
|
75
|
+
CRASHED = "crashed"
|
|
76
|
+
|
|
77
|
+
# `signal` is the unexplained death, and it is not permanent, because a signal says how a process died and
|
|
78
|
+
# never why. The supervisor knows it sent SIGKILL for a deadline and says so. Every other signal arrived
|
|
79
|
+
# from somewhere it cannot see: a cgroup OOM kill chosen on aggregate pressure across concurrent workers,
|
|
80
|
+
# or one worker signalling another — they share a uid, and nothing stops that. Attributing either to the
|
|
81
|
+
# input this worker happened to be holding condemns a file for something it did not do.
|
|
82
|
+
#
|
|
83
|
+
# So the supervisor never infers either of them from a signal. They are decided in the worker, which is
|
|
84
|
+
# the process that holds the request: `memory` when it catches NoMemoryError itself, and `fsize` when a
|
|
85
|
+
# write of its own returns EFBIG. See Worker#disarm_file_size_signal for why the file-size verdict has
|
|
86
|
+
# to be earned that way rather than read off a wait status.
|
|
87
|
+
#
|
|
88
|
+
# That is a narrower guarantee than "a permanent verdict cannot be forged", and the difference matters.
|
|
89
|
+
# It removes the supervisor as an instrument: a sibling's signal no longer travels through a wait status
|
|
90
|
+
# into someone else's blob. It does nothing about a cell that has been compromised outright and answers
|
|
91
|
+
# a connection itself — `from_wire` believes a `permanent` boolean off the wire, and a worker that stole
|
|
92
|
+
# `work.sock` writes whatever it likes. That is the socket-theft residual `docs/DESIGN.md` records, and
|
|
93
|
+
# it is not closed here.
|
|
94
|
+
PERMANENT_BY_CAUSE = {
|
|
95
|
+
FSIZE => true,
|
|
96
|
+
MEMORY => true,
|
|
97
|
+
DEADLINE => false,
|
|
98
|
+
CRASHED => false,
|
|
99
|
+
}.freeze
|
|
100
|
+
|
|
101
|
+
KILLED = "killed"
|
|
102
|
+
|
|
103
|
+
class << self
|
|
104
|
+
def permanent?(code, cause: nil)
|
|
105
|
+
code = code.to_s
|
|
106
|
+
return PERMANENT_BY_CAUSE.fetch(cause.to_s, false) if code == KILLED
|
|
107
|
+
|
|
108
|
+
PERMANENT.fetch(code) do
|
|
109
|
+
raise ArgumentError, "unknown error code #{code.inspect}"
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def known?(code)
|
|
114
|
+
code.to_s == KILLED || PERMANENT.key?(code.to_s)
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module HotCell
|
|
6
|
+
# One request per connection: the cold side connects, sends one request with its descriptors, reads
|
|
7
|
+
# one response line, and closes. The hot side never initiates.
|
|
8
|
+
#
|
|
9
|
+
# SOCK_STREAM rather than SOCK_SEQPACKET, and the receiver pays for it. Real message boundaries would
|
|
10
|
+
# remove the framing problem outright, and Darwin has no AF_UNIX SOCK_SEQPACKET, so a stream socket is
|
|
11
|
+
# what there is. A stream socket does not promise that one sendmsg arrives as one recvmsg, and the
|
|
12
|
+
# ancillary data rides on whichever bytes land first. So read to the newline in a loop, and take the
|
|
13
|
+
# descriptors from whichever recvmsg carried them rather than from the one that completes the line.
|
|
14
|
+
#
|
|
15
|
+
# An implementation that assumes one sendmsg is one recvmsg passes every test small enough not to
|
|
16
|
+
# fragment, which is why this is written down here rather than left to be rediscovered.
|
|
17
|
+
class Connection
|
|
18
|
+
CHUNK_BYTES = 4096
|
|
19
|
+
|
|
20
|
+
attr_reader :socket
|
|
21
|
+
|
|
22
|
+
def initialize(socket)
|
|
23
|
+
@socket = socket
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# So that a connection can itself be passed over SCM_RIGHTS. That is how the supervisor hands an
|
|
27
|
+
# accepted connection to a worker without reading it: the caller's descriptors stay queued on the
|
|
28
|
+
# connection until somebody calls recvmsg, and the worker is the one who does.
|
|
29
|
+
def to_io
|
|
30
|
+
socket
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# The descriptors ride the first sendmsg and the rest of the line follows as ordinary writes, because a
|
|
34
|
+
# stream socket does not promise that one sendmsg sends all of it. The return value used to be ignored:
|
|
35
|
+
# a short send that carried the ancillary data left the receiver holding the descriptors and waiting for
|
|
36
|
+
# a newline that never came, while this side moved on to waiting for a response. Both ends then sat
|
|
37
|
+
# until something else timed them out — and the caller's descriptors were installed in the cell either
|
|
38
|
+
# way, so it is not a case that fails safe.
|
|
39
|
+
#
|
|
40
|
+
# Ancillary data goes exactly once. Sending it again with a later chunk would install a second copy of
|
|
41
|
+
# every descriptor in the receiver.
|
|
42
|
+
def send_message(line, descriptors: [])
|
|
43
|
+
sent = if descriptors.empty?
|
|
44
|
+
socket.sendmsg line
|
|
45
|
+
else
|
|
46
|
+
socket.sendmsg line, 0, nil, Socket::AncillaryData.unix_rights(*descriptors.map(&:to_io))
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
write_all line.byteslice(sent..) if sent < line.bytesize
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Returns [line, descriptors]. The line is nil when the peer closed without sending anything.
|
|
53
|
+
#
|
|
54
|
+
# The caller owns the descriptors and must close every one of them, including any it will not use.
|
|
55
|
+
# A request that is refused still arrives with its descriptors installed in this process.
|
|
56
|
+
def receive_message(limit: MAX_REQUEST_BYTES)
|
|
57
|
+
buffer = "".b
|
|
58
|
+
descriptors = []
|
|
59
|
+
|
|
60
|
+
loop do
|
|
61
|
+
chunk, _, _, *controls = socket.recvmsg(CHUNK_BYTES, 0, nil, scm_rights: true)
|
|
62
|
+
descriptors.concat controls.flat_map(&:unix_rights)
|
|
63
|
+
|
|
64
|
+
break if chunk.nil? || chunk.empty?
|
|
65
|
+
buffer << chunk
|
|
66
|
+
break if buffer.include?("\n")
|
|
67
|
+
|
|
68
|
+
raise MessageError, "message passed #{limit} bytes with no newline" if buffer.bytesize > limit
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
[ line_from(buffer, limit), descriptors ]
|
|
72
|
+
rescue StandardError
|
|
73
|
+
descriptors.each { |descriptor| descriptor.close unless descriptor.closed? }
|
|
74
|
+
raise
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# IO#write already loops until everything is written or it raises, which sendmsg does not.
|
|
78
|
+
def write_line(line)
|
|
79
|
+
write_all line
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# UTF-8 for the same reason receive_message forces it: a socket read comes back ASCII-8BIT, where every
|
|
83
|
+
# byte is "valid" and nothing downstream can tell a mis-encoded message from a good one. Both directions
|
|
84
|
+
# should behave the same way, and the one that scrubs is Failure.
|
|
85
|
+
#
|
|
86
|
+
# `deadline` is an absolute instant covering the whole line rather than a per-read timeout, because a
|
|
87
|
+
# per-read one bounds nothing. Waiting for the socket to be readable and then calling a blocking `gets`
|
|
88
|
+
# meant a peer that sent a single byte inside the timeout and then stopped held the caller until the
|
|
89
|
+
# cell's own deadline, or forever against a peer that never closed. Returns nil at end of stream, and
|
|
90
|
+
# raises Timeout when the deadline passes with the line incomplete.
|
|
91
|
+
def read_line(limit: MAX_RESPONSE_BYTES, deadline: nil)
|
|
92
|
+
buffer = "".b
|
|
93
|
+
|
|
94
|
+
until buffer.end_with?("\n")
|
|
95
|
+
chunk = read_chunk(deadline)
|
|
96
|
+
|
|
97
|
+
if chunk.nil?
|
|
98
|
+
return nil if buffer.empty?
|
|
99
|
+
|
|
100
|
+
raise MessageError, "message ended after #{buffer.bytesize} bytes with no newline"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
buffer << chunk
|
|
104
|
+
raise MessageError, "message passed #{limit} bytes with no newline" if buffer.bytesize > limit
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
buffer.force_encoding Encoding::UTF_8
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def close
|
|
111
|
+
socket.close unless socket.closed?
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
private
|
|
115
|
+
def write_all(bytes)
|
|
116
|
+
socket.write bytes
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Returns nil at end of stream. With no deadline this blocks, which is what the cell's own side wants —
|
|
120
|
+
# it is bounded by the supervisor rather than by a clock here.
|
|
121
|
+
def read_chunk(deadline)
|
|
122
|
+
if deadline
|
|
123
|
+
remaining = deadline - Clock.now
|
|
124
|
+
|
|
125
|
+
unless remaining.positive? && socket.wait_readable(remaining)
|
|
126
|
+
raise ReadTimeout, "the peer stopped mid-message and the deadline passed"
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
socket.readpartial CHUNK_BYTES
|
|
131
|
+
rescue EOFError
|
|
132
|
+
nil
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def line_from(buffer, limit)
|
|
136
|
+
return nil if buffer.empty?
|
|
137
|
+
|
|
138
|
+
newline = buffer.index("\n")
|
|
139
|
+
raise MessageError, "message ended after #{buffer.bytesize} bytes with no newline" if newline.nil?
|
|
140
|
+
|
|
141
|
+
if newline < buffer.bytesize - 1
|
|
142
|
+
raise MessageError, "#{buffer.bytesize - newline - 1} bytes follow the message line, and one " \
|
|
143
|
+
"connection carries one message"
|
|
144
|
+
end
|
|
145
|
+
raise MessageError, "message is #{buffer.bytesize} bytes, over the #{limit} byte limit" if buffer.bytesize > limit
|
|
146
|
+
|
|
147
|
+
buffer.force_encoding Encoding::UTF_8
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "hot_cell/core/version"
|
|
4
|
+
require "hot_cell/protocol"
|
|
5
|
+
require "hot_cell/errors"
|
|
6
|
+
require "hot_cell/codes"
|
|
7
|
+
require "hot_cell/clock"
|
|
8
|
+
require "hot_cell/naming"
|
|
9
|
+
require "hot_cell/declarations"
|
|
10
|
+
require "hot_cell/payload"
|
|
11
|
+
require "hot_cell/fields"
|
|
12
|
+
require "hot_cell/descriptors"
|
|
13
|
+
require "hot_cell/failure"
|
|
14
|
+
require "hot_cell/request"
|
|
15
|
+
require "hot_cell/response"
|
|
16
|
+
require "hot_cell/connection"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# How a class-level declaration is read back on both sides of the socket.
|
|
5
|
+
#
|
|
6
|
+
# `HotCell::Client` and `HotCell::Operation` are the same shape of thing — a class whose body declares what
|
|
7
|
+
# it is, with a subclass free to override one declaration and inherit the rest. Both needed the same lookup
|
|
8
|
+
# and both had their own copy of it, in gems that are never loaded together and so could never diverge
|
|
9
|
+
# loudly. This is the shared home they already have.
|
|
10
|
+
#
|
|
11
|
+
# A class-level instance variable is not visible to a subclass, which is what makes `@limits` on a base class
|
|
12
|
+
# invisible to the operation that inherits from it. Walking `ancestors` is what turns that into inheritance.
|
|
13
|
+
# `Class` only, because a module cannot carry one of these declarations.
|
|
14
|
+
module Declarations
|
|
15
|
+
private
|
|
16
|
+
# The first ancestor that set it, or nil. `false` is a value and survives; only nil means unset.
|
|
17
|
+
def inherited_value(variable)
|
|
18
|
+
ancestors.grep(Class).each do |ancestor|
|
|
19
|
+
value = ancestor.instance_variable_get(variable)
|
|
20
|
+
return value unless value.nil?
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
nil
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fcntl"
|
|
4
|
+
|
|
5
|
+
module HotCell
|
|
6
|
+
# A descriptor, not a path. The cold side opens the file and passes the open descriptor, so no path a
|
|
7
|
+
# hot side chooses is ever opened by the cold side, and no path the cold side chose is ever visible
|
|
8
|
+
# to a tool.
|
|
9
|
+
#
|
|
10
|
+
# These verify rather than merely tag, and both sides verify. An access mode is fixed at open and
|
|
11
|
+
# cannot be narrowed afterward, so a cell handed a read-write descriptor as an input cannot correct
|
|
12
|
+
# it — it can only decline the request.
|
|
13
|
+
class Descriptor
|
|
14
|
+
MODES = {
|
|
15
|
+
Fcntl::O_RDONLY => "read-only",
|
|
16
|
+
Fcntl::O_WRONLY => "write-only",
|
|
17
|
+
Fcntl::O_RDWR => "read-write",
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
attr_reader :io
|
|
21
|
+
|
|
22
|
+
def initialize(io, scratch: nil)
|
|
23
|
+
@io = io
|
|
24
|
+
@scratch = scratch
|
|
25
|
+
verify_regular_file!
|
|
26
|
+
verify_access_mode!
|
|
27
|
+
verify_position!
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def to_io
|
|
31
|
+
io
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# The path that reads or writes this descriptor in place, without a copy onto scratch. `/dev/fd/N`
|
|
35
|
+
# names the open file behind fd N: the worker itself may open it (libvips does), and a spawned tool
|
|
36
|
+
# sees it too when the worker hands the fd across the exec at the same number — see Operation#run_tool.
|
|
37
|
+
#
|
|
38
|
+
# This is what keeps a multi-gigabyte input off a small tmpfs: the descriptor is the caller's own file,
|
|
39
|
+
# readable at any size, where staging it would be a write and RLIMIT_FSIZE bounds writes. Reopening
|
|
40
|
+
# `/dev/fd/N` gives a fresh file description at offset zero, so a read here does not disturb the
|
|
41
|
+
# descriptor the supervisor still holds.
|
|
42
|
+
def fd_path
|
|
43
|
+
"/dev/fd/#{io.fileno}"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def close
|
|
47
|
+
io.close unless io.closed?
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Whether this descriptor has been given a filename on the worker's own scratch.
|
|
51
|
+
def staged?
|
|
52
|
+
!@path.nil?
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
# The worker hands each descriptor the scratch it may stage onto; the client hands over nothing,
|
|
57
|
+
# because a filename is the worker's concern and no path the cold side names must ever matter.
|
|
58
|
+
def scratch_path
|
|
59
|
+
raise Error, "#{self.class.name} has no scratch, so a path is not a question for this side" if @scratch.nil?
|
|
60
|
+
|
|
61
|
+
@scratch.call
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def verify_access_mode!
|
|
65
|
+
mode = io.fcntl(Fcntl::F_GETFL) & Fcntl::O_ACCMODE
|
|
66
|
+
return if mode == self.class::ACCESS_MODE
|
|
67
|
+
|
|
68
|
+
raise AccessModeError, "#{self.class.name} needs a #{MODES.fetch(self.class::ACCESS_MODE)} " \
|
|
69
|
+
"descriptor, and this one is #{MODES.fetch(mode, "mode #{mode}")}"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# O_APPEND is a caller bug of the same shape as handing over a read-write descriptor, and it fails as
|
|
73
|
+
# quietly: every write lands at the end whatever the cell does, so a conversion is appended to
|
|
74
|
+
# whatever the file already held and the caller reads its old bytes followed by new ones. Like the
|
|
75
|
+
# access mode this is fixed at open, so the only thing a cell can do about it is decline.
|
|
76
|
+
def verify_position!
|
|
77
|
+
return unless (io.fcntl(Fcntl::F_GETFL) & Fcntl::O_APPEND).positive?
|
|
78
|
+
|
|
79
|
+
raise AccessModeError, "#{self.class.name} was opened with O_APPEND, so a conversion would be " \
|
|
80
|
+
"appended to what the file already holds rather than becoming its contents"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# A pipe as an output can deadlock a single streaming write against a cold side that is not
|
|
84
|
+
# draining it, and a character device is nobody's conversion.
|
|
85
|
+
def verify_regular_file!
|
|
86
|
+
return if io.stat.file?
|
|
87
|
+
|
|
88
|
+
raise AccessModeError, "#{self.class.name} needs a regular file, and this one is a #{io.stat.ftype}"
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
class Input < Descriptor
|
|
93
|
+
ACCESS_MODE = Fcntl::O_RDONLY
|
|
94
|
+
|
|
95
|
+
# Copies the bytes onto the worker's own scratch on the first call and returns the filename. This is the
|
|
96
|
+
# fallback, for an operation that genuinely needs a real file. Prefer `fd_path`, which reads the
|
|
97
|
+
# descriptor in place: staging is a write, so RLIMIT_FSIZE bounds it, and an input larger than the
|
|
98
|
+
# operation's file_size dies here as a permanent `fsize` verdict — a ceiling Rails does not have. The
|
|
99
|
+
# Active Storage operations all read `fd_path` for exactly that reason; nothing should reach for `path`
|
|
100
|
+
# without a specific need for a distinct on-disk copy.
|
|
101
|
+
#
|
|
102
|
+
# On call rather than up front, so an operation that never asks for a staged path never pays for the copy.
|
|
103
|
+
def path
|
|
104
|
+
@path ||= copied_to(scratch_path)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
private
|
|
108
|
+
def copied_to(path)
|
|
109
|
+
File.open(path, "wb") { |file| IO.copy_stream(io, file) }
|
|
110
|
+
path
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
class Output < Descriptor
|
|
115
|
+
ACCESS_MODE = Fcntl::O_WRONLY
|
|
116
|
+
|
|
117
|
+
# Names the file the operation is to write, on the first call. Nothing is copied yet: post sends
|
|
118
|
+
# the file back out through the descriptor, and an operation that writes the descriptor directly
|
|
119
|
+
# never names one at all.
|
|
120
|
+
#
|
|
121
|
+
# `extension` names a suffixed sibling on the same scratch instead, for a producer that picks its
|
|
122
|
+
# saver from the extension (ImageProcessing) or appends one of its own (pdftoppm). The sibling is
|
|
123
|
+
# not what post ships: adopt renames it into place.
|
|
124
|
+
def path(extension: nil)
|
|
125
|
+
base = (@path ||= scratch_path)
|
|
126
|
+
extension.nil? ? base : "#{base}.#{extension}"
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Renames a staged sibling onto `path`, so post ships it. A sibling the tool never wrote is left
|
|
130
|
+
# to post's zero-byte accounting.
|
|
131
|
+
def adopt(staged)
|
|
132
|
+
File.rename staged, path if File.exist?(staged)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Sends whatever the operation produced back out through the descriptor, and returns the byte count
|
|
136
|
+
# the cold side can now read. Outputs are posted and flushed before success is reported, so a
|
|
137
|
+
# caller may read as soon as it sees `ok`.
|
|
138
|
+
#
|
|
139
|
+
# Two shapes, and an operation chooses by which path it wrote. A staged output was written to `path` on
|
|
140
|
+
# scratch and is copied out here; a file that does not exist means the operation wrote nothing, reported
|
|
141
|
+
# as zero the way a full tmpfs is. An output the operation wrote straight through the descriptor was
|
|
142
|
+
# never staged, so there is nothing to copy — the bytes are already in the caller's file — and this only
|
|
143
|
+
# flushes and measures. The direct form leaves partial bytes in the caller's file if the operation
|
|
144
|
+
# fails mid-write, where the staged form leaves it empty; every operation that writes directly turns a
|
|
145
|
+
# failure into a refusal the caller acts on rather than reading those bytes as a result.
|
|
146
|
+
def post
|
|
147
|
+
if staged?
|
|
148
|
+
return 0 unless File.exist?(path)
|
|
149
|
+
|
|
150
|
+
File.open(path, "rb") { |file| IO.copy_stream(file, io) }.tap { io.flush }
|
|
151
|
+
else
|
|
152
|
+
io.flush
|
|
153
|
+
io.stat.size
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# What this gem raises when a call is wrong, as distinct from a cell's verdict on a conversion.
|
|
5
|
+
#
|
|
6
|
+
# These must be raised outside the client's transport rescue. An application injects its own
|
|
7
|
+
# exception classes for a cell's verdicts, and if one of those descends from IOError then a bad call
|
|
8
|
+
# swallowed by the transport rescue comes back as a socket failure and gets retried forever.
|
|
9
|
+
class Error < StandardError; end
|
|
10
|
+
|
|
11
|
+
# A payload or result value JSON cannot carry faithfully. Raised before anything is sent, because
|
|
12
|
+
# to_json is not a check: it turns a Symbol into a String, a Time into a String, and an arbitrary
|
|
13
|
+
# object into whatever its own to_json says, all silently and none of it reversible.
|
|
14
|
+
class SerializationError < Error; end
|
|
15
|
+
|
|
16
|
+
# A descriptor offered as an input that is not read-only, or as an output that is not write-only.
|
|
17
|
+
# An access mode is fixed at open and cannot be narrowed afterward, so this can only be declined.
|
|
18
|
+
class AccessModeError < Error; end
|
|
19
|
+
|
|
20
|
+
# A message that is not what the protocol says it is: unparseable, too long, or missing a field.
|
|
21
|
+
class MessageError < Error; end
|
|
22
|
+
|
|
23
|
+
# Either side configured wrongly. Raised at boot rather than at the first request, because a cell that
|
|
24
|
+
# cannot hold its limits and a client that cannot classify a failure are both worse discovered in traffic.
|
|
25
|
+
class ConfigurationError < Error; end
|
|
26
|
+
|
|
27
|
+
# A peer that stopped mid-message. Distinct from a cell answering `timeout`, which is a verdict: this is
|
|
28
|
+
# the caller's own deadline passing with the response incomplete.
|
|
29
|
+
class ReadTimeout < Error; end
|
|
30
|
+
|
|
31
|
+
# A client naming a cell nobody registered.
|
|
32
|
+
class UnregisteredCell < Error; end
|
|
33
|
+
|
|
34
|
+
# A client calling a cell whose socket directory is unset, which means the deployment has not turned this
|
|
35
|
+
# path on. The caller is the one that knows what to do instead, so this is loud rather than silent.
|
|
36
|
+
class CellNotConfigured < Error; end
|
|
37
|
+
end
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# A cell's verdict on a request that did not succeed.
|
|
5
|
+
#
|
|
6
|
+
# The message is untrusted and it outlives the request. It comes out of a worker that has just parsed
|
|
7
|
+
# a hostile file, and Vips::Error#message routinely contains the input filename. Applications store
|
|
8
|
+
# these as durable blob metadata so they can re-decide later against a newer library, which means an
|
|
9
|
+
# unscrubbed byte sequence becomes a permanently poisoned row, and an invalid UTF-8 sequence makes a
|
|
10
|
+
# downstream regex raise ArgumentError instead of answering false.
|
|
11
|
+
#
|
|
12
|
+
# So the message is capped and scrubbed here, and that is not only hygiene: a cell that could not
|
|
13
|
+
# serialize its own error could not answer at all. The client scrubs again on receipt, because
|
|
14
|
+
# JSON.parse is not a filter — a \uD800 escape parses into an invalid UTF-8 String without complaint.
|
|
15
|
+
class Failure
|
|
16
|
+
MAX_MESSAGE_BYTES = 512
|
|
17
|
+
|
|
18
|
+
attr_reader :code, :message, :error_class, :cause, :signal
|
|
19
|
+
|
|
20
|
+
# Every field is sanitized, not only the message. All five arrive from the wire on the client side, so
|
|
21
|
+
# all five carry whatever the peer put there — and they travel further than the message does, into
|
|
22
|
+
# `to_s`, into the `perform.hot_cell` event, and into whatever a subscriber writes down. `code` in
|
|
23
|
+
# particular is the field applications store. Scrubbing one and not the other four left the same
|
|
24
|
+
# poisoned row the scrub exists to prevent, reachable through a different key.
|
|
25
|
+
def initialize(code:, permanent: nil, message: nil, error_class: nil, cause: nil, signal: nil)
|
|
26
|
+
@code = self.class.sanitize(code).to_s
|
|
27
|
+
@cause = self.class.sanitize(cause)
|
|
28
|
+
@signal = self.class.sanitize(signal)
|
|
29
|
+
@error_class = self.class.sanitize(error_class)
|
|
30
|
+
@message = self.class.sanitize(message)
|
|
31
|
+
@permanent = permanent.nil? ? Codes.permanent?(@code, cause: @cause) : permanent
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def permanent?
|
|
35
|
+
@permanent
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# compact rather than four guards: the constructor puts every one of these through `&.to_s` or
|
|
39
|
+
# `sanitize`, so each is a String or nil and there is no falsey-but-meaningful value to protect.
|
|
40
|
+
# `permanent` is the exception and survives, because compact drops only nil.
|
|
41
|
+
def to_h
|
|
42
|
+
{ code: code, permanent: permanent? }
|
|
43
|
+
.merge(cause: cause, signal: signal, class: error_class, message: message).compact
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def to_s
|
|
47
|
+
[ code, cause, error_class, message ].compact.join(": ")
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
class << self
|
|
51
|
+
# Builds from either a message String or an Exception. An Exception has to become two wire fields, and
|
|
52
|
+
# that rule was written out at three call sites across two gems — the worker, the supervisor's control
|
|
53
|
+
# answer, and the client's transport.
|
|
54
|
+
def for(code, detail, cause: nil)
|
|
55
|
+
if detail.is_a?(Exception)
|
|
56
|
+
new code: code, cause: cause, error_class: detail.class.name, message: detail.message
|
|
57
|
+
else
|
|
58
|
+
new code: code, cause: cause, message: detail
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# A code this client has never heard of is not permanent. An old client will meet a code added
|
|
63
|
+
# later, and the harm of the two mistakes is not symmetrical: retrying something permanent costs
|
|
64
|
+
# some work, while writing down a verdict that was temporary is irreversible.
|
|
65
|
+
# A `permanent` that is present but not a boolean is derived rather than believed. Truthiness would make
|
|
66
|
+
# any non-nil value permanent, and permanent is the answer that cannot be taken back — so a garbled
|
|
67
|
+
# field must not be able to say it.
|
|
68
|
+
def from_wire(wire)
|
|
69
|
+
permanent = if [ true, false ].include?(wire[:permanent])
|
|
70
|
+
wire[:permanent]
|
|
71
|
+
else
|
|
72
|
+
Codes.known?(wire[:code]) && Codes.permanent?(wire[:code], cause: wire[:cause])
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
new code: wire[:code], permanent: permanent, cause: wire[:cause], signal: wire[:signal],
|
|
76
|
+
error_class: wire[:class], message: wire[:message]
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# For a message that is about to be written as one line of a log. `sanitize` leaves CR and LF alone,
|
|
80
|
+
# which is right for a message an application stores or re-raises, but a peer that puts a newline in
|
|
81
|
+
# one writes a second log line of its own — formatted and indented like the real ones. Escape them,
|
|
82
|
+
# and the rest of the control characters with them.
|
|
83
|
+
def one_line(text)
|
|
84
|
+
sanitize(text)&.gsub(/[[:cntrl:]]/) { |character| character.dump[1..-2] }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def sanitize(message)
|
|
88
|
+
return nil if message.nil?
|
|
89
|
+
|
|
90
|
+
String(message).dup.force_encoding(Encoding::UTF_8)
|
|
91
|
+
.scrub("").byteslice(0, MAX_MESSAGE_BYTES).scrub("")
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# How a wire message reads its own fields, shared by Request and Response.
|
|
5
|
+
#
|
|
6
|
+
# The two had a copy each of the same parse preamble and the same typed-field check, differing only in
|
|
7
|
+
# whether the error said "request" or "response" — so a fix to the wording landed in one and not the other.
|
|
8
|
+
#
|
|
9
|
+
# The includer states its own `noun`. Deriving it from the class name looks tidier and does not work:
|
|
10
|
+
# Request's singleton defines `name(parsed, key)` for its own String field, which shadows Module#name.
|
|
11
|
+
module Fields
|
|
12
|
+
private
|
|
13
|
+
# Wraps the two errors every parse can produce: a line that is not JSON, and a line that is JSON but
|
|
14
|
+
# not an object. The block builds the message from the parsed Hash.
|
|
15
|
+
#
|
|
16
|
+
# Payload.parse already answers with MessageError however the JSON layer failed, so this only adds
|
|
17
|
+
# the noun. It rescues around that call alone rather than around the block, because the block's own
|
|
18
|
+
# field checks raise MessageError too and re-wording those as "not valid JSON" would be a lie.
|
|
19
|
+
def parse_message(line)
|
|
20
|
+
parsed = begin
|
|
21
|
+
Payload.parse(line)
|
|
22
|
+
rescue MessageError => error
|
|
23
|
+
raise MessageError, "#{noun} is not valid JSON: #{error.message}"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
raise MessageError, "#{noun} is not a JSON object" unless parsed.is_a?(Hash)
|
|
27
|
+
|
|
28
|
+
yield parsed
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def object(parsed, key)
|
|
32
|
+
value = parsed[key]
|
|
33
|
+
return value if value.is_a?(Hash)
|
|
34
|
+
|
|
35
|
+
raise MessageError, "#{noun} #{key} is #{value.inspect} and must be a JSON object"
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# An operation name is namespaced, and the namespace is what stops two operation sets colliding on one
|
|
5
|
+
# cell. Both sides derive the same default from the same rule, so a client and an operation that were
|
|
6
|
+
# never written together still agree.
|
|
7
|
+
#
|
|
8
|
+
# A trailing Operation is stripped, because the naming convention puts it on the cell-side class and not
|
|
9
|
+
# on the client — so TransformImage in the application and TransformImageOperation in the cell both
|
|
10
|
+
# derive "transform_image", the way Rails strips Controller from a controller's route name.
|
|
11
|
+
#
|
|
12
|
+
# This runs in one direction only. A name that arrived on the wire is never turned back into a constant:
|
|
13
|
+
# it is looked up in a registry that only ever holds classes the cell already loaded.
|
|
14
|
+
module Naming
|
|
15
|
+
class << self
|
|
16
|
+
def default_operation_name(klass)
|
|
17
|
+
if klass.name.nil?
|
|
18
|
+
raise ConfigurationError, "#{klass.inspect} is anonymous, so it needs an explicit name"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
*namespaces, base = klass.name.split("::")
|
|
22
|
+
base = base.delete_suffix("Operation") unless base == "Operation"
|
|
23
|
+
|
|
24
|
+
[ *namespaces, base ].map { |part| underscore(part) }.join(".")
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def underscore(camel)
|
|
28
|
+
camel.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module HotCell
|
|
6
|
+
# A payload is a JSON object, and the rules are stricter than JSON's in one direction and looser in
|
|
7
|
+
# another.
|
|
8
|
+
#
|
|
9
|
+
# Values must be JSON-native, because to_json is not a check. It serializes a Symbol to a String, a
|
|
10
|
+
# Time to a String, and an arbitrary object through whatever to_json that object happens to define,
|
|
11
|
+
# all silently and none of it faithfully. So validate the values, then serialize.
|
|
12
|
+
#
|
|
13
|
+
# Keys may be Strings or Symbols, because both serialize to the same JSON string and both arrive
|
|
14
|
+
# symbolized. `{ format: "png" }` is how a payload is naturally written and must not be rejected.
|
|
15
|
+
module Payload
|
|
16
|
+
# A payload sits one level inside the message envelope, so it gets one level less than the line.
|
|
17
|
+
MAX_DEPTH = MAX_NESTING - 1
|
|
18
|
+
|
|
19
|
+
class << self
|
|
20
|
+
def generate(object, name)
|
|
21
|
+
validate! object, name
|
|
22
|
+
JSON.generate object
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Keys are deep-symbolized here rather than in an operation, so an operation never has to know
|
|
26
|
+
# whether to reach for payload[:format] or payload["format"], and a nested hash can be splatted
|
|
27
|
+
# straight into a library's keyword arguments. Only keys: a Symbol value would not survive the
|
|
28
|
+
# round trip, which the JSON-native rule already forbids.
|
|
29
|
+
#
|
|
30
|
+
# JSON.parse only. Never JSON.load, and never create_additions, both of which instantiate
|
|
31
|
+
# arbitrary classes named by a json_class key in the document.
|
|
32
|
+
#
|
|
33
|
+
# Every way this can fail becomes one named failure, and the catch-all is the point rather than
|
|
34
|
+
# laziness. Callers used to name what JSON.parse raises, and naming it has now been wrong twice: a
|
|
35
|
+
# report that was not an object raised TypeError past a rescue for JSON::ParserError, and a key
|
|
36
|
+
# holding bytes that are not valid UTF-8 raises EncodingError past both. The supervisor reads worker
|
|
37
|
+
# reports through here inside the loop that enforces every request's deadline, and nothing above it
|
|
38
|
+
# rescues anything, so each miss is a one-line denial of service against every request in the cell.
|
|
39
|
+
#
|
|
40
|
+
# The body is a single JSON.parse call, so this is scoped to "the JSON layer failed" and cannot
|
|
41
|
+
# swallow a bug in our own code. NoMemoryError is deliberately not caught: it is not a StandardError,
|
|
42
|
+
# and a document large enough to raise it is the worker's own memory verdict rather than a bad line.
|
|
43
|
+
def parse(json)
|
|
44
|
+
JSON.parse json, symbolize_names: true, max_nesting: MAX_NESTING, allow_nan: false,
|
|
45
|
+
create_additions: false
|
|
46
|
+
rescue StandardError => error
|
|
47
|
+
raise MessageError, "#{error.class}: #{Failure.sanitize(error.message)}"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def validate!(object, name)
|
|
51
|
+
unless object.is_a?(Hash)
|
|
52
|
+
raise SerializationError, "#{name} is a #{object.class} and must be a Hash"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
walk object, name, 1
|
|
56
|
+
object
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
def walk(value, path, depth)
|
|
61
|
+
case value
|
|
62
|
+
when Hash
|
|
63
|
+
too_deep! path, depth
|
|
64
|
+
seen = {}
|
|
65
|
+
|
|
66
|
+
value.each do |key, nested|
|
|
67
|
+
unless key.is_a?(String) || key.is_a?(Symbol)
|
|
68
|
+
raise SerializationError,
|
|
69
|
+
"#{path} has a #{key.class} key #{key.inspect}; keys must be String or Symbol"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# A Hash holding both :a and "a" is legal Ruby and two distinct keys. JSON has one string key
|
|
73
|
+
# space, so it serializes to a document with "a" twice, and parsing that back keeps whichever
|
|
74
|
+
# came last — one of the values is gone and nothing said so. Refused here rather than silently
|
|
75
|
+
# dropped, because this runs on results too, where the loss would be the caller's data.
|
|
76
|
+
if (clash = seen[key.to_s])
|
|
77
|
+
raise SerializationError,
|
|
78
|
+
"#{path} has both #{clash.inspect} and #{key.inspect}, which are one key in JSON"
|
|
79
|
+
end
|
|
80
|
+
seen[key.to_s] = key
|
|
81
|
+
|
|
82
|
+
walk nested, "#{path}[#{key.inspect}]", depth + 1
|
|
83
|
+
end
|
|
84
|
+
when Array
|
|
85
|
+
too_deep! path, depth
|
|
86
|
+
value.each_with_index { |nested, index| walk nested, "#{path}[#{index}]", depth + 1 }
|
|
87
|
+
when Float
|
|
88
|
+
unless value.finite?
|
|
89
|
+
raise SerializationError, "#{path} is #{value}, which JSON cannot carry"
|
|
90
|
+
end
|
|
91
|
+
when String
|
|
92
|
+
# JSON.generate refuses a String whose bytes are not valid UTF-8, and it refuses it *after* this
|
|
93
|
+
# has said the structure is fine. In a worker that meant dying with no answer at all, on a
|
|
94
|
+
# response that had already been decided — so the caller read a closed socket and retried
|
|
95
|
+
# something that will fail the same way every time. Tools produce these: a filename, or a
|
|
96
|
+
# line of stderr. Scrub it in the operation if you want it; it is not scrubbed here, because a
|
|
97
|
+
# result is data the caller acts on rather than a diagnostic.
|
|
98
|
+
unless value.valid_encoding?
|
|
99
|
+
raise SerializationError, "#{path} is a String whose bytes are not valid #{value.encoding}"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
value
|
|
103
|
+
when Integer, true, false, nil
|
|
104
|
+
value
|
|
105
|
+
else
|
|
106
|
+
raise SerializationError, "#{path} is a #{value.class}, which JSON cannot carry faithfully"
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def too_deep!(path, depth)
|
|
111
|
+
return if depth <= MAX_DEPTH
|
|
112
|
+
raise SerializationError, "#{path} nests deeper than #{MAX_DEPTH} levels"
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# Checked, never negotiated. A cell answers `protocol` to anything else, which is transient rather
|
|
5
|
+
# than a bug: during a rolling deploy the app moves before the accessory reboots, so every request
|
|
6
|
+
# mismatches until it does.
|
|
7
|
+
PROTOCOL_VERSION = 1
|
|
8
|
+
|
|
9
|
+
# A request is a control message from the trusted side, and the worker parses it before it has
|
|
10
|
+
# narrowed to the operation's limits. Capping it is what bounds that parse.
|
|
11
|
+
MAX_REQUEST_BYTES = 8192
|
|
12
|
+
|
|
13
|
+
# A response carries metadata rather than bytes, so this is generous rather than tight.
|
|
14
|
+
MAX_RESPONSE_BYTES = 65_536
|
|
15
|
+
|
|
16
|
+
# The kernel caps SCM_RIGHTS at 253 descriptors per message. Nothing here wants more than a handful.
|
|
17
|
+
MAX_DESCRIPTORS = 16
|
|
18
|
+
|
|
19
|
+
# Nesting depth of a whole message line. Deep symbolization walks the structure, so this is what
|
|
20
|
+
# bounds the recursion.
|
|
21
|
+
MAX_NESTING = 8
|
|
22
|
+
|
|
23
|
+
# The two operations a cell answers on control.sock rather than work.sock. The names live here because
|
|
24
|
+
# both sides need them and neither owns them.
|
|
25
|
+
DESCRIBE = "hotcell.describe"
|
|
26
|
+
METRICS = "hotcell.metrics"
|
|
27
|
+
CONTROL_OPERATIONS = [ DESCRIBE, METRICS ].freeze
|
|
28
|
+
end
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# One line of UTF-8 JSON terminated by a newline, sent with a single sendmsg carrying one SCM_RIGHTS
|
|
5
|
+
# message.
|
|
6
|
+
#
|
|
7
|
+
# {"v":1,"op":"active_storage.transform_image","inputs":1,"outputs":1,"payload":{"format":"png"}}
|
|
8
|
+
#
|
|
9
|
+
# Inputs are the leading descriptors and outputs the trailing ones, so two counts are the whole
|
|
10
|
+
# framing and there is no naming layer to keep in step with an operation.
|
|
11
|
+
class Request
|
|
12
|
+
attr_reader :version, :op, :inputs, :outputs, :payload
|
|
13
|
+
|
|
14
|
+
def initialize(op:, inputs: 0, outputs: 0, payload: {}, version: PROTOCOL_VERSION)
|
|
15
|
+
@version = version
|
|
16
|
+
@op = op
|
|
17
|
+
@inputs = inputs
|
|
18
|
+
@outputs = outputs
|
|
19
|
+
@payload = payload
|
|
20
|
+
|
|
21
|
+
verify_descriptor_count!
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def descriptor_count
|
|
25
|
+
inputs + outputs
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def current_version?
|
|
29
|
+
version == PROTOCOL_VERSION
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Next to the predicate it explains, because both sockets answer with it and a caller may well be
|
|
33
|
+
# grepping for it — the one failure that arrives at a hundred percent during a rolling deploy.
|
|
34
|
+
def version_mismatch
|
|
35
|
+
"this cell speaks v#{PROTOCOL_VERSION} and the request is v#{version}"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def to_line
|
|
39
|
+
Payload.validate! payload, "payload"
|
|
40
|
+
|
|
41
|
+
line = JSON.generate({ v: version, op: op, inputs: inputs, outputs: outputs, payload: payload }) << "\n"
|
|
42
|
+
if line.bytesize > MAX_REQUEST_BYTES
|
|
43
|
+
raise MessageError, "request is #{line.bytesize} bytes, over the #{MAX_REQUEST_BYTES} byte limit"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
line
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
class << self
|
|
50
|
+
include Fields
|
|
51
|
+
|
|
52
|
+
def noun
|
|
53
|
+
"request"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def parse(line)
|
|
57
|
+
parse_message(line) do |parsed|
|
|
58
|
+
new version: integer(parsed, :v), op: name(parsed, :op), inputs: count(parsed, :inputs),
|
|
59
|
+
outputs: count(parsed, :outputs), payload: object(parsed, :payload)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
def integer(parsed, key)
|
|
65
|
+
value = parsed[key]
|
|
66
|
+
return value if value.is_a?(Integer)
|
|
67
|
+
|
|
68
|
+
raise MessageError, "request #{key} is #{value.inspect} and must be an Integer"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def count(parsed, key)
|
|
72
|
+
integer(parsed, key).tap do |value|
|
|
73
|
+
raise MessageError, "request #{key} is #{value} and must not be negative" if value.negative?
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def name(parsed, key)
|
|
78
|
+
value = parsed[key]
|
|
79
|
+
return value if value.is_a?(String) && !value.empty?
|
|
80
|
+
|
|
81
|
+
raise MessageError, "request #{key} is #{value.inspect} and must be a non-empty String"
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
def verify_descriptor_count!
|
|
87
|
+
if descriptor_count > MAX_DESCRIPTORS
|
|
88
|
+
raise MessageError, "request wants #{descriptor_count} descriptors, over the #{MAX_DESCRIPTORS} limit"
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HotCell
|
|
4
|
+
# One line of JSON. Outputs are posted and flushed before success is reported, so the cold side may
|
|
5
|
+
# read as soon as it sees `ok`.
|
|
6
|
+
#
|
|
7
|
+
# `timing` is present on every response, success or failure, and it is also the cell's tracing
|
|
8
|
+
# channel. A cell cannot reach a trace collector, so anything it knows about its own internals
|
|
9
|
+
# travels back here or not at all. An operation may add its own keys, and a client should treat the
|
|
10
|
+
# map as open.
|
|
11
|
+
class Response
|
|
12
|
+
attr_reader :version, :result, :failure, :timing
|
|
13
|
+
|
|
14
|
+
class << self
|
|
15
|
+
include Fields
|
|
16
|
+
|
|
17
|
+
def noun
|
|
18
|
+
"response"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def ok(result: {}, timing: {})
|
|
22
|
+
new result: result, timing: timing
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def failed(failure, timing: {})
|
|
26
|
+
new failure: failure, timing: timing
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# `ok` has to be the boolean it says it is rather than anything truthy. This is the field the whole
|
|
30
|
+
# taxonomy turns on, and Ruby's truthiness would read `"false"`, `0` and `[]` as success — so a
|
|
31
|
+
# malformed or hostile response could turn a failure into an `ok` carrying no result at all.
|
|
32
|
+
#
|
|
33
|
+
# **Accepted risk.** Nothing inside `result` is checked, and a compromised cell chooses all of it.
|
|
34
|
+
# `Payload.validate!` is not run on the way in, so a value JSON can carry but Ruby cannot serialize
|
|
35
|
+
# again reaches the caller — `1e400` parses to `Float::INFINITY`, and an application that writes a
|
|
36
|
+
# result to a JSON column raises there rather than here. The premise is that a result is data the
|
|
37
|
+
# caller acts on and only the caller knows its shape, so the framework has no rule to apply and a
|
|
38
|
+
# client with a specific expectation states it itself. `ok` is the exception because the taxonomy
|
|
39
|
+
# turns on it and every caller depends on it equally.
|
|
40
|
+
def parse(line)
|
|
41
|
+
parse_message(line) do |parsed|
|
|
42
|
+
ok = parsed[:ok]
|
|
43
|
+
unless ok == true || ok == false
|
|
44
|
+
raise MessageError, "response ok is #{ok.inspect} and must be true or false"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
timing = parsed[:timing].is_a?(Hash) ? parsed[:timing] : {}
|
|
48
|
+
|
|
49
|
+
if ok
|
|
50
|
+
new version: parsed[:v], result: object(parsed, :result), timing: timing
|
|
51
|
+
else
|
|
52
|
+
new version: parsed[:v], failure: Failure.from_wire(object(parsed, :error)), timing: timing
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def initialize(result: nil, failure: nil, timing: {}, version: PROTOCOL_VERSION)
|
|
59
|
+
@result = result
|
|
60
|
+
@failure = failure
|
|
61
|
+
@timing = timing
|
|
62
|
+
@version = version
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def ok?
|
|
66
|
+
failure.nil?
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def to_line
|
|
70
|
+
body = { v: version, ok: ok? }
|
|
71
|
+
if ok?
|
|
72
|
+
body[:result] = Payload.validate!(result, "result")
|
|
73
|
+
else
|
|
74
|
+
body[:error] = failure.to_h
|
|
75
|
+
end
|
|
76
|
+
body[:timing] = Payload.validate!(timing, "timing")
|
|
77
|
+
|
|
78
|
+
line = JSON.generate(body) << "\n"
|
|
79
|
+
if line.bytesize > MAX_RESPONSE_BYTES
|
|
80
|
+
raise MessageError, "response is #{line.bytesize} bytes, over the #{MAX_RESPONSE_BYTES} byte limit"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
line
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tempfile"
|
|
4
|
+
|
|
5
|
+
module HotCell
|
|
6
|
+
# Fixtures and waits for the suites of all three gems, shipped rather than copied.
|
|
7
|
+
#
|
|
8
|
+
# Same reasoning as `hot_cell/test_cell.rb` shipping from hotcell-server: a helper every consumer needs is
|
|
9
|
+
# one every consumer should get, not one each writes again. Before this, `with_file` had three
|
|
10
|
+
# byte-identical copies, `wait_until` had two — one of them inside a test file rather than a helper — and
|
|
11
|
+
# the monotonic clock was hand-rolled six times in code that already loads HotCell::Clock.
|
|
12
|
+
#
|
|
13
|
+
# It lives in hotcell-core because that is the gem all three load. Requiring it is opt-in, so nothing here
|
|
14
|
+
# reaches a production graph unless a suite asks for it.
|
|
15
|
+
#
|
|
16
|
+
# Include into a Minitest::Test subclass; `skip` and `flunk` are called on the including test.
|
|
17
|
+
module TestSupport
|
|
18
|
+
private
|
|
19
|
+
# Descriptors have to be regular files, so every fixture is a real file on disk.
|
|
20
|
+
def with_file(contents = "")
|
|
21
|
+
Tempfile.create([ "hotcell", ".bin" ], binmode: true) do |file|
|
|
22
|
+
file.write contents
|
|
23
|
+
file.flush
|
|
24
|
+
yield file.path
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# A source and a destination, which is the shape of most requests.
|
|
29
|
+
def with_files(contents = "source bytes")
|
|
30
|
+
with_file(contents) { |source| with_file { |destination| yield source, destination } }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def reading(path, &block)
|
|
34
|
+
File.open path, "rb", &block
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def writing(path, &block)
|
|
38
|
+
File.open path, "wb", &block
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def updating(path, &block)
|
|
42
|
+
File.open path, "r+b", &block
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def open_descriptors
|
|
46
|
+
skip "counting open descriptors needs Linux" unless File.directory?("/proc/self/fd")
|
|
47
|
+
|
|
48
|
+
Dir.children("/proc/self/fd").size
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# True while the process exists and is not a zombie. A killed process lingers as a zombie until its
|
|
52
|
+
# parent reaps it, and a zombie is dead for a caller watching for a sweep. The state is the first
|
|
53
|
+
# token after the final `)` of /proc/<pid>/stat, which is where the comm field ends.
|
|
54
|
+
# ESRCH alongside ENOENT: the entry can be torn down between the open and the read, and a caller
|
|
55
|
+
# polling for a kill is exactly who reads during the teardown.
|
|
56
|
+
def process_running?(pid)
|
|
57
|
+
state = File.read("/proc/#{pid}/stat")[/\)\s+(\S)/, 1]
|
|
58
|
+
!state.nil? && state != "Z"
|
|
59
|
+
rescue Errno::ENOENT, Errno::ESRCH
|
|
60
|
+
false
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# A cell writes its log and removes its scratch after it has answered, so a few properties are
|
|
64
|
+
# genuinely asynchronous with respect to the caller. Bounded polling says so; a sleep would not.
|
|
65
|
+
def wait_until(within: 5, what: "the condition")
|
|
66
|
+
deadline = Clock.now + within
|
|
67
|
+
|
|
68
|
+
until yield
|
|
69
|
+
flunk "#{what} did not happen within #{within}s" if Clock.now > deadline
|
|
70
|
+
|
|
71
|
+
sleep 0.01
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def elapsed
|
|
76
|
+
at = Clock.now
|
|
77
|
+
yield
|
|
78
|
+
Clock.now - at
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
data/lib/hotcell-core.rb
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Bundler auto-requires a gem named "hotcell-core" as "hotcell-core", then as "hotcell/core". This gem
|
|
4
|
+
# uses neither path, because hot_cell/ is what yields the HotCell constant under the default inflection.
|
|
5
|
+
# Without this file, `gem "hotcell-core"` in a Gemfile silently loads nothing at all.
|
|
6
|
+
require "hot_cell/core"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: hotcell-core
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mike Dalessio
|
|
@@ -9,16 +9,44 @@ bindir: bin
|
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies: []
|
|
12
|
-
description:
|
|
12
|
+
description: |
|
|
13
|
+
The wire protocol between an application and a HotCell container: the request and response format,
|
|
14
|
+
file descriptor passing over SCM_RIGHTS, payload validation, and the error taxonomy. Both
|
|
15
|
+
hotcell-client and hotcell-server depend on it. Install one of those rather than this.
|
|
13
16
|
email:
|
|
14
17
|
- mike@37signals.com
|
|
15
18
|
executables: []
|
|
16
19
|
extensions: []
|
|
17
20
|
extra_rdoc_files: []
|
|
18
|
-
files:
|
|
21
|
+
files:
|
|
22
|
+
- MIT-LICENSE
|
|
23
|
+
- README.md
|
|
24
|
+
- lib/hot_cell/clock.rb
|
|
25
|
+
- lib/hot_cell/codes.rb
|
|
26
|
+
- lib/hot_cell/connection.rb
|
|
27
|
+
- lib/hot_cell/core.rb
|
|
28
|
+
- lib/hot_cell/core/version.rb
|
|
29
|
+
- lib/hot_cell/declarations.rb
|
|
30
|
+
- lib/hot_cell/descriptors.rb
|
|
31
|
+
- lib/hot_cell/errors.rb
|
|
32
|
+
- lib/hot_cell/failure.rb
|
|
33
|
+
- lib/hot_cell/fields.rb
|
|
34
|
+
- lib/hot_cell/naming.rb
|
|
35
|
+
- lib/hot_cell/payload.rb
|
|
36
|
+
- lib/hot_cell/protocol.rb
|
|
37
|
+
- lib/hot_cell/request.rb
|
|
38
|
+
- lib/hot_cell/response.rb
|
|
39
|
+
- lib/hot_cell/test_support.rb
|
|
40
|
+
- lib/hotcell-core.rb
|
|
41
|
+
homepage: https://github.com/basecamp/hotcell
|
|
19
42
|
licenses:
|
|
20
43
|
- MIT
|
|
21
|
-
metadata:
|
|
44
|
+
metadata:
|
|
45
|
+
homepage_uri: https://github.com/basecamp/hotcell
|
|
46
|
+
source_code_uri: https://github.com/basecamp/hotcell/tree/v0.2.0/hotcell-core
|
|
47
|
+
changelog_uri: https://github.com/basecamp/hotcell/blob/v0.2.0/CHANGELOG.md
|
|
48
|
+
bug_tracker_uri: https://github.com/basecamp/hotcell/issues
|
|
49
|
+
rubygems_mfa_required: 'true'
|
|
22
50
|
rdoc_options: []
|
|
23
51
|
require_paths:
|
|
24
52
|
- lib
|
|
@@ -26,7 +54,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
26
54
|
requirements:
|
|
27
55
|
- - ">="
|
|
28
56
|
- !ruby/object:Gem::Version
|
|
29
|
-
version: '
|
|
57
|
+
version: '3.3'
|
|
30
58
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
31
59
|
requirements:
|
|
32
60
|
- - ">="
|
|
@@ -35,5 +63,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
35
63
|
requirements: []
|
|
36
64
|
rubygems_version: 4.0.16
|
|
37
65
|
specification_version: 4
|
|
38
|
-
summary:
|
|
66
|
+
summary: The wire protocol shared by both sides of a HotCell.
|
|
39
67
|
test_files: []
|