hotcell-core 0.0.0 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f6b2888231865d0473516c75ab8b1caf8fe162d414b5d63978dad521e0cf65e5
4
- data.tar.gz: dae5536d6035f67546136f12ffe1fc52d602bdffc8dd5eeba120fec2313c4cd7
3
+ metadata.gz: b90b3c31e988d71fe6f9b1da6c1e727d8c25b934c5085757367fe6e38ffd5aef
4
+ data.tar.gz: d8eed61e4eeb2d52fe65571ff75b34ff0f73d37ef96c615ec5ab88b7e4d64696
5
5
  SHA512:
6
- metadata.gz: ba01bcb50b70fb26fd0c01e5e1a7d60fac57cc7d965638289fd8a603f1edd4294a21786305231e022628f9211e4b2d51da66483d52f183f898343e7a27550800
7
- data.tar.gz: 8cf8d948c812075b068479e25737a0d43999831f2dea87a018d6d62fe8c1049d96bb27076f88173ed91d4bd4ae1939669d768402ee9e8ce8e5c215c14014d909
6
+ metadata.gz: 5b8e87d2713d7d8aae5091a469f3a16be1a1dc860e37c4a0b40aa2b3819a1ee3336f7807e88cbb4ae35892f4a6ad5ad892a3c239a6f4928108ecd584946d1d19
7
+ data.tar.gz: 16ea6b1a37ab997e25d4553f697a6fd390b3b4dbe6869a6b703e0bccf1a1a8467fb14e95921e19c0125e48462aecc4ac578214729573106d4f00e5c0a8ac509c
data/MIT-LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 37signals, LLC
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,3 @@
1
+ # hotcell-core
2
+
3
+ Part of [HotCell](https://github.com/basecamp/hotcell). See the repository README.
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # Monotonic, because every duration here is reported to a caller that will alarm on it, and a clock
5
+ # that can step backwards produces negative latencies during an NTP correction.
6
+ module Clock
7
+ class << self
8
+ def now
9
+ Process.clock_gettime Process::CLOCK_MONOTONIC
10
+ end
11
+
12
+ def ms_since(at)
13
+ ((now - at) * 1000).round(2)
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,106 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # Every failure carries a code and a `permanent` flag, and `permanent` is the only distinction that
5
+ # changes what a caller must do.
6
+ #
7
+ # Terminal means the same request fails the same way until the input or the code changes — not until
8
+ # the load or the deployment changes. A permanent failure may be recorded against a blob and served
9
+ # from a cache. A non-permanent one must be retried and must never be written down.
10
+ #
11
+ # The flag travels on the wire, set by the side that knows, rather than being derived by each caller
12
+ # from the code. That is what makes a code added later safe: an old client will not recognise it but
13
+ # will still dispose of it correctly.
14
+ module Codes
15
+ PERMANENT = {
16
+ "unreadable" => true, # the input could not be decoded — the operation said so explicitly
17
+ "invalid" => true, # malformed request, or a descriptor that failed its access-mode check
18
+ "failed" => false, # the operation raised something nobody classified — see below
19
+ "unsupported" => false, # this cell does not carry that operation — see below
20
+ "protocol" => false, # version mismatch, which heals when the accessory reboots
21
+ "capacity" => false, # the queue is full
22
+ "unavailable" => false, # no connection, or a connection closed with no response
23
+ "timeout" => false, # the client's own deadline fired
24
+ }.freeze
25
+
26
+ # **`failed` is what an unclassified exception becomes, so it cannot be permanent.**
27
+ #
28
+ # A worker rescues StandardError around the whole request and calls it `failed`. Errno::ENOSPC is a
29
+ # StandardError, and so are EMFILE, EIO, ENOENT and ENOMEM. A shared tmpfs filled by concurrent requests,
30
+ # a full disk under the caller's own output, a descriptor table exhausted by load, a fork that cannot get
31
+ # memory under host pressure, a tool missing during a broken deploy — every one of those raises inside
32
+ # staging or writeback and arrives here. ENOMEM is the one that looks like the input's fault and is not:
33
+ # the worker's own out-of-memory, where the input drove it past its limit, is a NoMemoryError, which the
34
+ # worker classifies `killed`/`memory` instead.
35
+ #
36
+ # Terminal meant each of them was written down against a customer's file and served from a cache forever,
37
+ # for a condition that would have succeeded on retry. Permanence has to be claimed, never inferred from
38
+ # not knowing: an operation says `unreadable` for an input it could not decode, and the protocol says
39
+ # `invalid` for a caller that broke its own contract. Everything else is this cell having a bad day.
40
+ #
41
+ # The cost of the other mistake is a genuinely broken operation being retried. That is bounded by the
42
+ # job's attempts, it is visible in the `failed` rate, and it is recoverable.
43
+
44
+ # **`unsupported` is transient, and the design document says otherwise.** Its reasoning was that an
45
+ # unknown *operation* is a caller bug that never heals, where an unknown *version* is a deploy window
46
+ # that does. That holds for a typo and not for the case that actually happens: an accessory is not
47
+ # updated by a deploy, so an application that ships a client for a new operation before anybody reboots
48
+ # the cell gets `unsupported` at one hundred percent for as long as that takes.
49
+ #
50
+ # The two mistakes are not symmetrical. Retrying a caller's typo costs some work and shows up in the
51
+ # `unsupported` rate and in the client's boot-time warning. Recording a deploy window as permanent
52
+ # condemns every blob uploaded during it, and needs a hand-written backfill to undo.
53
+
54
+ # `killed` splits on what the worker hit, because a caller cannot otherwise tell a decompression
55
+ # bomb from a slow afternoon. Size and memory are properties of the input, so the same bytes will
56
+ # do it again on an idle cell. A deadline is as much a property of the load, and treating it as
57
+ # permanent means a busy hour permanently condemns whatever was uploaded during it.
58
+ # `crashed` is the cell's own fault rather than the input's — a worker that died without answering, which
59
+ # a misconfigured cell does on every request. Recording that against a blob would condemn everything
60
+ # uploaded during a broken deploy, so it is transient. An older client that has never heard of it still
61
+ # disposes of it correctly, because `permanent` travels on the wire.
62
+ #
63
+ # **A limit this table has never heard of is not permanent, and that default is the point of the table.**
64
+ # A cell mints these; adding a kill reason to the supervisor without adding a row here used to make it
65
+ # permanent, silently, and permanent is the answer that cannot be taken back. The names below are
66
+ # constants so that the two places that mint them cannot spell one the table does not carry.
67
+ #
68
+ # Raising instead — the way an unknown *code* raises — would be worse than the bug. `Supervisor#answer_for`
69
+ # builds its Failure as an argument to `answer`, so the raise would land before that method's rescue, and
70
+ # neither `reap` nor `drain_signals` nor `run` catches it. A typo would take the whole cell down from the
71
+ # one path whose job is reporting a dead worker.
72
+ FSIZE = "fsize"
73
+ MEMORY = "memory"
74
+ DEADLINE = "deadline"
75
+ CRASHED = "crashed"
76
+
77
+ # `signal` is the unexplained death, and it is not permanent, because a signal says how a process died and
78
+ # never why. The supervisor knows it sent SIGKILL for a deadline and says so. Every other signal arrived
79
+ # from somewhere it cannot see: a cgroup OOM kill chosen on aggregate pressure across concurrent workers,
80
+ # or one worker signalling another — they share a uid, and nothing stops that. Attributing either to the
81
+ # input this worker happened to be holding condemns a file for something it did not do.
82
+ PERMANENT_BY_CAUSE = {
83
+ FSIZE => true,
84
+ MEMORY => true,
85
+ DEADLINE => false,
86
+ CRASHED => false,
87
+ }.freeze
88
+
89
+ KILLED = "killed"
90
+
91
+ class << self
92
+ def permanent?(code, cause: nil)
93
+ code = code.to_s
94
+ return PERMANENT_BY_CAUSE.fetch(cause.to_s, false) if code == KILLED
95
+
96
+ PERMANENT.fetch(code) do
97
+ raise ArgumentError, "unknown error code #{code.inspect}"
98
+ end
99
+ end
100
+
101
+ def known?(code)
102
+ code.to_s == KILLED || PERMANENT.key?(code.to_s)
103
+ end
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,150 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module HotCell
6
+ # One request per connection: the cold side connects, sends one request with its descriptors, reads
7
+ # one response line, and closes. The hot side never initiates.
8
+ #
9
+ # SOCK_STREAM rather than SOCK_SEQPACKET, and the receiver pays for it. Real message boundaries would
10
+ # remove the framing problem outright, and Darwin has no AF_UNIX SOCK_SEQPACKET, so a stream socket is
11
+ # what there is. A stream socket does not promise that one sendmsg arrives as one recvmsg, and the
12
+ # ancillary data rides on whichever bytes land first. So read to the newline in a loop, and take the
13
+ # descriptors from whichever recvmsg carried them rather than from the one that completes the line.
14
+ #
15
+ # An implementation that assumes one sendmsg is one recvmsg passes every test small enough not to
16
+ # fragment, which is why this is written down here rather than left to be rediscovered.
17
+ class Connection
18
+ CHUNK_BYTES = 4096
19
+
20
+ attr_reader :socket
21
+
22
+ def initialize(socket)
23
+ @socket = socket
24
+ end
25
+
26
+ # So that a connection can itself be passed over SCM_RIGHTS. That is how the supervisor hands an
27
+ # accepted connection to a worker without reading it: the caller's descriptors stay queued on the
28
+ # connection until somebody calls recvmsg, and the worker is the one who does.
29
+ def to_io
30
+ socket
31
+ end
32
+
33
+ # The descriptors ride the first sendmsg and the rest of the line follows as ordinary writes, because a
34
+ # stream socket does not promise that one sendmsg sends all of it. The return value used to be ignored:
35
+ # a short send that carried the ancillary data left the receiver holding the descriptors and waiting for
36
+ # a newline that never came, while this side moved on to waiting for a response. Both ends then sat
37
+ # until something else timed them out — and the caller's descriptors were installed in the cell either
38
+ # way, so it is not a case that fails safe.
39
+ #
40
+ # Ancillary data goes exactly once. Sending it again with a later chunk would install a second copy of
41
+ # every descriptor in the receiver.
42
+ def send_message(line, descriptors: [])
43
+ sent = if descriptors.empty?
44
+ socket.sendmsg line
45
+ else
46
+ socket.sendmsg line, 0, nil, Socket::AncillaryData.unix_rights(*descriptors.map(&:to_io))
47
+ end
48
+
49
+ write_all line.byteslice(sent..) if sent < line.bytesize
50
+ end
51
+
52
+ # Returns [line, descriptors]. The line is nil when the peer closed without sending anything.
53
+ #
54
+ # The caller owns the descriptors and must close every one of them, including any it will not use.
55
+ # A request that is refused still arrives with its descriptors installed in this process.
56
+ def receive_message(limit: MAX_REQUEST_BYTES)
57
+ buffer = "".b
58
+ descriptors = []
59
+
60
+ loop do
61
+ chunk, _, _, *controls = socket.recvmsg(CHUNK_BYTES, 0, nil, scm_rights: true)
62
+ descriptors.concat controls.flat_map(&:unix_rights)
63
+
64
+ break if chunk.nil? || chunk.empty?
65
+ buffer << chunk
66
+ break if buffer.include?("\n")
67
+
68
+ raise MessageError, "message passed #{limit} bytes with no newline" if buffer.bytesize > limit
69
+ end
70
+
71
+ [ line_from(buffer, limit), descriptors ]
72
+ rescue StandardError
73
+ descriptors.each { |descriptor| descriptor.close unless descriptor.closed? }
74
+ raise
75
+ end
76
+
77
+ # IO#write already loops until everything is written or it raises, which sendmsg does not.
78
+ def write_line(line)
79
+ write_all line
80
+ end
81
+
82
+ # UTF-8 for the same reason receive_message forces it: a socket read comes back ASCII-8BIT, where every
83
+ # byte is "valid" and nothing downstream can tell a mis-encoded message from a good one. Both directions
84
+ # should behave the same way, and the one that scrubs is Failure.
85
+ #
86
+ # `deadline` is an absolute instant covering the whole line rather than a per-read timeout, because a
87
+ # per-read one bounds nothing. Waiting for the socket to be readable and then calling a blocking `gets`
88
+ # meant a peer that sent a single byte inside the timeout and then stopped held the caller until the
89
+ # cell's own deadline, or forever against a peer that never closed. Returns nil at end of stream, and
90
+ # raises Timeout when the deadline passes with the line incomplete.
91
+ def read_line(limit: MAX_RESPONSE_BYTES, deadline: nil)
92
+ buffer = "".b
93
+
94
+ until buffer.end_with?("\n")
95
+ chunk = read_chunk(deadline)
96
+
97
+ if chunk.nil?
98
+ return nil if buffer.empty?
99
+
100
+ raise MessageError, "message ended after #{buffer.bytesize} bytes with no newline"
101
+ end
102
+
103
+ buffer << chunk
104
+ raise MessageError, "message passed #{limit} bytes with no newline" if buffer.bytesize > limit
105
+ end
106
+
107
+ buffer.force_encoding Encoding::UTF_8
108
+ end
109
+
110
+ def close
111
+ socket.close unless socket.closed?
112
+ end
113
+
114
+ private
115
+ def write_all(bytes)
116
+ socket.write bytes
117
+ end
118
+
119
+ # Returns nil at end of stream. With no deadline this blocks, which is what the cell's own side wants —
120
+ # it is bounded by the supervisor rather than by a clock here.
121
+ def read_chunk(deadline)
122
+ if deadline
123
+ remaining = deadline - Clock.now
124
+
125
+ unless remaining.positive? && socket.wait_readable(remaining)
126
+ raise ReadTimeout, "the peer stopped mid-message and the deadline passed"
127
+ end
128
+ end
129
+
130
+ socket.readpartial CHUNK_BYTES
131
+ rescue EOFError
132
+ nil
133
+ end
134
+
135
+ def line_from(buffer, limit)
136
+ return nil if buffer.empty?
137
+
138
+ newline = buffer.index("\n")
139
+ raise MessageError, "message ended after #{buffer.bytesize} bytes with no newline" if newline.nil?
140
+
141
+ if newline < buffer.bytesize - 1
142
+ raise MessageError, "#{buffer.bytesize - newline - 1} bytes follow the message line, and one " \
143
+ "connection carries one message"
144
+ end
145
+ raise MessageError, "message is #{buffer.bytesize} bytes, over the #{limit} byte limit" if buffer.bytesize > limit
146
+
147
+ buffer.force_encoding Encoding::UTF_8
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ VERSION = "0.1.0"
5
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "hot_cell/core/version"
4
+ require "hot_cell/protocol"
5
+ require "hot_cell/errors"
6
+ require "hot_cell/codes"
7
+ require "hot_cell/clock"
8
+ require "hot_cell/naming"
9
+ require "hot_cell/declarations"
10
+ require "hot_cell/payload"
11
+ require "hot_cell/fields"
12
+ require "hot_cell/descriptors"
13
+ require "hot_cell/failure"
14
+ require "hot_cell/request"
15
+ require "hot_cell/response"
16
+ require "hot_cell/connection"
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # How a class-level declaration is read back on both sides of the socket.
5
+ #
6
+ # `HotCell::Client` and `HotCell::Operation` are the same shape of thing — a class whose body declares what
7
+ # it is, with a subclass free to override one declaration and inherit the rest. Both needed the same lookup
8
+ # and both had their own copy of it, in gems that are never loaded together and so could never diverge
9
+ # loudly. This is the shared home they already have.
10
+ #
11
+ # A class-level instance variable is not visible to a subclass, which is what makes `@limits` on a base class
12
+ # invisible to the operation that inherits from it. Walking `ancestors` is what turns that into inheritance.
13
+ # `Class` only, because a module cannot carry one of these declarations.
14
+ module Declarations
15
+ private
16
+ # The first ancestor that set it, or nil. `false` is a value and survives; only nil means unset.
17
+ def inherited_value(variable)
18
+ ancestors.grep(Class).each do |ancestor|
19
+ value = ancestor.instance_variable_get(variable)
20
+ return value unless value.nil?
21
+ end
22
+
23
+ nil
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,157 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fcntl"
4
+
5
+ module HotCell
6
+ # A descriptor, not a path. The cold side opens the file and passes the open descriptor, so no path a
7
+ # hot side chooses is ever opened by the cold side, and no path the cold side chose is ever visible
8
+ # to a tool.
9
+ #
10
+ # These verify rather than merely tag, and both sides verify. An access mode is fixed at open and
11
+ # cannot be narrowed afterward, so a cell handed a read-write descriptor as an input cannot correct
12
+ # it — it can only decline the request.
13
+ class Descriptor
14
+ MODES = {
15
+ Fcntl::O_RDONLY => "read-only",
16
+ Fcntl::O_WRONLY => "write-only",
17
+ Fcntl::O_RDWR => "read-write",
18
+ }.freeze
19
+
20
+ attr_reader :io
21
+
22
+ def initialize(io, scratch: nil)
23
+ @io = io
24
+ @scratch = scratch
25
+ verify_regular_file!
26
+ verify_access_mode!
27
+ verify_position!
28
+ end
29
+
30
+ def to_io
31
+ io
32
+ end
33
+
34
+ # The path that reads or writes this descriptor in place, without a copy onto scratch. `/dev/fd/N`
35
+ # names the open file behind fd N: the worker itself may open it (libvips does), and a spawned tool
36
+ # sees it too when the worker hands the fd across the exec at the same number — see Operation#run_tool.
37
+ #
38
+ # This is what keeps a multi-gigabyte input off a small tmpfs: the descriptor is the caller's own file,
39
+ # readable at any size, where staging it would be a write and RLIMIT_FSIZE bounds writes. Reopening
40
+ # `/dev/fd/N` gives a fresh file description at offset zero, so a read here does not disturb the
41
+ # descriptor the supervisor still holds.
42
+ def fd_path
43
+ "/dev/fd/#{io.fileno}"
44
+ end
45
+
46
+ def close
47
+ io.close unless io.closed?
48
+ end
49
+
50
+ # Whether this descriptor has been given a filename on the worker's own scratch.
51
+ def staged?
52
+ !@path.nil?
53
+ end
54
+
55
+ private
56
+ # The worker hands each descriptor the scratch it may stage onto; the client hands over nothing,
57
+ # because a filename is the worker's concern and no path the cold side names must ever matter.
58
+ def scratch_path
59
+ raise Error, "#{self.class.name} has no scratch, so a path is not a question for this side" if @scratch.nil?
60
+
61
+ @scratch.call
62
+ end
63
+
64
+ def verify_access_mode!
65
+ mode = io.fcntl(Fcntl::F_GETFL) & Fcntl::O_ACCMODE
66
+ return if mode == self.class::ACCESS_MODE
67
+
68
+ raise AccessModeError, "#{self.class.name} needs a #{MODES.fetch(self.class::ACCESS_MODE)} " \
69
+ "descriptor, and this one is #{MODES.fetch(mode, "mode #{mode}")}"
70
+ end
71
+
72
+ # O_APPEND is a caller bug of the same shape as handing over a read-write descriptor, and it fails as
73
+ # quietly: every write lands at the end whatever the cell does, so a conversion is appended to
74
+ # whatever the file already held and the caller reads its old bytes followed by new ones. Like the
75
+ # access mode this is fixed at open, so the only thing a cell can do about it is decline.
76
+ def verify_position!
77
+ return unless (io.fcntl(Fcntl::F_GETFL) & Fcntl::O_APPEND).positive?
78
+
79
+ raise AccessModeError, "#{self.class.name} was opened with O_APPEND, so a conversion would be " \
80
+ "appended to what the file already holds rather than becoming its contents"
81
+ end
82
+
83
+ # A pipe as an output can deadlock a single streaming write against a cold side that is not
84
+ # draining it, and a character device is nobody's conversion.
85
+ def verify_regular_file!
86
+ return if io.stat.file?
87
+
88
+ raise AccessModeError, "#{self.class.name} needs a regular file, and this one is a #{io.stat.ftype}"
89
+ end
90
+ end
91
+
92
+ class Input < Descriptor
93
+ ACCESS_MODE = Fcntl::O_RDONLY
94
+
95
+ # Copies the bytes onto the worker's own scratch on the first call and returns the filename. This is the
96
+ # fallback, for an operation that genuinely needs a real file. Prefer `fd_path`, which reads the
97
+ # descriptor in place: staging is a write, so RLIMIT_FSIZE bounds it, and an input larger than the
98
+ # operation's file_size dies here as a permanent `fsize` verdict — a ceiling Rails does not have. The
99
+ # Active Storage operations all read `fd_path` for exactly that reason; nothing should reach for `path`
100
+ # without a specific need for a distinct on-disk copy.
101
+ #
102
+ # On call rather than up front, so an operation that never asks for a staged path never pays for the copy.
103
+ def path
104
+ @path ||= copied_to(scratch_path)
105
+ end
106
+
107
+ private
108
+ def copied_to(path)
109
+ File.open(path, "wb") { |file| IO.copy_stream(io, file) }
110
+ path
111
+ end
112
+ end
113
+
114
+ class Output < Descriptor
115
+ ACCESS_MODE = Fcntl::O_WRONLY
116
+
117
+ # Names the file the operation is to write, on the first call. Nothing is copied yet: post sends
118
+ # the file back out through the descriptor, and an operation that writes the descriptor directly
119
+ # never names one at all.
120
+ #
121
+ # `extension` names a suffixed sibling on the same scratch instead, for a producer that picks its
122
+ # saver from the extension (ImageProcessing) or appends one of its own (pdftoppm). The sibling is
123
+ # not what post ships: adopt renames it into place.
124
+ def path(extension: nil)
125
+ base = (@path ||= scratch_path)
126
+ extension.nil? ? base : "#{base}.#{extension}"
127
+ end
128
+
129
+ # Renames a staged sibling onto `path`, so post ships it. A sibling the tool never wrote is left
130
+ # to post's zero-byte accounting.
131
+ def adopt(staged)
132
+ File.rename staged, path if File.exist?(staged)
133
+ end
134
+
135
+ # Sends whatever the operation produced back out through the descriptor, and returns the byte count
136
+ # the cold side can now read. Outputs are posted and flushed before success is reported, so a
137
+ # caller may read as soon as it sees `ok`.
138
+ #
139
+ # Two shapes, and an operation chooses by which path it wrote. A staged output was written to `path` on
140
+ # scratch and is copied out here; a file that does not exist means the operation wrote nothing, reported
141
+ # as zero the way a full tmpfs is. An output the operation wrote straight through the descriptor was
142
+ # never staged, so there is nothing to copy — the bytes are already in the caller's file — and this only
143
+ # flushes and measures. The direct form leaves partial bytes in the caller's file if the operation
144
+ # fails mid-write, where the staged form leaves it empty; every operation that writes directly turns a
145
+ # failure into a refusal the caller acts on rather than reading those bytes as a result.
146
+ def post
147
+ if staged?
148
+ return 0 unless File.exist?(path)
149
+
150
+ File.open(path, "rb") { |file| IO.copy_stream(file, io) }.tap { io.flush }
151
+ else
152
+ io.flush
153
+ io.stat.size
154
+ end
155
+ end
156
+ end
157
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # What this gem raises when a call is wrong, as distinct from a cell's verdict on a conversion.
5
+ #
6
+ # These must be raised outside the client's transport rescue. An application injects its own
7
+ # exception classes for a cell's verdicts, and if one of those descends from IOError then a bad call
8
+ # swallowed by the transport rescue comes back as a socket failure and gets retried forever.
9
+ class Error < StandardError; end
10
+
11
+ # A payload or result value JSON cannot carry faithfully. Raised before anything is sent, because
12
+ # to_json is not a check: it turns a Symbol into a String, a Time into a String, and an arbitrary
13
+ # object into whatever its own to_json says, all silently and none of it reversible.
14
+ class SerializationError < Error; end
15
+
16
+ # A descriptor offered as an input that is not read-only, or as an output that is not write-only.
17
+ # An access mode is fixed at open and cannot be narrowed afterward, so this can only be declined.
18
+ class AccessModeError < Error; end
19
+
20
+ # A message that is not what the protocol says it is: unparseable, too long, or missing a field.
21
+ class MessageError < Error; end
22
+
23
+ # Either side configured wrongly. Raised at boot rather than at the first request, because a cell that
24
+ # cannot hold its limits and a client that cannot classify a failure are both worse discovered in traffic.
25
+ class ConfigurationError < Error; end
26
+
27
+ # A peer that stopped mid-message. Distinct from a cell answering `timeout`, which is a verdict: this is
28
+ # the caller's own deadline passing with the response incomplete.
29
+ class ReadTimeout < Error; end
30
+
31
+ # A client naming a cell nobody registered.
32
+ class UnregisteredCell < Error; end
33
+
34
+ # A client calling a cell whose socket directory is unset, which means the deployment has not turned this
35
+ # path on. The caller is the one that knows what to do instead, so this is loud rather than silent.
36
+ class CellNotConfigured < Error; end
37
+ end
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # A cell's verdict on a request that did not succeed.
5
+ #
6
+ # The message is untrusted and it outlives the request. It comes out of a worker that has just parsed
7
+ # a hostile file, and Vips::Error#message routinely contains the input filename. Applications store
8
+ # these as durable blob metadata so they can re-decide later against a newer library, which means an
9
+ # unscrubbed byte sequence becomes a permanently poisoned row, and an invalid UTF-8 sequence makes a
10
+ # downstream regex raise ArgumentError instead of answering false.
11
+ #
12
+ # So the message is capped and scrubbed here, and that is not only hygiene: a cell that could not
13
+ # serialize its own error could not answer at all. The client scrubs again on receipt, because
14
+ # JSON.parse is not a filter — a \uD800 escape parses into an invalid UTF-8 String without complaint.
15
+ class Failure
16
+ MAX_MESSAGE_BYTES = 512
17
+
18
+ attr_reader :code, :message, :error_class, :cause, :signal
19
+
20
+ # Every field is sanitized, not only the message. All five arrive from the wire on the client side, so
21
+ # all five carry whatever the peer put there — and they travel further than the message does, into
22
+ # `to_s`, into the `perform.hot_cell` event, and into whatever a subscriber writes down. `code` in
23
+ # particular is the field applications store. Scrubbing one and not the other four left the same
24
+ # poisoned row the scrub exists to prevent, reachable through a different key.
25
+ def initialize(code:, permanent: nil, message: nil, error_class: nil, cause: nil, signal: nil)
26
+ @code = self.class.sanitize(code).to_s
27
+ @cause = self.class.sanitize(cause)
28
+ @signal = self.class.sanitize(signal)
29
+ @error_class = self.class.sanitize(error_class)
30
+ @message = self.class.sanitize(message)
31
+ @permanent = permanent.nil? ? Codes.permanent?(@code, cause: @cause) : permanent
32
+ end
33
+
34
+ def permanent?
35
+ @permanent
36
+ end
37
+
38
+ # compact rather than four guards: the constructor puts every one of these through `&.to_s` or
39
+ # `sanitize`, so each is a String or nil and there is no falsey-but-meaningful value to protect.
40
+ # `permanent` is the exception and survives, because compact drops only nil.
41
+ def to_h
42
+ { code: code, permanent: permanent? }
43
+ .merge(cause: cause, signal: signal, class: error_class, message: message).compact
44
+ end
45
+
46
+ def to_s
47
+ [ code, cause, error_class, message ].compact.join(": ")
48
+ end
49
+
50
+ class << self
51
+ # Builds from either a message String or an Exception. An Exception has to become two wire fields, and
52
+ # that rule was written out at three call sites across two gems — the worker, the supervisor's control
53
+ # answer, and the client's transport.
54
+ def for(code, detail, cause: nil)
55
+ if detail.is_a?(Exception)
56
+ new code: code, cause: cause, error_class: detail.class.name, message: detail.message
57
+ else
58
+ new code: code, cause: cause, message: detail
59
+ end
60
+ end
61
+
62
+ # A code this client has never heard of is not permanent. An old client will meet a code added
63
+ # later, and the harm of the two mistakes is not symmetrical: retrying something permanent costs
64
+ # some work, while writing down a verdict that was temporary is irreversible.
65
+ # A `permanent` that is present but not a boolean is derived rather than believed. Truthiness would make
66
+ # any non-nil value permanent, and permanent is the answer that cannot be taken back — so a garbled
67
+ # field must not be able to say it.
68
+ def from_wire(wire)
69
+ permanent = if [ true, false ].include?(wire[:permanent])
70
+ wire[:permanent]
71
+ else
72
+ Codes.known?(wire[:code]) && Codes.permanent?(wire[:code], cause: wire[:cause])
73
+ end
74
+
75
+ new code: wire[:code], permanent: permanent, cause: wire[:cause], signal: wire[:signal],
76
+ error_class: wire[:class], message: wire[:message]
77
+ end
78
+
79
+ def sanitize(message)
80
+ return nil if message.nil?
81
+
82
+ String(message).dup.force_encoding(Encoding::UTF_8)
83
+ .scrub("").byteslice(0, MAX_MESSAGE_BYTES).scrub("")
84
+ end
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # How a wire message reads its own fields, shared by Request and Response.
5
+ #
6
+ # The two had a copy each of the same parse preamble and the same typed-field check, differing only in
7
+ # whether the error said "request" or "response" — so a fix to the wording landed in one and not the other.
8
+ #
9
+ # The includer states its own `noun`. Deriving it from the class name looks tidier and does not work:
10
+ # Request's singleton defines `name(parsed, key)` for its own String field, which shadows Module#name.
11
+ module Fields
12
+ private
13
+ # Wraps the two errors every parse can produce: a line that is not JSON, and a line that is JSON but
14
+ # not an object. The block builds the message from the parsed Hash.
15
+ #
16
+ # Payload.parse already answers with MessageError however the JSON layer failed, so this only adds
17
+ # the noun. It rescues around that call alone rather than around the block, because the block's own
18
+ # field checks raise MessageError too and re-wording those as "not valid JSON" would be a lie.
19
+ def parse_message(line)
20
+ parsed = begin
21
+ Payload.parse(line)
22
+ rescue MessageError => error
23
+ raise MessageError, "#{noun} is not valid JSON: #{error.message}"
24
+ end
25
+
26
+ raise MessageError, "#{noun} is not a JSON object" unless parsed.is_a?(Hash)
27
+
28
+ yield parsed
29
+ end
30
+
31
+ def object(parsed, key)
32
+ value = parsed[key]
33
+ return value if value.is_a?(Hash)
34
+
35
+ raise MessageError, "#{noun} #{key} is #{value.inspect} and must be a JSON object"
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # An operation name is namespaced, and the namespace is what stops two operation sets colliding on one
5
+ # cell. Both sides derive the same default from the same rule, so a client and an operation that were
6
+ # never written together still agree.
7
+ #
8
+ # A trailing Operation is stripped, because the naming convention puts it on the cell-side class and not
9
+ # on the client — so TransformImage in the application and TransformImageOperation in the cell both
10
+ # derive "transform_image", the way Rails strips Controller from a controller's route name.
11
+ #
12
+ # This runs in one direction only. A name that arrived on the wire is never turned back into a constant:
13
+ # it is looked up in a registry that only ever holds classes the cell already loaded.
14
+ module Naming
15
+ class << self
16
+ def default_operation_name(klass)
17
+ if klass.name.nil?
18
+ raise ConfigurationError, "#{klass.inspect} is anonymous, so it needs an explicit name"
19
+ end
20
+
21
+ *namespaces, base = klass.name.split("::")
22
+ base = base.delete_suffix("Operation") unless base == "Operation"
23
+
24
+ [ *namespaces, base ].map { |part| underscore(part) }.join(".")
25
+ end
26
+
27
+ def underscore(camel)
28
+ camel.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module HotCell
6
+ # A payload is a JSON object, and the rules are stricter than JSON's in one direction and looser in
7
+ # another.
8
+ #
9
+ # Values must be JSON-native, because to_json is not a check. It serializes a Symbol to a String, a
10
+ # Time to a String, and an arbitrary object through whatever to_json that object happens to define,
11
+ # all silently and none of it faithfully. So validate the values, then serialize.
12
+ #
13
+ # Keys may be Strings or Symbols, because both serialize to the same JSON string and both arrive
14
+ # symbolized. `{ format: "png" }` is how a payload is naturally written and must not be rejected.
15
+ module Payload
16
+ # A payload sits one level inside the message envelope, so it gets one level less than the line.
17
+ MAX_DEPTH = MAX_NESTING - 1
18
+
19
+ class << self
20
+ def generate(object, name)
21
+ validate! object, name
22
+ JSON.generate object
23
+ end
24
+
25
+ # Keys are deep-symbolized here rather than in an operation, so an operation never has to know
26
+ # whether to reach for payload[:format] or payload["format"], and a nested hash can be splatted
27
+ # straight into a library's keyword arguments. Only keys: a Symbol value would not survive the
28
+ # round trip, which the JSON-native rule already forbids.
29
+ #
30
+ # JSON.parse only. Never JSON.load, and never create_additions, both of which instantiate
31
+ # arbitrary classes named by a json_class key in the document.
32
+ #
33
+ # Every way this can fail becomes one named failure, and the catch-all is the point rather than
34
+ # laziness. Callers used to name what JSON.parse raises, and naming it has now been wrong twice: a
35
+ # report that was not an object raised TypeError past a rescue for JSON::ParserError, and a key
36
+ # holding bytes that are not valid UTF-8 raises EncodingError past both. The supervisor reads worker
37
+ # reports through here inside the loop that enforces every request's deadline, and nothing above it
38
+ # rescues anything, so each miss is a one-line denial of service against every request in the cell.
39
+ #
40
+ # The body is a single JSON.parse call, so this is scoped to "the JSON layer failed" and cannot
41
+ # swallow a bug in our own code. NoMemoryError is deliberately not caught: it is not a StandardError,
42
+ # and a document large enough to raise it is the worker's own memory verdict rather than a bad line.
43
+ def parse(json)
44
+ JSON.parse json, symbolize_names: true, max_nesting: MAX_NESTING, allow_nan: false,
45
+ create_additions: false
46
+ rescue StandardError => error
47
+ raise MessageError, "#{error.class}: #{Failure.sanitize(error.message)}"
48
+ end
49
+
50
+ def validate!(object, name)
51
+ unless object.is_a?(Hash)
52
+ raise SerializationError, "#{name} is a #{object.class} and must be a Hash"
53
+ end
54
+
55
+ walk object, name, 1
56
+ object
57
+ end
58
+
59
+ private
60
+ def walk(value, path, depth)
61
+ case value
62
+ when Hash
63
+ too_deep! path, depth
64
+ seen = {}
65
+
66
+ value.each do |key, nested|
67
+ unless key.is_a?(String) || key.is_a?(Symbol)
68
+ raise SerializationError,
69
+ "#{path} has a #{key.class} key #{key.inspect}; keys must be String or Symbol"
70
+ end
71
+
72
+ # A Hash holding both :a and "a" is legal Ruby and two distinct keys. JSON has one string key
73
+ # space, so it serializes to a document with "a" twice, and parsing that back keeps whichever
74
+ # came last — one of the values is gone and nothing said so. Refused here rather than silently
75
+ # dropped, because this runs on results too, where the loss would be the caller's data.
76
+ if (clash = seen[key.to_s])
77
+ raise SerializationError,
78
+ "#{path} has both #{clash.inspect} and #{key.inspect}, which are one key in JSON"
79
+ end
80
+ seen[key.to_s] = key
81
+
82
+ walk nested, "#{path}[#{key.inspect}]", depth + 1
83
+ end
84
+ when Array
85
+ too_deep! path, depth
86
+ value.each_with_index { |nested, index| walk nested, "#{path}[#{index}]", depth + 1 }
87
+ when Float
88
+ unless value.finite?
89
+ raise SerializationError, "#{path} is #{value}, which JSON cannot carry"
90
+ end
91
+ when String
92
+ # JSON.generate refuses a String whose bytes are not valid UTF-8, and it refuses it *after* this
93
+ # has said the structure is fine. In a worker that meant dying with no answer at all, on a
94
+ # response that had already been decided — so the caller read a closed socket and retried
95
+ # something that will fail the same way every time. Tools produce these: a filename, or a
96
+ # line of stderr. Scrub it in the operation if you want it; it is not scrubbed here, because a
97
+ # result is data the caller acts on rather than a diagnostic.
98
+ unless value.valid_encoding?
99
+ raise SerializationError, "#{path} is a String whose bytes are not valid #{value.encoding}"
100
+ end
101
+
102
+ value
103
+ when Integer, true, false, nil
104
+ value
105
+ else
106
+ raise SerializationError, "#{path} is a #{value.class}, which JSON cannot carry faithfully"
107
+ end
108
+ end
109
+
110
+ def too_deep!(path, depth)
111
+ return if depth <= MAX_DEPTH
112
+ raise SerializationError, "#{path} nests deeper than #{MAX_DEPTH} levels"
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # Checked, never negotiated. A cell answers `protocol` to anything else, which is transient rather
5
+ # than a bug: during a rolling deploy the app moves before the accessory reboots, so every request
6
+ # mismatches until it does.
7
+ PROTOCOL_VERSION = 1
8
+
9
+ # A request is a control message from the trusted side, and the worker parses it before it has
10
+ # narrowed to the operation's limits. Capping it is what bounds that parse.
11
+ MAX_REQUEST_BYTES = 8192
12
+
13
+ # A response carries metadata rather than bytes, so this is generous rather than tight.
14
+ MAX_RESPONSE_BYTES = 65_536
15
+
16
+ # The kernel caps SCM_RIGHTS at 253 descriptors per message. Nothing here wants more than a handful.
17
+ MAX_DESCRIPTORS = 16
18
+
19
+ # Nesting depth of a whole message line. Deep symbolization walks the structure, so this is what
20
+ # bounds the recursion.
21
+ MAX_NESTING = 8
22
+
23
+ # The two operations a cell answers on control.sock rather than work.sock. The names live here because
24
+ # both sides need them and neither owns them.
25
+ DESCRIBE = "hotcell.describe"
26
+ METRICS = "hotcell.metrics"
27
+ CONTROL_OPERATIONS = [ DESCRIBE, METRICS ].freeze
28
+ end
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # One line of UTF-8 JSON terminated by a newline, sent with a single sendmsg carrying one SCM_RIGHTS
5
+ # message.
6
+ #
7
+ # {"v":1,"op":"active_storage.transform_image","inputs":1,"outputs":1,"payload":{"format":"png"}}
8
+ #
9
+ # Inputs are the leading descriptors and outputs the trailing ones, so two counts are the whole
10
+ # framing and there is no naming layer to keep in step with an operation.
11
+ class Request
12
+ attr_reader :version, :op, :inputs, :outputs, :payload
13
+
14
+ def initialize(op:, inputs: 0, outputs: 0, payload: {}, version: PROTOCOL_VERSION)
15
+ @version = version
16
+ @op = op
17
+ @inputs = inputs
18
+ @outputs = outputs
19
+ @payload = payload
20
+
21
+ verify_descriptor_count!
22
+ end
23
+
24
+ def descriptor_count
25
+ inputs + outputs
26
+ end
27
+
28
+ def current_version?
29
+ version == PROTOCOL_VERSION
30
+ end
31
+
32
+ # Next to the predicate it explains, because both sockets answer with it and a caller may well be
33
+ # grepping for it — the one failure that arrives at a hundred percent during a rolling deploy.
34
+ def version_mismatch
35
+ "this cell speaks v#{PROTOCOL_VERSION} and the request is v#{version}"
36
+ end
37
+
38
+ def to_line
39
+ Payload.validate! payload, "payload"
40
+
41
+ line = JSON.generate({ v: version, op: op, inputs: inputs, outputs: outputs, payload: payload }) << "\n"
42
+ if line.bytesize > MAX_REQUEST_BYTES
43
+ raise MessageError, "request is #{line.bytesize} bytes, over the #{MAX_REQUEST_BYTES} byte limit"
44
+ end
45
+
46
+ line
47
+ end
48
+
49
+ class << self
50
+ include Fields
51
+
52
+ def noun
53
+ "request"
54
+ end
55
+
56
+ def parse(line)
57
+ parse_message(line) do |parsed|
58
+ new version: integer(parsed, :v), op: name(parsed, :op), inputs: count(parsed, :inputs),
59
+ outputs: count(parsed, :outputs), payload: object(parsed, :payload)
60
+ end
61
+ end
62
+
63
+ private
64
+ def integer(parsed, key)
65
+ value = parsed[key]
66
+ return value if value.is_a?(Integer)
67
+
68
+ raise MessageError, "request #{key} is #{value.inspect} and must be an Integer"
69
+ end
70
+
71
+ def count(parsed, key)
72
+ integer(parsed, key).tap do |value|
73
+ raise MessageError, "request #{key} is #{value} and must not be negative" if value.negative?
74
+ end
75
+ end
76
+
77
+ def name(parsed, key)
78
+ value = parsed[key]
79
+ return value if value.is_a?(String) && !value.empty?
80
+
81
+ raise MessageError, "request #{key} is #{value.inspect} and must be a non-empty String"
82
+ end
83
+ end
84
+
85
+ private
86
+ def verify_descriptor_count!
87
+ if descriptor_count > MAX_DESCRIPTORS
88
+ raise MessageError, "request wants #{descriptor_count} descriptors, over the #{MAX_DESCRIPTORS} limit"
89
+ end
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HotCell
4
+ # One line of JSON. Outputs are posted and flushed before success is reported, so the cold side may
5
+ # read as soon as it sees `ok`.
6
+ #
7
+ # `timing` is present on every response, success or failure, and it is also the cell's tracing
8
+ # channel. A cell cannot reach a trace collector, so anything it knows about its own internals
9
+ # travels back here or not at all. An operation may add its own keys, and a client should treat the
10
+ # map as open.
11
+ class Response
12
+ attr_reader :version, :result, :failure, :timing
13
+
14
+ class << self
15
+ include Fields
16
+
17
+ def noun
18
+ "response"
19
+ end
20
+
21
+ def ok(result: {}, timing: {})
22
+ new result: result, timing: timing
23
+ end
24
+
25
+ def failed(failure, timing: {})
26
+ new failure: failure, timing: timing
27
+ end
28
+
29
+ # `ok` has to be the boolean it says it is rather than anything truthy. This is the field the whole
30
+ # taxonomy turns on, and Ruby's truthiness would read `"false"`, `0` and `[]` as success — so a
31
+ # malformed or hostile response could turn a failure into an `ok` carrying no result at all.
32
+ #
33
+ # **Accepted risk.** Nothing inside `result` is checked, and a compromised cell chooses all of it.
34
+ # `Payload.validate!` is not run on the way in, so a value JSON can carry but Ruby cannot serialize
35
+ # again reaches the caller — `1e400` parses to `Float::INFINITY`, and an application that writes a
36
+ # result to a JSON column raises there rather than here. The premise is that a result is data the
37
+ # caller acts on and only the caller knows its shape, so the framework has no rule to apply and a
38
+ # client with a specific expectation states it itself. `ok` is the exception because the taxonomy
39
+ # turns on it and every caller depends on it equally.
40
+ def parse(line)
41
+ parse_message(line) do |parsed|
42
+ ok = parsed[:ok]
43
+ unless ok == true || ok == false
44
+ raise MessageError, "response ok is #{ok.inspect} and must be true or false"
45
+ end
46
+
47
+ timing = parsed[:timing].is_a?(Hash) ? parsed[:timing] : {}
48
+
49
+ if ok
50
+ new version: parsed[:v], result: object(parsed, :result), timing: timing
51
+ else
52
+ new version: parsed[:v], failure: Failure.from_wire(object(parsed, :error)), timing: timing
53
+ end
54
+ end
55
+ end
56
+ end
57
+
58
+ def initialize(result: nil, failure: nil, timing: {}, version: PROTOCOL_VERSION)
59
+ @result = result
60
+ @failure = failure
61
+ @timing = timing
62
+ @version = version
63
+ end
64
+
65
+ def ok?
66
+ failure.nil?
67
+ end
68
+
69
+ def to_line
70
+ body = { v: version, ok: ok? }
71
+ if ok?
72
+ body[:result] = Payload.validate!(result, "result")
73
+ else
74
+ body[:error] = failure.to_h
75
+ end
76
+ body[:timing] = Payload.validate!(timing, "timing")
77
+
78
+ line = JSON.generate(body) << "\n"
79
+ if line.bytesize > MAX_RESPONSE_BYTES
80
+ raise MessageError, "response is #{line.bytesize} bytes, over the #{MAX_RESPONSE_BYTES} byte limit"
81
+ end
82
+
83
+ line
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "tempfile"
4
+
5
+ module HotCell
6
+ # Fixtures and waits for the suites of all three gems, shipped rather than copied.
7
+ #
8
+ # Same reasoning as `hot_cell/test_cell.rb` shipping from hotcell-server: a helper every consumer needs is
9
+ # one every consumer should get, not one each writes again. Before this, `with_file` had three
10
+ # byte-identical copies, `wait_until` had two — one of them inside a test file rather than a helper — and
11
+ # the monotonic clock was hand-rolled six times in code that already loads HotCell::Clock.
12
+ #
13
+ # It lives in hotcell-core because that is the gem all three load. Requiring it is opt-in, so nothing here
14
+ # reaches a production graph unless a suite asks for it.
15
+ #
16
+ # Include into a Minitest::Test subclass; `skip` and `flunk` are called on the including test.
17
+ module TestSupport
18
+ private
19
+ # Descriptors have to be regular files, so every fixture is a real file on disk.
20
+ def with_file(contents = "")
21
+ Tempfile.create([ "hotcell", ".bin" ], binmode: true) do |file|
22
+ file.write contents
23
+ file.flush
24
+ yield file.path
25
+ end
26
+ end
27
+
28
+ # A source and a destination, which is the shape of most requests.
29
+ def with_files(contents = "source bytes")
30
+ with_file(contents) { |source| with_file { |destination| yield source, destination } }
31
+ end
32
+
33
+ def reading(path, &block)
34
+ File.open path, "rb", &block
35
+ end
36
+
37
+ def writing(path, &block)
38
+ File.open path, "wb", &block
39
+ end
40
+
41
+ def updating(path, &block)
42
+ File.open path, "r+b", &block
43
+ end
44
+
45
+ def open_descriptors
46
+ skip "counting open descriptors needs Linux" unless File.directory?("/proc/self/fd")
47
+
48
+ Dir.children("/proc/self/fd").size
49
+ end
50
+
51
+ # True while the process exists and is not a zombie. A killed process lingers as a zombie until its
52
+ # parent reaps it, and a zombie is dead for a caller watching for a sweep. The state is the first
53
+ # token after the final `)` of /proc/<pid>/stat, which is where the comm field ends.
54
+ # ESRCH alongside ENOENT: the entry can be torn down between the open and the read, and a caller
55
+ # polling for a kill is exactly who reads during the teardown.
56
+ def process_running?(pid)
57
+ state = File.read("/proc/#{pid}/stat")[/\)\s+(\S)/, 1]
58
+ !state.nil? && state != "Z"
59
+ rescue Errno::ENOENT, Errno::ESRCH
60
+ false
61
+ end
62
+
63
+ # A cell writes its log and removes its scratch after it has answered, so a few properties are
64
+ # genuinely asynchronous with respect to the caller. Bounded polling says so; a sleep would not.
65
+ def wait_until(within: 5, what: "the condition")
66
+ deadline = Clock.now + within
67
+
68
+ until yield
69
+ flunk "#{what} did not happen within #{within}s" if Clock.now > deadline
70
+
71
+ sleep 0.01
72
+ end
73
+ end
74
+
75
+ def elapsed
76
+ at = Clock.now
77
+ yield
78
+ Clock.now - at
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Bundler auto-requires a gem named "hotcell-core" as "hotcell-core", then as "hotcell/core". This gem
4
+ # uses neither path, because hot_cell/ is what yields the HotCell constant under the default inflection.
5
+ # Without this file, `gem "hotcell-core"` in a Gemfile silently loads nothing at all.
6
+ require "hot_cell/core"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hotcell-core
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.0
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mike Dalessio
@@ -9,16 +9,47 @@ bindir: bin
9
9
  cert_chain: []
10
10
  date: 1980-01-02 00:00:00.000000000 Z
11
11
  dependencies: []
12
- description: To be released soon, secure sidecar for Rails
12
+ description: |
13
+ HotCell runs untrusted media conversion in an unprivileged sibling container with no network,
14
+ reached over a Unix socket that carries file descriptors rather than paths or bytes.
15
+
16
+ This gem holds what both sides must agree on: the request and response format, SCM_RIGHTS
17
+ marshalling, descriptor access-mode verification, payload validation, and the error taxonomy.
18
+ It performs no I/O of its own and loads no media library.
13
19
  email:
14
20
  - mike@37signals.com
15
21
  executables: []
16
22
  extensions: []
17
23
  extra_rdoc_files: []
18
- files: []
24
+ files:
25
+ - MIT-LICENSE
26
+ - README.md
27
+ - lib/hot_cell/clock.rb
28
+ - lib/hot_cell/codes.rb
29
+ - lib/hot_cell/connection.rb
30
+ - lib/hot_cell/core.rb
31
+ - lib/hot_cell/core/version.rb
32
+ - lib/hot_cell/declarations.rb
33
+ - lib/hot_cell/descriptors.rb
34
+ - lib/hot_cell/errors.rb
35
+ - lib/hot_cell/failure.rb
36
+ - lib/hot_cell/fields.rb
37
+ - lib/hot_cell/naming.rb
38
+ - lib/hot_cell/payload.rb
39
+ - lib/hot_cell/protocol.rb
40
+ - lib/hot_cell/request.rb
41
+ - lib/hot_cell/response.rb
42
+ - lib/hot_cell/test_support.rb
43
+ - lib/hotcell-core.rb
44
+ homepage: https://github.com/basecamp/hotcell
19
45
  licenses:
20
46
  - MIT
21
- metadata: {}
47
+ metadata:
48
+ homepage_uri: https://github.com/basecamp/hotcell
49
+ source_code_uri: https://github.com/basecamp/hotcell/tree/v0.1.0/hotcell-core
50
+ changelog_uri: https://github.com/basecamp/hotcell/blob/v0.1.0/CHANGELOG.md
51
+ bug_tracker_uri: https://github.com/basecamp/hotcell/issues
52
+ rubygems_mfa_required: 'true'
22
53
  rdoc_options: []
23
54
  require_paths:
24
55
  - lib
@@ -26,7 +57,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
26
57
  requirements:
27
58
  - - ">="
28
59
  - !ruby/object:Gem::Version
29
- version: '0'
60
+ version: '3.3'
30
61
  required_rubygems_version: !ruby/object:Gem::Requirement
31
62
  requirements:
32
63
  - - ">="
@@ -35,5 +66,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
35
66
  requirements: []
36
67
  rubygems_version: 4.0.16
37
68
  specification_version: 4
38
- summary: To be released soon, secure sidecar for Rails
69
+ summary: The wire protocol shared by both sides of a HotCell.
39
70
  test_files: []