hotcell-server 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db6e168c58fc1e84ea4411d8935574649f5464059f1e785d43d2388440bcef21
-  data.tar.gz: 9c2e43635ad2233707d59aa48d76f4b20823ae64bf67d7fc651f8e6d8178a88b
+  metadata.gz: 96b9d4b602f5a5430605a97aec5db12ea0a67a6cdac0b1fb42e5174c2fa447a7
+  data.tar.gz: 4beed8c182fa5cfbb1bb4713ff3cb6e19772c29ac4e4dfec6d5456b217be054d
 SHA512:
-  metadata.gz: 81c839e7cd18af7add06ba34c0e3953787734eaf7906f9c0c6eaf7144e039d4d5dc7caef119fb9d98c006a7c09ce93b99576fa615cddff65c1868bb102a1567a
-  data.tar.gz: 0744a62b685fd7e96b39f9169869813e7ccc11e9331052742cc3a017be1464090aad6ee6cc0a36de4ec52e0e2802ced1d04215230a2d186e23c6ad2cfd5d41d1
+  metadata.gz: 4974036dd70e168a833dbb03cf0aef47d6ce0be264ddc8327e5b4d6265c1b65d85a55767815cbf2471e6c9a534a910dcc8597bc6913cf0d1c5bec93d4d198c9d
+  data.tar.gz: 433c2c6f0e55f72b849f67a3abb9bc55129a0d5ececb9fa9b8c50c277dabee74f69d2ce95259d30c9fc0d0bbfd9b7db48cdcbd061be85677d98aaf3c76f8dd91

data/lib/hot_cell/control.rb

@@ -39,8 +39,7 @@ module HotCell
     end
 
     # Static, and called once per registered cell at app boot. It is the cheapest way to catch a client
-    # pointed at a cell that does not carry the operation it wants, which is otherwise an `unsupported` on
-    # the first real request, and to catch a client whose own timeout is below what this cell may take.
+    # whose own timeout is below what this cell may take, and it is what `bin/hotcell describe` reads.
     def describe
       { v: PROTOCOL_VERSION, operations: Registry.names, groups: groups, **@configuration.to_h }
     end

data/lib/hot_cell/log.rb

@@ -65,9 +65,11 @@ module HotCell
       # deadline loop until the runtime resumed. A non-blocking write answers `:wait_writable` for the full
       # pipe instead, and the line is dropped like any other.
       #
-      # A short count would be a torn line, which a write at or under PIPE_BUF cannot produce. The one line
-      # that can exceed it is cell.boot's inventory, written once before the loop enforces anything and
-      # against an empty pipe.
+      # A short count would be a torn line, which a write at or under PIPE_BUF cannot produce. Torn is worse
+      # than dropped: the stub has no newline, so the next line written is glued to it and a reader loses
+      # both. cell.boot's inventory can exceed it, written once against an empty pipe before the loop
+      # enforces anything, and so can worker.killed — 512 bytes of stderr become 3072 when every one is a
+      # control character, and the operation name and envelope spend most of what is left.
       def emit(line)
         @io.write_nonblock line, exception: false
       rescue SystemCallError, IOError

data/lib/hot_cell/operation.rb

@@ -196,8 +196,12 @@ module HotCell
         kept
       end
 
+      # The OpenMP variables come from the cell's environment rather than being named here: the number
+      # belongs to the image, which is configured to match the container's CPU quota. Without them
+      # `unsetenv_others: true` would hand an exec'd tool the pool the image's bound was meant to take away.
       def tool_environment(overrides)
-        { "HOME" => ENV["HOME"], "PATH" => ENV["PATH"], "LANG" => "C.UTF-8", "LC_ALL" => "C.UTF-8" }
+        { "HOME" => ENV["HOME"], "PATH" => ENV["PATH"], "LANG" => "C.UTF-8", "LC_ALL" => "C.UTF-8",
+          "OMP_NUM_THREADS" => ENV["OMP_NUM_THREADS"], "OMP_THREAD_LIMIT" => ENV["OMP_THREAD_LIMIT"] }
           .merge(overrides.transform_keys(&:to_s))
           .compact
       end

data/lib/hot_cell/server/version.rb

@@ -2,6 +2,6 @@
 
 module HotCell
   module Server
-    VERSION = "0.1.0"
+    VERSION = "0.3.0"
   end
 end

data/lib/hot_cell/slot.rb

@@ -11,8 +11,7 @@ module HotCell
   #
   # **A slot has one directory per request and it is that request's `$HOME`.** It is created when the
   # request starts and removed when the request ends, so nothing a tool writes under `$HOME` reaches the
-  # next request on this slot. The name is stable and the directory behind it is not, which is why a
-  # reused worker reports the same path twice.
+  # next request on this slot.
   #
   # This directory used to survive, to give a tool with an expensive per-user profile a warm one. That is
   # withdrawn. What a tool reads from `$HOME` is configuration, and for the toolchains a cell carries
@@ -22,23 +21,48 @@ module HotCell
   # `max_requests_per_worker: 1` is supposed to hold. adr/0003 records the reversal and adr/0002 the
   # reasoning it supersedes.
   #
-  # There is one directory and not two. A request's staged inputs and outputs are named inside `$HOME`
-  # rather than in a scratch directory of their own, because the two had the same lifetime and the same
-  # owner once the home stopped surviving. Staging used to create its directory on demand, which is what
-  # kept a descriptor-only operation from paying for one; `$HOME` has to exist for every request either
-  # way, so that laziness bought nothing and is gone with it.
+  # **The name is fresh for every request, and that is what makes the removal above a guarantee rather than
+  # an intention.** A stable name only holds while the deletion behind it works, and the tool that filled
+  # the directory runs as the user that owns it: `chmod 0500` on a directory it has written makes its own
+  # configuration unremovable, and the same mode on the slot directory makes it unrenameable too. Both
+  # cleanups answer false and both callers log — and a stable name then handed the next request the tree
+  # that had just refused to go. A name no earlier request has held is not a name an earlier request could
+  # have prepared, so what a failed cleanup now costs is disk rather than the isolation the cell is for.
+  #
+  # The mode of the slot directory is reasserted for the same reason. It is the one name here that is
+  # predictable, so it is the one an earlier request can lock, and a mode on a directory this uid owns is
+  # ours to put back.
+  #
+  # **This bounds what an earlier request left behind, and not what a live one is doing.** Workers share a
+  # uid and `0700` is the owner's own mode, so a concurrent sibling — or a `setsid` descendant of a finished
+  # request, which process groups do not contain — can still write into a home the moment it exists. The
+  # slot directory itself can be renamed aside and replaced, and `chmod` follows what it finds. Both are the
+  # residuals `docs/DESIGN.md` records under worker isolation, and neither is closed here.
+  #
+  # There is one directory per request and not two. A request's staged inputs and outputs are named inside
+  # `$HOME` rather than in a scratch directory of their own, because the two had the same lifetime and the
+  # same owner once the home stopped surviving. Staging used to create its directory on demand, which is
+  # what kept a descriptor-only operation from paying for one; `$HOME` has to exist for every request
+  # either way, so that laziness bought nothing and is gone with it.
   #
   # The filesystem behaviour belongs here rather than in the two processes that call it. The directory is
   # removed from both — the worker before it answers, the supervisor at finish and at reap — so the guard
   # and the swallowed SystemCallError are a rule that has to hold on both sides of a fork, and it had a
   # copy on each.
-  Slot = Struct.new(:number, :home) do
+  Slot = Struct.new(:number, :directory, :home) do
     def self.build(workspace, number)
-      new number, File.join(workspace, number.to_s, "home")
+      new number, File.join(workspace, number.to_s)
     end
 
+    # A name no request has held before, so nothing an earlier one did to the tree it was given reaches this
+    # one. The suffix is random rather than a counter for the reason the discarded name's is: the previous
+    # request could write to this directory, and a predictable name is one it can pre-create.
     def make_home
-      FileUtils.mkdir_p home, mode: 0o700
+      FileUtils.mkdir_p directory, mode: 0o700
+      FileUtils.chmod 0o700, directory
+
+      self.home = File.join(directory, "home-#{SecureRandom.hex(8)}")
+      Dir.mkdir home, 0o700
       home
     end
 
@@ -55,10 +79,11 @@ module HotCell
     # response with a crash, and the supervisor calls discard_home from finish and reap, where nothing above
     # rescues anything and a raise stops the cell with every request it holds.
     def remove_home
-      FileUtils.remove_entry home if Dir.exist?(home)
+      return true if home.nil?
+      return false unless remove_tree(home)
+
+      self.home = nil
       true
-    rescue SystemCallError
-      false
     end
 
     # **The supervisor renames rather than deletes, and that is a scheduling decision.**
@@ -73,34 +98,74 @@ module HotCell
     # where the unlinking costs nobody's latency.
     #
     # The destination carries a random suffix rather than a counter, because the tool that filled the
-    # directory runs as this user and can write to the slot's workspace. A predictable name lets it
+    # directory runs as this user and can write to the slot's directory. A predictable name lets it
     # pre-create a colliding entry, fail the rename, and send the supervisor into the recursive delete the
     # rename exists to avoid. On any rename failure the tree is left where it is, for a later worker's own
     # cleanup to remove off the hot path. The supervisor never deletes a tree inline, whatever goes wrong.
+    # It renames what is there rather than the name it is holding, because the supervisor is the caller that
+    # matters and it does not know the name. `home` is assigned in the worker after the fork, so the
+    # supervisor's copy of the slot is still nil when it discards a killed worker's tree. At most one worker
+    # holds a slot at a time, so everything matching `home-*` under it is that worker's and nobody else's.
     def discard_home
-      return true unless Dir.exist?(home)
+      FileUtils.chmod 0o700, directory if Dir.exist?(directory)
 
-      File.rename home, "#{home}.discarded-#{Process.pid}-#{SecureRandom.hex(8)}"
+      Dir.glob(File.join(directory, "home-*")).each do |path|
+        File.rename path, File.join(directory, "discarded-#{Process.pid}-#{SecureRandom.hex(8)}")
+      end
+
+      self.home = nil
       true
     rescue SystemCallError
       false
     end
 
     # Nothing here is created at boot, because nothing survives a request. This only clears what an earlier
-    # boot left behind.
+    # boot left behind — the whole slot directory, since the names inside it are an earlier boot's and not
+    # this one's to reconstruct.
+    #
+    # `Dir.exist?` is not the guard, because it follows symlinks and answers false for a dangling one. An
+    # entry a tool left in the slot's place is exactly what this has to remove, and it used to survive the
+    # sweep and raise from every later `make_home`.
     def prepare
-      # Both, rather than short-circuiting: a home that could not be removed must not stop the sweep.
-      cleared = remove_home
-      sweep && cleared
+      remove_tree directory
     end
 
     # Unlinks whatever discard_home renamed out of the way. Partial progress is fine: a sweep killed
     # part-way leaves fewer entries for the next one, so this converges rather than repeating.
     def sweep
-      Dir.glob("#{home}.discarded-*").each { |path| FileUtils.remove_entry path }
-      true
+      Dir.glob(File.join(directory, "discarded-*")).map { |path| remove_tree(path) }.all?
     rescue SystemCallError
+      # The glob itself can fail, because the slot directory is a name a tool can replace — a symlink loop
+      # in its place answers ELOOP here rather than for any one entry. This runs from the worker's ensure,
+      # where a raise would replace the caller's response with a crash.
       false
     end
+
+    private
+      # **A mode is the only thing a tool needs to make its own tree unremovable, and a mode on a tree this
+      # uid owns is ours to put back.** `chmod 0500` on a directory a conversion wrote is enough to fail the
+      # recursive delete underneath it, and the delete failing used to be the end of it: one tree per
+      # request stayed on the tmpfs until the container ended. The repair is not a permission the process
+      # gains, it is one it never lost.
+      #
+      # Repair only after a failure, never before, so the common request pays for a walk of its own tree
+      # once rather than twice. `force:` on the chmod because a partial repair that removes most of the tree
+      # is better than none, and the remove that follows is what reports the outcome either way.
+      def remove_tree(path)
+        return true unless File.exist?(path) || File.symlink?(path)
+
+        FileUtils.remove_entry path
+        true
+      rescue SystemCallError
+        repair_and_remove path
+      end
+
+      def repair_and_remove(path)
+        FileUtils.chmod_R 0o700, path, force: true
+        FileUtils.remove_entry path
+        true
+      rescue SystemCallError
+        false
+      end
   end
 end

data/lib/hot_cell/supervisor.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "socket"
+require "fcntl"
 require "fileutils"
 require "tmpdir"
 
@@ -13,6 +14,8 @@ module HotCell
   # the trusted side on a bounded buffer — but the supervisor does not need to, and staying out of the
   # request is what lets it dispatch a connection whose descriptors are still queued on it.
   #
+  # `peeked_op` is the exception, on the one path where no worker will read the request.
+  #
   # Dispatching rather than letting workers accept is what makes the rest work. The supervisor needs to own
   # the accept anyway, for the queue, for queued_ms, and to answer `capacity`. It also means the supervisor
   # knows when every worker started its current request, which is what the deadline needs.
@@ -21,16 +24,36 @@ module HotCell
     # `busy?` and the supervisor assigning the four fields `busy?` is computed from are the same fact. Spread
     # across the caller, a new field is one the next transition forgets to clear.
     Child = Struct.new(:slot, :pid, :control, :connection, :dispatched_at, :deadline, :served, :killed_for,
-                       :retired_at, :buffer, keyword_init: true) do
-      def self.build(slot:, pid:, control:, deadline:)
-        new slot: slot, pid: pid, control: control, deadline: deadline, served: 0, buffer: "".b
+                       :op, :retired_at, :buffer, :stderr, :captured, keyword_init: true) do
+      def self.build(slot:, pid:, control:, deadline:, stderr: nil)
+        new slot: slot, pid: pid, control: control, deadline: deadline, served: 0, buffer: "".b,
+            stderr: stderr, captured: "".b
+      end
+
+      # Keeps the last MAX_MESSAGE_BYTES rather than the first, because the line that ended the request is
+      # the last one written. Trimmed on every read, so a worker that never stops printing costs no more
+      # than that.
+      def capture_stderr(chunk)
+        captured << chunk
+        excess = captured.bytesize - Failure::MAX_MESSAGE_BYTES
+        captured.slice! 0, excess if excess.positive?
+      end
+
+      # A silent worker gets no field at all: `Log#document` does not compact the hotcell namespace, so a nil
+      # would put `"stderr":null` on every death a cell reports.
+      def stderr_field
+        captured.empty? ? {} : { stderr: Failure.sanitize(captured, keep: :tail) }
       end
 
       def dispatched(connection, deadline, at:)
+        # An earlier request's warning is not this request's death. Not a boundary, though: a descendant
+        # holding fd 2 writes whenever it likes, and a sibling can open /proc/<pid>/fd/2 and write there too.
+        captured.clear
         self.connection = connection
         self.dispatched_at = at
         self.deadline = deadline
         self.killed_for = nil
+        self.op = nil
         self.served += 1
       end
 
@@ -90,31 +113,6 @@ module HotCell
     # bytes than Linux, and control.sock is the longer of the two names, so it overflows first.
     SUN_PATH_MAX = RUBY_PLATFORM.include?("darwin") ? 104 : 108
 
-    # The signals this cell can attribute to the request the worker was holding. XFSZ is that worker passing
-    # RLIMIT_FSIZE, and SEGV, ABRT and TRAP are how libvips and GLib die on their own allocation failures —
-    # libvips dereferences null after printing the correct diagnostic, and g_malloc aborts.
-    #
-    # These three are the worker hitting its own per-worker RLIMIT_DATA, which is a property of the input
-    # this worker held, so the same bytes do it again and the verdict is permanent. `Codes` says a signal
-    # tells how a process died and never why, so a signal is transient by default — and that is not in
-    # conflict with a permanent verdict here, because aggregate pressure the worker did not cause arrives as
-    # SIGKILL, not as these. The two are different signals, and SIGKILL is excluded below.
-    #
-    # SIGKILL is deliberately absent, and its absence is the point. The supervisor's own deadline kill is
-    # already named by `killed_for`, so a SIGKILL reaching this table came from somewhere this process
-    # cannot see: a cgroup OOM chosen on aggregate pressure, or a sibling worker, which shares a uid and is
-    # not prevented from signalling. Reading it as this request's memory condemned an input for someone
-    # else's pressure.
-    #
-    # Anything not here is `crashed`, which is also where a worker that exited without a signal lands. They
-    # were two names, `signal` and `crashed`, for one amount of knowledge: the worker died and nothing says
-    # why. One name is honest about that.
-    SIGNAL_CAUSES = {
-      "XFSZ" => Codes::FSIZE,
-      "SEGV" => Codes::MEMORY,
-      "ABRT" => Codes::MEMORY,
-      "TRAP" => Codes::MEMORY,
-    }.freeze
 
     SOCKETS = [ "work.sock", "control.sock" ].freeze
 
@@ -130,6 +128,12 @@ module HotCell
     # far above any real scrape rate rather than as a throttle.
     CONTROL_BACKLOG = 64
 
+    # What one pass reads off a worker's fd 2, and how many reads the reap's final drain gets. Sized to
+    # clear a full pipe rather than to bound memory — `Child#capture_stderr` does that — so reading further
+    # would only mean a fresher tail.
+    STDERR_READ_BYTES = 16 * 1024
+    STDERR_FINAL_READS = 8
+
     attr_reader :configuration, :counters, :log, :directory, :workspace
 
     def initialize(directory:, workspace: nil, configuration: HotCell.configuration, log: Log.new,
@@ -188,6 +192,7 @@ module HotCell
       def sources
         [ @signals ].tap do |list|
           list.concat @children.each_value.map { |child| child.control.socket }.reject(&:closed?)
+          list.concat @children.each_value.filter_map(&:stderr).reject(&:closed?)
           list.concat @control_pending.map { |pending| pending.connection.socket }.reject(&:closed?)
           list.push @work, @control unless @stopping
         end
@@ -212,8 +217,13 @@ module HotCell
         when @work then accept_work
         when @control then accept_control
         else
-          pending = pending_control(source)
-          pending ? read_control(pending) : child_reported(source)
+          if (pending = pending_control(source))
+            read_control pending
+          elsif (child = child_writing_stderr(source))
+            drain_stderr child
+          else
+            child_reported source
+          end
         end
       end
 
@@ -383,7 +393,7 @@ module HotCell
         true
       rescue SystemCallError, IOError => error
         log.write "worker.undispatchable", pid: child.pid, slot: child.slot.number,
-                                           error: error.class.name
+                                           op: peeked_op(connection), error: error.class.name
 
         # Released rather than finished: this is the only path that hands the connection back to be answered
         # here, so the client connection must not be closed on the way out. `retire` closes the worker's
@@ -395,6 +405,20 @@ module HotCell
         false
       end
 
+      # Safe only because the dispatch failed: no worker will serve this request. A peek and a bytes-only
+      # `recv`, never `recvmsg` — a partial `send_message` can leave a worker holding this connection, and
+      # consuming the request would take it from that worker. `recv_nonblock` rather than `MSG_DONTWAIT`,
+      # whose blocking `recv` parks on an empty socket; it leaves O_NONBLOCK alone on 3.3, 3.4 and 4.0,
+      # which a worker's SCM_RIGHTS duplicate would otherwise share.
+      def peeked_op(connection)
+        peeked = connection.socket.recv_nonblock(MAX_REQUEST_BYTES, Socket::MSG_PEEK, exception: false)
+        return nil unless peeked.is_a?(String) && peeked.include?("\n")
+
+        Request.parse(peeked.force_encoding(Encoding::UTF_8)).op
+      rescue StandardError
+        nil
+      end
+
       def available_child
         @children.each_value.find(&:available?) || spawn
       end
@@ -403,34 +427,38 @@ module HotCell
         number = free_slot or return nil
         slot = Slot.build(workspace, number)
         supervisor_side, worker_side = UNIXSocket.pair(:STREAM)
+        stderr_reader, stderr_writer = IO.pipe
 
         # A fork that fails is a host under pressure, not a reason to stop serving. The request stays queued
         # and is either dispatched on a later pass or answered `capacity` when its wait runs out.
         pid = begin
           fork do
-            become_worker supervisor_side
+            become_worker supervisor_side, stderr_reader, stderr_writer
             Worker.new(slot: slot, configuration: configuration, control: Connection.new(worker_side),
                        log: log).run
           end
         rescue SystemCallError => error
           log.write "worker.unforkable", slot: number, error: error.class.name, message: error.message
-          supervisor_side.close
-          worker_side.close
+          [ supervisor_side, worker_side, stderr_reader, stderr_writer ].each(&:close)
           return nil
         end
 
+        # The write end goes now rather than at the reap. Held here, the pipe never reports end of stream,
+        # `sources` keeps a dead worker's read end forever, and a worker that said nothing is
+        # indistinguishable from one that has not finished saying it.
         worker_side.close
+        stderr_writer.close
         log.write "worker.forked", pid: pid, slot: number
 
         @children[number] = Child.build(slot: slot, pid: pid, control: Connection.new(supervisor_side),
-                                        deadline: configuration.limits.deadline)
+                                        deadline: configuration.limits.deadline, stderr: stderr_reader)
       end
 
       # Everything the supervisor holds and the worker must not: the listener, the signal pipe, the other
       # children's control sockets, and every connection the supervisor is still holding for somebody else.
       # The connection this worker is about to serve arrives over SCM_RIGHTS a moment from now, so closing
       # the inherited copy here costs nothing and stops it lingering for the worker's whole life.
-      def become_worker(supervisor_side)
+      def become_worker(supervisor_side, stderr_reader, stderr_writer)
         [ "CHLD", "INT", "TERM" ].each { |signal| trap signal, "DEFAULT" }
 
         # Its own process group, so the deadline reaches the tools this request started rather than only the
@@ -463,9 +491,66 @@ module HotCell
         @children.each_value do |child|
           child.control.close
           child.connection&.close
+          child.stderr&.close
         end
         @queue.each { |(connection, _)| connection.close }
         @control_pending.each { |pending| pending.connection.close }
+
+        # fd 2 becomes the pipe, and it stays non-blocking. `IO.pipe` already returns both ends O_NONBLOCK
+        # and `reopen` is a dup2, which shares the file description — so the flag would ride along on its
+        # own. It is set here anyway, because a decision this load-bearing should be in the code rather than
+        # only in a comment, and because it then survives a Ruby that stops handing out non-blocking pipes.
+        #
+        # A blocking fd 2 would put backpressure into the image-processing path, which was never designed
+        # for it: a warning written from inside libvips is a `write(2)` in a C call Ruby cannot interrupt, so
+        # the conversion would wait on the supervisor's scheduling — nearest exactly when the host is under
+        # pressure and the supervisor is scheduled least. That trade is refused; the conversion path must
+        # never wait on the supervisor. What non-blocking loses instead is in docs/LOGS.md, and a test pins
+        # the flag so that a well-meaning fix has to argue with it.
+        stderr_reader.close
+        stderr_writer.fcntl Fcntl::F_SETFL, stderr_writer.fcntl(Fcntl::F_GETFL) | Fcntl::O_NONBLOCK
+        $stderr.reopen stderr_writer
+        stderr_writer.close
+      end
+
+      # One bounded read per pass, never a loop until the pipe is empty: this runs inside the loop that
+      # enforces every request's deadline, and the peer is a worker that can print as fast as it likes.
+      #
+      # End of stream closes the read end. It has to: an EOF pipe is permanently readable, so leaving it in
+      # `sources` turns the run loop into a spin between the worker's exit and its reap.
+      def drain_stderr(child)
+        chunk = begin
+          child.stderr.read_nonblock(STDERR_READ_BYTES, exception: false)
+        rescue SystemCallError
+          nil
+        end
+        return if chunk == :wait_readable
+        return child.stderr.close if chunk.nil?
+
+        child.capture_stderr chunk
+      end
+
+      # A worker's last line sits in the pipe after the process is gone, so the reap reads once more — and
+      # under a flat bound rather than to end of stream. End of stream never arrives while a descendant
+      # holds the write end, and "until the pipe is momentarily empty" terminates only by winning a race
+      # against whoever is writing.
+      def drain_stderr_after_exit(child)
+        STDERR_FINAL_READS.times do
+          break if child.stderr.nil? || child.stderr.closed?
+
+          chunk = begin
+            child.stderr.read_nonblock(STDERR_READ_BYTES, exception: false)
+          rescue SystemCallError
+            nil
+          end
+          break if chunk.nil? || chunk == :wait_readable
+
+          child.capture_stderr chunk
+        end
+      end
+
+      def child_writing_stderr(source)
+        @children.each_value.find { |child| child.stderr.equal?(source) }
       end
 
       # Buffered and non-blocking, for the same reason read_control is, and more so: readability means a byte
@@ -554,10 +639,11 @@ module HotCell
 
         if message[:deadline]
           child.deadline = narrowed_deadline(message[:deadline])
+          child.op = reported_op(message[:op])
         elsif message[:idle]
           return unreadable_report child, "idle report from a worker with no request" unless child.busy?
 
-          finish child, message[:code]
+          finish child, message[:code], message[:cause]
         end
       rescue MessageError => error
         unreadable_report child, Failure.sanitize(error.message)
@@ -568,10 +654,13 @@ module HotCell
       # off the hot path, because the supervisor must never delete one inline. It is not tolerated silently:
       # the random suffix exists because a tool running as this user can pre-create a colliding name, so a
       # rename that fails is the shape of that attempt as well as of an ordinary error.
+      # The slot directory rather than the request's home, because the supervisor does not know the home. It
+      # is named in the worker after the fork, so this copy of the slot holds nil for the whole life of the
+      # child — and logging it said `null` on every failure, which is worse than saying nothing.
       def discard(child)
         return if child.slot.discard_home
 
-        log.write "slot.undiscarded", pid: child.pid, slot: child.slot.number, home: child.slot.home
+        log.write "slot.undiscarded", pid: child.pid, slot: child.slot.number, home: child.slot.directory
       end
 
       def unreadable_report(child, message)
@@ -579,8 +668,9 @@ module HotCell
         nil
       end
 
-      def finish(child, code)
+      def finish(child, code, cause = nil)
         counters.record outcome_code(code)
+        counters.record_kill cause if reported_cause?(code, cause)
         child.finished
         discard child
 
@@ -592,6 +682,19 @@ module HotCell
       # NoMethodError on `to_sym` past `apply_report`'s rescue and took the cell down, and a stream of unique
       # strings grew the counters without bound. An unknown code is recorded, so a misreporting worker is
       # visible rather than silent, but under one fixed bucket.
+      # A worker reports its own kill cause now, so this is a value from a process that may be compromised.
+      # It is checked against the known causes rather than passed through, because `record_kill` interns it
+      # as a symbol and an unchecked one is an unbounded symbol table keyed by whatever a tool decides.
+      def reported_cause?(code, cause)
+        outcome_code(code) == Codes::KILLED && cause.is_a?(String) &&
+          Codes::PERMANENT_BY_CAUSE.key?(cause)
+      end
+
+      # Bounded to a registered name: this rides an untrusted worker report straight into a log line.
+      def reported_op(reported)
+        reported if reported.is_a?(String) && Registry.lookup(reported)
+      end
+
       def outcome_code(reported)
         return reported if reported.is_a?(String) && (reported == "ok" || Codes.known?(reported))
 
@@ -712,11 +815,19 @@ module HotCell
             child_reported child.control.socket
           end
 
+          # Sweep before the last drain, not after. fd 2 is never close-on-exec, so everything this worker
+          # spawned holds the write end too — swept first, what those wrote is waiting to be read rather
+          # than arriving after the last read. The sweep does not make the pipe quiet: SIGKILL is
+          # asynchronous, and a `setsid` descendant is deliberately never reached, so its final bytes are
+          # best effort like everything else here.
           sweep_group child
+          drain_stderr_after_exit child
+
           @children.delete child.slot.number
           answer_for child, status
           discard child
           child.control.close
+          child.stderr&.close
 
           log.write "worker.reaped", pid: pid, slot: child.slot.number, served: child.served,
                                      signal: signal_name(status), exit_code: status.exitstatus
@@ -725,26 +836,45 @@ module HotCell
         nil
       end
 
-      # A killed worker cannot report its own death, because RLIMIT_FSIZE and the deadline KILL are
-      # enforced by a signal. So the supervisor holds its copy of every dispatched connection and writes
-      # the verdict itself. Without this the cold side sees a bare end of stream and cannot tell a limit
+      # A killed worker cannot report its own death, because the deadline KILL is enforced by a signal. So
+      # the supervisor holds its copy of every dispatched connection and writes the verdict itself. Without
+      # this the cold side sees a bare end of stream and cannot tell a limit
       # breach from a crash.
       # A worker still holding a connection at reap time never answered: it reports itself idle after writing,
       # and that report is drained above. So this is the only thing that can answer, and whether it says the
       # input did this or the cell did turns on how the worker died.
+      # **The only cause this can authenticate is its own.** `killed_for` is the deadline kill this
+      # supervisor sent, and it is the one thing here that knows why a worker died. Everything else is a
+      # wait status, which says how — and workers share a uid, so any signal in one may have come from a
+      # sibling rather than from the kernel. Reading XFSZ, SEGV, ABRT or TRAP as this request's file size or
+      # memory let one compromised worker write a permanent verdict against another request's unrelated
+      # input, which Active Storage then kept. `Codes` already stated the rule this broke.
+      #
+      # The verdicts themselves are not gone, they moved to where they can be earned: the worker answers
+      # `memory` when it catches NoMemoryError and `fsize` when a write of its own returns EFBIG, on the
+      # connection it is holding. See Worker#disarm_file_size_signal.
+      #
+      # What that buys is bounded, and it is worth being exact. The supervisor is no longer an instrument
+      # for one worker to condemn another's input. A cell compromised outright still answers whatever it
+      # likes on a connection it holds, which is the socket-theft residual under "Worker isolation".
       def answer_for(child, status)
         return child.connection&.close unless child.busy?
 
-        cause = child.killed_for ||
-                SIGNAL_CAUSES.fetch(signal_name(status), Codes::CRASHED)
+        cause = child.killed_for || Codes::CRASHED
         counters.record Codes::KILLED
         counters.record_kill cause
 
-        log.write "worker.killed", pid: child.pid, slot: child.slot.number, cause: cause,
-                                   signal: signal_name(status), duration_ms: Clock.ms_since(child.dispatched_at)
+        captured = child.stderr_field
+
+        log.write "worker.killed", pid: child.pid, slot: child.slot.number, op: child.op, cause: cause,
+                                   signal: signal_name(status), duration_ms: Clock.ms_since(child.dispatched_at),
+                                   **captured
 
+        # The capture rides the verdict as well as the log line, so an application logs
+        # `killed: crashed (libgomp: ...)` rather than a bare `crashed`. Additive on the wire: `from_wire`
+        # reads named keys, so an old client ignores the field and a new one against an old cell meets nil.
         answer child.connection,
-               Failure.new(code: Codes::KILLED, cause: cause, signal: signal_name(status)),
+               Failure.new(code: Codes::KILLED, cause: cause, signal: signal_name(status), **captured),
                timing: { perform_ms: Clock.ms_since(child.dispatched_at) }
       end
 
@@ -860,7 +990,7 @@ module HotCell
           slot = Slot.build(workspace, number)
           next if slot.prepare
 
-          log.write "slot.uncleaned", slot: number, home: slot.home,
+          log.write "slot.uncleaned", slot: number, home: slot.directory,
                                       message: "an earlier boot's files are still here"
         end
       end

data/lib/hot_cell/test_operations.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "digest"
+require "fcntl"
 require "fileutils"
 
 # Fixture operations, so the whole surface can be exercised in milliseconds, with no tool installed and no
@@ -124,6 +125,26 @@ module HotCell
       end
     end
 
+    # `Exception` so it escapes `serve`'s `rescue StandardError` and reaches `run`, the only path that
+    # writes `worker.crashed`.
+    class Fatal < HotCell::Operation
+      operation "test.fatal"
+
+      def perform(_inputs, _outputs)
+        raise Exception, "a worker cannot answer for this one"
+      end
+    end
+
+    # Hangs in the boot hook, which runs before the worker reports its operation.
+    class SlowBoot < HotCell::Operation
+      operation "test.slow_boot"
+      before_worker_boot { sleep 60 }
+
+      def perform(_inputs, _outputs)
+        {}
+      end
+    end
+
     class Undecodable < HotCell::Operation
       operation "test.undecodable"
 
@@ -164,6 +185,46 @@ module HotCell
       end
     end
 
+    class SignalsSibling < HotCell::Operation
+      operation "test.signals_sibling"
+
+      # Workers share a uid and a pid namespace, so one finds another by looking for a process the
+      # supervisor also fathered. This is the reproducer for the forged verdict: nothing here touches the
+      # victim's input, and the victim is holding an unrelated one.
+      def perform(_inputs, _outputs, signal:)
+        sibling = siblings.first
+        Process.kill signal, sibling if sibling
+
+        { signalled: sibling }
+      end
+
+      private
+        # Two ways to ask the same question, because the suite runs on macOS as well and only one of them
+        # has /proc. An attacker inside a cell has whichever the image gives it; the point of the reproducer
+        # is that the answer is obtainable at all.
+        def siblings
+          pids = Dir.exist?("/proc") ? procfs_children : ps_children
+
+          pids.reject { |pid| pid == Process.pid }
+        end
+
+        def procfs_children
+          Dir.glob("/proc/[0-9]*").filter_map do |path|
+            status = File.read(File.join(path, "status"))
+            File.basename(path).to_i if status[/^PPid:\s+(\d+)/, 1].to_i == Process.ppid
+          rescue SystemCallError
+            nil
+          end
+        end
+
+        def ps_children
+          `ps -A -o pid=,ppid=`.lines.filter_map do |line|
+            pid, ppid = line.split.map(&:to_i)
+            pid if ppid == Process.ppid
+          end
+        end
+    end
+
     # Spawns a process and does not wait for it, the way a worker that crashed mid-request would leave a
     # tool behind. The spawned process inherits the worker's process group, so the supervisor's group sweep
     # at reap is what must kill it — nothing else is watching it, and it has no deadline. Returns the pid so
@@ -421,5 +482,57 @@ module HotCell
         { slept: seconds, pid: Process.pid }
       end
     end
+
+    # Writes to fd 2 and then either dies the way a C library does — `exit()` with no Ruby exception, so
+    # there is no `worker.crashed` line and nothing on the connection — or returns normally, which is the
+    # warning a cell deliberately does not report. `noise:` goes out before `text:`, so a test can ask which
+    # end of an oversized transcript was kept.
+    class StderrWriter < HotCell::Operation
+      operation "test.stderr_writer"
+
+      def perform(_inputs, _outputs, text:, noise: 0, fatal: false)
+        $stderr.write "noise\n" * noise
+        $stderr.write text
+        $stderr.flush
+        exit! 1 if fatal
+
+        { wrote: text.bytesize }
+      end
+    end
+
+    # Bytes that are not valid UTF-8, which a payload cannot carry — so this is a fixture rather than an
+    # argument to StderrWriter. A decoder writing a filename out of a hostile file is where these come from.
+    class GarbledStderr < HotCell::Operation
+      operation "test.garbled_stderr"
+
+      def perform(_inputs, _outputs)
+        $stderr.write "libgomp: \xFF\xFE failed\n".b
+        $stderr.flush
+        exit! 1
+      end
+    end
+
+    # Reports whether fd 2 is non-blocking, which is the load-bearing decision behind the capture: a
+    # blocking fd 2 would put the supervisor's scheduling in the middle of a libvips `write(2)`.
+    class StderrFlags < HotCell::Operation
+      operation "test.stderr_flags"
+
+      def perform(_inputs, _outputs)
+        { nonblock: ($stderr.fcntl(Fcntl::F_GETFL) & Fcntl::O_NONBLOCK).positive? }
+      end
+    end
+
+    # A tool that keeps writing to fd 2 after the worker itself is gone. fd 2 is never close-on-exec, so
+    # everything a worker spawned holds the write end, and the pipe reports no end of stream until the
+    # reap's group sweep kills them.
+    class StderrDescendant < HotCell::Operation
+      operation "test.stderr_descendant"
+
+      def perform(_inputs, _outputs)
+        spawn "sh", "-c", "while :; do echo from the descendant >&2; done"
+        sleep 0.5
+        exit! 1
+      end
+    end
   end
 end

data/lib/hot_cell/worker.rb

@@ -21,6 +21,7 @@ module HotCell
       @log = log
       @booted = nil
       @effective = {}
+      @op = nil
     end
 
     # exit! rather than exit, so that no finalizer and no library teardown ever runs. There is deliberately no
@@ -40,7 +41,7 @@ module HotCell
     # It swallows nothing: `exit! 1` runs whatever was caught.
     def run
       configuration.limits.apply
-      ENV["HOME"] = slot.home
+      disarm_file_size_signal
 
       while (dispatch = await_dispatch)
         serve(*dispatch)
@@ -48,19 +49,49 @@ module HotCell
 
       exit! 0
     rescue Exception => error
-      log.write "worker.crashed", pid: Process.pid, slot: slot.number, error: error.class.name,
-                                  message: Failure.sanitize(error.message)
+      log.write "worker.crashed", pid: Process.pid, slot: slot.number, op: @op,
+                                  error: error.class.name, message: Failure.sanitize(error.message)
       exit! 1
     end
 
     private
       attr_reader :slot, :configuration, :control, :log
 
+      # **The handler carries nothing, and that is the whole point.**
+      #
+      # RLIMIT_FSIZE is enforced by SIGXFSZ, and the supervisor used to read that signal off a wait status
+      # and answer `fsize`, permanently, against whatever input the worker was holding. Workers share a uid,
+      # so a sibling sends SIGXFSZ as easily as the kernel does and a wait status cannot tell them apart —
+      # nor can a handler, because Ruby hands one only the signal number and no siginfo, so SI_KERNEL and
+      # SI_USER are not reachable from here.
+      #
+      # Catching it makes the kernel fail the offending write with EFBIG rather than killing the process,
+      # and that error return is what a signal is not: it is raised by a write this process made, and no
+      # signal any sibling sends produces one. So the verdict below keys on Errno::EFBIG and this handler
+      # does nothing at all. A handler that set so much as a flag the verdict consulted would hand the
+      # forgery straight back.
+      #
+      # EFBIG is evidence of this request's own write and not proof of which limit stopped it. A filesystem
+      # maximum, or the caller's own file, answers the same errno — so this says the bytes did not go, by
+      # something this request did, rather than naming RLIMIT_FSIZE. Narrowing it further means checking the
+      # written file against the effective limit at each write site, which is not what this changes.
+      #
+      # A block rather than "IGNORE", because an ignored disposition survives execve and a handled one does
+      # not. A tool must keep dying on its own RLIMIT_FSIZE rather than writing past it, and a tool that
+      # died by signal is `crashed` and transient — its wait status is no more trustworthy than a worker's,
+      # since a sibling can signal a tool too.
+      def disarm_file_size_signal
+        Signal.trap("XFSZ") { nil }
+      end
+
       # Returns [connection, queued_ms], or nil once the supervisor has retired this worker by closing the
       # control socket. The connection arrives as a descriptor: the supervisor accepted it and never
       # called recvmsg, so the caller's own descriptors are still queued on it and this worker's recvmsg
       # is what installs them.
       def await_dispatch
+        # Not in `serve`'s ensure: `worker.crashed` is written from `run`, past it.
+        @op = nil
+
         line, descriptors = control.receive_message(limit: DISPATCH_BYTES)
         return nil if line.nil?
 
@@ -75,12 +106,17 @@ module HotCell
         received = []
         response = nil
 
-        # Before the request rather than at boot, and one directory rather than two. A tool reads its
-        # configuration from $HOME and that configuration is executable, so a home that outlived the request
-        # let one compromised conversion reconfigure every later one on this slot. adr/0003.
-        slot.make_home
-
         begin
+          # Before the request rather than at boot, and one directory rather than two. A tool reads its
+          # configuration from $HOME and that configuration is executable, so a home that outlived the
+          # request let one compromised conversion reconfigure every later one on this slot. adr/0003.
+          #
+          # Inside the begin, because a home that cannot be created is a broken deployment and answers
+          # `failed`, which is transient. Outside it the raise skipped every response path and reached
+          # `run`, which exits the worker — after this ensure had already reported idle `"ok"`, counting a
+          # success for a request that never ran.
+          ENV["HOME"] = slot.make_home
+
           line, received = connection.receive_message
           response = if line.nil?
             # The caller closed before sending a request. Transient, so it is never written against a blob,
@@ -98,6 +134,10 @@ module HotCell
         # transient. Adding it back would condemn a blob for the cell's own bad moment.
         rescue NoMemoryError, MemoryExhausted => error
           response = refuse(Codes::KILLED, error, timing, cause: Codes::MEMORY)
+        # The one place a file-size verdict can be earned. EFBIG comes back from a write this worker made
+        # past its own RLIMIT_FSIZE, so unlike the signal it cannot arrive from anywhere else.
+        rescue Errno::EFBIG => error
+          response = refuse(Codes::KILLED, error, timing, cause: Codes::FSIZE)
         rescue StandardError => error
           response = refuse("failed", error, timing)
         end
@@ -107,6 +147,7 @@ module HotCell
       ensure
         received.each(&:close)
         connection.close
+        home = slot.home
         swept = slot.remove_home
 
         # After the answer and before reporting idle, which is the only window where this costs nobody. How
@@ -115,21 +156,29 @@ module HotCell
         # staging, where it would spend the next request's deadline on the previous request's mess. Here the
         # caller already has its response, and `report_idle` is what makes this worker available — so the
         # supervisor will not dispatch into a worker that is still sweeping.
-        slot.sweep
-        report_uncleaned unless swept
-        report_idle response&.failure&.code
+        report_uncleaned home unless swept
+        report_unswept unless slot.sweep
+        report_idle response&.failure
       end
 
       # A removal that failed is the one thing here nobody else can see. The bytes stay on the shared tmpfs
       # after the caller has been told the request is over, and a sibling worker can cause it by writing into
       # the tree while remove_entry walks it. It cannot raise from an ensure, so it says so instead. One line
       # per request, from the ensure, because that is the attempt that knows the final state.
-      def report_uncleaned
-        log.write "slot.uncleaned", pid: Process.pid, slot: slot.number, home: slot.home
+      def report_uncleaned(home)
+        log.write "slot.uncleaned", pid: Process.pid, slot: slot.number, home: home
+      end
+
+      # The other half of the same fact. A sweep removes what the supervisor renamed out of the way after a
+      # killed request, and its failure was the one cleanup outcome nobody said anything about — so a slot
+      # accumulating one tree per request looked exactly like a slot that was clean.
+      def report_unswept
+        log.write "slot.unswept", pid: Process.pid, slot: slot.number, home: slot.directory
       end
 
       def handle(line, received, timing)
         request = Request.parse(line)
+        @op = request.op
 
         unless request.current_version?
           return refuse("protocol", request.version_mismatch, timing)
@@ -220,12 +269,38 @@ module HotCell
       # The supervisor enforces the deadline and never reads a request, so it cannot know that this
       # operation asked for less than the cell's maximum. The worker is the only thing that knows, and it
       # says so before it touches an untrusted byte.
+      #
+      # The name rides along because a killed worker cannot write its own `worker.killed`.
       def report_deadline(operation)
-        tell deadline: effective(operation).deadline
+        named = { deadline: effective(operation).deadline, op: @op }
+
+        tell(**(fits?(named) ? named : named.merge(op: nil)))
+      end
+
+      # The supervisor drops an over-limit report whole, so an oversized name would take the narrowed
+      # deadline with it. Measured on the encoded line and not the name's length, because escaping decides:
+      # JSON writes a NUL as six bytes. Dropped rather than truncated, which can end mid-character and
+      # raise out of `tell`, or match the registry as a different operation.
+      def fits?(message)
+        (JSON.generate(message) << "\n").bytesize <= DISPATCH_BYTES
+      rescue StandardError
+        false
       end
 
-      def report_idle(code)
-        tell idle: true, code: code || "ok"
+      # The cause travels with the code so the supervisor can still count a kill by cause. It used to read
+      # that off a wait status; it reads the worker's own report now, and only when there is a cause to send,
+      # so an ordinary idle report is the two keys it always was.
+      #
+      # **This is a metric and not a verdict.** The verdict went to the caller on the work connection before
+      # this line runs. A compromised worker can report a cause its request never had, or withhold one it
+      # did, so `killed_by` is what workers said rather than what happened — which is why the supervisor
+      # checks the value against the known causes before interning it, and why nothing downstream may treat
+      # it as evidence about a blob.
+      def report_idle(failure)
+        return tell(idle: true, code: "ok") if failure.nil?
+        return tell(idle: true, code: failure.code) if failure.cause.nil?
+
+        tell idle: true, code: failure.code, cause: failure.cause
       end
 
       def tell(**message)
@@ -243,7 +318,7 @@ module HotCell
 
         connection.write_line line_for(response)
       rescue SystemCallError, IOError
-        log.write "request.abandoned", pid: Process.pid, slot: slot.number
+        log.write "request.abandoned", pid: Process.pid, slot: slot.number, op: @op
       end
 
       def line_for(response)
@@ -255,7 +330,8 @@ module HotCell
       def record(response, timing)
         return if response.nil?
 
-        log.write "request", pid: Process.pid, slot: slot.number, code: response.failure&.code || "ok",
+        log.write "request", pid: Process.pid, slot: slot.number, op: @op,
+                             code: response.failure&.code || "ok",
                              permanent: response.failure&.permanent?,
                              outcome: response.failure ? "failure" : "success",
                              duration_ms: timing.elapsed_ms, timing: response.timing

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hotcell-server
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Mike Dalessio
@@ -15,22 +15,18 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.3.0
 description: |
-  The cell side of HotCell. A supervisor listens on two Unix sockets, forks a worker per request,
-  holds each worker to a wall-clock deadline and a set of resource limits, and answers for one that
-  dies without reporting.
-
-  An operation subclasses HotCell::Operation, declares its limits, and implements perform. This gem
-  deliberately depends on no application framework: nothing about it loads ActiveSupport, because a
-  sandbox should carry only what the conversion needs.
+  Runs a HotCell container. A supervisor listens on two Unix sockets, forks a worker for each request,
+  and enforces a wall clock deadline and resource limits on it. Write the work as a subclass of
+  HotCell::Operation.
 email:
 - mike@37signals.com
 executables:
@@ -65,8 +61,8 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/basecamp/hotcell
-  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.1.0/hotcell-server
-  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.1.0/CHANGELOG.md
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.3.0/hotcell-server
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.3.0/CHANGELOG.md
   bug_tracker_uri: https://github.com/basecamp/hotcell/issues
   rubygems_mfa_required: 'true'
 rdoc_options: []