hotcell-server 0.4.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 390fc0f10953763de1f745cc3002fa0e548b362cbb21c82faba7dfa807b49dd5
-  data.tar.gz: fd6932e090acccea7964e38d76e65ea6163b1ebf6b5f401f6854fe8fa1e38034
+  metadata.gz: 14fae075c3d4fef73a330453eff2c295ac7e67895d50e2b9ad73d800b50ff2f8
+  data.tar.gz: d3730524324b2283332bdf2f04c90923a6dc3b0ed4a8b6821b10dc46a7e1fd3a
 SHA512:
-  metadata.gz: a34f827e6dbb2616e4e20b122ff6a8fdd78e40c19bfd8297c767c450a06ea4efaebc245d3ca3fb117caf13b70d922236e3c51f6a0a93207b5fc0a815e60e2583
-  data.tar.gz: '0769c12d7f26f35380dd5532edd8073c160f6a823128dcc4c88fddc697fd896be8cabec1eaa21dedf3c505a26ccda5a90cc38716a23461fa70410eb92d2ccc1e'
+  metadata.gz: 063e11ec9aac319a707f3ca11d2d92b02d4aeeeffc35fdf2ec7e169ab8e14b82f4949b0d6f03255a4285186d43620ccb102584feaa616c02677ce6cefbee39b4
+  data.tar.gz: f7ca005517c6bc32c6e3b803b3c577b4e5e103bcc34b603e64534526430189b8f5b39cc87ee3235832ec4d2c078088091251fa1c66e409111ef20020d6d3e5ff

data/lib/hot_cell/configuration.rb

@@ -23,6 +23,7 @@ module HotCell
       queue_wait:       10,  # seconds a queued connection may wait before it is answered `capacity`
       max_requests_per_worker:            1,   # requests a worker serves before it is discarded
       control_deadline: 5,   # seconds a control connection may take to send its request
+      sweep_interval:   10,  # seconds between the supervisor's checks for a killed request's tree to unlink
     }.freeze
 
     LIMITS = {
@@ -51,6 +52,7 @@ module HotCell
       # in describe's JSON, and they may arrive as Active Support durations.
       @queue_wait = @queue_wait.to_f
       @control_deadline = @control_deadline.to_f
+      @sweep_interval = @sweep_interval.to_f
 
       # A nil is not "use the default" here, it is a missing number. A cell whose deadline is nil accepts
       # every request and then dies on the first arithmetic the supervisor does with it, so an explicit nil
@@ -103,6 +105,7 @@ module HotCell
         positive! :concurrency, integer: true
         positive! :queue_wait
         positive! :control_deadline
+        positive! :sweep_interval
 
         unless queue_size.is_a?(Integer) && !queue_size.negative?
           raise ConfigurationError, "queue_size: #{queue_size} must not be negative"

data/lib/hot_cell/control.rb

@@ -40,8 +40,12 @@ module HotCell
 
     # Static, and called once per registered cell at app boot. It is the cheapest way to catch a client
     # whose own timeout is below what this cell may take, and it is what `bin/hotcell describe` reads.
+    #
+    # `v` is the wire format and changes only when the format does; `server_version` says which release of
+    # hotcell-server is answering.
     def describe
-      { v: PROTOCOL_VERSION, operations: Registry.names, groups: groups, **@configuration.to_h }
+      { v: PROTOCOL_VERSION, server_version: Server::VERSION, operations: Registry.names, groups: groups,
+        **@configuration.to_h }
     end
 
     private

data/lib/hot_cell/filesystem.rb

@@ -18,8 +18,12 @@ module HotCell
     #
     # `Dir.exist?` is not the guard, because it follows symlinks and answers false for a dangling one, and
     # an entry a tool left in a directory's place is exactly what this has to remove.
+    #
+    # A tree that is gone by the time the removal fails is the outcome this wants, however it went. Two
+    # sweepers can meet on one discarded tree — the worker that answered on the slot and the supervisor's
+    # own — and the loser's walk fails on an entry the winner unlinked first.
     def self.remove_tree(path)
-      return true unless File.exist?(path) || File.symlink?(path)
+      return true if gone?(path)
 
       FileUtils.remove_entry path
       true
@@ -27,12 +31,26 @@ module HotCell
       repair_and_remove path
     end
 
+    # `lstat` rather than `File.exist?`, which answers false for a path it cannot stat as well as for one that
+    # is gone. Only ENOENT means gone; a tree behind a directory a tool made unsearchable is still there.
+    def self.gone?(path)
+      File.lstat path
+      false
+    rescue Errno::ENOENT
+      true
+    rescue SystemCallError
+      false
+    end
+    private_class_method :gone?
+
     def self.repair_and_remove(path)
+      return true if gone?(path)
+
       FileUtils.chmod_R 0o700, path, force: true
       FileUtils.remove_entry path
       true
     rescue SystemCallError
-      false
+      gone?(path)
     end
     private_class_method :repair_and_remove
   end

data/lib/hot_cell/health_operations.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "hot_cell/server"
+
+# The two round trips an application runs against its own cell for a health check. Shipped in the
+# gem because every deployment was copying them from examples/, and copies drift.
+#
+# A cell serves them only if it requires this file.
+module HotCell
+  module Health
+    # The message goes in through the caller's input descriptor and out through the caller's output
+    # descriptor, with no copy onto scratch, so one round trip proves descriptor passing end to end.
+    class Echo < HotCell::Operation
+      operation "health.echo"
+
+      def perform(inputs, outputs)
+        bytes = outputs.first.to_io.write(inputs.first.to_io.read)
+
+        { bytes: bytes, staged: inputs.first.staged? }
+      end
+    end
+
+    # `fd_path` is a fresh open by name — `/dev/fd/N` on Linux, the file's own path on macOS — checked
+    # against this process's uid and the file's mode, so this succeeds only when the cell can open the
+    # caller's files by name — which echo never checks.
+    # It reads the input and writes the output, because a tool may need either permission.
+    class Reopen < HotCell::Operation
+      operation "health.reopen"
+
+      def perform(inputs, outputs)
+        source, = inputs
+        destination, = outputs
+
+        bytes = File.open(source.fd_path, "rb") do |input|
+          File.open(destination.fd_path, "wb") { |output| output.write(input.read) }
+        end
+
+        { bytes: bytes, staged: source.staged? || destination.staged? }
+      end
+    end
+  end
+end

data/lib/hot_cell/log.rb

@@ -35,11 +35,18 @@ module HotCell
       "worker.unforkable" => "ERROR",
       "worker.undispatchable" => "ERROR",
       "worker.unreadable_report" => "ERROR",
+      "sweeper.forked" => "INFO",
+      "sweeper.deadline" => "WARN",
+      "sweeper.unforkable" => "ERROR",
+      "sweeper.crashed" => "ERROR",
+      "scratch.swept" => "INFO",
       "control.abandoned" => "WARN",
       "control.unanswerable" => "WARN",
       "slot.uncleaned" => "WARN",
       "scratch.unswept" => "WARN",
       "slot.undiscarded" => "WARN",
+      "slot.unswept" => "WARN",
+      "sweeper.died" => "WARN",
     }.freeze
 
     def self.null

data/lib/hot_cell/server/version.rb

@@ -2,6 +2,6 @@
 
 module HotCell
   module Server
-    VERSION = "0.4.1"
+    VERSION = "0.6.0"
   end
 end

data/lib/hot_cell/server.rb

@@ -15,6 +15,7 @@ require "hot_cell/timing"
 require "hot_cell/counters"
 require "hot_cell/control"
 require "hot_cell/worker"
+require "hot_cell/sweeper"
 require "hot_cell/supervisor"
 
 module HotCell

data/lib/hot_cell/slot.rb

@@ -95,7 +95,8 @@ module HotCell
     #
     # A rename within one filesystem is O(1) and takes the tree out of the way. A worker sweeps it later,
     # after it has answered and before it reports itself idle — see Worker#serve, which is the one window
-    # where the unlinking costs nobody's latency.
+    # where the unlinking costs nobody's latency. A worker killed at its deadline never reaches that window,
+    # so the supervisor also forks a Sweeper on a timer, which unlinks in a process of its own.
     #
     # The destination carries a random suffix rather than a counter, because the tool that filled the
     # directory runs as this user and can write to the slot's directory. A predictable name lets it
@@ -133,12 +134,25 @@ module HotCell
     # Unlinks whatever discard_home renamed out of the way. Partial progress is fine: a sweep killed
     # part-way leaves fewer entries for the next one, so this converges rather than repeating.
     def sweep
-      Dir.glob(File.join(directory, "discarded-*")).map { |path| Filesystem.remove_tree(path) }.all?
+      discarded.map { |path| Filesystem.remove_tree(path) }.all?
     rescue SystemCallError
       # The glob itself can fail, because the slot directory is a name a tool can replace — a symlink loop
       # in its place answers ELOOP here rather than for any one entry. This runs from the worker's ensure,
       # where a raise would replace the caller's response with a crash.
       false
     end
+
+    # What discard_home has renamed aside and nobody has unlinked yet.
+    def discarded
+      Dir.glob(File.join(directory, "discarded-*"))
+    end
+
+    # Streams the directory and stops at the first match, because the supervisor asks this in its loop and
+    # a tool can put as many entries beside the discarded ones as it likes; a glob would list and sort them all.
+    def discarded?
+      Dir.each_child(directory).any? { |name| name.start_with?("discarded-") }
+    rescue Errno::ENOENT
+      false
+    end
   end
 end

data/lib/hot_cell/supervisor.rb

@@ -22,11 +22,27 @@ module HotCell
   # the accept anyway, for the queue, for queued_ms, and to answer `capacity`. It also means the supervisor
   # knows when every worker started its current request, which is what the deadline needs.
   class Supervisor
+    # The one deadline the supervisor enforces, on a worker and on the sweeper alike: from `started_at`,
+    # for `deadline` seconds, killed once. `killed_for` is the one-kill latch — a killed process stays here
+    # until the reap, and without the latch it would be re-killed and re-logged on every pass until then.
+    # See `Child#overdue?` for the measurement behind that.
+    module Timed
+      def overdue?(now)
+        timed? && now - started_at >= deadline
+      end
+
+      def expires_at
+        started_at + deadline if timed?
+      end
+    end
+
     # Owns "is this worker busy" and the two transitions that change the answer, because the supervisor asking
     # `busy?` and the supervisor assigning the four fields `busy?` is computed from are the same fact. Spread
     # across the caller, a new field is one the next transition forgets to clear.
     Child = Struct.new(:slot, :pid, :control, :connection, :dispatched_at, :deadline, :served, :killed_for,
                        :op, :retired_at, :buffer, :stderr, :captured, keyword_init: true) do
+      include Timed
+
       def self.build(slot:, pid:, control:, deadline:, stderr: nil)
         new slot: slot, pid: pid, control: control, deadline: deadline, served: 0, buffer: "".b,
             stderr: stderr, captured: "".b
@@ -87,12 +103,12 @@ module HotCell
       # SIGKILLs and 72 synchronous stdout writes for one breach. The window is longest exactly when the host
       # is already struggling — a worker in uninterruptible sleep, or one tearing down gigabytes of mappings —
       # and the loop it starves is the one enforcing every other request's deadline.
-      def overdue?(now)
-        busy? && killed_for.nil? && now - dispatched_at >= deadline
+      def timed?
+        busy? && killed_for.nil?
       end
 
-      def expires_at
-        dispatched_at + deadline if busy? && killed_for.nil?
+      def started_at
+        dispatched_at
       end
 
       # The retirement analogue of `overdue?`, and the only timer that can reach a worker whose idle
@@ -111,6 +127,18 @@ module HotCell
       end
     end
 
+    # The one sweeper the supervisor runs at a time. Not a `Child`: it is dispatched no request, holds no
+    # slot and answers nobody, so none of `busy?`, retirement or the idle report applies to it. What it
+    # shares with a worker is the deadline, and that is `Timed` — the same measurement, the same latch, the
+    # same `deadline` value from the configuration, and the same kill.
+    Sweep = Struct.new(:pid, :started_at, :deadline, :killed_for, keyword_init: true) do
+      include Timed
+
+      def timed?
+        killed_for.nil?
+      end
+    end
+
     # A path longer than this fails to bind with an error that does not say so. Darwin allows four fewer
     # bytes than Linux, and control.sock is the longer of the two names, so it overflows first.
     SUN_PATH_MAX = RUBY_PLATFORM.include?("darwin") ? 104 : 108
@@ -151,6 +179,8 @@ module HotCell
       @control_pending = []
       @counters = Counters.new
       @stopping = false
+      @sweep = nil
+      @next_sweep_at = Clock.now + configuration.sweep_interval
     end
 
     def boot
@@ -161,6 +191,7 @@ module HotCell
       verify_scratches!
       prepare_directories
       preload
+      warm_up
       @work = listen "work.sock"
       @control = listen "control.sock"
       @control_handler = Control.new(configuration: configuration, counters: counters)
@@ -172,16 +203,20 @@ module HotCell
     end
 
     def run
+      prefork
+
       until stopped?
         readable, = IO.select(sources, nil, nil, wait_for)
         Array(readable).each { |source| handle source }
 
         enforce_deadlines
         enforce_retirements
+        enforce_sweep_deadline
         expire_queue
         expire_control
         retire_idle if @stopping
         pump
+        sweep_if_due
       end
     ensure
       shutdown
@@ -204,13 +239,16 @@ module HotCell
       end
 
       # The nearest thing that needs doing without anybody knocking: a deadline, a queued connection that
-      # has waited long enough to be told so, or a control client that never said what it wanted.
+      # has waited long enough to be told so, a control client that never said what it wanted, or the
+      # sweeper's next tick.
       def wait_for
         now = Clock.now
         nearest = [ *@children.each_value.filter_map(&:expires_at),
                     *@children.each_value.filter_map { |child| child.lingers_until(Configuration::KILL_GRACE) },
                     *@queue.map { |(_, queued_at)| queued_at + configuration.queue_wait },
-                    *@control_pending.map { |pending| pending.accepted_at + configuration.control_deadline } ].min
+                    *@control_pending.map { |pending| pending.accepted_at + configuration.control_deadline },
+                    @sweep&.expires_at,
+                    (@next_sweep_at unless @stopping) ].compact.min
         return nil if nearest.nil?
 
         [ nearest - now, 0 ].max
@@ -428,8 +466,20 @@ module HotCell
         @children.each_value.find(&:available?) || spawn
       end
 
+      # Forks a worker into each free slot, so that a request does not wait for `fork`. If a fork fails,
+      # `spawn` logs it and this stops. The next request forks its own worker.
+      def prefork
+        return if @stopping
+
+        while free_slot
+          break unless spawn
+        end
+      end
+
       def spawn
-        number = free_slot or return nil
+        number = free_slot
+        return nil if number.nil?
+
         slot = Slot.build(workspace, number)
         supervisor_side, worker_side = UNIXSocket.pair(:STREAM)
         stderr_reader, stderr_writer = IO.pipe
@@ -459,12 +509,12 @@ module HotCell
                                         deadline: configuration.limits.deadline, stderr: stderr_reader)
       end
 
-      # Everything the supervisor holds and the worker must not: the listener, the signal pipe, the other
-      # children's control sockets, and every connection the supervisor is still holding for somebody else.
       # The connection this worker is about to serve arrives over SCM_RIGHTS a moment from now, so closing
-      # the inherited copy here costs nothing and stops it lingering for the worker's whole life.
+      # the inherited copy in `leave_supervisor` costs nothing and stops it lingering for the worker's whole
+      # life.
       def become_worker(supervisor_side, stderr_reader, stderr_writer)
-        [ "CHLD", "INT", "TERM" ].each { |signal| trap signal, "DEFAULT" }
+        leave_supervisor
+        supervisor_side.close
 
         # Its own process group, so the deadline reaches the tools this request started rather than only the
         # Ruby process that started them. A tool is a grandchild — the worker spawns it — and killing the
@@ -487,20 +537,6 @@ module HotCell
         # reasons that section records. Landlock is the candidate that fits, tracked at basecamp/hotcell#13.
         Process.setpgid 0, 0
 
-        supervisor_side.close
-        @signals.close
-        @signal_writer.close
-        @work.close
-        @control.close
-
-        @children.each_value do |child|
-          child.control.close
-          child.connection&.close
-          child.stderr&.close
-        end
-        @queue.each { |(connection, _)| connection.close }
-        @control_pending.each { |pending| pending.connection.close }
-
         # fd 2 becomes the pipe, and it stays non-blocking. `IO.pipe` already returns both ends O_NONBLOCK
         # and `reopen` is a dup2, which shares the file description — so the flag would ride along on its
         # own. It is set here anyway, because a decision this load-bearing should be in the code rather than
@@ -518,6 +554,80 @@ module HotCell
         stderr_writer.close
       end
 
+      # Everything the supervisor holds and a child of its must not: the listeners, the signal pipe, the
+      # children's control sockets, and every connection the supervisor is still holding for somebody else.
+      def leave_supervisor
+        [ "CHLD", "INT", "TERM" ].each { |signal| trap signal, "DEFAULT" }
+
+        @signals.close
+        @signal_writer.close
+        @work.close
+        @control.close
+
+        @children.each_value do |child|
+          child.control.close
+          child.connection&.close
+          child.stderr&.close
+        end
+        @queue.each { |(connection, _)| connection.close }
+        @control_pending.each { |pending| pending.connection.close }
+      end
+
+      # Ticks whether or not it forks, so a sweep that outlives an interval is left to its deadline rather
+      # than joined by a second one. Forking only when a slot holds a tree keeps an idle cell from paying
+      # for a child every interval; the glob it costs is the one `Slot#discard_home` already runs inline.
+      def sweep_if_due
+        return if @stopping || Clock.now < @next_sweep_at
+
+        @next_sweep_at = Clock.now + configuration.sweep_interval
+        spawn_sweeper if @sweep.nil? && discarded_anywhere?
+      end
+
+      # A glob that raises is a slot directory a tool replaced, and the sweeper is the process that reports
+      # that, so it is forked to find out.
+      def discarded_anywhere?
+        (0...configuration.concurrency).any? { |number| Slot.build(workspace, number).discarded? }
+      rescue SystemCallError
+        true
+      end
+
+      # Held to the cell's deadline: a tree that takes longer to unlink than a request is allowed to run is
+      # one an input built to, and the next tick sweeps what this one left. A fork that fails is the host
+      # under pressure, as in `spawn`; the trees wait for the next tick.
+      def spawn_sweeper
+        pid = fork do
+          leave_supervisor
+          Sweeper.new(workspace: workspace, configuration: configuration, log: log).run
+        end
+
+        log.write "sweeper.forked", pid: pid
+        @sweep = Sweep.new(pid: pid, started_at: Clock.now, deadline: configuration.limits.deadline)
+      rescue SystemCallError => error
+        log.write "sweeper.unforkable", error: error.class.name, message: error.message
+      end
+
+      # Kill first, log second, as `enforce_deadlines` does. The sweeper spawns nothing and leads no group,
+      # so `kill_group` reaches it through its fallback to the bare pid.
+      def enforce_sweep_deadline
+        return unless @sweep&.overdue?(Clock.now)
+
+        @sweep.killed_for = Codes::DEADLINE
+        kill_group @sweep
+        log.write "sweeper.deadline", pid: @sweep.pid, deadline_s: @sweep.deadline
+      end
+
+      # A sweeper this supervisor killed already has its `sweeper.deadline` line. Any other abnormal end —
+      # the OOM killer, a sibling's signal, a crash `Sweeper#run` could not catch — would otherwise leave
+      # only `sweeper.forked` behind, and look exactly like a sweep that finished.
+      def reap_sweeper(status)
+        unless @sweep.killed_for || status.success?
+          log.write "sweeper.died", pid: @sweep.pid, signal: signal_name(status), exit_code: status.exitstatus
+        end
+
+        @sweep = nil
+      end
+
+
       # One bounded read per pass, never a loop until the pipe is empty: this runs inside the loop that
       # enforces every request's deadline, and the peer is a worker that can print as fast as it likes.
       #
@@ -762,18 +872,23 @@ module HotCell
       # the sweep must not signal it. An empty group is the common case, so ESRCH is expected. It is safe to
       # kill the group by the leader's pid even though the leader is reaped, because the supervisor is single
       # threaded and mints group leaders only in `spawn`, which cannot run between the `wait2` above and here.
+      #
+      # macOS answers EPERM rather than ESRCH for a group whose only members are zombies, and a tool killed
+      # with its worker stays one until launchd reaps it. Raised, it unwound `run` and ended the cell before
+      # this reap answered the caller. EPERM means no member received the signal, the same outcome as ESRCH.
       def sweep_group(child)
         Process.kill :KILL, -child.pid
-      rescue Errno::ESRCH
+      rescue Errno::ESRCH, Errno::EPERM
         nil
       end
 
       # The whole process group, which is the worker and everything it started. Negative pid is the group.
       # Falls back to the worker alone if the group is already gone, so a worker that died between the check
-      # and the signal is not an error.
+      # and the signal is not an error. A group answering EPERM falls back as well, for the reason
+      # `sweep_group` gives; the bare pid does not, because macOS signals a zombie pid without complaint.
       def kill_group(child)
         Process.kill :KILL, -child.pid
-      rescue Errno::ESRCH
+      rescue Errno::ESRCH, Errno::EPERM
         begin
           Process.kill :KILL, child.pid
         rescue Errno::ESRCH
@@ -810,6 +925,11 @@ module HotCell
           pid, status = Process.wait2(-1, Process::WNOHANG)
           break if pid.nil?
 
+          if @sweep&.pid == pid
+            reap_sweeper status
+            next
+          end
+
           child = @children.each_value.find { |candidate| candidate.pid == pid }
           next if child.nil?
 
@@ -836,6 +956,12 @@ module HotCell
 
           log.write "worker.reaped", pid: pid, slot: child.slot.number, served: child.served,
                                      signal: signal_name(status), exit_code: status.exitstatus
+
+          # A worker that exited without serving a request is not replaced. This guards against a worker
+          # that crashes during its startup, for example after a bad deploy. A replacement would crash too,
+          # and the supervisor would fork again, with no requests arriving. Instead, the next request forks
+          # its own worker.
+          prefork if child.served.positive?
         end
       rescue Errno::ECHILD
         nil
@@ -1137,11 +1263,19 @@ module HotCell
         Registry.operations.each { |operation| operation.before_fork.each(&:call) }
       end
 
+      # Runs after `preload`, so that the objects the operations loaded are compacted too. Runs before the
+      # first fork, so that each worker inherits the compacted heap. The compacted objects use fewer pages,
+      # so a worker copies fewer pages when it writes to them.
+      def warm_up
+        Process.warmup
+      end
+
 
       def shutdown
         refuse_queue "the cell is stopping"
         @control_pending.each { |pending| pending.connection.close }
         @children.each_value { |child| child.control.close unless child.control.socket.closed? }
+        kill_group @sweep if @sweep
         @work&.close
         @control&.close
         SOCKETS.each { |name| File.unlink socket_path(name) if File.socket?(socket_path(name)) }

data/lib/hot_cell/sweeper.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module HotCell
+  # Unlinks what the supervisor renamed aside, in a process of its own.
+  #
+  # How long a recursive delete takes is chosen by the input that filled the tree, so it runs in neither
+  # the supervisor, whose loop enforces every deadline, nor a worker's request. A worker sweeps its own
+  # slot after it has answered, but a worker killed at its deadline never reaches that ensure — and a slot
+  # whose every request is killed stacked one tree per kill until the scratch was full. This is the sweep
+  # that needs no request to run: the supervisor forks it on a timer, holds it to a deadline, and never
+  # runs two at once.
+  class Sweeper
+    def initialize(workspace:, configuration:, log:)
+      @workspace = workspace
+      @configuration = configuration
+      @log = log
+    end
+
+    # exit! for the reason Worker#run does: nothing inherited from the supervisor may run its teardown here.
+    #
+    # The cell's memory limit goes on first, and only that one. `FileUtils.remove_entry` lists a directory
+    # before it unlinks anything in it, so a tree one directory wide enough allocates in proportion to its
+    # width; RLIMIT_DATA makes that this process's NoMemoryError and a `sweeper.crashed` line. It is a bound
+    # on this process and not on the cell: the cgroup counts every worker and the tmpfs too, and can run out
+    # first. A sweep that dies this way makes no progress on that directory, and that is accepted here.
+    #
+    # Not `file_size`: this process writes nothing but log lines, and a log that is a regular file is past
+    # any worker's limit already, so the first line would have killed the sweeper with SIGXFSZ.
+    def run
+      configuration.limits.merge(file_size: nil, open_files: nil).apply
+      started = Clock.now
+      swept = slots.sum { |slot| sweep slot }
+
+      log.write "scratch.swept", pid: Process.pid, swept: swept, duration_ms: Clock.ms_since(started)
+      exit! 0
+    rescue Exception => error
+      log.write "sweeper.crashed", pid: Process.pid, error: error.class.name, message: Failure.sanitize(error.message)
+      exit! 1
+    end
+
+    private
+      attr_reader :workspace, :configuration, :log
+
+      def slots
+        (0...configuration.concurrency).map { |number| Slot.build(workspace, number) }
+      end
+
+      # A tree that would not go is the worker's `slot.unswept`, from this pid: the same fact, whoever
+      # noticed it. The glob can raise — the slot directory is a name a tool can replace — and that too is
+      # a slot left unswept rather than the end of the sweep.
+      def sweep(slot)
+        trees = slot.discarded
+        removed = trees.count { |path| Filesystem.remove_tree(path) }
+        report_unswept slot if removed < trees.size
+        removed
+      rescue SystemCallError
+        report_unswept slot
+        0
+      end
+
+      def report_unswept(slot)
+        log.write "slot.unswept", pid: Process.pid, slot: slot.number, home: slot.directory
+      end
+  end
+end

data/lib/hot_cell/test_cell.rb

@@ -26,6 +26,9 @@ module HotCell
   # which is the only place such a library may be loaded.
   #
   #   HotCell::TestCell.boot(operations: -> { require "active_storage/hot_cell/server" })
+  #
+  # The cell's sockets go beneath `Dir.tmpdir`, and a Unix socket path holds 104 bytes on macOS, whose per-user
+  # `TMPDIR` spends about half of that. A suite on macOS should set a short `TMPDIR`, such as `/tmp`.
   class TestCell
     READY = "up"
 

data/lib/hot_cell/test_operations.rb

@@ -70,6 +70,16 @@ module HotCell
       end
     end
 
+    # A worker inherits the supervisor's GC counters at the fork. So this reports what the supervisor did to
+    # its heap before the fork.
+    class HeapState < HotCell::Operation
+      operation "test.heap_state"
+
+      def perform(_inputs, _outputs)
+        { compactions: GC.stat(:compact_count) }
+      end
+    end
+
     # Leaves its $HOME in a state the worker cannot remove, by taking write permission off a subdirectory
     # that still has a file in it. A tool running as this user can do the same to a sibling's directory,
     # which is why a removal that fails has to be reported rather than swallowed.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hotcell-server
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Mike Dalessio
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.6.0
 description: |
   Runs a HotCell container. A supervisor listens on two Unix sockets, forks a worker for each request,
   and enforces a wall clock deadline and resource limits on it. Write the work as a subclass of
@@ -43,6 +43,7 @@ files:
 - lib/hot_cell/control.rb
 - lib/hot_cell/counters.rb
 - lib/hot_cell/filesystem.rb
+- lib/hot_cell/health_operations.rb
 - lib/hot_cell/limits.rb
 - lib/hot_cell/log.rb
 - lib/hot_cell/operation.rb
@@ -52,6 +53,7 @@ files:
 - lib/hot_cell/server/version.rb
 - lib/hot_cell/slot.rb
 - lib/hot_cell/supervisor.rb
+- lib/hot_cell/sweeper.rb
 - lib/hot_cell/test_cell.rb
 - lib/hot_cell/test_operations.rb
 - lib/hot_cell/timing.rb
@@ -62,8 +64,8 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/basecamp/hotcell
-  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.4.1/hotcell-server
-  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.4.1/CHANGELOG.md
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.6.0/hotcell-server
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.6.0/CHANGELOG.md
   bug_tracker_uri: https://github.com/basecamp/hotcell/issues
   rubygems_mfa_required: 'true'
 rdoc_options: []
@@ -80,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.6
 specification_version: 4
 summary: 'Run a HotCell: the supervisor, the worker, and the operation API.'
 test_files: []