hotcell-server 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 390fc0f10953763de1f745cc3002fa0e548b362cbb21c82faba7dfa807b49dd5
-  data.tar.gz: fd6932e090acccea7964e38d76e65ea6163b1ebf6b5f401f6854fe8fa1e38034
+  metadata.gz: b69f25f352406df26617692f42c1ce78c7191df4d7b617360c62c2371e488bed
+  data.tar.gz: 34ca8c1eea5a9de3cf739ce548151ac5c9487e185739f684d179844c13257b1d
 SHA512:
-  metadata.gz: a34f827e6dbb2616e4e20b122ff6a8fdd78e40c19bfd8297c767c450a06ea4efaebc245d3ca3fb117caf13b70d922236e3c51f6a0a93207b5fc0a815e60e2583
-  data.tar.gz: '0769c12d7f26f35380dd5532edd8073c160f6a823128dcc4c88fddc697fd896be8cabec1eaa21dedf3c505a26ccda5a90cc38716a23461fa70410eb92d2ccc1e'
+  metadata.gz: 5667ac9be32cd60ab131df56fb4be0a0354f007a19d31990c6210076dcd6e72c019a903b8e4f4f89ed5bf4ee444269de2b30d4776aa6373750f02ab21fa2869b
+  data.tar.gz: 9567d005d357239081a4c804b081939714c1ead9afb2588fcbd95d6795c638770348959d00549d49d9b76ec7cdc7bb0e3b2f7dd226a1a001d0c532fb524a05d2

data/lib/hot_cell/configuration.rb

@@ -23,6 +23,7 @@ module HotCell
       queue_wait:       10,  # seconds a queued connection may wait before it is answered `capacity`
       max_requests_per_worker:            1,   # requests a worker serves before it is discarded
       control_deadline: 5,   # seconds a control connection may take to send its request
+      sweep_interval:   10,  # seconds between the supervisor's checks for a killed request's tree to unlink
     }.freeze
 
     LIMITS = {
@@ -51,6 +52,7 @@ module HotCell
       # in describe's JSON, and they may arrive as Active Support durations.
       @queue_wait = @queue_wait.to_f
       @control_deadline = @control_deadline.to_f
+      @sweep_interval = @sweep_interval.to_f
 
       # A nil is not "use the default" here, it is a missing number. A cell whose deadline is nil accepts
       # every request and then dies on the first arithmetic the supervisor does with it, so an explicit nil
@@ -103,6 +105,7 @@ module HotCell
         positive! :concurrency, integer: true
         positive! :queue_wait
         positive! :control_deadline
+        positive! :sweep_interval
 
         unless queue_size.is_a?(Integer) && !queue_size.negative?
           raise ConfigurationError, "queue_size: #{queue_size} must not be negative"

data/lib/hot_cell/filesystem.rb

@@ -18,8 +18,12 @@ module HotCell
     #
     # `Dir.exist?` is not the guard, because it follows symlinks and answers false for a dangling one, and
     # an entry a tool left in a directory's place is exactly what this has to remove.
+    #
+    # A tree that is gone by the time the removal fails is the outcome this wants, however it went. Two
+    # sweepers can meet on one discarded tree — the worker that answered on the slot and the supervisor's
+    # own — and the loser's walk fails on an entry the winner unlinked first.
     def self.remove_tree(path)
-      return true unless File.exist?(path) || File.symlink?(path)
+      return true if gone?(path)
 
       FileUtils.remove_entry path
       true
@@ -27,12 +31,26 @@ module HotCell
       repair_and_remove path
     end
 
+    # `lstat` rather than `File.exist?`, which answers false for a path it cannot stat as well as for one that
+    # is gone. Only ENOENT means gone; a tree behind a directory a tool made unsearchable is still there.
+    def self.gone?(path)
+      File.lstat path
+      false
+    rescue Errno::ENOENT
+      true
+    rescue SystemCallError
+      false
+    end
+    private_class_method :gone?
+
     def self.repair_and_remove(path)
+      return true if gone?(path)
+
       FileUtils.chmod_R 0o700, path, force: true
       FileUtils.remove_entry path
       true
     rescue SystemCallError
-      false
+      gone?(path)
     end
     private_class_method :repair_and_remove
   end

data/lib/hot_cell/log.rb

@@ -35,11 +35,18 @@ module HotCell
       "worker.unforkable" => "ERROR",
       "worker.undispatchable" => "ERROR",
       "worker.unreadable_report" => "ERROR",
+      "sweeper.forked" => "INFO",
+      "sweeper.deadline" => "WARN",
+      "sweeper.unforkable" => "ERROR",
+      "sweeper.crashed" => "ERROR",
+      "scratch.swept" => "INFO",
       "control.abandoned" => "WARN",
       "control.unanswerable" => "WARN",
       "slot.uncleaned" => "WARN",
       "scratch.unswept" => "WARN",
       "slot.undiscarded" => "WARN",
+      "slot.unswept" => "WARN",
+      "sweeper.died" => "WARN",
     }.freeze
 
     def self.null

data/lib/hot_cell/server/version.rb

@@ -2,6 +2,6 @@
 
 module HotCell
   module Server
-    VERSION = "0.4.1"
+    VERSION = "0.5.0"
   end
 end

data/lib/hot_cell/server.rb

@@ -15,6 +15,7 @@ require "hot_cell/timing"
 require "hot_cell/counters"
 require "hot_cell/control"
 require "hot_cell/worker"
+require "hot_cell/sweeper"
 require "hot_cell/supervisor"
 
 module HotCell

data/lib/hot_cell/slot.rb

@@ -95,7 +95,8 @@ module HotCell
     #
     # A rename within one filesystem is O(1) and takes the tree out of the way. A worker sweeps it later,
     # after it has answered and before it reports itself idle — see Worker#serve, which is the one window
-    # where the unlinking costs nobody's latency.
+    # where the unlinking costs nobody's latency. A worker killed at its deadline never reaches that window,
+    # so the supervisor also forks a Sweeper on a timer, which unlinks in a process of its own.
     #
     # The destination carries a random suffix rather than a counter, because the tool that filled the
     # directory runs as this user and can write to the slot's directory. A predictable name lets it
@@ -133,12 +134,25 @@ module HotCell
     # Unlinks whatever discard_home renamed out of the way. Partial progress is fine: a sweep killed
     # part-way leaves fewer entries for the next one, so this converges rather than repeating.
     def sweep
-      Dir.glob(File.join(directory, "discarded-*")).map { |path| Filesystem.remove_tree(path) }.all?
+      discarded.map { |path| Filesystem.remove_tree(path) }.all?
     rescue SystemCallError
       # The glob itself can fail, because the slot directory is a name a tool can replace — a symlink loop
       # in its place answers ELOOP here rather than for any one entry. This runs from the worker's ensure,
       # where a raise would replace the caller's response with a crash.
       false
     end
+
+    # What discard_home has renamed aside and nobody has unlinked yet.
+    def discarded
+      Dir.glob(File.join(directory, "discarded-*"))
+    end
+
+    # Streams the directory and stops at the first match, because the supervisor asks this in its loop and
+    # a tool can put as many entries beside the discarded ones as it likes; a glob would list and sort them all.
+    def discarded?
+      Dir.each_child(directory).any? { |name| name.start_with?("discarded-") }
+    rescue Errno::ENOENT
+      false
+    end
   end
 end

data/lib/hot_cell/supervisor.rb

@@ -22,11 +22,27 @@ module HotCell
   # the accept anyway, for the queue, for queued_ms, and to answer `capacity`. It also means the supervisor
   # knows when every worker started its current request, which is what the deadline needs.
   class Supervisor
+    # The one deadline the supervisor enforces, on a worker and on the sweeper alike: from `started_at`,
+    # for `deadline` seconds, killed once. `killed_for` is the one-kill latch — a killed process stays here
+    # until the reap, and without the latch it would be re-killed and re-logged on every pass until then.
+    # See `Child#overdue?` for the measurement behind that.
+    module Timed
+      def overdue?(now)
+        timed? && now - started_at >= deadline
+      end
+
+      def expires_at
+        started_at + deadline if timed?
+      end
+    end
+
     # Owns "is this worker busy" and the two transitions that change the answer, because the supervisor asking
     # `busy?` and the supervisor assigning the four fields `busy?` is computed from are the same fact. Spread
     # across the caller, a new field is one the next transition forgets to clear.
     Child = Struct.new(:slot, :pid, :control, :connection, :dispatched_at, :deadline, :served, :killed_for,
                        :op, :retired_at, :buffer, :stderr, :captured, keyword_init: true) do
+      include Timed
+
       def self.build(slot:, pid:, control:, deadline:, stderr: nil)
         new slot: slot, pid: pid, control: control, deadline: deadline, served: 0, buffer: "".b,
             stderr: stderr, captured: "".b
@@ -87,12 +103,12 @@ module HotCell
       # SIGKILLs and 72 synchronous stdout writes for one breach. The window is longest exactly when the host
       # is already struggling — a worker in uninterruptible sleep, or one tearing down gigabytes of mappings —
       # and the loop it starves is the one enforcing every other request's deadline.
-      def overdue?(now)
-        busy? && killed_for.nil? && now - dispatched_at >= deadline
+      def timed?
+        busy? && killed_for.nil?
       end
 
-      def expires_at
-        dispatched_at + deadline if busy? && killed_for.nil?
+      def started_at
+        dispatched_at
       end
 
       # The retirement analogue of `overdue?`, and the only timer that can reach a worker whose idle
@@ -111,6 +127,18 @@ module HotCell
       end
     end
 
+    # The one sweeper the supervisor runs at a time. Not a `Child`: it is dispatched no request, holds no
+    # slot and answers nobody, so none of `busy?`, retirement or the idle report applies to it. What it
+    # shares with a worker is the deadline, and that is `Timed` — the same measurement, the same latch, the
+    # same `deadline` value from the configuration, and the same kill.
+    Sweep = Struct.new(:pid, :started_at, :deadline, :killed_for, keyword_init: true) do
+      include Timed
+
+      def timed?
+        killed_for.nil?
+      end
+    end
+
     # A path longer than this fails to bind with an error that does not say so. Darwin allows four fewer
     # bytes than Linux, and control.sock is the longer of the two names, so it overflows first.
     SUN_PATH_MAX = RUBY_PLATFORM.include?("darwin") ? 104 : 108
@@ -151,6 +179,8 @@ module HotCell
       @control_pending = []
       @counters = Counters.new
       @stopping = false
+      @sweep = nil
+      @next_sweep_at = Clock.now + configuration.sweep_interval
     end
 
     def boot
@@ -178,10 +208,12 @@ module HotCell
 
         enforce_deadlines
         enforce_retirements
+        enforce_sweep_deadline
         expire_queue
         expire_control
         retire_idle if @stopping
         pump
+        sweep_if_due
       end
     ensure
       shutdown
@@ -204,13 +236,16 @@ module HotCell
       end
 
       # The nearest thing that needs doing without anybody knocking: a deadline, a queued connection that
-      # has waited long enough to be told so, or a control client that never said what it wanted.
+      # has waited long enough to be told so, a control client that never said what it wanted, or the
+      # sweeper's next tick.
       def wait_for
         now = Clock.now
         nearest = [ *@children.each_value.filter_map(&:expires_at),
                     *@children.each_value.filter_map { |child| child.lingers_until(Configuration::KILL_GRACE) },
                     *@queue.map { |(_, queued_at)| queued_at + configuration.queue_wait },
-                    *@control_pending.map { |pending| pending.accepted_at + configuration.control_deadline } ].min
+                    *@control_pending.map { |pending| pending.accepted_at + configuration.control_deadline },
+                    @sweep&.expires_at,
+                    (@next_sweep_at unless @stopping) ].compact.min
         return nil if nearest.nil?
 
         [ nearest - now, 0 ].max
@@ -459,12 +494,12 @@ module HotCell
                                         deadline: configuration.limits.deadline, stderr: stderr_reader)
       end
 
-      # Everything the supervisor holds and the worker must not: the listener, the signal pipe, the other
-      # children's control sockets, and every connection the supervisor is still holding for somebody else.
       # The connection this worker is about to serve arrives over SCM_RIGHTS a moment from now, so closing
-      # the inherited copy here costs nothing and stops it lingering for the worker's whole life.
+      # the inherited copy in `leave_supervisor` costs nothing and stops it lingering for the worker's whole
+      # life.
       def become_worker(supervisor_side, stderr_reader, stderr_writer)
-        [ "CHLD", "INT", "TERM" ].each { |signal| trap signal, "DEFAULT" }
+        leave_supervisor
+        supervisor_side.close
 
         # Its own process group, so the deadline reaches the tools this request started rather than only the
         # Ruby process that started them. A tool is a grandchild — the worker spawns it — and killing the
@@ -487,20 +522,6 @@ module HotCell
         # reasons that section records. Landlock is the candidate that fits, tracked at basecamp/hotcell#13.
         Process.setpgid 0, 0
 
-        supervisor_side.close
-        @signals.close
-        @signal_writer.close
-        @work.close
-        @control.close
-
-        @children.each_value do |child|
-          child.control.close
-          child.connection&.close
-          child.stderr&.close
-        end
-        @queue.each { |(connection, _)| connection.close }
-        @control_pending.each { |pending| pending.connection.close }
-
         # fd 2 becomes the pipe, and it stays non-blocking. `IO.pipe` already returns both ends O_NONBLOCK
         # and `reopen` is a dup2, which shares the file description — so the flag would ride along on its
         # own. It is set here anyway, because a decision this load-bearing should be in the code rather than
@@ -518,6 +539,80 @@ module HotCell
         stderr_writer.close
       end
 
+      # Everything the supervisor holds and a child of its must not: the listeners, the signal pipe, the
+      # children's control sockets, and every connection the supervisor is still holding for somebody else.
+      def leave_supervisor
+        [ "CHLD", "INT", "TERM" ].each { |signal| trap signal, "DEFAULT" }
+
+        @signals.close
+        @signal_writer.close
+        @work.close
+        @control.close
+
+        @children.each_value do |child|
+          child.control.close
+          child.connection&.close
+          child.stderr&.close
+        end
+        @queue.each { |(connection, _)| connection.close }
+        @control_pending.each { |pending| pending.connection.close }
+      end
+
+      # Ticks whether or not it forks, so a sweep that outlives an interval is left to its deadline rather
+      # than joined by a second one. Forking only when a slot holds a tree keeps an idle cell from paying
+      # for a child every interval; the glob it costs is the one `Slot#discard_home` already runs inline.
+      def sweep_if_due
+        return if @stopping || Clock.now < @next_sweep_at
+
+        @next_sweep_at = Clock.now + configuration.sweep_interval
+        spawn_sweeper if @sweep.nil? && discarded_anywhere?
+      end
+
+      # A glob that raises is a slot directory a tool replaced, and the sweeper is the process that reports
+      # that, so it is forked to find out.
+      def discarded_anywhere?
+        (0...configuration.concurrency).any? { |number| Slot.build(workspace, number).discarded? }
+      rescue SystemCallError
+        true
+      end
+
+      # Held to the cell's deadline: a tree that takes longer to unlink than a request is allowed to run is
+      # one an input built to, and the next tick sweeps what this one left. A fork that fails is the host
+      # under pressure, as in `spawn`; the trees wait for the next tick.
+      def spawn_sweeper
+        pid = fork do
+          leave_supervisor
+          Sweeper.new(workspace: workspace, configuration: configuration, log: log).run
+        end
+
+        log.write "sweeper.forked", pid: pid
+        @sweep = Sweep.new(pid: pid, started_at: Clock.now, deadline: configuration.limits.deadline)
+      rescue SystemCallError => error
+        log.write "sweeper.unforkable", error: error.class.name, message: error.message
+      end
+
+      # Kill first, log second, as `enforce_deadlines` does. The sweeper spawns nothing and leads no group,
+      # so `kill_group` reaches it through its fallback to the bare pid.
+      def enforce_sweep_deadline
+        return unless @sweep&.overdue?(Clock.now)
+
+        @sweep.killed_for = Codes::DEADLINE
+        kill_group @sweep
+        log.write "sweeper.deadline", pid: @sweep.pid, deadline_s: @sweep.deadline
+      end
+
+      # A sweeper this supervisor killed already has its `sweeper.deadline` line. Any other abnormal end —
+      # the OOM killer, a sibling's signal, a crash `Sweeper#run` could not catch — would otherwise leave
+      # only `sweeper.forked` behind, and look exactly like a sweep that finished.
+      def reap_sweeper(status)
+        unless @sweep.killed_for || status.success?
+          log.write "sweeper.died", pid: @sweep.pid, signal: signal_name(status), exit_code: status.exitstatus
+        end
+
+        @sweep = nil
+      end
+
+
       # One bounded read per pass, never a loop until the pipe is empty: this runs inside the loop that
       # enforces every request's deadline, and the peer is a worker that can print as fast as it likes.
       #
@@ -810,6 +905,11 @@ module HotCell
           pid, status = Process.wait2(-1, Process::WNOHANG)
           break if pid.nil?
 
+          if @sweep&.pid == pid
+            reap_sweeper status
+            next
+          end
+
           child = @children.each_value.find { |candidate| candidate.pid == pid }
           next if child.nil?
 
@@ -1142,6 +1242,7 @@ module HotCell
         refuse_queue "the cell is stopping"
         @control_pending.each { |pending| pending.connection.close }
         @children.each_value { |child| child.control.close unless child.control.socket.closed? }
+        kill_group @sweep if @sweep
         @work&.close
         @control&.close
         SOCKETS.each { |name| File.unlink socket_path(name) if File.socket?(socket_path(name)) }

data/lib/hot_cell/sweeper.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module HotCell
+  # Unlinks what the supervisor renamed aside, in a process of its own.
+  #
+  # How long a recursive delete takes is chosen by the input that filled the tree, so it runs in neither
+  # the supervisor, whose loop enforces every deadline, nor a worker's request. A worker sweeps its own
+  # slot after it has answered, but a worker killed at its deadline never reaches that ensure — and a slot
+  # whose every request is killed stacked one tree per kill until the scratch was full. This is the sweep
+  # that needs no request to run: the supervisor forks it on a timer, holds it to a deadline, and never
+  # runs two at once.
+  class Sweeper
+    def initialize(workspace:, configuration:, log:)
+      @workspace = workspace
+      @configuration = configuration
+      @log = log
+    end
+
+    # exit! for the reason Worker#run does: nothing inherited from the supervisor may run its teardown here.
+    #
+    # The cell's memory limit goes on first, and only that one. `FileUtils.remove_entry` lists a directory
+    # before it unlinks anything in it, so a tree one directory wide enough allocates in proportion to its
+    # width; RLIMIT_DATA makes that this process's NoMemoryError and a `sweeper.crashed` line. It is a bound
+    # on this process and not on the cell: the cgroup counts every worker and the tmpfs too, and can run out
+    # first. A sweep that dies this way makes no progress on that directory, and that is accepted here.
+    #
+    # Not `file_size`: this process writes nothing but log lines, and a log that is a regular file is past
+    # any worker's limit already, so the first line would have killed the sweeper with SIGXFSZ.
+    def run
+      configuration.limits.merge(file_size: nil, open_files: nil).apply
+      started = Clock.now
+      swept = slots.sum { |slot| sweep slot }
+
+      log.write "scratch.swept", pid: Process.pid, swept: swept, duration_ms: Clock.ms_since(started)
+      exit! 0
+    rescue Exception => error
+      log.write "sweeper.crashed", pid: Process.pid, error: error.class.name, message: Failure.sanitize(error.message)
+      exit! 1
+    end
+
+    private
+      attr_reader :workspace, :configuration, :log
+
+      def slots
+        (0...configuration.concurrency).map { |number| Slot.build(workspace, number) }
+      end
+
+      # A tree that would not go is the worker's `slot.unswept`, from this pid: the same fact, whoever
+      # noticed it. The glob can raise — the slot directory is a name a tool can replace — and that too is
+      # a slot left unswept rather than the end of the sweep.
+      def sweep(slot)
+        trees = slot.discarded
+        removed = trees.count { |path| Filesystem.remove_tree(path) }
+        report_unswept slot if removed < trees.size
+        removed
+      rescue SystemCallError
+        report_unswept slot
+        0
+      end
+
+      def report_unswept(slot)
+        log.write "slot.unswept", pid: Process.pid, slot: slot.number, home: slot.directory
+      end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hotcell-server
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Mike Dalessio
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.5.0
 description: |
   Runs a HotCell container. A supervisor listens on two Unix sockets, forks a worker for each request,
   and enforces a wall clock deadline and resource limits on it. Write the work as a subclass of
@@ -52,6 +52,7 @@ files:
 - lib/hot_cell/server/version.rb
 - lib/hot_cell/slot.rb
 - lib/hot_cell/supervisor.rb
+- lib/hot_cell/sweeper.rb
 - lib/hot_cell/test_cell.rb
 - lib/hot_cell/test_operations.rb
 - lib/hot_cell/timing.rb
@@ -62,8 +63,8 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/basecamp/hotcell
-  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.4.1/hotcell-server
-  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.4.1/CHANGELOG.md
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.5.0/hotcell-server
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.5.0/CHANGELOG.md
   bug_tracker_uri: https://github.com/basecamp/hotcell/issues
   rubygems_mfa_required: 'true'
 rdoc_options: []