hotcell-server 0.5.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b69f25f352406df26617692f42c1ce78c7191df4d7b617360c62c2371e488bed
-  data.tar.gz: 34ca8c1eea5a9de3cf739ce548151ac5c9487e185739f684d179844c13257b1d
+  metadata.gz: bbb296d96887a9a1bc312a14acef431caa03fc30d05a3f77bd6708a8a9d596a4
+  data.tar.gz: 6f3a72abba2e2e90ca8b805c15b5bff8feef52d1b1ef5c0a73832e190cc4f182
 SHA512:
-  metadata.gz: 5667ac9be32cd60ab131df56fb4be0a0354f007a19d31990c6210076dcd6e72c019a903b8e4f4f89ed5bf4ee444269de2b30d4776aa6373750f02ab21fa2869b
-  data.tar.gz: 9567d005d357239081a4c804b081939714c1ead9afb2588fcbd95d6795c638770348959d00549d49d9b76ec7cdc7bb0e3b2f7dd226a1a001d0c532fb524a05d2
+  metadata.gz: 3facfa26f280439cbf43dc21aff1e3a4cfb3ba3212402f1d1e5f396b2773e629409f926cf8fcf00debcd68fdb33d126acfe7f2d487afdff194180eb724c4029f
+  data.tar.gz: 52b1e35eb13e510c912ca7bb9cc706186e36380ee166f5f6d2bff0c36a22088d84d4bed69db0e81cd987eb493b450dd7f71bfeb0d5db63531e3d2aa9cb7602c8

data/lib/hot_cell/control.rb

@@ -40,8 +40,12 @@ module HotCell
 
     # Static, and called once per registered cell at app boot. It is the cheapest way to catch a client
     # whose own timeout is below what this cell may take, and it is what `bin/hotcell describe` reads.
+    #
+    # `v` is the wire format and changes only when the format does; `server_version` says which release of
+    # hotcell-server is answering.
     def describe
-      { v: PROTOCOL_VERSION, operations: Registry.names, groups: groups, **@configuration.to_h }
+      { v: PROTOCOL_VERSION, server_version: Server::VERSION, operations: Registry.names, groups: groups,
+        **@configuration.to_h }
     end
 
     private

data/lib/hot_cell/health_operations.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "hot_cell/server"
+
+# The two round trips an application runs against its own cell for a health check. Shipped in the
+# gem because every deployment was copying them from examples/, and copies drift.
+#
+# A cell serves them only if it requires this file.
+module HotCell
+  module Health
+    # The message goes in through the caller's input descriptor and out through the caller's output
+    # descriptor, with no copy onto scratch, so one round trip proves descriptor passing end to end.
+    class Echo < HotCell::Operation
+      operation "health.echo"
+
+      def perform(inputs, outputs)
+        bytes = outputs.first.to_io.write(inputs.first.to_io.read)
+
+        { bytes: bytes, staged: inputs.first.staged? }
+      end
+    end
+
+    # `fd_path` is a fresh open by name — `/dev/fd/N` on Linux, the file's own path on macOS — checked
+    # against this process's uid and the file's mode, so this succeeds only when the cell can open the
+    # caller's files by name — which echo never checks.
+    # It reads the input and writes the output, because a tool may need either permission.
+    class Reopen < HotCell::Operation
+      operation "health.reopen"
+
+      def perform(inputs, outputs)
+        source, = inputs
+        destination, = outputs
+
+        bytes = File.open(source.fd_path, "rb") do |input|
+          File.open(destination.fd_path, "wb") { |output| output.write(input.read) }
+        end
+
+        { bytes: bytes, staged: source.staged? || destination.staged? }
+      end
+    end
+  end
+end

data/lib/hot_cell/log.rb

@@ -12,7 +12,7 @@ module HotCell
   # This is the channel for everything no response can carry: deadline kills, reaps that found a signal,
   # boot checks, and queue high-water.
   #
-  # Lines follow docs/LOGS.md: ECS field names, domain fields under the hotcell namespace. The fleet's
+  # Lines follow docs/observability.md: ECS field names, domain fields under the hotcell namespace. The fleet's
   # collector routes on service.name, takes the record timestamp from @timestamp, and severity from
   # log.level, so those three are load-bearing: renaming any of them silently drops or mislabels every
   # cell log line in production.

data/lib/hot_cell/operation.rb

@@ -57,7 +57,7 @@ module HotCell
       # class's own declaration, or of the nearest ancestor's when this class has none yet. That is what
       # lets a subclass narrow a single number, and what lets an operator give a shipped operation a
       # different budget from an operations file, after the operation loads, without editing the gem. The
-      # cell's own limits still clamp whatever is declared. docs/DEPLOYMENT.md, "Changing a shipped
+      # cell's own limits still clamp whatever is declared. docs/operation-api.md, "Changing a shipped
       # operation's limits".
       def limits(**values)
         return inherited_value(:@limits) || Limits.new if values.empty?

data/lib/hot_cell/server/version.rb

@@ -2,6 +2,6 @@
 
 module HotCell
   module Server
-    VERSION = "0.5.0"
+    VERSION = "1.0.0"
   end
 end

data/lib/hot_cell/slot.rb

@@ -37,7 +37,7 @@ module HotCell
   # uid and `0700` is the owner's own mode, so a concurrent sibling — or a `setsid` descendant of a finished
   # request, which process groups do not contain — can still write into a home the moment it exists. The
   # slot directory itself can be renamed aside and replaced, and `chmod` follows what it finds. Both are the
-  # residuals `docs/DESIGN.md` records under worker isolation, and neither is closed here.
+  # residuals `docs/design/worker-isolation.md` records, and neither is closed here.
   #
   # There is one directory per request and not two. A request's staged inputs and outputs are named inside
   # `$HOME` rather than in a scratch directory of their own, because the two had the same lifetime and the

data/lib/hot_cell/supervisor.rb

@@ -191,6 +191,7 @@ module HotCell
       verify_scratches!
       prepare_directories
       preload
+      warm_up
       @work = listen "work.sock"
       @control = listen "control.sock"
       @control_handler = Control.new(configuration: configuration, counters: counters)
@@ -202,6 +203,8 @@ module HotCell
     end
 
     def run
+      prefork
+
       until stopped?
         readable, = IO.select(sources, nil, nil, wait_for)
         Array(readable).each { |source| handle source }
@@ -278,7 +281,7 @@ module HotCell
       # `Codes` already relies on when it refuses to attribute a signal to the input a worker was holding.
       # The premise is that handling TERM is not optional — it is how an orchestrator stops a cell without
       # killing requests in flight — and that denial of service against a cell is out of scope per
-      # docs/DESIGN.md.
+      # docs/design/threat-model.md.
       def trap_signals
         @signals, @signal_writer = IO.pipe
         trap("CHLD") { @signal_writer.write_nonblock "C", exception: false }
@@ -463,8 +466,20 @@ module HotCell
         @children.each_value.find(&:available?) || spawn
       end
 
+      # Forks a worker into each free slot, so that a request does not wait for `fork`. If a fork fails,
+      # `spawn` logs it and this stops. The next request forks its own worker.
+      def prefork
+        return if @stopping
+
+        while free_slot
+          break unless spawn
+        end
+      end
+
       def spawn
-        number = free_slot or return nil
+        number = free_slot
+        return nil if number.nil?
+
         slot = Slot.build(workspace, number)
         supervisor_side, worker_side = UNIXSocket.pair(:STREAM)
         stderr_reader, stderr_writer = IO.pipe
@@ -512,9 +527,9 @@ module HotCell
         # capability and so survives `cap-drop ALL`, and neither the deadline kill nor the reap sweep reaches
         # it afterwards. It then runs untimed until `pids-limit` or the cgroup ends it.
         #
-        # It reaches further than that, so this is not the denial of service docs/DESIGN.md puts out of
+        # It reaches further than that, so this is not the denial of service docs/design/threat-model.md puts out of
         # scope. A stolen listener needs a live process to hold it, and this escape is the only way one
-        # outlives the reap — so it is what turns the socket theft under "Worker isolation" from a dead
+        # outlives the reap — so it is what turns the socket theft in docs/design/worker-isolation.md from a dead
         # socket path into a cell that intercepts every later request.
         #
         # The premise is that nothing in this process prevents it. A process group is the only bound the
@@ -531,7 +546,7 @@ module HotCell
         # for it: a warning written from inside libvips is a `write(2)` in a C call Ruby cannot interrupt, so
         # the conversion would wait on the supervisor's scheduling — nearest exactly when the host is under
         # pressure and the supervisor is scheduled least. That trade is refused; the conversion path must
-        # never wait on the supervisor. What non-blocking loses instead is in docs/LOGS.md, and a test pins
+        # never wait on the supervisor. What non-blocking loses instead is in docs/observability.md, and a test pins
         # the flag so that a well-meaning fix has to argue with it.
         stderr_reader.close
         stderr_writer.fcntl Fcntl::F_SETFL, stderr_writer.fcntl(Fcntl::F_GETFL) | Fcntl::O_NONBLOCK
@@ -857,18 +872,23 @@ module HotCell
       # the sweep must not signal it. An empty group is the common case, so ESRCH is expected. It is safe to
       # kill the group by the leader's pid even though the leader is reaped, because the supervisor is single
       # threaded and mints group leaders only in `spawn`, which cannot run between the `wait2` above and here.
+      #
+      # macOS answers EPERM rather than ESRCH for a group whose only members are zombies, and a tool killed
+      # with its worker stays one until launchd reaps it. Raised, it unwound `run` and ended the cell before
+      # this reap answered the caller. EPERM means no member received the signal, the same outcome as ESRCH.
       def sweep_group(child)
         Process.kill :KILL, -child.pid
-      rescue Errno::ESRCH
+      rescue Errno::ESRCH, Errno::EPERM
         nil
       end
 
       # The whole process group, which is the worker and everything it started. Negative pid is the group.
       # Falls back to the worker alone if the group is already gone, so a worker that died between the check
-      # and the signal is not an error.
+      # and the signal is not an error. A group answering EPERM falls back as well, for the reason
+      # `sweep_group` gives; the bare pid does not, because macOS signals a zombie pid without complaint.
       def kill_group(child)
         Process.kill :KILL, -child.pid
-      rescue Errno::ESRCH
+      rescue Errno::ESRCH, Errno::EPERM
         begin
           Process.kill :KILL, child.pid
         rescue Errno::ESRCH
@@ -936,6 +956,12 @@ module HotCell
 
           log.write "worker.reaped", pid: pid, slot: child.slot.number, served: child.served,
                                      signal: signal_name(status), exit_code: status.exitstatus
+
+          # A worker that exited without serving a request is not replaced. This guards against a worker
+          # that crashes during its startup, for example after a bad deploy. A replacement would crash too,
+          # and the supervisor would fork again, with no requests arriving. Instead, the next request forks
+          # its own worker.
+          prefork if child.served.positive?
         end
       rescue Errno::ECHILD
         nil
@@ -1237,6 +1263,13 @@ module HotCell
         Registry.operations.each { |operation| operation.before_fork.each(&:call) }
       end
 
+      # Runs after `preload`, so that the objects the operations loaded are compacted too. Runs before the
+      # first fork, so that each worker inherits the compacted heap. The compacted objects use fewer pages,
+      # so a worker copies fewer pages when it writes to them.
+      def warm_up
+        Process.warmup
+      end
+
 
       def shutdown
         refuse_queue "the cell is stopping"

data/lib/hot_cell/test_cell.rb

@@ -26,6 +26,9 @@ module HotCell
   # which is the only place such a library may be loaded.
   #
   #   HotCell::TestCell.boot(operations: -> { require "active_storage/hot_cell/server" })
+  #
+  # The cell's sockets go beneath `Dir.tmpdir`, and a Unix socket path holds 104 bytes on macOS, whose per-user
+  # `TMPDIR` spends about half of that. A suite on macOS should set a short `TMPDIR`, such as `/tmp`.
   class TestCell
     READY = "up"
 

data/lib/hot_cell/test_operations.rb

@@ -70,6 +70,16 @@ module HotCell
       end
     end
 
+    # A worker inherits the supervisor's GC counters at the fork. So this reports what the supervisor did to
+    # its heap before the fork.
+    class HeapState < HotCell::Operation
+      operation "test.heap_state"
+
+      def perform(_inputs, _outputs)
+        { compactions: GC.stat(:compact_count) }
+      end
+    end
+
     # Leaves its $HOME in a state the worker cannot remove, by taking write permission off a subdirectory
     # that still has a file in it. A tool running as this user can do the same to a sibling's directory,
     # which is why a removal that fails has to be reported rather than swallowed.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hotcell-server
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Mike Dalessio
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 1.0.0
 description: |
   Runs a HotCell container. A supervisor listens on two Unix sockets, forks a worker for each request,
   and enforces a wall clock deadline and resource limits on it. Write the work as a subclass of
@@ -43,6 +43,7 @@ files:
 - lib/hot_cell/control.rb
 - lib/hot_cell/counters.rb
 - lib/hot_cell/filesystem.rb
+- lib/hot_cell/health_operations.rb
 - lib/hot_cell/limits.rb
 - lib/hot_cell/log.rb
 - lib/hot_cell/operation.rb
@@ -63,8 +64,9 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/basecamp/hotcell
-  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.5.0/hotcell-server
-  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.5.0/CHANGELOG.md
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v1.0.0/hotcell-server
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v1.0.0/CHANGELOG.md
+  documentation_uri: https://basecamp.github.io/hotcell/
   bug_tracker_uri: https://github.com/basecamp/hotcell/issues
   rubygems_mfa_required: 'true'
 rdoc_options: []
@@ -81,7 +83,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.6
 specification_version: 4
 summary: 'Run a HotCell: the supervisor, the worker, and the operation API.'
 test_files: []