hotcell-server 0.0.0 → 0.2.0

data/lib/hot_cell/test_cell.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "hot_cell/server"
+require "fileutils"
+require "tmpdir"
+
+module HotCell
+  # A real cell in a real process, for anybody's test suite.
+  #
+  # This ships here rather than being written again by each consumer, because every consumer otherwise
+  # writes its own stub cell and they drift. It is not a stub: it forks, it passes descriptors, it applies
+  # limits, and it reaps — none of which an in-process double would exercise, and all of which is where the
+  # interesting failures are.
+  #
+  # The operations it carries are whatever the calling process has defined, because a worker inherits the
+  # registry through the fork, exactly as a real cell does at boot.
+  #
+  #   HotCell::TestCell.boot(concurrency: 2) do |cell|
+  #     HotCell.root = File.dirname(cell.directory)
+  #     ...
+  #   end
+  #
+  # Some operations cannot be loaded in a test process at all, and libvips is the reason this matters: its
+  # thread pool does not survive a fork, so a suite that required it before booting a cell would make every
+  # worker deadlock. Pass `operations:` a callable and it runs inside the cell's own process, before it boots —
+  # which is the only place such a library may be loaded.
+  #
+  #   HotCell::TestCell.boot(operations: -> { require "active_storage/hot_cell/server" })
+  class TestCell
+    READY = "up"
+
+    attr_reader :name, :directory, :workspace, :log_path
+
+    def self.boot(**options)
+      new(**options).start.tap do |cell|
+        return cell unless block_given?
+
+        begin
+          yield cell
+        ensure
+          cell.stop
+          cell.cleanup
+        end
+      end
+    end
+
+    # Anything in `supervisor:` goes to Supervisor.new; everything else is the cell's own limits.
+    def initialize(name: "test", supervisor: {}, operations: nil, **options)
+      @name = name
+      @supervisor_options = supervisor
+      @operations = operations
+      @options = options
+      @root = Dir.mktmpdir "hotcell-test"
+      @directory = File.join(@root, name)
+      @workspace = File.join(@root, "workspace")
+      @log_path = File.join(@root, "cell.log")
+    end
+
+    # The parent of the cell's own directory, which is what a client registers as HotCell.root.
+    def socket_root
+      @root
+    end
+
+    # Writes a byte down a pipe once it is listening, so nothing here waits on a sleep.
+    def start
+      reader, writer = IO.pipe
+
+      @pid = fork do
+        reader.close
+
+        # The cell must not hold the test runner's stdout. A cell that outlives its test would otherwise keep
+        # the runner's pipe open, and a failing assertion becomes a hang rather than a failure.
+        $stdout.reopen log_path, "a"
+        $stderr.reopen log_path, "a"
+
+        HotCell.limits(**@options) unless @options.empty?
+
+        supervisor = Supervisor.new(directory: directory, workspace: workspace,
+                                    log: Log.new(File.open(log_path, "w")), **@supervisor_options)
+        begin
+          @operations&.call
+          supervisor.boot
+          writer.write READY
+          writer.close
+          supervisor.run
+        # StandardError is enough. What must not happen is an exception escaping this block, because Ruby would
+        # then run at_exit in the child — including minitest's autorun, which starts the whole suite over inside
+        # a forked cell. The `ensure exit!` below is what prevents that, for anything raised. This rescue only
+        # writes the diagnostic the parent reads.
+        rescue StandardError => error
+          File.write log_path, "#{error.class}: #{error.message}\n" \
+                               "#{error.backtrace&.first(10)&.join("\n")}\n", mode: "a"
+        ensure
+          exit! 0
+        end
+      end
+
+      writer.close
+      ready = reader.read(READY.bytesize)
+      reader.close
+      raise "the cell did not boot: #{log}" unless ready == READY
+
+      self
+    end
+
+    # Terminates the cell and leaves its files, so a test can read the log once everything that was going to
+    # write to it has exited.
+    def stop
+      return if @pid.nil?
+
+      begin
+        Process.kill :TERM, @pid
+        wait_for_exit
+      rescue Errno::ESRCH, Errno::ECHILD
+        nil
+      end
+
+      @pid = nil
+    end
+
+    def cleanup
+      FileUtils.remove_entry @root if Dir.exist?(@root)
+    end
+
+    def log
+      File.exist?(log_path) ? File.read(log_path) : ""
+    end
+
+    def log_events(event)
+      log.lines.filter_map do |line|
+        parsed = begin
+          JSON.parse line, symbolize_names: true
+        rescue JSON::ParserError
+          nil
+        end
+
+        parsed if parsed.is_a?(Hash) && parsed[:event].is_a?(Hash) && parsed[:event][:action] == event
+      end
+    end
+
+    private
+      def wait_for_exit(within: 5)
+        deadline = Clock.now + within
+
+        loop do
+          return if Process.wait(@pid, Process::WNOHANG)
+          break if Clock.now > deadline
+
+          sleep 0.01
+        end
+
+        Process.kill :KILL, @pid
+        Process.wait @pid
+      end
+  end
+end

data/lib/hot_cell/test_operations.rb

@@ -0,0 +1,465 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+
+# Fixture operations, so the whole surface can be exercised in milliseconds, with no tool installed and no
+# container running.
+#
+# They ship rather than sitting in this gem's own test directory, for the reason hot_cell/test_cell.rb does:
+# hotcell-client boots a real cell and needs an inventory to point it at. It had written its own, and five of
+# those answered to routing names these already claim — so `test.uppercase` meant one thing when the client
+# suite proved it and another when this one did.
+#
+# Namespaced because this is a shipped file and `Fixtures` at the top level belongs to the application. Both
+# suites alias it in their own helper.
+module HotCell
+  module Fixtures
+    class Uppercase < HotCell::Operation
+      operation "test.uppercase"
+
+      def perform(inputs, outputs)
+        source, = inputs
+        destination, = outputs
+        File.binwrite destination.path, File.binread(source.path).upcase
+
+        { bytes: File.size(destination.path) }
+      end
+    end
+
+    # Two inputs, one output, so the request shape with several inputs is covered.
+    class Concatenate < HotCell::Operation
+      operation "test.concatenate"
+
+      def perform(inputs, outputs, separator: "")
+        destination, = outputs
+        File.binwrite destination.path, inputs.map { |input| File.binread(input.path) }.join(separator.to_s)
+
+        { inputs: inputs.size }
+      end
+    end
+
+    # Analysis: metadata out and no bytes, which is the shape with no outputs at all.
+    class Measure < HotCell::Operation
+      operation "test.measure"
+
+      def perform(inputs, _outputs, asked_for: nil)
+        source, = inputs
+
+        { bytes: File.size(source.path), digest: Digest::SHA256.file(source.path).hexdigest[0, 8],
+          asked_for: asked_for }
+      end
+    end
+
+    # No inputs and no outputs, like rendering an initials avatar from nothing but a payload.
+    class Echo < HotCell::Operation
+      operation "test.echo"
+
+      def perform(_inputs, _outputs, **payload)
+        { echoed: payload }
+      end
+    end
+
+    class WhoAmI < HotCell::Operation
+      operation "test.whoami"
+
+      def perform(_inputs, _outputs)
+        { pid: Process.pid, home: ENV["HOME"] }
+      end
+    end
+
+    # Leaves its $HOME in a state the worker cannot remove, by taking write permission off a subdirectory
+    # that still has a file in it. A tool running as this user can do the same to a sibling's directory,
+    # which is why a removal that fails has to be reported rather than swallowed.
+    class UnremovableHome < HotCell::Operation
+      operation "test.unremovable_home"
+
+      def perform(_inputs, _outputs)
+        blocked = File.join(ENV["HOME"].to_s, "blocked")
+        FileUtils.mkdir_p blocked
+        File.write File.join(blocked, "file"), "x"
+        File.chmod 0o500, blocked
+
+        { blocked: blocked }
+      end
+    end
+
+    # Leaves a file in $HOME and says whether an earlier request already left one. A tool reads its
+    # configuration from $HOME and a configuration file is executable — ImageMagick runs the command lines
+    # in delegates.xml and applies the rights in policy.xml — so a home that outlives its request lets one
+    # compromised conversion reconfigure every later one on that slot. See adr/0003.
+    class HomeMarker < HotCell::Operation
+      operation "test.home_marker"
+
+      def perform(_inputs, _outputs)
+        found = File.exist?(marker)
+        File.write marker, "planted"
+
+        { found: found, home: ENV["HOME"] }
+      end
+
+      private
+        def marker
+          File.join ENV["HOME"].to_s, "planted-by-an-earlier-request"
+        end
+    end
+
+    # An operation that reads what a caller gave it without a copy onto scratch: it never asks for a path,
+    # which is what an operation reading only a container header wants rather than a multi-gigabyte copy.
+    class Reverse < HotCell::Operation
+      operation "test.reverse"
+
+      def perform(inputs, outputs)
+        outputs.first.to_io.write inputs.first.to_io.read.reverse
+
+        { copied: inputs.first.staged? }
+      end
+    end
+
+    class Broken < HotCell::Operation
+      operation "test.broken"
+
+      def perform(_inputs, _outputs)
+        raise "the operation itself is broken"
+      end
+    end
+
+    class Undecodable < HotCell::Operation
+      operation "test.undecodable"
+
+      def perform(_inputs, _outputs)
+        raise HotCell::UnreadableInput, "not an image at all"
+      end
+    end
+
+    # A library exception an operation declares as meaning "the input could not be decoded", the way the
+    # Active Storage operations will declare Vips::Error.
+    class LibraryError < StandardError; end
+
+    class DeclaredUnreadable < HotCell::Operation
+      operation "test.declared_unreadable"
+      unreadable LibraryError
+
+      def perform(_inputs, _outputs)
+        raise LibraryError, "the library says no"
+      end
+    end
+
+    class Hungry < HotCell::Operation
+      operation "test.hungry"
+
+      def perform(_inputs, _outputs)
+        raise HotCell::MemoryExhausted, "out of memory -- size == 732MB"
+      end
+    end
+
+    # A tool spawn that fails because the host is out of memory. fork raises Errno::ENOMEM under host
+    # pressure, before the tool has read a byte of the input, so this is the cell having a bad moment
+    # rather than a decompression bomb — and it must not be recorded as a permanent memory verdict.
+    class StarvedSpawn < HotCell::Operation
+      operation "test.starved_spawn"
+
+      def perform(_inputs, _outputs)
+        raise Errno::ENOMEM, "Cannot allocate memory - fork(2)"
+      end
+    end
+
+    class SignalsSibling < HotCell::Operation
+      operation "test.signals_sibling"
+
+      # Workers share a uid and a pid namespace, so one finds another by looking for a process the
+      # supervisor also fathered. This is the reproducer for the forged verdict: nothing here touches the
+      # victim's input, and the victim is holding an unrelated one.
+      def perform(_inputs, _outputs, signal:)
+        sibling = siblings.first
+        Process.kill signal, sibling if sibling
+
+        { signalled: sibling }
+      end
+
+      private
+        # Two ways to ask the same question, because the suite runs on macOS as well and only one of them
+        # has /proc. An attacker inside a cell has whichever the image gives it; the point of the reproducer
+        # is that the answer is obtainable at all.
+        def siblings
+          pids = Dir.exist?("/proc") ? procfs_children : ps_children
+
+          pids.reject { |pid| pid == Process.pid }
+        end
+
+        def procfs_children
+          Dir.glob("/proc/[0-9]*").filter_map do |path|
+            status = File.read(File.join(path, "status"))
+            File.basename(path).to_i if status[/^PPid:\s+(\d+)/, 1].to_i == Process.ppid
+          rescue SystemCallError
+            nil
+          end
+        end
+
+        def ps_children
+          `ps -A -o pid=,ppid=`.lines.filter_map do |line|
+            pid, ppid = line.split.map(&:to_i)
+            pid if ppid == Process.ppid
+          end
+        end
+    end
+
+    # Spawns a process and does not wait for it, the way a worker that crashed mid-request would leave a
+    # tool behind. The spawned process inherits the worker's process group, so the supervisor's group sweep
+    # at reap is what must kill it — nothing else is watching it, and it has no deadline. Returns the pid so
+    # a test can watch for it.
+    class Orphaner < HotCell::Operation
+      operation "test.orphaner"
+
+      def perform(_inputs, _outputs)
+        { spawned: Process.spawn("sleep", "300") }
+      end
+    end
+
+    class BadResult < HotCell::Operation
+      operation "test.bad_result"
+
+      def perform(_inputs, _outputs)
+        "a String is not a result"
+      end
+    end
+
+    class UnserializableResult < HotCell::Operation
+      operation "test.unserializable_result"
+
+      def perform(_inputs, _outputs)
+        { format: :png }
+      end
+    end
+
+    # Returns without writing anything, which is how a full tmpfs arrives too.
+    class Silent < HotCell::Operation
+      operation "test.silent"
+
+      def perform(_inputs, _outputs)
+        {}
+      end
+    end
+
+    class Overflowing < HotCell::Operation
+      operation "test.overflowing"
+
+      def perform(_inputs, outputs, megabytes:)
+        File.open(outputs.first.path, "wb") do |file|
+          megabytes.times { file.write "x" * (1024 * 1024) }
+          file.flush
+        end
+
+        {}
+      end
+    end
+
+    # Pins the worker where Ruby cannot interrupt it.
+    #
+    # A deadline test built on sleep passes against a self-enforcing implementation that could never work in
+    # production, because Timeout raises at an interrupt checkpoint and a thread inside a C extension does
+    # not reach one until it returns. libvips is the real case. Deferring the raise reproduces that on every
+    # Ruby and every build, which a long computation cannot: Integer#** takes seven seconds without GMP and
+    # a fraction of a second with it. SIGKILL is not deferrable, so the supervisor still gets through.
+    class Uninterruptible < HotCell::Operation
+      operation "test.uninterruptible"
+      SECONDS = 60
+
+      def perform(_inputs, _outputs)
+        Thread.handle_interrupt(Exception => :never) { sleep SECONDS }
+        {}
+      end
+    end
+
+    # Starts a grandchild that outlives the worker, and reports its pid so a test can ask whether the
+    # deadline reached it. `spawn` rather than a tool, so this needs no toolchain installed.
+    class Spawns < HotCell::Operation
+      operation "test.spawns"
+
+      def perform(_inputs, _outputs)
+        pid = spawn("sleep", "60")
+        tell_and_wait pid
+      end
+
+      private
+        def tell_and_wait(pid)
+          File.write ENV.fetch("HOTCELL_SPAWNED_PID_PATH"), pid.to_s
+          sleep 60
+          { pid: pid }
+        end
+    end
+
+    # Declares less than the cell allows, so the supervisor has to learn the narrower number from the worker
+    # rather than from the request it never reads.
+    class Impatient < Uninterruptible
+      operation "test.impatient"
+      limits deadline: 1
+    end
+
+    # Declares more than the cell allows, which the cell clamps. Invariant 6.
+    class Patient < Uninterruptible
+      operation "test.patient"
+      limits deadline: 300
+    end
+
+    # Hands an exec'd tool the input descriptor and has it read the bytes back through /dev/fd, so a test
+    # can prove run_tool's `pass:` exposes a descriptor to a tool without staging it. Reports whether the
+    # input was copied onto scratch, which must be false.
+    class ReadThroughFd < HotCell::Operation
+      operation "test.read_through_fd"
+
+      def perform(inputs, _outputs)
+        source, = inputs
+        result = run_tool "cat", source.fd_path, pass: [ source.to_io ]
+
+        { content: result.out, ok: result.ok?, staged: source.staged? }
+      end
+    end
+
+    # Reports what an exec'd tool can see of its environment, and what the worker itself can see, so a
+    # test can prove the canary was really there before proving the tool never got it.
+    class Environment < HotCell::Operation
+      operation "test.environment"
+
+      def perform(_inputs, _outputs, env: {}, canary: nil)
+        result = run_tool "env", env: env
+
+        { seen: result.out.lines.map(&:chomp).sort, worker_saw: ENV[canary.to_s],
+          ok: result.ok? }
+      end
+    end
+
+    # Prints far more than any sane capture, on the stream the payload names, then exits. For proving that
+    # a noisy tool costs a bounded amount of this worker's memory rather than all of it.
+    class Noisy < HotCell::Operation
+      operation "test.noisy"
+
+      def perform(_inputs, _outputs, stream: "out")
+        stream = stream == "err" ? "STDERR" : "STDOUT"
+        script = "40.times { #{stream}.write('x' * 1_000_000) }"
+        result = run_tool "ruby", "-e", script, capture: 1024
+
+        { out: result.out.bytesize, err: result.err.bytesize, ok: result.ok? }
+      end
+    end
+
+    # Two operations that configure the same global, so a worker serving A, B, A can be asked what the
+    # global says at the moment each one ran.
+    CONFIGURED = []
+
+    class ConfiguresGlobal < HotCell::Operation
+      abstract_operation
+
+      def perform(_inputs, _outputs)
+        { configured_for: CONFIGURED.last }
+      end
+    end
+
+    class ConfiguresAlpha < ConfiguresGlobal
+      operation "test.configures_alpha"
+      before_worker_boot { CONFIGURED << "alpha" }
+    end
+
+    class ConfiguresBeta < ConfiguresGlobal
+      operation "test.configures_beta"
+      before_worker_boot { CONFIGURED << "beta" }
+    end
+
+    # Writes the first output and leaves the second alone, which is a positive total and reads as success to
+    # anything checking the aggregate.
+    class HalfWritten < HotCell::Operation
+      operation "test.half_written"
+
+      def perform(_inputs, outputs)
+        File.binwrite outputs.first.path, "only the first"
+
+        { wrote: 1 }
+      end
+    end
+
+    class Rlimits < HotCell::Operation
+      operation "test.rlimits"
+
+      def perform(_inputs, _outputs)
+        { memory: Process.getrlimit(Process::RLIMIT_DATA), file_size: Process.getrlimit(Process::RLIMIT_FSIZE),
+          open_files: Process.getrlimit(Process::RLIMIT_NOFILE), core: Process.getrlimit(Process::RLIMIT_CORE) }
+      end
+    end
+
+    # Asks for less than the cell allows, so the soft limit narrows and the hard limit does not.
+    class Frugal < Rlimits
+      operation "test.frugal"
+      limits memory: 1024 * 1024**2, file_size: 4 * 1024 * 1024, open_files: 64
+    end
+
+    # Asks for more than any cell will allow, on every limit. Unclamped, the worker would try to set a soft limit
+    # above its own hard limit and die before it could answer.
+    class Extravagant < Rlimits
+      operation "test.extravagant"
+      limits memory: 8 * 1024**3, file_size: 512 * 1024**2, open_files: 4096, deadline: 3600
+    end
+
+    class Greedy < HotCell::Operation
+      operation "test.greedy"
+
+      def perform(_inputs, _outputs, megabytes:)
+        { bytes: ("x" * (megabytes * 1024 * 1024)).bytesize }
+      end
+    end
+
+    # A result carrying bytes a tool produced, which is where invalid UTF-8 comes from in practice.
+    class Mojibake < HotCell::Operation
+      operation "test.mojibake"
+
+      def perform(_inputs, _outputs)
+        { filename: "caf\xFF.jpg".dup.force_encoding(Encoding::UTF_8) }
+      end
+    end
+
+    # Dies mid-request without answering and without a signal, which is what a cell fault looks like as
+    # distinct from an input fault.
+    class Vanishes < HotCell::Operation
+      operation "test.vanishes"
+
+      def perform(_inputs, _outputs)
+        exit! 3
+      end
+    end
+
+    # Reports itself idle while its request is still running, so the report and the truth disagree from
+    # that moment on. The control socket is private to the Worker, but it lives in the operation's own
+    # process, so reaching it takes one ObjectSpace walk. With `exit_after`, the worker then exits without
+    # ever reading its control socket again, so whatever the supervisor wrote there in the meantime is
+    # queued and unread when it goes.
+    class EarlyIdle < HotCell::Operation
+      operation "test.early_idle"
+
+      def perform(_inputs, _outputs, pid_path:, exit_after: nil)
+        # The one with an open control socket, not `.first`: a suite that builds a Worker in its own
+        # process leaves it on the heap for the fork to inherit, with its sockets closed by teardown.
+        worker = ObjectSpace.each_object(HotCell::Worker).find do |candidate|
+          !candidate.instance_variable_get(:@control).socket.closed?
+        end
+        worker.instance_variable_get(:@control).write_line JSON.generate(idle: true, code: "ok") << "\n"
+        File.write pid_path, Process.pid.to_s
+
+        if exit_after
+          sleep exit_after
+          exit! 0
+        else
+          sleep 300
+        end
+      end
+    end
+
+    class Blocking < HotCell::Operation
+      operation "test.blocking"
+
+      def perform(_inputs, _outputs, seconds:)
+        sleep seconds
+
+        { slept: seconds, pid: Process.pid }
+      end
+    end
+  end
+end

data/lib/hot_cell/timing.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module HotCell
+  # One request's timing ledger, carried through the worker instead of the two loose values it replaces.
+  #
+  # It exists for the base instant rather than for tidiness. `perform_ms` means time spent performing where
+  # performing happened, and time since the request arrived where it did not — a request refused for a
+  # protocol mismatch never performed anything, so measuring it from the start is the only meaningful number.
+  # That distinction used to live in the argument every caller passed: `refuse`'s `since` was `started` on
+  # three paths and `began` on a fourth, and getting it wrong would have been invisible. The base moves once,
+  # here, when `performing` is called.
+  #
+  # Phases accumulate as they complete, so a refusal reports the ones that finished before the failure. An
+  # `unreadable` verdict that arrives with no `operation_ms` says exactly where it got to.
+  class Timing
+    attr_reader :queued_ms, :started
+
+    def initialize(queued_ms)
+      @queued_ms = queued_ms
+      @started = Clock.now
+      @base = @started
+      @phases = {}
+    end
+
+    # Performing starts now, so this is what perform_ms measures from.
+    def performing
+      @base = Clock.now
+    end
+
+    def measure(phase)
+      at = Clock.now
+      result = yield
+      @phases[phase] = Clock.ms_since(at)
+      result
+    end
+
+    def to_h
+      { queued_ms: queued_ms, **@phases, perform_ms: Clock.ms_since(@base) }
+    end
+
+    # The whole request as the cell saw it, including reading the message. For the log line rather
+    # than for the caller, who is told what performing cost.
+    def elapsed_ms
+      Clock.ms_since started
+    end
+  end
+end