hotcell-server 0.0.0 → 0.2.0

data/lib/hot_cell/operation.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+module HotCell
+  # Every operation takes inputs, outputs, and the payload — and the payload arrives as keyword
+  # arguments. There is no argument schema and no generated code, but an operation may declare the
+  # specific keywords it wants (`format:, operations: {}`), and Ruby validates them on arrival; one that
+  # wants the payload as a plain Hash declares `**payload` as its third argument, and one that takes no
+  # options declares neither. A mismatched call raises inside perform and reports `failed`, and the
+  # blast radius of that is one worker with no network.
+  #
+  # This is the one place application code deliberately runs inside a cell. The invariant is not "no
+  # application code": it is that the code running there has no credentials, no database, no network, and
+  # no application configuration.
+  class Operation
+    READ_BYTES = 16 * 1024
+
+    class << self
+      include Declarations
+
+      def inherited(subclass)
+        super
+        Registry.register subclass
+      end
+
+      # Declares a class that exists to be inherited from, not to be dispatched to.
+      #
+      # Every subclass registers, because that is what makes an operation reachable by writing it. An
+      # intermediate that gathers shared setup registers too, and a cell then advertises it in `describe` and
+      # accepts it on the wire — where it reaches a `perform` that raises NotImplementedError and answers
+      # `failed`, as though the caller's document were the problem.
+      #
+      # Deliberately not inherited: a subclass of an abstract operation is concrete unless it says otherwise,
+      # and a class-level instance variable is not visible to a subclass, so that falls out for free.
+      def abstract_operation
+        @abstract_operation = true
+        Registry.reload!
+      end
+
+      def abstract_operation?
+        @abstract_operation == true
+      end
+
+      # Writes the routing name, defaulting to the underscored class name with namespaces as dots.
+      def operation(name = nil)
+        return operation_name if name.nil?
+
+        @operation_name = name.to_s
+        Registry.reload!
+        @operation_name
+      end
+
+      def operation_name
+        @operation_name || derived_operation_name
+      end
+
+      # A class-level declaration that accumulates. Naming one limit changes one and keeps the rest — of this
+      # class's own declaration, or of the nearest ancestor's when this class has none yet. That is what
+      # lets a subclass narrow a single number, and what lets an operator give a shipped operation a
+      # different budget from an operations file, after the operation loads, without editing the gem. The
+      # cell's own limits still clamp whatever is declared. docs/DEPLOYMENT.md, "Changing a shipped
+      # operation's limits".
+      def limits(**values)
+        return inherited_value(:@limits) || Limits.new if values.empty?
+
+        @limits = limits.merge(**values)
+      end
+
+      # Runs in the supervisor, once, at boot. It may require and it may configure, and it must never
+      # evaluate an image.
+      #
+      # That rule is what the whole process model rests on, and breaking it is a silent hang rather than a
+      # crash. Measured: a parent that has only required image_processing/vips and set concurrency has
+      # three threads and forks children that work. One additional 1x1 evaluation takes it to five, and
+      # from then on every forked worker blocks forever in futex_do_wait, because the GLib thread pool
+      # does not survive fork and the child waits on a pool with no threads. Not the first worker — every
+      # worker. A later change that pre-warms the pool to save the fork cost is exactly what this forbids.
+      #
+      # Note that a hotcell forks per request, continuously, so "before the fork" and "at boot" are not
+      # the same moment the way they are in Puma. This runs once.
+      def before_fork(&block)
+        return collected(:@before_fork) if block.nil?
+
+        (@before_fork ||= []) << block
+      end
+
+      # Runs in the worker after the fork and before it serves anything. This is where an operation sizes
+      # its library, and the framework deliberately does not do it for anyone: Vips.concurrency_set 4 in a
+      # cell given two CPUs running twenty workers is forty threads on two cores, and getting that right
+      # is the operation author's problem against the cell's own numbers.
+      #
+      # Configuration belongs here rather than in before_fork because it is global and singular. Two
+      # operations configuring the same library in the supervisor would silently disagree, with the last one
+      # registered winning.
+      #
+      # A worker is not one operation, which is what makes this the right place rather than a safe one. Above
+      # `max_requests_per_worker: 1` it can serve A, then B, then A, and these hooks re-run whenever the operation
+      # changes — so what the library is set up for always matches what is about to run. Write them to be
+      # re-entrant: they are setters against shared state, not one-time initialization.
+      def before_worker_boot(&block)
+        return collected(:@before_worker_boot) if block.nil?
+
+        (@before_worker_boot ||= []) << block
+      end
+
+      # Library exceptions that mean the input could not be decoded, rather than that the operation broke.
+      def unreadable(*classes)
+        return collected(:@unreadable) + [ UnreadableInput ] if classes.empty?
+
+        (@unreadable ||= []).concat classes
+      end
+
+      private
+        def derived_operation_name
+          Naming.default_operation_name self
+        end
+
+        # Superclass first, so a base class's hooks run before the ones that specialize it.
+        def collected(variable)
+          ancestors.grep(Class).reverse.flat_map { |ancestor| ancestor.instance_variable_get(variable) || [] }
+        end
+    end
+
+    # A fresh instance per request, so that nothing an operation puts in an instance variable survives
+    # into the next request a reused worker serves.
+    def perform(inputs, outputs, **)
+      raise NotImplementedError, "#{self.class} must implement perform(inputs, outputs, <keywords>)"
+    end
+
+    ToolResult = Struct.new(:status, :out, :err) do
+      def ok?
+        status.success?
+      end
+    end
+
+    # Runs a tool with `unsetenv_others` and a fully written environment, never a filtered copy of this
+    # worker's own. This is invariant 9, and it is the only point in the whole design where we control what a
+    # tool's /proc/<pid>/environ shows.
+    #
+    # Filtering would not work. The worker is forked, so its own /proc/self/environ is the exec-time
+    # environment of the process it was forked from, and ENV.delete changes nothing that a sibling worker can
+    # read. An exec'd child is different: it gets a fresh environ, and this is where that gets written.
+    #
+    # Bounded output, because a tool's stdout is attacker-influenced — and worth remembering when an
+    # operation parses it, because doing so brings the bytes a tool was isolating back into this worker.
+    #
+    # `capture` bounds what is kept AND what is read, which capture3 could not do. It accumulates both
+    # streams in full and hands them over at exit, so slicing afterwards bounded the Strings this method
+    # returns and nothing else: an input that makes a tool print gigabytes of diagnostics had already
+    # cost gigabytes of this worker's address space, and took RLIMIT_DATA with it — arriving as a `memory`
+    # verdict, which is permanent, for a document whose only crime was being noisy.
+    # `pass` hands the tool a set of the worker's own descriptors — an input to read, an output to write —
+    # at their existing fd numbers, so the tool reaches them as `/dev/fd/N` (Descriptor#fd_path) and no
+    # byte is copied onto scratch to give it a filename. A fd handed to a child this way loses its
+    # close-on-exec, which is exactly the inheritance wanted, and only for these; the worker's other
+    # descriptors are untouched. Passing an fd at its own number cannot collide with the stdio pipes popen3
+    # installs on 0, 1 and 2, because a received descriptor is never one of those.
+    def run_tool(*command, env: {}, capture: 64 * 1024, pass: [])
+      require "open3"
+
+      inherit = pass.to_h { |io| [ io.fileno, io.fileno ] }
+
+      Open3.popen3(tool_environment(env), *command, unsetenv_others: true, **inherit) do |stdin, out, err, thread|
+        stdin.close
+        captured = drain(out, err, capture)
+
+        ToolResult.new thread.value, captured[out], captured[err]
+      end
+    end
+
+    private
+      # Reads both streams until they close, keeping only the first `limit` bytes of each and discarding the
+      # rest as it arrives. Both have to be read, not just the one being kept: a tool blocks writing to
+      # a pipe nobody drains, and a tool blocked on stderr never exits, which turns a noisy document
+      # into a deadline kill.
+      def drain(*streams, limit)
+        kept = streams.to_h { |stream| [ stream, +"".b ] }
+        open = streams.dup
+
+        until open.empty?
+          readable, = IO.select(open)
+
+          Array(readable).each do |stream|
+            chunk = stream.read_nonblock(READ_BYTES, exception: false)
+            next if chunk == :wait_readable
+
+            if chunk.nil?
+              open.delete stream
+              next
+            end
+
+            room = limit - kept[stream].bytesize
+            kept[stream] << chunk.byteslice(0, room) if room.positive?
+          end
+        end
+
+        kept
+      end
+
+      def tool_environment(overrides)
+        { "HOME" => ENV["HOME"], "PATH" => ENV["PATH"], "LANG" => "C.UTF-8", "LC_ALL" => "C.UTF-8" }
+          .merge(overrides.transform_keys(&:to_s))
+          .compact
+      end
+  end
+end

data/lib/hot_cell/registry.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module HotCell
+  # The `op` field on the wire is looked up here. A name that arrived on the wire is never used to derive a constant, so the
+  # index only ever holds classes this cell already loaded.
+  module Registry
+    class << self
+      def register(operation)
+        operations << operation unless operations.include?(operation)
+        reload!
+        operation
+      end
+
+      def operations
+        @operations ||= []
+      end
+
+      def names
+        index.keys.sort
+      end
+
+      def lookup(name)
+        index[name]
+      end
+
+      def reload!
+        @index = nil
+      end
+
+      def clear
+        @operations = []
+        reload!
+      end
+
+      private
+        def index
+          @index ||= operations.each_with_object({}) do |operation, names|
+            next if operation.abstract_operation?
+
+            name = operation.operation_name
+
+            if (claimed = names[name])
+              raise ConfigurationError, "#{operation} and #{claimed} both answer to #{name.inspect}"
+            end
+
+            names[name] = operation
+          end
+        end
+    end
+  end
+end

data/lib/hot_cell/server/errors.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module HotCell
+  # What an operation raises to choose its own verdict. An operation may also declare the library
+  # exceptions that mean the same thing, with `unreadable Vips::Error`.
+  class OperationError < StandardError; end
+
+  # The input could not be decoded. Terminal: the same bytes fail the same way on an idle cell. Common
+  # rather than exceptional — it covers truncated uploads, formats the build was not compiled with, and
+  # formats deliberately refused.
+  class UnreadableInput < OperationError; end
+
+  # Reported as `killed` with `cause: memory` rather than as an ordinary failure, because it is the
+  # decompression-bomb case and a caller must be able to act on it without parsing a message.
+  class MemoryExhausted < OperationError; end
+end

data/lib/hot_cell/server/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module HotCell
+  module Server
+    VERSION = "0.2.0"
+  end
+end

data/lib/hot_cell/server.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "hot_cell/core"
+
+require "hot_cell/server/version"
+require "hot_cell/server/errors"
+require "hot_cell/limits"
+require "hot_cell/configuration"
+require "hot_cell/registry"
+require "hot_cell/operation"
+require "hot_cell/slot"
+require "hot_cell/log"
+require "hot_cell/timing"
+require "hot_cell/counters"
+require "hot_cell/control"
+require "hot_cell/worker"
+require "hot_cell/supervisor"
+
+module HotCell
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    # A cell is configured once, and everything about scheduling lives here rather than on an operation.
+    #
+    #   HotCell.limits concurrency: 4, queue_size: 8, deadline: 60, queue_wait: 10, max_requests_per_worker: 1,
+    #                  memory: 1536 * 1024**2, file_size: 48 * 1024**2
+    def limits(**options)
+      return configuration if options.empty?
+
+      @configuration = Configuration.new(**options)
+    end
+
+    # Boots a cell's code: the configuration file first, explicitly, then every operation file in sorted
+    # order. A derived image adds these by copying files in, never by changing this gem, and a cell with
+    # no config.rb takes every default.
+    def load!(config: ENV.fetch("HOTCELL_CONFIG", "/hotcell/config.rb"),
+              operations: ENV.fetch("HOTCELL_OPERATIONS", "/hotcell/operations"))
+      require config if File.exist?(config)
+
+      Dir.glob(File.join(operations, "**", "*.rb")).sort.each { |file| require file }
+    end
+  end
+end

data/lib/hot_cell/slot.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "securerandom"
+
+module HotCell
+  # Slots are a consequence of the concurrency limit rather than something to configure. At most
+  # `concurrency` workers run, so number them and hand each worker its number at fork. There is no
+  # leasing: a slot is always free when a worker starts, because the thing that bounds workers is the same
+  # thing that counts slots. A request never waits for a slot, it waits in the cell's queue.
+  #
+  # **A slot has one directory per request and it is that request's `$HOME`.** It is created when the
+  # request starts and removed when the request ends, so nothing a tool writes under `$HOME` reaches the
+  # next request on this slot.
+  #
+  # This directory used to survive, to give a tool with an expensive per-user profile a warm one. That is
+  # withdrawn. What a tool reads from `$HOME` is configuration, and for the toolchains a cell carries
+  # configuration is executable: ImageMagick runs the command lines in `delegates.xml` and applies the
+  # rights in `policy.xml`, both read from `$HOME/.config/ImageMagick`. A surviving home therefore let an
+  # input that achieved code execution reconfigure every later request on the slot, which is the bound
+  # `max_requests_per_worker: 1` is supposed to hold. adr/0003 records the reversal and adr/0002 the
+  # reasoning it supersedes.
+  #
+  # **The name is fresh for every request, and that is what makes the removal above a guarantee rather than
+  # an intention.** A stable name only holds while the deletion behind it works, and the tool that filled
+  # the directory runs as the user that owns it: `chmod 0500` on a directory it has written makes its own
+  # configuration unremovable, and the same mode on the slot directory makes it unrenameable too. Both
+  # cleanups answer false and both callers log — and a stable name then handed the next request the tree
+  # that had just refused to go. A name no earlier request has held is not a name an earlier request could
+  # have prepared, so what a failed cleanup now costs is disk rather than the isolation the cell is for.
+  #
+  # The mode of the slot directory is reasserted for the same reason. It is the one name here that is
+  # predictable, so it is the one an earlier request can lock, and a mode on a directory this uid owns is
+  # ours to put back.
+  #
+  # **This bounds what an earlier request left behind, and not what a live one is doing.** Workers share a
+  # uid and `0700` is the owner's own mode, so a concurrent sibling — or a `setsid` descendant of a finished
+  # request, which process groups do not contain — can still write into a home the moment it exists. The
+  # slot directory itself can be renamed aside and replaced, and `chmod` follows what it finds. Both are the
+  # residuals `docs/DESIGN.md` records under worker isolation, and neither is closed here.
+  #
+  # There is one directory per request and not two. A request's staged inputs and outputs are named inside
+  # `$HOME` rather than in a scratch directory of their own, because the two had the same lifetime and the
+  # same owner once the home stopped surviving. Staging used to create its directory on demand, which is
+  # what kept a descriptor-only operation from paying for one; `$HOME` has to exist for every request
+  # either way, so that laziness bought nothing and is gone with it.
+  #
+  # The filesystem behaviour belongs here rather than in the two processes that call it. The directory is
+  # removed from both — the worker before it answers, the supervisor at finish and at reap — so the guard
+  # and the swallowed SystemCallError are a rule that has to hold on both sides of a fork, and it had a
+  # copy on each.
+  Slot = Struct.new(:number, :directory, :home) do
+    def self.build(workspace, number)
+      new number, File.join(workspace, number.to_s)
+    end
+
+    # A name no request has held before, so nothing an earlier one did to the tree it was given reaches this
+    # one. The suffix is random rather than a counter for the reason the discarded name's is: the previous
+    # request could write to this directory, and a predictable name is one it can pre-create.
+    def make_home
+      FileUtils.mkdir_p directory, mode: 0o700
+      FileUtils.chmod 0o700, directory
+
+      self.home = File.join(directory, "home-#{SecureRandom.hex(8)}")
+      Dir.mkdir home, 0o700
+      home
+    end
+
+    # **Returns whether the directory is gone, and every caller logs when it is not.**
+    #
+    # A home that was already removed, or that another process removed between the check and the unlink, is
+    # the outcome this wants either way, so that answers true. Failing to remove one is a different fact and
+    # it used to arrive as the same swallowed nil. This is the call that deletes a request's staged input and
+    # output before the caller is told anything, so a failure leaves those bytes on the tmpfs for the life of
+    # the container — and a sibling worker can cause one, by creating entries under the tree while
+    # remove_entry walks it. Same uid, no defence, and it was silent.
+    #
+    # It still cannot raise. Worker#serve calls this from an ensure, where a raise would replace the caller's
+    # response with a crash, and the supervisor calls discard_home from finish and reap, where nothing above
+    # rescues anything and a raise stops the cell with every request it holds.
+    def remove_home
+      return true if home.nil?
+      return false unless remove_tree(home)
+
+      self.home = nil
+      true
+    end
+
+    # **The supervisor renames rather than deletes, and that is a scheduling decision.**
+    #
+    # How long a recursive delete takes is chosen by the operation that filled the directory. Nothing bounds
+    # the number of entries — RLIMIT_FSIZE caps one file, not a million tiny ones — so an input that makes a
+    # tool write an enormous tree and then hang buys a deletion the supervisor performs synchronously,
+    # after the kill, inside the loop enforcing every other request's deadline.
+    #
+    # A rename within one filesystem is O(1) and takes the tree out of the way. A worker sweeps it later,
+    # after it has answered and before it reports itself idle — see Worker#serve, which is the one window
+    # where the unlinking costs nobody's latency.
+    #
+    # The destination carries a random suffix rather than a counter, because the tool that filled the
+    # directory runs as this user and can write to the slot's directory. A predictable name lets it
+    # pre-create a colliding entry, fail the rename, and send the supervisor into the recursive delete the
+    # rename exists to avoid. On any rename failure the tree is left where it is, for a later worker's own
+    # cleanup to remove off the hot path. The supervisor never deletes a tree inline, whatever goes wrong.
+    # It renames what is there rather than the name it is holding, because the supervisor is the caller that
+    # matters and it does not know the name. `home` is assigned in the worker after the fork, so the
+    # supervisor's copy of the slot is still nil when it discards a killed worker's tree. At most one worker
+    # holds a slot at a time, so everything matching `home-*` under it is that worker's and nobody else's.
+    def discard_home
+      FileUtils.chmod 0o700, directory if Dir.exist?(directory)
+
+      Dir.glob(File.join(directory, "home-*")).each do |path|
+        File.rename path, File.join(directory, "discarded-#{Process.pid}-#{SecureRandom.hex(8)}")
+      end
+
+      self.home = nil
+      true
+    rescue SystemCallError
+      false
+    end
+
+    # Nothing here is created at boot, because nothing survives a request. This only clears what an earlier
+    # boot left behind — the whole slot directory, since the names inside it are an earlier boot's and not
+    # this one's to reconstruct.
+    #
+    # `Dir.exist?` is not the guard, because it follows symlinks and answers false for a dangling one. An
+    # entry a tool left in the slot's place is exactly what this has to remove, and it used to survive the
+    # sweep and raise from every later `make_home`.
+    def prepare
+      remove_tree directory
+    end
+
+    # Unlinks whatever discard_home renamed out of the way. Partial progress is fine: a sweep killed
+    # part-way leaves fewer entries for the next one, so this converges rather than repeating.
+    def sweep
+      Dir.glob(File.join(directory, "discarded-*")).map { |path| remove_tree(path) }.all?
+    rescue SystemCallError
+      # The glob itself can fail, because the slot directory is a name a tool can replace — a symlink loop
+      # in its place answers ELOOP here rather than for any one entry. This runs from the worker's ensure,
+      # where a raise would replace the caller's response with a crash.
+      false
+    end
+
+    private
+      # **A mode is the only thing a tool needs to make its own tree unremovable, and a mode on a tree this
+      # uid owns is ours to put back.** `chmod 0500` on a directory a conversion wrote is enough to fail the
+      # recursive delete underneath it, and the delete failing used to be the end of it: one tree per
+      # request stayed on the tmpfs until the container ended. The repair is not a permission the process
+      # gains, it is one it never lost.
+      #
+      # Repair only after a failure, never before, so the common request pays for a walk of its own tree
+      # once rather than twice. `force:` on the chmod because a partial repair that removes most of the tree
+      # is better than none, and the remove that follows is what reports the outcome either way.
+      def remove_tree(path)
+        return true unless File.exist?(path) || File.symlink?(path)
+
+        FileUtils.remove_entry path
+        true
+      rescue SystemCallError
+        repair_and_remove path
+      end
+
+      def repair_and_remove(path)
+        FileUtils.chmod_R 0o700, path, force: true
+        FileUtils.remove_entry path
+        true
+      rescue SystemCallError
+        false
+      end
+  end
+end