hotcell-client 0.0.0 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a30252d4ffced6434f939fc313543c682d36abfdcb34d5b70431b9dc5002561a
-  data.tar.gz: dae5536d6035f67546136f12ffe1fc52d602bdffc8dd5eeba120fec2313c4cd7
+  metadata.gz: d6808ab8f7c6d12e3917dca07053e1c1ca88b686f43626b895f96a84e2625611
+  data.tar.gz: ee7b60efeaf1b1baed200abe2bc2756d80b75507259b98171c55e78b652ca1d8
 SHA512:
-  metadata.gz: f81150fc7be64160288e5ba3433d7561e9d500de3b5d248721da66ba03a75110f9692d5c40f04f5e86e7e6c9b2a45174092ddbea0a7adc381a36d99643697940
-  data.tar.gz: 8cf8d948c812075b068479e25737a0d43999831f2dea87a018d6d62fe8c1049d96bb27076f88173ed91d4bd4ae1939669d768402ee9e8ce8e5c215c14014d909
+  metadata.gz: 0135bebaa226693ef7c11f5e34ab1562bf6c58694ffbfb0af7aa3666acdeca05930a814696ed0b7097126deb1b8b3683033e3dead4134cecb12edfce65b20d3f
+  data.tar.gz: 1998e3e48ded09c5a5975574ce4f961652c8f0e6f635ab875dafc8cc2eb0553301c9dc0271386657f6348cc22d80358e29f0961d046b417a1cfbcae0931b86da

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 37signals, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# hotcell-client
+
+Part of [HotCell](https://github.com/basecamp/hotcell). See the repository README.

data/lib/hot_cell/cell.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module HotCell
+  # One registered cell: where its sockets are, how long this application will wait, and which of its own
+  # exception classes to raise for each side of the permanent split.
+  #
+  # Both socket paths are derived from one directory, so the volume mounts are mechanical rather than
+  # something to remember and the two sockets cannot end up apart.
+  class Cell
+    attr_reader :name, :timeout, :control_timeout, :permanent, :transient, :transport
+
+    # `timeout` covers work, so it is sized to clear the cell's `answer_within` and a saturated cell reports
+    # its own verdict rather than a transport failure. `control_timeout` covers `describe` and `metrics`,
+    # which the supervisor answers inline with no fork or queue — so it is short on purpose. Sharing one
+    # number would give the call whose job is to say "this cell is down" the patience of a video transcode.
+    #
+    # Both bound the answer rather than the whole call: connecting is not covered. Transport::Socket says
+    # why that is left alone.
+    def initialize(name, dir: nil, timeout: 30, control_timeout: 5,
+                   permanent: PermanentFailure, transient: TransientFailure,
+                   on_contract_skew: nil, transport: Transport::Socket.new)
+      @name = name.to_s
+      @dir = dir
+      @timeout = timeout
+      @control_timeout = control_timeout
+      @permanent = permanent
+      @transient = transient
+      @on_contract_skew = on_contract_skew
+      @transport = transport
+
+      verify_classification!
+    end
+
+    # Resolved on every call rather than at registration, which is what makes turning a path on a
+    # configuration change instead of a release. A directory consulted once in HotCell.register would make
+    # every flip a deploy, and reverting one too.
+    def directory
+      return @dir.call if @dir.respond_to?(:call)
+
+      @dir || (HotCell.root && File.join(HotCell.root, name))
+    end
+
+    # Unset means this path is off, and the caller runs in process exactly as it did before.
+    def enabled?
+      !directory.nil?
+    end
+
+    def work_socket
+      File.join directory, "work.sock"
+    end
+
+    def control_socket
+      File.join directory, "control.sock"
+    end
+
+    def exception_for(failure)
+      failure.permanent? ? permanent : transient
+    end
+
+    # Contract skew needs its own reporting hook because applications rescue broadly around
+    # representations, so "raise" is indistinguishable from "placeholder" and the skew is otherwise
+    # invisible. An application running several clients against several independently-booted cells needs to
+    # know which one skewed.
+    def report_contract_skew(error)
+      @on_contract_skew&.call error, self
+    end
+
+    # Static, and called once at boot. The cheapest way to catch a client pointed at a cell that does not
+    # carry the operation it wants, which is otherwise an `unsupported` on the first real request.
+    #
+    # Boot must not fail when a cell does not answer. A cell that is down at app boot is a degraded
+    # deployment rather than a broken one, and an application that refuses to start because its thumbnail
+    # cell is restarting is worse than one that serves placeholders. So this warns and carries on.
+    def describe
+      return nil unless enabled?
+
+      response = control(DESCRIBE)
+      unless response.ok?
+        HotCell.logger.warn "hotcell #{name}: #{unreachable_because response.failure}"
+        return nil
+      end
+
+      warn_about_timeout response.result
+      warn_about_missing_operations response.result
+      warn_about_group_skew response.result
+      response.result
+    end
+
+    def metrics
+      enabled? ? control(METRICS) : nil
+    end
+
+    private
+      # A cell's sockets are `0660`, so the group that lets a worker re-open a descriptor is also the group
+      # that admits a caller. EACCES therefore means one thing, and it is worth saying rather than leaving an
+      # operator to read "could not describe the cell" as "the cell is down". Every other failure reads that
+      # way correctly, because a restarting accessory is the common one.
+      def unreachable_because(failure)
+        if failure.error_class == "Errno::EACCES"
+          "this process may not open the cell's socket. Both sides share a group, and this one is in " \
+          "#{Process.groups.sort.inspect}. Add the cell's gid to this container (Kamal: `group-add` under " \
+          "the role's `options:`)."
+        else
+          "could not describe the cell (#{failure})"
+        end
+      end
+
+      def control(op)
+        transport.call self, Request.new(op: op).to_line, [], socket: control_socket, timeout: control_timeout
+      end
+
+      # The client's timeout is a sum rather than a comparison: a request may wait queue_wait in the queue
+      # and then run for deadline, and only then does the cell get to say what happened.
+      #
+      # Being bound tighter than that is defensible and it is a choice, not a mistake. A synchronous
+      # representation request wants it tighter, because a thread held for sixty seconds is a thread not
+      # serving traffic. A background job wants it looser, so it receives `capacity` or `killed` and can act
+      # on them rather than guessing from a socket error. Both outcomes are transient, so neither is
+      # misclassified — which is the only reason this is safe.
+      # The cell states this; adding it up here would mean guessing at stages only the supervisor knows about.
+      # A cell too old to report it says nothing, which is the right answer for a number we cannot know.
+      def warn_about_timeout(described)
+        needed = described[:answer_within]
+        return if needed.nil? || timeout.nil? || timeout >= needed
+
+        HotCell.logger.warn "hotcell #{name}: this client waits #{seconds timeout} and the cell says it may " \
+                            "take #{seconds needed} to answer (queue_wait #{seconds described[:queue_wait]} + " \
+                            "deadline #{seconds described[:deadline]} + the time to kill and reply), so a " \
+                            "saturated cell will arrive here as a transport failure rather than as its own " \
+                            "verdict. Deliberate on a synchronous path; a mistake for a background job."
+      end
+
+      # The cell reports seconds as floats, and "41.0s" is a worse sentence than "41s".
+      def seconds(value)
+        "#{format("%g", value)}s"
+      end
+
+      # HotCell.group is a number in this application's deploy file, and the cell's gid is baked into an
+      # image built somewhere else. Nothing else compares them, so a cell image that changed its gid would
+      # be an EACCES on every conversion — with a probe that was green the day before.
+      #
+      # A cell too old to report its groups says nothing, which is the right answer for a number we cannot
+      # know. So is a client with no group configured, which is the one-user case.
+      def warn_about_group_skew(described)
+        carried = described[:groups]
+        return if HotCell.group.nil? || carried.nil? || carried.include?(HotCell.group)
+
+        HotCell.logger.warn "hotcell #{name}: HotCell.group is #{HotCell.group} and this cell runs in " \
+                            "#{carried.inspect}, so it cannot open a file this application hands it and " \
+                            "every operation that gives a tool a filename will fail with EACCES."
+      end
+
+      def warn_about_missing_operations(described)
+        carried = Array(described[:operations])
+        wanted = HotCell.clients.select { |client| client.hotcell == name }
+
+        wanted.reject { |client| carried.include?(client.operation) }.each do |client|
+          HotCell.logger.warn "hotcell #{name}: #{client} wants #{client.operation.inspect} and this cell " \
+                              "carries #{carried.inspect}"
+        end
+      end
+
+      def verify_classification!
+        return if permanent.nil? || transient.nil?
+        return unless transient <= permanent
+
+        raise ConfigurationError,
+              "transient: #{transient} descends from permanent: #{permanent}, so every retryable failure " \
+              "would be recorded as a permanent one. The inheritance graph is the classification."
+      end
+  end
+end

data/lib/hot_cell/cells.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module HotCell
+  class << self
+    # The directory a cell's name resolves under, holding one subdirectory per cell. Unset means no cell is
+    # reachable at all, which is the off position of the whole rollout.
+    attr_accessor :root
+
+    # The group both sides hold, so a cell can open a caller's file by name. An operation that hands a tool
+    # a filename re-opens the descriptor as `/dev/fd/N`, and the kernel rechecks that open against the
+    # cell's own credentials rather than the caller's — so a file only this application can read fails with
+    # EACCES however the descriptor was passed. Set this to the cell's gid and put the application in that
+    # group; the client then narrows each descriptor's mode on the way out. See Client#wrap.
+    #
+    # Leave it unset where both sides already run as one user, which is how development runs.
+    attr_reader :group
+
+    # Takes the String an environment variable holds, so the initializer is one assignment with no
+    # coercion. `to_i` would read garbage as gid 0 — root's group — so a value that is not a number
+    # raises instead.
+    def group=(value)
+      @group = value.nil? ? nil : Integer(value)
+    rescue ArgumentError, TypeError
+      raise ConfigurationError, "HotCell.group is #{value.inspect} and must be a numeric gid"
+    end
+
+    attr_writer :logger
+
+    def logger
+      @logger ||= Logger.new($stderr)
+    end
+
+    # Cells are registered once.
+    #
+    #   HotCell.root = ENV.fetch("HOTCELL_ROOT", "/run/hotcell")
+    #
+    #   HotCell.register "active_storage",
+    #     permanent: ActiveStorage::PreviewError,
+    #     transient: MyApp::ConversionTemporarilyUnavailable
+    #
+    #   HotCell.register "archiver", dir: -> { ENV["HOTCELL_ARCHIVER_DIR"] }, timeout: 300
+    def register(name, **options)
+      Cell.new(name, **options).tap { |cell| cells[cell.name] = cell }
+    end
+
+    def cells
+      @cells ||= {}
+    end
+
+    def cell(name)
+      cells.fetch(name.to_s) do
+        raise UnregisteredCell, "no cell named #{name.to_s.inspect} is registered (#{cells.keys.inspect})"
+      end
+    end
+
+    # Whether a name is registered, for a caller with a fallback rather than a requirement.
+    def cell?(name)
+      cells.key?(name.to_s)
+    end
+
+    # Every HotCell::Client subclass, so a boot check can tell which cell is expected to carry what. This
+    # records what the process has loaded rather than what it has configured, so resetting registrations
+    # leaves it alone: the classes are still defined either way.
+    def clients
+      @clients ||= []
+    end
+
+    # Call once at boot, after registering. Warns and carries on; see Cell#describe.
+    def describe_cells
+      warn_about_group
+      cells.each_value.to_h { |cell| [ cell.name, cell.describe ] }
+    end
+
+    # A group this process does not hold cannot be given to a file, so without this the first conversion
+    # fails as EPERM from the client's own chown — after the deployment is live and carrying traffic. The
+    # check is local and needs no cell, so it reports a missing `group-add` even when every cell is down.
+    def warn_about_group
+      return if group.nil? || group == Process.gid || group == Process.egid
+      return if Process.groups.include?(group)
+
+      logger.warn "hotcell: HotCell.group is #{group} and this process is in #{Process.groups.sort.inspect}, " \
+                  "so it cannot put a descriptor in that group and every conversion will fail with EPERM. " \
+                  "Add the group to this container (Kamal: `group-add` under the role's `options:`), or " \
+                  "unset HotCell.group where both sides run as one user."
+    end
+
+    # Test support. Named apart from the server gem's own reset, because both gems open this module and a
+    # shared name would mean whichever loaded last silently won.
+    def reset_registrations!
+      @cells = nil
+      @root = nil
+      @group = nil
+    end
+  end
+end

data/lib/hot_cell/client/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module HotCell
+  # A class and not a module: this file loads before hot_cell/client.rb opens the same name.
+  class Client
+    VERSION = "0.1.0"
+  end
+end

data/lib/hot_cell/client.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require "hot_cell/core"
+
+# Notifications reaches for IsolatedExecutionState the moment something is actually subscribed, and not
+# before — so requiring only notifications works until the first subscriber appears and then raises
+# NameError in production.
+require "active_support/isolated_execution_state"
+require "active_support/notifications"
+
+require "hot_cell/client/version"
+require "hot_cell/failures"
+require "hot_cell/transport"
+require "hot_cell/cell"
+require "hot_cell/cells"
+
+require "hot_cell/railtie" if defined?(::Rails::Railtie)
+
+module HotCell
+  # The application side. A client class names the cell that serves it, and the call carries the same
+  # three things an operation receives on the other side — with the payload Hash arriving there as
+  # keyword arguments.
+  #
+  #   class ArchiveFolder < HotCell::Client
+  #     hotcell "archiver"
+  #   end
+  #
+  #   ArchiveFolder.perform_in_hotcell(inputs, outputs, payload)
+  #
+  # Routing is a class-level declaration rather than a call-site argument, so call sites carry no deployment
+  # detail and several clients may name the same cell.
+  class Client
+    # What a shared group is narrowed to on the way out, and the reason the group is safe to give away. A
+    # cell may read an input and never write one; it may write an output and never read one. The kernel
+    # applies these on every re-open by name, and the cell cannot widen either — changing a mode needs
+    # ownership, the caller owns these files, and `cap-drop ALL` leaves no capability that overrides it.
+    INPUT_MODE = 0o640
+    OUTPUT_MODE = 0o620
+
+    class << self
+      include Declarations
+
+      def inherited(subclass)
+        super
+        HotCell.clients << subclass
+      end
+
+      def hotcell(name = nil)
+        return inherited_value(:@cell_name) if name.nil?
+
+        @cell_name = name.to_s
+      end
+
+      def operation(name = nil)
+        return @operation_name || Naming.default_operation_name(self) if name.nil?
+
+        @operation_name = name.to_s
+      end
+
+      def cell
+        name = hotcell
+        raise ConfigurationError, "#{self} must name its cell with `hotcell \"a_name\"`" if name.nil?
+
+        HotCell.cell name
+      end
+
+      # Whether this client's named cell is registered, for a boot-time hook that must not raise on an
+      # application that has bundled the gem and not yet written the initializer.
+      def registered?
+        !hotcell.nil? && HotCell.cell?(hotcell)
+      end
+
+      # Whether this path is turned on. A caller that finds it off runs in process exactly as it did before,
+      # which is the whole rollout mechanism.
+      def enabled?
+        cell.enabled?
+      end
+
+      def perform_in_hotcell(inputs, outputs, payload = {})
+        new.perform_in_hotcell inputs, outputs, payload
+      end
+    end
+
+    def perform_in_hotcell(inputs, outputs, payload = {})
+      # Explicit wrapping rather than Array(): an IO is Enumerable, so Array(io) reads the stream line by
+      # line instead of wrapping it.
+      inputs = [ inputs ] unless inputs.is_a?(Array)
+      outputs = [ outputs ] unless outputs.is_a?(Array)
+
+      cell = self.class.cell
+
+      unless cell.enabled?
+        raise CellNotConfigured, "cell #{cell.name.inspect} has no socket directory, so this path is off"
+      end
+
+      # Everything up to here raises for itself, above the transport's rescue and on purpose. A payload
+      # value JSON cannot carry, a descriptor with the wrong access mode, or a request over the byte limit
+      # are this caller's bugs. An application whose transient class descends from IOError would otherwise
+      # have its own bad call reclassified as a socket failure and retried forever.
+      descriptors = wrap(inputs, outputs)
+      line = request_line(inputs, outputs, payload)
+
+      response = nil
+      ActiveSupport::Notifications.instrument "perform.hot_cell" do |event|
+        response = verify_output(cell.transport.call(cell, line, descriptors), outputs)
+        publish event, cell, response, inputs, outputs
+      end
+
+      raise_for response, cell
+      response.result
+    end
+
+    private
+      def wrap(inputs, outputs)
+        inputs.map { |io| Input.new(shared(io, INPUT_MODE)) } +
+          outputs.map { |io| Output.new(shared(io, OUTPUT_MODE)) }
+      end
+
+      # Through the descriptor rather than the path, so this names no file: fchown and fchmod take the open
+      # file the caller already gave us. Both need ownership, which the caller has and the cell does not.
+      #
+      # **Accepted risk.** These are set and not put back, so the caller's file keeps this group and this
+      # mode after the request — a `0600` file of the caller's own returns readable by the cell's group. The
+      # premise is that restoring is worse than carrying it: the cell may still hold the descriptor, undoing
+      # it mid-request adds a failure path to the answer, and a caller that could not share the file could
+      # not use this at all. Active Storage hands over tempfiles it then unlinks, so nothing survives there.
+      # docs/DEPLOYMENT.md tells anyone writing their own client to pass files they are willing to share.
+      def shared(io, mode)
+        return io if HotCell.group.nil?
+
+        io.chown nil, HotCell.group
+        io.chmod mode
+        io
+      end
+
+      def request_line(inputs, outputs, payload)
+        Request.new(op: self.class.operation, inputs: inputs.size, outputs: outputs.size,
+                    payload: payload).to_line
+      end
+
+      # `ok` with zero bytes written is a failure, and the client is where it has to be caught. The worker
+      # flushes before reporting success, so this should not happen — which is precisely why it must be
+      # handled rather than assumed away. A full tmpfs on the cell arrives this way, as ENOSPC from the copy
+      # rather than from the socket, and a full filesystem must never be recorded as "this document is
+      # unprocessable". So it is transient, not a valid empty image.
+      # Only a measured zero, never a stat that failed. `byte_count` used to answer 0 for both, which turned
+      # a descriptor this process could no longer stat into "the cell wrote nothing" — a transient failure
+      # manufactured on a request that had succeeded, and a variant thrown away for it.
+      def verify_output(response, outputs)
+        return response if !response.ok? || outputs.empty?
+        return response unless byte_count(outputs)&.zero?
+
+        Response.failed Failure.new(code: "unavailable",
+                                    message: "the cell reported success and wrote no bytes"),
+                        timing: response.timing
+      end
+
+      # `code` belongs on the event rather than only on an exception. A caller configured to treat
+      # `unreadable` as data would otherwise make those requests invisible, and unreadable rates are exactly
+      # what you want to watch after a library upgrade. `capacity` matters for the same reason: without its
+      # rate you cannot size a worker pool.
+      #
+      # The cause, signal and verdict go with it, because the code alone cannot classify a kill: `killed`
+      # is permanent for fsize and memory and transient for deadline and crashed. A subscriber with only
+      # the code filed every fsize kill as transient. `permanent` is the Failure's own answer, so no
+      # subscriber re-derives it.
+      #
+      # A subscriber's own duration minus perform_ms is transport plus queueing, and those want separate
+      # metrics: a rising perform_ms means the work got more expensive, and a rising difference means the
+      # cell is saturated.
+      def publish(event, cell, response, inputs, outputs)
+        failure = response.failure
+
+        event[:operation] = self.class.operation
+        event[:cell] = cell.name
+        event[:code] = failure&.code
+        event[:cause] = failure&.cause
+        event[:signal] = failure&.signal
+        event[:permanent] = failure&.permanent?
+        event[:bytes_in] = byte_count(inputs)
+        event[:bytes_out] = byte_count(outputs)
+        event[:perform_ms] = response.timing[:perform_ms]
+        event[:timing] = response.timing
+      end
+
+      # nil means "could not measure", which is not the same fact as zero and must not be confused with it.
+      def byte_count(ios)
+        ios.sum { |io| io.stat.size }
+      rescue SystemCallError, IOError
+        nil
+      end
+
+      def raise_for(response, cell)
+        return if response.ok?
+
+        failure = response.failure
+        error = cell.exception_for(failure).new(failure.to_s)
+        cell.report_contract_skew error if failure.code == "protocol"
+
+        raise error
+      end
+  end
+end

data/lib/hot_cell/failures.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module HotCell
+  # The gem cannot pick an application's exception classes, because two applications already do
+  # irreconcilable things with the same one. One puts ActiveStorage::PreviewError in an
+  # UNPROCESSABLE_ERRORS list and writes metadata["unprocessable"] = true, which no code path anywhere in
+  # that application ever un-writes. Another never marks at representation time and splits on the cache
+  # header instead, serving a file-icon placeholder with a hundred-year expiry for a permanent failure and
+  # no-store for anything else.
+  #
+  # So a gem that raised PreviewError for a capacity refusal would, in the first, permanently destroy the
+  # thumbnail of every blob viewed during a cell restart, with no recovery short of a hand-written backfill.
+  # `permanent` on the wire says which class to raise; injection is what stops the gem from guessing.
+  #
+  # These two are the fallback for a registration that injected neither. They are deliberately classes no
+  # application already rescues, so an unclassified failure surfaces as a loud 500 rather than as a silent
+  # permanent mark. Loud is the right way round.
+  class PermanentFailure < StandardError; end
+
+  # Must not descend from PermanentFailure, and that is not a stylistic point: the inheritance graph is the
+  # classification, so a later tidying pass that gives both a common ancestor silently turns every retryable
+  # failure into a permanent one.
+  class TransientFailure < StandardError; end
+end

data/lib/hot_cell/install/Dockerfile.tt

@@ -0,0 +1,55 @@
+# The cell's image. HotCell installed this file once; it is yours to customize, and nothing inherits
+# from a published base image — this is the whole recipe.
+#
+# A cell carries a Ruby runtime and every gem in its loaded graph, all inside the blast radius, so treat
+# this file as a budget rather than an inventory. Which tools it holds is what decides its blast radius.
+
+ARG RUBY_VERSION=3.4
+FROM ruby:${RUBY_VERSION}-slim
+
+# Install the tools and libraries your operations run — and nothing else. For example:
+#
+# RUN apt-get update && \
+#     apt-get install -y --no-install-recommends libvips42 mupdf-tools ffmpeg && \
+#     rm -rf /var/lib/apt/lists/*
+
+# The cell's user: no home directory and no shell. Pick a uid your host does not also use for a person —
+# without user namespace remapping this uid is the host's uid, and Debian's own range runs to 60000, so
+# 10001 is not outside it. What isolates the cell is the container, not the number.
+RUN groupadd --gid 10001 hotcell && \
+    useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin hotcell
+
+# The socket volume's mount point must exist here, owned by the cell's user, because a new named volume
+# takes its ownership from the directory it covers. /hotcell is not: see the chown back to root below.
+RUN mkdir -p /run/hotcell/cell /hotcell/operations && chown -R hotcell:hotcell /run/hotcell /hotcell
+
+# HOME is /tmp because the cell's user has no home directory, and bundler wants one. A worker replaces
+# it with its slot's home before it serves anything.
+ENV HOME=/tmp \
+    BUNDLE_PATH=/hotcell/bundle \
+    BUNDLE_WITHOUT=development \
+    HOTCELL_OPERATIONS=/hotcell/operations \
+    HOTCELL_DIR=/run/hotcell/cell
+
+WORKDIR /hotcell
+
+COPY --chown=hotcell:hotcell Gemfile* ./
+USER hotcell
+RUN bundle install
+
+COPY config.rb /hotcell/config.rb
+COPY operations/ /hotcell/operations/
+
+# The cell reads its own code and must not be able to write it. `HotCell.load!` requires config.rb and
+# every operation file at boot, so a worker that could write here would choose what the next supervisor
+# runs. Bundler needed to write during the build and nothing needs to afterwards, so hand the tree back to
+# root once the build is done. `read-only: true` covers this as well, and this holds without it.
+USER root
+RUN chown -R root:root /hotcell && chmod -R go-w /hotcell
+USER hotcell
+
+# Probes the supervisor's control socket from inside the container, where network: none does not apply.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+  CMD ["bundle", "exec", "hotcell-health"]
+
+CMD ["bundle", "exec", "hotcell"]

data/lib/hot_cell/install/Gemfile.tt

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# The cell's Gemfile. Keep it short: every gem loaded here is inside the blast radius and paid for on
+# every request. hotcell-client and rails do not belong here.
+
+source "https://rubygems.org"
+
+# Pinned to the client that installed it: the two are halves of one wire contract, and a skew between
+# them answers `protocol` on every request.
+gem "hotcell-server", "<%= HotCell::Client::VERSION %>"
+
+# Gems your operations need go here. For example:
+#
+# gem "image_processing"
+# gem "ruby-vips"

data/lib/hot_cell/install/config.rb.tt

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Loaded when the cell boots, before any operation.
+#
+# deadline and queue_wait are seconds; memory and file_size are bytes.
+HotCell.limits concurrency: 4, queue_size: 8, deadline: 60, memory: 1536 * 1024**2

data/lib/hot_cell/install.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "erb"
+require "fileutils"
+
+require "hot_cell/client/version"
+
+module HotCell
+  # Writes the hotcell/ directory into an application: a complete Dockerfile to customize, the cell's
+  # Gemfile, and an operations/ directory the cell loads at boot. Everything installs inline rather than
+  # deriving from a published base image, so the whole cell is the application's to read and change.
+  #
+  # A file that already exists is left exactly as it is, so running this again after customizing is safe.
+  module Install
+    TEMPLATES = File.expand_path("install", __dir__)
+
+    class << self
+      def call(root, out: $stdout)
+        templates.each do |template|
+          relative = template.delete_prefix("#{TEMPLATES}/").delete_suffix(".tt")
+          install template, File.join(root, "hotcell", relative), "hotcell/#{relative}", out
+        end
+      end
+
+      private
+        def templates
+          Dir.glob("#{TEMPLATES}/**/*.tt", File::FNM_DOTMATCH).sort
+        end
+
+        def install(template, destination, label, out)
+          if File.exist?(destination)
+            out.puts "   skip  #{label} (already exists)"
+          else
+            FileUtils.mkdir_p File.dirname(destination)
+            File.write destination, render(template)
+            out.puts " create  #{label}"
+          end
+        end
+
+        # A .tt is ERB, so a template can name something only the installing gem knows. The Gemfile uses it
+        # to pin the cell to this client's version, which is the one number the two sides must agree on.
+        def render(template)
+          ERB.new(File.read(template), trim_mode: "-").result(binding)
+        end
+    end
+  end
+end

data/lib/hot_cell/railtie.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module HotCell
+  # Loads the hotcell:install task into a Rails application. The gem works without Rails, so this file
+  # is required only when Rails::Railtie is already defined.
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load File.expand_path("tasks/hotcell.rake", __dir__)
+    end
+  end
+end

data/lib/hot_cell/tasks/hotcell.rake

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+namespace :hotcell do
+  desc "Install the hotcell/ directory: a Dockerfile to customize, the cell's Gemfile, and operations/"
+  task :install do
+    require "hot_cell/install"
+
+    HotCell::Install.call Rails.root
+  end
+end

data/lib/hot_cell/transport.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module HotCell
+  # The transport is a seam, and the socket one is the only implementation used outside tests.
+  #
+  # It never reconnects and never retries. One request per connection invites exactly that on
+  # ECONNREFUSED, and a silent retry doubles a cell's load at the moment it is least able to take it.
+  # Retry belongs in the job layer, which is the whole reason the transient class exists.
+  module Transport
+    class Socket
+      # **Accepted risk.** `timeout` covers the answer and not the connection. `UNIXSocket.new` blocks, and
+      # `connect` to a Unix socket blocks while the listener's backlog is full — a supervisor that is alive
+      # and no longer calling `accept`. A caller can be held there with no bound, on a path an application
+      # may be calling from a web request.
+      #
+      # The premise is that the state is nearly unreachable rather than tolerable. A supervisor that dies
+      # gives ECONNREFUSED, not a hang, so this needs one that lives and stops accepting — and the loop is
+      # built to make that not happen: `Log#emit` writes non-blocking so a stalled container log pipe cannot
+      # park it, and a recursive delete is renamed out of the loop rather than performed inside it. Bounding
+      # it means `connect_nonblock` plus `wait_writable` against the deadline `receive` already builds, which
+      # is worth doing the day this is observed and not before.
+      def call(cell, line, descriptors, socket: cell.work_socket, timeout: cell.timeout)
+        connection = Connection.new(UNIXSocket.new(socket))
+        connection.send_message line, descriptors: descriptors
+        receive connection, timeout
+      rescue SystemCallError, IOError => error
+        # A socket that does not exist, a cell that is restarting, an accessory not yet booted. These are
+        # the most likely failures in production and they produce no wire response at all, so they belong
+        # in the taxonomy rather than outside it — otherwise the code on the instrumentation event is blank
+        # for exactly the outage you most want to see.
+        unavailable error
+      ensure
+        connection&.close
+      end
+
+      private
+        # One absolute deadline across the whole response, not a wait for the first byte. Waiting for
+        # readability and then calling a blocking read bounded nothing: a peer that sent one byte inside the
+        # timeout and then stopped held this caller until the cell's own deadline, and a peer that never
+        # closed held it forever — on a path an application may well be calling from a web request.
+        def receive(connection, timeout)
+          line = connection.read_line(deadline: timeout && Clock.now + timeout)
+          return supervisor_gone if line.nil?
+
+          Response.parse line
+        rescue ReadTimeout
+          failed "timeout", "the cell did not answer within #{timeout}s"
+        rescue MessageError => error
+          unavailable "the cell's answer could not be read: #{error.message}"
+        end
+
+        # A live supervisor answers `killed` for a worker that died, which is the whole reason it holds a
+        # copy of the connection. So a connection that closes with no response at all means the supervisor
+        # itself is gone.
+        def supervisor_gone
+          unavailable "the connection closed with no response, so the cell's supervisor is gone"
+        end
+
+        # Takes a String or an Exception; Failure.for splits an Exception into its two wire fields.
+        def unavailable(detail)
+          failed "unavailable", detail
+        end
+
+        def failed(code, detail)
+          Response.failed Failure.for(code, detail)
+        end
+    end
+  end
+end

data/lib/hotcell-client.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Bundler auto-requires a gem named "hotcell-client" as "hotcell-client", then as "hotcell/client". This
+# gem uses neither path, because hot_cell/ is what yields the HotCell constant under the default inflection.
+# Without this file, `gem "hotcell-client"` in a Gemfile silently loads nothing at all.
+require "hot_cell/client"

metadata

@@ -1,24 +1,80 @@
 --- !ruby/object:Gem::Specification
 name: hotcell-client
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.0
 platform: ruby
 authors:
 - Mike Dalessio
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
-description: To be released soon, secure sidecar for Rails
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hotcell-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+description: |
+  The application side of HotCell. Register the cells a deployment runs, subclass HotCell::Client to
+  name one, and call it with descriptors and a payload.
+
+  The client owns everything a caller needs to respond correctly to a cell that is saturated,
+  restarting, or absent: the classification of every error into permanent and transient, the
+  exception classes an application injects for each, and instrumentation through
+  ActiveSupport::Notifications.
 email:
 - mike@37signals.com
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- MIT-LICENSE
+- README.md
+- lib/hot_cell/cell.rb
+- lib/hot_cell/cells.rb
+- lib/hot_cell/client.rb
+- lib/hot_cell/client/version.rb
+- lib/hot_cell/failures.rb
+- lib/hot_cell/install.rb
+- lib/hot_cell/install/Dockerfile.tt
+- lib/hot_cell/install/Gemfile.tt
+- lib/hot_cell/install/config.rb.tt
+- lib/hot_cell/railtie.rb
+- lib/hot_cell/tasks/hotcell.rake
+- lib/hot_cell/transport.rb
+- lib/hotcell-client.rb
+homepage: https://github.com/basecamp/hotcell
 licenses:
 - MIT
-metadata: {}
+metadata:
+  homepage_uri: https://github.com/basecamp/hotcell
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.1.0/hotcell-client
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.1.0/CHANGELOG.md
+  bug_tracker_uri: https://github.com/basecamp/hotcell/issues
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -26,7 +82,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -35,5 +91,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.16
 specification_version: 4
-summary: To be released soon, secure sidecar for Rails
+summary: Call a HotCell from an application.
 test_files: []