hotcell-server 0.0.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a38580413ef3b772a435be5c014e81e0b259fcf30d0f8644915185ab4b62959d
-  data.tar.gz: dae5536d6035f67546136f12ffe1fc52d602bdffc8dd5eeba120fec2313c4cd7
+  metadata.gz: 50d90e8b9f7ecb2a139d9c678a3cfdb228288d590f33cdf19390b6cbd92a924e
+  data.tar.gz: 66c3bcb911c378507bc9623cb1badc8c3372b6a58c3cd0312eae81c72b003877
 SHA512:
-  metadata.gz: cfd2cef68fba36bd8fb723e0c08468a91b1f99a5bd2b5e9cb657d8a267df5e7b438f922af66511439f92f6cf4335389566adf7a2b7e6add8d6dc25dd543c5c97
-  data.tar.gz: 8cf8d948c812075b068479e25737a0d43999831f2dea87a018d6d62fe8c1049d96bb27076f88173ed91d4bd4ae1939669d768402ee9e8ce8e5c215c14014d909
+  metadata.gz: c489f02a8b5670abda9df8c52f62fc4657a725abf01c6e49a39ad1065671535f96942e9ed5168fed5062d9b1ce3f55cef5aba0c2dfab2ec2f1c9c35b14980490
+  data.tar.gz: 5b633520243d9f70ab9adfaa1f2427516012ca477e3acd4d5917cbfe5a02ca01dc830e6d6855faeda5e11e8d3b8978634cadfa39712938aeeaeb6560e9f8032c

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 37signals, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# hotcell-server
+
+Part of [HotCell](https://github.com/basecamp/hotcell). See the repository README.

data/exe/hotcell

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "hot_cell/server"
+
+HotCell.load!
+
+HotCell::Supervisor
+  .new(directory: ENV.fetch("HOTCELL_DIR", "/run/hotcell/cell"), workspace: ENV["HOTCELL_WORKSPACE"])
+  .boot
+  .run

data/exe/hotcell-health

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Exits 0 when the local cell answers describe on its control socket, and 1 with a reason otherwise.
+#
+# Made for a Docker HEALTHCHECK, which execs inside the container — the one place a probe can reach a
+# cell that has no network. The control socket answers even when the work socket is saturated, so
+# healthy means the supervisor is alive and answering, not that a worker is free.
+
+require "hot_cell/core"
+require "socket"
+
+def unhealthy(reason)
+  puts "unhealthy: #{reason}"
+  exit 1
+end
+
+directory = ENV.fetch("HOTCELL_DIR", "/run/hotcell/cell")
+deadline = HotCell::Clock.now + Float(ENV.fetch("HOTCELL_HEALTH_TIMEOUT", "5"))
+
+begin
+  socket = UNIXSocket.new(File.join(directory, "control.sock"))
+  connection = HotCell::Connection.new(socket)
+
+  connection.write_line HotCell::Request.new(op: HotCell::DESCRIBE).to_line
+
+  # nil is end of stream, which is a supervisor that accepted and then died. Passing it to the parser
+  # raised TypeError from inside JSON, so the probe reported unhealthy with a backtrace instead of a reason.
+  line = connection.read_line(deadline: deadline)
+  unhealthy "the cell closed the connection without answering" if line.nil?
+
+  response = HotCell::Response.parse(line)
+  unhealthy "the cell answered #{response.failure}" unless response.ok?
+rescue SystemCallError, IOError, HotCell::Error => error
+  unhealthy "#{error.class}: #{error.message}"
+end
+
+puts "healthy"

data/lib/hot_cell/configuration.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module HotCell
+  # A cell is configured once, and everything about scheduling lives here rather than on an operation.
+  #
+  # These numbers are arithmetic against the container's own flags, not defaults to copy. Against a cell
+  # given two CPUs, 2GB, and a 512MB tmpfs: concurrency 4 because a request spends much of its life off
+  # the CPU, so twice `cpus` is where to start; file_size 64MB because it bounds what one worker writes
+  # onto that tmpfs, and four of them share it; and memory 1536MB because that is the measured working
+  # value for RLIMIT_DATA, which is per worker and mostly reservation rather than resident bytes.
+  #
+  # What a worker writes is its outputs plus a copy of any input an operation asked for by path. An
+  # operation that consumes the descriptor never pays for its input, which is what lets a small file_size
+  # accept a large upload — but the kernel does not distinguish the two writes, so one number covers both.
+  #
+  # memory does not multiply by concurrency, and that is the easiest mistake to make here. It is an
+  # address-space charge on one worker. The cgroup limit is what bounds real memory across the cell, and
+  # it counts the tmpfs too, so size the two separately.
+  class Configuration
+    SCHEDULING = {
+      concurrency:      4,   # workers running at once, and therefore the number of slots
+      queue_size:       8,   # connections that may be accepted and waiting for one
+      queue_wait:       10,  # seconds a queued connection may wait before it is answered `capacity`
+      max_requests_per_worker:            1,   # requests a worker serves before it is discarded
+      control_deadline: 5,   # seconds a control connection may take to send its request
+    }.freeze
+
+    LIMITS = {
+      deadline:   60,
+      memory:     1536 * 1024**2,
+      file_size:  64 * 1024**2,
+      open_files: 256,
+    }.freeze
+
+    UNLIMITED = :unlimited
+
+    # Noticing an overdue worker, signalling it, reaping it, and writing the answer. Also the budget a
+    # retired worker gets to exit before the supervisor kills its group — see Supervisor#enforce_retirements.
+    KILL_GRACE = 1
+
+    attr_reader(*SCHEDULING.keys)
+    attr_reader :limits
+
+    def initialize(**options)
+      unknown = options.keys - SCHEDULING.keys - Limits::KEYS
+      raise ConfigurationError, "unknown setting #{unknown.join(", ")}" if unknown.any?
+
+      SCHEDULING.each { |key, default| instance_variable_set :"@#{key}", options.fetch(key, default) }
+
+      # The two second-valued settings, coerced for the same reason Limits coerces deadline: they appear
+      # in describe's JSON, and they may arrive as Active Support durations.
+      @queue_wait = @queue_wait.to_f
+      @control_deadline = @control_deadline.to_f
+
+      # A nil is not "use the default" here, it is a missing number. A cell whose deadline is nil accepts
+      # every request and then dies on the first arithmetic the supervisor does with it, so an explicit nil
+      # has to be refused where it is written rather than where it is used.
+      declared = options.slice(*Limits::KEYS)
+      if (empty = declared.select { |_, value| value.nil? }.keys).any?
+        raise ConfigurationError, "#{empty.join(", ")} cannot be nil; leave it out to take the default"
+      end
+
+      @limits = Limits.new(**LIMITS.merge(declared))
+
+      verify!
+    end
+
+    def unlimited_requests?
+      max_requests_per_worker == UNLIMITED
+    end
+
+    def retire?(served)
+      !unlimited_requests? && served >= max_requests_per_worker
+    end
+
+    # What this cell expects to answer within, and deliberately not a bound it can keep: a worker in
+    # uninterruptible sleep does not die when it is signalled, and the supervisor answers only after the
+    # reap. Treat it as the threshold a client's timeout should clear, not as a guarantee to build a
+    # correctness argument on.
+    #
+    # Derived here rather than reassembled in the client: a client adding up `queue_wait + deadline` plus its
+    # own guess at the kill-to-answer step cannot follow a change to any of them, and the error points the
+    # unsafe way — the client believes its timeout is generous and takes a transport failure instead of the
+    # cell's verdict. A stage added later that costs a caller time belongs in this sum.
+    def answer_within
+      queue_wait + limits.deadline + KILL_GRACE
+    end
+
+    # Goes on the wire as the answer to hotcell.describe, so every value has to be JSON-native.
+    # `max_requests_per_worker` is the one that is not: :unlimited is a Symbol, and a cell configured with it could
+    # not describe itself at all.
+    def to_h
+      SCHEDULING.keys.to_h { |key| [ key, public_send(key) ] }
+        .merge(limits.to_h)
+        .merge(answer_within: answer_within,
+               max_requests_per_worker: unlimited_requests? ? UNLIMITED.to_s : max_requests_per_worker)
+    end
+
+    private
+      # `integer:` is a real distinction rather than an oversight, so it is written as one: a count of workers
+      # or of queue places has to be whole, where a number of seconds does not.
+      def verify!
+        positive! :concurrency, integer: true
+        positive! :queue_wait
+        positive! :control_deadline
+
+        unless queue_size.is_a?(Integer) && !queue_size.negative?
+          raise ConfigurationError, "queue_size: #{queue_size} must not be negative"
+        end
+
+        unless unlimited_requests? || (max_requests_per_worker.is_a?(Integer) && max_requests_per_worker.positive?)
+          raise ConfigurationError, "max_requests_per_worker: #{max_requests_per_worker.inspect} must be " \
+                                    "a positive Integer or :unlimited"
+        end
+      end
+
+      def positive!(key, integer: false)
+        value = public_send(key)
+        return if value.positive? && (!integer || value.is_a?(Integer))
+
+        raise ConfigurationError, "#{key}: #{value} must be positive"
+      end
+  end
+end

data/lib/hot_cell/control.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module HotCell
+  # The two things a cell answers that are not conversions, on control.sock rather than work.sock.
+  #
+  # Keeping control off the data path buys three things. It answers when the cell is saturated, where a
+  # health check or a metrics scrape sharing the work queue would fail under load and report the same thing
+  # as an outage. It gets its own allowance and its own much shorter deadline, which are not the same
+  # numbers — a scrape is milliseconds and a conversion is seconds. And the socket a connection arrived on
+  # is itself the discriminator, so routing costs nothing and cannot be confused by a payload.
+  #
+  # The supervisor answers these itself rather than forking a worker for them, which is a deliberate
+  # departure from the original design. Forking would work — a child inherits the counters and could read
+  # them without being told — but the whole value of this channel is being available when nothing else is,
+  # and a channel that needs a fork to answer is a channel that goes quiet exactly when a fork is what is
+  # failing. Neither operation takes a descriptor, touches a tool, or evaluates a byte of image data,
+  # so none of the reasons the supervisor stays out of a conversion apply. Reading a bounded control line
+  # from the trusted side starts no thread pool and cannot deadlock a later fork.
+  class Control
+    def initialize(configuration:, counters:)
+      @configuration = configuration
+      @counters = counters
+    end
+
+    def answer(line, running:, queued:)
+      request = Request.parse(line)
+
+      unless request.current_version?
+        return failed("protocol", request.version_mismatch)
+      end
+
+      case request.op
+      when DESCRIBE then Response.ok(result: describe)
+      when METRICS then Response.ok(result: @counters.to_h(running: running, queued: queued))
+      else failed "unsupported", "control.sock answers #{CONTROL_OPERATIONS.join(" and ")}, not #{request.op.inspect}"
+      end
+    rescue MessageError => error
+      failed "invalid", error.message
+    end
+
+    # Static, and called once per registered cell at app boot. It is the cheapest way to catch a client
+    # pointed at a cell that does not carry the operation it wants, which is otherwise an `unsupported` on
+    # the first real request, and to catch a client whose own timeout is below what this cell may take.
+    def describe
+      { v: PROTOCOL_VERSION, operations: Registry.names, groups: groups, **@configuration.to_h }
+    end
+
+    private
+      # What a caller's files must carry for this cell to open one by name. The number in an application's
+      # deploy file has to agree with the gid baked into this image, and nothing else compares them — so a
+      # cell whose gid moved is a boot warning in the client rather than an EACCES on every conversion.
+      #
+      # The primary gid is added because getgroups is not required to report it.
+      def groups
+        (Process.groups + [ Process.gid ]).uniq.sort
+      end
+
+      def failed(code, message)
+        Response.failed Failure.new(code: code, message: message)
+      end
+  end
+end

data/lib/hot_cell/counters.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module HotCell
+  # These live in the supervisor, and a forked worker can still report them: a child inherits the
+  # supervisor's memory at fork, so the worker answering a metrics request reads a consistent snapshot of
+  # counters it never had to be told. This is the one place fork-per-request is a convenience rather than
+  # a cost.
+  #
+  # Three of these are not derivable on the client side, which is why the control channel exists at all.
+  # queue_high_water is the leading saturation signal and no single caller sees it. killed_by separates a
+  # decompression bomb from a slow afternoon in aggregate. cancelled counts callers that gave up before
+  # the cell answered, which by definition appears on no response.
+  #
+  # cancelled is a floor rather than a total, and the reason is where each answer is written from. The
+  # supervisor writes refusals, kills and deadline breaches, so it sees the broken pipe and counts it. A
+  # successful conversion is written by the worker, which exits without telling anyone — so a caller that
+  # hangs up during one is not counted. Read it as "at least this many", and read a rise as real.
+  class Counters
+    def initialize
+      @started_at = Clock.now
+      @requests = Hash.new(0)
+      @killed_by = Hash.new(0)
+      @cancelled = 0
+      @queue_high_water = 0
+    end
+
+    def record(code)
+      @requests[:total] += 1
+      @requests[(code || :ok).to_sym] += 1
+    end
+
+    def record_kill(cause)
+      @killed_by[cause.to_sym] += 1
+    end
+
+    def cancelled!
+      @cancelled += 1
+    end
+
+    def observe_queue(depth)
+      @queue_high_water = depth if depth > @queue_high_water
+    end
+
+    def to_h(running: 0, queued: 0)
+      { uptime_s: (Clock.now - @started_at).round, running: running, queued: queued,
+        queue_high_water: @queue_high_water, requests: @requests, killed_by: @killed_by,
+        cancelled: @cancelled }
+    end
+  end
+end

data/lib/hot_cell/limits.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module HotCell
+  # What a worker may consume, and what a cell will let an operation ask for. `deadline` is seconds;
+  # `memory` and `file_size` are bytes; `open_files` is a count. Active Support's helpers work —
+  # `1280.megabytes` is a plain Integer already, and a `30.seconds` duration flattens to one on arrival.
+  #
+  # There is deliberately no RLIMIT_CPU. The deadline strictly covers it — anything that burns CPU also
+  # burns wall clock, and the deadline additionally catches a worker blocked on a wedged subprocess,
+  # which trips no CPU limit because a stuck worker consumes no CPU at all. The two numbers are related
+  # by a factor nobody can predict, measuring 1.0x at libvips concurrency 1 and about 1.5x at 4, so a CPU
+  # limit cannot be derived from a latency budget. And RLIMIT_CPU is cumulative over a process's life, so
+  # it stops meaning "per request" the moment a worker serves a second one.
+  #
+  # What is given up is real: RLIMIT_CPU was kernel-enforced and would still fire if the supervisor's
+  # timer logic were wrong. That is the reason to keep the supervisor's loop boring.
+  class Limits
+    # Below this a worker does not fail gracefully, it dies before it can answer, as SIGABRT or ENOMEM
+    # during boot. Roughly 450MB of any RLIMIT_DATA is Ruby's own: since 3.3 the interpreter reserves a
+    # single ~404MB writable anonymous region at boot that it never touches, and RLIMIT_DATA charges all
+    # of it. So this is not "how much a bomb may consume" — subtract 450MB before reading it that way.
+    MEMORY_FLOOR = 1024 * 1024**2
+
+    # RLIMIT_DATA rather than RLIMIT_AS. RLIMIT_DATA charges private writable anonymous mappings and
+    # ignores PROT_NONE reservations, read-only file mappings, and MAP_SHARED of any kind. Against a real
+    # variant whose peak RSS is 45MB, RLIMIT_DATA works from 704MB where RLIMIT_AS needs 1536MB, and
+    # RLIMIT_AS fails nondeterministically for a 400MB band below its floor. Do not add RLIMIT_AS as a
+    # backstop: any value clearing that band already exceeds the container's own memory limit.
+    RESOURCES = {
+      memory:     Process::RLIMIT_DATA,
+      file_size:  Process::RLIMIT_FSIZE,
+      open_files: Process::RLIMIT_NOFILE,
+    }.freeze
+
+    KEYS = [ :deadline, *RESOURCES.keys ].freeze
+
+    attr_reader(*KEYS)
+
+    # to_f and to_i, because these travel as JSON and 30.seconds is an ActiveSupport::Duration until it
+    # is asked to be a number.
+    def initialize(deadline: nil, memory: nil, file_size: nil, open_files: nil)
+      @deadline = deadline&.to_f
+      @memory = memory&.to_i
+      @file_size = file_size&.to_i
+      @open_files = open_files&.to_i
+
+      verify_positive!
+      verify_memory_floor!
+    end
+
+    def [](key)
+      public_send key
+    end
+
+    def to_h
+      KEYS.to_h { |key| [ key, self[key] ] }
+    end
+
+    def declared
+      to_h.compact
+    end
+
+    # A new Limits with these values over this one's — what a redeclaration means. A key that is not
+    # named keeps its value; one that is named to nil is withdrawn, so a redeclaration can also hand a
+    # limit back to the cell.
+    def merge(**values)
+      self.class.new(**to_h.merge(values))
+    end
+
+    # An operation cannot exceed its cell's limits, whatever it declares. This is invariant 6, and a
+    # clamp that silently stops clamping looks exactly like a clamp, which is why it is tested.
+    def clamped_to(ceiling)
+      self.class.new(**KEYS.to_h { |key| [ key, smaller(self[key], ceiling[key]) ] })
+    end
+
+    # The soft limit narrows to the operation and the hard limit stays at the cell's ceiling. An
+    # unprivileged process can raise a soft limit up to its hard limit but can never raise a hard one, so
+    # this is what lets a reused worker widen back for an operation with a different budget. Setting both
+    # to the operation's value would make the first request the tightest the worker could ever be.
+    def apply(ceiling: self)
+      Process.setrlimit Process::RLIMIT_CORE, 0
+
+      RESOURCES.each do |key, resource|
+        soft = self[key]
+        next if soft.nil?
+        next if key == :memory && !self.class.memory_enforceable?
+
+        begin
+          Process.setrlimit resource, soft, ceiling[key] || soft
+        rescue Errno::EINVAL
+          raise unless key == :memory
+
+          self.class.memory_unenforceable!
+        end
+      end
+    end
+
+    # macOS/XNU has no finite RLIMIT_DATA and returns EINVAL for any value, so the memory clamp cannot be set
+    # there. Rather than crash every worker, warn once and run unclamped — the memory limit is a Linux property
+    # and production is Linux. The suite skips the enforcement assertions off Linux.
+    @memory_enforceable = true
+
+    class << self
+      def memory_enforceable?
+        @memory_enforceable
+      end
+
+      def memory_unenforceable!
+        return unless @memory_enforceable
+
+        @memory_enforceable = false
+        warn "hotcell: RLIMIT_DATA is not settable on #{RUBY_PLATFORM}; the cell's memory limit is not enforced"
+      end
+    end
+
+    private
+      def smaller(mine, theirs)
+        return theirs if mine.nil?
+        return mine if theirs.nil?
+
+        [ mine, theirs ].min
+      end
+
+      def verify_positive!
+        KEYS.each do |key|
+          value = self[key]
+          next if value.nil? || value.positive?
+
+          raise ConfigurationError, "#{key}: #{value} must be positive"
+        end
+      end
+
+      def verify_memory_floor!
+        return if memory.nil? || memory >= MEMORY_FLOOR
+
+        raise ConfigurationError,
+              "memory: #{memory} is below the #{MEMORY_FLOOR} byte floor. About 450MB of RLIMIT_DATA is " \
+              "Ruby's own untouched reservation, so a worker under the floor dies during boot rather " \
+              "than failing gracefully."
+      end
+  end
+end

data/lib/hot_cell/log.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module HotCell
+  # A cell has no network, so nothing inside it can dial a collector. That rules out less than it appears
+  # to, because stdout is not a network: it is a pipe to the container runtime, whose log driver runs in
+  # that daemon with the host's network. So logs ship normally under network: none, and a cell writes
+  # structured JSON lines with the standard library and gets a real log for free.
+  #
+  # This is the channel for everything no response can carry: deadline kills, reaps that found a signal,
+  # boot checks, and queue high-water.
+  #
+  # Lines follow docs/LOGS.md: ECS field names, domain fields under the hotcell namespace. The fleet's
+  # collector routes on service.name, takes the record timestamp from @timestamp, and severity from
+  # log.level, so those three are load-bearing: renaming any of them silently drops or mislabels every
+  # cell log line in production.
+  class Log
+    # Severity lives here rather than in the collector so that adding an event never needs a collector
+    # change. An event missing from this table logs as INFO rather than not at all.
+    LEVELS = {
+      "cell.boot" => "INFO",
+      "cell.stopping" => "INFO",
+      "cell.stopped" => "INFO",
+      "cell.ptrace_scope_unknown" => "ERROR",
+      "request" => "INFO",
+      "request.abandoned" => "WARN",
+      "worker.forked" => "INFO",
+      "worker.reaped" => "INFO",
+      "worker.crashed" => "ERROR",
+      "worker.killed" => "WARN",
+      "worker.deadline" => "WARN",
+      "worker.lingered" => "WARN",
+      "worker.unforkable" => "ERROR",
+      "worker.undispatchable" => "ERROR",
+      "worker.unreadable_report" => "ERROR",
+      "control.abandoned" => "WARN",
+      "control.unanswerable" => "WARN",
+      "slot.uncleaned" => "WARN",
+      "slot.undiscarded" => "WARN",
+    }.freeze
+
+    def self.null
+      new File.open(File::NULL, "w")
+    end
+
+    def initialize(io = $stdout)
+      @io = io
+      @io.sync = true
+    end
+
+    # A log line is never worth the cell. This runs inside the loop that enforces every request's deadline,
+    # and the sink is a pipe to a runtime that can go away or stop draining. Losing the line is the correct
+    # trade against losing the process.
+    def write(event, **fields)
+      emit line(event, fields)
+    rescue JSON::GeneratorError
+      emit line(event, unloggable: fields.keys)
+    end
+
+    private
+      # Never blocks. A closed reader is EPIPE and a full disk behind the driver is ENOSPC, both rescued —
+      # but a reader that is alive and not draining is neither: a blocking write would simply park the
+      # deadline loop until the runtime resumed. A non-blocking write answers `:wait_writable` for the full
+      # pipe instead, and the line is dropped like any other.
+      #
+      # A short count would be a torn line, which a write at or under PIPE_BUF cannot produce. The one line
+      # that can exceed it is cell.boot's inventory, written once before the loop enforces anything and
+      # against an empty pipe.
+      def emit(line)
+        @io.write_nonblock line, exception: false
+      rescue SystemCallError, IOError
+        nil
+      end
+
+      def line(event, fields)
+        JSON.generate(document(event, fields.dup)) << "\n"
+      end
+
+      def document(event, fields)
+        {
+          "@timestamp": Time.now.utc.iso8601(3),
+          service: { name: "hotcell" },
+          event: event_fields(event, fields),
+          log: { level: LEVELS.fetch(event, "INFO") },
+          **process_fields(fields),
+          **prose_fields(fields),
+          **({ hotcell: fields } unless fields.empty?).to_h,
+        }
+      end
+
+      def event_fields(event, fields)
+        { action: event }.tap do |result|
+          result[:outcome] = fields.delete(:outcome) if fields.key?(:outcome)
+          result[:duration] = { ms: fields.delete(:duration_ms) } if fields.key?(:duration_ms)
+        end
+      end
+
+      def process_fields(fields)
+        process = {
+          pid: fields.delete(:pid),
+          exit_code: fields.delete(:exit_code),
+        }.compact
+
+        process.empty? ? {} : { process: process }
+      end
+
+      # `message` beside an exception is that exception's message; alone it is the line's prose.
+      def prose_fields(fields)
+        if fields.key?(:error)
+          { error: { type: fields.delete(:error), message: fields.delete(:message) }.compact }
+        elsif fields.key?(:message)
+          { message: fields.delete(:message) }
+        else
+          {}
+        end
+      end
+  end
+end