hotcell-client 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1bd5fbf6b1ac7ad757c4b8215f377b51322beb36994948d1cd384f71b1122e4f
-  data.tar.gz: d9a8dd19d861d0a67080a749dc86969da12d1604a23cc6b6001b516356ed6f33
+  metadata.gz: acb581746375493247ece1d1962ef0f534cbca23e97851c1539f6b75e0eaa495
+  data.tar.gz: 621d326e537edb6bdd1125cbb229e20588a3312b7f63bbba56f039ff9c42bf45
 SHA512:
-  metadata.gz: dbf9f630feb6d583574f38af75752caed8c54266a05a06b2e4559d53763fba3e173c2fd8a4b8d6c2362da0fed3b0fe538cbf05874c66334f60f44361cfd598d0
-  data.tar.gz: 94a49d9fb871cdeba4792d4b52a855635048b6fc7aac633d1f58960a4730c24b167ac4afd3a9dfe1cef7beb8a06e6dc1f5b944432f2b774bb2179f3a11e53e4f
+  metadata.gz: 9b94270d6241921e1f8b9793e1fafa3dd3c3928d21a68ff14b2012c50eb2fc612007b31c072462ac1b90adb738c61bbf1ab64ee3827a3c31825a20d1718f3e4c
+  data.tar.gz: 536992cc18c2777bb5e7ae31e0b7445ebec24987b3d691c8a50d9e8b251d0b00283fdf93cfd94b58c2a6a3f224833f7c3ab4462950db9218073779c3142245d1

data/lib/hot_cell/cell.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "tmpdir"
+
 module HotCell
   # One registered cell: where its sockets are, how long this application will wait, and which of its own
   # exception classes to raise for each side of the permanent split.
@@ -15,7 +17,7 @@ module HotCell
     # number would give the call whose job is to say "this cell is down" the patience of a video transcode.
     #
     # Both bound the answer rather than the whole call: connecting is not covered. Transport::Socket says
-    # why that is left alone.
+    # why that does not hang.
     def initialize(name, dir: nil, timeout: 30, control_timeout: 5,
                    permanent: PermanentFailure, transient: TransientFailure,
                    on_contract_skew: nil, transport: Transport::Socket.new)
@@ -72,7 +74,7 @@ module HotCell
     # deployment rather than a broken one, and an application that refuses to start because its thumbnail
     # cell is restarting is worse than one that serves placeholders. So this warns and carries on.
     #
-    # Nor when a cell answers something this client cannot read. The two warnings below reach into the
+    # Nor when a cell answers something this client cannot read. The warnings below reach into the
     # description without checking types, so a cell that sends the wrong ones raises here — and the README
     # calls `describe_cells` from `after_initialize`, where that is not a failed check but an application
     # that does not boot. The process that wrote the description is the one that runs untrusted content.
@@ -86,6 +88,7 @@ module HotCell
       response.result.tap do |described|
         warn_about_timeout described
         warn_about_group_skew described
+        warn_about_version_skew described
       end
     rescue StandardError => error
       HotCell.logger.warn "hotcell #{name}: this cell's description could not be read and is being " \
@@ -98,7 +101,65 @@ module HotCell
       enabled? ? control(METRICS) : nil
     end
 
+    # The control socket takes no worker but carries no descriptor, so its checks pass on a cell whose work
+    # socket this application cannot use. Only the round trips see that, and echo alone passes a cell with
+    # the wrong group. `control` rather than `describe`, whose boot warnings would repeat on every poll.
+    def diagnose(work: false)
+      checks = { describe: check { answered control(DESCRIBE) },
+                 metrics: check { answered control(METRICS) } }
+      return checks unless work
+
+      checks.merge echo: check { round_trip "health.echo" }, reopen: check { round_trip "health.reopen" }
+    end
+
     private
+      class CheckFailed < StandardError; end
+
+      def check
+        return { ok: false, error: "no socket directory, so this cell is off" } unless enabled?
+
+        { ok: true, result: yield }
+      rescue CheckFailed => error
+        { ok: false, error: error.message }
+      rescue StandardError => error
+        { ok: false, error: "#{error.class}: #{Failure.one_line error.message}" }
+      end
+
+      def answered(response)
+        raise CheckFailed, response.failure.to_s unless response.ok?
+
+        response.result
+      end
+
+      def round_trip(operation_name, message = "hotcell")
+        client = client_for(operation_name)
+
+        Dir.mktmpdir "hotcell-diagnose" do |directory|
+          source = File.join(directory, "in")
+          destination = File.join(directory, "out")
+          File.write source, message
+
+          result = File.open(source, "rb") do |input|
+            File.open(destination, "wb") { |output| client.perform_in_hotcell input, output }
+          end
+
+          returned = File.binread(destination)
+          raise CheckFailed, "the cell returned other bytes than it was sent" unless returned == message
+          raise CheckFailed, "the input was staged, so this did not cross a descriptor" if result[:staged]
+
+          result
+        end
+      end
+
+      def client_for(operation_name)
+        cell = self
+
+        Class.new(Client) do
+          define_singleton_method(:cell) { cell }
+          operation operation_name
+        end
+      end
+
       # A cell's sockets are `0660`, so the group that lets a worker re-open a descriptor is also the group
       # that admits a caller. EACCES therefore means one thing, and it is worth saying rather than leaving an
       # operator to read "could not describe the cell" as "the cell is down". Every other failure reads that
@@ -162,6 +223,14 @@ module HotCell
                             "every operation that gives a tool a filename will fail with EACCES."
       end
 
+      def warn_about_version_skew(described)
+        reported = described[:server_version]
+        return if reported == Client::VERSION
+
+        HotCell.logger.warn "hotcell #{name}: this client is #{Client::VERSION} and the cell reports " \
+                            "hotcell-server #{Failure.one_line reported.inspect}."
+      end
+
       # A `nil` timeout reaches `Transport::Socket#receive` as `deadline: nil`, and reading with no deadline
       # blocks: a cell that accepts the connection and then never answers holds this caller for good. At
       # boot that is an application that never finishes starting, with no exception and nothing to rescue.

data/lib/hot_cell/cells.rb

@@ -79,6 +79,18 @@ module HotCell
                   "unset HotCell.group where both sides run as one user."
     end
 
+    def diagnose(work: false)
+      Diagnosis.new(cells.values, work: work)
+    end
+
+    # Read when HotCell::DiagnosticsController loads, so set it in an initializer. A development reload of
+    # the parent does not reach it.
+    attr_writer :diagnostics_controller_parent
+
+    def diagnostics_controller_parent
+      @diagnostics_controller_parent || "ActionController::Base"
+    end
+
     # Test support. Named apart from the server gem's own reset, because both gems open this module and a
     # shared name would mean whichever loaded last silently won.
     def reset_registrations!

data/lib/hot_cell/client/version.rb

@@ -3,6 +3,6 @@
 module HotCell
   # A class and not a module: this file loads before hot_cell/client.rb opens the same name.
   class Client
-    VERSION = "0.5.0"
+    VERSION = "0.6.0"
   end
 end

data/lib/hot_cell/client.rb

@@ -13,9 +13,16 @@ require "hot_cell/failures"
 require "hot_cell/transport"
 require "hot_cell/cell"
 require "hot_cell/cells"
+require "hot_cell/diagnosis"
 
 require "hot_cell/railtie" if defined?(::Rails::Railtie)
 
+module HotCell
+  # Loaded when a route first names them, so an application without Action Pack never requires it.
+  autoload :HealthController, "hot_cell/health_controller"
+  autoload :DiagnosticsController, "hot_cell/diagnostics_controller"
+end
+
 module HotCell
   # The application side. A client class names the cell that serves it, and the call carries the same
   # three things an operation receives on the other side — with the payload Hash arriving there as
@@ -96,7 +103,7 @@ module HotCell
       line = request_line(inputs, outputs, payload)
 
       response = nil
-      ActiveSupport::Notifications.instrument "perform.hot_cell" do |event|
+      ActiveSupport::Notifications.instrument "perform.hot_cell", operation: self.class.operation, cell: cell.name do |event|
         response = verify_output(cell.transport.call(cell, line, descriptors), outputs)
         publish event, cell, response, inputs, outputs
       end
@@ -171,8 +178,6 @@ module HotCell
       def publish(event, cell, response, inputs, outputs)
         failure = response.failure
 
-        event[:operation] = self.class.operation
-        event[:cell] = cell.name
         event[:code] = failure&.code
         event[:cause] = failure&.cause
         event[:signal] = failure&.signal

data/lib/hot_cell/diagnosis.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "socket"
+require "time"
+
+module HotCell
+  class Diagnosis
+    attr_reader :cells
+
+    def initialize(cells, work:)
+      @at = Time.now.utc.iso8601(3)
+      @host = Socket.gethostname
+      @cells = cells.to_h { |cell| [ cell.name, cell.diagnose(work: work) ] }
+    end
+
+    # `cells.any?`, so an application that never registered a cell does not answer OK.
+    def healthy?
+      cells.any? && cells.each_value.all? { |checks| checks.each_value.all? { |check| check[:ok] } }
+    end
+
+    # `host` because sockets are host-local, and a poll through a load balancer lands wherever it lands.
+    def as_json(*)
+      { at: @at, host: @host, healthy: healthy?, cells: cells }
+    end
+  end
+end

data/lib/hot_cell/diagnostics_controller.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "action_controller"
+
+module HotCell
+  # Each request takes a worker per round trip, so put this behind authentication.
+  class DiagnosticsController < HotCell.diagnostics_controller_parent.constantize
+    def show
+      diagnosis = HotCell.diagnose(work: true)
+
+      render json: diagnosis, status: diagnosis.healthy? ? :ok : :service_unavailable
+    end
+  end
+end

data/lib/hot_cell/health_controller.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "action_controller"
+
+module HotCell
+  # Safe to leave unauthenticated: it asks the control socket only, so a poll takes no worker, and it answers
+  # nothing but OK or FAIL.
+  class HealthController < ActionController::Base
+    def show
+      healthy = HotCell.diagnose.healthy?
+
+      render plain: healthy ? "OK" : "FAIL", status: healthy ? :ok : :service_unavailable
+    end
+  end
+end

data/lib/hot_cell/install/Gemfile.tt

@@ -5,8 +5,7 @@
 
 source "https://rubygems.org"
 
-# Pinned to the client that installed it: the two are halves of one wire contract, and a skew between
-# them answers `protocol` on every request.
+# Keep this at the application's hotcell-client version; HotCell.describe_cells warns at boot otherwise.
 gem "hotcell-server", "<%= HotCell::Client::VERSION %>"
 
 # Gems your operations need go here. For example:

data/lib/hot_cell/log_subscriber.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+
+# All three, in this order, on Rails main: log_subscriber needs ColorizeLogging, which only the top-level file
+# autoloads, and loading it before deprecation warns of a circular require.
+require "active_support"
+require "active_support/deprecation"
+require "active_support/log_subscriber"
+
+module HotCell
+  # One line per call, success or failure, from the `perform.hot_cell` event. The railtie attaches it; an
+  # application without Rails calls `HotCell::LogSubscriber.attach_to :hot_cell` and sets
+  # `ActiveSupport::LogSubscriber.logger`.
+  #
+  # Two clocks on purpose: `perform_ms` is what the cell measured inside the worker and `duration_ms` is what
+  # this process waited, and their difference is the queue and the socket.
+  #
+  # `stderr` is text a tool wrote while reading a hostile file. It goes into the JSON and nowhere else, so
+  # its newlines arrive escaped rather than as forged log lines. `ascii_only` because JSON leaves U+2028 and
+  # its kin raw, and a log viewer may break a line on them.
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def perform(event)
+      payload = event.payload
+      duration_ms = event.duration.round(1)
+
+      info do
+        "  HotCell (#{duration_ms}ms) " + JSON.generate({
+          cell: payload[:cell],
+          operation: payload[:operation],
+          code: payload[:code] || ("ok" unless payload[:exception]),
+          exception: payload[:exception]&.first,
+          cause: payload[:cause],
+          perform_ms: payload[:perform_ms],
+          duration_ms: duration_ms,
+          bytes_in: payload[:bytes_in],
+          bytes_out: payload[:bytes_out],
+          stderr: payload[:stderr],
+        }.compact, ascii_only: true)
+      end
+    end
+  end
+end

data/lib/hot_cell/railtie.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
+require "hot_cell/log_subscriber"
+
 module HotCell
-  # Loads the hotcell:install task into a Rails application. The gem works without Rails, so this file
-  # is required only when Rails::Railtie is already defined.
+  # Loads the hotcell:install task into a Rails application and logs each call. The gem works without
+  # Rails, so this file is required only when Rails::Railtie is already defined.
   class Railtie < ::Rails::Railtie
+    initializer "hot_cell.log_subscriber" do
+      HotCell::LogSubscriber.attach_to :hot_cell
+    end
+
     rake_tasks do
       load File.expand_path("tasks/hotcell.rake", __dir__)
     end

data/lib/hot_cell/transport.rb

@@ -10,20 +10,19 @@ module HotCell
   # Retry belongs in the job layer, which is the whole reason the transient class exists.
   module Transport
     class Socket
-      # **Accepted risk.** `timeout` covers the answer and not the connection. `UNIXSocket.new` blocks, and
-      # `connect` to a Unix socket blocks while the listener's backlog is full — a supervisor that is alive
-      # and no longer calling `accept`. A caller can be held there with no bound, on a path an application
-      # may be calling from a web request.
+      # `timeout` covers the answer and not the connection. On Linux a blocking `connect` to a Unix socket waits
+      # in the kernel while the listener's backlog is full, which is where a supervisor that stops calling
+      # `accept` leaves it. `UNIXSocket.new` does not wait: Ruby opens every socket non-blocking and takes the
+      # kernel's EAGAIN for a connection in progress, so it returns a socket that never connected, the send
+      # fails with ENOTCONN, and the caller gets `unavailable` at once. That is Ruby's behavior rather than
+      # this gem's, so ControlTimeoutTest holds it.
       #
-      # The premise is that the state is nearly unreachable rather than tolerable. A supervisor that dies
-      # gives ECONNREFUSED, not a hang, so this needs one that lives and stops accepting — and the loop is
-      # built to make that not happen: `Log#emit` writes non-blocking so a stalled container log pipe cannot
-      # park it, and a recursive delete is renamed out of the loop rather than performed inside it. Bounding
-      # it means `connect_nonblock` plus `wait_writable` against the deadline `receive` already builds, which
-      # is worth doing the day this is observed and not before.
+      # If Ruby ever starts waiting, `connect_nonblock` plus `wait_writable` against a deadline would bound the
+      # call only by spinning: on Linux an unconnected Unix socket is always writable, so it retries EAGAIN until
+      # the deadline.
       def call(cell, line, descriptors, socket: cell.work_socket, timeout: cell.timeout)
         connection = Connection.new(UNIXSocket.new(socket))
-        connection.send_message line, descriptors: descriptors
+        deliver connection, line, descriptors
         receive connection, timeout
       rescue SystemCallError, IOError => error
         # A socket that does not exist, a cell that is restarting, an accessory not yet booted. These are
@@ -36,6 +35,16 @@ module HotCell
       end
 
       private
+        # A full cell writes its answer and closes the connection without reading the request. The send can
+        # then fail while the answer is already waiting on the socket. So this ignores the failed send, and
+        # `receive` reads the answer. If the peer closed without an answer, `receive` reports that the
+        # supervisor is gone.
+        def deliver(connection, line, descriptors)
+          connection.send_message line, descriptors: descriptors
+        rescue Errno::EPIPE, Errno::ECONNRESET
+          nil
+        end
+
         # One absolute deadline across the whole response, not a wait for the first byte. Waiting for
         # readability and then calling a blocking read bounded nothing: a peer that sent one byte inside the
         # timeout and then stopped held this caller until the cell's own deadline, and a peer that never

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hotcell-client
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Mike Dalessio
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -53,11 +53,15 @@ files:
 - lib/hot_cell/cells.rb
 - lib/hot_cell/client.rb
 - lib/hot_cell/client/version.rb
+- lib/hot_cell/diagnosis.rb
+- lib/hot_cell/diagnostics_controller.rb
 - lib/hot_cell/failures.rb
+- lib/hot_cell/health_controller.rb
 - lib/hot_cell/install.rb
 - lib/hot_cell/install/Dockerfile.tt
 - lib/hot_cell/install/Gemfile.tt
 - lib/hot_cell/install/config.rb.tt
+- lib/hot_cell/log_subscriber.rb
 - lib/hot_cell/railtie.rb
 - lib/hot_cell/tasks/hotcell.rake
 - lib/hot_cell/transport.rb
@@ -67,8 +71,8 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/basecamp/hotcell
-  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.5.0/hotcell-client
-  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.5.0/CHANGELOG.md
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.6.0/hotcell-client
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.6.0/CHANGELOG.md
   bug_tracker_uri: https://github.com/basecamp/hotcell/issues
   rubygems_mfa_required: 'true'
 rdoc_options: []
@@ -85,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Call a HotCell from an application.
 test_files: []