shuttlebay 0.1.4

data/ext/shuttlebay/src/link.rs

@@ -0,0 +1,78 @@
+use state_machines::state_machine;
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub struct RequestId(u32);
+
+state_machine! {
+    name: Link,
+    dynamic: true,
+    initial: Idle,
+    states: [
+        Idle,
+        superstate Exchange(RequestId) {
+            state Requested,
+            state Streaming,
+            state Replied,
+            state Tunnelling,
+            state Finished,
+            initial: Requested,
+        },
+    ],
+    events {
+        hail {
+            transition: { from: Idle, to: Exchange }
+        }
+        reply_stream {
+            transition: { from: Requested, to: Streaming }
+        }
+        reply_whole {
+            transition: { from: Requested, to: Replied }
+        }
+        reply_tunnel {
+            transition: { from: Requested, to: Tunnelling }
+        }
+        finish {
+            transition: { from: [Streaming, Replied], to: Finished }
+        }
+        ready {
+            transition: { from: Finished, to: Idle }
+        }
+    }
+}
+
+pub type Machine = DynamicLink<()>;
+
+pub fn new() -> Machine {
+    DynamicLink::new(())
+}
+
+pub fn request_id(link: &Machine) -> Option<u32> {
+    link.exchange_data().map(|id| id.0)
+}
+
+pub fn begin(link: &mut Machine, request_id: u32) -> Result<(), &'static str> {
+    link.handle(LinkEvent::Hail).map_err(|_| "read_request called before the previous ready")?;
+    link.set_exchange_data(RequestId(request_id)).map_err(|_| "request id could not be recorded")
+}
+
+pub fn request_in(
+    link: &Machine,
+    wanted: LinkState,
+    without_request: &'static str,
+    elsewhere: &'static str,
+) -> Result<u32, &'static str> {
+    let request_id = request_id(link).ok_or(without_request)?;
+    if link.current_state() == wanted { Ok(request_id) } else { Err(elsewhere) }
+}
+
+pub fn advance(
+    link: &mut Machine,
+    event: LinkEvent,
+    refused: impl FnOnce(LinkState) -> &'static str,
+) -> Result<(u32, LinkState), &'static str> {
+    let from = link.current_state();
+    let refusal = refused(from);
+    let request_id = request_id(link).ok_or(refusal)?;
+    link.handle(event).map_err(|_| refusal)?;
+    Ok((request_id, from))
+}

data/lib/rackup/handler/shuttlebay.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'rackup/handler'
+require 'shuttlebay/attached'
+
+module Rackup
+  module Handler
+    module Shuttlebay
+      def self.run(app, **)
+        server = ::Shuttlebay::Attached.new(app, **)
+        yield server if block_given?
+        server.run
+      end
+
+      def self.valid_options
+        {
+          'Host=HOST' => 'Address Mothership listens on (default: 127.0.0.1)',
+          'Port=PORT' => 'Port Mothership listens on (default: 9292)',
+          'Threads=N' => 'Threads running the app (default: MS_BAY_THREADS or 3)'
+        }
+      end
+    end
+
+    register :shuttlebay, Shuttlebay
+  end
+end

data/lib/shuttlebay/attached.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'socket'
+require 'tmpdir'
+require_relative 'runtime'
+
+module Shuttlebay
+  class Attached
+    # sun_path holds 104 bytes on macOS (108 on Linux), terminator included
+    SOCKET_PATH_MAX = 103
+
+    attr_reader :host, :port, :threads
+
+    def initialize(app, trap_signals: true, mothership: Attached.mothership_binary, logger: nil, **options)
+      @app = app
+      @host = options.fetch(:Host, '127.0.0.1').to_s
+      @port = Integer(options.fetch(:Port, 9292))
+      @threads = Attached.threads(options[:Threads])
+      @silent = options.fetch(:Silent, false)
+      @trap_signals = trap_signals
+      @mothership = mothership
+      @logger = logger || (@silent ? Logger.new(File.open(File::NULL, 'w')) : Shuttlebay.logger)
+    end
+
+    def run
+      @dir, socket = Attached.socket_dir
+      listener = UNIXServer.new(socket)
+      manifest = File.join(@dir, 'ship-manifest.toml')
+      File.write(manifest, manifest_toml(socket))
+
+      @worker = Worker.new(app: @app, listener: listener, threads: threads, workers: 0, ship: 'app',
+                           logger: @logger)
+      Shuttlebay.running = true
+      @mothership_pid = spawn_mothership(manifest)
+      watch_mothership
+      at_exit { stop_mothership } unless @trap_signals
+      @logger.info('serving through mothership', url: url, threads: threads)
+      @worker.run(trap_signals: @trap_signals)
+      raise Error, "mothership exited (#{@mothership_exit}); see its output above" if @mothership_exit
+    ensure
+      stop_mothership
+      FileUtils.rm_rf(@dir) if @dir
+    end
+
+    def stop
+      @worker&.stop!
+    end
+
+    def self.mothership_binary(env = ENV)
+      env.fetch('MOTHERSHIP_BIN') do
+        Gem.bin_path('guardship', 'mothership')
+      rescue Gem::Exception
+        'mothership'
+      end
+    end
+
+    # A private directory for the app socket: under $TMPDIR (Dir.tmpdir),
+    # unless that makes the socket path too long for a Unix socket.
+    def self.socket_dir(roots = [Dir.tmpdir, '/tmp'])
+      roots.uniq.each do |root|
+        dir = Dir.mktmpdir('sb', root)
+        socket = File.join(dir, 'app.sock')
+        return [dir, socket] if socket.bytesize <= SOCKET_PATH_MAX
+
+        FileUtils.rm_rf(dir)
+      rescue SystemCallError
+        next
+      end
+      raise Error, "no temp directory gives a Unix socket path under #{SOCKET_PATH_MAX} bytes; " \
+                   'set TMPDIR to a shorter one'
+    end
+
+    def self.threads(option, env = ENV)
+      value = option || env.fetch('MS_BAY_THREADS', '3')
+      count = Integer(value.to_s, 10)
+      raise ArgumentError, "Threads must be at least 1 (got #{value.inspect})" if count < 1
+
+      count
+    end
+
+    private
+
+    def url
+      "http://#{address}"
+    end
+
+    def address
+      "#{host.include?(':') ? "[#{host}]" : host}:#{port}"
+    end
+
+    def manifest_toml(socket)
+      <<~TOML
+        [mothership.bind]
+        http = #{address.dump}
+
+        [[mothership.upstreams]]
+        name = "app"
+        bind = #{"unix://#{socket}".dump}
+        protocol = "docking"
+        routes = [{ bind = "http", pattern = "/.*" }]
+      TOML
+    end
+
+    def spawn_mothership(manifest)
+      output = @silent ? File::NULL : $stdout
+      Process.spawn(@mothership, 'run', '-c', manifest, out: output, err: @silent ? File::NULL : $stderr)
+    rescue SystemCallError => e
+      raise Error, "cannot start mothership (#{@mothership}): #{e.message}. " \
+                   'Install the guardship gem or set MOTHERSHIP_BIN.'
+    end
+
+    def watch_mothership
+      pid = @mothership_pid
+      @watcher = Thread.new do
+        _, status = Process.wait2(pid)
+        next unless @mothership_pid
+
+        @mothership_pid = nil
+        @mothership_exit = status.exitstatus ? "exit #{status.exitstatus}" : "signal #{status.termsig}"
+        @logger.error('mothership exited', status: @mothership_exit)
+        stop
+      rescue Errno::ECHILD
+        nil
+      end
+    end
+
+    def stop_mothership
+      pid = @mothership_pid
+      @mothership_pid = nil
+      return unless pid
+
+      Process.kill('TERM', pid)
+      @watcher&.join(30) || Process.wait(pid)
+    rescue Errno::ESRCH, Errno::ECHILD
+      nil
+    end
+  end
+end

data/lib/shuttlebay/capybara.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'capybara'
+require_relative 'attached'
+
+Capybara.register_server(:shuttlebay) do |app, port, host, **options|
+  Shuttlebay::Attached.new(app, Host: host, Port: port, Silent: true, trap_signals: false, **options).run
+end

data/lib/shuttlebay/engine.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+begin
+  ruby_abi = RUBY_VERSION[/\A\d+\.\d+/]
+  require_relative "#{ruby_abi}/shuttlebay"
+rescue LoadError
+  begin
+    require_relative 'shuttlebay'
+  rescue LoadError => e
+    raise LoadError, "shuttlebay native engine is missing (#{e.message}). " \
+                     'Install a platform shuttlebay gem, or run `bundle exec rake compile` in a checkout.'
+  end
+end
+
+require 'tempfile'
+
+module Shuttlebay
+  module Engine
+    def self.spool_file
+      Tempfile.new('shuttlebay-body', binmode: true).tap(&:unlink)
+    end
+  end
+end

data/lib/shuttlebay/handler.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Shuttlebay
+  class Handler
+    BODYLESS_STATUS = ->(status) { status < 200 || status == 204 || status == 304 }
+
+    LINK_LOST = [IOError, Errno::EPIPE, Errno::ECONNRESET, Errno::ENOTCONN, Errno::EBADF].freeze
+
+    INTERNAL_ERROR_BODY = 'Internal Server Error'
+
+    def initialize(app, template:, logger: Shuttlebay.logger)
+      @app = app
+      @template = template
+      @logger = logger
+    end
+
+    def serve(io, codec, stopping: -> { false }, after_request: -> {})
+      until stopping.call
+        env = codec.read_request(io, @template)
+        break unless env
+
+        outcome = handle(io, codec, env)
+        return :detached if outcome == :detached
+        break unless outcome
+
+        run_after_request(after_request)
+        codec.ready(io)
+      end
+    end
+
+    def handle(io, codec, env)
+      env['rack.response_finished'] = finished = []
+      status = headers = body = error = nil
+      tunnel = offer_hijack(env)
+
+      begin
+        status, headers, body = @app.call(env)
+        return detach(io, codec, tunnel.tap { tunnel = nil }) if tunnel && env['rack.hijack_io']
+
+        status = Integer(status)
+      rescue StandardError, ScriptError => e
+        error = e
+        @logger.error('request failed', path: env['PATH_INFO'], **Logger.error_fields(e))
+        return internal_error(io, codec)
+      end
+
+      kept, error = respond(io, codec, env, status, headers, body)
+      kept
+    ensure
+      tunnel&.discard
+      close_body(body)
+      run_finished(finished, env, status, headers, error)
+      close_input(env)
+    end
+
+    private
+
+    def offer_hijack(env)
+      return unless env['HTTP_UPGRADE']
+
+      tunnel = Tunnel.new(logger: @logger)
+      env['rack.hijack?'] = true
+      env['rack.hijack'] = lambda do
+        env['rack.hijack_io'] = tunnel.app_io
+      end
+      tunnel
+    end
+
+    def detach(io, codec, tunnel)
+      Thread.new { tunnel.start(io, codec) }
+      :detached
+    end
+
+    def respond(io, codec, env, status, headers, body)
+      head_sent = false
+      kind, path = reply_kind(env, status, body)
+      chunks = body.to_ary if kind == :stream && body.respond_to?(:to_ary)
+
+      codec.reply(io, status, headers, kind, path)
+      head_sent = true
+      write_body(io, codec, body, chunks) if kind == :stream
+      codec.finish(io)
+      [true, nil]
+    rescue *LINK_LOST => e
+      @logger.debug('link lost mid-response', **Logger.error_fields(e))
+      [false, e]
+    rescue StandardError, ScriptError => e
+      @logger.error('response failed', path: env['PATH_INFO'], head_sent: head_sent, **Logger.error_fields(e))
+      [head_sent ? false : internal_error(io, codec), e]
+    end
+
+    def reply_kind(env, status, body)
+      return [:empty, nil] if env['REQUEST_METHOD'] == 'HEAD' || BODYLESS_STATUS.call(status)
+
+      if status == 200 && body.respond_to?(:to_path)
+        path = File.expand_path(body.to_path)
+        return [:file, path] if File.file?(path) && File.readable?(path)
+      end
+
+      [:stream, nil]
+    end
+
+    def write_body(io, codec, body, chunks)
+      if chunks
+        chunks.each { |chunk| codec.write(io, chunk) }
+      elsif body.respond_to?(:each)
+        body.each do |chunk|
+          codec.write(io, chunk)
+          codec.flush(io)
+        end
+      else
+        body.call(Stream.new(io, codec))
+      end
+    end
+
+    def internal_error(io, codec)
+      codec.reply(io, 500, { 'content-type' => 'text/plain' }, :stream)
+      codec.write(io, INTERNAL_ERROR_BODY)
+      codec.finish(io)
+      true
+    rescue *LINK_LOST, Engine::ProtocolError
+      false
+    end
+
+    def close_input(env)
+      input = env['rack.input']
+      input.close if input.respond_to?(:close)
+    rescue StandardError => e
+      @logger.error('rack.input close failed', **Logger.error_fields(e))
+    end
+
+    def close_body(body)
+      body.close if body.respond_to?(:close)
+    rescue StandardError => e
+      @logger.error('body close failed', **Logger.error_fields(e))
+    end
+
+    def run_after_request(hook)
+      hook.call
+    rescue StandardError => e
+      @logger.error('after-request hook failed', **Logger.error_fields(e))
+    end
+
+    def run_finished(callbacks, env, status, headers, error)
+      callbacks.reverse_each do |callback|
+        callback.call(env, status, headers, error)
+      rescue StandardError => e
+        @logger.error('rack.response_finished callback failed', **Logger.error_fields(e))
+      end
+    end
+  end
+end

data/lib/shuttlebay/logger.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Shuttlebay
+  class Logger
+    LEVELS = %i[debug info warn error].freeze
+
+    def initialize(io = $stdout, ship: ENV.fetch('MS_SHIP', 'rack'))
+      @io = io
+      @io.sync = true
+      @ship = ship
+    end
+
+    LEVELS.each do |level|
+      define_method(level) { |msg, **fields| log(level, msg, **fields) }
+    end
+
+    def log(level, msg, **fields)
+      line = JSON.generate({ level: level, msg: msg, ship: @ship, pid: Process.pid, **fields })
+      @io.write("#{line}\n")
+    end
+
+    def self.error_fields(error)
+      { error: error.class.name, message: error.message, backtrace: error.backtrace.first(10) }
+    end
+  end
+end

data/lib/shuttlebay/process_control.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Shuttlebay
+  module ProcessControl
+    private
+
+    def trap_signals(signals, wake_w)
+      signals.to_h do |signal, code|
+        [signal, trap(signal) { wake_w.write_nonblock(code, exception: false) }]
+      end
+    end
+
+    def close_self_pipe(previous, wake_r, wake_w)
+      previous&.each { |signal, handler| trap(signal, handler || 'DEFAULT') }
+      [wake_r, wake_w].compact.each(&:close)
+    end
+
+    def run_worker_process(index, signals)
+      signals.each_key { |signal| trap(signal, 'DEFAULT') }
+      yield.run
+      $stdout.flush
+      exit!(0)
+    rescue StandardError, ScriptError => e
+      @logger.error('worker crashed', index: index, **Logger.error_fields(e))
+      $stdout.flush
+      exit!(1)
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/shuttlebay/railtie.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'rack/sendfile'
+
+module Shuttlebay
+  class Railtie < ::Rails::Railtie
+    initializer 'shuttlebay.mothership_serves_files' do |app|
+      app.config.middleware.delete(::Rack::Sendfile) if Shuttlebay.running?
+    end
+  end
+end

data/lib/shuttlebay/runtime.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require_relative 'version'
+require_relative 'engine'
+require_relative 'logger'
+
+module Shuttlebay
+  class Error < StandardError; end
+
+  HOOKS = %i[before_fork worker_boot worker_shutdown out_of_band].freeze
+
+  class << self
+    attr_accessor :config
+
+    attr_writer :running, :logger
+
+    def running?
+      @running == true
+    end
+
+    def logger
+      @logger ||= Logger.new
+    end
+
+    def before_fork(&block)
+      register_hook(:before_fork, block)
+    end
+
+    def on_worker_boot(&block)
+      register_hook(:worker_boot, block)
+    end
+
+    def on_worker_shutdown(&block)
+      register_hook(:worker_shutdown, block)
+    end
+
+    def out_of_band(&block)
+      register_hook(:out_of_band, block)
+    end
+
+    attr_reader :fiber_scheduler_factory, :fiber_reactors
+
+    def fiber_scheduler(reactors: 1, &factory)
+      raise ArgumentError, 'Shuttlebay.fiber_scheduler needs a block that builds a Fiber scheduler' unless factory
+
+      count = Integer(reactors)
+      raise ArgumentError, "Shuttlebay.fiber_scheduler needs reactors >= 1 (got #{reactors.inspect})" if count < 1
+
+      @fiber_reactors = count
+      @fiber_scheduler_factory = factory
+    end
+
+    def run_hooks(name, *)
+      hooks.fetch(name).each { |hook| hook.call(*) }
+    end
+
+    def clear_hooks!
+      HOOKS.each { |name| hooks[name].clear }
+      @fiber_scheduler_factory = @fiber_reactors = nil
+    end
+
+    private
+
+    def hooks
+      @hooks ||= HOOKS.to_h { |name| [name, []] }
+    end
+
+    def register_hook(name, block)
+      raise ArgumentError, "Shuttlebay.#{name} needs a block" unless block
+
+      hooks.fetch(name) << block
+      block
+    end
+  end
+end
+
+require_relative 'stream'
+require_relative 'tunnel'
+require_relative 'handler'
+require_relative 'process_control'
+require_relative 'worker'
+require_relative 'server'