shuttlebay 0.1.4-aarch64-linux

data/lib/shuttlebay/runtime.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require_relative 'version'
+require_relative 'engine'
+require_relative 'logger'
+
+module Shuttlebay
+  class Error < StandardError; end
+
+  HOOKS = %i[before_fork worker_boot worker_shutdown out_of_band].freeze
+
+  class << self
+    attr_accessor :config
+
+    attr_writer :running, :logger
+
+    def running?
+      @running == true
+    end
+
+    def logger
+      @logger ||= Logger.new
+    end
+
+    def before_fork(&block)
+      register_hook(:before_fork, block)
+    end
+
+    def on_worker_boot(&block)
+      register_hook(:worker_boot, block)
+    end
+
+    def on_worker_shutdown(&block)
+      register_hook(:worker_shutdown, block)
+    end
+
+    def out_of_band(&block)
+      register_hook(:out_of_band, block)
+    end
+
+    attr_reader :fiber_scheduler_factory, :fiber_reactors
+
+    def fiber_scheduler(reactors: 1, &factory)
+      raise ArgumentError, 'Shuttlebay.fiber_scheduler needs a block that builds a Fiber scheduler' unless factory
+
+      count = Integer(reactors)
+      raise ArgumentError, "Shuttlebay.fiber_scheduler needs reactors >= 1 (got #{reactors.inspect})" if count < 1
+
+      @fiber_reactors = count
+      @fiber_scheduler_factory = factory
+    end
+
+    def run_hooks(name, *)
+      hooks.fetch(name).each { |hook| hook.call(*) }
+    end
+
+    def clear_hooks!
+      HOOKS.each { |name| hooks[name].clear }
+      @fiber_scheduler_factory = @fiber_reactors = nil
+    end
+
+    private
+
+    def hooks
+      @hooks ||= HOOKS.to_h { |name| [name, []] }
+    end
+
+    def register_hook(name, block)
+      raise ArgumentError, "Shuttlebay.#{name} needs a block" unless block
+
+      hooks.fetch(name) << block
+      block
+    end
+  end
+end
+
+require_relative 'stream'
+require_relative 'tunnel'
+require_relative 'handler'
+require_relative 'process_control'
+require_relative 'worker'
+require_relative 'server'

data/lib/shuttlebay/server.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+module Shuttlebay
+  class Server
+    include ProcessControl
+
+    RESPAWN_MIN = 0.1
+    RESPAWN_MAX = 5.0
+    STABLE_AFTER = 30.0
+    SIGNALS = { 'TERM' => 'T', 'INT' => 'I', 'QUIT' => 'Q', 'CHLD' => 'C' }.freeze
+
+    Slot = Struct.new(:index, :pid, :started_at, :backoff, :respawn_at)
+
+    attr_reader :app_path, :socket_path, :workers, :threads, :ship
+
+    def self.from_env(argv, env = ENV)
+      socket_path = env.fetch('MS_SOCKET_PATH') do
+        raise ArgumentError, 'MS_SOCKET_PATH is not set: shuttlebay runs as a mothership [[bays.http]] bay'
+      end
+      new(
+        app_path: argv.first || 'config.ru',
+        socket_path: socket_path,
+        workers: Integer(env.fetch('MS_BAY_WORKERS', '0'), 10),
+        threads: Integer(env.fetch('MS_BAY_THREADS', '3'), 10),
+        ship: env.fetch('MS_SHIP', 'rack')
+      )
+    end
+
+    def initialize(app_path:, socket_path:, workers:, threads:, ship:, logger: Shuttlebay.logger)
+      raise ArgumentError, "workers must be >= 0 (got #{workers})" if workers.negative?
+      raise ArgumentError, "threads must be >= 1 (got #{threads})" if threads < 1
+
+      @app_path = app_path
+      @socket_path = socket_path
+      @workers = workers
+      @threads = threads
+      @ship = ship
+      @logger = logger
+      @master_pid = Process.pid
+    end
+
+    def run
+      Shuttlebay.running = true
+      app = load_app
+      @listener = bind
+      @logger.info('app loaded, socket bound', app: app_path, socket: socket_path,
+                                               workers: workers, threads: threads)
+
+      if workers.zero?
+        worker(app, index: nil).run
+      else
+        supervise(app)
+      end
+    ensure
+      cleanup if Process.pid == @master_pid
+    end
+
+    private
+
+    def load_app
+      require 'rack'
+      require 'rack/builder'
+      path = File.expand_path(app_path)
+      raise ArgumentError, "Rack app not found: #{path}" unless File.file?(path)
+
+      Rack::Builder.parse_file(path)
+    end
+
+    def bind
+      if File.exist?(socket_path)
+        raise ArgumentError, "#{socket_path} exists and is not a socket" unless File.socket?(socket_path)
+
+        File.unlink(socket_path)
+      end
+      UNIXServer.new(socket_path).tap { |server| server.listen(1024) }
+    end
+
+    def worker(app, index:)
+      Worker.new(app: app, listener: @listener, threads: threads, workers: workers, ship: ship,
+                 index: index, master_pid: index && @master_pid, logger: @logger)
+    end
+
+    def supervise(app)
+      wake_r, wake_w = IO.pipe
+      previous = trap_signals(SIGNALS, wake_w)
+      @app = app
+      Shuttlebay.run_hooks(:before_fork)
+      @slots = Array.new(workers) { |index| Slot.new(index, nil, nil, RESPAWN_MIN, monotonic) }
+      @stopping = false
+
+      until @stopping && @slots.none?(&:pid)
+        spawn_due unless @stopping
+        reap
+        wait_for_signal(wake_r)
+      end
+      @logger.info('all workers stopped')
+    ensure
+      close_self_pipe(previous, wake_r, wake_w)
+    end
+
+    def wait_for_signal(wake_r)
+      ready, = IO.select([wake_r], nil, nil, next_timeout)
+      codes = ready ? wake_r.read_nonblock(1024, exception: false) : nil
+      stop_code = codes.delete('C')[0] if codes.is_a?(String)
+      begin_stop(SIGNALS.key(stop_code)) if stop_code
+      stop_overdue if @stopping
+    end
+
+    def next_timeout
+      return 0.2 if @stopping
+
+      due = @slots.reject(&:pid).map(&:respawn_at).min
+      due ? [[due - monotonic, 0.0].max, 1.0].min : 1.0
+    end
+
+    def spawn_due
+      now = monotonic
+      @slots.each do |slot|
+        next if slot.pid || slot.respawn_at > now
+
+        slot.pid = spawn_worker(slot.index)
+        slot.started_at = now
+      end
+    end
+
+    def spawn_worker(index)
+      app = @app
+      fork { run_worker_process(index, SIGNALS) { worker(app, index: index) } }
+    end
+
+    def reap
+      loop do
+        pid, status = Process.wait2(-1, Process::WNOHANG)
+        break unless pid
+
+        exited(pid, status)
+      end
+    rescue Errno::ECHILD
+      @slots.each { |slot| slot.pid = nil }
+    end
+
+    def exited(pid, status)
+      slot = @slots.find { |candidate| candidate.pid == pid }
+      return unless slot
+
+      uptime = monotonic - slot.started_at
+      slot.pid = nil
+      slot.backoff = RESPAWN_MIN if uptime >= STABLE_AFTER
+      delay = slot.backoff
+      slot.backoff = [slot.backoff * 2, RESPAWN_MAX].min
+      slot.respawn_at = monotonic + delay
+      return if @stopping
+
+      @logger.warn('worker exited, respawning', index: slot.index, pid: pid,
+                                                status: status && (status.exitstatus || status.termsig),
+                                                uptime: uptime.round(3), respawn_in: delay)
+    end
+
+    def begin_stop(signal)
+      return if @stopping
+
+      @stopping = true
+      @stop_deadline = monotonic + Worker.stop_grace + 0.5
+      @logger.info('stopping workers', signal: signal)
+      signal_workers('TERM')
+    end
+
+    def stop_overdue
+      return unless monotonic >= @stop_deadline && running_pids.any?
+
+      @logger.warn('workers did not stop in time, killing', pids: running_pids)
+      signal_workers('KILL')
+      @stop_deadline = Float::INFINITY
+    end
+
+    def running_pids
+      @slots.filter_map(&:pid)
+    end
+
+    def signal_workers(signal)
+      running_pids.each { |pid| send_signal(pid, signal) }
+    end
+
+    def send_signal(pid, signal)
+      Process.kill(signal, pid)
+    rescue Errno::ESRCH
+      nil
+    end
+
+    def cleanup
+      @listener&.close
+      File.unlink(socket_path) if socket_path && File.socket?(socket_path)
+    rescue SystemCallError => e
+      @logger.warn('socket cleanup failed', socket: socket_path, error: e.message)
+    end
+  end
+end

data/lib/shuttlebay/stream.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Shuttlebay
+  class Stream
+    def initialize(io, codec)
+      @io = io
+      @codec = codec
+      @write_closed = false
+      @read_closed = false
+    end
+
+    def write(data)
+      raise IOError, 'closed stream' if @write_closed
+
+      data = String(data)
+      @codec.write(@io, data)
+      @codec.flush(@io)
+      data.bytesize
+    end
+
+    def <<(data)
+      write(data)
+      self
+    end
+
+    def flush
+      @codec.flush(@io) unless @write_closed
+      self
+    end
+
+    def read(_length = nil, _buffer = nil)
+      raise IOError, 'closed stream' if @read_closed
+
+      nil
+    end
+
+    def close_read
+      @read_closed = true
+      nil
+    end
+
+    def close_write
+      @write_closed = true
+      nil
+    end
+
+    def close
+      close_read
+      close_write
+    end
+
+    def closed?
+      @read_closed && @write_closed
+    end
+  end
+end

data/lib/shuttlebay/tunnel.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+module Shuttlebay
+  class Tunnel
+    MAX_HEAD = 64 * 1024
+    HEAD_TIMEOUT = 10
+    READ_CHUNK = 16 * 1024
+    HEAD_END = /\r?\n\r?\n/
+
+    class InvalidHead < StandardError; end
+
+    attr_reader :app_io
+
+    def initialize(logger: Shuttlebay.logger)
+      @app_io, @engine_io = UNIXSocket.pair
+      @logger = logger
+    end
+
+    def discard
+      [@app_io, @engine_io].each(&:close)
+    end
+
+    def start(io, codec)
+      status, headers, rest = read_head
+      codec.reply(io, status, headers, :tunnel)
+      codec.flush(io)
+      codec.tunnel_write(io, rest) unless rest.empty?
+      relay(io, codec)
+    rescue InvalidHead, *Handler::LINK_LOST, Engine::ProtocolError => e
+      @logger.warn('hijacked connection ended before its tunnel opened', **Logger.error_fields(e))
+      close(io)
+    end
+
+    private
+
+    def relay(io, codec)
+      to_client = Thread.new do
+        begin
+          loop { codec.tunnel_write(io, @engine_io.readpartial(READ_CHUNK)) }
+        rescue EOFError
+          codec.tunnel_close(io)
+        end
+      rescue *Handler::LINK_LOST, Engine::ProtocolError
+        nil
+      end
+      Thread.new do
+        while (data = codec.tunnel_read(io))
+          @engine_io.write(data)
+        end
+        @engine_io.close_write
+      rescue *Handler::LINK_LOST, Engine::ProtocolError
+        nil
+      ensure
+        to_client.join(1)
+        close(io)
+      end
+    end
+
+    def read_head
+      buffer = +''.b
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + HEAD_TIMEOUT
+      until (split = buffer =~ HEAD_END)
+        raise InvalidHead, "no response head within #{MAX_HEAD} bytes" if buffer.bytesize > MAX_HEAD
+
+        left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        raise InvalidHead, "app wrote no response head within #{HEAD_TIMEOUT}s" unless left.positive? &&
+                                                                                       @engine_io.wait_readable(left)
+
+        buffer << @engine_io.readpartial(READ_CHUNK)
+      end
+      [*parse_head(buffer.byteslice(0, split)), buffer.byteslice((split + Regexp.last_match(0).bytesize)..)]
+    rescue EOFError
+      raise InvalidHead, 'app closed the hijacked connection without a response head'
+    end
+
+    def parse_head(head)
+      status_line, *lines = head.split(/\r?\n/)
+      status = status_line[%r{\AHTTP/1\.[01] (\d{3})}, 1] or raise InvalidHead, "bad status line #{status_line.inspect}"
+      headers = Hash.new { |hash, key| hash[key] = [] }
+      lines.each do |line|
+        name, value = line.split(':', 2)
+        raise InvalidHead, "bad header line #{line.inspect}" unless value
+
+        headers[name.strip.downcase] << value.strip
+      end
+      [Integer(status, 10), headers.to_h]
+    end
+
+    def close(io)
+      [@engine_io, io].each(&:close)
+    end
+  end
+end

data/lib/shuttlebay/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Shuttlebay
+  VERSION = '0.1.4'
+end

data/lib/shuttlebay/worker.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Shuttlebay
+  class Worker
+    include ProcessControl
+
+    DEFAULT_STOP_GRACE = 4.0
+    SIGNALS = { 'TERM' => '.', 'INT' => '.', 'QUIT' => '.' }.freeze
+
+    def self.stop_grace(env = ENV)
+      Float(env.fetch('MS_BAY_DRAIN_TIMEOUT', DEFAULT_STOP_GRACE))
+    end
+
+    def initialize(app:, listener:, threads:, workers:, ship:, index: nil, master_pid: nil,
+                   logger: Shuttlebay.logger)
+      @app = app
+      @listener = listener
+      @threads = threads
+      @workers = workers
+      @ship = ship
+      @index = index
+      @master_pid = master_pid
+      @logger = logger
+      @handler = Handler.new(app, template: Worker.template(threads: threads, workers: workers), logger: logger)
+      @links = {}
+      @links_lock = Mutex.new
+      @stopping = false
+    end
+
+    def self.template(threads:, workers:)
+      {
+        'SCRIPT_NAME' => '',
+        'rack.errors' => $stderr,
+        'rack.multithread' => threads > 1,
+        'rack.multiprocess' => workers > 1,
+        'rack.run_once' => false,
+        'rack.hijack?' => false
+      }.freeze
+    end
+
+    def run(trap_signals: true)
+      @wake_r, @wake_w = IO.pipe
+      previous = trap_signals(SIGNALS, @wake_w) if trap_signals
+      Shuttlebay.run_hooks(:worker_boot, @index) if forked?
+      pool = spawn_pool
+      @logger.info('worker serving', index: @index, threads: @threads, reactors: @reactors)
+
+      wait_for_stop
+      stop(pool)
+      raise @reactor_error if @reactor_error
+    ensure
+      close_self_pipe(previous, @wake_r, @wake_w)
+    end
+
+    def stop!
+      @wake_w&.write_nonblock('.', exception: false)
+    rescue IOError
+      nil
+    end
+
+    private
+
+    def forked?
+      !@master_pid.nil?
+    end
+
+    def spawn_pool
+      factory = Shuttlebay.fiber_scheduler_factory
+      return Array.new(@threads) { |slot| Thread.new { accept_loop(slot) } } unless factory
+
+      reactors = (0...@threads).group_by { |slot| slot % Shuttlebay.fiber_reactors }.values
+      @reactors = reactors.size
+      reactors.map { |slots| Thread.new { run_fibers(factory, slots) } }
+    end
+
+    def run_fibers(factory, slots)
+      Fiber.set_scheduler(factory.call)
+      slots.each { |slot| Fiber.schedule { accept_loop(slot) } }
+      Fiber.set_scheduler(nil)
+    rescue StandardError, ScriptError => e
+      @reactor_error = e
+      stop!
+    end
+
+    def accept_loop(slot)
+      until @stopping
+        io = @listener.accept
+        serve_link(io)
+      end
+    rescue IOError, Errno::EBADF, Errno::EINVAL
+      nil
+    rescue StandardError => e
+      @logger.error('accept loop crashed', slot: slot, **Logger.error_fields(e))
+      return if @stopping
+
+      sleep 0.1
+      retry
+    end
+
+    LINK_BUFFER = 512 * 1024
+
+    def serve_link(io)
+      widen_buffers(io)
+      codec = Engine::Codec.new
+      @links_lock.synchronize { @links[io] = codec }
+      config = codec.dock(io, @ship, @threads)
+      Shuttlebay.config ||= config
+      outcome = @handler.serve(io, codec, stopping: -> { @stopping }, after_request: -> { out_of_band(codec) })
+    rescue Engine::ProtocolError => e
+      @logger.warn('docking link protocol error', **Logger.error_fields(e))
+    rescue *Handler::LINK_LOST
+      nil
+    ensure
+      @links_lock.synchronize { @links.delete(io) }
+      io.close unless outcome == :detached
+    end
+
+    def widen_buffers(io)
+      io.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDBUF, LINK_BUFFER)
+      io.setsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF, LINK_BUFFER)
+    end
+
+    def wait_for_stop
+      loop do
+        return if @wake_r.wait_readable(1.0)
+        return @logger.warn('master is gone, stopping worker', index: @index) if orphaned?
+      end
+    end
+
+    def orphaned?
+      forked? && Process.ppid != @master_pid
+    end
+
+    def stop(pool)
+      @stopping = true
+      @listener.close
+      close_idle_links
+      deadline = monotonic + Worker.stop_grace
+      pool.each do |thread|
+        thread.join([deadline - monotonic, 0].max)
+      end
+      busy = @links_lock.synchronize { @links.count { |_io, codec| codec.request_id } }
+      @logger.warn('stopping with requests still running', busy: busy) if busy.positive?
+      run_shutdown_hooks
+      @logger.info('worker stopped', index: @index)
+    end
+
+    def out_of_band(codec)
+      others_busy = @links_lock.synchronize do
+        @links.any? { |_io, other| !other.equal?(codec) && !other.request_id.nil? }
+      end
+      Shuttlebay.run_hooks(:out_of_band) unless others_busy
+    end
+
+    def close_idle_links
+      idle = @links_lock.synchronize { @links.select { |_io, codec| codec.request_id.nil? }.keys }
+      idle.each(&:close)
+    end
+
+    def run_shutdown_hooks
+      Shuttlebay.run_hooks(:worker_shutdown, @index)
+    rescue StandardError => e
+      @logger.error('on_worker_shutdown hook failed', **Logger.error_fields(e))
+    end
+  end
+end

data/lib/shuttlebay.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative 'shuttlebay/runtime'
+require_relative 'shuttlebay/railtie' if defined?(Rails::Railtie)