httpx 0.0.1

data/lib/httpx/channel.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "httpx/io"
+require "httpx/buffer"
+
+module HTTPX
+  # The Channel entity can be watched for IO events.
+  #
+  # It contains the +io+ object to read/write from, and knows what to do when it can.
+  #
+  # It defers connecting until absolutely necessary. Connection should be triggered from
+  # the IO selector (until then, any request will be queued).
+  #
+  # A channel boots up its parser after connection is established. All pending requests
+  # will be redirected there after connection.
+  #
+  # A channel can be prevented from closing by the parser, that is, if there are pending
+  # requests. This will signal that the channel was prematurely closed, due to a possible
+  # number of conditions:
+  #
+  # * Remote peer closed the connection ("Connection: close");
+  # * Remote peer doesn't support pipelining;
+  #
+  # A channel may also route requests for a different host for which the +io+ was connected
+  # to, provided that the IP is the same and the port and scheme as well. This will allow to
+  # share the same socket to send HTTP/2 requests to different hosts.
+  # TODO: For this to succeed, the certificates sent by the servers to the client must be
+  #       identical (or match both hosts).
+  #
+  class Channel
+    extend Forwardable
+    include Registry
+    include Loggable
+    include Callbacks
+
+    require "httpx/channel/http2"
+    require "httpx/channel/http1"
+
+    BUFFER_SIZE = 1 << 14
+
+    class << self
+      def by(uri, options)
+        io = case uri.scheme
+             when "http"
+               IO.registry("tcp").new(uri.host, uri.port, options)
+             when "https"
+               IO.registry("ssl").new(uri.host, uri.port, options)
+             else
+               raise Error, "#{uri}: #{uri.scheme}: unrecognized channel"
+        end
+        new(io, options)
+      end
+    end
+
+    def_delegator :@io, :closed?
+
+    def_delegator :@write_buffer, :empty?
+
+    def initialize(io, options)
+      @io = io
+      @options = Options.new(options)
+      @window_size = @options.window_size
+      @read_buffer = Buffer.new(BUFFER_SIZE)
+      @write_buffer = Buffer.new(BUFFER_SIZE)
+      @pending = []
+      @state = :idle
+    end
+
+    def match?(uri)
+      ip = begin
+             TCPSocket.getaddress(uri.host)
+           rescue StandardError
+             uri.host
+           end
+
+      ip == @io.ip &&
+        uri.port == @io.port &&
+        uri.scheme == @io.scheme
+    end
+
+    def interests
+      return :w if @state == :idle
+      readable = !@read_buffer.full?
+      writable = !@write_buffer.empty?
+      if readable
+        writable ? :rw : :r
+      else
+        writable ? :w : :r
+      end
+    end
+
+    def to_io
+      case @state
+      when :idle
+        transition(:open)
+      end
+      @io.to_io
+    end
+
+    def close(hard = false)
+      pr = @parser
+      transition(:closing)
+      if hard || (pr && pr.empty?)
+        pr.close
+        @parser = nil
+      else
+        transition(:idle)
+        @parser = pr
+        parser.reenqueue!
+        return
+      end
+    end
+
+    def reset
+      transition(:closing)
+      transition(:closed)
+      emit(:close)
+    end
+
+    def send(request, **args)
+      if @parser && !@write_buffer.full?
+        parser.send(request, **args)
+      else
+        @pending << [request, args]
+      end
+    end
+
+    def call
+      case @state
+      when :closed
+        return
+      when :closing
+        dwrite
+        transition(:closed)
+        emit(:close)
+      else
+        catch(:called) do
+          dread
+          dwrite
+          parser.consume
+        end
+      end
+      nil
+    end
+
+    def upgrade_parser(protocol)
+      @parser.reset if @parser
+      @parser = build_parser(protocol)
+    end
+
+    private
+
+    def dread(wsize = @window_size)
+      loop do
+        siz = @io.read(wsize, @read_buffer)
+        throw(:close, self) unless siz
+        return if siz.zero?
+        log { "READ: #{siz} bytes..." }
+        parser << @read_buffer.to_s
+      end
+    end
+
+    def dwrite
+      loop do
+        return if @write_buffer.empty?
+        siz = @io.write(@write_buffer)
+        throw(:close, self) unless siz
+        log { "WRITE: #{siz} bytes..." }
+        return if siz.zero?
+      end
+    end
+
+    def send_pending
+      while !@write_buffer.full? && (req_args = @pending.shift)
+        request, args = req_args
+        parser.send(request, **args)
+      end
+    end
+
+    def parser
+      @parser ||= build_parser
+    end
+
+    def build_parser(protocol = @io.protocol)
+      parser = registry(protocol).new(@write_buffer, @options)
+      parser.on(:response) do |*args|
+        emit(:response, *args)
+      end
+      parser.on(:promise) do |*args|
+        emit(:promise, *args)
+      end
+      # parser.inherit_callbacks(self)
+      parser.on(:complete) { throw(:close, self) }
+      parser.on(:close) do
+        transition(:closed)
+        emit(:close)
+      end
+      parser
+    end
+
+    def transition(nextstate)
+      case nextstate
+      # when :idle
+
+      when :open
+        return if @state == :closed
+        @io.connect
+        return if @io.closed?
+        send_pending
+      when :closing
+        return unless @state == :open
+      when :closed
+        return unless @state == :closing
+        return unless @write_buffer.empty?
+        @io.close
+        @read_buffer.clear
+      end
+      @state = nextstate
+    end
+  end
+end

data/lib/httpx/channel/http1.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+require "http_parser"
+
+module HTTPX
+  class Channel::HTTP1
+    include Callbacks
+    include Loggable
+
+    CRLF = "\r\n"
+
+    def initialize(buffer, options)
+      @options = Options.new(options)
+      @max_concurrent_requests = @options.max_concurrent_requests
+      @retries = options.max_retries
+      @parser = HTTP::Parser.new(self)
+      @parser.header_value_type = :arrays
+      @buffer = buffer
+      @version = [1, 1]
+      @pending = []
+      @requests = []
+      @has_response = false
+    end
+
+    def reset
+      @parser.reset!
+      @has_response = false
+    end
+
+    def close
+      reset
+      emit(:close)
+    end
+
+    def empty?
+      # this means that for every request there's an available
+      # partial response, so there are no in-flight requests waiting.
+      @requests.empty? || @requests.all? { |request| !request.response.nil? }
+    end
+
+    def <<(data)
+      @parser << data
+      dispatch if @has_response
+    end
+
+    def send(request, **)
+      if @requests.size >= @max_concurrent_requests
+        @pending << request
+        return
+      end
+      @requests << request unless @requests.include?(request)
+      handle(request)
+    end
+
+    def reenqueue!
+      requests = @requests.dup
+      @requests.clear
+      requests.each do |request|
+        send(request)
+      end
+    end
+
+    def consume
+      @requests.each do |request|
+        handle(request)
+      end
+    end
+
+    # HTTP Parser callbacks
+    #
+    # must be public methods, or else they won't be reachable
+
+    def on_message_begin
+      log(2) { "parsing begins" }
+    end
+
+    def on_headers_complete(h)
+      return on_trailer_headers_complete(h) if @parser_trailers
+      # Wait for fix: https://github.com/tmm1/http_parser.rb/issues/52
+      # callback is called 2 times when chunked
+      request = @requests.first
+      return if request.response
+
+      log(2) { "headers received" }
+      headers = @options.headers_class.new(h)
+      response = @options.response_class.new(@requests.last,
+                                             @parser.status_code,
+                                             @parser.http_version.join("."),
+                                             headers, @options)
+      log { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
+      log { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
+
+      request.response = response
+
+      @has_response = true if response.complete?
+    end
+
+    def on_body(chunk)
+      log { "-> DATA: #{chunk.bytesize} bytes..." }
+      log(2) { "-> #{chunk.inspect}" }
+      response = @requests.first.response
+
+      response << chunk
+
+      dispatch if response.complete?
+    end
+
+    def on_message_complete
+      log(2) { "parsing complete" }
+      request = @requests.first
+      response = request.response
+
+      if !@parser_trailers && response.headers.key?("trailer")
+        @parser_trailers = true
+        # this is needed, because the parser can't accept further headers.
+        # we need to reset it and artificially move it to receive headers state,
+        # hence the bogus headline
+        #
+        @parser.reset!
+        @parser << "#{request.verb.to_s.upcase} #{request.path} HTTP/#{response.version}#{CRLF}"
+      else
+        @has_response = true
+      end
+    end
+
+    def on_trailer_headers_complete(h)
+      response = @requests.first.response
+
+      response.merge_headers(h)
+    end
+
+    def dispatch
+      request = @requests.first
+      return handle(request) if request.expects?
+
+      @requests.shift
+      response = request.response
+      emit(:response, request, response)
+
+      if @parser.upgrade?
+        response << @parser.upgrade_data
+        throw(:called)
+      end
+      close
+      send(@pending.shift) unless @pending.empty?
+      return unless response.headers["connection"] == "close"
+      unless @requests.empty?
+        @requests.map { |r| r.transition(:idle) }
+        # server doesn't handle pipelining, and probably
+        # doesn't support keep-alive. Fallback to send only
+        # 1 keep alive request.
+        @max_concurrent_requests = 1
+      end
+      emit(:complete)
+    end
+
+    private
+
+    def set_request_headers(request)
+      request.headers["host"] ||= request.authority
+      request.headers["connection"] ||= "keep-alive"
+      if !request.headers.key?("content-length") &&
+         request.body.bytesize == Float::INFINITY
+        request.chunk!
+      end
+    end
+
+    def headline_uri(request)
+      request.path
+    end
+
+    def handle(request)
+      @has_response = false
+      set_request_headers(request)
+      catch(:buffer_full) do
+        request.transition(:headers)
+        join_headers(request) if request.state == :headers
+        request.transition(:body)
+        join_body(request) if request.state == :body
+        request.transition(:done)
+      end
+    end
+
+    def join_headers(request)
+      buffer = +""
+      buffer << "#{request.verb.to_s.upcase} #{headline_uri(request)} HTTP/#{@version.join(".")}" << CRLF
+      log { "<- HEADLINE: #{buffer.chomp.inspect}" }
+      @buffer << buffer
+      buffer.clear
+      request.headers.each do |field, value|
+        buffer << "#{capitalized(field)}: #{value}" << CRLF
+        log { "<- HEADER: #{buffer.chomp}" }
+        @buffer << buffer
+        buffer.clear
+      end
+      log { "<- " }
+      @buffer << CRLF
+    end
+
+    def join_body(request)
+      return if request.empty?
+      while (chunk = request.drain_body)
+        log { "<- DATA: #{chunk.bytesize} bytes..." }
+        log(2) { "<- #{chunk.inspect}" }
+        @buffer << chunk
+        throw(:buffer_full, request) if @buffer.full?
+      end
+    end
+
+    UPCASED = {
+      "www-authenticate" => "WWW-Authenticate",
+      "http2-settings" => "HTTP2-Settings",
+    }.freeze
+
+    def capitalized(field)
+      UPCASED[field] || field.to_s.split("-").map(&:capitalize).join("-")
+    end
+  end
+  Channel.register "http/1.1", Channel::HTTP1
+end

data/lib/httpx/channel/http2.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require "http/2"
+
+module HTTPX
+  class Channel::HTTP2
+    include Callbacks
+    include Loggable
+
+    attr_reader :streams, :pending
+
+    def initialize(buffer, options)
+      @options = Options.new(options)
+      @max_concurrent_requests = @options.max_concurrent_requests
+      init_connection
+      @retries = options.max_retries
+      @pending = []
+      @streams = {}
+      @drains  = {}
+      @buffer = buffer
+    end
+
+    def close
+      @connection.goaway
+    end
+
+    def empty?
+      @connection.state == :closed || @streams.empty?
+    end
+
+    def <<(data)
+      @connection << data
+    end
+
+    def send(request, **)
+      if @connection.active_stream_count >= @max_concurrent_requests
+        @pending << request
+        return
+      end
+      unless (stream = @streams[request])
+        stream = @connection.new_stream
+        handle_stream(stream, request)
+        @streams[request] = stream
+      end
+      handle(request, stream)
+    end
+
+    def reenqueue!
+      requests = @streams.keys
+      @streams.clear
+      init_connection
+      requests.each do |request|
+        send(request)
+      end
+    end
+
+    def consume
+      @streams.each do |request, stream|
+        handle(request, stream)
+      end
+    end
+
+    private
+
+    def headline_uri(request)
+      request.path
+    end
+
+    def set_request_headers(request); end
+
+    def handle(request, stream)
+      catch(:buffer_full) do
+        request.transition(:headers)
+        join_headers(stream, request) if request.state == :headers
+        request.transition(:body)
+        join_body(stream, request) if request.state == :body
+        request.transition(:done)
+      end
+    end
+
+    def init_connection
+      @connection = HTTP2::Client.new(@options.http2_settings)
+      @connection.on(:frame, &method(:on_frame))
+      @connection.on(:frame_sent, &method(:on_frame_sent))
+      @connection.on(:frame_received, &method(:on_frame_received))
+      @connection.on(:promise, &method(:on_promise))
+      @connection.on(:altsvc, &method(:on_altsvc))
+      @connection.on(:settings_ack, &method(:on_settings))
+      @connection.on(:goaway, &method(:on_close))
+    end
+
+    def handle_stream(stream, request)
+      stream.on(:close, &method(:on_stream_close).curry[stream, request])
+      stream.on(:half_close) do
+        log(2, "#{stream.id}: ") { "waiting for response..." }
+      end
+      # stream.on(:altsvc)
+      stream.on(:headers, &method(:on_stream_headers).curry[stream, request])
+      stream.on(:data, &method(:on_stream_data).curry[stream, request])
+    end
+
+    def join_headers(stream, request)
+      set_request_headers(request)
+      headers = {}
+      headers[":scheme"]    = request.scheme
+      headers[":method"]    = request.verb.to_s.upcase
+      headers[":path"]      = headline_uri(request)
+      headers[":authority"] = request.authority
+      headers = headers.merge(request.headers)
+      log(1, "#{stream.id}: ") do
+        headers.map { |k, v| "-> HEADER: #{k}: #{v}" }.join("\n")
+      end
+      stream.headers(headers, end_stream: request.empty?)
+    end
+
+    def join_body(stream, request)
+      chunk = @drains.delete(request) || request.drain_body
+      while chunk
+        next_chunk = request.drain_body
+        log(1, "#{stream.id}: ") { "-> DATA: #{chunk.bytesize} bytes..." }
+        log(2, "#{stream.id}: ") { "-> #{chunk.inspect}" }
+        stream.data(chunk, end_stream: !next_chunk)
+        if next_chunk && @buffer.full?
+          @drains[request] = next_chunk
+          throw(:buffer_full)
+        end
+        chunk = next_chunk
+      end
+    end
+
+    ######
+    # HTTP/2 Callbacks
+    ######
+
+    def on_stream_headers(stream, request, h)
+      log(stream.id) do
+        h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n")
+      end
+      _, status = h.shift
+      headers = @options.headers_class.new(h)
+      response = @options.response_class.new(request, status, "2.0", headers, @options)
+      request.response = response
+      @streams[request] = stream
+    end
+
+    def on_stream_data(stream, request, data)
+      log(1, "#{stream.id}: ") { "<- DATA: #{data.bytesize} bytes..." }
+      log(2, "#{stream.id}: ") { "<- #{data.inspect}" }
+      request.response << data
+    end
+
+    def on_stream_close(stream, request, error)
+      return handle(request, stream) if request.expects?
+      response = request.response || ErrorResponse.new(error, @retries)
+      emit(:response, request, response)
+      log(2, "#{stream.id}: ") { "closing stream" }
+
+      @streams.delete(request)
+      send(@pending.shift) unless @pending.empty?
+    end
+
+    def on_frame(bytes)
+      @buffer << bytes
+    end
+
+    def on_settings(*)
+      @max_concurrent_requests = [@max_concurrent_requests,
+                                  @connection.remote_settings[:settings_max_concurrent_streams]].min
+    end
+
+    def on_close(*)
+      return unless @connection.state == :closed && @connection.active_stream_count.zero?
+      emit(:complete)
+    end
+
+    def on_frame_sent(frame)
+      log(2, "#{frame[:stream]}: ") { "frame was sent!" }
+      log(2, "#{frame[:stream]}: ") do
+        case frame[:type]
+        when :data
+          frame.merge(payload: frame[:payload].bytesize).inspect
+        when :headers
+          "\e[33m#{frame.inspect}\e[0m"
+        else
+          frame.inspect
+        end
+      end
+    end
+
+    def on_frame_received(frame)
+      log(2, "#{frame[:stream]}: ") { "frame was received!" }
+      log(2, "#{frame[:stream]}: ") do
+        case frame[:type]
+        when :data
+          frame.merge(payload: frame[:payload].bytesize).inspect
+        else
+          frame.inspect
+        end
+      end
+    end
+
+    def on_altsvc(frame)
+      log(2, "#{frame[:stream]}: ") { "altsvc frame was received" }
+      log(2, "#{frame[:stream]}: ") { frame.inspect }
+    end
+
+    def on_promise(stream)
+      emit(:promise, self, stream)
+    end
+
+    def respond_to_missing?(meth, *args)
+      @connection.respond_to?(meth, *args) || super
+    end
+
+    def method_missing(meth, *args, &blk)
+      if @connection.respond_to?(meth)
+        @connection.__send__(meth, *args, &blk)
+      else
+        super
+      end
+    end
+  end
+  Channel.register "h2", Channel::HTTP2
+end