cool.io 0.9.0

data/lib/cool.io/http_client.rb

@@ -0,0 +1,419 @@
+#--
+# Copyright (C)2007-10 Tony Arcieri
+# Includes portions originally Copyright (C)2005 Zed Shaw
+# You can redistribute this under the terms of the Ruby license
+# See file LICENSE for details
+#++
+
+require 'http11_client'
+
+module Coolio
+  # A simple hash is returned for each request made by HttpClient with
+  # the headers that were given by the server for that request.
+  class HttpResponseHeader < Hash
+    # The reason returned in the http response ("OK","File not found",etc.)
+    attr_accessor :http_reason
+
+    # The HTTP version returned.
+    attr_accessor :http_version
+
+    # The status code (as a string!)
+    attr_accessor :http_status
+
+    # HTTP response status as an integer
+    def status
+      Integer(http_status) rescue nil
+    end
+
+    # Length of content as an integer, or nil if chunked/unspecified
+    def content_length
+      Integer(self[HttpClient::CONTENT_LENGTH]) rescue nil
+    end
+ 
+    # Is the transfer encoding chunked?
+    def chunked_encoding?
+      /chunked/i === self[HttpClient::TRANSFER_ENCODING]
+    end
+ end
+
+  class HttpChunkHeader < Hash
+    # When parsing chunked encodings this is set
+    attr_accessor :http_chunk_size
+
+    # Size of the chunk as an integer
+    def chunk_size
+      return @chunk_size unless @chunk_size.nil?
+      @chunk_size = @http_chunk_size ? @http_chunk_size.to_i(base=16) : 0
+    end
+  end
+
+  # Methods for building HTTP requests
+  module HttpEncoding
+    HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
+    FIELD_ENCODING = "%s: %s\r\n"
+
+    # Escapes a URI.
+    def escape(s)
+      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+        '%'+$1.unpack('H2'*$1.size).join('%').upcase
+      }.tr(' ', '+') 
+    end
+
+    # Unescapes a URI escaped string.
+    def unescape(s)
+      s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
+        [$1.delete('%')].pack('H*')
+      } 
+    end
+
+    # Map all header keys to a downcased string version
+    def munge_header_keys(head)
+      head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
+    end
+
+    # HTTP is kind of retarded that you have to specify
+    # a Host header, but if you include port 80 then further
+    # redirects will tack on the :80 which is annoying.
+    def encode_host
+      remote_host + (remote_port.to_i != 80 ? ":#{remote_port}" : "")
+    end
+
+    def encode_request(method, path, query)
+      HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query)]
+    end
+
+    def encode_query(path, query)
+      return path unless query
+      path + "?" + query.map { |k, v| encode_param(k, v) }.join('&')
+    end
+
+    # URL encodes a single k=v parameter.
+    def encode_param(k, v)
+      escape(k) + "=" + escape(v)
+    end
+
+    # Encode a field in an HTTP header
+    def encode_field(k, v)
+      FIELD_ENCODING % [k, v]
+    end
+
+    def encode_headers(head)
+      head.inject('') do |result, (key, value)|
+        # Munge keys from foo-bar-baz to Foo-Bar-Baz
+        key = key.split('-').map { |k| k.capitalize }.join('-')
+      result << encode_field(key, value)
+      end
+    end
+
+    def encode_cookies(cookies)
+      cookies.inject('') { |result, (k, v)| result << encode_field('Cookie', encode_param(k, v)) }
+    end
+  end
+
+  # HTTP client class implemented as a subclass of Coolio::TCPSocket.  Encodes
+  # requests and allows streaming consumption of the response.  Response is
+  # parsed with a Ragel-generated whitelist parser which supports chunked
+  # HTTP encoding.
+  #
+  # == Example
+  #
+  #   loop = Coolio::Loop.default
+  #   client = Coolio::HttpClient.connect("www.google.com").attach
+  #   client.get('/search', query: {q: 'foobar'})
+  #   loop.run
+  #
+  class HttpClient < TCPSocket
+    include HttpEncoding
+
+    ALLOWED_METHODS=[:put, :get, :post, :delete, :head]
+    TRANSFER_ENCODING="TRANSFER_ENCODING"
+    CONTENT_LENGTH="CONTENT_LENGTH"
+    SET_COOKIE="SET_COOKIE"
+    LOCATION="LOCATION"
+    HOST="HOST"
+    CRLF="\r\n"
+
+    # Connect to the given server, with port 80 as the default
+    def self.connect(addr, port = 80, *args)
+      super
+    end
+
+    def initialize(socket)
+      super
+
+      @parser = HttpClientParser.new
+      @parser_nbytes = 0
+
+      @state = :response_header
+      @data = ::IO::Buffer.new
+
+      @response_header = HttpResponseHeader.new
+      @chunk_header = HttpChunkHeader.new
+    end
+
+    # Send an HTTP request and consume the response.  
+    # Supports the following options:
+    #
+    #   head: {Key: Value}
+    #     Specify an HTTP header, e.g. {'Connection': 'close'} 
+    #
+    #   query: {Key: Value}
+    #     Specify query string parameters (auto-escaped)
+    #
+    #   cookies: {Key: Value}
+    #     Specify hash of cookies (auto-escaped)
+    #
+    #   body: String
+    #     Specify the request body (you must encode it for now)
+    #
+    def request(method, path, options = {})
+      raise ArgumentError, "invalid request path" unless /^\// === path
+      raise RuntimeError, "request already sent" if @requested
+
+      @method, @path, @options = method, path, options
+      @requested = true
+
+      return unless @connected
+      send_request
+    end
+    
+    # Enable the HttpClient if it has been disabled
+    def enable
+      super
+      dispatch unless @data.empty?
+    end
+
+    # Called when response header has been received
+    def on_response_header(response_header)
+    end
+
+    # Called when part of the body has been read
+    def on_body_data(data)
+      STDOUT.write data
+      STDOUT.flush
+    end
+
+    # Called when the request has completed
+    def on_request_complete
+      close
+    end
+
+    # Called when an error occurs dispatching the request
+    def on_error(reason)
+      close
+      raise RuntimeError, reason
+    end
+
+    #########
+    protected
+    #########
+
+    #
+    # Coolio callbacks
+    #
+        
+    def on_connect
+      @connected = true
+      send_request if @method and @path
+    end
+
+    def on_read(data)
+      @data << data
+      dispatch
+    end
+
+    #
+    # Request sending
+    #
+
+    def send_request
+      send_request_header
+      send_request_body
+    end
+
+    def send_request_header
+      query   = @options[:query]
+      head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
+      cookies = @options[:cookies]
+      body    = @options[:body]
+
+      # Set the Host header if it hasn't been specified already
+      head['host'] ||= encode_host
+
+      # Set the Content-Length if it hasn't been specified already and a body was given
+      head['content-length'] ||= body ? body.length : 0
+
+      # Set the User-Agent if it hasn't been specified
+      head['user-agent'] ||= "Coolio #{Coolio::VERSION}"
+
+      # Default to Connection: close
+      head['connection'] ||= 'close'
+
+      # Build the request
+      request_header = encode_request(@method, @path, query)
+      request_header << encode_headers(head)
+      request_header << encode_cookies(cookies) if cookies
+      request_header << CRLF
+
+      write request_header
+    end
+
+    def send_request_body
+      write @options[:body] if @options[:body]
+    end  
+ 
+    #
+    # Response processing
+    #
+
+    def dispatch
+      while enabled? and case @state
+        when :response_header
+          parse_response_header
+        when :chunk_header
+          parse_chunk_header
+        when :chunk_body
+          process_chunk_body
+        when :chunk_footer
+          process_chunk_footer
+        when :response_footer
+          process_response_footer
+        when :body
+          process_body
+        when :finished, :invalid
+          break
+        else raise RuntimeError, "invalid state: #{@state}"
+        end
+      end
+    end
+
+    def parse_header(header)
+      return false if @data.empty?
+      
+      begin
+        @parser_nbytes = @parser.execute(header, @data.to_str, @parser_nbytes)
+      rescue Coolio::HttpClientParserError
+        on_error "invalid HTTP format, parsing fails"
+        @state = :invalid
+      end
+      
+      return false unless @parser.finished?
+
+      # Clear parsed data from the buffer
+      @data.read(@parser_nbytes)
+      @parser.reset
+      @parser_nbytes = 0
+      
+      true
+    end
+
+    def parse_response_header
+      return false unless parse_header(@response_header)
+
+      unless @response_header.http_status and @response_header.http_reason
+        on_error "no HTTP response"
+        @state = :invalid
+        return false
+      end
+
+      on_response_header(@response_header)
+
+      if @response_header.chunked_encoding?
+        @state = :chunk_header
+      else
+        @state = :body
+        @bytes_remaining = @response_header.content_length
+      end
+      
+      true
+    end
+
+    def parse_chunk_header
+      return false unless parse_header(@chunk_header)
+
+      @bytes_remaining = @chunk_header.chunk_size
+      @chunk_header = HttpChunkHeader.new
+
+      @state = @bytes_remaining > 0 ? :chunk_body : :response_footer      
+      true
+    end
+
+    def process_chunk_body
+      if @data.size < @bytes_remaining
+        @bytes_remaining -= @data.size
+        on_body_data @data.read
+        return false
+      end
+
+      on_body_data @data.read(@bytes_remaining)
+      @bytes_remaining = 0
+      
+      @state = :chunk_footer      
+      true
+    end
+
+    def process_chunk_footer
+      return false if @data.size < 2
+
+      if @data.read(2) == CRLF
+        @state = :chunk_header
+      else
+        on_error "non-CRLF chunk footer"
+        @state = :invalid
+      end
+      
+      true
+    end
+
+    def process_response_footer
+      return false if @data.size < 2
+      
+      if @data.read(2) == CRLF
+        if @data.empty?
+          on_request_complete
+          @state = :finished
+        else
+          on_error "garbage at end of chunked response"
+          @state = :invalid
+        end
+      else
+        on_error "non-CRLF response footer"
+        @state = :invalid
+      end
+      
+      false
+    end
+
+    def process_body
+      if @bytes_remaining.nil?
+        on_body_data @data.read
+        return false
+      end
+      
+      if @bytes_remaining.zero?
+        on_request_complete
+        @state = :finished
+        return false
+      end
+
+      if @data.size < @bytes_remaining
+        @bytes_remaining -= @data.size
+        on_body_data @data.read
+        return false
+      end
+
+      on_body_data @data.read(@bytes_remaining)
+      @bytes_remaining = 0
+
+      if @data.empty?
+        on_request_complete
+        @state = :finished
+      else
+        on_error "garbage at end of body"
+        @state = :invalid
+      end
+
+      false
+    end
+  end
+end

data/lib/cool.io/io.rb

@@ -0,0 +1,174 @@
+#--
+# Copyright (C)2007-10 Tony Arcieri
+# You can redistribute this under the terms of the Ruby license
+# See file LICENSE for details
+#++
+
+module Coolio
+  # A buffered I/O class witch fits into the Coolio Watcher framework.
+  # It provides both an observer which reads data as it's received
+  # from the wire and a buffered write watcher which stores data and writes
+  # it out each time the socket becomes writable.
+  #
+  # This class is primarily meant as a base class for other streams
+  # which need non-blocking writing, and is used to implement Coolio's
+  # Socket class and its associated subclasses.
+  class IO
+    extend Meta
+    
+    # Maximum number of bytes to consume at once
+    INPUT_SIZE = 16384
+
+    def initialize(io)
+      @_io = io
+      @_write_buffer  ||= ::IO::Buffer.new
+      @_read_watcher  = Watcher.new(io, self, :r)
+      @_write_watcher = Watcher.new(io, self, :w)
+    end
+
+    #
+    # Watcher methods, delegated to @_read_watcher
+    #
+    
+    # Attach to the event loop
+    def attach(loop); @_read_watcher.attach loop; schedule_write if !@_write_buffer.empty?; self; end
+    
+    # Detach from the event loop
+    def detach; @_read_watcher.detach; self; end # TODO should these detect write buffers, as well?
+    
+    # Enable the watcher
+    def enable; @_read_watcher.enable; self; end
+    
+    # Disable the watcher
+    def disable; @_read_watcher.disable; self; end
+    
+    # Is the watcher attached?
+    def attached?; @_read_watcher.attached?; end
+    
+    # Is the watcher enabled?
+    def enabled?; @_read_watcher.enabled?; end
+    
+    # Obtain the event loop associated with this object
+    def evloop; @_read_watcher.evloop; end
+    
+    #
+    # Callbacks for asynchronous events
+    #
+      
+    # Called whenever the IO object receives data
+    def on_read(data); end
+    event_callback :on_read
+
+    # Called whenever a write completes and the output buffer is empty
+    def on_write_complete; end
+    event_callback :on_write_complete
+
+    # Called whenever the IO object hits EOF
+    def on_close; end
+    event_callback :on_close
+
+    #
+    # Write interface
+    #
+
+    # Write data in a buffered, non-blocking manner
+    def write(data)
+      @_write_buffer << data
+      schedule_write
+      data.size
+    end
+
+    # Number of bytes are currently in the output buffer
+    def output_buffer_size
+      @_write_buffer.size
+    end
+
+    # Close the IO stream
+    def close
+      detach if attached?
+      detach_write_watcher
+      @_io.close unless @_io.closed?
+
+      on_close
+      nil
+    end
+
+    # Is the IO object closed?
+    def closed?
+      @_io.nil? or @_io.closed?
+    end
+
+    #########
+    protected
+    #########
+
+    # Read from the input buffer and dispatch to on_read
+    def on_readable
+      begin
+        on_read @_io.read_nonblock(INPUT_SIZE)
+      rescue Errno::EAGAIN, Errno::EINTR
+        return
+
+      # SystemCallError catches Errno::ECONNRESET amongst others.
+      rescue SystemCallError, EOFError, IOError, SocketError
+        close
+      end
+    end
+    
+    # Write the contents of the output buffer
+    def on_writable
+      begin
+        @_write_buffer.write_to(@_io)
+      rescue Errno::EINTR
+        return
+
+      # SystemCallError catches Errno::EPIPE & Errno::ECONNRESET amongst others.
+      rescue SystemCallError, IOError, SocketError
+        return close
+      end
+      
+      if @_write_buffer.empty?
+        disable_write_watcher
+        on_write_complete
+      end
+    end
+
+    # Schedule a write to be performed when the IO object becomes writable 
+    def schedule_write
+      return unless @_io # this would mean 'we are still pre DNS here'
+      return unless attached? # this would mean 'currently unattached' -- ie still pre DNS, or just plain not attached, which is ok
+      begin
+        enable_write_watcher      
+      rescue IOError
+      end
+    end
+    
+    def enable_write_watcher
+      if @_write_watcher.attached?
+        @_write_watcher.enable unless @_write_watcher.enabled?
+      else
+        @_write_watcher.attach(evloop)
+      end
+    end
+    
+    def disable_write_watcher
+      @_write_watcher.disable if @_write_watcher and @_write_watcher.enabled?
+    end
+    
+    def detach_write_watcher
+      @_write_watcher.detach if @_write_watcher and @_write_watcher.attached?
+    end
+    
+    # Internal class implementing watchers used by Coolio::IO
+    class Watcher < IOWatcher
+      def initialize(ruby_io, coolio_io, flags)
+        @coolio_io = coolio_io
+        super(ruby_io, flags)
+      end
+
+      # Configure IOWatcher event callbacks to call the method passed to #initialize
+      def on_readable; @coolio_io.__send__(:on_readable); end
+      def on_writable; @coolio_io.__send__(:on_writable); end
+    end
+  end
+end