ftw 0.0.1 → 0.0.4

data/README.md

@@ -1,10 +1,12 @@
 # For The Web
 
-net/http is pretty much not good.
+net/http is pretty much not good. dns behavior in ruby changes quite frequently.
+
+Above all else, I want a consistent API and behavior that I can rely on. Ruby stdlib is not that thing.
 
 I want:
 
-* A HTTP client that acts as a full user agent, not just a single connection.
+* A HTTP client that acts as a full user agent, not just a single connections. (With connection reuse)
 * HTTP and SPDY support.
 * WebSockets support.
 * SSL/TLS support.
@@ -12,12 +14,6 @@ I want:
 * Server and Client modes.
 * Support for both normal operation and EventMachine would be nice.
 
-## DONE
-
-* TCP connection 
-* DNS resolution (wraps Socket.gethostname)
-* HTTP client partially done
-
 ## TODO
 
 * Tests, yo.
@@ -32,6 +28,9 @@ I want:
     request = agent.get("http://www.google.com/")
     response = request.execute
 
+    # Simpler
+    response = agent.get!("http://www.google.com/")
+
 ### SPDY
 
     # SPDY should automatically be attempted. The caller should be unaware.

data/lib/ftw.rb

@@ -0,0 +1,4 @@
+require "ftw/agent"
+require "ftw/connection"
+require "ftw/dns"
+require "ftw/version"

data/lib/ftw/agent.rb

@@ -1,40 +1,223 @@
 require "ftw/namespace"
 require "ftw/request"
 require "ftw/connection"
+require "ftw/pool"
+require "ftw/websocket"
 require "addressable/uri"
+require "cabin"
+require "logger"
 
-# This should act as a proper agent.
+# This should act as a proper web agent.
 #
-# * Keep cookies. Offer local-storage of cookies
-# * Reuse connections. HTTP 1.1 Connection: keep-alive
-# * HTTP Upgrade support
-# * Websockets
-# * SSL/TLS
+# * Reuse connections.
+# * SSL/TLS.
+# * HTTP Upgrade support.
+# * HTTP 1.1 (RFC2616).
+# * WebSockets (RFC6455).
+# * Support Cookies.
+#
+# All standard HTTP methods defined by RFC2616 are available as methods on 
+# this agent: get, head, put, etc.
+#
+# Example:
+#
+#     agent = FTW::Agent.new
+#     request = agent.get("http://www.google.com/")
+#     response = agent.execute(request)
+#     puts response.body.read
+#
+# For any standard http method (like 'get') you can invoke it with '!' on the end
+# and it will execute and return a FTW::Response object:
+#
+#     agent = FTW::Agent.new
+#     response = agent.get!("http://www.google.com/")
+#     puts response.body.head
+#
+# TODO(sissel): TBD: implement cookies... delicious chocolate chip cookies.
 class FTW::Agent
-  # TODO(sissel): All standard HTTP methods should be defined here.
-  # Also allow users to specify non-standard methods.
-  
+  STANDARD_METHODS = %w(options get head post put delete trace connect)
+
   def initialize
-  end
+    @pool = FTW::Pool.new
+    @logger = Cabin::Channel.get($0)
+    @logger.subscribe(Logger.new(STDOUT))
+    @logger.level = :warn
+
+    @redirect_max = 20
+  end # def initialize
+
+  # Define all the standard HTTP methods (Per RFC2616)
+  # As an example, for "get" method, this will define these methods:
+  # 
+  # * FTW::Agent#get(uri, options={})
+  # * FTW::Agent#get!(uri, options={})
+  #
+  # The first one returns a FTW::Request you must pass to Agent#execute(...)
+  # The second does the execute for you and returns a FTW::Response.
+  # 
+  # For a full list of these available methods, see STANDARD_METHODS.
+  STANDARD_METHODS.each do |name|
+    m = name.upcase
+
+    # define 'get' etc method.
+    define_method(name.to_sym) do |uri, options={}|
+      return request(m, uri, options)
+    end
+
+    # define 'get!' etc method.
+    define_method("#{name}!".to_sym) do |uri, options={}|
+      return execute(request(m, uri, options))
+    end
+  end # STANDARD_METHODS.each
 
-  # Returns a FTW::Request
-  # TODO(sissel): SSL/TLS support
-  def get(uri, options={})
-    return request("GET", uri, options)
-  end # def get
+  # Send the request as an HTTP upgrade.
+  # 
+  # Returns the response and the FTW::Connection for this connection.
+  # If the upgrade was denied, the connection returned will be nil.
+  def upgrade!(uri, protocol, options={})
+    req = request("GET", uri, options)
+    req.headers["Connection"] = "Upgrade"
+    req.headers["Upgrade"] = protocol
+    response = execute(req)
+    if response.status == 101
+      return response, response.body
+    else
+      return response, nil
+    end
+  end # def upgrade!
 
+  # Make a new websocket connection.
+  #
+  # This will send the http request. If the websocket handshake
+  # is successful, a FTW::WebSocket instance will be returned.
+  # Otherwise, a FTW::Response will be returned.
+  public
+  def websocket!(uri, options={})
+    # TODO(sissel): Use FTW::Agent#upgrade! ?
+    req = request("GET", uri, options)
+    ws = FTW::WebSocket.new(req)
+    response = execute(req)
+    if ws.handshake_ok?(response)
+      # response.body is a FTW::Connection
+      ws.connection = response.body
+
+      # TODO(sissel): Investigate this bug
+      # There seems to be a bug in http_parser.rb (or in this library) where
+      # websocket responses lead with a newline for some reason. Work around
+      # it.
+      data = response.body.read
+      if data[0] == "\n"
+        response.body.pushback(data[1..-1])
+      else
+        response.body.pushback(data)
+      end
+      return ws
+    else
+      return response
+    end
+  end # def websocket!
+
+  # Make a request. Returns a FTW::Request object.
+  #
+  # Arguments:
+  #
+  # * method - the http method
+  # * uri - the URI to make the request to
+  # * options - a hash of options
+  #
+  # uri can be a valid url or an Addressable::URI object.
+  # The uri will be used to choose the host/port to connect to. It also sets
+  # the protocol (https, etc). Further, it will set the 'Host' header.
+  #
+  # The 'options' hash supports the following keys:
+  # 
+  # * :headers => { string => string, ... }. This allows you to set header values.
+  public
   def request(method, uri, options)
+    @logger.info("Creating new request", :method => method, :uri => uri, :options => options)
     request = FTW::Request.new(uri)
     request.method = method
-    request.connection = connection(uri.host, uri.port)
+    request.headers.add("Connection", "keep-alive")
+
+    if options.include?(:headers)
+      options[:headers].each do |key, value|
+        request.headers.add(key, value)
+      end
+    end
+
     return request
   end # def request
 
+  # Execute a FTW::Request in this Agent.
+  #
+  # If an existing, idle connection is already open to the target server
+  # of this Request, it will be reused. Otherwise, a new connection
+  # is opened.
+  #
+  # Redirects are always followed.
+  public
+  def execute(request)
+    # TODO(sissel): Make redirection-following optional, but default.
+
+    connection = connect(request.headers["Host"], request.port)
+    connection.secure if request.protocol == "https"
+    response = request.execute(connection)
+
+    redirects = 0
+    while response.redirect? and response.headers.include?("Location")
+      redirects += 1
+      if redirects > @redirect_max
+        # TODO(sissel): Abort somehow...
+      end
+      # RFC2616 section 10.3.3 indicates HEAD redirects must not include a
+      # body. Otherwise, the redirect response can have a body, so let's
+      # throw it away.
+      if request.method == "HEAD" 
+        # Head requests have no body
+        connection.release
+      elsif response.content?
+        # Throw away the body
+        response.body = connection
+        # read_body will release the connection
+        response.read_body { |chunk| }
+      end
+
+      # TODO(sissel): If this response has any cookies, store them in the
+      # agent's cookie store
+
+      @logger.debug("Redirecting", :location => response.headers["Location"])
+      redirects += 1
+      request.use_uri(response.headers["Location"])
+      connection = connect(request.headers["Host"], request.port)
+      connection.secure if request.protocol == "https"
+      response = request.execute(connection)
+    end
+
+    # RFC 2616 section 9.4, HEAD requests MUST NOT have a message body.
+    if request.method != "HEAD"
+      response.body = connection
+    else
+      connection.release
+    end
+   
+    # TODO(sissel): If this response has any cookies, store them in the
+    # agent's cookie store
+    return response
+  end # def execute
+
   # Returns a FTW::Connection connected to this host:port.
-  # TODO(sissel): Implement connection reuse
-  # TODO(sissel): support SSL/TLS
   private
-  def connection(host, port)
-    return FTW::Connection.new("#{host}:#{port}")
+  def connect(host, port)
+    address = "#{host}:#{port}"
+    @logger.debug("Fetching from pool", :address => address)
+    connection = @pool.fetch(address) do
+      @logger.info("New connection to #{address}")
+      connection = FTW::Connection.new(address)
+      connection.connect
+      connection
+    end
+    @logger.debug("Pool fetched a connection", :connection => connection)
+    connection.mark
+    return connection
   end # def connect
 end # class FTW::Agent

data/lib/ftw/connection.rb

@@ -1,6 +1,7 @@
 require "cabin" # rubygem "cabin"
-require "net/ftw/dns"
-require "net/ftw/namespace"
+require "ftw/dns"
+require "ftw/poolable"
+require "ftw/namespace"
 require "socket"
 require "timeout" # ruby stdlib, just for the Timeout exception.
 require "backport-bij" # for Array#rotate, IO::WaitWritable, etc, in ruby < 1.9
@@ -9,7 +10,15 @@ require "backport-bij" # for Array#rotate, IO::WaitWritable, etc, in ruby < 1.9
 #
 # You can use IO::select on this objects of this type.
 # (at least, in MRI you can)
+#
+# You can activate SSL/TLS on this connection by invoking FTW::Connection#secure
 class FTW::Connection
+  class ConnectTimeout < StandardError; end
+  class ReadTimeout < StandardError; end
+  class WriteTimeout < StandardError; end
+  include FTW::Poolable
+  include Cabin::Inspectable
+
   # A new network connection.
   # The 'destination' argument can be an array of strings or a single string.
   # String format is expected to be "host:port"
@@ -28,21 +37,33 @@ class FTW::Connection
       @destinations = destinations
     end
 
+    @logger = Cabin::Channel.get($0)
     @connect_timeout = 2
 
     # Use a fixed-size string that we set to BINARY encoding.
     # Not all byte sequences are UTF-8 friendly :0
     @read_size = 16384
     @read_buffer = " " * @read_size
+    @pushback_buffer = ""
 
     # Tell Ruby 1.9 that this string is a binary string, not utf-8 or somesuch.
     if @read_buffer.respond_to?(:force_encoding)
       @read_buffer.force_encoding("BINARY")
     end
 
+    @inspectables = [:@destinations, :@connected, :@remote_address, :@secure]
+    @connected = false
+    @remote_address = nil
+    @secure = false
+
     # TODO(sissel): Validate @destinations
+    # TODO(sissel): Barf if a destination is not of the form "host:port"
   end # def initialize
 
+  # Connect now.
+  #
+  # Timeout value is optional. If no timeout is given, this method
+  # blocks until a connection is successful or an error occurs.
   public
   def connect(timeout=nil)
     # TODO(sissel): Raise if we're already connected?
@@ -53,6 +74,8 @@ class FTW::Connection
     # Do dns resolution on the host. If there are multiple
     # addresses resolved, return one at random.
     @remote_address = FTW::DNS.singleton.resolve_random(host)
+    @logger.debug("Connecting", :address => @remote_address,
+                  :host => host, :port => port)
 
     # Addresses with colon ':' in them are assumed to be IPv6
     family = @remote_address.include?(":") ? Socket::AF_INET6 : Socket::AF_INET
@@ -72,20 +95,22 @@ class FTW::Connection
       if writable?(timeout)
         begin
           @socket.connect_nonblock(sockaddr) # check connection failure
-        rescue Errno::EISCONN # Ignore, we're already connected.
+        rescue Errno::EISCONN 
+          # Ignore, we're already connected.
         rescue Errno::ECONNREFUSED => e
           # Fire 'disconnected' event with reason :refused
-          trigger(DISCONNECTED, :refused, e)
+          return e
         end
       else
         # Connection timeout
         # Fire 'disconnected' event with reason :timeout
-        trigger(DISCONNECTED, :connect_timeout, nil)
+        return ConnectTimeout.new
       end
     end
 
     # We're now connected.
-    trigger(CONNECTED, "#{host}:#{port}")
+    @connected = true
+    return true
   end # def connect
 
   # Is this Connection connected?
@@ -97,14 +122,16 @@ class FTW::Connection
   # Write data to this connection.
   # This method blocks until the write succeeds unless a timeout is given.
   #
-  # Returns the number of bytes written (See IO#syswrite)
+  # This method is not guaranteed to have written the full data given.
+  #
+  # Returns the number of bytes written (See also IO#syswrite)
   public
   def write(data, timeout=nil)
     #connect if !connected?
     if writable?(timeout)
       return @socket.syswrite(data)
     else
-      raise Timeout::Error.new
+      raise FTW::Connection::WriteTimeout.new(self.inspect)
     end
   end # def write
 
@@ -115,47 +142,53 @@ class FTW::Connection
   # IO#sysread
   public
   def read(timeout=nil)
-    if readable?(timeout)
-      if !@unread_buffer.empty?
-        data = @unread_buffer
-        @unread_buffer = ""
-        return data
-      end
+    data = ""
+    data.force_encoding("BINARY") if data.respond_to?(:force_encoding)
+    have_pushback = !@pushback_buffer.empty?
+    if have_pushback
+      data << @pushback_buffer
+      @pushback_buffer = ""
+      # We have data 'now' so don't wait.
+      timeout = 0
+    end
 
+    if readable?(timeout)
       begin
         @socket.sysread(@read_size, @read_buffer)
-        return @read_buffer
-      rescue EOFError
-        trigger(READER_CLOSED)
+        data << @read_buffer
+        return data
+      rescue EOFError => e
+        raise e
       end
     else
-      raise Timeout::Error.new
+      if have_pushback
+        return data
+      else
+        raise ReadTimeout.new
+      end
     end
   end # def read
 
-  # Un-read some data
+  # Push back some data onto the connection's read buffer.
   public
-  def unread(data)
-    @unread_buffer << data
-  end # def unread
+  def pushback(data)
+    @pushback_buffer << data
+  end # def pushback
 
   # End this connection, specifying why.
   public
   def disconnect(reason)
     begin 
-      #@reader_closed = true
       @socket.close_read
     rescue IOError => e
-      # Ignore
+      # Ignore, perhaps we shouldn't ignore.
     end
 
     begin 
       @socket.close_write
     rescue IOError => e
-      # Ignore
+      # Ignore, perhaps we shouldn't ignore.
     end
-
-    trigger(DISCONNECTED, reason)
   end # def disconnect
 
   # Is this connection writable? Returns true if it is writable within
@@ -179,53 +212,74 @@ class FTW::Connection
     return !ready.nil?
   end # def readable?
 
-  protected
-  def connected(address)
-    @remote_address = nil
-    @connected = true
-  end # def connected
-
-  protected
-  def disconnected(reason, error)
-    @remote_address = nil
-    @connected = false
-  end # def disconnected
-
   # The host:port
   public
   def peer
     return @remote_address
   end # def peer
 
-  # Run this Connection.
-  # This is generally meant for Threaded or synchronous operation. 
-  # For EventMachine, see TODO(sissel): Implement EventMachine support.
+  # Support 'to_io' so you can use IO::select on this object.
   public
-  def run
-    connect(@connect_timeout) if not connected?
-    while connected?
-      read_and_trigger
-    end
-  end # def run
+  def to_io
+    return @socket
+  end # def to_io
 
-  # Read data and trigger data callbacks.
-  #
-  # This is mainly useful if you are implementing your own run loops
-  # and IO::select shenanigans.
+  # Secure this connection with TLS.
   public
-  def read_and_trigger
-    data = read(@read_size)
-    if data.length == 0
-      disconnect(EOFError)
-    else
-      trigger(DATA, data)
+  def secure(timeout=nil, options={})
+    # Skip this if we're already secure.
+    return if secured?
+
+    @logger.debug("Securing this connection", :peer => peer, :connection => self)
+    # Wrap this connection with TLS/SSL
+    require "openssl"
+    sslcontext = OpenSSL::SSL::SSLContext.new
+    sslcontext.ssl_version = :TLSv1
+    # If you use VERIFY_NONE, you are removing an important piece 
+    sslcontext.verify_mode = OpenSSL::SSL::VERIFY_PEER
+    # TODO(sissel): Try to be smart about setting this default.
+    sslcontext.ca_path = "/etc/ssl/certs"
+    @socket = OpenSSL::SSL::SSLSocket.new(@socket, sslcontext)
+
+    # SSLSocket#connect_nonblock will do the SSL/TLS handshake.
+    # TODO(sissel): refactor this into a method that both secure and connect
+    # methods can call.
+    start = Time.now
+    begin
+      @socket.connect_nonblock
+    rescue IO::WaitReadable, IO::WaitWritable
+      # The ruby OpenSSL docs for 1.9.3 have example code saying I should use
+      # IO::WaitReadable, but in the real world it raises an SSLError with
+      # a specific string message instead of Errno::EAGAIN or IO::WaitReadable
+      # explicitly...
+      #
+      # This SSLSocket#connect_nonblock raising WaitReadable (Technically,
+      # OpenSSL::SSL::SSLError) is in contrast to what Socket#connect_nonblock
+      # raises, WaitWritable (ok, Errno::EINPROGRESS, technically)
+      # Ruby's SSL exception for 'this call would block' is pretty shitty.
+      #
+      # If the exception string is *not* 'read would block' we have a real
+      # problem.
+      
+      if !timeout.nil?
+        time_left = timeout - (Time.now - start)
+        raise ConnectTimeout.new if time_left < 0
+        r, w, e = IO.select([@socket], [@socket], nil, time_left)
+      else
+        r, w, e = IO.select([@socket], [@socket], nil, timeout)
+      end
+
+      # try connect_nonblock again if the socket is ready
+      retry if r.size > 0 || w.size > 0
     end
-  end # def read_and_trigger
 
-  # Support 'to_io' so you can use IO::select on this object.
+    @secure = true
+  end # def secure
+
+  # Has this connection been secured?
   public
-  def to_io
-    return @socket
-  end
+  def secured?
+    return @secure
+  end # def secured?
 end # class FTW::Connection