ftw 0.0.6 → 0.0.7

data/README.md

@@ -21,6 +21,7 @@ Desired features:
 * HTTP and SPDY support.
 * WebSockets support.
 * SSL/TLS support.
+* Browser Agent features like cookies and caching
 * An API that lets me do what I need.
 * Server and Client modes.
 * Support for both normal operation and EventMachine would be nice.
@@ -61,11 +62,13 @@ I do not plan on exposing any direct means for invoking SPDY.
 
 ## Server API
 
-TBD. Will likely surround 'rack'. Need to find out what servers actually can
-support HTTP Upgrade.
+Not sure yet...
 
-It's possible the 'cramp' gem supports all the server-side features we need
-(except for SPDY, I suppose, which I might be able to contribute upstream)
+Since Rack is not supported, I'll have to do a lot of legwork myself.
+
+* Implement a proper Socket Server api
+* Implement a HTTP server on top of that (add SPDY support later)
+* Implement a Sinatra-like DSL on top of HTTP
 
 ## Other Projects
 
@@ -76,3 +79,10 @@ Here are some related projects that I have no affiliation with:
 * https://github.com/lifo/cramp - real-time web framework (async, websockets)
 * https://github.com/igrigorik/em-http-request - HTTP client for EventMachine
 * https://github.com/geemus/excon - http client library
+
+## Missing Features
+
+* No Rack support, for now. There are technical requirements the Rack SPEC that
+  prevent rack applications from really servicing uploads, HTTP Upgrades, etc.
+  Details here: https://github.com/rack/rack/issues/347
+

data/lib/ftw.rb

@@ -2,3 +2,4 @@ require "ftw/agent"
 require "ftw/connection"
 require "ftw/dns"
 require "ftw/version"
+require "ftw/server"

data/lib/ftw/agent.rb

@@ -1,6 +1,7 @@
 require "ftw/namespace"
 require "ftw/request"
 require "ftw/connection"
+require "ftw/protocol"
 require "ftw/pool"
 require "ftw/websocket"
 require "addressable/uri"
@@ -35,6 +36,9 @@ require "logger"
 #
 # TODO(sissel): TBD: implement cookies... delicious chocolate chip cookies.
 class FTW::Agent
+  include FTW::Protocol
+
+  # List of standard HTTP methods described in RFC2616
   STANDARD_METHODS = %w(options get head post put delete trace connect)
 
   # Everything is private by default.
@@ -95,6 +99,8 @@ class FTW::Agent
   # This will send the http request. If the websocket handshake
   # is successful, a FTW::WebSocket instance will be returned.
   # Otherwise, a FTW::Response will be returned.
+  #
+  # See {#request} for what the 'uri' and 'options' parameters should be.
   def websocket!(uri, options={})
     # TODO(sissel): Use FTW::Agent#upgrade! ?
     req = request("GET", uri, options)
@@ -103,17 +109,6 @@ class FTW::Agent
     if ws.handshake_ok?(response)
       # response.body is a FTW::Connection
       ws.connection = response.body
-
-      # TODO(sissel): Investigate this bug
-      # There seems to be a bug in http_parser.rb (or in this library) where
-      # websocket responses lead with a newline for some reason. Work around
-      # it.
-      data = response.body.read
-      if data[0] == "\n"
-        response.body.pushback(data[1..-1])
-      else
-        response.body.pushback(data)
-      end
       return ws
     else
       return response
@@ -158,12 +153,16 @@ class FTW::Agent
   #
   # Redirects are always followed.
   #
-  # @params
+  # @param [FTW::Request]
   # @return [FTW::Response] the response for this request.
   def execute(request)
     # TODO(sissel): Make redirection-following optional, but default.
 
-    connection = connect(request.headers["Host"], request.port)
+    connection, error = connect(request.headers["Host"], request.port)
+    if !error.nil?
+      p :error => error
+      raise error
+    end
     connection.secure if request.protocol == "https"
     response = request.execute(connection)
 
@@ -192,7 +191,12 @@ class FTW::Agent
       @logger.debug("Redirecting", :location => response.headers["Location"])
       redirects += 1
       request.use_uri(response.headers["Location"])
-      connection = connect(request.headers["Host"], request.port)
+      connection, error = connect(request.headers["Host"], request.port)
+      # TODO(sissel): Do better error handling than raising.
+      if !error.nil?
+        p :error => error
+        raise error
+      end
       connection.secure if request.protocol == "https"
       response = request.execute(connection)
     end
@@ -213,15 +217,28 @@ class FTW::Agent
   def connect(host, port)
     address = "#{host}:#{port}"
     @logger.debug("Fetching from pool", :address => address)
+    error = nil
     connection = @pool.fetch(address) do
       @logger.info("New connection to #{address}")
       connection = FTW::Connection.new(address)
-      connection.connect
-      connection
+      error = connection.connect
+      if !error.nil?
+        # Return nil to the pool, so like, we failed..
+        nil
+      else
+        # Otherwise return our new connection
+        connection
+      end
     end
+
+    if !error.nil?
+      @logger.error("Connection failed", :destination => address, :error => error)
+      return nil, error
+    end
+
     @logger.debug("Pool fetched a connection", :connection => connection)
     connection.mark
-    return connection
+    return connection, nil
   end # def connect
 
   public(:initialize, :execute, :websocket!, :upgrade!)

data/lib/ftw/connection.rb

@@ -4,7 +4,7 @@ require "ftw/poolable"
 require "ftw/namespace"
 require "socket"
 require "timeout" # ruby stdlib, just for the Timeout exception.
-require "backport-bij" # for Array#rotate, IO::WaitWritable, etc, in ruby < 1.9
+require "backports" # for Array#rotate, IO::WaitWritable, etc, in ruby < 1.9
 
 # A network connection. This is TCP.
 #
@@ -12,12 +12,27 @@ require "backport-bij" # for Array#rotate, IO::WaitWritable, etc, in ruby < 1.9
 # (at least, in MRI you can)
 #
 # You can activate SSL/TLS on this connection by invoking FTW::Connection#secure
+#
+# This class also implements buffering itself because some IO-like classes
+# (OpenSSL::SSL::SSLSocket) do not support IO#ungetbyte
 class FTW::Connection
+  include FTW::Poolable
+  include Cabin::Inspectable
+
+  # A connection attempt timed out
   class ConnectTimeout < StandardError; end
+  
+  # A connection attempt was rejected
+  class ConnectRefused < StandardError; end
+
+  # A read timed out
   class ReadTimeout < StandardError; end
+
+  # A write timed out
   class WriteTimeout < StandardError; end
-  include FTW::Poolable
-  include Cabin::Inspectable
+
+  # Secure setup timed out
+  class SecureHandshakeTimeout < StandardError; end
 
   private
 
@@ -38,6 +53,10 @@ class FTW::Connection
       @destinations = destinations
     end
 
+    setup
+  end # def initialize
+
+  def setup
     @logger = Cabin::Channel.get($0)
     @connect_timeout = 2
 
@@ -61,13 +80,47 @@ class FTW::Connection
     # TODO(sissel): Barf if a destination is not of the form "host:port"
   end # def initialize
 
+  # Create a new connection from an existing IO instance (like a socket)
+  # 
+  # Valid modes are :server and :client.
+  #
+  # * specify :server if this connection is from a server (via Socket#accept)
+  # * specify :client if this connection is from a client (via Socket#connect)
+  def self.from_io(io, mode=:server)
+    valid_modes = [:server, :client]
+    if !valid_modes.include?(mode)
+      raise InvalidArgument.new("Invalid connection mode '#{mode}'. Valid modes: #{valid_modes.inspect}")
+    end
+
+    connection = self.new(nil) # New connection with no destinations
+    connection.instance_eval do
+      @socket = io
+      @connected = true
+      port, address = Socket.unpack_sockaddr_in(io.getpeername)
+      @remote_address = "#{address}:#{port}"
+      @mode = mode
+    end
+    return connection
+  end # def self.from_io
+
   # Connect now.
   #
   # Timeout value is optional. If no timeout is given, this method
   # blocks until a connection is successful or an error occurs.
+  #
+  # You should check the return value of this method to determine if
+  # a connection was successful.
+  #
+  # Possible return values are on error include:
+  #
+  # * FTW::Connection::ConnectRefused
+  # * FTW::Connection::ConnectTimeout
+  #
+  # @return [nil] if the connection was successful
+  # @return [StandardError or subclass] if the connection failed
   def connect(timeout=nil)
     # TODO(sissel): Raise if we're already connected?
-    close if connected?
+    disconnect("reconnecting") if connected?
     host, port = @destinations.first.split(":")
     @destinations = @destinations.rotate # round-robin
 
@@ -88,10 +141,11 @@ class FTW::Connection
     # Connect with timeout
     begin
       @socket.connect_nonblock(sockaddr)
-    rescue IO::WaitWritable
+    rescue IO::WaitWritable, Errno::EINPROGRESS
       # Ruby actually raises Errno::EINPROGRESS, but for some reason
       # the documentation says to use this IO::WaitWritable thing...
       # I don't get it, but whatever :(
+
       if writable?(timeout)
         begin
           @socket.connect_nonblock(sockaddr) # check connection failure
@@ -99,18 +153,25 @@ class FTW::Connection
           # Ignore, we're already connected.
         rescue Errno::ECONNREFUSED => e
           # Fire 'disconnected' event with reason :refused
-          return e
+          return ConnectRefused.new("#{host}[#{@remote_address}]:#{port}")
+        rescue Errno::ETIMEDOUT
+          # This occurs when the system's TCP timeout hits, we have no control
+          # over this, as far as I can tell. *maybe* setsockopt(2) has a flag
+          # for this, but I haven't checked..
+          # TODO(sissel): We should instead do 'retry' unless we've exceeded
+          # the timeout.
+          return ConnectTimeout.new("#{host}[#{@remote_address}]:#{port}")
         end
       else
         # Connection timeout
         # Fire 'disconnected' event with reason :timeout
-        return ConnectTimeout.new
+        return ConnectTimeout.new("#{host}[#{@remote_address}]:#{port}")
       end
     end
 
     # We're now connected.
     @connected = true
-    return true
+    return nil
   end # def connect
 
   # Is this Connection connected?
@@ -138,7 +199,7 @@ class FTW::Connection
   #
   # This method is not guaranteed to read exactly 'length' bytes. See
   # IO#sysread
-  def read(timeout=nil)
+  def read(length=16384, timeout=nil)
     data = ""
     data.force_encoding("BINARY") if data.respond_to?(:force_encoding)
     have_pushback = !@pushback_buffer.empty?
@@ -151,7 +212,8 @@ class FTW::Connection
 
     if readable?(timeout)
       begin
-        @socket.sysread(@read_size, @read_buffer)
+        # Read at most 'length' data, so read less from the socket
+        @socket.sysread(@read_size - data.length, @read_buffer)
         data << @read_buffer
         return data
       rescue EOFError => e
@@ -191,8 +253,8 @@ class FTW::Connection
   #
   # The time out is in seconds. Fractional seconds are OK.
   def writable?(timeout)
-    ready = IO.select(nil, [@socket], nil, timeout)
-    return !ready.nil?
+    readable, writable, errors = IO.select(nil, [@socket], nil, timeout)
+    return !writable.nil?
   end # def writable?
 
   # Is this connection readable? Returns true if it is readable within
@@ -200,9 +262,8 @@ class FTW::Connection
   #
   # The time out is in seconds. Fractional seconds are OK.
   def readable?(timeout)
-    #return false if @reader_closed
-    ready = IO.select([@socket], nil, nil, timeout)
-    return !ready.nil?
+    readable, writable, errors = IO.select([@socket], nil, nil, timeout)
+    return !readable.nil?
   end # def readable?
 
   # The host:port
@@ -225,18 +286,30 @@ class FTW::Connection
     require "openssl"
     sslcontext = OpenSSL::SSL::SSLContext.new
     sslcontext.ssl_version = :TLSv1
-    # If you use VERIFY_NONE, you are removing an important piece 
+    # If you use VERIFY_NONE, you are removing the trust feature of TLS. Don't do that.
+    # Encryption without trust means you don't know who you are talking to.
     sslcontext.verify_mode = OpenSSL::SSL::VERIFY_PEER
     # TODO(sissel): Try to be smart about setting this default.
     sslcontext.ca_path = "/etc/ssl/certs"
     @socket = OpenSSL::SSL::SSLSocket.new(@socket, sslcontext)
 
+    # TODO(sissel): Set up local certificat/key stuff. This is required for
+    # server-side ssl operation, I think.
+
+    if client?
+      do_secure(:connect_nonblock)
+    else
+      do_secure(:accept_nonblock)
+    end
+  end # def secure
+
+  def do_secure(handshake_method)
     # SSLSocket#connect_nonblock will do the SSL/TLS handshake.
     # TODO(sissel): refactor this into a method that both secure and connect
     # methods can call.
     start = Time.now
     begin
-      @socket.connect_nonblock
+      @socket.send(handshake_method)
     rescue IO::WaitReadable, IO::WaitWritable
       # The ruby OpenSSL docs for 1.9.3 have example code saying I should use
       # IO::WaitReadable, but in the real world it raises an SSLError with
@@ -248,30 +321,43 @@ class FTW::Connection
       # raises, WaitWritable (ok, Errno::EINPROGRESS, technically)
       # Ruby's SSL exception for 'this call would block' is pretty shitty.
       #
-      # If the exception string is *not* 'read would block' we have a real
-      # problem.
+      # So we rescue both IO::Wait{Readable,Writable} and keep trying
+      # until timeout occurs.
+      #
       
       if !timeout.nil?
         time_left = timeout - (Time.now - start)
-        raise ConnectTimeout.new if time_left < 0
+        raise SecureHandshakeTimeout.new if time_left < 0
         r, w, e = IO.select([@socket], [@socket], nil, time_left)
       else
         r, w, e = IO.select([@socket], [@socket], nil, timeout)
       end
 
-      # try connect_nonblock again if the socket is ready
+      # keep going if the socket is ready
       retry if r.size > 0 || w.size > 0
+    rescue => e
+      @logger.warn(e)
+      raise e
     end
 
     @secure = true
-  end # def secure
+  end # def do_secure
 
   # Has this connection been secured?
   def secured?
     return @secure
   end # def secured?
 
+  def client?
+    return @mode == :client
+  end # def client?
+
+  def server?
+    return @mode == :server
+  end # def server?
+
   public(:connect, :connected?, :write, :read, :pushback, :disconnect,
-         :writable?, :readable?, :peer, :to_io, :secure, :secured?)
+         :writable?, :readable?, :peer, :to_io, :secure, :secured?,
+         :client?, :server?)
 end # class FTW::Connection
 

data/lib/ftw/cookies.rb

@@ -3,9 +3,11 @@ require "cabin"
 
 # Based on behavior and things described in RFC6265
 class FTW::Cookies
+
+  # This is a Cookie. It expires, has a value, a name, etc.
+  # I could have used stdlib CGI::Cookie, but it actually parses cookie strings
+  # incorrectly and also lacks the 'httponly' attribute.
   class Cookie
-    # I could use stdlib CGI::Cookie, but it actually parses cookie strings
-    # incorrectly and also lacks the 'httponly' attribute
     attr_accessor :name
     attr_accessor :value
 
@@ -18,12 +20,16 @@ class FTW::Cookies
 
     # TODO(sissel): Support 'extension-av' ? RFC6265 section 4.1.1
     # extension-av      = <any CHAR except CTLs or ";">
-    
+ 
+    # List of standard cookie attributes
+    STANDARD_ATTRIBUTES = [:domain, :path, :comment, :expires, :secure, :httponly]
+
+    # A new cookie. Value and attributes are optional.
     def initialize(name, value=nil, attributes={})
       @name = name
       @value = value
       
-      [:domain, :path, :comment, :expires, :secure, :httponly].each do |iv|
+      STANDARD_ATTRIBUTES.each do |iv|
         instance_variable_set("@#{iv.to_s}", attributes.delete(iv))
       end
 
@@ -52,9 +58,9 @@ class FTW::Cookies
             # TODO(sissel): Parse the Max-Age value and convert it to 'expires'
             #extra[:expires] = 
           when /^Domain=/
-            extra[:domain] = attr[7:]
+            extra[:domain] = attr[7..-1]
           when /^Path=/
-            extra[:path] = attr[5:]
+            extra[:path] = attr[5..-1]
           when /^Secure/
             extra[:secure] = true
           when /^HttpOnly/
@@ -66,20 +72,24 @@ class FTW::Cookies
     end # def Cookie.parse
   end # class Cookie
 
+  # A new cookies store
   def initialize
     @cookies = []
   end # def initialize
 
+  # Add a cookie 
   def add(name, value=nil, attributes={})
     cookie = Cookie.new(name, value, attributes)
     @cookies << cookie
   end # def add
 
+  # Add a cookie from a header 'Set-Cookie' value
   def add_from_header(set_cookie_string)
     cookie = Cookie.parse(set_cookie_string)
     @cookies << cookie
   end # def add_from_header
 
+  # Get cookies for a URL
   def for_url(url)
     # TODO(sissel): only return cookies that are valid for the url
     return @cookies