ftw 0.0.1 → 0.0.4

data/lib/ftw/cookies.rb

@@ -0,0 +1,87 @@
+require "ftw/namespace"
+require "cabin"
+
+# Based on behavior and things described in RFC6265
+class FTW::Cookies
+  class Cookie
+    # I could use stdlib CGI::Cookie, but it actually parses cookie strings
+    # incorrectly and also lacks the 'httponly' attribute
+    attr_accessor :name
+    attr_accessor :value
+
+    attr_accessor :domain
+    attr_accessor :path
+    attr_accessor :comment
+    attr_accessor :expires # covers both 'expires' and 'max-age' behavior
+    attr_accessor :secure
+    attr_accessor :httponly # part of RFC6265
+
+    # TODO(sissel): Support 'extension-av' ? RFC6265 section 4.1.1
+    # extension-av      = <any CHAR except CTLs or ";">
+    
+    def initialize(name, value=nil, attributes={})
+      @name = name
+      @value = value
+      
+      [:domain, :path, :comment, :expires, :secure, :httponly].each do |iv|
+        instance_variable_set("@#{iv.to_s}", attributes.delete(iv))
+      end
+
+      if !attributes.empty?
+        raise InvalidArgument.new("Invalid Cookie attributes: #{attributes.inspect}")
+      end
+    end # def initialize
+
+    # See RFC6265 section 4.1.1
+    def self.parse(set_cookie_string)
+      @logger ||= Cabin::Channel.get($0)
+      # TODO(sissel): Implement
+      # grammar is:
+      #  set-cookie-string = cookie-pair *( ";" SP cookie-av )
+      #  cookie-pair       = cookie-name "=" cookie-value
+      #  cookie-name       = token
+      #  cookie-value      = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE )
+      pair, *attributes = set_cookie_string.split(/\s*;\s*/)
+      name, value = pair.split(/\s*=\s*/)
+      extra = {}
+      attributes.each do |attr|
+        case attr
+          when /^Expires=/
+            #extra[:expires] = 
+          when /^Max-Age=/
+            # TODO(sissel): Parse the Max-Age value and convert it to 'expires'
+            #extra[:expires] = 
+          when /^Domain=/
+            extra[:domain] = attr[7:]
+          when /^Path=/
+            extra[:path] = attr[5:]
+          when /^Secure/
+            extra[:secure] = true
+          when /^HttpOnly/
+            extra[:httponly] = true
+          else
+            
+        end
+      end
+    end # def Cookie.parse
+  end # class Cookie
+
+  def initialize
+    @cookies = []
+  end # def initialize
+
+  def add(name, value=nil, attributes={})
+    cookie = Cookie.new(name, value, attributes)
+    @cookies << cookie
+  end # def add
+
+  def add_from_header(set_cookie_string)
+    cookie = Cookie.parse(set_cookie_string)
+    @cookies << cookie
+  end # def add_from_header
+
+  def for_url(url)
+    # TODO(sissel): only return cookies that are valid for the url
+    return @cookies
+  end # def for_url
+end # class FTW::Cookies

data/lib/ftw/crlf.rb

@@ -1,4 +1,4 @@
-require "net/ftw/namespace"
+require "ftw/namespace"
 
 module FTW::CRLF
   # carriage-return + line-feed

data/lib/ftw/dns.rb

@@ -1,9 +1,6 @@
 require "ftw/namespace"
 require "socket" # for Socket.gethostbyname
 
-# TODO(sissel): Switch to using Resolv::DNS since it lets you (the programmer)
-# choose dns configuration (servers, etc)
-#
 # I wrap whatever Ruby provides because it is historically very
 # inconsistent in implementation behavior across ruby platforms and versions.
 # In the future, this will probably implement the DNS protocol, but for now
@@ -12,14 +9,21 @@ require "socket" # for Socket.gethostbyname
 # I didn't really want to write a DNS library, but a consistent API and
 # behavior is necessary for my continued sanity :)
 class FTW::DNS
+  # TODO(sissel): Switch to using Resolv::DNS since it lets you (the programmer)
+  # choose dns configuration (servers, etc)
+
   V4_IN_V6_PREFIX = "0:" * 12
 
+  # Get a singleton instance of FTW::DNS
+  public
   def self.singleton
     @resolver ||= self.new
   end # def self.singleton
 
-  # This method is only intended to do A or AAAA lookups
-  # I may add PTR lookups later.
+  # Resolve a hostname.
+  #
+  # It will return an array of all known addresses for the host.
+  public
   def resolve(hostname)
     official, aliases, family, *addresses = Socket.gethostbyname(hostname)
     # We ignore family, here. Ruby will return v6 *and* v4 addresses in
@@ -35,6 +39,11 @@ class FTW::DNS
     end
   end # def resolve
 
+  # Resolve hostname and choose one of the results at random.
+  # 
+  # Use this method if you are connecting to a hostname that resolves to
+  # multiple addresses.
+  public
   def resolve_random(hostname)
     addresses = resolve(hostname)
     return addresses[rand(addresses.size)]

data/lib/ftw/http/headers.rb

@@ -1,4 +1,4 @@
-require "net/ftw/namespace"
+require "ftw/namespace"
 require "ftw/crlf"
 
 # HTTP Headers
@@ -30,6 +30,8 @@ class FTW::HTTP::Headers
     @headers[field.downcase] = value
   end # def set
 
+  alias_method :[]=, :set
+
   # Set a header field to a specific value.
   # Any existing value(s) for this field are destroyed.
   def include?(field)
@@ -99,6 +101,8 @@ class FTW::HTTP::Headers
     return @headers[field]
   end # def get
 
+  alias_method :[], :get
+
   # Iterate over headers. Given to the block are two arguments, the field name
   # and the field value. For fields with multiple values, you will receive
   # that same field name multiple times, like:
@@ -115,8 +119,18 @@ class FTW::HTTP::Headers
     end
   end # end each
 
+  public
+  def to_hash
+    return @headers
+  end # def to_hash
+
   public
   def to_s
     return @headers.collect { |name, value| "#{name}: #{value}" }.join(CRLF) + CRLF
   end # def to_s
+  
+  public
+  def inspect
+    return "#{self.class.name} <#{to_hash.inspect}>"
+  end # def inspect
 end # class FTW::HTTP::Headers

data/lib/ftw/http/message.rb

@@ -58,7 +58,15 @@ module FTW::HTTP::Message
     return @body
   end # def body
 
-  # Does this message have a message body?
+  # Does this message have a message body / content?
+  public
+  def content?
+    # In HTTP 1.1, there is a body if response sets Content-Length *or*
+    # Transfer-Encoding, it has a body. Otherwise, there is no body.
+    return (headers.include?("Content-Length") and headers["Content-Length"].to_i > 0) \
+      || headers.include?("Transfer-Encoding")
+  end # def content?
+
   public
   def body?
     return @body.nil?

data/lib/ftw/namespace.rb

@@ -1,3 +1,4 @@
 module FTW
   module HTTP; end
+  class WebSocket; end
 end

data/lib/ftw/pool.rb

@@ -0,0 +1,50 @@
+require "ftw/namespace"
+require "thread"
+
+# A simple thread-safe resource pool.
+#
+# Resources in this pool must respond to 'available?'.
+# For best results, your resources should just 'include FTW::Poolable'
+#
+# The primary use case was as a way to pool FTW::Connection instances.
+class FTW::Pool
+  def initialize
+    # Pool is a hash of arrays.
+    @pool = Hash.new { |h,k| h[k] = Array.new }
+    @lock = Mutex.new
+  end # def initialize
+
+  # Add an object to the pool with a given identifier. For example:
+  #
+  #     pool.add("www.google.com:80", connection1)
+  #     pool.add("www.google.com:80", connection2)
+  #     pool.add("github.com:443", connection3)
+  def add(identifier, object)
+    @lock.synchronize do
+      @pool[identifier] << object
+    end
+    return object
+  end # def add
+
+  # Fetch a resource from this pool. If no available resources
+  # are found, the 'default_block' is invoked and expected to
+  # return a new resource to add to the pool that satisfies
+  # the fetch..
+  #
+  # Example:
+  #
+  #     pool.fetch("github.com:443") do 
+  #       conn = FTW::Connection.new("github.com:443")
+  #       conn.secure
+  #       conn
+  #     end
+  def fetch(identifier, &default_block)
+    @lock.synchronize do
+      object = @pool[identifier].find { |o| o.available? }
+      return object if !object.nil?
+    end
+    # Otherwise put the return value of default_block in the
+    # pool and return it.
+    return add(identifier, default_block.call)
+  end # def fetch
+end # class FTW::Pool

data/lib/ftw/poolable.rb

@@ -0,0 +1,19 @@
+require "ftw/namespace"
+
+# A poolable mixin. This is for use with the FTW::Pool class.
+module FTW::Poolable
+  # Mark that this resource is in use
+  def mark
+    @__in_use = true
+  end # def mark
+
+  # Release this resource
+  def release
+    @__in_use = false
+  end # def release
+
+  # Is this resource available for use?
+  def available?
+    return !@__in_use
+  end # def avialable?
+end # module FTW::Poolable

data/lib/ftw/request.rb

@@ -1,9 +1,11 @@
-require "ftw/namespace"
-require "ftw/http/message"
 require "addressable/uri" # gem addressable
-require "uri" # ruby stdlib
-require "http/parser" # gem http_parser.rb
+require "cabin" # gem cabin
 require "ftw/crlf"
+require "ftw/http/message"
+require "ftw/namespace"
+require "ftw/response"
+require "http/parser" # gem http_parser.rb
+require "uri" # ruby stdlib
 
 # An HTTP Request.
 #
@@ -11,6 +13,7 @@ require "ftw/crlf"
 class FTW::Request
   include FTW::HTTP::Message
   include FTW::CRLF
+  include Cabin::Inspectable
 
   # The http method. Like GET, PUT, POST, etc..
   # RFC2616 5.1.1 - <http://tools.ietf.org/html/rfc2616#section-5.1.1>
@@ -25,53 +28,114 @@ class FTW::Request
   # RFC2616 5.1.2 - <http://tools.ietf.org/html/rfc2616#section-5.1.2>
   attr_accessor :request_uri
 
-  # Lemmings. Everyone else calls Request-URI the 'path' - so I should too.
+  # Lemmings. Everyone else calls Request-URI the 'path' (including me, most of
+  # the time), so let's just follow along.
   alias_method :path, :request_uri
 
+  # RFC2616 section 14.23 allows the Host header to include a port, but I have
+  # never seen this in practice, and I shudder to think about what poorly-behaving
+  # web servers will barf if the Host header includes a port. So, instead of
+  # storing the port in the Host header, it is stored here. It is not included
+  # in the Request when sent from a client and it is not used on a server.
+  attr_accessor :port
+
+  # This is *not* an RFC2616 field. It exists so that the connection handling
+  # this request knows what protocol to use. The protocol for this request.
+  # Usually 'http' or 'https' or perhaps 'spdy' maybe?
+  attr_accessor :protocol
+
+  # Make a new request with a uri if given.
+  #
+  # The uri is used to set the address, protocol, Host header, etc.
   public
   def initialize(uri=nil)
     super()
-    use_uri(uri) if !uri.nil?
+    @port = 80
+    @protocol = "http"
     @version = 1.1
+    use_uri(uri) if !uri.nil?
   end # def initialize
 
-  # Set the connection to use for this request.
-  public
-  def connection=(connection)
-    @connection = connection
-  end # def connection=
-
+  # Execute this request on a given connection: Writes the request, returns a
+  # Response object.
+  #
+  # This method will block until the HTTP response header has been completely
+  # received. The body will not have been read yet at the time of this
+  # method's return.
+  #
+  # The 'connection' should be a FTW::Connection instance, but it might work
+  # with a normal IO object.
+  #
   public
   def execute(connection)
-    connection.write(to_s + CRLF)
+    tries = 3
+    begin
+      connection.write(to_s + CRLF)
+    rescue => e
+      # TODO(sissel): Rescue specific exceptions, not just anything.
+      # Reconnect and retry
+      if tries > 0
+        connection.connect
+        retry
+      else
+        raise e
+      end
+    end
+
+    # TODO(sissel): Support request a body.
 
     parser = HTTP::Parser.new
-    parser.on_headers_complete = proc { state = :body; :stop }
+    headers_done = false
+    parser.on_headers_complete = proc { headers_done = true; :stop }
 
-    data = connection.read(16384)
-    parser << data
-    # TODO(sissel): use connection.unread() if we finish reading headers
-    # and there's still some data left that is part of the body.
-  end # def execute
+    # headers_done will be set to true when parser finishes parsing the http
+    # headers for this request
+    while !headers_done
+      # TODO(sissel): This read could toss an exception of the server aborts
+      # prior to sending the full headers. Figure out a way to make this happy.
+      # Perhaps fabricating a 500 response?
+      data = connection.read
+
+      # Feed the data into the parser. Offset will be nonzero if there's 
+      # extra data beyond the header.
+      offset = parser << data
+    end
 
-  # TODO(sissel): Methods to write:
-  # 1. Parsing a request, use HTTP::Parser from http_parser.rb
-  # 2. Building a request from a URI or Addressable::URI
+    # Done reading response header
+    response = FTW::Response.new
+    response.version = "#{parser.http_major}.#{parser.http_minor}".to_f
+    response.status = parser.status_code
+    parser.headers.each { |field, value| response.headers.add(field, value) }
+
+    # If we consumed part of the body while parsing headers, put it back
+    # onto the connection's read buffer so the next consumer can use it.
+    if offset < data.length
+      connection.pushback(data[offset .. -1])
+    end
+    return response
+  end # def execute
 
+  # Use a URI to help fill in parts of this Request.
   public
   def use_uri(uri)
     # Convert URI objects to Addressable::URI
-    uri = Addressable::URI.parse(uri.to_s) if uri.is_a?(URI)
+    case uri
+      when URI, String
+        uri = Addressable::URI.parse(uri.to_s)
+    end
 
-    # TODO(sissel): Use normalized versions of these fields?
-    # uri.host
-    # uri.port
-    # uri.scheme
-    # uri.path
+    # TODO(sissel): Use uri.password and uri.user to set Authorization basic
+    # stuff.
     # uri.password
     # uri.user
     @request_uri = uri.path
     @headers.set("Host", uri.host)
+    @protocol = uri.scheme
+    if uri.port.nil?
+      # default to port 80
+      uri.port = { "http" => 80, "https" => 443 }.fetch(uri.scheme, 80)
+    end
+    @port = uri.port
     
     # TODO(sissel): support authentication
   end # def use_uri

data/lib/ftw/response.rb

@@ -0,0 +1,179 @@
+require "ftw/namespace"
+require "ftw/http/message"
+require "cabin" # gem cabin
+require "http/parser" # gem http_parser.rb
+
+# An HTTP Response.
+#
+# See RFC2616 section 6: <http://tools.ietf.org/html/rfc2616#section-6>
+class FTW::Response 
+  include FTW::HTTP::Message
+
+  # The HTTP version number
+  # See RFC2616 section 6.1: <http://tools.ietf.org/html/rfc2616#section-6.1>
+  attr_reader :version
+
+  # The http status code (RFC2616 6.1.1)
+  # See RFC2616 section 6.1.1: <http://tools.ietf.org/html/rfc2616#section-6.1.1>
+  attr_reader :status
+
+  # The reason phrase (RFC2616 6.1.1)
+  # See RFC2616 section 6.1.1: <http://tools.ietf.org/html/rfc2616#section-6.1.1>
+  attr_reader :reason
+
+  # Translated from the recommendations listed in RFC2616 section 6.1.1
+  # See RFC2616 section 6.1.1: <http://tools.ietf.org/html/rfc2616#section-6.1.1>
+  STATUS_REASON_MAP =  {
+    100 => "Continue",
+    101 => "Switching Protocols",
+    200 => "OK",
+    201 => "Created",
+    202 => "Accepted",
+    203 => "Non-Authoritative Information",
+    204 => "No Content",
+    205 => "Reset Content",
+    206 => "Partial Content",
+    300 => "Multiple Choices",
+    301 => "Moved Permanently",
+    302 => "Found",
+    303 => "See Other",
+    304 => "Not Modified",
+    305 => "Use Proxy",
+    307 => "Temporary Redirect",
+    400 => "Bad Request",
+    401 => "Unauthorized",
+    402 => "Payment Required",
+    403 => "Forbidden",
+    404 => "Not Found",
+    405 => "Method Not Allowed",
+    406 => "Not Acceptable"
+  } # STATUS_REASON_MAP
+
+  attr_accessor :body
+
+  # Create a new Response.
+  public
+  def initialize
+    super
+    @logger = Cabin::Channel.get
+    @reason = "" # Empty reason string by default. It is not required.
+  end # def initialize
+
+  # Is this response a redirect?
+  public
+  def redirect?
+    # redirects are 3xx
+    return @status >= 300 && @status < 400
+  end # redirect?
+
+  # Is this response an error?
+  public
+  def error?
+    # 4xx and 5xx are errors
+    return @status >= 400 && @status < 600
+  end # def error?
+
+  # Set the status code
+  public
+  def status=(code)
+    code = code.to_i if !code.is_a?(Fixnum)
+    # TODO(sissel): Validate that 'code' is a 3 digit number
+    @status = code
+
+    # Attempt to set the reason if the status code has a known reason
+    # recommendation. If one is not found, default to the current reason.
+    @reason = STATUS_REASON_MAP.fetch(@status, @reason)
+  end # def status=
+
+  # Get the status-line string, like "HTTP/1.0 200 OK"
+  public
+  def status_line
+    # First line is 'Status-Line' from RFC2616 section 6.1
+    # Status-Line = HTTP-Version SP Status-Code SP Reason-Phrase CRLF
+    # etc...
+    return "HTTP/#{version} #{status} #{reason}"
+  end # def status_line
+
+  # Define the Message's start_line as status_line
+  alias_method :start_line, :status_line
+
+  # Set the body of this response. In most cases this will be a FTW::Connection when
+  # Response objects are being created by a FTW::Agent. In Server cases,
+  # the body is likely to be a string or enumerable.
+  public
+  def body=(connection_or_string_or_enumerable)
+    @body = connection_or_string_or_enumerable
+  end # def body=
+
+  # Read the body of this Response. The block is called with chunks of the
+  # response as they are read in.
+  #
+  # This method is generally only called by http clients, not servers.
+  public
+  def read_body(&block)
+    if @body.respond_to?(:read)
+      if headers.include?("Content-Length") and headers["Content-Length"].to_i > 0
+        @logger.debug("Reading body with Content-Length")
+        read_body_length(headers["Content-Length"].to_i, &block)
+      elsif headers["Transfer-Encoding"] == "chunked"
+        @logger.debug("Reading body with chunked encoding")
+        read_body_chunked(&block)
+      end
+
+      # If this is a poolable resource, release it (like a FTW::Connection)
+      @body.release if @body.respond_to?(:release)
+    elsif !@body.nil?
+      yield @body
+    end
+  end # def read_body
+
+  # Read the length bytes from the body. Yield each chunk read to the block
+  # given. This method is generally only called by http clients, not servers.
+  private
+  def read_body_length(length, &block)
+    remaining = length
+    while remaining > 0
+      data = @body.read
+      @logger.debug("Read bytes", :length => data.size)
+      if data.size > remaining
+        # Read too much data, only wanted part of this. Push the rest back.
+        yield data[0..remaining]
+        remaining = 0
+        @body.pushback(data[remaining .. -1]) if remaining < 0
+      else
+        yield data
+        remaining -= data.size
+      end
+    end
+  end # def read_body_length
+
+  # This is kind of messed, need to fix it.
+  private
+  def read_body_chunked(&block)
+    parser = HTTP::Parser.new
+
+    # Fake fill-in the response we've already read into the parser.
+    parser << to_s
+    parser << CRLF
+    parser.on_body = block
+    done = false
+    parser.on_message_complete = proc { done = true }
+
+    while !done # will break on special conditions below
+      data = @body.read
+      offset = parser << data
+      if offset != data.length
+        raise "Parser dis not consume all data read?"
+      end
+    end
+  end # def read_body_chunked
+
+  # Is this Response the result of a successful Upgrade request?
+  public
+  def upgrade?
+    return false unless status == 101 # "Switching Protocols"
+    return false unless headers["Connection"] == "Upgrade"
+    return true
+  end # def upgrade?
+end # class FTW::Response
+