ftw 0.0.1

data/README.md

@@ -0,0 +1,49 @@
+# For The Web
+
+net/http is pretty much not good.
+
+I want:
+
+* A HTTP client that acts as a full user agent, not just a single connection.
+* HTTP and SPDY support.
+* WebSockets support.
+* SSL/TLS support.
+* An API that lets me do what I need.
+* Server and Client modes.
+* Support for both normal operation and EventMachine would be nice.
+
+## DONE
+
+* TCP connection 
+* DNS resolution (wraps Socket.gethostname)
+* HTTP client partially done
+
+## TODO
+
+* Tests, yo.
+* Logging, yo. With cabin, obviously.
+* [DNS in Ruby stdlib is broken](https://github.com/jordansissel/experiments/tree/master/ruby/dns-resolving-bug), I need to write my own
+
+## API Scratch
+
+### Common case
+
+    agent = FTW::Agent.new
+    request = agent.get("http://www.google.com/")
+    response = request.execute
+
+### SPDY
+
+    # SPDY should automatically be attempted. The caller should be unaware.
+
+### WebSockets
+
+    # 'http(s)' or 'ws(s)' urls are valid here. They will mean the same thing.
+    request = agent.websocket("http://somehost/endpoint")
+    # Set auth header
+    request["Authorization"] = ...
+    request["Cookie"] = ...
+
+    websocket, error = request.execute
+    # Now websocket.read receives a message, websocket.write sends a message.
+

data/lib/ftw/agent.rb

@@ -0,0 +1,40 @@
+require "ftw/namespace"
+require "ftw/request"
+require "ftw/connection"
+require "addressable/uri"
+
+# This should act as a proper agent.
+#
+# * Keep cookies. Offer local-storage of cookies
+# * Reuse connections. HTTP 1.1 Connection: keep-alive
+# * HTTP Upgrade support
+# * Websockets
+# * SSL/TLS
+class FTW::Agent
+  # TODO(sissel): All standard HTTP methods should be defined here.
+  # Also allow users to specify non-standard methods.
+  
+  def initialize
+  end
+
+  # Returns a FTW::Request
+  # TODO(sissel): SSL/TLS support
+  def get(uri, options={})
+    return request("GET", uri, options)
+  end # def get
+
+  def request(method, uri, options)
+    request = FTW::Request.new(uri)
+    request.method = method
+    request.connection = connection(uri.host, uri.port)
+    return request
+  end # def request
+
+  # Returns a FTW::Connection connected to this host:port.
+  # TODO(sissel): Implement connection reuse
+  # TODO(sissel): support SSL/TLS
+  private
+  def connection(host, port)
+    return FTW::Connection.new("#{host}:#{port}")
+  end # def connect
+end # class FTW::Agent

data/lib/ftw/connection.rb

@@ -0,0 +1,231 @@
+require "cabin" # rubygem "cabin"
+require "net/ftw/dns"
+require "net/ftw/namespace"
+require "socket"
+require "timeout" # ruby stdlib, just for the Timeout exception.
+require "backport-bij" # for Array#rotate, IO::WaitWritable, etc, in ruby < 1.9
+
+# A network connection. This is TCP.
+#
+# You can use IO::select on this objects of this type.
+# (at least, in MRI you can)
+class FTW::Connection
+  # A new network connection.
+  # The 'destination' argument can be an array of strings or a single string.
+  # String format is expected to be "host:port"
+  #
+  # Example:
+  #
+  #     conn = FTW::Connection.new(["1.2.3.4:80", "1.2.3.5:80"])
+  #
+  # If you specify multiple destinations, they are used in a round-robin
+  # decision made during reconnection.
+  public
+  def initialize(destinations)
+    if destinations.is_a?(String)
+      @destinations = [destinations]
+    else
+      @destinations = destinations
+    end
+
+    @connect_timeout = 2
+
+    # Use a fixed-size string that we set to BINARY encoding.
+    # Not all byte sequences are UTF-8 friendly :0
+    @read_size = 16384
+    @read_buffer = " " * @read_size
+
+    # Tell Ruby 1.9 that this string is a binary string, not utf-8 or somesuch.
+    if @read_buffer.respond_to?(:force_encoding)
+      @read_buffer.force_encoding("BINARY")
+    end
+
+    # TODO(sissel): Validate @destinations
+  end # def initialize
+
+  public
+  def connect(timeout=nil)
+    # TODO(sissel): Raise if we're already connected?
+    close if connected?
+    host, port = @destinations.first.split(":")
+    @destinations = @destinations.rotate # round-robin
+
+    # Do dns resolution on the host. If there are multiple
+    # addresses resolved, return one at random.
+    @remote_address = FTW::DNS.singleton.resolve_random(host)
+
+    # Addresses with colon ':' in them are assumed to be IPv6
+    family = @remote_address.include?(":") ? Socket::AF_INET6 : Socket::AF_INET
+    @socket = Socket.new(family, Socket::SOCK_STREAM, 0)
+
+    # This api is terrible. pack_sockaddr_in? This isn't C, man...
+    sockaddr = Socket.pack_sockaddr_in(port, @remote_address)
+    # TODO(sissel): Support local address binding
+
+    # Connect with timeout
+    begin
+      @socket.connect_nonblock(sockaddr)
+    rescue IO::WaitWritable
+      # Ruby actually raises Errno::EINPROGRESS, but for some reason
+      # the documentation says to use this IO::WaitWritable thing...
+      # I don't get it, but whatever :(
+      if writable?(timeout)
+        begin
+          @socket.connect_nonblock(sockaddr) # check connection failure
+        rescue Errno::EISCONN # Ignore, we're already connected.
+        rescue Errno::ECONNREFUSED => e
+          # Fire 'disconnected' event with reason :refused
+          trigger(DISCONNECTED, :refused, e)
+        end
+      else
+        # Connection timeout
+        # Fire 'disconnected' event with reason :timeout
+        trigger(DISCONNECTED, :connect_timeout, nil)
+      end
+    end
+
+    # We're now connected.
+    trigger(CONNECTED, "#{host}:#{port}")
+  end # def connect
+
+  # Is this Connection connected?
+  public
+  def connected?
+    return @connected
+  end # def connected?
+
+  # Write data to this connection.
+  # This method blocks until the write succeeds unless a timeout is given.
+  #
+  # Returns the number of bytes written (See IO#syswrite)
+  public
+  def write(data, timeout=nil)
+    #connect if !connected?
+    if writable?(timeout)
+      return @socket.syswrite(data)
+    else
+      raise Timeout::Error.new
+    end
+  end # def write
+
+  # Read data from this connection
+  # This method blocks until the read succeeds unless a timeout is given.
+  #
+  # This method is not guaranteed to read exactly 'length' bytes. See
+  # IO#sysread
+  public
+  def read(timeout=nil)
+    if readable?(timeout)
+      if !@unread_buffer.empty?
+        data = @unread_buffer
+        @unread_buffer = ""
+        return data
+      end
+
+      begin
+        @socket.sysread(@read_size, @read_buffer)
+        return @read_buffer
+      rescue EOFError
+        trigger(READER_CLOSED)
+      end
+    else
+      raise Timeout::Error.new
+    end
+  end # def read
+
+  # Un-read some data
+  public
+  def unread(data)
+    @unread_buffer << data
+  end # def unread
+
+  # End this connection, specifying why.
+  public
+  def disconnect(reason)
+    begin 
+      #@reader_closed = true
+      @socket.close_read
+    rescue IOError => e
+      # Ignore
+    end
+
+    begin 
+      @socket.close_write
+    rescue IOError => e
+      # Ignore
+    end
+
+    trigger(DISCONNECTED, reason)
+  end # def disconnect
+
+  # Is this connection writable? Returns true if it is writable within
+  # the timeout period. False otherwise.
+  #
+  # The time out is in seconds. Fractional seconds are OK.
+  public
+  def writable?(timeout)
+    ready = IO.select(nil, [@socket], nil, timeout)
+    return !ready.nil?
+  end # def writable?
+
+  # Is this connection readable? Returns true if it is readable within
+  # the timeout period. False otherwise.
+  #
+  # The time out is in seconds. Fractional seconds are OK.
+  public
+  def readable?(timeout)
+    #return false if @reader_closed
+    ready = IO.select([@socket], nil, nil, timeout)
+    return !ready.nil?
+  end # def readable?
+
+  protected
+  def connected(address)
+    @remote_address = nil
+    @connected = true
+  end # def connected
+
+  protected
+  def disconnected(reason, error)
+    @remote_address = nil
+    @connected = false
+  end # def disconnected
+
+  # The host:port
+  public
+  def peer
+    return @remote_address
+  end # def peer
+
+  # Run this Connection.
+  # This is generally meant for Threaded or synchronous operation. 
+  # For EventMachine, see TODO(sissel): Implement EventMachine support.
+  public
+  def run
+    connect(@connect_timeout) if not connected?
+    while connected?
+      read_and_trigger
+    end
+  end # def run
+
+  # Read data and trigger data callbacks.
+  #
+  # This is mainly useful if you are implementing your own run loops
+  # and IO::select shenanigans.
+  public
+  def read_and_trigger
+    data = read(@read_size)
+    if data.length == 0
+      disconnect(EOFError)
+    else
+      trigger(DATA, data)
+    end
+  end # def read_and_trigger
+
+  # Support 'to_io' so you can use IO::select on this object.
+  public
+  def to_io
+    return @socket
+  end
+end # class FTW::Connection
+

data/lib/ftw/crlf.rb

@@ -0,0 +1,6 @@
+require "net/ftw/namespace"
+
+module FTW::CRLF
+  # carriage-return + line-feed
+  CRLF = "\r\n"
+end

data/lib/ftw/dns.rb

@@ -0,0 +1,62 @@
+require "ftw/namespace"
+require "socket" # for Socket.gethostbyname
+
+# TODO(sissel): Switch to using Resolv::DNS since it lets you (the programmer)
+# choose dns configuration (servers, etc)
+#
+# I wrap whatever Ruby provides because it is historically very
+# inconsistent in implementation behavior across ruby platforms and versions.
+# In the future, this will probably implement the DNS protocol, but for now
+# chill in the awkward, but already-written, ruby stdlib.
+#
+# I didn't really want to write a DNS library, but a consistent API and
+# behavior is necessary for my continued sanity :)
+class FTW::DNS
+  V4_IN_V6_PREFIX = "0:" * 12
+
+  def self.singleton
+    @resolver ||= self.new
+  end # def self.singleton
+
+  # This method is only intended to do A or AAAA lookups
+  # I may add PTR lookups later.
+  def resolve(hostname)
+    official, aliases, family, *addresses = Socket.gethostbyname(hostname)
+    # We ignore family, here. Ruby will return v6 *and* v4 addresses in
+    # the same gethostbyname() call. It is confusing. 
+    #
+    # Let's just rely entirely on the length of the address string.
+    return addresses.collect do |address|
+      if address.length == 16
+        unpack_v6(address)
+      else
+        unpack_v4(address)
+      end
+    end
+  end # def resolve
+
+  def resolve_random(hostname)
+    addresses = resolve(hostname)
+    return addresses[rand(addresses.size)]
+  end # def resolve_random
+
+  private
+  def unpack_v4(address)
+    return address.unpack("C4").join(".")
+  end # def unpack_v4
+
+  private
+  def unpack_v6(address)
+    if address.length == 16
+      # Unpack 16 bit chunks, convert to hex, join with ":"
+      address.unpack("n8").collect { |p| p.to_s(16) } \
+        .join(":").sub(/(?:0:(?:0:)+)/, "::")
+    else 
+      # assume ipv4
+      # Per the following sites, "::127.0.0.1" is valid and correct
+      # http://en.wikipedia.org/wiki/IPv6#IPv4-mapped_IPv6_addresses
+      # http://www.tcpipguide.com/free/t_IPv6IPv4AddressEmbedding.htm
+      "::" + unpack_v4(address)
+    end
+  end # def unpack_v6
+end # class FTW::DNS

data/lib/ftw/http/headers.rb

@@ -0,0 +1,122 @@
+require "net/ftw/namespace"
+require "ftw/crlf"
+
+# HTTP Headers
+#
+# See RFC2616 section 4.2: <http://tools.ietf.org/html/rfc2616#section-4.2>
+#
+# Section 14.44 says Field Names in the header are case-insensitive, so
+# this library always forces field names to be lowercase. This includes
+# get() calls.
+#
+#    headers.set("HELLO", "world")
+#    headers.get("hello")   # ===> "world"
+#
+class FTW::HTTP::Headers
+  include Enumerable
+  include FTW::CRLF
+
+  # Make a new headers container. You can pass a hash of 
+  public
+  def initialize(headers={})
+    super()
+    @version = 1.1
+    @headers = headers
+  end # def initialize
+
+  # Set a header field to a specific value.
+  # Any existing value(s) for this field are destroyed.
+  def set(field, value)
+    @headers[field.downcase] = value
+  end # def set
+
+  # Set a header field to a specific value.
+  # Any existing value(s) for this field are destroyed.
+  def include?(field)
+    @headers.include?(field.downcase)
+  end # def include?
+
+  # Add a header field with a value.
+  #
+  # If this field already exists, another value is added.
+  # If this field does not already exist, it is set.
+  def add(field, value)
+    field = field.downcase
+    if @headers.include?(field)
+      if @headers[field].is_a?(Array)
+        @headers[field] << value
+      else
+        @headers[field] = [@headers[field], value]
+      end
+    else
+      set(field, value)
+    end
+  end # def add
+
+  # Removes a header entry. If the header has multiple values
+  # (like X-Forwarded-For can), you can delete a specific entry
+  # by passing the value of the header field to remove.
+  #
+  #     # Remove all X-Forwarded-For entries
+  #     headers.remove("X-Forwarded-For") 
+  #     # Remove a specific X-Forwarded-For entry
+  #     headers.remove("X-Forwarded-For", "1.2.3.4")
+  #
+  # * If you remove a field that doesn't exist, no error will occur.
+  # * If you remove a field value that doesn't exist, no error will occur.
+  # * If you remove a field value that is the only value, it is the same as
+  #   removing that field by name.
+  def remove(field, value=nil)
+    field = field.downcase
+    if value.nil?
+      # no value, given, remove the entire field.
+      @headers.delete(field)
+    else
+      field_value = @headers[field]
+      if field_value.is_a?(Array)
+        # remove a specific value
+        field_value.delete(value)
+        # Down to a String again if there's only one value.
+        if field_value.size == 1
+          set(field, field_value.first)
+        end
+      else
+        # Remove this field if the value matches
+        if field_value == value
+          remove(field)
+        end
+      end
+    end
+  end # def remove
+
+  # Get a field value. 
+  # 
+  # This will return:
+  #   * String if there is only a single value for this field
+  #   * Array of String if there are multiple values for this field
+  def get(field)
+    field = field.downcase
+    return @headers[field]
+  end # def get
+
+  # Iterate over headers. Given to the block are two arguments, the field name
+  # and the field value. For fields with multiple values, you will receive
+  # that same field name multiple times, like:
+  #    yield "Host", "www.example.com"
+  #    yield "X-Forwarded-For", "1.2.3.4"
+  #    yield "X-Forwarded-For", "1.2.3.5"
+  def each(&block)
+    @headers.each do |field_name, field_value|
+      if field_value.is_a?(Array)
+        field_value.map { |value| yield field_name, v }
+      else
+        yield field_name, field_value
+      end
+    end
+  end # end each
+
+  public
+  def to_s
+    return @headers.collect { |name, value| "#{name}: #{value}" }.join(CRLF) + CRLF
+  end # def to_s
+end # class FTW::HTTP::Headers