ftw 0.0.1

data/lib/net/ftw/connection2.rb

@@ -0,0 +1,247 @@
+require "cabin" # rubygem "cabin"
+require "net/ftw/dns"
+require "net/ftw/namespace"
+require "socket"
+require "timeout" # ruby stdlib, just for the Timeout exception.
+require "backport-bij" # for Array#rotate, IO::WaitWritable, etc, in ruby < 1.9
+
+# A network connection. This is TCP.
+#
+# Example:
+#
+#     conn = Net::FTW::Connection.new("www.google.com:80")
+#     conn.on(CONNECTED) do |address| 
+#       puts "Connected to #{address} (#{conn.peer})"
+#       conn.write("GET / HTTP/1.0\r\n\r\n")
+#     end
+#     conn.on(DATA) do |data|
+#       puts data
+#     end
+#     conn.run
+#
+# You can use IO::select on this objects of this type.
+class Net::FTW::Connection2
+
+  # Events
+  CONNECTED = :connected
+  DISCONNECTED = :disconnected
+  READER_CLOSED = :reader_closed
+  DATA = :data
+
+  # Disconnection reasons
+  TIMEOUT = :timeout
+  REFUSED = :refused
+  LOST = :lost
+  INTENTIONAL = :intentional
+
+  # A new network connection.
+  # The 'destination' argument can be an array of strings or a single string.
+  # String format is expected to be "host:port"
+  #
+  # Example:
+  #
+  #     conn = Net::FTW::Connection.new(["1.2.3.4:80", "1.2.3.5:80"])
+  #
+  # If you specify multiple destinations, they are used in a round-robin
+  # decision made during reconnection.
+  public
+  def initialize(destinations)
+    if destinations.is_a?(String)
+      @destinations = [destinations]
+    else
+      @destinations = destinations
+    end
+
+    # Handlers are key => array of callbacks
+    @handlers = Hash.new { |h,k| h[k] = [] }
+
+    @connect_timeout = 2
+
+    # Use a fixed-size string that we set to BINARY encoding.
+    # Not all byte sequences are UTF-8 friendly :0
+    @read_size = 16384
+    @read_buffer = " " * @read_size
+
+    # Tell Ruby 1.9 that this string is a binary string, not utf-8 or somesuch.
+    if @read_buffer.respond_to?(:force_encoding)
+      @read_buffer.force_encoding("BINARY")
+    end
+
+    # TODO(sissel): Validate @destinations
+  end # def initialize
+
+  public
+  def connect(timeout=nil)
+    # TODO(sissel): Raise if we're already connected?
+    close if connected?
+    host, port = @destinations.first.split(":")
+    @destinations = @destinations.rotate # round-robin
+
+    # Do dns resolution on the host. If there are multiple
+    # addresses resolved, return one at random.
+    @remote_address = Net::FTW::DNS.singleton.resolve_random(host)
+
+    family = @remote_address.include?(":") ? Socket::AF_INET6 : Socket::AF_INET
+    @socket = Socket.new(family, Socket::SOCK_STREAM, 0)
+    sockaddr = Socket.pack_sockaddr_in(port, @remote_address)
+    # TODO(sissel): Support local address binding
+
+    # Connect with timeout
+    begin
+      @socket.connect_nonblock(sockaddr)
+    rescue IO::WaitWritable
+      # Ruby actually raises Errno::EINPROGRESS, but for some reason
+      # the documentation says to use this IO::WaitWritable thing...
+      # I don't get it, but whatever :(
+      if writable?(timeout)
+        begin
+          @socket.connect_nonblock(sockaddr) # check connection failure
+        rescue Errno::EISCONN # Ignore, we're already connected.
+        rescue Errno::ECONNREFUSED => e
+          # Fire 'disconnected' event with reason :refused
+          trigger(DISCONNECTED, :refused, e)
+        end
+      else
+        # Connection timeout
+        # Fire 'disconnected' event with reason :timeout
+        trigger(DISCONNECTED, :connect_timeout, nil)
+      end
+    end
+
+    # We're now connected.
+    trigger(CONNECTED, "#{host}:#{port}")
+  end # def connect
+
+  # Is this Connection connected?
+  public
+  def connected?
+    return @connected
+  end # def connected?
+
+  # Write data to this connection.
+  # This method blocks until the write succeeds unless a timeout is given.
+  #
+  # Returns the number of bytes written (See IO#syswrite)
+  public
+  def write(data, timeout=nil)
+    #connect if !connected?
+    if writable?(timeout)
+      return @socket.syswrite(data)
+    else
+      raise Timeout::Error.new
+    end
+  end # def write
+
+  # Read data from this connection
+  # This method blocks until the read succeeds unless a timeout is given.
+  #
+  # This method is not guaranteed to read exactly 'length' bytes. See
+  # IO#sysread
+  public
+  def read(length, timeout=nil)
+    if readable?(timeout)
+      begin
+        @socket.sysread(length, @read_buffer)
+        return @read_buffer
+      rescue EOFError
+        trigger(READER_CLOSED)
+      end
+    else
+      raise Timeout::Error.new
+    end
+  end # def read
+
+  # End this connection
+  public
+  def disconnect(reason=INTENTIONAL)
+    begin 
+      #@reader_closed = true
+      @socket.close_read
+    rescue IOError => e
+      # Ignore
+    end
+
+    begin 
+      @socket.close_write
+    rescue IOError => e
+      # Ignore
+    end
+
+    trigger(DISCONNECTED, reason)
+  end # def disconnect
+
+  # Is this connection writable? Returns true if it is writable within
+  # the timeout period. False otherwise.
+  #
+  # The time out is in seconds. Fractional seconds are OK.
+  public
+  def writable?(timeout)
+    ready = IO.select(nil, [@socket], nil, timeout)
+    return !ready.nil?
+  end # def writable?
+
+  # Is this connection readable? Returns true if it is readable within
+  # the timeout period. False otherwise.
+  #
+  # The time out is in seconds. Fractional seconds are OK.
+  public
+  def readable?(timeout)
+    #return false if @reader_closed
+    ready = IO.select([@socket], nil, nil, timeout)
+    return !ready.nil?
+  end # def readable?
+
+  protected
+  def connected(address)
+    @remote_address = nil
+    @connected = true
+  end # def connected
+
+  protected
+  def disconnected(reason, error)
+    @remote_address = nil
+    @connected = false
+  end # def disconnected
+
+  # The host:port
+  public
+  def peer
+    return @remote_address
+  end # def peer
+
+  # Run this Connection.
+  # This is generally meant for Threaded or synchronous operation. 
+  # For EventMachine, see TODO(sissel): Implement EventMachine support.
+  public
+  def run
+    connect(@connect_timeout) if not connected?
+    while connected?
+      read_and_trigger
+    end
+  end # def run
+
+  # Read data and trigger data callbacks.
+  #
+  # This is mainly useful if you are implementing your own run loops
+  # and IO::select shenanigans.
+  public
+  def read_and_trigger
+    data = read(@read_size)
+    if data.length == 0
+      disconnect(EOFError)
+    else
+      trigger(DATA, data)
+    end
+  end # def read_and_trigger
+
+  # Support 'to_io' so you can use IO::select on this object.
+  public
+  def to_io
+    return @socket
+  end
+
+  def trigger(*args)
+    p :trigger => args
+  end
+end # class Net::FTW::Connection
+

data/lib/net/ftw/crlf.rb

@@ -0,0 +1,6 @@
+require "net/ftw/namespace"
+
+module Net::FTW::CRLF
+  # carriage-return + line-feed
+  CRLF = "\r\n"
+end

data/lib/net/ftw/dns.rb

@@ -0,0 +1,57 @@
+require "net/ftw/namespace"
+require "socket"
+# TODO(sissel): Switch to using Resolv::DNS since it lets you (the programmer)
+# choose dns configuration (servers, etc)
+# I still need to wrap whatever Ruby provides because it is historically very
+# inconsistent in implementation behavior across ruby platforms and versions.
+#
+# I didn't really want to write a DNS library.
+class Net::FTW::DNS
+  V4_IN_V6_PREFIX = "0:" * 12
+
+  def self.singleton
+    @resolver ||= self.new
+  end # def self.singleton
+
+  # This method is only intended to do A or AAAA lookups
+  # I may add PTR lookups later.
+  def resolve(hostname)
+    official, aliases, family, *addresses = Socket.gethostbyname(hostname)
+    # We ignore family, here. Ruby will return v6 *and* v4 addresses in
+    # the same gethostbyname() call. It is confusing. 
+    #
+    # Let's just rely entirely on the length of the address string.
+    return addresses.collect do |address|
+      if address.length == 16
+        unpack_v6(address)
+      else
+        unpack_v4(address)
+      end
+    end
+  end # def resolve
+
+  def resolve_random(hostname)
+    addresses = resolve(hostname)
+    return addresses[rand(addresses.size)]
+  end # def resolve_random
+
+  private
+  def unpack_v4(address)
+    return address.unpack("C4").join(".")
+  end # def unpack_v4
+
+  private
+  def unpack_v6(address)
+    if address.length == 16
+      # Unpack 16 bit chunks, convert to hex, join with ":"
+      address.unpack("n8").collect { |p| p.to_s(16) } \
+        .join(":").sub(/(?:0:(?:0:)+)/, "::")
+    else 
+      # assume ipv4
+      # Per the following sites, "::127.0.0.1" is valid and correct
+      # http://en.wikipedia.org/wiki/IPv6#IPv4-mapped_IPv6_addresses
+      # http://www.tcpipguide.com/free/t_IPv6IPv4AddressEmbedding.htm
+      "::" + unpack_v4(address)
+    end
+  end # def unpack_v6
+end # class Net::FTW::DNS

data/lib/net/ftw/http.rb

@@ -0,0 +1,2 @@
+require "net/ftw/namespace"
+require "net/ftw/http/client"

data/lib/net/ftw/http/client.rb

@@ -0,0 +1,116 @@
+require "net/ftw/http/connection"
+require "net/ftw/http/request"
+require "net/ftw/http/response"
+require "net/ftw/namespace"
+require "socket" # ruby stdlib
+
+# TODO(sissel): Split this out into a general 'client' class (outside http)
+# TODO(sissel): EventMachine support
+
+# A client should be like a web browser. It should support lots of active
+# connections.
+class Net::FTW::HTTP::Client
+  include Net::FTW::CRLF
+
+  # Create a new HTTP client. You probably only need one of these.
+  def initialize
+    @connections = []
+  end # def initialize
+  
+  # TODO(sissel): This method may not stay. I dunno yet.
+  public
+  def get(uri, headers={})
+    # TODO(sissel): enforce uri scheme options? (ws, wss, http, https?)
+    prepare("GET", uri, headers)
+  end # def get
+
+  public
+  def prepare(method, uri, headers={})
+    uri = Addressable::URI.parse(uri.to_s) if uri.is_a?(URI)
+    uri.port ||= 80
+
+    request = Net::FTW::HTTP::Request.new(uri)
+    response = Net::FTW::HTTP::Response.new
+    request.method = method
+    request.version = 1.1
+    headers.each do |key, value|
+      request.headers[key] = value
+    end
+
+    # TODO(sissel): This is starting to feel like not the best way to implement
+    # protocols.
+    connection = Net::FTW::HTTP::Connection.new("#{uri.host}:#{uri.port}")
+    connection.on(connection.class::CONNECTED) do |address|
+      connection.write(request.to_s)
+      connection.write(CRLF)
+    end
+    connection.on(connection.class::HEADERS_COMPLETE) do |version, status, headers|
+      response.status = status
+      response.version = version
+      headers.each { |field, value| response.headers.add(field, value) }
+
+      # TODO(sissel): Split these BODY handlers into separate body-handling
+      # classes.
+      if response.headers.include?("Content-Length")
+        length = response.headers.get("Content-Length").to_i
+        connection.on(connection.class::MESSAGE_BODY) do |data|
+          length -= data.size
+          #$stdout.write data
+          if length <= 0
+            if response.headers.get("Connection") == "close"
+              connection.disconnect
+            else
+              p :response_complete => response.headers.get("Content-Length")
+              # TODO(sissel): This connection is now ready for another HTTP
+              # request.
+            end
+
+            # TODO(sissel): What to do with the extra bytes?
+            if length < 0
+              # Length is negative, will be offset on end of data string
+              $stderr.puts :TOOMANYBYTES => data[length .. -1]
+            end
+          end
+        end 
+      elsif response.headers.get("Transfer-Encoding") == "chunked"
+        connection.on(connection.class::MESSAGE_BODY) do |data|
+          # TODO(sissel): Handle chunked encoding
+          p :chunked => data
+        end
+      elsif response.version == 1.1
+        # No content-length nor transfer-encoding. If this is HTTP/1.1, this is
+        # an error, I think. I need to find the specific part of RFC2616 that
+        # specifies this.
+        connection.disconnect("Invalid HTTP Response received. Response " \
+          "version claimed 1.1 but no Content-Length nor Transfer-Encoding "\
+          "header was set in the response.")
+      end
+    end # connection.on HEADERS_COMPLETE
+    #connection.run
+    return connection
+  end # def prepare
+
+  def prepare2(method, uri, headers={})
+    uri = Addressable::URI.parse(uri.to_s) if uri.is_a?(URI)
+    uri.port ||= 80
+
+    request = Net::FTW::HTTP::Request.new(uri)
+    response = Net::FTW::HTTP::Response.new
+    request.method = method
+    request.version = 1.1
+    headers.each do |key, value|
+      request.headers[key] = value
+    end
+
+    # TODO(sissel): This is starting to feel like not the best way to implement
+    # protocols.
+    id = "#{uri.scheme}://#{uri.host}:#{uri.port}/..."
+    connection = Net::FTW::HTTP::Connection.new("#{uri.host}:#{uri.port}")
+    @connections[id] = connection
+  end # def prepare2
+
+  # TODO(sissel): 
+  def run
+    # Select across all active connections, do read_and_trigger, etc.
+  end # def run
+end # class Net::FTW::HTTP::Client

data/lib/net/ftw/http/client2.rb

@@ -0,0 +1,80 @@
+require "net/ftw/connection2"
+require "net/ftw/http/request"
+require "net/ftw/http/response"
+require "net/ftw/namespace"
+require "socket" # ruby stdlib
+
+# TODO(sissel): Split this out into a general 'client' class (outside http)
+# TODO(sissel): EventMachine support
+
+# A client should be like a web browser. It should support lots of active
+# connections.
+class Net::FTW::HTTP::Client2
+  include Net::FTW::CRLF
+
+  # Create a new HTTP client. You probably only need one of these.
+  def initialize
+    @connections = []
+  end # def initialize
+  
+  # TODO(sissel): This method may not stay. I dunno yet.
+  public
+  def get(uri, headers={})
+    # TODO(sissel): enforce uri scheme options? (ws, wss, http, https?)
+    return prepare("GET", uri, headers)
+  end # def get
+
+  public
+  def prepare(method, uri, headers={})
+    uri = Addressable::URI.parse(uri.to_s) if uri.is_a?(URI)
+    uri.port ||= 80
+
+    request = Net::FTW::HTTP::Request.new(uri)
+    response = Net::FTW::HTTP::Response.new
+    request.method = method
+    request.version = 1.1
+    headers.each do |key, value|
+      request.headers[key] = value
+    end
+
+    connection = Net::FTW::Connection2.new("#{uri.host}:#{uri.port}")
+    return fiberup(connection, request, response)
+  end # def prepare
+
+  def fiberup(connection, request, response)
+    # Body just passes through
+    body = Fiber.new do |data|
+      Fiber.yield data
+    end
+
+    # Parse the HTTP headers
+    headers = Fiber.new do |data|
+      parser = HTTP::Parser.new
+      headers_done = false
+      parser.on_headers_complete = proc { headers_done = true; :stop }
+      while true do
+        offset = parser << data
+        if headers_done
+          version = "#{parser.http_major}.#{parser.http_minor}".to_f
+          p :processing
+          Fiber.yield [version, parser.status_code, parser.headers]
+          p :processing
+          # Transfer control to the 'body' fiber.
+          body.transfer(data[offset..-1])
+        end
+        p :waiting
+        data = Fiber.resume
+      end
+    end
+
+    connect = Fiber.new do
+      connection.connect
+      connection.write(request.to_s + CRLF)
+      while true do
+        data = connection.read(16384)
+        headers.resume data
+      end
+    end
+    return connect
+  end # def fiberup
+end # class Net::FTW::HTTP::Client2