rstyx 0.2.0 → 0.3.0

data/lib/rstyx/client.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/ruby
 #
-# Copyright (C) 2005 Rafael Sevilla
+# Copyright (C) 2005-2007 Rafael Sevilla
 # This file is part of RStyx
 #
 # RStyx is free software; you can redistribute it and/or modify
@@ -15,646 +15,599 @@
 #
 # You should have received a copy of the GNU Lesser General Public
 # License along with RStyx; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# 02111-1307 USA.
+# Foundation, Inc., 51 Franklin St., Fifth Floor, Boston, MA
+# 02110-1301 USA.
 #
 # Styx Client
 #
 # Author:: Rafael R. Sevilla (mailto:dido@imperium.ph)
-# Copyright:: Copyright (c) 2005 Rafael R. Sevilla
+# Copyright:: Copyright (c) 2005-2007 Rafael R. Sevilla
 # License:: GNU Lesser General Public License
 #
-# $Id: client.rb,v 1.10 2005/11/01 13:02:18 dido Exp $
+# $Id: client.rb 201 2007-06-08 17:13:58Z dido $
 #
 
-require 'English'
-require 'socket'
 require 'thread'
 require 'timeout'
-require 'rstyx/errors'
-require 'rstyx/messages'
+require 'rubygems'
+require 'eventmachine'
 
 module RStyx
 
   module Client
 
     ##
-    # Utility functions
+    # Message receiving module for the Styx client.  The client will
+    # assemble all inbound messages. 
     #
-    class Util
+    module StyxClient
+      attr_accessor :sentmessages
+
+      def post_init
+        @msgbuffer = ""
+        @lock = Mutex.new
+        @sentmessages = {}
+      end
+
       ##
-      # Gets the next unused tag or fid.
+      # Send a message asynchronously.
       #
-      # +limit+:: [Fixnum] maximum value
-      # +func+:: [Proc] predicate that determines if tag or fid i is in use
-      # +thing+:: [String] what we're trying to get
-      # return:: an unused value
+      # +message+:: [StyxMessage] the message to be sent
+      # +block+:: [Proc] the callback to use
+      # return:: [Fixnum] the tag number used.
       #
-      def Util.get_first_unused(limit, func, thing)
-        found = false
-        val = nil
-        0.upto(limit-1) do |i|
-          if func.call(i)
-            i += 1
-          else
-            val = i
-            break
+      def send_message_async(message, &block)
+        # store the message and callback indexed by tag
+        @lock.synchronize do
+          if message.tag.nil?
+            # If a tag has not been explicitly specified, get
+            # a new tag for the message. We use the current
+            # thread's object ID as the base and use what
+            # amounts to a linear probing algorithm to
+            # determine a new tag in case of collisions.
+            tag = Thread.current.object_id % MAX_TAG
+            while @sentmessages.has_key?(tag)
+              tag += 1
+            end
+            message.tag = tag
           end
+          @sentmessages[message.tag] = [message, block]
         end
 
-        if val.nil?
-          # this should probably be impossible
-          raise StyxException.new(sprintf("No more free %s!", thing))
-        end
-        return(val)
+        DEBUG > 0 && puts(" >> #{message.to_s}")
+        DEBUG > 1 && puts(" >> #{message.to_bytes.unpack("H*").inspect}")
+        send_data(message.to_bytes)
+        return(message.tag)
       end
 
-    end
-
-    ##
-    # An abstract class for Styx connections. DO NOT INSTANTIATE THIS
-    # CLASS DIRECTLY!
-    #
-    # A +Connection+ subclass instance acts in the same way as the
-    # +File+ class would under Ruby, and many of the class methods for
-    # File are implemented as instance methods for Connection, and are
-    # relevant to the Styx filesystem the Connection is connected to.
-    # There are a few differences: for instance, new is not equivalent
-    # to open, for obvious reasons.
-    #
-    class Connection
-      attr_reader :msize, :version, :rootfid, :user
-
       ##
-      # Create a new Styx connection.  If a block is passed, yield the
-      # new connection to the block and do a disconnect when the block
-      # finishes.
+      # Send a message synchronously.  If an error occurs, a
+      # StyxException is raised.
       #
-      # +user+:: [String] the user to connect as
+      # +message+:: The Styx message to send.
+      # +timeout+:: optional timeout for receiving the response.
       #
-      def initialize(user="")
-        @lock = Mutex.new       # used to synchronize threads
-        @condvar = ConditionVariable.new
-        @rmessages = Hash.new   # filled with rmessages that have arrived
-        @usedfids = Array.new   # list of fids currently in use
-        @pendingclunks = Hash.new # outstanding Tclunk messages (tags and fids)
-        @conn = self.startconn  # get the connection
-
-        # start message receiving thread
-        @receiver = Thread.new { receive_messages }
-
-        tv = Message::Tversion.new
-        rver = send_message(tv)
-        @msize = rver.msize
-        @version = rver.version
-        @rootfid = get_free_fid # the fid representing the serv. root
-        rattach = send_message(Message::Tattach.new(@rootfid, user))
-        @user = user
+      def send_message(message, timeout=0)
+        # The queue here holds the response message, and is used
+        # to communicate with the receive_data thread, which ultimately
+        # calls the block passed to send_message_async.
+        queue = Queue.new
+        send_message_async(message) do |tx,rx|
+          # Enqueue the response message -- this runs in the
+          # receive_data thread
+          queue << rx
+        end
+        Timeout::timeout(timeout, StyxException.new("timeout waiting for a reply to #{message.to_s}")) do
+          # This will block until the queue contains something
+          resp = queue.shift
+          # Check the response to see if it is the response to
+          # the transmitted message.
+          if resp.class == Message::Rerror
+            raise StyxException.new("#{resp.ename}")
+          end
 
-        if block_given?
-          begin
-            yield self
-          ensure
-            self.disconnect
+          if resp.ident != message.ident + 1
+            raise StyxException.new("Unexpected #{resp.to_s} received in response to #{message.to_s}")
           end
-        else
-          return(self)
+          return(resp)
         end
       end
 
-      protected
-
       ##
-      # This method should be overridden by subclasses, and should return
-      # a subclass of IO (e.g. a TCPSocket) on which the read and write
-      # methods can be used.
+      # Receive data from the network connection, called by EventMachine.
       #
-      def startconn
-        return(nil)             # Override me!
-      end
+      def receive_data(data)
+        @msgbuffer << data
+        DEBUG > 1 && puts(" << #{data.unpack("H*").inspect}")
+        while @msgbuffer.length > 4
+          length = @msgbuffer.unpack("V")[0]
+          # Break out if there is not enough data in the message
+          # buffer to construct a message.
+          if @msgbuffer.length < length
+            break
+          end
 
-      ##
-      # Thread that listens for incoming messages on the socket.
-      #
-      # FIXME: Frankly, I think this is quite ugly...
-      #
-      def receive_messages
-        buffer = ""
-        data = nil
-        done = false
-        loop do
-          begin
-            data = @conn.read(1)
-          rescue Exception => e
-            done = true
-            raise StyxException.new("error reading from socket: #{e.message}")
+          # Decode the received data
+          message, @msgbuffer = @msgbuffer.unpack("a#{length}a*")
+          styxmsg = Message::StyxMessage.from_bytes(message)
+          DEBUG > 0 && puts(" << #{styxmsg.to_s}")
+          # and look for its tag in the hash of received messages
+          tmsg, cb = @lock.synchronize do
+            @sentmessages.delete(styxmsg.tag)
+          end
+
+          if tmsg.nil?
+            # Discard unrecognized messages.
+            next
           end
 
-          # There may be more than one message (or a partial message)
-          buffer << data
-          if buffer.length >= 4
-            length = buffer[0..3].unpack("V")[0]
-            if buffer.length >= length
-              message = buffer[0..(length-1)]
-              buffer = buffer[length..(buffer.length)]
-              styxmsg = Message::StyxMessage.decode(message)
-
-              # let everyone know there's a new message available
-              @lock.synchronize do
-                @rmessages[styxmsg.tag] = styxmsg
-                @condvar.signal
+          if styxmsg.class == Message::Rflush
+            # We need to delete the oldtag as well, and send the
+            # rflush to the original sender if possible, so they
+            # don't wait forever.
+            if tmsg.respond_to?(:oldtag)
+              otmsg, ocb = @lock.synchronize do
+                @sentmessages.delete(tmsg.oldtag)
               end
             end
+
+            if !otmsg.nil? && !ocb.nil?
+              ocb.call(otmsg, styxmsg)
+            end
           end
-        end
-      end
 
-      ##
-      # Get a new tag for use in new messages.
-      #
-      def get_free_tag
-        return(Util.get_first_unused(0xffff,
-                                     lambda {|i| @rmessages.has_key?(i) },
-                                     "tags"))
+          # Now, activate the callback block.
+          if !(tmsg.nil? || cb.nil?)
+            cb.call(tmsg, styxmsg)
+          end
+
+          # after all this is done, there may still be enough data in
+          # the message buffer for more messages so keep looping.
+        end
+        # If we get here, we don't have enough data in the buffer to
+        # build a new message.
       end
 
       ##
-      # Send a message and return immediately, returning the message's tag
-      #
-      # +message+:: [StyxMessage] the message to be sent
-      # return:: [Fixnum] the tag number used
+      # Disconnect from the remote server.
       #
-      def send_message_async(message)
-        if (message.tag.nil?)
-          message.tag = get_free_tag
+      def disconnect
+        # flush all outstanding messages before disconnect
+        sentmessages.each_key do |tag|
+          rflush = send_message(Message::Tflush.new(:oldtag => tag))
         end
 
-        # If this is a Tclunk message we are trying to close the file.
-        # Remember the fid so we can free it from the list of used fids
-        # when we get the Rclunk corresponding to it.
-        if message.mtype == Message::StyxMessage::TCLUNK
-          @pendingclunks[message.tag] = message.fid
-        end
+        EventMachine::stop_event_loop
+      end
 
-        # puts "sending message #{message.to_s}"
-        @conn.write(message.to_bytes)
-        return(message.tag)
+    end                         # module StyxClient
+
+    class Connection
+      attr_accessor :usedfids, :pendingclunks, :umask
+      attr_reader :connectstate, :msize, :version
+      attr_reader :rootdirectory, :rootfid
+
+      def initialize(auth=nil)
+        @usedfids = []
+        @pendingclunks = {}
+        @rpendingclunks = {}
+        @conn = nil
+        @rootfid = nil
+        @eventthread = nil
+        @authenticator = auth
+        @clunklock = Mutex.new
+        @umask = ::File.umask
       end
 
       ##
-      # Waits for the message with the given tag to arrive, then returns it.
-      #
-      # +tag+:: [Fixnum] Tag to wait for
-      # return:: [Message::StyxMessage] the message received
+      # Get a new free FID.
       #
-      def get_reply(tag)
-        msg = nil
-        @lock.synchronize do
-          # wait for a new message to arrive
-          while @rmessages[tag].nil?
-            @condvar.wait(@lock)
+      def get_free_fid
+        found = false
+        val = nil
+        0.upto(MAX_FID) do |i|
+          unless @usedfids.include?(i)
+            val = i
+            break
           end
-          msg = @rmessages[tag]
-          @rmessages.delete(tag)
-        end
-
-        # puts "Received message #{msg.to_s}"
-        # Special things to do for certain messages
-        case msg.mtype
-        when Message::StyxMessage::RERROR
-          # raise an exception when we receive an Rerror
-          raise StyxException.new(msg.ename)
-        when Message::StyxMessage::RCLUNK
-          # find the fid corresponding to the original Tclunk and free it
-          fid = @pendingclunks[tag]
-          @pendingclunks.delete(tag)
-          return_fid(fid)
         end
-        return(msg)
-      end
 
-      public
-
-      ##
-      # Get a new fid where it's needed.
-      #
-      def get_free_fid
-        fid = Util.get_first_unused(0xffffffff,
-                                    lambda {|i| @usedfids.include?(i) },
-                                    "fids")
-        @usedfids << fid
-        return(fid)
+        if val.nil?
+          raise StyxException.new("No more free fids")
+        end
+        @usedfids << val
+        return(val)
       end
 
       ##
-      # returns a fid after we are finished with it.
-      #
-      # +fid+:: [Fixnum] the fid to return
+      # Returns a fid after we're done using it.
       #
       def return_fid(fid)
         @usedfids.delete(fid)
       end
 
-      ##
-      # Send a message and wait for the reply
-      #
-      # +message+:: [StyxMessage] the message to be sent
-      # return:: [StyxMessage] the reply to our message
-      #
-      def send_message(message)
-        tag = send_message_async(message)
-        return(get_reply(tag))
-      end
+      protected
 
       ##
-      # Disconnect from the remote server.
+      # This method is used to prepare the connection, and should be
+      # defined by subclasses.
       #
-      def disconnect
-        # Clunk all outstanding fids in reverse order so root fid gets
-        # clunked last.
-        while (@usedfids.length > 0)
-          # The fid is automatically removed from @usedfids when the
-          # Rclunk message is received
-          rclunk = send_message(Message::Tclunk.new(@usedfids[-1]))
-        end
-
-        # Flush all outstanding messages
-        @rmessages.each_key do |tag|
-          rflush = send_message(Message::Tflush.new(tag))
-        end
-        # kill the receiver thread (FIXME: there has got to be a cleaner way!)
-        @receiver.kill
-        @conn.close
+      def prepare_connection
+        raise StyxException.new("No prepare_connection method defined")
       end
 
+      public
+
       ##
-      # Open a file on the connection.  Throws a StyxException if it doesn't
-      # exist.  If a block is specified along with the file, it is passed
-      # the new StyxFile, and the StyxFile is closed when the block
-      # terminates.
-      #
-      # +path+:: the path to the Styx file to be opened
-      # +mode+:: the mode to open the file (which can be r, w, r+, or e)
-      # +perm+:: the (optional) permissions bit mask.  This is only
-      #          relevant if the open creates a file.
-      # +rfid+:: the root fid.  If this is nil, use the root fid we got
-      #          when we connected.
-      #
-      # All parameters are optional.  If no path is specified, it defaults
-      # to reading the root directory on the Styx server.
-      #
-      # FIXME: This doesn't deal with the case where MAXWELEM is reached
-      #        yet.
-      # TODO: Allow the caller to make use of the Styx file access mode
-      #       values (OREAD, OWRITE, ORDWR, OEXEC, OTRUNC, ORCLOSE).
+      # Connect to a remote Styx server.  If a block is passed, yield
+      # self to the block and then do a disconnect when the block
+      # finishes.
       #
-      def open(path="", mode='r', perm=0644, rfid=nil)
-        # get a new fid for the file
-        fid = get_free_fid
+      def connect(&block)
+        prepare_connection()
 
-        if rfid.nil?
-          rfid = @rootfid
-        end
-
-        #
-        # The mode string is the standard Ruby mode string, viz.
-        # "r" - read-only, start at beginning of file
-        # "r+" - read/write, start at beginning of file
-        # "w" - write only, truncate existing file or create it if it
-        #       doesn't exist.
-        # "w+" - read/write, truncate existing file or create it if it
-        #        doesn't exist
-        # "a" - write only, starts at end of file if it exists otherwise
-        #       creates a new file for writing.
-        # "a+" - read/write, starts at end of file if it exists, otherwise
-        #        creates a new file for writing.
-        #
-        # There is also a mode specific to RStyx, not a standard mode
-        # for Ruby:
+        # Connection has been established.  Begin the handshaking process
+        # with the remote server.
         #
-        # "e" - execute the file
+        # 1. Send a Tversion message and check the response from the
+        #    remote Styx server.
         #
-        # The "b" suffix on DOS/Windows is not recognized as valid.
-        #
-        read = true
-        write = false
-        create = false
-        truncate = false
-        append = false
-        mval = Message::Topen::OREAD
-        case mode
-        when "r"
-          # all default above
-        when "r+"
-          write = true
-          mval = Message::Topen::ORDWR
-        when "w"
-          read = false
-          write = true
-          create = true
-          truncate = true
-          mval = Message::Topen::OWRITE
-        when "w+"
-          read = true
-          write = true
-          create = true
-          truncate = true
-          mval = Message::Topen::ORDWR
-        when "a"
-          read = false
-          write = true
-          create = true
-          append = true
-          mval = Message::Topen::OWRITE
-        when "a+"
-          read = true
-          write = true
-          create = true
-          append = true
-          mval = Message::Topen::ORDWR
-        when "e"
-          read = true
-          write = true
-          mval = Message::Topen::OEXEC
-        else
-          raise StyxException.new("Invalid mode string '#{mode}'")
-        end
-
-        if truncate
-          mval |= Message::Topen::OTRUNC
+        rver = send_message(Message::Tversion.new(:msize => 8216,
+                                                  :version => "9P2000"))
+        if (rver.version != "9P2000")
+          raise StyxException.new("Server uses unsupported Styx version #{rver.version}")
         end
-
-        iclass = StyxIO
-        begin
-          twalk = Message::Twalk.new(rfid, fid, path)
-          rwalk = send_message(twalk)
-          # if the rwalk has some other length than the number of path
-          # elements in the original twalk, we have failed.
-          if rwalk.qids.length != twalk.path_elements.length
-            raise StyxException.new(sprintf("%s: no such file or directory", twalk.path_elements[rwalk.qids.length]))
-          end
-          open_msg = Message::Topen.new(fid, mval)
-          # Determine if the file is actually a directory.  Such a file
-          # would have its Qid.qtype high bit set to 1.  In this case,
-          # mode can only be 'r', and instead of a StyxIO, a StyxDir
-          # instance is created instead.
+        @msize = rver.msize
+        @version = rver.version
+        rfid = nil
+        # 2. Attach to the remote server
+        if @auth.nil?
+          # unauthenticated connection
+          rfid = get_free_fid
+          rattach = send_message(Message::Tattach.new(:fid => rfid,
+                                                      :afid => NOFID,
+                                                      :uname => ENV['USER'],
+                                                      :aname => ""))
+        else
           #
-          # If the number of path elements involved is zero, assume a
-          # directory, as the root fid is always one.
+          # 3. Perform authentication based on the passed authenticator
+          #    object.
           #
-          if twalk.path_elements.length == 0 ||
-              rwalk.qids[twalk.path_elements.length-1].qtype & 0x80 != 0
-            iclass = StyxDir
-          end
-
-        rescue StyxException => se
-          # If the walk generated an error, see if the create flag
-          # was true.  If so, the error may be ignored, and the
-          # open_msgclass set to Message::Tcreate.  Otherwise, return
-          # the fid and reraise the exception.
-          if create
-            # Walk to the directory of the file to be made, if possible
-            begin
-              rwalk = send_message(Message::Twalk.new(rfid, fid,
-                                                      File.dirname(path)))
-            rescue StyxException => se
-              return_fid(fid)
-              raise se            # reraise the exception
-            end
-            open_msg = Message::Tcreate.new(fid, File.basename(path),
-                                            mval, perm)
-          else
-            return_fid(fid)
-            raise se            # reraise the exception
-          end
+          # If we have an authenticator object, we call its authenticator
+          # method with ourself.
+          #
+          rfid = @auth.authenticate(self)
         end
 
-        begin
-          # Create, open, and return a StyxIO object
-          ropen = send_message(open_msg)
-          fp = iclass.new(self, path, fid, ropen.iounit, mode)
-
-          # if we're supposed to append, seek to the end of the file
-          if append
-            rstat = send_message(Message::Tstat.new(fid))
-            fp.seek(rstat.stat.length, :seek_set)
-          end
+        # If we get here, we're connected, and rfid represents the root
+        # fid of the connection
+        @rootfid = rfid
 
-          if block_given?
-            begin
-            yield fp
-            ensure
-              fp.close
-            end
-          else
-            return(fp)
+        if block_given?
+          begin
+            yield self
+          ensure
+            self.disconnect
           end
-        rescue StyxException => se
-          return_fid(fid)
-          raise se            # reraise the exception
+        else
+          return(self)
         end
       end
 
       ##
-      # Delete the files specified.  Unlike File.delete, this method
-      # should also work properly on directories
+      # Disconnect from the remote server.
       #
-      def delete(*files)
-        files.each do |file|
-          fid = get_free_fid
-          # walk to the file to be deleted
+      def disconnect
+        # Clunk all outstanding fids in reverse order so the root fid
+        # gets clunked last.
+        while (@usedfids.length > 0)
           begin
-            rwalk = send_message(Message::Twalk.new(@rootfid, fid, file))
-          rescue StyxException => se
-            return_fid(fid)
-            raise se
+            rclunk = tclunk(@usedfids[-1], true)
+          rescue
+            # An error is most likely a no such fid error.  Return the fid
+            # manually in this case.
+            return_fid(@usedfids[-1])
           end
+        end
 
-          begin
-            rremove = send_message(Message::Tremove.new(fid))
-            return_fid(fid)
-          rescue StyxException => se
-            # because the Twalk succeeded, we need to clunk the fid
-            begin
-              rclunk = send_message(Message::Tclunk.new(fid))
-            rescue StyxException => se
-              return_fid(fid)
+        @conn.disconnect
+        @eventthread.kill
+      end
+
+      ##
+      # Send a message, and return the response.  Delegates to
+      # @conn#send_message.  Do not use this method to send Tclunk
+      # messages!
+      #
+      def send_message(msg, timeout=0)
+        @conn.send_message(msg, timeout)
+      end
+
+      ##
+      # Fire and forget a Tclunk for some fid.  When the Rclunk is
+      # received, return the fid.  USE THIS METHOD, AND THIS METHOD
+      # ONLY, to send Tclunk messages.
+      #
+      def tclunk(fid, sync=false)
+        if @rpendingclunks.has_key?(fid)
+          return
+        end
+        q = nil
+        if sync
+          q = Queue.new
+        end
+        tag = @conn.send_message_async(Message::Tclunk.new(:fid => fid)) do |tx,rx|
+          # Test whether the response is an Rclunk.
+          if rx.class != Message::Rclunk
+            # this is an error condition, but it will only get reported
+            # if Thread.abort_on_exception is set to true, or if
+            # the tclunk is synchronous
+            exc = StyxException.new("#{tx.to_s} received #{rx.to_s}")
+            if sync
+              q << exc
+            else
+              raise exc
             end
-            raise se
+          end
+          # return the FID
+          fid = @pendingclunks.delete(tag)
+          @rpendingclunks.delete(fid)
+          return_fid(fid)
+          if sync
+            q << fid
+          end
+        end
+        @pendingclunks[tag] = fid
+        @rpendingclunks[fid] = tag
+        if sync
+          res = q.shift
+          if res.class == StyxException
+            raise res
           end
         end
       end
 
-      alias rmdir delete
-      alias unlink delete
-
       ##
-      # Create a new directory named by +dirname+, with permissions
-      # specified by an optional parameter +perm+.
+      # Open a file on the remote server, throwing a StyxException if the
+      # file can't be found or opened in a given mode.
       #
-      def mkdir(dirname, perm=0755)
-        begin
-          fid = get_free_fid
-          perm |= Message::Tcreate::DMDIR
-          rwalk = send_message(Message::Twalk.new(@rootfid, fid,
-                                                  File.dirname(dirname)))
-        rescue StyxException => se
-          return_fid(fid)
-          raise se
+      # +path+:: The path of the file relative to the server root.
+      # +mode+:: Integer representing the mode, or one of "r", "r+",
+      #          "w", "w+", "a", "a+", "e" as aliases
+      # +return+:: A File object representing the opened file, or
+      #          possibly a Directory object if the file was a directory.
+      #
+      # If a block is passed, it will yield the file object to the block
+      # and close the file when the block finishes (actually it will pass
+      # the block on to the StyxFile#open method, which does just that).
+      #
+      def open(path, mode="r", perm=0666, &block)
+        file = File.new(self, path)
+
+        append = false
+        create = false
+        numeric_mode = nil
+        if mode.is_a?(Integer)
+          numeric_mode = mode
+        else
+          case mode.to_s
+          when "r"
+            numeric_mode = OREAD
+          when "r+"
+            numeric_mode = ORDWR
+          when "w"
+            numeric_mode = OTRUNC | OWRITE
+            create = true
+          when "w+"
+            numeric_mode = OTRUNC | ORDWR
+            create = true
+          when "a"
+            numeric_mode = OWRITE
+            append = true
+            create = true
+          when "a+"
+            numeric_mode = ORDWR
+            append = true
+            create = true
+          when "e"
+            numeric_mode = OEXEC
+          end
         end
 
-        begin
-          rcreate = send_message(Message::Tcreate.new(fid,
-                                                      File.basename(dirname),
-                                                      Message::Topen::OREAD,
-                                                      perm))
-        rescue StyxException => se
-          return_fid(fid)
-          raise se
-        ensure
-          rclunk = send_message(Message::Tclunk.new(fid))
+        fp = file.open(numeric_mode, perm, create, &block)
+        if append
+          fp.seek(0, 2)
         end
+        return(fp)
       end
 
-    end
+    end                         # class Connection
 
     ##
-    # TCP connection instance of a Styx connection
+    # TCP connection subclass.
     #
     class TCPConnection < Connection
-      attr_reader :host, :port
-
-      ##
-      # Create a Styx connection over TCP
-      #
-      # +host+:: [String] the host to connect to
-      # +port+:: [Fixnum] the port on which to connect
-      # +user+:: [String] the user to connect as
-      #
-      def initialize(host, port, user="")
+      def initialize(host, port, auth=nil)
         @host = host
         @port = port
-        super(user)
+        super(auth)
       end
 
+      protected
+
       ##
-      # TCP connection version of startconn
+      # Prepare a TCP connection to the Styx server
       #
-      def startconn
-        sock = TCPSocket.new(@host, @port)
-        return(sock)
+      def prepare_connection
+        queue = Queue.new
+        @eventthread = Thread.new do
+          EventMachine::run do
+            queue << EventMachine::connect(@host, @port, StyxClient)
+          end
+        end
+
+        @conn = queue.shift
       end
 
-    end
+      public
+
+    end                         # class TCPConnection
 
     ##
-    # Internally used class for buffers
-    class Buffer
-      attr_reader :bufsize
+    # A Styx client's view of a file.  This class should probably
+    # never be directly instantiated, but only via Connection#open.
+    # The buffering algorithm in use here is somewhat based on the
+    # Buffering mix-in module in the Ruby OpenSSL module written by
+    # Goto Yuuzou, but modified a bit to provide for offset
+    # handling.  Note that this class isn't thread-safe.
+    #
+    class File
+      include Enumerable
 
-      def initialize(bufsize)
-        @bufsize = bufsize    # maximum size of the buffer
-        @data = ""            # data in the buffer
+      attr_reader :conn, :path
+      attr_accessor :mode, :fid, :qid, :iounit, :sync
+
+      def initialize(conn, path)
+        @conn = conn            # the connection on which the file sits
+        @path = path            # pathname of the file
+        @fid = -1               # the client's file identifier
+        @qid = nil              # the server's unique identifier for this file
+        # maximum number of bytes that can be read or written to the file at a time
+        @iounit = 0
+        @mode = -1              # mode under which the file is opened, -1 == not open
+        @offset = 0             # File offset.  This is the same as @boffset only after a seek
+        @rboffset = 0           # Read buffer offset
+        @eof = false
+        @rbuffer = ""
+        @sync = false           # whether or not to buffer writes
       end
 
       ##
-      # Returns true if the buffer is empty
+      # Open the file on the server.  If a block is passed to this method
+      # it will pass the file to the block and close the file automatically
+      # when the block terminates.
       #
-      def empty?
-        return(@data.length == 0)
-      end
+      # This follows more or less the same semantics as sys-open(2)
+      # on Inferno, performing an open with truncate, when a file
+      # is opened that doesn't exist.
+      #
+      # XXX - twalk should handle the case of MAXWELEM, as well as for
+      #       when the twalk message is too large to fit in the server's
+      #       designated msize.
+      #
+      # +mode+:: Integer representing the mode - see the constants in common.rb
+      # +perm+:: Permissions of the file (only used on open/create)
+      # +create+:: should we create the file if it doesn't exist?
+      #
+      def open(mode, perm, create, &block)
+        dfid = @conn.get_free_fid
+        basename = ::File.basename(@path)
+        dirname = ::File.dirname(@path)
+        # Walk to the dirname first
+        twalk = Message::Twalk.new(:fid => @conn.rootfid, :newfid => dfid)
+        twalk.path = dirname
+        rwalk = @conn.send_message(twalk)
+        # if the rwalk has some other length than the number of path
+        # elements in the original twalk, we have failed.
+        if rwalk.qids.length != twalk.wnames.length
+          raise StyxException.new(("#{path} no such file or directory"))
+        end
+        ropen = fid = nil
+        begin
+          # Next, walk to the basename from there, using a new fid
+          fid = @conn.get_free_fid
+          twalk = Message::Twalk.new(:fid => dfid, :newfid => fid)
+          twalk.path = basename
+          rwalk = @conn.send_message(twalk)
+          # Do a Topen if this succeeds
+          open_msg = Message::Topen.new(:fid => fid, :mode => mode)
+          ropen = @conn.send_message(open_msg)
+          @conn.tclunk(dfid)
+        rescue Exception => e
+          # If we are being directed to create the file if it doesn't
+          # already exist, send a Tcreate message, and use the response
+          # for it as the ropen (the two classes respond to exactly the
+          # same messages--ah the wonders of duck typing!).  If not,
+          # or if that fails, this should propagate an exception upwards.
+          if create
+            # Alter the submitted permissions mask according to
+            # the connection's umask
+            perm &= ~(@conn.umask)
+            create_msg = Message::Tcreate.new(:fid => dfid, :name => basename,
+                                              :perm => perm, :mode => mode)
+            ropen = @conn.send_message(create_msg)
+            fid = dfid
+          else
+            raise e
+          end
+        end
 
-      ##
-      # Returns true if the buffer is full
-      def full?
-        return(@data.length == @bufsize)
+        # If we get here, we were able to successfully open or create
+        # the file.
+        @mode = mode
+        @fid = fid
+        @qid = ropen.qid
+        @iounit = ropen.iounit
+        # Determine if the file is actually a directory.  Such a file
+        # would have its Qid.qtype high bit set to 1.  In this case, instead
+        # of returning self, we return self wrapped in a Directory object.
+        retval = self
+        if twalk.wnames.length == 0 || (rwalk.qids[-1].qtype & 0x80 != 0)
+          retval = Directory.new(self)
+        end
+        if block_given?
+          begin
+            yield retval
+          ensure
+            retval.close
+          end
+        else
+          return(retval)
+        end
       end
 
       ##
-      # How much data is needed to fill the buffer?
+      # Read the Stat information for the file.
       #
-      def slack
-        return(@bufsize - @data.length)
-      end
-
-      ##
-      # Empty the buffer.  Return all the data in the buffer
+      # returns: the RStyx::Message::Stat instance corresponding to
+      #          the file
       #
-      def empty
-        retval = @data
-        @data = ""
-        return(retval)
+      def stat
+        rstat = @conn.send_message(Message::Tstat.new(:fid => @fid))
+        return(rstat.stat)
       end
 
       ##
-      # Consume data from the buffer.  Returns a string of at most +size+
-      # length from the buffer.
+      # Close the file.  This will flush all unwritten buffered data
+      # if any.
       #
-      def consume(size)
-        retval = @data[0..(size-1)]
-        @data = @data[size..@data.length]
-        if @data == nil      # if the buffer goes empty
-          @data = ""
+      def close
+        flush rescue nil
+        if @fid >= 0
+          # Clunk the fid
+          @conn.tclunk(@fid)
         end
-        return(retval)
+        @fid = -1
+        @iounit = 0
+        @mode = -1
       end
 
-      ##
-      # Write data to the buffer.  Returns the data that was not actually
-      # written to the buffer (the excess) if the buffer is full.
-      def fill(data)
-        if data.length > slack
-          excess = data[slack..(data.length)]
-          data = data[0..(slack-1)]
-        end
-        @data << data
-        return(excess)
+      def closed?
+        return(@mode < 0)
       end
 
-    end
-
-    ##
-    # A file on a Styx server.  This should probably not be instantiated
-    # directly, but only by the use of Connection#open.
-    #
-    # TODO: This should later become a true subclass of IO.
-    #
-    class StyxIO
-      attr_reader :name, :mode, :closed, :offset, :iounit, :fid
-      attr_accessor :sync
-
-      # Whether or not to buffer writes.  Unlike regular IO objects,
-      # StyxIO has this true by default.
-      attr_accessor :sync
-      include Enumerable
-
-      def initialize(conn, name, fid, iounit, mode="r")
-        @conn = conn
-        @name = name
-        @mode = mode
-        @closed = false         # closed flag
-        @fid = fid
-        @iounit = iounit        # maximum payload of a Twrite or Rread message
-        @offset = 0             # file offset
-        @sync = true
-
-        @buffer = Buffer.new(iounit) # the read buffer
-        @boffset = 0   # the offset of the last buffered read or write
-        @lastop = :none         # the last operation performed
-        @sync = false           # Buffer or not buffer writes
-      end
+      private
 
       ##
       # Read at most +size+ bytes from +offset+
       # If the size argument is negative or omitted, read until EOF.
-      # This is an unbuffered read!  This function should probably
-      # not be used directly.
+      # This should probably not be used directly.
       #
       # +size+:: number of bytes to read from the file
       # +offset+:: the offset to read from.
       # return:: the data followed by the new offset
       #
-      def sysread_int(size=-1, offset=0)
+      def _sysread(size=-1, offset=0)
         contents = ""
         bytes_to_read = size
         loop do
@@ -665,7 +618,10 @@ module RStyx
           else
             n = bytes_to_read
           end
-          rread = @conn.send_message(Message::Tread.new(@fid, offset, n))
+          rread =
+            @conn.send_message(Message::Tread.new(:fid => @fid,
+                                                  :offset => offset,
+                                                  :count => n))
           if rread.data.length == 0
             break                 # EOF
           end
@@ -681,16 +637,17 @@ module RStyx
       ##
       #
       # Write data to the file at +offset+.  No buffering is
-      # performed.  This function should probably not be used
-      # directly.
+      # performed.  This should probably not be used directly.
       #
-      # +str+:: data to be written
+      # +d+:: data to be written
       # +offset+:: the offset to write at
       #
-      # returns the new offset
+      # returns the new offset and the number of bytes written
       #
-      def syswrite_int(str, offset)
+      def _syswrite(d, offset)
+        str = d.to_s
         pos = 0
+        count = 0
         loop do
           bytes_left = str.length - pos
           if bytes_left <= 0
@@ -700,256 +657,323 @@ module RStyx
           else
             n = bytes_left
           end
-          rwrite = @conn.send_message(Message::Twrite.new(@fid, offset,
-                                                          str[pos..(pos+n)]))
-          if rwrite.count != n
-            raise StyxException.new("error writing data")
-          end
+          rwrite = @conn.send_message(Message::Twrite.new(:fid => @fid,
+                                                          :offset => offset,
+                                                          :data => str[pos..(pos+n)]))
           pos += n
           offset += n
+          count += rwrite.count
         end
-        return(offset)
+        return([offset, count])
       end
 
-      public
-
       ##
-      # closes the file.  Once this has been done, no more read or write
-      # operations should be possible.
+      # Add up to @iounit bytes to the read buffer.
       #
-      def close
-        flush                     # flush any bytes buffered
-        rclunk = @conn.send_message(Message::Tclunk.new(@fid))
-        # FIXME: abort all outstanding messages with flushes
+      def fill_rbuff
+        d, @rboffset = _sysread(@iounit, @rboffset)
+        if d.empty?
+          @eof = true
+        end
+        @rbuffer << d
       end
 
       ##
-      # Read at most +size+ bytes from the current file position.
-      # If the size argument is negative or omitted, read until EOF.
-      # Returns nil if we read from the end of stream.
-      #
-      # This is a buffered read.
-      #
-      # +size+:: number of bytes to read from the file
+      # Consume +size+ bytes from the read buffer.
       #
-      def read(size=-1)
-        # flush any buffered data first, so the buffer becomes empty
-        if @lastop == :write
-          flush
+      def consume_rbuff(size=nil)
+        if @rbuffer.empty?
+          nil
+        else
+          size ||= @rbuffer.size
+          ret = @rbuffer[0, size]
+          @rbuffer[0, size] = ""
+          return(ret)
         end
-        @lastop = :read
-
-        bytes_to_read = size
-        contents = ""
-        loop do
-          if @buffer.empty?
-            # fill the buffer if the buffer goes empty
-            rdata, @boffset = sysread_int(@buffer.slack, @boffset)
-            if rdata.length == 0
-              break
-            end
-            @buffer.fill(rdata)
-          end
-
-          d = ""
-
-          if size < 0 || bytes_to_read > @buffer.bufsize
-            d = @buffer.empty
-          else
-            d = @buffer.consume(bytes_to_read)
-          end
+      end
 
-          @offset += d.length
-          contents << d
-          if size >= 0
-            bytes_to_read -= d.length
-          end
+      public
 
-          if bytes_to_read == 0
+      ##
+      # Read at most +size+ bytes from the Styx file or to the end of
+      # file if omitted.  Returns nil if called at end of file.
+      #
+      def read(size=-1)
+        until @eof
+          # Fill up the buffer until we have at least the requested
+          # size, or until end of file if size is negative.
+          if size > 0 && size <= @rbuffer.size
             break
           end
+          fill_rbuff
         end
-        if (contents.length == 0)
-          return(nil)
+
+        # We managed to slurp the entire file!
+        if size < 0
+          size = @rbuffer.size
         end
-        return(contents)
+
+        @offset += size
+        retval = consume_rbuff(size) || ""
+        (size && retval.empty?) ? nil : retval
       end
 
       ##
-      # Do an unbuffered read.
+      # Reads the next "line" from the Styx file; lines are separated by
+      # +eol+.  An +eol+ of nil reads the entire contents.  Returns nil
+      # if called at end of file.
       #
-      def sysread(size=-1)
-        unless @buffer.empty?
-          raise IOError.new("sysread for buffered IO")
+      def gets(eol=$/)
+        idx = @rbuffer.index(eol)
+        until @eof
+          if idx
+            break
+          end
+          fill_rbuff
+          idx = @rbuffer.index(eol)
         end
-        data, @offset = sysread_int(size, @offset)
-        if (data.length == 0)
-          return(nil)
+        if eol.is_a?(Regexp)
+          size = idx ? idx+$&.size : nil
+        else
+          size = idx ? idx+eol.size : nil
         end
-        return(data)
+        @offset += size
+        return(consume_rbuff(size))
       end
 
       ##
-      # Calls the given block once for each byte (0..255) in the StyxIO,
-      # passing the byte as an argument.  The Styx file must be opened
-      # for reading.
+      # Executes the block for evely line in the Styx file, where lines
+      # are separated by +eol+.
       #
-      def each_byte
-        if !block_given?
-          raise LocalJumpError("no block given")
-        end
-
-        while (!(c=read(1)).nil?)
-          yield c[0]
+      def each(eol=$/)
+        while line = self.gets(eol)
+          yield line
         end
       end
 
+      alias each_line each
+
       ##
-      # Reads the next "line" from the Styx file; lines are separated by
-      # sep_str.  A separator of nil reads the entire contents, [and a
-      # zero length separator reads the input a paragraph at a time (two
-      # successive newlines in the input separate paragraphs) TODO: this
-      # needs to be implemented].  The Styxfile
-      # must be opened for reading.  The line read in will be returned and
-      # also assigned to $LAST_READ_LINE.  Returns nil if called at end
-      # of file.
-      #
-      def gets(sep_str=$RS)
-        line = ""
-        while (!(c=read(1)).nil?)
-          line << c
-          if c == $RS
-            break
-          end
-        end
-        if line.length == 0
-          return(nil)
+      # Reads all of the lines in the Styx file, and returns them in an
+      # array.  Lines are separated by an optional separator +eol+.
+      #
+      def readlines(eol=$/)
+        ary = []
+        while line = self.gets(eol)
+          ary << line
         end
-        return(line)
+        ary
       end
 
       ##
-      # Executes the passed block for every line in styxio, where lines
-      # are separated by sep_str.  The Styx file must be opened for
-      # reading.
-      #
-      def each
-        if !block_given?
-          raise LocalJumpError("no block given")
-        end
+      # Reads a line as with gets, but 
+      def readline(eol=$/)
+        raise EOFError if eof?
+        return(gets(eol))
+      end
 
-        while (!(line = gets).nil?)
-          yield line
+      def getc
+        c = read(1)
+        return(c ? c[0] : nil)
+      end
+
+      def readchar
+        raise EOFError if eof?
+        getc
+      end
+
+      def ungetc(c)
+        @rbuffer[0,0] = c.chr
+        @offset -= 1
+      end
+
+      def eof?
+        if !@eof && @rbuffer.empty?
+          fill_rbuff
         end
+        return(@eof && @rbuffer.empty?)
       end
 
+      alias eof eof?
+
+      private
+
       ##
-      # A buffered write of data to the file.
+      # Write data to the buffer if the buffer is not yet full, or
+      # if it has been determined (e.g. by an end of line marker) that
+      # we should flush the buffer, and actually write.
       #
-      def write(data)
-        if @lastop == :read
-          @buffer.empty         # empty the buffer
+      def do_write(s)
+        unless defined?(@wbuffer)
+          @wbuffer = ""
+          # we obviously start writing at the current offset
+          @wboffset = @offset
         end
-        @lastop = :write
-
-        # If the sync flag is set, simply write the data straight
-        # to the file with no buffering at all.
-        if @sync
-          @offset = syswrite_int(data, @offset)
-          return(data.length)
+        @wbuffer << s
+        @offset += s.length
+        #
+        # We flush the buffer if at least one of the following conditions
+        # has been met:
+        # 
+        # 1. The sync flag is set to true.
+        # 2. The write buffer size has equalled or exceeded the connection's
+        #    iounit.
+        # 3. The write buffer now contains an end of line marker, in which
+        #    cas we flush only until the end of line marker.
+        #
+        if @sync || @wbuffer.size >= @iounit || (idx = @wbuffer.rindex($/))
+          remain = idx ? idx + $/.size : @wbuffer.length
+          nwritten = 0
+          ofs = @wboffset
+          while remain > 0
+            str = @wbuffer[nwritten, remain]
+            ofs, nwrote = _syswrite(str, ofs)
+            remain -= nwrote
+            nwritten += nwrote
+          end
+          @wbuffer[0, nwritten] = ""
+          @wboffset += nwritten
         end
+      end
+
+      public
+
+      def write(s)
+        do_write(s)
+        return(s.length)
+      end
+
+      def <<(s)
+        do_write(s)
+        return(self)
+      end
 
-        # If the sync flag is not set, try to write the data to the buffer
-        # until the buffer becomes full.
-        bytes_written = data.length
-        while !data.nil?
-          if @buffer.full?
-            # flush the buffer if the buffer fills up
-            flush
+      def puts(*args)
+        s = ""
+        if args.empty?
+          s << "\n"
+        end
+        args.each do |arg|
+          s << arg.to_s
+          if $/ && /\n\z/ !~ s
+            s << "\n"
           end
-          data = @buffer.fill(data)
         end
-        @offset += bytes_written
-        return(bytes_written)
+        do_write(s)
+        return(nil)
+      end
+
+      def print(*args)
+        s = ""
+        args.each{ |arg| s << arg.to_s }
+        do_write(s)
+        return(nil)
+      end
+
+      def printf(s, *args)
+        do_write(s % args)
+        return(nil)
       end
 
-      ##
-      # Flushes any buffered data
-      #
       def flush
-        if @lastop == :write
-          @boffset = syswrite_int(@buffer.empty, @boffset)
-          return(self)
-        end
+        osync = @sync
+        @sync = true
+        do_write ""
+        @sync = osync
       end
 
       ##
-      # Move to a new seek position
+      # Seek in the file.  The whence values may be one of SEEK_SET
+      # SEEK_CUR, or SEEK_END, as defined in rstyx/common, and they
+      # result in the offset being taken from the beginning of the
+      # file, relative to the current offset, or from the end of the
+      # file respectively.
       #
-      # TODO: implement :seek_end
+      # XXX: A seek, no matter where it goes, will always invalidate
+      #      any buffering.  This is fine for write buffers, but is
+      #      somewhat wasteful for the read buffers.
       #
-      def seek(offset, whence=:seek_set)
-        # flush the buffers before doing the seek
+      def seek(offset, whence)
+        # Before seeking, flush the write buffers
         flush
+        s = self.stat
         case whence
-        when :seek_set: @offset = offset
-        when :seek_cur: @offset += offset
-          # FIXME: do :seek_end
-        when :seek_end: raise StyxException.new("Can't seek from end of file")
-        else raise StyxException.new("whence must be :seek_set, :seek_cur, or :seek_end")
+        when 0
+          @offset = offset
+        when 1
+          @offset += offset
+        when 2
+          # We have to obtain the stat of the file to do this kind of seek.
+          @offset = s.length + offset
+        else
+          raise StyxException.new("Invalid seek")
         end
-        @boffset = @offset
-        @buffer.empty
+        # After seeking, discard the read buffers
+        @rbuffer = ""
+        @rboffset = @offset
+        @eof = (@offset >= s.length)
+        return(@offset)
+      end
+
+      def tell
+        return(@offset)
       end
 
-      ##
-      # Seek to the start of the file
-      #
       def rewind
-        seek(0, :seek_set)
+        seek(0, 0)
       end
 
       ##
-      # Return the current file position
+      # Reads +size+ bytes from the Styx file and returns them as a string.
+      # Do not mix with other methods that read from the Styx file or you
+      # may get unpredictable results.
       #
-      def tell
-        return @offset
+      def sysread(size=-1)
+        data, @offset = _sysread(size, @offset)
+        if data.length == 0
+          return(nil)
+        end
+        return(data)
       end
 
       ##
-      # Give the DirEntry corresponding to this file
-      #
-      def stat
-        rstat = @conn.send_message(Message::Tstat.new(@fid))
-        return(rstat.stat)
+      # Writes +data+ to the Styx file.  Returns the number of bytes written.
+      # do not mix with other methods that write to the Styx file or you
+      # may get unpredictable results.
+      def syswrite(data)
+        @offset, count = _syswrite(data, @offset)
+        return(count)
       end
 
-    end
+    end                         # class File
 
     ##
-    # A directory on a Styx server.  This obtains the entries inside
-    # a directory.  This actually delegates to StyxIO.
+    # Styx directory.  This obtains the entries inside a directory, and
+    # works by delegating to File.
     #
-    class StyxDir
+    class Directory
       include Enumerable
 
-      def initialize(conn, name, fid, iounit, mode="r")
-        @io = StyxIO.new(conn, name, fid, iounit, mode)
+      def initialize(fp)
+        @io = fp
         # directory entry buffer
         @read_direntries = []
+        # byte buffer
+        @data = ""
       end
 
       def close
         @io.close
       end
 
-      ##
-      # get the fid corresponding to this directory
-      #
       def fid
         @io.fid
       end
 
+      def qid
+        @io.qid
+      end
+
       ##
       # Read the next directory entry from the dir and return the file
       # name as a string.  Returns nil at the end of stream.
@@ -961,18 +985,25 @@ module RStyx
           return(@read_direntries.shift.name)
         end
         # read iounit bytes from the directory--this must be unbuffered
-        data = @io.read(@io.iounit)
+        d = @io.sysread
+        if d.nil?
+          return(nil)
+        end
+        @data << d
 
-        if (data.nil?)
+        if (@data.empty?)
           return(nil)
         end
 
         # decode the directory entries in the iounit
-        while data.length != 0
-          delen = data[0..1].unpack("v")[0]
-          edirent = data[0..(delen + 1)]
-          data = data[(delen + 2)..(data.length)]
-          @read_direntries << Message::DirEntry.decode(edirent)
+        loop do
+          delen = @data.unpack("v")[0]
+          if delen.nil? || delen + 1 > @data.length
+            break
+          end
+          edirent = @data[0..(delen + 1)]
+          @data = @data[(delen + 2)..-1]
+          @read_direntries << Message::Stat.from_bytes(edirent)
         end
         de = @read_direntries.shift
         if (de.nil?)
@@ -982,36 +1013,21 @@ module RStyx
       end
 
       ##
-      # Seek to the beginning of the directory.  This is the only type of
-      # seek allowed.
-      #
-      def rewind
-        @io.rewind
-      end
-
-      ##
-      # same as StyxIO#stat
-      #
-      def stat
-        return(@io.stat)
-      end
-
-      ##
-      # Call the block once for each entry in the directory, passing the
-      # filename of each entry as a parameter to the block.
+      # Call the block once for each entry in the directory, passing
+      # the filename of each entry as a parameter to the block.
       #
       def each
         if !block_given?
-          raise LocalJumpError("no block given")
+          raise LocalJumpError.new("no block given")
         end
 
         self.rewind
-        while (!(de = self.read).nil?)
+        until (de = read).nil?
           yield de
         end
       end
 
-    end
+    end                         # class Directory
 
   end                           # module Client