distribustream 0.1.0 → 0.2.0

data/lib/pdtp/client/file_buffer.rb

@@ -8,120 +8,110 @@
 # See http://distribustream.rubyforge.org/
 #++
 
-module PDTP
-  # Handle a file buffer, which may be written to and read from randomly
-  class FileBuffer
-    def initialize(io = nil)
-      @io = io
-      @written = 0
-      @entries = []
-    end
+require 'rubygems'
+require 'mongrel'
+require 'tempfile'
 
-    # Write data starting at start_pos. Overwrites any existing data in that block
-    def write(start_pos, data)
-      return if data.size == 0
-      
-      # create and entry and attempt to combine it with old entries
-      new_entry = Entry.new(start_pos,data)
+module PDTP
+  class Client
+    # Handle a file buffer, which may be written to and read from randomly
+    class FileBuffer  
+      def initialize(output = nil)
+        # The output stream.  Treated as non-seekable since it may be a pipe
+        @output = output
+        @cursor = 0
+        
+        # The tempfile backbuffer.  This will always be seekable
+        @buffer = Tempfile.new 'distribustream'
+        @populated = []
+      end
       
-      intersections = true
-      while intersections
-        intersections = false
-        @entries.each do |e|
-          if intersects?(new_entry, e)
-            new_entry = combine(e, new_entry)
-            @entries.delete(e)
-            intersections = true
-          end 
+      # Write to the backbuffer. Write contiguous data to the output stream
+      def write(position, data)
+        length = data.size
+        return 0 if length.zero?
+        
+        range = position..(position + length - 1)
+        
+        # Insert the chunk into the buffer and return a new range of contiguous data
+        writeable_range = insert_chunk range, data
+        
+        # If this chunk begins at the cursor, write it out
+        if @output and writeable_range.begin == @cursor
+          @cursor += @output.write read(writeable_range) 
         end
+        
+        # Return the length we just wrote
+        length
       end
-
-      # Add entry to the local store
-      @entries << new_entry
       
-      # Write contiguous blocks we receive to our internal IO cursor
-      if @io and start_pos == @written
-        data_begin = @written - new_entry.start_pos
-        bytes_written = @io.write(new_entry.data[data_begin..new_entry.data.length])
-        @written += bytes_written
-      else
-        bytes_written = data.size
+      def read(range)
+        chunk = containing_chunk range.begin
+        
+        unless chunk and chunk.end >= range.end
+          raise RuntimeError, "#{chunk.inspect} cannot satisfy range #{range.inspect}" 
+        end
+        
+        @buffer.pos = range.begin
+        @buffer.read(range.end - range.begin + 1)
       end
       
-      bytes_written
-    end
-
-    # Returns a string containing the desired data. 
-    # Returns nil if the data is not all there
-    def read(range)
-      return nil if range.first>range.last
-      current_byte=range.first
-
-      buffer = ''
-
-      while current_byte <= range.last do
-        # find an entry that contains this byte
-
-        found=false
-        @entries.each do |e|
-          if e.range.include?(current_byte)
-            internal_start=current_byte-e.start_pos #start position inside this entry's data
-            internal_end=(range.last<e.end_pos ? range.last : e.end_pos) - e.start_pos 
-            buffer << e.data[internal_start..internal_end]
-            current_byte+=internal_end-internal_start+1
-            found=true
-            break if current_byte>range.last
+      #######
+      private
+      #######
+      
+      def insert_chunk(range, data)
+        i = 0
+        
+        # Find the first slot in the populated array that this range exceeds
+        i += 1 while i < @populated.size and range.begin > @populated[i].end
+        
+        if i == 0
+          unless @populated.empty? or @populated.first.begin > range.end
+            raise RuntimeError, "chunks in output buffer overlap" 
           end
+          
+          @populated = [range] + @populated
+        elsif i < @populated.size
+          head = @populated[0..(i - 1)]
+          tail = @populated[i..(@populated.size - 1)]
+          
+          raise RuntimeError, "chunks in output buffer overlap" if tail.first.begin <= range.end
+          
+          @populated = head + [range] + tail
+        else
+          @populated << range
         end
-        return nil if found==false
-      end   
-      
-      buffer
-    end
-
-    # Returns true if two entries intersect
-    def intersects?(entry1, entry2)
-      first, last = entry1.start_pos <= entry2.start_pos ? [entry1, entry2] : [entry2, entry1]
-      first.end_pos + 1 >= last.start_pos 
-    end
-
-    # Takes two Entries
-    # Returns nil if there is no intersection
-    # Returns the union if they intersect
-    def combine(old_entry, new_entry)
-      start = old_entry.start_pos < new_entry.start_pos ? old_entry.start_pos: new_entry.start_pos
-
-      stringio = StringIO.new
-      stringio.seek(old_entry.start_pos - start)
-      stringio.write(old_entry.data)
-      stringio.seek(new_entry.start_pos - start)
-      stringio.write(new_entry.data)
-      return Entry.new(start, stringio.string)    
-    end
-
-    # Return number of bytes currently in the buffer
-    def bytes_stored
-      bytes=0
-      @entries.each do |e|
-        bytes=bytes+e.data.size
-      end  
-      return bytes
-    end
-
-    # Container for an entry in the buffer
-    class Entry
-      def initialize(start_pos,data)
-        @start_pos,@data=start_pos,data
+        
+        @buffer.pos = range.begin
+        @buffer.write data
+        
+        compact_chunks
+        position = range.begin
+        
+        # Find the range beginning at the current position and ending at the last contiguous chunk
+        position..(containing_chunk(position).end)
       end
-
-      attr_accessor :start_pos, :data
-
-      def end_pos
-        @start_pos + data.length - 1
+      
+      def compact_chunks
+        output = []
+        current_range = @populated.shift
+        
+        @populated.each do |next_range|
+          if current_range.end + 1 == next_range.begin
+            current_range = current_range.begin..next_range.end
+          else
+            output << current_range
+            current_range = next_range
+          end
+        end
+        
+        output << current_range
+        @populated = output
       end
-
-      def range
-        Range.new(@start_pos,end_pos)
+      
+      def containing_chunk(position)
+        @populated.detect { |range| range.include? position }
       end
     end
   end

data/lib/pdtp/client/file_service.rb

@@ -8,18 +8,21 @@
 # See http://distribustream.rubyforge.org/
 #++
 
+require 'rubygems'
+require 'mongrel'
 require 'uri'
 require 'pathname'
+
 require File.dirname(__FILE__) + '/../common/file_service.rb'
 require File.dirname(__FILE__) + '/file_buffer.rb'    
 
 module PDTP
-  class Client < Mongrel::HttpHandler
+  class Client
     # The client specific file utilities. Most importantly, handling
     # the data buffer.
     class FileInfo < PDTP::FileInfo
-      def initialize(filename)
-        @buffer = FileBuffer.new open(filename, 'w')
+      def initialize(filename, io = nil)
+        @buffer = FileBuffer.new io || open(filename, 'w')
         @lock = Mutex.new
       end
       

data/lib/pdtp/client/http_handler.rb

@@ -0,0 +1,77 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require 'rubygems'
+require 'eventmachine'
+require 'mongrel'
+
+require File.dirname(__FILE__) + '/transfer'
+
+module PDTP
+  class Client
+    # Mongrel::HttpHandler to handle incoming HTTP chunk transfers
+    class HttpHandler < Mongrel::HttpHandler
+      attr_accessor :client
+
+      def initialize(client)
+        @client = client
+      end
+
+      # This method is called after a connection to the server
+      # has been successfully established.
+      def connection_created(connection)
+        @@log.debug("[mongrel] Opened connection...");
+      end
+
+      # This method is called when the server connection is destroyed
+      def connection_destroyed(connection)
+        @@log.debug("[mongrel] Closed connection...")
+      end
+
+      # Returns a transfer object if the given connection is a peer associated with 
+      # that transfer. Otherwise returns nil.
+      def get_transfer(connection)
+        @client.transfers.each { |t| return t if t.peer == connection }
+        nil
+      end
+
+      # This method is called when an HTTP request is received. It is called in 
+      # a separate thread, one for each request.
+      def process(request,response)
+        begin
+          @@log.debug "Creating Transfer::Listener"
+          transfer = Transfer::Listener.new(
+            @client.connection,
+            request, 
+            response,
+            client.file_service
+          )
+
+          #Needs to be locked because multiple threads could attempt to append a transfer at once
+          @client.lock.synchronize { @client.transfers << transfer }
+          transfer.handle_header 
+        rescue Exception => e
+          raise e if transfer.nil?
+          transfer.write_http_exception(e)
+        end
+
+        transfer.send_completed_message transfer.hash
+      end
+
+      # Returns true if the given message refers to the given transfer  
+      def transfer_matches?(transfer, message)
+        transfer.peer       == message["peer"] and 
+        transfer.url        == message["url"] and
+        transfer.byte_range == message["range"] and
+        transfer.peer_id    == message["peer_id"]
+      end
+    end
+  end
+end

data/lib/pdtp/client/transfer.rb

@@ -15,7 +15,7 @@ require "uri"
 require "digest/sha2"
 
 module PDTP
-  class Client < Mongrel::HttpHandler
+  class Client
     module Transfer
       # Generic HTTP Exception to be used on error
       class HTTPException < Exception
@@ -29,8 +29,7 @@ module PDTP
       # The base information and methods needed by client transfers
       class Base
         attr_reader :peer, :peer_id, :url, :byte_range
-        attr_reader :server_connection, :file_service
-        attr_reader :method, :client, :hash
+        attr_reader :file_service, :method, :connection, :hash
 
         # Returns true if a server message matches this transfer
         def matches_message?(message)
@@ -52,7 +51,7 @@ module PDTP
         # Notify the server of transfer completion.
         # Hash field is used to denote success or failure
         def send_completed_message(hash)
-          @server_connection.send_message(:completed,
+          @connection.send_message(:completed,
             :url => @url,
             :peer => @peer,
             :range => @byte_range,
@@ -62,7 +61,7 @@ module PDTP
         end
 
         def send_ask_verify_message
-          @server_connection.send_message(:ask_verify,
+          @connection.send_message(:ask_verify,
             :url => @url,
             :peer => @peer,
             :range => @byte_range,
@@ -76,11 +75,11 @@ module PDTP
         attr :request, :response
 
         # Called with the request and response parameters given by Mongrel
-        def initialize(request,response,server_connection,file_service,client)
-          @request,@response=request,response
-          @server_connection,@file_service=server_connection,file_service
-          @authorized=false
-          @client=client
+        def initialize(connection, request, response, file_service)
+          @request, @response = request,response
+          @file_service = file_service
+          @authorized = false
+          @connection = connection
         end
 
         # Send an HTTP error response to requester
@@ -177,14 +176,14 @@ module PDTP
 
       # Implements http transfer between two peers from the connector's (client) perspective
       class Connector < Base
-        def initialize(message,server_connection,file_service,client)
-          @server_connection,@file_service=server_connection,file_service
+        def initialize(connection, message, file_service)
+          @file_service=file_service
           @peer,@port=message["host"],message["port"]
           @method = message["method"]
           @url=message["url"]
           @byte_range=message["range"]
           @peer_id=message["peer_id"]
-          @client=client
+          @connection=connection
         end
 
         # Perform the transfer
@@ -211,7 +210,7 @@ module PDTP
 
           req.add_field("Range", "bytes=#{@byte_range.begin}-#{@byte_range.end}")
           req.add_field("Host",vhost)
-          req.add_field("X-PDTP-Peer-Id",@client.my_id)
+          req.add_field("X-PDTP-Peer-Id", @connection.client.client_id)
           res = Net::HTTP.start(@peer,@port) {|http| http.request(req,body) }
 
           if res.code == '206' and @method == 'get'

data/lib/pdtp/client.rb

@@ -10,186 +10,103 @@
 
 require 'rubygems'
 require 'eventmachine'
-require 'mongrel'
-require 'net/http'
 require 'thread'
 require 'digest/md5'
 
-require File.dirname(__FILE__) + '/common/common_init'
-require File.dirname(__FILE__) + '/common/protocol'
-require File.dirname(__FILE__) + '/client/protocol'
+require File.dirname(__FILE__) + '/client/connection'
+require File.dirname(__FILE__) + '/client/callbacks'
 require File.dirname(__FILE__) + '/client/file_service'
-require File.dirname(__FILE__) + '/client/transfer'
-require File.dirname(__FILE__) + '/server/file_service'
+require File.dirname(__FILE__) + '/client/http_handler'
 
 module PDTP
-  # This is the main driver for the client-side implementation
-  # of PDTP. It maintains a single connection to a server and 
-  # all the necessary connections to peers. It is responsible
-  # for handling all messages corresponding to these connections.
-
-  # Client inherits from Mongrel::HttpHandler in order to handle
-  # incoming HTTP connections
-  class Client < Mongrel::HttpHandler
-    # Accessor for a client file service instance
-    attr_accessor :file_service
-    attr_accessor :server_connection
-    attr_accessor :my_id
-    
-    def self.get(host, path, options = {})
-      path = '/' + path unless path[0] == ?/
-      
-      opts = {
-        :host => host,
-        :port => 6086,
-        :file_root => '.',
-        :quiet => true,
-        :listen_port => 8000,
-        :request_url => "http://#{host}#{path}"
-      }.merge(options)
-      
-      common_init $0, opts
+  # PDTP::Client provides an interface for accessing resources on PDTP servers
+  class Client
+    # None of these should be publically accessible, and will be factored away in time
+    attr_reader :connection, :client_id, :listen_port, :file_service, :transfers, :lock
+
+    # Create a PDTP::Client object for accessing the PDTP server at the given host and port
+    def initialize(host, port = 6086, options = {})
+      @host, @port = host, port
+            
+      @listen_addr = options[:listen_addr] || '0.0.0.0'
+      @listen_port = options[:listen_port] || 60860
       
-      # Run the EventMachine reactor loop
-      EventMachine::run do
-        connection = EventMachine::connect host, opts[:port], Client::Protocol
-        @@log.info "connecting with ev=#{EventMachine::VERSION}"
-        @@log.info "host= #{host} port=#{opts[:port]}"
-      end
-    end
-
-    def initialize
+      @file_service = PDTP::Client::FileService.new
       @transfers = []
-      @mutex = Mutex.new
-    end
-
-    # This method is called after a connection to the server
-    # has been successfully established.
-    def connection_created(connection)
-      @@log.debug("[mongrel] Opened connection...");
-    end
-
-    # This method is called when the server connection is destroyed
-    def connection_destroyed(connection)
-      @@log.debug("[mongrel] Closed connection...")
-    end
+      @lock = Mutex.new
 
-    # Returns a transfer object if the given connection is a peer associated with 
-    # that transfer. Otherwise returns nil.
-    def get_transfer(connection)
-      @transfers.each { |t| return t if t.peer == connection }
-      nil
-    end
-
-    # This method is called when an HTTP request is received. It is called in 
-    # a separate thread, one for each request.
-    def process(request,response)
+      # Start a Mongrel server on the specified port.  If it isnt available, keep trying higher ports
       begin
-        @@log.debug "Creating Transfer::Listener"
-        transfer = Transfer::Listener.new(
-          request, 
-          response, 
-          @server_connection, 
-          @file_service, 
-          self
-        )
-
-        #Needs to be locked because multiple threads could attempt to append a transfer at once
-        @mutex.synchronize { @transfers << transfer }
-        transfer.handle_header 
-      rescue Exception=>e
-        transfer.write_http_exception(e)
+        @http_server = Mongrel::HttpServer.new @listen_addr, @listen_port
+      rescue Exception => e
+        @listen_port += 1
+        retry
       end
 
-      transfer.send_completed_message transfer.hash
-    end
+      @client_id = Digest::MD5.hexdigest "#{Time.now.to_f}#{$$}"
 
-    # Returns true if the given message refers to the given transfer  
-    def transfer_matches?(transfer, message)
-      transfer.peer       == message["peer"] and 
-      transfer.url        == message["url"] and
-      transfer.byte_range == message["range"] and
-      transfer.peer_id    == message["peer_id"]
+      #@@log.info "listening on port #{@listen_port}"
+      @http_handler = HttpHandler.new(self)
+      @http_server.register '/', @http_handler
+      @http_server.run
     end
 
-    # Called when any server message is received. This is the brains of
-    # the client's protocol handling.
-    def dispatch_message(command, message, connection)
-      case command
-      when "tell_info" # Receive and store information for this url  
-        info = FileInfo.new message["url"].split('/').last
-        info.file_size = message["size"]
-        info.base_chunk_size = message["chunk_size"]
-        info.streaming = message["streaming"]
-        @file_service.set_info(message["url"], info)   
-      when "transfer" # Begin a transfer as a connector
-        transfer = Transfer::Connector.new(message,@server_connection,@file_service,self)
-
-        @@log.debug "TRANSFER STARTING"
-
-        # Run each transfer in its own thread and notify the server upon completion
-        Thread.new(transfer) do |t|
-          begin
-            t.run
-          rescue Exception=>e
-            @@log.info("Exception in dispatch_message: " + e.exception + "\n" + e.backtrace.join("\n")) 
-          end
-          t.send_completed_message(t.hash)
-        end
-      when "tell_verify"
-        # We are a listener, and asked for verification of a transfer from a server.
-        # After asking for verification, we stopped running, and must be restarted
-        # if verification is successful
-        
-        found=false
-        @transfers.each do |t|
-          if t.matches_message?(message)
-            finished(t)
-            t.tell_verify(message["is_authorized"])
-            found=true
-            break
-          end
+    # Connect to the PDTP server. This is a blocking call which runs the client event loop
+    def connect(callbacks = nil)
+      klass = if(callbacks and callbacks.is_a?(Class))
+        callbacks
+      else
+        Class.new(Callbacks) {callbacks and include callbacks}
+      end
+      
+      # Run the EventMachine reactor loop
+      EventMachine.run do
+        EventMachine.connect(@host, @port, Connection) do |c|
+          # Store reference to the connection
+          @connection = c
+          
+          # Set connection variables and configure callbacks
+          c.client = self
+          c.callbacks = klass.new(self)
+          c.callbacks.client = self
+          
+          # Allow users to populate instance variables in their callback class
+          yield c.callbacks if block_given?
         end
 
-        unless found
-          puts "BUG: Tell verify sent for an unknown transfer"
-          exit!
-        end
-      when "hash_verify"
-        @@log.debug "Hash verified for url=#{message["url"]} range=#{message["range"]} hash_ok=#{message["hash_ok"]}"
-      when "protocol_error", "protocol_warn" #ignore
-      else raise "Server sent an unknown message type: #{command} "
+        #@@log.info "connecting with ev=#{EventMachine::VERSION}"
+        #@@log.info "host= #{host} port=#{opts[:port]}"
       end
     end
 
-    #Prints the number of transfers associated with this client
-    def print_stats
-      @@log.debug "client:  num_transfers=#{@transfers.size}"
+    # Are we currently connected to a server?
+    def connected?
+      !!@connection
     end
 
-    #Provides a threadsafe mechanism for transfers to report themselves finished
-    def finished(transfer)
-      @mutex.synchronize do
-        @transfers.delete(transfer)
-      end
-    end
-
-    # Generate and set the client ID for an instance
-    def generate_client_id(port = 0)
-      @my_id = Client.generate_client_id port
-    end
+    # Retrieve the resource at the given path and write it to the given IO object
+    def get(path, io, options = {})
+      raise RuntimeError, "not connected to server yet" unless connected?
+      
+      path = '/' + path unless path[0] == ?/
+      url = "http://#{@host}#{path}"
+      filename = path.split('/').last
+      
+      # Register the file and its IO object with the local file service
+      file_service.set_info url, FileInfo.new(filename, io)
+      
+      # Ask the server for some information on the file we want
+      @connection.send_message :ask_info, :url => url
 
-    # Client ID generator routine
-    def self.generate_client_id(port = 0)
-      md5 = Digest::MD5::new
-      now = Time::now
-      md5.update now.to_s
-      md5.update String(now.usec)
-      md5.update String(rand(0))
-      md5.update String($$)
+      # Request the file (should probably be done after receiving :tell_info)
+      @connection.send_message :request, :url => url
 
-      #return md5.hexdigest+":#{port}" # long id
-      return md5.hexdigest[0..5] # short id
+      #@@log.info "This client is requesting"
+    end
+    
+    # Stop the client event loop.  This only works within callbacks given to the #connect method
+    def stop
+      EventMachine.stop_event_loop
     end
   end
 end