distribustream 0.4.1 → 0.5.0

data/lib/pdtp/server/dispatcher.rb

@@ -20,10 +20,7 @@
 # See http://distribustream.org/
 #++
 
-require File.dirname(__FILE__) + '/../common/protocol'
-require File.dirname(__FILE__) + '/file_service'
-require File.dirname(__FILE__) + '/client_info'
-require File.dirname(__FILE__) + '/transfer'
+require File.dirname(__FILE__) + '/transfer_manager'
 
 module PDTP
   class Server
@@ -36,10 +33,10 @@ module PDTP
         @file_service = file_service
         @connections = []
         @used_client_ids = {} #keeps a list of client ids in use, they must be unique
-        @updated_clients = {} #a set of clients that have been modified and need transfers spawned
+        @transfer_manager = TransferManager.new @connections, @file_service
       end
 
-      #called by pdtp_protocol when a connection is created
+      # Register a PDTP::Server::Connection with the Dispatcher
       def connection_created(connection)
         addr, port = connection.get_peer_info
         
@@ -53,203 +50,31 @@ module PDTP
           @server.log "client connected: #{connection.get_peer_info.inspect}"
         end
         
-        connection.user_data = ClientInfo.new
         @connections << connection 
       end  
 
-      #called by pdtp_protocol when a connection is destroyed
+      # Unregister a PDTP::Server::Connection from the Dispatcher
       def connection_destroyed(connection)
         @server.log "client disconnected: #{connection.get_peer_info.inspect}"
         @connections.delete connection
       end
-
-      # returns the ClientInfo object associated with this connection
-      def client_info(connection)
-        connection.user_data ||= ClientInfo.new
-      end
-
-      # called when a transfer either finishes, successfully or not
-      def transfer_completed(transfer,connection,chunk_hash,send_response=true)      
-        # did the transfer complete successfully?
-        local_hash=@file_service.get_chunk_hash(transfer.url,transfer.chunkid)
-
-        c1=client_info(transfer.taker)
-        c2=client_info(transfer.giver)
-
-        if connection==transfer.taker
-          success= (chunk_hash==local_hash)
-
-          if success
-            #the taker now has the file, so he can provide it
-            client_info(transfer.taker).chunk_info.provide(transfer.url,transfer.chunkid..transfer.chunkid)
-            c1.trust.success(c2.trust)
-          else
-            #transfer failed, the client still wants the chunk
-            client_info(transfer.taker).chunk_info.request(transfer.url,transfer.chunkid..transfer.chunkid)
-            c1.trust.failure(c2.trust)
-          end 
-
-          transfer.taker.send_message(:hash_verify, 
-            :url => transfer.url, 
-            :range => transfer.byte_range, 
-            :hash_ok => success
-          ) if send_response
-        end
-
-        #remove this transfer from whoever sent it
-        client_info(connection).transfers.delete(transfer.transfer_id)
-        @updated_clients[connection]=true #flag this client for transfer creation
-      end
-
-      #Creates a new transfer between two peers
-      #returns true on success, or false if the specified transfer is already in progress
-      def begin_transfer(taker, giver, url, chunkid)
-        byte_range = @file_service.get_info(url).chunk_range(chunkid) 
-        t = Transfer.new(taker, giver, url, chunkid, byte_range)
-
-        #make sure this transfer doesnt already exist
-        t1 = client_info(taker).transfers[t.transfer_id]
-        t2 = client_info(giver).transfers[t.transfer_id]
-        return false unless t1.nil? and t2.nil?
-
-        client_info(taker).chunk_info.transfer(url, chunkid..chunkid) 
-        client_info(taker).transfers[t.transfer_id] = t
-        client_info(giver).transfers[t.transfer_id] = t
-
-        #send transfer message to the connector
-        addr, port = t.acceptor.get_peer_info
       
-        t.connector.send_message(:transfer,
-          :host => addr,
-          :port => t.acceptor.user_data.listen_port,
-          :method => t.connector == t.taker ? "get" : "put",
-          :url => url,
-          :range => byte_range,
-          :peer_id => client_info(t.acceptor).client_id
-        )
-        true
-      end
-
-      #this function removes all stalled transfers from the list
-      #and spawns new transfers as appropriate
-      #it must be called periodically by EventMachine
+      # This function removes all stalled transfers from the list
+      # and spawns new transfers as appropriate
+      # It must be called periodically by EventMachine
       def clear_all_stalled_transfers
         @connections.each { |connection| clear_stalled_transfers_for_client connection }  
-        spawn_all_transfers
-      end
-
-      #removes all stalled transfers that this client is a part of
-      def clear_stalled_transfers_for_client(client_connection)
-        client_info(client_connection).get_stalled_transfers.each do |transfer|
-          transfer_completed transfer, client_connection, nil, false
-        end  
-      end
-
-      #spawns uploads and downloads for this client.
-      #should be called every time there is a change that would affect 
-      #what this client has or wants
-      def spawn_transfers_for_client(client_connection)
-        info = client_info client_connection
-
-        while info.wants_download? do
-          break if spawn_download_for_client(client_connection) == false
-        end
-
-        while info.wants_upload? do
-          break if spawn_upload_for_client(client_connection) == false
-        end
+        @transfer_manager.spawn_all_transfers
       end
-
-      #creates a single download for the specified client
-      #returns true on success, false on failure
-      def spawn_download_for_client(client_connection)
-        feasible_peers=[]
-
-        c1info=client_info(client_connection)
-        begin
-          url,chunkid=c1info.chunk_info.high_priority_chunk
-        rescue
-          return false
-        end
-
-        @connections.each do |c2|
-          next if client_connection==c2
-          next if client_info(c2).wants_upload? == false
-          if client_info(c2).chunk_info.provided?(url,chunkid)
-            feasible_peers << c2
-            break if feasible_peers.size > 5
-          end
-        end
-
-        # we now have a list of clients that have the requested chunk.
-        # pick one and start the transfer
-        if feasible_peers.size > 0
-          #FIXME base this on the trust model
-          giver=feasible_peers[rand(feasible_peers.size)]
-          return begin_transfer(client_connection,giver,url,chunkid)
-          #FIXME should we try again if begin_transfer fails?
-        end
-
-        false
-      end
-
-      #creates a single upload for the specified client
-      #returns true on success, false on failure
-      def spawn_upload_for_client(client_connection)
-        c1info=client_info(client_connection)
-
-        @connections.each do |c2|
-          next if client_connection==c2
-          next if client_info(c2).wants_download? == false
-
-          begin
-            url,chunkid=client_info(c2).chunk_info.high_priority_chunk
-          rescue
-            next
-          end
-
-          if c1info.chunk_info.provided?(url,chunkid)
-            return begin_transfer(c2,client_connection,url,chunkid)
-          end
-        end
-
-        false
-      end
-
-      #creates new transfers for all clients that have been updated
-      def spawn_all_transfers
-        while @updated_clients.size > 0 do
-          tmp=@updated_clients
-          @updated_clients=Hash.new
-          tmp.each do |client,true_key| 
-            spawn_transfers_for_client(client)
-          end
-        end    
-      end
-
-      #handles the request, provide, unrequest, unprovide messages
-      def handle_requestprovide(connection,message)
-        type=message["type"]
-        url=message["url"]
-        info=@file_service.get_info(url) rescue nil
-        raise ProtocolWarn.new("Requested URL: '#{url}' not found") if info.nil?
-
-        exclude_partial= (type=="provide") #only exclude partial chunks from provides
-        range=info.chunk_range_from_byte_range(message["range"],exclude_partial)
-
-        #call request, provide, unrequest, or unprovide
-        client_info(connection).chunk_info.send( type.to_sym, url, range)
-        @updated_clients[connection]=true #add to the list of client that need new transfers
-      end
-
-      #handles all incoming messages from clients
+      
+      # Handles all incoming messages from clients
       def dispatch_message(command, message, connection)
-        # store the command in the message hash
+        # Store the command in the message hash
         message["type"] = command
       
-        #require the client to register their client id and listen port before doing anything
-        if command != "register" and client_info(connection).client_id.nil?
-          raise ProtocolError.new("You need to send a 'client_info' message first")
+        # Require the client to register their client id and listen port before doing anything
+        if command != "register" and connection.client_id.nil?
+          raise ProtocolError, "You need to send a 'register' message first"
         end 
 
         case command
@@ -257,12 +82,12 @@ module PDTP
           cid = message["client_id"]
           #make sure this id isnt in use
           if @used_client_ids[cid]
-            raise ProtocolError.new("Your client id: #{cid} is already in use.")   
+            raise ProtocolError, "Your client id: #{cid} is already in use."
           end
         
           @used_client_ids[cid] = true 
-          client_info(connection).listen_port = message["listen_port"]
-          client_info(connection).client_id = cid
+          connection.listen_port = message["listen_port"]
+          connection.client_id = cid
         when "ask_info"
           info = @file_service.get_info(message["url"])
           response = { :url => message["url"] }
@@ -276,45 +101,92 @@ module PDTP
           handle_requestprovide connection, message
         when "ask_verify"
           #check if the specified transfer is a real one
-          my_id = client_info(connection).client_id
-          transfer_id=Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
-          ok = !!client_info(connection).transfers[transfer_id]
-          client_info(connection).transfers[transfer_id].verification_asked=true if ok
-          @server.debug "AskVerify not ok: id=#{transfer_id}" unless ok
+          my_id = connection.client_id
+          transfer_id = Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
+          authorized = !connection.transfers[transfer_id].nil?
+          
+          connection.transfers[transfer_id].verification_asked = true if authorized
+          @server.debug "AskVerify not ok: id=#{transfer_id}" unless authorized
           connection.send_message(:tell_verify,
             :url     => message["url"],
             :peer_id => message["peer_id"],
             :range   => message["range"],
             :peer    => message["peer"],
-            :authorized=>ok
+            :authorized=>authorized
           )
         when "completed"
-          my_id = client_info(connection).client_id
+          my_id = connection.client_id
           transfer_id = Transfer::gen_transfer_id(
             my_id,
             message["peer_id"],
             message["url"],
             message["range"]
           )
-          transfer=client_info(connection).transfers[transfer_id]
-          @server.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
+          transfer = connection.transfers[transfer_id]
+          @server.debug "Completed: id=#{transfer_id} ok=#{transfer != nil}"
           if transfer
-            transfer_completed(transfer,connection,message["hash"])
+            transfer_completed transfer, connection, message["hash"]
           else
-            raise ProtocolWarn.new("You sent me a transfer completed message for unknown transfer: #{transfer_id}")
+            raise ProtocolWarn, "You sent me a transfer completed message for unknown transfer: #{transfer_id}"
           end
         when 'protocol_error', 'protocol_warn' #ignore
-        else raise ProtocolError.new("Unhandled message type: #{command}")
+        else raise ProtocolError, "Unhandled message type: #{command}"
         end
 
-        spawn_all_transfers
+        # Process all clients that are in need of new transfers
+        @transfer_manager.spawn_all_transfers
       end
 
-      #returns a string representing the specified connection
-      def connection_name(c)
-        #host,port=c.get_peer_info
-        #return "#{get_id(c)}: #{host}:#{port}"
-        client_info(c).client_id
+      #########
+      protected
+      #########
+      
+      # Removes all stalled transfers that this client is a part of
+      def clear_stalled_transfers_for_client(connection)
+        connection.stalled_transfers.each do |transfer|
+          transfer_completed transfer, connection, nil, false
+        end  
+      end
+
+      # Called when a transfer either finishes, successfully or not
+      def transfer_completed(transfer, connection, chunk_hash, send_response=true)      
+        # Compute the SHA256 hash for the given file
+        local_hash = @file_service.get_chunk_hash transfer.url, transfer.chunkid
+
+        if connection == transfer.taker
+          success = (chunk_hash == local_hash)
+          
+          if success
+            transfer.success
+          else
+            transfer.failure
+          end
+          
+          transfer.taker.send_message(:hash_verify, 
+            :url => transfer.url, 
+            :range => transfer.byte_range, 
+            :hash_ok => success
+          ) if send_response
+        end
+
+        # Remove this transfer from whoever sent it
+        connection.transfers.delete(transfer.transfer_id)
+        @transfer_manager.process_client(connection)
+      end
+
+      # Handles the request, provide, unrequest, unprovide messages
+      def handle_requestprovide(connection, message)
+        type = message["type"]
+        url = message["url"]
+        info = @file_service.get_info(url) rescue nil
+        raise ProtocolWarn, "Requested URL: '#{url}' not found" if info.nil?
+
+        exclude_partial = (type=="provide") #only exclude partial chunks from provides
+        range = info.chunk_range_from_byte_range(message["range"],exclude_partial)
+
+        #call request, provide, unrequest, or unprovide
+        connection.chunk_info.send(type.to_sym, url, range)
+        @transfer_manager.process_client(connection)
       end
     end
   end

data/lib/pdtp/server/file_service.rb

@@ -38,7 +38,7 @@ module PDTP
           range = 0..chunk_size(chunkid) - 1 if range.nil? # full range of chunk if range isnt specified
           raise if range.first < 0 or range.last >= chunk_size(chunkid)
           start = range.first + chunkid * @base_chunk_size 
-          size = range.last-range.first + 1 
+          size = range.last - range.first + 1 
           file = open @path
           file.pos = start
           file.read size
@@ -72,7 +72,8 @@ module PDTP
       # by checking whether the registered file exists 
       def get_info(url)
         begin
-          host = URI.split(url)[2]
+          host = URI.parse(url).host
+          
           #FIXME we should check host against a list of known hosts here
           info = FileInfo.new
           info.streaming = false
@@ -90,13 +91,13 @@ module PDTP
 
       #returns the path of this file on the local filesystem
       def get_local_path(url)
-        path = URI.split(url)[5]
+        path = URI.parse(url).path
         path = path[1..path.size-1] #remove leading /
         (Pathname.new(@root) + path).to_s  
       end
 
       #returns the SHA256 hash of the specified chunk
-      def get_chunk_hash(url,chunk_id)
+      def get_chunk_hash(url, chunk_id)
         Digest::SHA256.hexdigest(get_info(url).chunk_data(chunk_id)) rescue nil
       end
     end

data/lib/pdtp/server/status_helper.rb

@@ -43,12 +43,12 @@ module PDTP
       # Iterate over all of a peer's active transfers
       def each_transfer(peer)
         raise ArgumentError, "no block given" unless block_given?
-        @dispatcher.client_info(peer).transfers.each { |_, transfer| yield transfer }
+        peer.transfers.each { |_, transfer| yield transfer }
       end
 
       # Iterate over all of a peer's active files
       def each_file(peer, &block)
-        @dispatcher.client_info(peer).chunk_info.get_file_stats.each(&block)
+        peer.chunk_info.get_file_stats.each(&block)
       end
 
       # Name of a peer (client ID or file service)
@@ -56,7 +56,7 @@ module PDTP
         if peer.file_service?
           "<b>File Service</b>"
         else
-          @dispatcher.connection_name(peer)
+          peer.client_id
         end
       end
       
@@ -65,6 +65,22 @@ module PDTP
         host, port = peer.get_peer_info
         "#{host}:#{port}"
       end
+      
+      # Upstream bandwidth of a peer
+      def upstream_bandwidth(peer)
+        bandwidth = peer.upstream_bandwidth
+        return 'N/A' if bandwidth.nil?
+        
+        "#{bandwidth / 1024} kBps"
+      end
+      
+      # Downstream bandwidth of a peer
+      def downstream_bandwidth(peer)
+        bandwidth = peer.downstream_bandwidth
+        return 'N/A' if bandwidth.nil?
+        
+        "#{bandwidth / 1024} kBps"
+      end
 
       # Information about an active transfer
       def transfer_info(peer, transfer)

data/lib/pdtp/server/transfer.rb

@@ -30,34 +30,46 @@ module PDTP
       attr_accessor :creation_time
       attr_accessor :verification_asked
 
-      #generates a transfer id based on 2 client ids, a url, and a byte range
+      # Generates a transfer id based on 2 client ids, a url, and a byte range
       def self.gen_transfer_id(id1,id2,url,byte_range)
-        a = id1<id2 ? id1 : id2
-        b = id1<id2 ? id2 : id1
+        a = id1 < id2 ? id1 : id2
+        b = id1 < id2 ? id2 : id1
         "#{a}$#{b}$#{url}$#{byte_range}"
       end
 
-      def initialize(taker,giver,url,chunkid,byte_range,connector_receives=true)
-        @taker,@giver,@url,@chunkid,@byte_range=taker,giver,url,chunkid,byte_range
+      def initialize(taker, giver, url, chunkid, byte_range, connector_receives = true)
+        @taker, @giver, @url, @chunkid, @byte_range = taker, giver, url, chunkid, byte_range
 
         @verification_asked = false
         @creation_time = Time.now
+        
         if connector_receives
-          @connector=@taker
-          @acceptor=@giver
+          @connector = @taker
+          @acceptor = @giver
         else
-          @connector=@giver
-          @acceptor=@taker
+          @connector = @giver
+          @acceptor = @taker
         end
 
         recompute_transfer_id
       end
 
-      #calculates the transfer id for this transfer based on the local data
+      # Calculates the transfer id for this transfer based on the local data
       def recompute_transfer_id
-        id1=connector.user_data.client_id
-        id2=acceptor.user_data.client_id
-        @transfer_id=Transfer::gen_transfer_id(id1,id2,@url,@byte_range)
+        id1 = connector.client_id
+        id2 = acceptor.client_id
+        @transfer_id = Transfer::gen_transfer_id id1, id2, @url, @byte_range
+      end
+      
+      # Update internal data upon a successful transfer
+      def success
+        giver.success self
+        taker.success self
+      end
+      
+      # Update internal data upon a failed transfer
+      def failure
+        taker.failure self
       end
 
       def to_s
@@ -67,7 +79,7 @@ module PDTP
       def debug_str
         str = ''
         str << "to_s=#{to_s}"
-        str << "   taker_id=#{@taker.user_data.client_id} giver_id=#{@giver.user_data.client_id}" 
+        str << "   taker_id=#{@taker.client_id} giver_id=#{@giver.client_id}" 
       end
     end
   end