distribustream 0.1.0 → 0.2.0

data/lib/pdtp/server/dispatcher.rb

@@ -0,0 +1,370 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require File.dirname(__FILE__) + '/../common/protocol'
+require File.dirname(__FILE__) + '/../common/common_init'
+require File.dirname(__FILE__) + '/file_service'
+require File.dirname(__FILE__) + '/client_info'
+require File.dirname(__FILE__) + '/transfer'
+
+require 'thread'
+require 'erb'
+
+module PDTP
+  class Server
+    # Core dispatching and control logic for PDTP servers
+    class Dispatcher
+      attr_reader :connections
+      attr_accessor :file_service
+      def initialize
+        @connections = []
+        @used_client_ids = {} #keeps a list of client ids in use, they must be unique
+        @updated_clients = {} #a set of clients that have been modified and need transfers spawned
+        @stats_mutex=Mutex.new
+      end
+
+      #called by pdtp_protocol when a connection is created
+      def connection_created(connection)
+        @stats_mutex.synchronize do
+          @@log.info "Client connected: #{connection.get_peer_info.inspect}"
+          connection.user_data = ClientInfo.new
+          @connections << connection 
+        end
+      end  
+
+      #called by pdtp_protocol when a connection is destroyed
+      def connection_destroyed(connection)
+        @stats_mutex.synchronize do
+          @@log.info "Client connection closed: #{connection.get_peer_info.inspect}"
+          @connections.delete connection
+        end
+      end
+
+      # returns the ClientInfo object associated with this connection
+      def client_info(connection)
+        connection.user_data ||= ClientInfo.new
+      end
+
+      # called when a transfer either finishes, successfully or not
+      def transfer_completed(transfer,connection,chunk_hash,send_response=true)      
+        # did the transfer complete successfully?
+        local_hash=@file_service.get_chunk_hash(transfer.url,transfer.chunkid)
+
+        c1=client_info(transfer.taker)
+        c2=client_info(transfer.giver)
+
+        if connection==transfer.taker
+          success= (chunk_hash==local_hash)
+
+          if success
+            #the taker now has the file, so he can provide it
+            client_info(transfer.taker).chunk_info.provide(transfer.url,transfer.chunkid..transfer.chunkid)
+            c1.trust.success(c2.trust)
+          else
+            #transfer failed, the client still wants the chunk
+            client_info(transfer.taker).chunk_info.request(transfer.url,transfer.chunkid..transfer.chunkid)
+            c1.trust.failure(c2.trust)
+          end 
+
+          transfer.taker.send_message(:hash_verify, 
+            :url => transfer.url, 
+            :range => transfer.byte_range, 
+            :hash_ok => success
+          ) if send_response
+        end
+
+        #outstr="#{@ids[transfer.giver]}->#{@ids[transfer.taker]} transfer completed: #{transfer}"
+        #outstr=outstr+" t->g=#{c1.trust.weight(c2.trust)} g->t=#{c2.trust.weight(c1.trust)}" 
+        #outstr=outstr+"sent_by: "+ ( connection==transfer.taker ? "taker" : "giver" )
+        #outstr=outstr+" success=#{success} "
+        #@@log.debug(outstr)
+
+        #remove this transfer from whoever sent it
+        client_info(connection).transfers.delete(transfer.transfer_id)
+        @updated_clients[connection]=true #flag this client for transfer creation
+      end
+
+      #Creates a new transfer between two peers
+      #returns true on success, or false if the specified transfer is already in progress
+      def begin_transfer(taker, giver, url, chunkid)
+        byte_range = @file_service.get_info(url).chunk_range(chunkid) 
+        t = Transfer.new(taker, giver, url, chunkid, byte_range)
+
+        #make sure this transfer doesnt already exist
+        t1 = client_info(taker).transfers[t.transfer_id]
+        t2 = client_info(giver).transfers[t.transfer_id]
+        return false unless t1.nil? and t2.nil?
+
+        client_info(taker).chunk_info.transfer(url, chunkid..chunkid) 
+        client_info(taker).transfers[t.transfer_id] = t
+        client_info(giver).transfers[t.transfer_id] = t
+
+        #send transfer message to the connector
+        addr, port = t.acceptor.get_peer_info
+      
+        t.connector.send_message(:transfer,
+          :host => addr,
+          :port => t.acceptor.user_data.listen_port,
+          :method => t.connector == t.taker ? "get" : "put",
+          :url => url,
+          :range => byte_range,
+          :peer_id => client_info(t.acceptor).client_id
+        )
+        true
+      end
+
+      #this function removes all stalled transfers from the list
+      #and spawns new transfers as appropriate
+      #it must be called periodically by EventMachine
+      def clear_all_stalled_transfers
+        @connections.each { |connection| clear_stalled_transfers_for_client connection }  
+        spawn_all_transfers
+      end
+
+      #removes all stalled transfers that this client is a part of
+      def clear_stalled_transfers_for_client(client_connection)
+        client_info(client_connection).get_stalled_transfers.each do |transfer|
+          transfer_completed transfer, client_connection, nil, false
+        end  
+      end
+
+      #spawns uploads and downloads for this client.
+      #should be called every time there is a change that would affect 
+      #what this client has or wants
+      def spawn_transfers_for_client(client_connection)
+        info = client_info client_connection
+
+        while info.wants_download? do
+          break if spawn_download_for_client(client_connection) == false
+        end
+
+        while info.wants_upload? do
+          break if spawn_upload_for_client(client_connection) == false
+        end
+      end
+
+      #creates a single download for the specified client
+      #returns true on success, false on failure
+      def spawn_download_for_client(client_connection)
+        feasible_peers=[]
+
+        c1info=client_info(client_connection)
+        begin
+          url,chunkid=c1info.chunk_info.high_priority_chunk
+        rescue
+          return false
+        end
+
+        @connections.each do |c2|
+          next if client_connection==c2
+          next if client_info(c2).wants_upload? == false
+          if client_info(c2).chunk_info.provided?(url,chunkid)
+            feasible_peers << c2
+            break if feasible_peers.size > 5
+          end
+        end
+
+        # we now have a list of clients that have the requested chunk.
+        # pick one and start the transfer
+        if feasible_peers.size > 0
+          #FIXME base this on the trust model
+          giver=feasible_peers[rand(feasible_peers.size)]
+          return begin_transfer(client_connection,giver,url,chunkid)
+          #FIXME should we try again if begin_transfer fails?
+        end
+
+        false
+      end
+
+      #creates a single upload for the specified client
+      #returns true on success, false on failure
+      def spawn_upload_for_client(client_connection)
+        c1info=client_info(client_connection)
+
+        @connections.each do |c2|
+          next if client_connection==c2
+          next if client_info(c2).wants_download? == false
+
+          begin
+            url,chunkid=client_info(c2).chunk_info.high_priority_chunk
+          rescue
+            next
+          end
+
+          if c1info.chunk_info.provided?(url,chunkid)
+            return begin_transfer(c2,client_connection,url,chunkid)
+          end
+        end
+
+        false
+      end
+
+      #called by pdtp_protocol for each message that comes in from the wire
+      def dispatch_message(command, message, connection)
+        @stats_mutex.synchronize do
+          dispatch_message_needslock command, message, connection
+        end
+      end
+
+      #creates new transfers for all clients that have been updated
+      def spawn_all_transfers
+        while @updated_clients.size > 0 do
+          tmp=@updated_clients
+          @updated_clients=Hash.new
+          tmp.each do |client,true_key| 
+            spawn_transfers_for_client(client)
+          end
+        end    
+      end
+
+      #handles the request, provide, unrequest, unprovide messages
+      def handle_requestprovide(connection,message)
+        type=message["type"]
+        url=message["url"]
+        info=@file_service.get_info(url) rescue nil
+        raise ProtocolWarn.new("Requested URL: '#{url}' not found") if info.nil?
+
+        exclude_partial= (type=="provide") #only exclude partial chunks from provides
+        range=info.chunk_range_from_byte_range(message["range"],exclude_partial)
+
+        #call request, provide, unrequest, or unprovide
+        client_info(connection).chunk_info.send( type.to_sym, url, range)
+        @updated_clients[connection]=true #add to the list of client that need new transfers
+      end
+
+      #handles all incoming messages from clients
+      def dispatch_message_needslock(command, message, connection)
+        # store the command in the message hash
+        message["type"] = command
+      
+        #require the client to register their client id and listen port before doing anything
+        if command != "register" and client_info(connection).client_id.nil?
+          raise ProtocolError.new("You need to send a 'client_info' message first")
+        end 
+
+        case command
+        when "register"
+          cid = message["client_id"]
+          #make sure this id isnt in use
+          if @used_client_ids[cid]
+            raise ProtocolError.new("Your client id: #{cid} is already in use.")   
+          end
+        
+          @used_client_ids[cid] = true 
+          client_info(connection).listen_port = message["listen_port"]
+          client_info(connection).client_id = cid
+        when "ask_info"
+          info = file_service.get_info(message["url"])
+          response = { :url => message["url"] }
+          unless info.nil?
+            response[:size] = info.file_size
+            response[:chunk_size] = info.base_chunk_size
+            response[:streaming] = info.streaming
+          end
+          connection.send_message :tell_info, response
+        when "request", "provide", "unrequest", "unprovide"
+          handle_requestprovide connection, message
+        when "ask_verify"
+          #check if the specified transfer is a real one
+          my_id = client_info(connection).client_id
+          transfer_id=Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
+          ok = !!client_info(connection).transfers[transfer_id]
+          client_info(connection).transfers[transfer_id].verification_asked=true if ok
+          @@log.debug "AskVerify not ok: id=#{transfer_id}" unless ok
+          connection.send_message(:tell_verify,
+            :url     => message["url"],
+            :peer_id => message["peer_id"],
+            :range   => message["range"],
+            :peer    => message["peer"],
+            :authorized=>ok
+          )
+        when "completed"
+          my_id = client_info(connection).client_id
+          transfer_id=  Transfer::gen_transfer_id(
+            my_id,message["peer_id"],
+            message["url"],
+            message["range"]
+          )
+          transfer=client_info(connection).transfers[transfer_id]
+          @@log.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
+          if transfer
+            transfer_completed(transfer,connection,message["hash"])
+          else
+            raise ProtocolWarn.new("You sent me a transfer completed message for unknown transfer: #{transfer_id}")
+          end
+        when 'protocol_error', 'protocol_warn' #ignore
+        else raise ProtocolError.new("Unhandled message type: #{command}")
+        end
+
+        spawn_all_transfers
+      end
+
+      #returns a string representing the specified connection
+      def connection_name(c)
+        #host,port=c.get_peer_info
+        #return "#{get_id(c)}: #{host}:#{port}"
+        client_info(c).client_id
+      end
+
+      def generate_html_stats
+        @stats_mutex.synchronize { generate_html_stats_needslock }
+      end
+
+      #builds an html page with information about the server's internal workings
+      def generate_html_stats_needslock
+        s = ERB.new <<EOF
+<html><head><title>DistribuStream Statistics</title></head>
+<body>
+<h1>DistribuStream Statistics</h1>
+Time=<%= Time.new %><br> Connected Clients=<%= @connections.size %>
+<center><table border=1>
+<tr><th>Client</th><th>Transfers</th><th>Files</th></tr>
+<% @connections.each do |c| %>
+  <tr><td>
+  <% host, port = c.get_peer_info %>
+  <%= connection_name(c) %><br><%= host %>:<%= port %>
+  </td>
+  <td>
+  <%
+  client_info(c).transfers.each do |key,t|
+    if c==t.giver
+      type="UP: "
+      peer=t.taker
+    else
+      type="DOWN: "
+      peer=t.giver
+    end
+    %>
+    <%= type %> id=<%= t.transfer_id %><br>
+    <%
+  end
+  %>
+  </td>
+  <td>
+  <%
+  client_info(c).chunk_info.get_file_stats.each do |fs|
+    %>
+    <%= fs.url %> size=<%= fs.file_chunks %> req=<%= fs.chunks_requested %>
+    prov=<%= fs.chunks_provided %> transf=<%= fs.chunks_transferring %><br>    
+    <%
+  end      
+  %>
+  </td></tr>
+  <%
+end
+%>
+</table>
+</body></html>
+EOF
+        s.result binding
+      end
+    end
+  end
+end

data/lib/pdtp/server/file_service.rb

@@ -48,7 +48,7 @@ module PDTP
 
     #The file service provides utilities for determining various information about files.
     class FileService < PDTP::FileService
-      attr_accessor :root,:default_chunk_size
+      attr_accessor :root, :default_chunk_size
 
       def initialize     
         @root = ''

data/lib/pdtp/server/file_service_protocol.rb

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require 'digest/md5'
+require 'thread'
+
+require File.dirname(__FILE__) + '/../common/protocol'
+require File.dirname(__FILE__) + '/../server/file_service'
+require File.dirname(__FILE__) + '/../client/connection'
+require File.dirname(__FILE__) + '/../client/http_handler'
+
+module PDTP
+  class Server
+    class FileService
+      # Implements the file service for the pdtp protocol
+      class Protocol < PDTP::Client::Connection
+        attr_reader :client, :connection, :file_service, :client_id, :transfers, :lock
+        
+        def initialize(data)
+          @transfers = []
+          @lock = Mutex.new
+          @client = self
+          @connection = self
+          @client_id = Digest::MD5.hexdigest "#{Time.now.to_f}#{$$}"
+          
+          super data
+        end
+        
+        # Called after a connection to the server has been established
+        def connection_completed
+          begin
+            @listen_addr = '0.0.0.0'
+            @listen_port = @@config[:listen_port]
+
+            #create the client
+            #client = PDTP::Client.new
+            #PDTP::Protocol.listener = client
+            #client.server_connection = self
+            #client.generate_client_id listen_port
+
+            # Start a mongrel server on the specified port.  If it isnt available, keep trying higher ports
+            begin
+              mongrel_server = Mongrel::HttpServer.new @listen_addr, @listen_port
+            rescue Exception => e
+              @listen_port += 1
+              retry
+            end
+
+            #@@log.info "listening on port #{@listen_port}"
+            @http_handler = Client::HttpHandler.new(self)
+            
+            @@log.info "listening on port #{@listen_port}"
+            mongrel_server.register "/", @http_handler
+            mongrel_server.run
+
+            # Register our client_id and listen_port
+            send_message :register, :listen_port => @listen_port, :client_id => @client_id
+
+            @@log.info 'This client is providing'
+            @file_service = PDTP::Server::FileService.new
+            @file_service.root = @base_path
+            #client.file_service = sfs #give this client access to all data
+
+            hostname = @@config[:vhost]
+
+            # Provide all the files in the root directory
+            files = find_files @base_path
+            files.each { |file| send_message :provide, :url => "http://#{hostname}/#{file}" }
+          rescue Exception => e
+            puts "Exception in connection_completed: #{e}"
+            puts e.backtrace.join("\n")
+            exit
+          end
+        end  
+
+        def unbind
+          super
+          puts "Disconnected from PDTP server."
+        end
+        
+        #########
+        protected
+        #########
+        
+        # Fine all suitable files in the give path
+        def find_files(base_path)
+          require 'find'
+
+          found = []
+          excludes = %w{.svn CVS}
+          base_full = File.expand_path(base_path)
+
+          Find.find(base_full) do |path|
+            if FileTest.directory?(path)
+              next unless excludes.include?(File.basename(path))
+              Find.prune
+            else
+              filename = path[(base_path.size - path.size + 1)..-1] #the entire file path after the base_path
+              found << filename
+            end
+          end
+
+          found
+        end
+      end
+    end
+  end
+end

data/lib/pdtp/server/stats_handler.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'mongrel'
+
+module PDTP
+  class Server
+    #set up the mongrel server for serving the stats page
+    class StatsHandler < Mongrel::HttpHandler
+      def initialize(server)
+        @server = server
+      end
+
+      def process(request,response)
+        response.start(200) do |head, out|
+          out.write begin
+            outstr = @server.generate_html_stats
+          rescue Exception=>e
+            outstr = "Exception: #{e}\n#{e.backtrace.join("\n")}"
+          end
+        end    
+      end
+    end
+  end
+end

data/lib/pdtp/server/trust.rb

@@ -9,76 +9,78 @@
 #++
 
 module PDTP
-  #maintains trust information for a single client
-  class Trust
-    #struct for storing a single trust relationship
-    class Edge
-      attr_accessor :trust, :success, :transfers
+  class Server
+    #maintains trust information for a single client
+    class Trust
+      #struct for storing a single trust relationship
+      class Edge
+        attr_accessor :trust, :success, :transfers
 
-      def initialize(trust = 1.0, success = 1.0, transfers = 1.0)
-        @trust = trust
-        @success = success
-        @transfers = transfers
+        def initialize(trust = 1.0, success = 1.0, transfers = 1.0)
+          @trust = trust
+          @success = success
+          @transfers = transfers
+        end
       end
-    end
 
-    attr_reader :outgoing, :implicit
+      attr_reader :outgoing, :implicit
 
-    def initialize(incoming = {}, outgoing = {}, implicit = {})
-      @incoming = incoming
-      @outgoing = outgoing
-      @implicit = implicit
-    end
+      def initialize(incoming = {}, outgoing = {}, implicit = {})
+        @incoming = incoming
+        @outgoing = outgoing
+        @implicit = implicit
+      end
 
-    #I have successfully downloaded a chunk from 'node'
-    def success(node)
-      if @outgoing[node].nil?
-        @outgoing[node] = Edge.new
-      else
-        @outgoing[node].success += 1.0
-        @outgoing[node].transfers += 1.0
+      #I have successfully downloaded a chunk from 'node'
+      def success(node)
+        if @outgoing[node].nil?
+          @outgoing[node] = Edge.new
+        else
+          @outgoing[node].success += 1.0
+          @outgoing[node].transfers += 1.0
+        end
+        normalize
       end
-      normalize
-    end
 
-    #I have failed to download a chunk from 'node'
-    def failure(node)
-      @outgoing[node] = Edge.new if @outgoing[node].nil?
-      @outgoing[node].transfers += 1.0
-      normalize
-    end
+      #I have failed to download a chunk from 'node'
+      def failure(node)
+        @outgoing[node] = Edge.new if @outgoing[node].nil?
+        @outgoing[node].transfers += 1.0
+        normalize
+      end
 
-    #returns a number from 0 to 1 saying how much I trust 'node'
-    def weight(node)
-      return @outgoing[node].trust unless @outgoing[node].nil?
-      return @implicit[node].trust unless @implicit[node].nil?
-      0
-    end
+      #returns a number from 0 to 1 saying how much I trust 'node'
+      def weight(node)
+        return @outgoing[node].trust unless @outgoing[node].nil?
+        return @implicit[node].trust unless @implicit[node].nil?
+        0
+      end
 
-    # brings all trust values between 0 and 1
-    def normalize
-      total_success = 0
-      total_transfers = 0
+      # brings all trust values between 0 and 1
+      def normalize
+        total_success = 0
+        total_transfers = 0
 
-      @outgoing.each do |_, link|
-        total_success += link.success
-        total_transfers += link.transfers
-      end
+        @outgoing.each do |_, link|
+          total_success += link.success
+          total_transfers += link.transfers
+        end
 
-      @outgoing.each { |_, link| link.trust = link.success / total_transfers }
-      @outgoing.each do |target, link|
-        [target.outgoing, target.implicit].each do |links|
-          links.each do |nextlinkedge|
-            nextlinktarget = nextlinkedge[0]
-            nextlink = nextlinkedge[1]
-            next unless outgoing[nextlinktarget].nil?
+        @outgoing.each { |_, link| link.trust = link.success / total_transfers }
+        @outgoing.each do |target, link|
+          [target.outgoing, target.implicit].each do |links|
+            links.each do |nextlinkedge|
+              nextlinktarget = nextlinkedge[0]
+              nextlink = nextlinkedge[1]
+              next unless outgoing[nextlinktarget].nil?
 
-            if implicit[nextlinktarget].nil? || implicit[nextlinktarget].trust < (link.trust * nextlink.trust)
-              implicit[nextlinktarget] = Edge.new(
-              link.trust * nextlink.trust, 
-              nextlink.success, 
-              nextlink.transfers
-              )
+              if implicit[nextlinktarget].nil? || implicit[nextlinktarget].trust < (link.trust * nextlink.trust)
+                implicit[nextlinktarget] = Edge.new(
+                link.trust * nextlink.trust, 
+                nextlink.success, 
+                nextlink.transfers
+                )
+              end
             end
           end
         end