distribustream 0.5.0 → 0.5.1

data/CHANGES

@@ -1,3 +1,11 @@
+Version 0.5.1
+
+* Add #score method to peers to compare ideal transfer candidates
+
+* Convert PDTP::Server::ChunkInfo to store chunk data in RangeMaps
+
+* Add daemonization support
+
 Version 0.5.0
 
 * Factor traffic routing code into PDTP::Server::TransferManager

data/bin/dstream

@@ -21,6 +21,7 @@
 # See http://distribustream.org/
 #++
 
+require 'rubygems'
 require 'optparse'
 require File.dirname(__FILE__) + '/../lib/pdtp/server'
 
@@ -28,6 +29,7 @@ program_name = File.basename($0)
 config_filename = nil
 listen_addr = nil
 listen_port = nil
+daemonize = true
 
 OptionParser.new do |opts|
   opts.banner = "Usage: #{program_name} [options]"
@@ -40,11 +42,14 @@ OptionParser.new do |opts|
   opts.on("--port PORT", "Listen on the specified port number.") do |p|
     listen_port = p
   end
+  opts.on("--foreground", "Run in the foreground instead of daemonizing.") do
+    daemonize = false
+  end
   opts.on("--help", "Prints this usage info.") do
     puts opts
     exit
   end
-  opts.on("--version", "Print version information.") do
+ opts.on("--version", "Print version information.") do
     puts "#{program_name} #{PDTP::VERSION} - DistribuStream server application"
     puts "Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)"
     exit
@@ -57,7 +62,9 @@ config = {
   :listen_port      => 8000, #client listen port
   :file_root        => '.',
   :chunk_size       => 5000,
-  :quiet            => true
+  :quiet            => true,
+  :logfile          => nil,
+  :daemonize        => true
 }
 
 if config_filename.nil?
@@ -96,4 +103,10 @@ logger.level = Logger::INFO if config[:quiet]
 server.enable_logging logger
 server.enable_status_page
 server.enable_file_service config[:file_root], :chunk_size => config[:chunk_size]
-server.run
+
+if daemonize and config[:daemonize]
+  require 'daemons/daemonize'
+  Daemonize.daemonize config[:logfile], program_name
+end
+
+server.run

data/conf/bigchunk.yml

@@ -15,4 +15,10 @@
 :file_root: /Users/tony/dstest/files
 
 # Size of segments to be distributed (in bytes)
-:chunk_size: 500000
+:chunk_size: 512000
+
+# Path to logfile
+:logfile: dstream.log
+
+# Daemonize or run in foreground
+:daemonize: true

data/conf/debug.yml

@@ -16,3 +16,9 @@
 
 # Size of segments to be distributed (in bytes)
 :chunk_size: 100000
+
+# Path to logfile
+:logfile: dstream.log
+
+# Daemonize or run in foreground
+:daemonize: true

data/conf/example.yml

@@ -16,3 +16,9 @@
 
 # Size of segments to be distributed (in bytes)
 :chunk_size: 100000
+
+# Path to logfile
+:logfile: dstream.log
+
+# Daemonize or run in foreground
+:daemonize: true

data/distribustream.gemspec

@@ -2,10 +2,10 @@ require 'rubygems'
 
 GEMSPEC = Gem::Specification.new do |s|
   s.name = "distribustream"
-  s.version = "0.5.0"
+  s.version = "0.5.1"
   s.authors = ["Tony Arcieri", "Ashvin Mysore", "Galen Pahlke", "James Sanders", "Tom Stapleton"]
   s.email = "tony@clickcaster.com"
-  s.date = "2008-11-27"
+  s.date = "2008-12-04"
   s.summary = "DistribuStream is a fully open peercasting system allowing on-demand or live streaming media to be delivered at a fraction of the normal cost"
   s.platform = Gem::Platform::RUBY
 
@@ -17,6 +17,7 @@ GEMSPEC = Gem::Specification.new do |s|
   s.add_dependency("eventmachine", ">= 0.9.0")
   s.add_dependency("mongrel", ">= 1.0.2")
   s.add_dependency("json", ">= 1.1.0")
+  s.add_dependency("activesupport", ">= 1.4.0")
 
   # RubyForge info
   s.homepage = "http://distribustream.org"

data/lib/pdtp/common.rb

@@ -20,9 +20,12 @@
 # See http://distribustream.org/
 #++
 
+require 'rubygems'
+require 'active_support'
+
 # Namespace for all PDTP components
 module PDTP
-  PDTP::VERSION = '0.5.0' unless defined? PDTP::VERSION
+  PDTP::VERSION = '0.5.1' unless defined? PDTP::VERSION
   def self.version() VERSION end
 
   PDTP::DEFAULT_PORT = 6086 unless defined? PDTP::DEFAULT_PORT

data/lib/pdtp/common/length_prefix_protocol.rb

@@ -50,10 +50,10 @@ module PDTP
         new_data = data[0..(toread - 1)]
         
         @data << new_data        
-        @read += new_data.length
+        @read += new_data.size
         return nil unless @read == @size
         
-        @length = @data.unpack(@size == 2 ? 'n' : 'N').first
+        @payload_length = @data.unpack(@size == 2 ? 'n' : 'N').first
         result = data[toread..data.length]
         
         return nil if result.nil? or result.empty?
@@ -61,13 +61,13 @@ module PDTP
       end
       
       # Length of the payload extracted from the prefix
-      def length
-        raise RuntimeError, 'length called before prefix extracted' unless read?
-        @length
+      def payload_length
+        raise RuntimeError, 'payload_length called before prefix extracted' unless read?
+        @payload_length
       end
       
       def reset!
-        @length = nil
+        @payload_length = nil
         @read = 0
         @data = ''        
       end
@@ -97,13 +97,13 @@ module PDTP
         @buffer << data
         
         # Don't do anything until we receive the specified amount of data
-        return unless @buffer.length >= @prefix.length
+        return unless @buffer.size >= @prefix.payload_length
         
         # Extract the specified amount of data and process it
-        data = @buffer[0..(@prefix.length - 1)]
+        data = @buffer[0..(@prefix.payload_length - 1)]
         
         # Store any remaining data
-        remainder = @buffer[@prefix.length..@buffer.length]
+        remainder = @buffer[@prefix.payload_length..@buffer.length]
         
         # Invoke receive_packet and allow the user to process the data
         receive_packet data
@@ -127,4 +127,4 @@ module PDTP
       send_data [data.size].pack(@prefix.size == 2 ? 'n' : 'N') << data
     end
   end
-end
+end

data/lib/pdtp/common/range_map.rb

@@ -0,0 +1,138 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.org/
+#++
+
+module PDTP
+  # Structure which maps non-overlapping ranges to objects 
+  class RangeMap
+    include Enumerable
+    
+    def initialize
+      @ranges = []
+    end
+    
+    # Insert a range into the RangeMap
+    def []=(range, obj)
+      range = range..range if range.is_a?(Integer) or range.is_a?(Float)
+      raise ArgumentError, 'key must be a number or range' unless range.is_a?(Range)
+                
+      index = binary_search range.begin
+      
+      # Correct total overlap
+      [index, index + 1].each do |i|
+        while @ranges[i] and @ranges[i][0].begin >= range.begin and @ranges[i][0].end <= range.end
+          @ranges.delete_at i
+        end
+      end
+        
+      # Correct overlap with leftmost member
+      if @ranges[index] and @ranges[index][0].begin <= range.begin
+        if range.end < @ranges[index][0].end
+          @ranges[index][0] = (range.end + 1)..(@ranges[index][0].end)
+        else
+          @ranges[index][0] = (@ranges[index][0].begin)..(range.begin - 1)
+          index += 1
+        end
+      end
+      
+      # Correct overlap with rightmost member
+      if @ranges[index] and @ranges[index][0].begin <= range.end
+        @ranges[index][0] = (range.end + 1)..(@ranges[index][0].end)
+      end
+      
+      @ranges.insert(index, [range, obj])
+      obj
+    end
+    
+    # Find a value in the RangeMap
+    def [](value)
+      return nil if empty?
+      range, obj = @ranges[binary_search(value)]
+      return nil if range.nil?
+      
+      case value
+      when Integer, Float
+        return nil unless range.include?(value)
+      else raise ArgumentError, 'key must be a number'
+      end
+      
+      obj
+    end
+    
+    # Iterate over all ranges and objects
+    def each(&block)
+      @ranges.each(&block)
+    end
+            
+    # Number of entries in the RangeMap
+    def size
+      @ranges.size
+    end
+    
+    # First range
+    def first
+      @ranges.first[0]
+    end
+    
+    # Last range
+    def last
+      @ranges.last[0]
+    end
+    
+    # Is the RangeMap empty?
+    def empty?
+      @ranges.empty?
+    end
+    
+    # Remove all elements from the RangeMap
+    def clear
+      @ranges.clear
+      self
+    end
+    
+    # Inspect the RangeMap
+    def inspect
+      "#<PDTP::RangeMap {#{@ranges.map { |r| "#{r.first}=>#{r.last.inspect}" }.join(", ")}}>"
+    end
+    
+    #########
+    protected
+    #########
+    
+    # Find the index of the range nearest the given value
+    def binary_search(value, a = 0, b = @ranges.size)
+      pivot = (a + b) / 2          
+      range, _ = @ranges[pivot]
+      
+      return b if range.nil?
+      
+      if value < range.begin
+        return a if a == pivot
+        binary_search(value, a, pivot) 
+      elsif value > range.end
+        return b if b == pivot
+        binary_search(value, pivot + 1, b)
+      else
+        pivot
+      end
+    end
+  end
+end

data/lib/pdtp/server/bandwidth_estimator.rb

@@ -20,6 +20,8 @@
 # See http://distribustream.org/
 #++
 
+require File.dirname(__FILE__) + '/../common'
+
 module PDTP
   class Server
     # BandwidthEstimator logs the durations of chunk transfers over time,
@@ -76,7 +78,7 @@ module PDTP
         return 0 if pruned.size.zero?
         
         # Calculate the mean of the pruned estimates
-        @estimate = pruned.inject(0) { |a, v| a + v } / pruned.size
+        @estimate = pruned.sum / pruned.size
       end
       
       #########
@@ -95,12 +97,12 @@ module PDTP
         end
         
         # Sum the overlapping transfer rates and store them as estimates
-        @estimates ||= overlapping_transfers.map { |t| t.inject(0) { |a,v| a + v } }
+        @estimates ||= overlapping_transfers.sum
       end
       
       # Compute the mean of the bandwidth estimates
       def estimates_mean
-        @mean ||= estimates.inject(0) { |a, v| a + v } / estimates.size
+        @mean ||= estimates.sum / estimates.size
       end
       
       # Compute the variance of the bandwidth estimates

data/lib/pdtp/server/chunk_info.rb

@@ -20,77 +20,84 @@
 # See http://distribustream.org/
 #++
 
+require File.dirname(__FILE__) + '/../common/range_map'
+
 module PDTP
   class Server
     # Stores information about the chunks requested or provided by a client
-    class ChunkInfo
+    # FIXME this entire class is in need of a rewrite
+    class ChunkInfo      
       def initialize
         @files = {}
       end
 
-      #each chunk can either be provided, requested, transfer, or none
-      #FIXME some metaprogramming is probably in order here
-      def provide(filename,range); set(filename, range, :provided); end
-      def unprovide(filename,range); set(filename,range, :none); end
-      def request(filename,range); set(filename,range, :requested); end
-      def unrequest(filename,range); set(filename,range, :none); end
-      def transfer(filename,range); set(filename,range, :transfer); end
+      # Set the state for a given chunk range
+      def set(state, filename, range)        
+        chunks = @files[filename] ||= RangeMap.new
+        chunks[range] = state
+      end
 
-      def provided?(filename,chunk); get(filename,chunk)  == :provided; end
-      def requested?(filename,chunk); get(filename,chunk) == :requested; end
+      def provided?(filename, chunk)
+        get(filename, chunk) == :provided
+      end
+      
+      def requested?(filename,chunk)
+        get(filename, chunk) == :requested
+      end
 
       #returns a high priority requested chunk
       def high_priority_chunk
         #right now return any chunk
-        @files.each do |name,file|
-          file.each_index do |i|
-            return [name, i] if file[i] == :requested
-          end
+        @files.each do |name, chunkmap|
+          range, _ = chunkmap.find { |_, state| state == :requested }
+          next unless range
+          
+          return name, range.begin
         end  
 
         nil
       end
-
-      class FileStats
-        attr_accessor :file_chunks, :chunks_requested, :url
-        attr_accessor :chunks_provided, :chunks_transferring
-
-        def initialize
-          @url = ""
-          @file_chunks = 0
-          @chunks_requested = 0
-          @chunks_provided = 0
-          @chunks_transferring = 0
-        end
-      end
-
+      
       # Returns an array of FileStats objects for debug output
       def get_file_stats
-        @files.map do |name, file|
+        @files.map do |name, chunkmap|
           fs = FileStats.new
-          fs.file_chunks = file.size
+          fs.file_chunks = chunkmap.last.end + 1
           fs.url = name
-          file.each do |chunk|
-            fs.chunks_requested += 1 if chunk == :requested
-            fs.chunks_provided += 1 if chunk == :provided
-            fs.chunks_transferring += 1 if chunk == :transfer
+          
+          chunkmap.each do |range, state|
+            length = range.end - range.begin + 1
+            
+            case state
+            when :requested    then fs.chunks_requested += length
+            when :provided     then fs.chunks_provided += length
+            when :transferring then fs.chunks_transferring += length
+            end
           end
           
           fs
         end
       end
-
+            
       #########
       protected
       #########
 
-      def get(filename,chunk)
+      def get(filename, chunk)
         @files[filename][chunk] rescue :neither
       end
+      
+      class FileStats
+        attr_accessor :file_chunks, :chunks_requested, :url
+        attr_accessor :chunks_provided, :chunks_transferring
 
-      def set(filename,range,state)
-        chunks=@files[filename]||=Array.new
-        range.each { |i| chunks[i]=state }
+        def initialize
+          @url = ""
+          @file_chunks = 0
+          @chunks_requested = 0
+          @chunks_provided = 0
+          @chunks_transferring = 0
+        end
       end
     end
   end

data/lib/pdtp/server/connection.rb

@@ -20,6 +20,7 @@
 # See http://distribustream.org/
 #++
 
+require File.dirname(__FILE__) + '/../common'
 require File.dirname(__FILE__) + '/../common/protocol'
 require File.dirname(__FILE__) + '/chunk_info'
 require File.dirname(__FILE__) + '/trust'
@@ -57,11 +58,41 @@ module PDTP
         
         super
       end
+      
+      # Address prefix of this peer
+      def prefix(mask = 24)
+        raise ArgumentError, "mask must be 8, 16, or 24" unless [8, 16, 24].include?(mask)
+        addr, _ = @cached_peer_info
+        
+        octets = mask / 8
+        addr.split('.')[0..(octets - 1)].join('.')
+      end
+      
+      # Calculate a score from 0 to 10 for two clients being a good match
+      def score(peer)
+        s = 0.0
+        
+        # One point for each matching prefix
+        [8, 16, 24].each { |mask| s += 1 if prefix(mask) == peer.prefix(mask) }
+        
+        # Up to three points for having ample bandwidth compared to us
+        [0.5, 1, 2].each { |divisor| s += 1 if peer.upstream_bandwidth > downstream_bandwidth / divisor }
+        
+        # 0 - 4 points for trust
+        s += 4 * @trust.weight(peer.trust)
+        
+        #puts "#{peer} trust: #{@trust.weight(peer.trust)}"
+        
+        # Deprioritize file service by cutting its score in half
+        s /= 2 if peer.file_service?
+        
+        s
+      end
 
       # Log a completed transfer and update internal data
       def success(transfer)
         if transfer.taker == self
-          @chunk_info.provide transfer.url, transfer.chunkid..transfer.chunkid
+          @chunk_info.set :provided, transfer.url, transfer.chunkid..transfer.chunkid
           @trust.success transfer.giver.trust
           bandwidth_estimator = @downstream
         else
@@ -91,65 +122,82 @@ module PDTP
       def downstream_bandwidth
         @downstream.estimate rescue nil
       end
-
-      # Returns true if this client wants the server to spawn a transfer for it
+      
+      # Number of concurrent downloads desired desired
+      # FIXME hardcoded, should probably be computed or client-specified
+      def max_concurrent_downloads; 8; end
+      
+      # For now, keep concurrent_downloads equal to uploads
+      alias_method :max_concurrent_uploads, :max_concurrent_downloads
+      
+      # Maximum number of "half open" transfers allowed
+      # FIXME hardcoded, should probably be computed
+      def max_half_open; 8; end
+      
+      # Are we below the limit on half-open transfer slots? 
+      def empty_transfer_slots?
+        @transfers.select { |_, t| not t.verification_asked }.size < max_half_open
+      end
+      
+      # List of all active downloads
+      def downloads
+        @transfers.select { |_, t| t.taker == self and t.verification_asked }
+      end
+      
+      # List of all active uplaods
+      def uploads
+        @transfers.select { |_, t| t.giver == self and t.verification_asked }
+      end
+      
+      # Returns true if this client wants the server to spawn a download for it
       def wants_download?
-        transfer_state_allowed = 5
-        total_allowed = 10
-        transferring = 0
-        @transfers.each do |key, t|
-          transferring += 1 if t.verification_asked
-          return false if transferring >= transfer_state_allowed
-        end
-
-        @transfers.size < total_allowed
-      end 
-
-      # This could have a different definition, but it works fine to use wants_download?
-      alias_method :wants_upload?, :wants_download?
+        return false unless @chunk_info.high_priority_chunk
+        return false unless empty_transfer_slots?
+                
+        # Are we at our concurrent download limit?
+        downloads.size < max_concurrent_downloads
+      end
+      
+      # Returns true if this client wants the server to spawn an uplaod for it
+      def wants_upload?
+        return false unless empty_transfer_slots?
+        
+        # Are we at our concurrent upload limit?
+        uploads.size < max_concurrent_uploads
+      end
 
       # Returns a list of all the stalled transfers this client is a part of
       def stalled_transfers
-        stalled = []
         timeout = 20.0
         now = Time.now
-        @transfers.each do |key,t|
-          #only delete if we are the acceptor to prevent race conditions
-          next unless t.acceptor == self 
-          if now - t.creation_time > timeout and not t.verification_asked
-            stalled << t
+        
+        @transfers.inject([]) do |stalled, (_, t)|
+          # only delete if we are the acceptor to prevent race conditions
+          unless t.acceptor == self 
+            stalled << t if now - t.creation_time > timeout and not t.verification_asked
           end
+          
+          stalled
         end
-        stalled
       end
                   
       # Is this connection the file service?
       def file_service?
-        @file_service
+        false
       end
-      
-      # Mark this connection as being a file service
-      def mark_as_file_service
-        @file_service = true
-      end
-            
+                  
       #
-      # EventMachine callbacks to delegate to the dispatcher
+      # Delegate PDTP::Protocol callbacks to the dispatcher
       #
-            
-      def connection_completed
-        raise(RuntimeError, 'server was never initialized') unless @dispatcher
-        @dispatcher.connection_created self
-      end
-      
-      def connection_destroyed
-        raise(RuntimeError, 'server was never initialized') unless @dispatcher
-        @dispatcher.connection_destroyed self
-      end
       
-      def receive_message(command, message)
-        raise(RuntimeError, 'server was never initialized') unless @dispatcher
-        @dispatcher.dispatch_message command, message, self
+      %w{connection_completed connection_destroyed receive_message}.each do |method|
+        module_eval <<-EOD
+          def #{method}(*args)
+            raise(RuntimeError, 'server was never initialized') unless @dispatcher
+            args << self
+            @dispatcher.#{method}(*args)
+          end
+        EOD
       end
     end
   end

data/lib/pdtp/server/dispatcher.rb

@@ -21,12 +21,13 @@
 #++
 
 require File.dirname(__FILE__) + '/transfer_manager'
+require File.dirname(__FILE__) + '/file_service_connection'
 
 module PDTP
   class Server
     # Core dispatching and control logic for PDTP servers
     class Dispatcher
-      attr_reader :connections
+      attr_reader :connections, :server
       
       def initialize(server, file_service)
         @server = server
@@ -37,16 +38,22 @@ module PDTP
       end
 
       # Register a PDTP::Server::Connection with the Dispatcher
-      def connection_created(connection)
+      def connection_completed(connection)
         addr, port = connection.get_peer_info
         
+        # FIXME hacked file service registration.  There really ought to be a better way
+        # to both register and authenticate file services
         if @server.file_service_enabled? and not @seen_file_service and addr == @server.addr
-          #display file service greeting when we see it connect
+          # Display file service greeting when we see it connect
           @server.log "file service running at #{addr}:#{@server.instance_eval { @http_port }}"
           @seen_file_service = true
-          connection.mark_as_file_service
+          
+          # Extend connection with file service-specific method implementations
+          connection.extend FileServiceConnection
+        elsif not @seen_file_service
+          raise RuntimeError, "File service failed to initialize.  Ensure listen address is correct."
         else
-          #display greeting for normal client
+          # Display greeting for normal client
           @server.log "client connected: #{connection.get_peer_info.inspect}"
         end
         
@@ -59,16 +66,8 @@ module PDTP
         @connections.delete connection
       end
       
-      # This function removes all stalled transfers from the list
-      # and spawns new transfers as appropriate
-      # It must be called periodically by EventMachine
-      def clear_all_stalled_transfers
-        @connections.each { |connection| clear_stalled_transfers_for_client connection }  
-        @transfer_manager.spawn_all_transfers
-      end
-      
       # Handles all incoming messages from clients
-      def dispatch_message(command, message, connection)
+      def receive_message(command, message, connection)
         # Store the command in the message hash
         message["type"] = command
       
@@ -98,7 +97,7 @@ module PDTP
           end
           connection.send_message :tell_info, response
         when "request", "provide", "unrequest", "unprovide"
-          handle_requestprovide connection, message
+          handle_requestprovide connection, command, message
         when "ask_verify"
           #check if the specified transfer is a real one
           my_id = connection.client_id
@@ -136,6 +135,14 @@ module PDTP
         # Process all clients that are in need of new transfers
         @transfer_manager.spawn_all_transfers
       end
+      
+      # This function removes all stalled transfers from the list
+      # and spawns new transfers as appropriate
+      # It must be called periodically by EventMachine
+      def clear_all_stalled_transfers
+        @connections.each { |connection| clear_stalled_transfers_for_client connection }  
+        @transfer_manager.spawn_all_transfers
+      end
 
       #########
       protected
@@ -154,13 +161,8 @@ module PDTP
         local_hash = @file_service.get_chunk_hash transfer.url, transfer.chunkid
 
         if connection == transfer.taker
-          success = (chunk_hash == local_hash)
-          
-          if success
-            transfer.success
-          else
-            transfer.failure
-          end
+          success = (chunk_hash == local_hash) # Capture for hash_verify message
+          success ? transfer.success : transfer.failure
           
           transfer.taker.send_message(:hash_verify, 
             :url => transfer.url, 
@@ -174,18 +176,24 @@ module PDTP
         @transfer_manager.process_client(connection)
       end
 
+      CHUNK_STATES  = {
+        'request'   => :requested,
+        'provide'   => :provided,
+        'unrequest' => :none,
+        'unprovide' => :none,
+      } unless defined? CHUNK_STATES
+      
       # Handles the request, provide, unrequest, unprovide messages
-      def handle_requestprovide(connection, message)
-        type = message["type"]
+      def handle_requestprovide(connection, command, message)
         url = message["url"]
         info = @file_service.get_info(url) rescue nil
         raise ProtocolWarn, "Requested URL: '#{url}' not found" if info.nil?
 
-        exclude_partial = (type=="provide") #only exclude partial chunks from provides
+        exclude_partial = (command=="provide") #only exclude partial chunks from provides
         range = info.chunk_range_from_byte_range(message["range"],exclude_partial)
 
         #call request, provide, unrequest, or unprovide
-        connection.chunk_info.send(type.to_sym, url, range)
+        connection.chunk_info.set(CHUNK_STATES[command], url, range)
         @transfer_manager.process_client(connection)
       end
     end

data/lib/pdtp/server/file_info.rb

@@ -0,0 +1,58 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.org/
+#++
+
+require File.dirname(__FILE__) + '/../common/file_service'
+
+module PDTP
+  class Server
+    #The server specific file utilities
+    class FileInfo < PDTP::FileInfo
+      attr_accessor :path
+
+      #Return a raw string of chunk data. The range parameter is local to this chunk
+      #and zero based
+      def chunk_data(chunkid, range = nil)
+        begin
+          range = 0..chunk_size(chunkid) - 1 if range.nil? # full range of chunk if range isnt specified
+          raise if range.first < 0 or range.last >= chunk_size(chunkid)
+          start = range.first + chunkid * @base_chunk_size 
+          size = range.last - range.first + 1 
+          file = open @path
+          file.pos = start
+          file.read size
+        rescue nil
+        end
+      end
+
+      #reads the specified byte range from the file and returns it as a string
+      def read(range)
+        #puts "READING: range=#{range}"
+        begin
+          file = open @path
+          file.pos = range.first
+          file.read range.last - range.first + 1
+        rescue nil
+        end
+      end 
+    end
+  end
+end

data/lib/pdtp/server/file_service.rb

@@ -20,44 +20,14 @@
 # See http://distribustream.org/
 #++
 
-require "uri"    
-require "pathname"
-require "digest/sha2"
-require File.dirname(__FILE__) + '/../common/file_service.rb'    
+require 'uri'
+require 'pathname'
+require 'digest/sha2'
+require File.dirname(__FILE__) + '/../common/file_service'
+require File.dirname(__FILE__) + '/file_info'
 
 module PDTP
   class Server
-    #The server specific file utilities
-    class FileInfo < PDTP::FileInfo
-      attr_accessor :path
-
-      #Return a raw string of chunk data. The range parameter is local to this chunk
-      #and zero based
-      def chunk_data(chunkid, range = nil)
-        begin
-          range = 0..chunk_size(chunkid) - 1 if range.nil? # full range of chunk if range isnt specified
-          raise if range.first < 0 or range.last >= chunk_size(chunkid)
-          start = range.first + chunkid * @base_chunk_size 
-          size = range.last - range.first + 1 
-          file = open @path
-          file.pos = start
-          file.read size
-        rescue nil
-        end
-      end
-
-      #reads the specified byte range from the file and returns it as a string
-      def read(range)
-        #puts "READING: range=#{range}"
-        begin
-          file = open @path
-          file.pos = range.first
-          file.read range.last - range.first + 1
-        rescue nil
-        end
-      end 
-    end
-
     #The file service provides utilities for determining various information about files.
     class FileService < PDTP::FileService
       attr_accessor :root, :default_chunk_size

data/lib/pdtp/server/file_service_connection.rb

@@ -0,0 +1,44 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.org/
+#++
+
+require File.dirname(__FILE__) + '/../common'
+require File.dirname(__FILE__) + '/../common/protocol'
+require File.dirname(__FILE__) + '/chunk_info'
+require File.dirname(__FILE__) + '/trust'
+require File.dirname(__FILE__) + '/bandwidth_estimator'
+
+module PDTP
+  class Server
+    # Mixin for PDTP::Server::Connection to give it file service capibility
+    module FileServiceConnection
+      # We are the file service
+      def file_service?
+        true
+      end
+      
+      # The file service does not explicitly want uploads
+      #def wants_upload?
+      #  false
+      #end
+    end
+  end
+end

data/lib/pdtp/server/status_handler.rb

@@ -35,6 +35,7 @@ module PDTP
       
       def initialize(vhost, dispatcher)
         @vhost, @dispatcher = vhost, dispatcher
+        @status_erb = File.expand_path(File.dirname(__FILE__) + '/../../../status/index.erb')
       end
 
       # Process an incoming request to generate the status page
@@ -42,7 +43,7 @@ module PDTP
         response.start(200) do |head, out|
           out.write begin
             # Read the status page ERb template
-            erb_data = File.read(File.dirname(__FILE__) + '/../../../status/index.erb')
+            erb_data = File.read @status_erb
             
             # Render the status ERb template
             html = ERB.new(erb_data).result(binding)
@@ -59,4 +60,4 @@ module PDTP
       end      
     end
   end
-end
+end

data/lib/pdtp/server/transfer_manager.rb

@@ -30,22 +30,22 @@ module PDTP
         @connections, @file_service = connections, file_service
         @updated_clients = []
       end
-      
+
       # Add client to list of ones needing updates
       def process_client(client)
         @updated_clients << client
       end
-      
+
       # Creates new transfers for all clients that have been updated
       def spawn_all_transfers
         @updated_clients.each { |client| spawn_transfers_for_client(client) }
         @updated_clients.clear
       end
-      
+
       #########
       protected
       #########
-      
+
       # Spawns uploads and downloads for this client.
       # Should be called every time there is a change that would affect 
       # what this client has or wants
@@ -70,48 +70,62 @@ module PDTP
           return false
         end
 
-        @connections.each do |c2|
-          next if connection == c2
-          next unless c2.wants_upload?
-          if c2.chunk_info.provided?(url, chunkid)
-            feasible_peers << c2
-            break if feasible_peers.size > 5
-          end
-        end
-
-        # we now have a list of clients that have the requested chunk.
-        # pick one and start the transfer
-        if feasible_peers.size > 0
-          #FIXME base this on the trust model
-          giver = feasible_peers[rand(feasible_peers.size)]
-          return begin_transfer(connection,giver,url,chunkid)
-          #FIXME should we try again if begin_transfer fails?
+        @connections.each do |peer|
+          next if connection == peer
+          next unless peer.wants_upload?
+          feasible_peers << peer if peer.chunk_info.provided?(url, chunkid)
         end
-
+        
+        # debug info
+        # puts "Feasible peers: " + feasible_peers.map { |peer| peer.to_s }.join(", ")
+        
+        giver = optimal_peer(connection, feasible_peers)
+        return begin_transfer(connection, giver, url, chunkid) if giver
+        
+        #FIXME should we try again if begin_transfer fails?
         false
       end
+      
+      # Apply constraints to the list of potential peers to select
+      # the optimal one
+      def optimal_peer(connection, peer_list)
+        # We can't do anything if there's zero or one candidates
+        return peer_list.first if peer_list.size <= 1
+                
+        # Build a list of peers and their respective scores
+        scored_peers = peer_list.map { |peer| [peer, connection.score(peer)] }
+        
+        # Sort the peers by score
+        sorted_peers = scored_peers.sort { |(_, a), (_, b)| b <=> a }
+        
+        # debug info
+        #puts "Sorted peers: " + sorted_peers.map { |peer, score| "#{peer}(#{score})" }.join(", ")
+        
+        # Return the top choice (from a nested array)
+        sorted_peers.first.first
+      end
 
       # Creates a single upload for the specified client
       # Returns true on success, false on failure
       def spawn_upload_for_client(connection)
-        @connections.each do |c2|
-          next if connection == c2
-          next unless c2.wants_download?
+        @connections.each do |peer|
+          next if connection == peer
+          next unless peer.wants_download?
 
           begin
-            url, chunkid = c2.chunk_info.high_priority_chunk
+            url, chunkid = peer.chunk_info.high_priority_chunk
           rescue
             next
           end
 
           if connection.chunk_info.provided?(url, chunkid)
-            return begin_transfer(c2, connection, url, chunkid)
+            return begin_transfer(peer, connection, url, chunkid)
           end
         end
 
         false
       end
-            
+
       # Creates a new transfer between two peers
       # Returns true on success, or false if the specified transfer is already in progress
       def begin_transfer(taker, giver, url, chunkid)
@@ -123,13 +137,13 @@ module PDTP
         t2 = giver.transfers[t.transfer_id]
         return false unless t1.nil? and t2.nil?
 
-        taker.chunk_info.transfer(url, chunkid..chunkid) 
+        taker.chunk_info.set(:transfer, url, chunkid..chunkid) 
         taker.transfers[t.transfer_id] = t
         giver.transfers[t.transfer_id] = t
 
         #send transfer message to the connector
         addr, port = t.acceptor.get_peer_info
-      
+
         t.connector.send_message(:transfer,
           :host => addr,
           :port => t.acceptor.listen_port,
@@ -138,9 +152,9 @@ module PDTP
           :range => byte_range,
           :peer_id => t.acceptor.client_id
         )
-        
+
         true
       end
     end
   end
-end
+end

data/lib/pdtp/server/trust.rb

@@ -81,18 +81,15 @@ module PDTP
         @outgoing.each { |_, link| link.trust = link.success / total_transfers }
         @outgoing.each do |target, link|
           [target.outgoing, target.implicit].each do |links|
-            links.each do |nextlinkedge|
-              nextlinktarget = nextlinkedge[0]
-              nextlink = nextlinkedge[1]
+            links.each do |nextlinktarget, nextlink|
               next unless outgoing[nextlinktarget].nil?
-
-              if implicit[nextlinktarget].nil? || implicit[nextlinktarget].trust < (link.trust * nextlink.trust)
-                implicit[nextlinktarget] = Edge.new(
+              next unless implicit[nextlinktarget].nil? or implicit[nextlinktarget].trust < link.trust * nextlink.trust
+              
+              implicit[nextlinktarget] = Edge.new(
                 link.trust * nextlink.trust, 
                 nextlink.success, 
                 nextlink.transfers
-                )
-              end
+              )
             end
           end
         end

data/status/index.erb

@@ -22,8 +22,8 @@
       <% each_peer do |peer| %>
         <tr class="row<%= cycle(' row_alternate', '') %>">
 					<td>
-	          <%= peer_name(peer) %><br />
-	          <%= peer_address(peer) %>
+						<%= peer_name(peer) %><br />
+						<%= peer_address(peer) unless peer.file_service? %>
 	        </td>
 					<td>
 						Upstream: <%= upstream_bandwidth(peer) %><br />

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.4
+rubygems_version: 0.9.0
 specification_version: 1
 name: distribustream
 version: !ruby/object:Gem::Version 
-  version: 0.5.0
-date: 2008-11-27 00:00:00 -07:00
+  version: 0.5.1
+date: 2008-12-04 00:00:00 -07:00
 summary: DistribuStream is a fully open peercasting system allowing on-demand or live streaming media to be delivered at a fraction of the normal cost
 require_paths: 
 - lib
@@ -33,45 +33,48 @@ authors:
 - James Sanders
 - Tom Stapleton
 files: 
-- bin/dsclient
 - bin/dstream
+- bin/dsclient
 - lib/pdtp
+- lib/pdtp/server
+- lib/pdtp/common.rb
 - lib/pdtp/client
-- lib/pdtp/client/callbacks.rb
-- lib/pdtp/client/connection.rb
+- lib/pdtp/common
+- lib/pdtp/client.rb
+- lib/pdtp/server.rb
+- lib/pdtp/server/file_info.rb
+- lib/pdtp/server/file_service_protocol.rb
+- lib/pdtp/server/chunk_info.rb
+- lib/pdtp/server/transfer_manager.rb
+- lib/pdtp/server/status_handler.rb
+- lib/pdtp/server/trust.rb
+- lib/pdtp/server/status_helper.rb
+- lib/pdtp/server/bandwidth_estimator.rb
+- lib/pdtp/server/file_service_connection.rb
+- lib/pdtp/server/transfer.rb
+- lib/pdtp/server/connection.rb
+- lib/pdtp/server/file_service.rb
+- lib/pdtp/server/dispatcher.rb
 - lib/pdtp/client/file_buffer.rb
-- lib/pdtp/client/file_service.rb
 - lib/pdtp/client/http_client.rb
 - lib/pdtp/client/http_handler.rb
+- lib/pdtp/client/callbacks.rb
 - lib/pdtp/client/transfer.rb
-- lib/pdtp/client.rb
-- lib/pdtp/common
-- lib/pdtp/common/file_service.rb
-- lib/pdtp/common/http_server.rb
+- lib/pdtp/client/connection.rb
+- lib/pdtp/client/file_service.rb
 - lib/pdtp/common/length_prefix_protocol.rb
+- lib/pdtp/common/http_server.rb
 - lib/pdtp/common/protocol.rb
-- lib/pdtp/common.rb
-- lib/pdtp/server
-- lib/pdtp/server/bandwidth_estimator.rb
-- lib/pdtp/server/chunk_info.rb
-- lib/pdtp/server/connection.rb
-- lib/pdtp/server/dispatcher.rb
-- lib/pdtp/server/file_service.rb
-- lib/pdtp/server/file_service_protocol.rb
-- lib/pdtp/server/status_handler.rb
-- lib/pdtp/server/status_helper.rb
-- lib/pdtp/server/transfer.rb
-- lib/pdtp/server/transfer_manager.rb
-- lib/pdtp/server/trust.rb
-- lib/pdtp/server.rb
+- lib/pdtp/common/range_map.rb
+- lib/pdtp/common/file_service.rb
+- conf/example.yml
 - conf/bigchunk.yml
 - conf/debug.yml
-- conf/example.yml
+- status/stylesheets
 - status/images
-- status/images/logo.png
 - status/index.erb
-- status/stylesheets
 - status/stylesheets/style.css
+- status/images/logo.png
 - Rakefile
 - distribustream.gemspec
 - COPYING
@@ -126,3 +129,12 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.1.0
     version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.0
+    version: