distribustream 0.4.1 → 0.5.0

data/CHANGES

@@ -1,3 +1,16 @@
+Version 0.5.0
+
+* Factor traffic routing code into PDTP::Server::TransferManager
+
+* Cleanup transfer reporting and factor into PDTP::Server::Transfer
+
+* Add bandwidth estimator class and integrate into transfer reporting
+
+* Add bandwidth estimates to the status page with associated helpers
+
+* Eliminate the ClientInfo class and factor all ClientInfo-related code
+  into the PDTP::Server::Connection class
+
 Version 0.4.1
 * Add -v / --version flags to dsclient and dstream
 

data/README

@@ -1,11 +1,9 @@
-Welcome to DistribuStream!
+Welcome to DistribuStream! (http://distribustream.org)
 
 DistribuStream is a fully open peercasting system which allows on-demand
 or live streaming media to be delivered at a fraction of the normal cost. 
 
---
-
-Usage:
+USAGE:
 
 The DistribuStream gem includes three config files that can be located in
 the conf directory of the gem:
@@ -47,9 +45,7 @@ media as it downloads, you can:
 
 dsclient -o pdtp://myserver.url/file.ext | mediaplayer -
 
---
-
-Development Roadmap:
+ROADMAP:
 
 Short-term goals focus on improving the efficiency of peer-to-peer traffic
 routing, by incorporating all of the following constraints:

data/Rakefile

@@ -10,10 +10,10 @@ task :default => :rdoc
 Rake::RDocTask.new(:rdoc) do |task|
   task.rdoc_dir = 'doc'
   task.title    = 'DistribuStream'
+  task.options = %w(--title PDTP --main README --line-numbers)
   task.rdoc_files.include('bin/**/*.rb')
   task.rdoc_files.include('lib/**/*.rb')
-  task.rdoc_files.include('simulation/**/*.rb')
-  task.rdoc_files.include('test/**/*.rb')
+  task.rdoc_files.include('README')
 end
 
 # Gem

data/distribustream.gemspec

@@ -2,19 +2,28 @@ require 'rubygems'
 
 GEMSPEC = Gem::Specification.new do |s|
   s.name = "distribustream"
-  s.version = "0.4.1"
-  s.date = "2008-11-15"
-  s.summary = "DistribuStream is a fully open peercasting system allowing on-demand or live streaming media to be delivered at a fraction of the normal cost"
-  s.email = "tony@clickcaster.com"
-  s.homepage = "http://distribustream.org"
-  s.rubyforge_project = "distribustream"
-  s.has_rdoc = true
-  s.rdoc_options = ["--exclude", "definitions", "--exclude", "indexes"]
-  s.extra_rdoc_files = ["COPYING", "README", "CHANGES", "pdtp-specification.xml"]
+  s.version = "0.5.0"
   s.authors = ["Tony Arcieri", "Ashvin Mysore", "Galen Pahlke", "James Sanders", "Tom Stapleton"]
+  s.email = "tony@clickcaster.com"
+  s.date = "2008-11-27"
+  s.summary = "DistribuStream is a fully open peercasting system allowing on-demand or live streaming media to be delivered at a fraction of the normal cost"
+  s.platform = Gem::Platform::RUBY
+
+  # Gem contents
   s.files = Dir.glob("{bin,lib,conf,status}/**/*") + ['Rakefile', 'distribustream.gemspec']
   s.executables = %w{dstream dsclient}
+
+  # Dependencies
   s.add_dependency("eventmachine", ">= 0.9.0")
   s.add_dependency("mongrel", ">= 1.0.2")
   s.add_dependency("json", ">= 1.1.0")
+
+  # RubyForge info
+  s.homepage = "http://distribustream.org"
+  s.rubyforge_project = "distribustream"
+
+  # RDoc settings
+  s.has_rdoc = true
+  s.rdoc_options = %w(--title PDTP --main README --line-numbers)
+  s.extra_rdoc_files = ["COPYING", "README", "CHANGES", "pdtp-specification.xml"]
 end

data/lib/pdtp/client/callbacks.rb

@@ -26,12 +26,15 @@ module PDTP
     class Callbacks
       attr_accessor :client
       
-      def initialize(*args)
+      def initialize(*args) # :nodoc:
       end
       
+      # Callback fired when a client has successfully connected to a server
       def connected(client)
       end
       
+      # Callback fired when a client has been disconnected from a server
+      # Also fired if the connection attempt to a server fails
       def disconnected(client)
         client.stop
       end

data/lib/pdtp/common.rb

@@ -22,7 +22,7 @@
 
 # Namespace for all PDTP components
 module PDTP
-  PDTP::VERSION = '0.4.1' unless defined? PDTP::VERSION
+  PDTP::VERSION = '0.5.0' unless defined? PDTP::VERSION
   def self.version() VERSION end
 
   PDTP::DEFAULT_PORT = 6086 unless defined? PDTP::DEFAULT_PORT

data/lib/pdtp/common/http_server.rb

@@ -1,8 +1,13 @@
+#--
+# Copyright (C)2007 Kirk Haines
+# Licensed under the Ruby License. See http://www.ruby-lang.org/en/LICENSE.txt
+#
 # This module rewrites pieces of the very good Mongrel web server in
 # order to change it from a threaded application to an event based
 # application running inside an EventMachine event loop.  It should
 # be compatible with the existing Mongrel handlers for Rails,
 # Camping, Nitro, etc....
+#++
 
 require 'rubygems'
 require 'eventmachine'

data/lib/pdtp/common/protocol.rb

@@ -51,11 +51,6 @@ module PDTP
       @connection_open
     end
 
-    def initialize(*args)
-      user_data = nil
-      super
-    end
-
     #called by EventMachine after a connection has been established
     def post_init
       # a cache of the peer info because eventmachine seems to drop it before we want
@@ -72,8 +67,6 @@ module PDTP
       connection_created if respond_to? :connection_created
     end
 
-    attr_accessor :user_data #users of this class may store arbitrary data here
-
     #close a connection, but first send the specified error message
     def error_close_connection(error) 
       if PROTOCOL_DEBUG
@@ -86,7 +79,7 @@ module PDTP
 
     #debug routine: returns id of remote peer on this connection
     def remote_peer_id
-      ret = user_data.client_id rescue nil
+      ret = client_info.client_id rescue nil
       ret || 'NOID'
     end
 

data/lib/pdtp/server/bandwidth_estimator.rb

@@ -0,0 +1,148 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.org/
+#++
+
+module PDTP
+  class Server
+    # BandwidthEstimator logs the durations of chunk transfers over time,
+    # allowing the server to estimate the bandwidth of individual peers
+    class BandwidthEstimator
+      # Maximum number of transfers to store in a given history object
+      MAXSIZE = 100 unless defined? MAXSIZE
+      
+      # Inner class for storing a transfer
+      class Transfer
+        attr_reader :start_time, :end_time, :rate
+        
+        def initialize(start_time, end_time, bytes_transferred)
+          # Avoid division by zero or negative transfer rates
+          raise ArgumentError, "end_time must exceed start_time" unless end_time > start_time
+          
+          @start_time, @end_time = start_time, end_time
+          
+          # Calculate transfer rate in bytes per second
+          @rate = (bytes_transferred / (end_time - start_time)).to_i
+        end
+      end
+      
+      def initialize
+        @transfers = []
+        
+        # Clear caches of all calculated values
+        clear_caches
+      end
+      
+      # Log when a transfer began, ended, and how much data was transferred 
+      def log(start_time, end_time, bytes_transferred)
+        # Remove an old transfer if we've exceeded MAXSIZE
+        @transfers.shift if @transfers.size >= MAXSIZE
+        
+        # Clear caches of all calculated values because the sample has changed
+        clear_caches
+        
+        # Log the transfer
+        @transfers << Transfer.new(start_time, end_time, bytes_transferred)
+        
+        nil
+      end
+      
+      # Compute an estimate of the bandwidth, culling statistical outliers
+      def estimate
+        return @estimate unless @estimate.nil?
+        
+        # Prune outliers beyond 2 standard deviations of the mean
+        # FIXME maybe this should be 3 standard deviations?
+        pruned = estimates.select { |n| (n - estimates_mean).abs < standard_deviation * 2}
+        
+        # Avoid dividing by zero
+        return 0 if pruned.size.zero?
+        
+        # Calculate the mean of the pruned estimates
+        @estimate = pruned.inject(0) { |a, v| a + v } / pruned.size
+      end
+      
+      #########
+      protected
+      #########
+      
+      # Retrieve an array of bandwidth estimates over time
+      # FIXME Some O(n^2) nastiness... could definitely be improved
+      def estimates
+        # Generate a list of transfers which overlap at transfer endpoints
+        overlapping_transfers = @transfers.map do |t1|
+          @transfers.inject([]) do |array, t2| 
+            array << t2.rate if (t2.start_time..t2.end_time).include? t1.end_time
+            array
+          end
+        end
+        
+        # Sum the overlapping transfer rates and store them as estimates
+        @estimates ||= overlapping_transfers.map { |t| t.inject(0) { |a,v| a + v } }
+      end
+      
+      # Compute the mean of the bandwidth estimates
+      def estimates_mean
+        @mean ||= estimates.inject(0) { |a, v| a + v } / estimates.size
+      end
+      
+      # Compute the variance of the bandwidth estimates
+      def variance
+        # Return cached value if available
+        return @variance unless @variance.nil?
+        
+        mean = 0.0
+        s = 0.0
+        
+        estimates.each_with_index do |rate, n|
+          delta = rate - mean
+          mean += delta / (n + 1)
+          s += delta * (rate - mean)
+        end
+        
+        @mean = mean
+        @variance = s / estimates.size
+      end
+      
+      # Compute the standard deviation of the bandwidth estimates
+      def standard_deviation
+        @std ||= Math.sqrt variance
+      end
+      
+      # Clear all cached values for transfer statistics
+      def clear_caches
+        # Array of bandwidth estimates sampled at transfer completions
+        @estimates = nil
+        
+        # Mean of the bandwidth estimates
+        @mean = nil
+        
+        # Variance of @estimates
+        @variance = nil
+        
+        # Standard deviation of @estimates
+        @std = nil
+        
+        # Final estimate (arithmetic mean) after outliers have been culled from @estimates
+        @estimate = nil
+      end
+    end
+  end
+end

data/lib/pdtp/server/chunk_info.rb

@@ -0,0 +1,97 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.org/
+#++
+
+module PDTP
+  class Server
+    # Stores information about the chunks requested or provided by a client
+    class ChunkInfo
+      def initialize
+        @files = {}
+      end
+
+      #each chunk can either be provided, requested, transfer, or none
+      #FIXME some metaprogramming is probably in order here
+      def provide(filename,range); set(filename, range, :provided); end
+      def unprovide(filename,range); set(filename,range, :none); end
+      def request(filename,range); set(filename,range, :requested); end
+      def unrequest(filename,range); set(filename,range, :none); end
+      def transfer(filename,range); set(filename,range, :transfer); end
+
+      def provided?(filename,chunk); get(filename,chunk)  == :provided; end
+      def requested?(filename,chunk); get(filename,chunk) == :requested; end
+
+      #returns a high priority requested chunk
+      def high_priority_chunk
+        #right now return any chunk
+        @files.each do |name,file|
+          file.each_index do |i|
+            return [name, i] if file[i] == :requested
+          end
+        end  
+
+        nil
+      end
+
+      class FileStats
+        attr_accessor :file_chunks, :chunks_requested, :url
+        attr_accessor :chunks_provided, :chunks_transferring
+
+        def initialize
+          @url = ""
+          @file_chunks = 0
+          @chunks_requested = 0
+          @chunks_provided = 0
+          @chunks_transferring = 0
+        end
+      end
+
+      # Returns an array of FileStats objects for debug output
+      def get_file_stats
+        @files.map do |name, file|
+          fs = FileStats.new
+          fs.file_chunks = file.size
+          fs.url = name
+          file.each do |chunk|
+            fs.chunks_requested += 1 if chunk == :requested
+            fs.chunks_provided += 1 if chunk == :provided
+            fs.chunks_transferring += 1 if chunk == :transfer
+          end
+          
+          fs
+        end
+      end
+
+      #########
+      protected
+      #########
+
+      def get(filename,chunk)
+        @files[filename][chunk] rescue :neither
+      end
+
+      def set(filename,range,state)
+        chunks=@files[filename]||=Array.new
+        range.each { |i| chunks[i]=state }
+      end
+    end
+  end
+end

data/lib/pdtp/server/connection.rb

@@ -21,13 +21,108 @@
 #++
 
 require File.dirname(__FILE__) + '/../common/protocol'
+require File.dirname(__FILE__) + '/chunk_info'
+require File.dirname(__FILE__) + '/trust'
+require File.dirname(__FILE__) + '/bandwidth_estimator'
 
 module PDTP
   class Server
+    # Server's internal representation of a client connection
     class Connection < PDTP::Protocol
-      attr_accessor :dispatcher
-      attr_accessor :user_data
+      # Handle to the dispatcher which can hopefully be eliminated in future versions
+      # of EventMachine
+      attr_writer :dispatcher
       
+      # Accessors which can hopefully be eliminated in future versions
+      attr_accessor :chunk_info, :trust
+      attr_accessor :listen_port, :client_id
+      attr_accessor :transfers
+      
+      def initialize(*args)
+        # Information about what chunks the client is requesting/providing
+        @chunk_info = ChunkInfo.new
+        
+        # Chunk transfers the client is actively participating in
+        @transfers = Hash.new
+        
+        # Port the client is listening on        
+        @listen_port = 6000 #default
+        
+        # Trust relatipnships with other peers
+        @trust = Trust.new
+        
+        # Bandwidth estimators
+        @upstream = BandwidthEstimator.new
+        @downstream = BandwidthEstimator.new
+        
+        super
+      end
+
+      # Log a completed transfer and update internal data
+      def success(transfer)
+        if transfer.taker == self
+          @chunk_info.provide transfer.url, transfer.chunkid..transfer.chunkid
+          @trust.success transfer.giver.trust
+          bandwidth_estimator = @downstream
+        else
+          bandwidth_estimator = @upstream
+        end
+        
+        bandwidth_estimator.log(
+          transfer.creation_time, 
+          Time.now, 
+          transfer.byte_range.end - transfer.byte_range.begin
+        )
+      end
+      
+      # Log a failed transfer and update internal data
+      def failure(transfer)
+        raise ArgumentError, "not taker for this transfer" unless transfer.taker == self
+        @chunk_info.request transfer.url, transfer.chunkid..transfer.chunkid
+        @trust.failure transfer.giver.trust
+      end
+      
+      # Estimate of a client's upstream bandwidth
+      def upstream_bandwidth
+        @upstream.estimate rescue nil
+      end
+      
+      # Estimate of a client's downstream bandwidth
+      def downstream_bandwidth
+        @downstream.estimate rescue nil
+      end
+
+      # Returns true if this client wants the server to spawn a transfer for it
+      def wants_download?
+        transfer_state_allowed = 5
+        total_allowed = 10
+        transferring = 0
+        @transfers.each do |key, t|
+          transferring += 1 if t.verification_asked
+          return false if transferring >= transfer_state_allowed
+        end
+
+        @transfers.size < total_allowed
+      end 
+
+      # This could have a different definition, but it works fine to use wants_download?
+      alias_method :wants_upload?, :wants_download?
+
+      # Returns a list of all the stalled transfers this client is a part of
+      def stalled_transfers
+        stalled = []
+        timeout = 20.0
+        now = Time.now
+        @transfers.each do |key,t|
+          #only delete if we are the acceptor to prevent race conditions
+          next unless t.acceptor == self 
+          if now - t.creation_time > timeout and not t.verification_asked
+            stalled << t
+          end
+        end
+        stalled
+      end
+                  
       # Is this connection the file service?
       def file_service?
         @file_service
@@ -37,7 +132,11 @@ module PDTP
       def mark_as_file_service
         @file_service = true
       end
-      
+            
+      #
+      # EventMachine callbacks to delegate to the dispatcher
+      #
+            
       def connection_completed
         raise(RuntimeError, 'server was never initialized') unless @dispatcher
         @dispatcher.connection_created self