distribustream 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/CHANGES CHANGED
@@ -1,3 +1,16 @@
1
+ Version 0.5.0
2
+
3
+ * Factor traffic routing code into PDTP::Server::TransferManager
4
+
5
+ * Cleanup transfer reporting and factor into PDTP::Server::Transfer
6
+
7
+ * Add bandwidth estimator class and integrate into transfer reporting
8
+
9
+ * Add bandwidth estimates to the status page with associated helpers
10
+
11
+ * Eliminate the ClientInfo class and factor all ClientInfo-related code
12
+ into the PDTP::Server::Connection class
13
+
1
14
  Version 0.4.1
2
15
  * Add -v / --version flags to dsclient and dstream
3
16
 
data/README CHANGED
@@ -1,11 +1,9 @@
1
- Welcome to DistribuStream!
1
+ Welcome to DistribuStream! (http://distribustream.org)
2
2
 
3
3
  DistribuStream is a fully open peercasting system which allows on-demand
4
4
  or live streaming media to be delivered at a fraction of the normal cost.
5
5
 
6
- --
7
-
8
- Usage:
6
+ USAGE:
9
7
 
10
8
  The DistribuStream gem includes three config files that can be located in
11
9
  the conf directory of the gem:
@@ -47,9 +45,7 @@ media as it downloads, you can:
47
45
 
48
46
  dsclient -o pdtp://myserver.url/file.ext | mediaplayer -
49
47
 
50
- --
51
-
52
- Development Roadmap:
48
+ ROADMAP:
53
49
 
54
50
  Short-term goals focus on improving the efficiency of peer-to-peer traffic
55
51
  routing, by incorporating all of the following constraints:
data/Rakefile CHANGED
@@ -10,10 +10,10 @@ task :default => :rdoc
10
10
  Rake::RDocTask.new(:rdoc) do |task|
11
11
  task.rdoc_dir = 'doc'
12
12
  task.title = 'DistribuStream'
13
+ task.options = %w(--title PDTP --main README --line-numbers)
13
14
  task.rdoc_files.include('bin/**/*.rb')
14
15
  task.rdoc_files.include('lib/**/*.rb')
15
- task.rdoc_files.include('simulation/**/*.rb')
16
- task.rdoc_files.include('test/**/*.rb')
16
+ task.rdoc_files.include('README')
17
17
  end
18
18
 
19
19
  # Gem
@@ -2,19 +2,28 @@ require 'rubygems'
2
2
 
3
3
  GEMSPEC = Gem::Specification.new do |s|
4
4
  s.name = "distribustream"
5
- s.version = "0.4.1"
6
- s.date = "2008-11-15"
7
- s.summary = "DistribuStream is a fully open peercasting system allowing on-demand or live streaming media to be delivered at a fraction of the normal cost"
8
- s.email = "tony@clickcaster.com"
9
- s.homepage = "http://distribustream.org"
10
- s.rubyforge_project = "distribustream"
11
- s.has_rdoc = true
12
- s.rdoc_options = ["--exclude", "definitions", "--exclude", "indexes"]
13
- s.extra_rdoc_files = ["COPYING", "README", "CHANGES", "pdtp-specification.xml"]
5
+ s.version = "0.5.0"
14
6
  s.authors = ["Tony Arcieri", "Ashvin Mysore", "Galen Pahlke", "James Sanders", "Tom Stapleton"]
7
+ s.email = "tony@clickcaster.com"
8
+ s.date = "2008-11-27"
9
+ s.summary = "DistribuStream is a fully open peercasting system allowing on-demand or live streaming media to be delivered at a fraction of the normal cost"
10
+ s.platform = Gem::Platform::RUBY
11
+
12
+ # Gem contents
15
13
  s.files = Dir.glob("{bin,lib,conf,status}/**/*") + ['Rakefile', 'distribustream.gemspec']
16
14
  s.executables = %w{dstream dsclient}
15
+
16
+ # Dependencies
17
17
  s.add_dependency("eventmachine", ">= 0.9.0")
18
18
  s.add_dependency("mongrel", ">= 1.0.2")
19
19
  s.add_dependency("json", ">= 1.1.0")
20
+
21
+ # RubyForge info
22
+ s.homepage = "http://distribustream.org"
23
+ s.rubyforge_project = "distribustream"
24
+
25
+ # RDoc settings
26
+ s.has_rdoc = true
27
+ s.rdoc_options = %w(--title PDTP --main README --line-numbers)
28
+ s.extra_rdoc_files = ["COPYING", "README", "CHANGES", "pdtp-specification.xml"]
20
29
  end
@@ -26,12 +26,15 @@ module PDTP
26
26
  class Callbacks
27
27
  attr_accessor :client
28
28
 
29
- def initialize(*args)
29
+ def initialize(*args) # :nodoc:
30
30
  end
31
31
 
32
+ # Callback fired when a client has successfully connected to a server
32
33
  def connected(client)
33
34
  end
34
35
 
36
+ # Callback fired when a client has been disconnected from a server
37
+ # Also fired if the connection attempt to a server fails
35
38
  def disconnected(client)
36
39
  client.stop
37
40
  end
data/lib/pdtp/common.rb CHANGED
@@ -22,7 +22,7 @@
22
22
 
23
23
  # Namespace for all PDTP components
24
24
  module PDTP
25
- PDTP::VERSION = '0.4.1' unless defined? PDTP::VERSION
25
+ PDTP::VERSION = '0.5.0' unless defined? PDTP::VERSION
26
26
  def self.version() VERSION end
27
27
 
28
28
  PDTP::DEFAULT_PORT = 6086 unless defined? PDTP::DEFAULT_PORT
@@ -1,8 +1,13 @@
1
+ #--
2
+ # Copyright (C)2007 Kirk Haines
3
+ # Licensed under the Ruby License. See http://www.ruby-lang.org/en/LICENSE.txt
4
+ #
1
5
  # This module rewrites pieces of the very good Mongrel web server in
2
6
  # order to change it from a threaded application to an event based
3
7
  # application running inside an EventMachine event loop. It should
4
8
  # be compatible with the existing Mongrel handlers for Rails,
5
9
  # Camping, Nitro, etc....
10
+ #++
6
11
 
7
12
  require 'rubygems'
8
13
  require 'eventmachine'
@@ -51,11 +51,6 @@ module PDTP
51
51
  @connection_open
52
52
  end
53
53
 
54
- def initialize(*args)
55
- user_data = nil
56
- super
57
- end
58
-
59
54
  #called by EventMachine after a connection has been established
60
55
  def post_init
61
56
  # a cache of the peer info because eventmachine seems to drop it before we want
@@ -72,8 +67,6 @@ module PDTP
72
67
  connection_created if respond_to? :connection_created
73
68
  end
74
69
 
75
- attr_accessor :user_data #users of this class may store arbitrary data here
76
-
77
70
  #close a connection, but first send the specified error message
78
71
  def error_close_connection(error)
79
72
  if PROTOCOL_DEBUG
@@ -86,7 +79,7 @@ module PDTP
86
79
 
87
80
  #debug routine: returns id of remote peer on this connection
88
81
  def remote_peer_id
89
- ret = user_data.client_id rescue nil
82
+ ret = client_info.client_id rescue nil
90
83
  ret || 'NOID'
91
84
  end
92
85
 
@@ -0,0 +1,148 @@
1
+ #--
2
+ # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
+ #
17
+ # This source file is distributed as part of the
18
+ # DistribuStream file transfer system.
19
+ #
20
+ # See http://distribustream.org/
21
+ #++
22
+
23
+ module PDTP
24
+ class Server
25
+ # BandwidthEstimator logs the durations of chunk transfers over time,
26
+ # allowing the server to estimate the bandwidth of individual peers
27
+ class BandwidthEstimator
28
+ # Maximum number of transfers to store in a given history object
29
+ MAXSIZE = 100 unless defined? MAXSIZE
30
+
31
+ # Inner class for storing a transfer
32
+ class Transfer
33
+ attr_reader :start_time, :end_time, :rate
34
+
35
+ def initialize(start_time, end_time, bytes_transferred)
36
+ # Avoid division by zero or negative transfer rates
37
+ raise ArgumentError, "end_time must exceed start_time" unless end_time > start_time
38
+
39
+ @start_time, @end_time = start_time, end_time
40
+
41
+ # Calculate transfer rate in bytes per second
42
+ @rate = (bytes_transferred / (end_time - start_time)).to_i
43
+ end
44
+ end
45
+
46
+ def initialize
47
+ @transfers = []
48
+
49
+ # Clear caches of all calculated values
50
+ clear_caches
51
+ end
52
+
53
+ # Log when a transfer began, ended, and how much data was transferred
54
+ def log(start_time, end_time, bytes_transferred)
55
+ # Remove an old transfer if we've exceeded MAXSIZE
56
+ @transfers.shift if @transfers.size >= MAXSIZE
57
+
58
+ # Clear caches of all calculated values because the sample has changed
59
+ clear_caches
60
+
61
+ # Log the transfer
62
+ @transfers << Transfer.new(start_time, end_time, bytes_transferred)
63
+
64
+ nil
65
+ end
66
+
67
+ # Compute an estimate of the bandwidth, culling statistical outliers
68
+ def estimate
69
+ return @estimate unless @estimate.nil?
70
+
71
+ # Prune outliers beyond 2 standard deviations of the mean
72
+ # FIXME maybe this should be 3 standard deviations?
73
+ pruned = estimates.select { |n| (n - estimates_mean).abs < standard_deviation * 2}
74
+
75
+ # Avoid dividing by zero
76
+ return 0 if pruned.size.zero?
77
+
78
+ # Calculate the mean of the pruned estimates
79
+ @estimate = pruned.inject(0) { |a, v| a + v } / pruned.size
80
+ end
81
+
82
+ #########
83
+ protected
84
+ #########
85
+
86
+ # Retrieve an array of bandwidth estimates over time
87
+ # FIXME Some O(n^2) nastiness... could definitely be improved
88
+ def estimates
89
+ # Generate a list of transfers which overlap at transfer endpoints
90
+ overlapping_transfers = @transfers.map do |t1|
91
+ @transfers.inject([]) do |array, t2|
92
+ array << t2.rate if (t2.start_time..t2.end_time).include? t1.end_time
93
+ array
94
+ end
95
+ end
96
+
97
+ # Sum the overlapping transfer rates and store them as estimates
98
+ @estimates ||= overlapping_transfers.map { |t| t.inject(0) { |a,v| a + v } }
99
+ end
100
+
101
+ # Compute the mean of the bandwidth estimates
102
+ def estimates_mean
103
+ @mean ||= estimates.inject(0) { |a, v| a + v } / estimates.size
104
+ end
105
+
106
+ # Compute the variance of the bandwidth estimates
107
+ def variance
108
+ # Return cached value if available
109
+ return @variance unless @variance.nil?
110
+
111
+ mean = 0.0
112
+ s = 0.0
113
+
114
+ estimates.each_with_index do |rate, n|
115
+ delta = rate - mean
116
+ mean += delta / (n + 1)
117
+ s += delta * (rate - mean)
118
+ end
119
+
120
+ @mean = mean
121
+ @variance = s / estimates.size
122
+ end
123
+
124
+ # Compute the standard deviation of the bandwidth estimates
125
+ def standard_deviation
126
+ @std ||= Math.sqrt variance
127
+ end
128
+
129
+ # Clear all cached values for transfer statistics
130
+ def clear_caches
131
+ # Array of bandwidth estimates sampled at transfer completions
132
+ @estimates = nil
133
+
134
+ # Mean of the bandwidth estimates
135
+ @mean = nil
136
+
137
+ # Variance of @estimates
138
+ @variance = nil
139
+
140
+ # Standard deviation of @estimates
141
+ @std = nil
142
+
143
+ # Final estimate (arithmetic mean) after outliers have been culled from @estimates
144
+ @estimate = nil
145
+ end
146
+ end
147
+ end
148
+ end
@@ -0,0 +1,97 @@
1
+ #--
2
+ # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
16
+ #
17
+ # This source file is distributed as part of the
18
+ # DistribuStream file transfer system.
19
+ #
20
+ # See http://distribustream.org/
21
+ #++
22
+
23
+ module PDTP
24
+ class Server
25
+ # Stores information about the chunks requested or provided by a client
26
+ class ChunkInfo
27
+ def initialize
28
+ @files = {}
29
+ end
30
+
31
+ #each chunk can either be provided, requested, transfer, or none
32
+ #FIXME some metaprogramming is probably in order here
33
+ def provide(filename,range); set(filename, range, :provided); end
34
+ def unprovide(filename,range); set(filename,range, :none); end
35
+ def request(filename,range); set(filename,range, :requested); end
36
+ def unrequest(filename,range); set(filename,range, :none); end
37
+ def transfer(filename,range); set(filename,range, :transfer); end
38
+
39
+ def provided?(filename,chunk); get(filename,chunk) == :provided; end
40
+ def requested?(filename,chunk); get(filename,chunk) == :requested; end
41
+
42
+ #returns a high priority requested chunk
43
+ def high_priority_chunk
44
+ #right now return any chunk
45
+ @files.each do |name,file|
46
+ file.each_index do |i|
47
+ return [name, i] if file[i] == :requested
48
+ end
49
+ end
50
+
51
+ nil
52
+ end
53
+
54
+ class FileStats
55
+ attr_accessor :file_chunks, :chunks_requested, :url
56
+ attr_accessor :chunks_provided, :chunks_transferring
57
+
58
+ def initialize
59
+ @url = ""
60
+ @file_chunks = 0
61
+ @chunks_requested = 0
62
+ @chunks_provided = 0
63
+ @chunks_transferring = 0
64
+ end
65
+ end
66
+
67
+ # Returns an array of FileStats objects for debug output
68
+ def get_file_stats
69
+ @files.map do |name, file|
70
+ fs = FileStats.new
71
+ fs.file_chunks = file.size
72
+ fs.url = name
73
+ file.each do |chunk|
74
+ fs.chunks_requested += 1 if chunk == :requested
75
+ fs.chunks_provided += 1 if chunk == :provided
76
+ fs.chunks_transferring += 1 if chunk == :transfer
77
+ end
78
+
79
+ fs
80
+ end
81
+ end
82
+
83
+ #########
84
+ protected
85
+ #########
86
+
87
+ def get(filename,chunk)
88
+ @files[filename][chunk] rescue :neither
89
+ end
90
+
91
+ def set(filename,range,state)
92
+ chunks=@files[filename]||=Array.new
93
+ range.each { |i| chunks[i]=state }
94
+ end
95
+ end
96
+ end
97
+ end
@@ -21,13 +21,108 @@
21
21
  #++
22
22
 
23
23
  require File.dirname(__FILE__) + '/../common/protocol'
24
+ require File.dirname(__FILE__) + '/chunk_info'
25
+ require File.dirname(__FILE__) + '/trust'
26
+ require File.dirname(__FILE__) + '/bandwidth_estimator'
24
27
 
25
28
  module PDTP
26
29
  class Server
30
+ # Server's internal representation of a client connection
27
31
  class Connection < PDTP::Protocol
28
- attr_accessor :dispatcher
29
- attr_accessor :user_data
32
+ # Handle to the dispatcher which can hopefully be eliminated in future versions
33
+ # of EventMachine
34
+ attr_writer :dispatcher
30
35
 
36
+ # Accessors which can hopefully be eliminated in future versions
37
+ attr_accessor :chunk_info, :trust
38
+ attr_accessor :listen_port, :client_id
39
+ attr_accessor :transfers
40
+
41
+ def initialize(*args)
42
+ # Information about what chunks the client is requesting/providing
43
+ @chunk_info = ChunkInfo.new
44
+
45
+ # Chunk transfers the client is actively participating in
46
+ @transfers = Hash.new
47
+
48
+ # Port the client is listening on
49
+ @listen_port = 6000 #default
50
+
51
+ # Trust relatipnships with other peers
52
+ @trust = Trust.new
53
+
54
+ # Bandwidth estimators
55
+ @upstream = BandwidthEstimator.new
56
+ @downstream = BandwidthEstimator.new
57
+
58
+ super
59
+ end
60
+
61
+ # Log a completed transfer and update internal data
62
+ def success(transfer)
63
+ if transfer.taker == self
64
+ @chunk_info.provide transfer.url, transfer.chunkid..transfer.chunkid
65
+ @trust.success transfer.giver.trust
66
+ bandwidth_estimator = @downstream
67
+ else
68
+ bandwidth_estimator = @upstream
69
+ end
70
+
71
+ bandwidth_estimator.log(
72
+ transfer.creation_time,
73
+ Time.now,
74
+ transfer.byte_range.end - transfer.byte_range.begin
75
+ )
76
+ end
77
+
78
+ # Log a failed transfer and update internal data
79
+ def failure(transfer)
80
+ raise ArgumentError, "not taker for this transfer" unless transfer.taker == self
81
+ @chunk_info.request transfer.url, transfer.chunkid..transfer.chunkid
82
+ @trust.failure transfer.giver.trust
83
+ end
84
+
85
+ # Estimate of a client's upstream bandwidth
86
+ def upstream_bandwidth
87
+ @upstream.estimate rescue nil
88
+ end
89
+
90
+ # Estimate of a client's downstream bandwidth
91
+ def downstream_bandwidth
92
+ @downstream.estimate rescue nil
93
+ end
94
+
95
+ # Returns true if this client wants the server to spawn a transfer for it
96
+ def wants_download?
97
+ transfer_state_allowed = 5
98
+ total_allowed = 10
99
+ transferring = 0
100
+ @transfers.each do |key, t|
101
+ transferring += 1 if t.verification_asked
102
+ return false if transferring >= transfer_state_allowed
103
+ end
104
+
105
+ @transfers.size < total_allowed
106
+ end
107
+
108
+ # This could have a different definition, but it works fine to use wants_download?
109
+ alias_method :wants_upload?, :wants_download?
110
+
111
+ # Returns a list of all the stalled transfers this client is a part of
112
+ def stalled_transfers
113
+ stalled = []
114
+ timeout = 20.0
115
+ now = Time.now
116
+ @transfers.each do |key,t|
117
+ #only delete if we are the acceptor to prevent race conditions
118
+ next unless t.acceptor == self
119
+ if now - t.creation_time > timeout and not t.verification_asked
120
+ stalled << t
121
+ end
122
+ end
123
+ stalled
124
+ end
125
+
31
126
  # Is this connection the file service?
32
127
  def file_service?
33
128
  @file_service
@@ -37,7 +132,11 @@ module PDTP
37
132
  def mark_as_file_service
38
133
  @file_service = true
39
134
  end
40
-
135
+
136
+ #
137
+ # EventMachine callbacks to delegate to the dispatcher
138
+ #
139
+
41
140
  def connection_completed
42
141
  raise(RuntimeError, 'server was never initialized') unless @dispatcher
43
142
  @dispatcher.connection_created self