distribustream 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,10 +20,7 @@
20
20
  # See http://distribustream.org/
21
21
  #++
22
22
 
23
- require File.dirname(__FILE__) + '/../common/protocol'
24
- require File.dirname(__FILE__) + '/file_service'
25
- require File.dirname(__FILE__) + '/client_info'
26
- require File.dirname(__FILE__) + '/transfer'
23
+ require File.dirname(__FILE__) + '/transfer_manager'
27
24
 
28
25
  module PDTP
29
26
  class Server
@@ -36,10 +33,10 @@ module PDTP
36
33
  @file_service = file_service
37
34
  @connections = []
38
35
  @used_client_ids = {} #keeps a list of client ids in use, they must be unique
39
- @updated_clients = {} #a set of clients that have been modified and need transfers spawned
36
+ @transfer_manager = TransferManager.new @connections, @file_service
40
37
  end
41
38
 
42
- #called by pdtp_protocol when a connection is created
39
+ # Register a PDTP::Server::Connection with the Dispatcher
43
40
  def connection_created(connection)
44
41
  addr, port = connection.get_peer_info
45
42
 
@@ -53,203 +50,31 @@ module PDTP
53
50
  @server.log "client connected: #{connection.get_peer_info.inspect}"
54
51
  end
55
52
 
56
- connection.user_data = ClientInfo.new
57
53
  @connections << connection
58
54
  end
59
55
 
60
- #called by pdtp_protocol when a connection is destroyed
56
+ # Unregister a PDTP::Server::Connection from the Dispatcher
61
57
  def connection_destroyed(connection)
62
58
  @server.log "client disconnected: #{connection.get_peer_info.inspect}"
63
59
  @connections.delete connection
64
60
  end
65
-
66
- # returns the ClientInfo object associated with this connection
67
- def client_info(connection)
68
- connection.user_data ||= ClientInfo.new
69
- end
70
-
71
- # called when a transfer either finishes, successfully or not
72
- def transfer_completed(transfer,connection,chunk_hash,send_response=true)
73
- # did the transfer complete successfully?
74
- local_hash=@file_service.get_chunk_hash(transfer.url,transfer.chunkid)
75
-
76
- c1=client_info(transfer.taker)
77
- c2=client_info(transfer.giver)
78
-
79
- if connection==transfer.taker
80
- success= (chunk_hash==local_hash)
81
-
82
- if success
83
- #the taker now has the file, so he can provide it
84
- client_info(transfer.taker).chunk_info.provide(transfer.url,transfer.chunkid..transfer.chunkid)
85
- c1.trust.success(c2.trust)
86
- else
87
- #transfer failed, the client still wants the chunk
88
- client_info(transfer.taker).chunk_info.request(transfer.url,transfer.chunkid..transfer.chunkid)
89
- c1.trust.failure(c2.trust)
90
- end
91
-
92
- transfer.taker.send_message(:hash_verify,
93
- :url => transfer.url,
94
- :range => transfer.byte_range,
95
- :hash_ok => success
96
- ) if send_response
97
- end
98
-
99
- #remove this transfer from whoever sent it
100
- client_info(connection).transfers.delete(transfer.transfer_id)
101
- @updated_clients[connection]=true #flag this client for transfer creation
102
- end
103
-
104
- #Creates a new transfer between two peers
105
- #returns true on success, or false if the specified transfer is already in progress
106
- def begin_transfer(taker, giver, url, chunkid)
107
- byte_range = @file_service.get_info(url).chunk_range(chunkid)
108
- t = Transfer.new(taker, giver, url, chunkid, byte_range)
109
-
110
- #make sure this transfer doesnt already exist
111
- t1 = client_info(taker).transfers[t.transfer_id]
112
- t2 = client_info(giver).transfers[t.transfer_id]
113
- return false unless t1.nil? and t2.nil?
114
-
115
- client_info(taker).chunk_info.transfer(url, chunkid..chunkid)
116
- client_info(taker).transfers[t.transfer_id] = t
117
- client_info(giver).transfers[t.transfer_id] = t
118
-
119
- #send transfer message to the connector
120
- addr, port = t.acceptor.get_peer_info
121
61
 
122
- t.connector.send_message(:transfer,
123
- :host => addr,
124
- :port => t.acceptor.user_data.listen_port,
125
- :method => t.connector == t.taker ? "get" : "put",
126
- :url => url,
127
- :range => byte_range,
128
- :peer_id => client_info(t.acceptor).client_id
129
- )
130
- true
131
- end
132
-
133
- #this function removes all stalled transfers from the list
134
- #and spawns new transfers as appropriate
135
- #it must be called periodically by EventMachine
62
+ # This function removes all stalled transfers from the list
63
+ # and spawns new transfers as appropriate
64
+ # It must be called periodically by EventMachine
136
65
  def clear_all_stalled_transfers
137
66
  @connections.each { |connection| clear_stalled_transfers_for_client connection }
138
- spawn_all_transfers
139
- end
140
-
141
- #removes all stalled transfers that this client is a part of
142
- def clear_stalled_transfers_for_client(client_connection)
143
- client_info(client_connection).get_stalled_transfers.each do |transfer|
144
- transfer_completed transfer, client_connection, nil, false
145
- end
146
- end
147
-
148
- #spawns uploads and downloads for this client.
149
- #should be called every time there is a change that would affect
150
- #what this client has or wants
151
- def spawn_transfers_for_client(client_connection)
152
- info = client_info client_connection
153
-
154
- while info.wants_download? do
155
- break if spawn_download_for_client(client_connection) == false
156
- end
157
-
158
- while info.wants_upload? do
159
- break if spawn_upload_for_client(client_connection) == false
160
- end
67
+ @transfer_manager.spawn_all_transfers
161
68
  end
162
-
163
- #creates a single download for the specified client
164
- #returns true on success, false on failure
165
- def spawn_download_for_client(client_connection)
166
- feasible_peers=[]
167
-
168
- c1info=client_info(client_connection)
169
- begin
170
- url,chunkid=c1info.chunk_info.high_priority_chunk
171
- rescue
172
- return false
173
- end
174
-
175
- @connections.each do |c2|
176
- next if client_connection==c2
177
- next if client_info(c2).wants_upload? == false
178
- if client_info(c2).chunk_info.provided?(url,chunkid)
179
- feasible_peers << c2
180
- break if feasible_peers.size > 5
181
- end
182
- end
183
-
184
- # we now have a list of clients that have the requested chunk.
185
- # pick one and start the transfer
186
- if feasible_peers.size > 0
187
- #FIXME base this on the trust model
188
- giver=feasible_peers[rand(feasible_peers.size)]
189
- return begin_transfer(client_connection,giver,url,chunkid)
190
- #FIXME should we try again if begin_transfer fails?
191
- end
192
-
193
- false
194
- end
195
-
196
- #creates a single upload for the specified client
197
- #returns true on success, false on failure
198
- def spawn_upload_for_client(client_connection)
199
- c1info=client_info(client_connection)
200
-
201
- @connections.each do |c2|
202
- next if client_connection==c2
203
- next if client_info(c2).wants_download? == false
204
-
205
- begin
206
- url,chunkid=client_info(c2).chunk_info.high_priority_chunk
207
- rescue
208
- next
209
- end
210
-
211
- if c1info.chunk_info.provided?(url,chunkid)
212
- return begin_transfer(c2,client_connection,url,chunkid)
213
- end
214
- end
215
-
216
- false
217
- end
218
-
219
- #creates new transfers for all clients that have been updated
220
- def spawn_all_transfers
221
- while @updated_clients.size > 0 do
222
- tmp=@updated_clients
223
- @updated_clients=Hash.new
224
- tmp.each do |client,true_key|
225
- spawn_transfers_for_client(client)
226
- end
227
- end
228
- end
229
-
230
- #handles the request, provide, unrequest, unprovide messages
231
- def handle_requestprovide(connection,message)
232
- type=message["type"]
233
- url=message["url"]
234
- info=@file_service.get_info(url) rescue nil
235
- raise ProtocolWarn.new("Requested URL: '#{url}' not found") if info.nil?
236
-
237
- exclude_partial= (type=="provide") #only exclude partial chunks from provides
238
- range=info.chunk_range_from_byte_range(message["range"],exclude_partial)
239
-
240
- #call request, provide, unrequest, or unprovide
241
- client_info(connection).chunk_info.send( type.to_sym, url, range)
242
- @updated_clients[connection]=true #add to the list of client that need new transfers
243
- end
244
-
245
- #handles all incoming messages from clients
69
+
70
+ # Handles all incoming messages from clients
246
71
  def dispatch_message(command, message, connection)
247
- # store the command in the message hash
72
+ # Store the command in the message hash
248
73
  message["type"] = command
249
74
 
250
- #require the client to register their client id and listen port before doing anything
251
- if command != "register" and client_info(connection).client_id.nil?
252
- raise ProtocolError.new("You need to send a 'client_info' message first")
75
+ # Require the client to register their client id and listen port before doing anything
76
+ if command != "register" and connection.client_id.nil?
77
+ raise ProtocolError, "You need to send a 'register' message first"
253
78
  end
254
79
 
255
80
  case command
@@ -257,12 +82,12 @@ module PDTP
257
82
  cid = message["client_id"]
258
83
  #make sure this id isnt in use
259
84
  if @used_client_ids[cid]
260
- raise ProtocolError.new("Your client id: #{cid} is already in use.")
85
+ raise ProtocolError, "Your client id: #{cid} is already in use."
261
86
  end
262
87
 
263
88
  @used_client_ids[cid] = true
264
- client_info(connection).listen_port = message["listen_port"]
265
- client_info(connection).client_id = cid
89
+ connection.listen_port = message["listen_port"]
90
+ connection.client_id = cid
266
91
  when "ask_info"
267
92
  info = @file_service.get_info(message["url"])
268
93
  response = { :url => message["url"] }
@@ -276,45 +101,92 @@ module PDTP
276
101
  handle_requestprovide connection, message
277
102
  when "ask_verify"
278
103
  #check if the specified transfer is a real one
279
- my_id = client_info(connection).client_id
280
- transfer_id=Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
281
- ok = !!client_info(connection).transfers[transfer_id]
282
- client_info(connection).transfers[transfer_id].verification_asked=true if ok
283
- @server.debug "AskVerify not ok: id=#{transfer_id}" unless ok
104
+ my_id = connection.client_id
105
+ transfer_id = Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
106
+ authorized = !connection.transfers[transfer_id].nil?
107
+
108
+ connection.transfers[transfer_id].verification_asked = true if authorized
109
+ @server.debug "AskVerify not ok: id=#{transfer_id}" unless authorized
284
110
  connection.send_message(:tell_verify,
285
111
  :url => message["url"],
286
112
  :peer_id => message["peer_id"],
287
113
  :range => message["range"],
288
114
  :peer => message["peer"],
289
- :authorized=>ok
115
+ :authorized=>authorized
290
116
  )
291
117
  when "completed"
292
- my_id = client_info(connection).client_id
118
+ my_id = connection.client_id
293
119
  transfer_id = Transfer::gen_transfer_id(
294
120
  my_id,
295
121
  message["peer_id"],
296
122
  message["url"],
297
123
  message["range"]
298
124
  )
299
- transfer=client_info(connection).transfers[transfer_id]
300
- @server.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
125
+ transfer = connection.transfers[transfer_id]
126
+ @server.debug "Completed: id=#{transfer_id} ok=#{transfer != nil}"
301
127
  if transfer
302
- transfer_completed(transfer,connection,message["hash"])
128
+ transfer_completed transfer, connection, message["hash"]
303
129
  else
304
- raise ProtocolWarn.new("You sent me a transfer completed message for unknown transfer: #{transfer_id}")
130
+ raise ProtocolWarn, "You sent me a transfer completed message for unknown transfer: #{transfer_id}"
305
131
  end
306
132
  when 'protocol_error', 'protocol_warn' #ignore
307
- else raise ProtocolError.new("Unhandled message type: #{command}")
133
+ else raise ProtocolError, "Unhandled message type: #{command}"
308
134
  end
309
135
 
310
- spawn_all_transfers
136
+ # Process all clients that are in need of new transfers
137
+ @transfer_manager.spawn_all_transfers
311
138
  end
312
139
 
313
- #returns a string representing the specified connection
314
- def connection_name(c)
315
- #host,port=c.get_peer_info
316
- #return "#{get_id(c)}: #{host}:#{port}"
317
- client_info(c).client_id
140
+ #########
141
+ protected
142
+ #########
143
+
144
+ # Removes all stalled transfers that this client is a part of
145
+ def clear_stalled_transfers_for_client(connection)
146
+ connection.stalled_transfers.each do |transfer|
147
+ transfer_completed transfer, connection, nil, false
148
+ end
149
+ end
150
+
151
+ # Called when a transfer either finishes, successfully or not
152
+ def transfer_completed(transfer, connection, chunk_hash, send_response=true)
153
+ # Compute the SHA256 hash for the given file
154
+ local_hash = @file_service.get_chunk_hash transfer.url, transfer.chunkid
155
+
156
+ if connection == transfer.taker
157
+ success = (chunk_hash == local_hash)
158
+
159
+ if success
160
+ transfer.success
161
+ else
162
+ transfer.failure
163
+ end
164
+
165
+ transfer.taker.send_message(:hash_verify,
166
+ :url => transfer.url,
167
+ :range => transfer.byte_range,
168
+ :hash_ok => success
169
+ ) if send_response
170
+ end
171
+
172
+ # Remove this transfer from whoever sent it
173
+ connection.transfers.delete(transfer.transfer_id)
174
+ @transfer_manager.process_client(connection)
175
+ end
176
+
177
+ # Handles the request, provide, unrequest, unprovide messages
178
+ def handle_requestprovide(connection, message)
179
+ type = message["type"]
180
+ url = message["url"]
181
+ info = @file_service.get_info(url) rescue nil
182
+ raise ProtocolWarn, "Requested URL: '#{url}' not found" if info.nil?
183
+
184
+ exclude_partial = (type=="provide") #only exclude partial chunks from provides
185
+ range = info.chunk_range_from_byte_range(message["range"],exclude_partial)
186
+
187
+ #call request, provide, unrequest, or unprovide
188
+ connection.chunk_info.send(type.to_sym, url, range)
189
+ @transfer_manager.process_client(connection)
318
190
  end
319
191
  end
320
192
  end
@@ -38,7 +38,7 @@ module PDTP
38
38
  range = 0..chunk_size(chunkid) - 1 if range.nil? # full range of chunk if range isnt specified
39
39
  raise if range.first < 0 or range.last >= chunk_size(chunkid)
40
40
  start = range.first + chunkid * @base_chunk_size
41
- size = range.last-range.first + 1
41
+ size = range.last - range.first + 1
42
42
  file = open @path
43
43
  file.pos = start
44
44
  file.read size
@@ -72,7 +72,8 @@ module PDTP
72
72
  # by checking whether the registered file exists
73
73
  def get_info(url)
74
74
  begin
75
- host = URI.split(url)[2]
75
+ host = URI.parse(url).host
76
+
76
77
  #FIXME we should check host against a list of known hosts here
77
78
  info = FileInfo.new
78
79
  info.streaming = false
@@ -90,13 +91,13 @@ module PDTP
90
91
 
91
92
  #returns the path of this file on the local filesystem
92
93
  def get_local_path(url)
93
- path = URI.split(url)[5]
94
+ path = URI.parse(url).path
94
95
  path = path[1..path.size-1] #remove leading /
95
96
  (Pathname.new(@root) + path).to_s
96
97
  end
97
98
 
98
99
  #returns the SHA256 hash of the specified chunk
99
- def get_chunk_hash(url,chunk_id)
100
+ def get_chunk_hash(url, chunk_id)
100
101
  Digest::SHA256.hexdigest(get_info(url).chunk_data(chunk_id)) rescue nil
101
102
  end
102
103
  end
@@ -43,12 +43,12 @@ module PDTP
43
43
  # Iterate over all of a peer's active transfers
44
44
  def each_transfer(peer)
45
45
  raise ArgumentError, "no block given" unless block_given?
46
- @dispatcher.client_info(peer).transfers.each { |_, transfer| yield transfer }
46
+ peer.transfers.each { |_, transfer| yield transfer }
47
47
  end
48
48
 
49
49
  # Iterate over all of a peer's active files
50
50
  def each_file(peer, &block)
51
- @dispatcher.client_info(peer).chunk_info.get_file_stats.each(&block)
51
+ peer.chunk_info.get_file_stats.each(&block)
52
52
  end
53
53
 
54
54
  # Name of a peer (client ID or file service)
@@ -56,7 +56,7 @@ module PDTP
56
56
  if peer.file_service?
57
57
  "<b>File Service</b>"
58
58
  else
59
- @dispatcher.connection_name(peer)
59
+ peer.client_id
60
60
  end
61
61
  end
62
62
 
@@ -65,6 +65,22 @@ module PDTP
65
65
  host, port = peer.get_peer_info
66
66
  "#{host}:#{port}"
67
67
  end
68
+
69
+ # Upstream bandwidth of a peer
70
+ def upstream_bandwidth(peer)
71
+ bandwidth = peer.upstream_bandwidth
72
+ return 'N/A' if bandwidth.nil?
73
+
74
+ "#{bandwidth / 1024} kBps"
75
+ end
76
+
77
+ # Downstream bandwidth of a peer
78
+ def downstream_bandwidth(peer)
79
+ bandwidth = peer.downstream_bandwidth
80
+ return 'N/A' if bandwidth.nil?
81
+
82
+ "#{bandwidth / 1024} kBps"
83
+ end
68
84
 
69
85
  # Information about an active transfer
70
86
  def transfer_info(peer, transfer)
@@ -30,34 +30,46 @@ module PDTP
30
30
  attr_accessor :creation_time
31
31
  attr_accessor :verification_asked
32
32
 
33
- #generates a transfer id based on 2 client ids, a url, and a byte range
33
+ # Generates a transfer id based on 2 client ids, a url, and a byte range
34
34
  def self.gen_transfer_id(id1,id2,url,byte_range)
35
- a = id1<id2 ? id1 : id2
36
- b = id1<id2 ? id2 : id1
35
+ a = id1 < id2 ? id1 : id2
36
+ b = id1 < id2 ? id2 : id1
37
37
  "#{a}$#{b}$#{url}$#{byte_range}"
38
38
  end
39
39
 
40
- def initialize(taker,giver,url,chunkid,byte_range,connector_receives=true)
41
- @taker,@giver,@url,@chunkid,@byte_range=taker,giver,url,chunkid,byte_range
40
+ def initialize(taker, giver, url, chunkid, byte_range, connector_receives = true)
41
+ @taker, @giver, @url, @chunkid, @byte_range = taker, giver, url, chunkid, byte_range
42
42
 
43
43
  @verification_asked = false
44
44
  @creation_time = Time.now
45
+
45
46
  if connector_receives
46
- @connector=@taker
47
- @acceptor=@giver
47
+ @connector = @taker
48
+ @acceptor = @giver
48
49
  else
49
- @connector=@giver
50
- @acceptor=@taker
50
+ @connector = @giver
51
+ @acceptor = @taker
51
52
  end
52
53
 
53
54
  recompute_transfer_id
54
55
  end
55
56
 
56
- #calculates the transfer id for this transfer based on the local data
57
+ # Calculates the transfer id for this transfer based on the local data
57
58
  def recompute_transfer_id
58
- id1=connector.user_data.client_id
59
- id2=acceptor.user_data.client_id
60
- @transfer_id=Transfer::gen_transfer_id(id1,id2,@url,@byte_range)
59
+ id1 = connector.client_id
60
+ id2 = acceptor.client_id
61
+ @transfer_id = Transfer::gen_transfer_id id1, id2, @url, @byte_range
62
+ end
63
+
64
+ # Update internal data upon a successful transfer
65
+ def success
66
+ giver.success self
67
+ taker.success self
68
+ end
69
+
70
+ # Update internal data upon a failed transfer
71
+ def failure
72
+ taker.failure self
61
73
  end
62
74
 
63
75
  def to_s
@@ -67,7 +79,7 @@ module PDTP
67
79
  def debug_str
68
80
  str = ''
69
81
  str << "to_s=#{to_s}"
70
- str << " taker_id=#{@taker.user_data.client_id} giver_id=#{@giver.user_data.client_id}"
82
+ str << " taker_id=#{@taker.client_id} giver_id=#{@giver.client_id}"
71
83
  end
72
84
  end
73
85
  end