distribustream 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,370 @@
1
+ #--
2
+ # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
+ # All rights reserved. See COPYING for permissions.
4
+ #
5
+ # This source file is distributed as part of the
6
+ # DistribuStream file transfer system.
7
+ #
8
+ # See http://distribustream.rubyforge.org/
9
+ #++
10
+
11
+ require File.dirname(__FILE__) + '/../common/protocol'
12
+ require File.dirname(__FILE__) + '/../common/common_init'
13
+ require File.dirname(__FILE__) + '/file_service'
14
+ require File.dirname(__FILE__) + '/client_info'
15
+ require File.dirname(__FILE__) + '/transfer'
16
+
17
+ require 'thread'
18
+ require 'erb'
19
+
20
+ module PDTP
21
+ class Server
22
+ # Core dispatching and control logic for PDTP servers
23
+ class Dispatcher
24
+ attr_reader :connections
25
+ attr_accessor :file_service
26
+ def initialize
27
+ @connections = []
28
+ @used_client_ids = {} #keeps a list of client ids in use, they must be unique
29
+ @updated_clients = {} #a set of clients that have been modified and need transfers spawned
30
+ @stats_mutex=Mutex.new
31
+ end
32
+
33
+ #called by pdtp_protocol when a connection is created
34
+ def connection_created(connection)
35
+ @stats_mutex.synchronize do
36
+ @@log.info "Client connected: #{connection.get_peer_info.inspect}"
37
+ connection.user_data = ClientInfo.new
38
+ @connections << connection
39
+ end
40
+ end
41
+
42
+ #called by pdtp_protocol when a connection is destroyed
43
+ def connection_destroyed(connection)
44
+ @stats_mutex.synchronize do
45
+ @@log.info "Client connection closed: #{connection.get_peer_info.inspect}"
46
+ @connections.delete connection
47
+ end
48
+ end
49
+
50
+ # returns the ClientInfo object associated with this connection
51
+ def client_info(connection)
52
+ connection.user_data ||= ClientInfo.new
53
+ end
54
+
55
+ # called when a transfer either finishes, successfully or not
56
+ def transfer_completed(transfer,connection,chunk_hash,send_response=true)
57
+ # did the transfer complete successfully?
58
+ local_hash=@file_service.get_chunk_hash(transfer.url,transfer.chunkid)
59
+
60
+ c1=client_info(transfer.taker)
61
+ c2=client_info(transfer.giver)
62
+
63
+ if connection==transfer.taker
64
+ success= (chunk_hash==local_hash)
65
+
66
+ if success
67
+ #the taker now has the file, so he can provide it
68
+ client_info(transfer.taker).chunk_info.provide(transfer.url,transfer.chunkid..transfer.chunkid)
69
+ c1.trust.success(c2.trust)
70
+ else
71
+ #transfer failed, the client still wants the chunk
72
+ client_info(transfer.taker).chunk_info.request(transfer.url,transfer.chunkid..transfer.chunkid)
73
+ c1.trust.failure(c2.trust)
74
+ end
75
+
76
+ transfer.taker.send_message(:hash_verify,
77
+ :url => transfer.url,
78
+ :range => transfer.byte_range,
79
+ :hash_ok => success
80
+ ) if send_response
81
+ end
82
+
83
+ #outstr="#{@ids[transfer.giver]}->#{@ids[transfer.taker]} transfer completed: #{transfer}"
84
+ #outstr=outstr+" t->g=#{c1.trust.weight(c2.trust)} g->t=#{c2.trust.weight(c1.trust)}"
85
+ #outstr=outstr+"sent_by: "+ ( connection==transfer.taker ? "taker" : "giver" )
86
+ #outstr=outstr+" success=#{success} "
87
+ #@@log.debug(outstr)
88
+
89
+ #remove this transfer from whoever sent it
90
+ client_info(connection).transfers.delete(transfer.transfer_id)
91
+ @updated_clients[connection]=true #flag this client for transfer creation
92
+ end
93
+
94
+ #Creates a new transfer between two peers
95
+ #returns true on success, or false if the specified transfer is already in progress
96
+ def begin_transfer(taker, giver, url, chunkid)
97
+ byte_range = @file_service.get_info(url).chunk_range(chunkid)
98
+ t = Transfer.new(taker, giver, url, chunkid, byte_range)
99
+
100
+ #make sure this transfer doesnt already exist
101
+ t1 = client_info(taker).transfers[t.transfer_id]
102
+ t2 = client_info(giver).transfers[t.transfer_id]
103
+ return false unless t1.nil? and t2.nil?
104
+
105
+ client_info(taker).chunk_info.transfer(url, chunkid..chunkid)
106
+ client_info(taker).transfers[t.transfer_id] = t
107
+ client_info(giver).transfers[t.transfer_id] = t
108
+
109
+ #send transfer message to the connector
110
+ addr, port = t.acceptor.get_peer_info
111
+
112
+ t.connector.send_message(:transfer,
113
+ :host => addr,
114
+ :port => t.acceptor.user_data.listen_port,
115
+ :method => t.connector == t.taker ? "get" : "put",
116
+ :url => url,
117
+ :range => byte_range,
118
+ :peer_id => client_info(t.acceptor).client_id
119
+ )
120
+ true
121
+ end
122
+
123
+ #this function removes all stalled transfers from the list
124
+ #and spawns new transfers as appropriate
125
+ #it must be called periodically by EventMachine
126
+ def clear_all_stalled_transfers
127
+ @connections.each { |connection| clear_stalled_transfers_for_client connection }
128
+ spawn_all_transfers
129
+ end
130
+
131
+ #removes all stalled transfers that this client is a part of
132
+ def clear_stalled_transfers_for_client(client_connection)
133
+ client_info(client_connection).get_stalled_transfers.each do |transfer|
134
+ transfer_completed transfer, client_connection, nil, false
135
+ end
136
+ end
137
+
138
+ #spawns uploads and downloads for this client.
139
+ #should be called every time there is a change that would affect
140
+ #what this client has or wants
141
+ def spawn_transfers_for_client(client_connection)
142
+ info = client_info client_connection
143
+
144
+ while info.wants_download? do
145
+ break if spawn_download_for_client(client_connection) == false
146
+ end
147
+
148
+ while info.wants_upload? do
149
+ break if spawn_upload_for_client(client_connection) == false
150
+ end
151
+ end
152
+
153
+ #creates a single download for the specified client
154
+ #returns true on success, false on failure
155
+ def spawn_download_for_client(client_connection)
156
+ feasible_peers=[]
157
+
158
+ c1info=client_info(client_connection)
159
+ begin
160
+ url,chunkid=c1info.chunk_info.high_priority_chunk
161
+ rescue
162
+ return false
163
+ end
164
+
165
+ @connections.each do |c2|
166
+ next if client_connection==c2
167
+ next if client_info(c2).wants_upload? == false
168
+ if client_info(c2).chunk_info.provided?(url,chunkid)
169
+ feasible_peers << c2
170
+ break if feasible_peers.size > 5
171
+ end
172
+ end
173
+
174
+ # we now have a list of clients that have the requested chunk.
175
+ # pick one and start the transfer
176
+ if feasible_peers.size > 0
177
+ #FIXME base this on the trust model
178
+ giver=feasible_peers[rand(feasible_peers.size)]
179
+ return begin_transfer(client_connection,giver,url,chunkid)
180
+ #FIXME should we try again if begin_transfer fails?
181
+ end
182
+
183
+ false
184
+ end
185
+
186
+ #creates a single upload for the specified client
187
+ #returns true on success, false on failure
188
+ def spawn_upload_for_client(client_connection)
189
+ c1info=client_info(client_connection)
190
+
191
+ @connections.each do |c2|
192
+ next if client_connection==c2
193
+ next if client_info(c2).wants_download? == false
194
+
195
+ begin
196
+ url,chunkid=client_info(c2).chunk_info.high_priority_chunk
197
+ rescue
198
+ next
199
+ end
200
+
201
+ if c1info.chunk_info.provided?(url,chunkid)
202
+ return begin_transfer(c2,client_connection,url,chunkid)
203
+ end
204
+ end
205
+
206
+ false
207
+ end
208
+
209
+ #called by pdtp_protocol for each message that comes in from the wire
210
+ def dispatch_message(command, message, connection)
211
+ @stats_mutex.synchronize do
212
+ dispatch_message_needslock command, message, connection
213
+ end
214
+ end
215
+
216
+ #creates new transfers for all clients that have been updated
217
+ def spawn_all_transfers
218
+ while @updated_clients.size > 0 do
219
+ tmp=@updated_clients
220
+ @updated_clients=Hash.new
221
+ tmp.each do |client,true_key|
222
+ spawn_transfers_for_client(client)
223
+ end
224
+ end
225
+ end
226
+
227
+ #handles the request, provide, unrequest, unprovide messages
228
+ def handle_requestprovide(connection,message)
229
+ type=message["type"]
230
+ url=message["url"]
231
+ info=@file_service.get_info(url) rescue nil
232
+ raise ProtocolWarn.new("Requested URL: '#{url}' not found") if info.nil?
233
+
234
+ exclude_partial= (type=="provide") #only exclude partial chunks from provides
235
+ range=info.chunk_range_from_byte_range(message["range"],exclude_partial)
236
+
237
+ #call request, provide, unrequest, or unprovide
238
+ client_info(connection).chunk_info.send( type.to_sym, url, range)
239
+ @updated_clients[connection]=true #add to the list of client that need new transfers
240
+ end
241
+
242
+ #handles all incoming messages from clients
243
+ def dispatch_message_needslock(command, message, connection)
244
+ # store the command in the message hash
245
+ message["type"] = command
246
+
247
+ #require the client to register their client id and listen port before doing anything
248
+ if command != "register" and client_info(connection).client_id.nil?
249
+ raise ProtocolError.new("You need to send a 'client_info' message first")
250
+ end
251
+
252
+ case command
253
+ when "register"
254
+ cid = message["client_id"]
255
+ #make sure this id isnt in use
256
+ if @used_client_ids[cid]
257
+ raise ProtocolError.new("Your client id: #{cid} is already in use.")
258
+ end
259
+
260
+ @used_client_ids[cid] = true
261
+ client_info(connection).listen_port = message["listen_port"]
262
+ client_info(connection).client_id = cid
263
+ when "ask_info"
264
+ info = file_service.get_info(message["url"])
265
+ response = { :url => message["url"] }
266
+ unless info.nil?
267
+ response[:size] = info.file_size
268
+ response[:chunk_size] = info.base_chunk_size
269
+ response[:streaming] = info.streaming
270
+ end
271
+ connection.send_message :tell_info, response
272
+ when "request", "provide", "unrequest", "unprovide"
273
+ handle_requestprovide connection, message
274
+ when "ask_verify"
275
+ #check if the specified transfer is a real one
276
+ my_id = client_info(connection).client_id
277
+ transfer_id=Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
278
+ ok = !!client_info(connection).transfers[transfer_id]
279
+ client_info(connection).transfers[transfer_id].verification_asked=true if ok
280
+ @@log.debug "AskVerify not ok: id=#{transfer_id}" unless ok
281
+ connection.send_message(:tell_verify,
282
+ :url => message["url"],
283
+ :peer_id => message["peer_id"],
284
+ :range => message["range"],
285
+ :peer => message["peer"],
286
+ :authorized=>ok
287
+ )
288
+ when "completed"
289
+ my_id = client_info(connection).client_id
290
+ transfer_id= Transfer::gen_transfer_id(
291
+ my_id,message["peer_id"],
292
+ message["url"],
293
+ message["range"]
294
+ )
295
+ transfer=client_info(connection).transfers[transfer_id]
296
+ @@log.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
297
+ if transfer
298
+ transfer_completed(transfer,connection,message["hash"])
299
+ else
300
+ raise ProtocolWarn.new("You sent me a transfer completed message for unknown transfer: #{transfer_id}")
301
+ end
302
+ when 'protocol_error', 'protocol_warn' #ignore
303
+ else raise ProtocolError.new("Unhandled message type: #{command}")
304
+ end
305
+
306
+ spawn_all_transfers
307
+ end
308
+
309
+ #returns a string representing the specified connection
310
+ def connection_name(c)
311
+ #host,port=c.get_peer_info
312
+ #return "#{get_id(c)}: #{host}:#{port}"
313
+ client_info(c).client_id
314
+ end
315
+
316
+ def generate_html_stats
317
+ @stats_mutex.synchronize { generate_html_stats_needslock }
318
+ end
319
+
320
+ #builds an html page with information about the server's internal workings
321
+ def generate_html_stats_needslock
322
+ s = ERB.new <<EOF
323
+ <html><head><title>DistribuStream Statistics</title></head>
324
+ <body>
325
+ <h1>DistribuStream Statistics</h1>
326
+ Time=<%= Time.new %><br> Connected Clients=<%= @connections.size %>
327
+ <center><table border=1>
328
+ <tr><th>Client</th><th>Transfers</th><th>Files</th></tr>
329
+ <% @connections.each do |c| %>
330
+ <tr><td>
331
+ <% host, port = c.get_peer_info %>
332
+ <%= connection_name(c) %><br><%= host %>:<%= port %>
333
+ </td>
334
+ <td>
335
+ <%
336
+ client_info(c).transfers.each do |key,t|
337
+ if c==t.giver
338
+ type="UP: "
339
+ peer=t.taker
340
+ else
341
+ type="DOWN: "
342
+ peer=t.giver
343
+ end
344
+ %>
345
+ <%= type %> id=<%= t.transfer_id %><br>
346
+ <%
347
+ end
348
+ %>
349
+ </td>
350
+ <td>
351
+ <%
352
+ client_info(c).chunk_info.get_file_stats.each do |fs|
353
+ %>
354
+ <%= fs.url %> size=<%= fs.file_chunks %> req=<%= fs.chunks_requested %>
355
+ prov=<%= fs.chunks_provided %> transf=<%= fs.chunks_transferring %><br>
356
+ <%
357
+ end
358
+ %>
359
+ </td></tr>
360
+ <%
361
+ end
362
+ %>
363
+ </table>
364
+ </body></html>
365
+ EOF
366
+ s.result binding
367
+ end
368
+ end
369
+ end
370
+ end
@@ -48,7 +48,7 @@ module PDTP
48
48
 
49
49
  #The file service provides utilities for determining various information about files.
50
50
  class FileService < PDTP::FileService
51
- attr_accessor :root,:default_chunk_size
51
+ attr_accessor :root, :default_chunk_size
52
52
 
53
53
  def initialize
54
54
  @root = ''
@@ -0,0 +1,116 @@
1
+ #!/usr/bin/env ruby
2
+ #--
3
+ # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
4
+ # All rights reserved. See COPYING for permissions.
5
+ #
6
+ # This source file is distributed as part of the
7
+ # DistribuStream file transfer system.
8
+ #
9
+ # See http://distribustream.rubyforge.org/
10
+ #++
11
+
12
+ require 'digest/md5'
13
+ require 'thread'
14
+
15
+ require File.dirname(__FILE__) + '/../common/protocol'
16
+ require File.dirname(__FILE__) + '/../server/file_service'
17
+ require File.dirname(__FILE__) + '/../client/connection'
18
+ require File.dirname(__FILE__) + '/../client/http_handler'
19
+
20
+ module PDTP
21
+ class Server
22
+ class FileService
23
+ # Implements the file service for the pdtp protocol
24
+ class Protocol < PDTP::Client::Connection
25
+ attr_reader :client, :connection, :file_service, :client_id, :transfers, :lock
26
+
27
+ def initialize(data)
28
+ @transfers = []
29
+ @lock = Mutex.new
30
+ @client = self
31
+ @connection = self
32
+ @client_id = Digest::MD5.hexdigest "#{Time.now.to_f}#{$$}"
33
+
34
+ super data
35
+ end
36
+
37
+ # Called after a connection to the server has been established
38
+ def connection_completed
39
+ begin
40
+ @listen_addr = '0.0.0.0'
41
+ @listen_port = @@config[:listen_port]
42
+
43
+ #create the client
44
+ #client = PDTP::Client.new
45
+ #PDTP::Protocol.listener = client
46
+ #client.server_connection = self
47
+ #client.generate_client_id listen_port
48
+
49
+ # Start a mongrel server on the specified port. If it isnt available, keep trying higher ports
50
+ begin
51
+ mongrel_server = Mongrel::HttpServer.new @listen_addr, @listen_port
52
+ rescue Exception => e
53
+ @listen_port += 1
54
+ retry
55
+ end
56
+
57
+ #@@log.info "listening on port #{@listen_port}"
58
+ @http_handler = Client::HttpHandler.new(self)
59
+
60
+ @@log.info "listening on port #{@listen_port}"
61
+ mongrel_server.register "/", @http_handler
62
+ mongrel_server.run
63
+
64
+ # Register our client_id and listen_port
65
+ send_message :register, :listen_port => @listen_port, :client_id => @client_id
66
+
67
+ @@log.info 'This client is providing'
68
+ @file_service = PDTP::Server::FileService.new
69
+ @file_service.root = @base_path
70
+ #client.file_service = sfs #give this client access to all data
71
+
72
+ hostname = @@config[:vhost]
73
+
74
+ # Provide all the files in the root directory
75
+ files = find_files @base_path
76
+ files.each { |file| send_message :provide, :url => "http://#{hostname}/#{file}" }
77
+ rescue Exception => e
78
+ puts "Exception in connection_completed: #{e}"
79
+ puts e.backtrace.join("\n")
80
+ exit
81
+ end
82
+ end
83
+
84
+ def unbind
85
+ super
86
+ puts "Disconnected from PDTP server."
87
+ end
88
+
89
+ #########
90
+ protected
91
+ #########
92
+
93
+ # Fine all suitable files in the give path
94
+ def find_files(base_path)
95
+ require 'find'
96
+
97
+ found = []
98
+ excludes = %w{.svn CVS}
99
+ base_full = File.expand_path(base_path)
100
+
101
+ Find.find(base_full) do |path|
102
+ if FileTest.directory?(path)
103
+ next unless excludes.include?(File.basename(path))
104
+ Find.prune
105
+ else
106
+ filename = path[(base_path.size - path.size + 1)..-1] #the entire file path after the base_path
107
+ found << filename
108
+ end
109
+ end
110
+
111
+ found
112
+ end
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,23 @@
1
+ require 'rubygems'
2
+ require 'mongrel'
3
+
4
+ module PDTP
5
+ class Server
6
+ #set up the mongrel server for serving the stats page
7
+ class StatsHandler < Mongrel::HttpHandler
8
+ def initialize(server)
9
+ @server = server
10
+ end
11
+
12
+ def process(request,response)
13
+ response.start(200) do |head, out|
14
+ out.write begin
15
+ outstr = @server.generate_html_stats
16
+ rescue Exception=>e
17
+ outstr = "Exception: #{e}\n#{e.backtrace.join("\n")}"
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -9,76 +9,78 @@
9
9
  #++
10
10
 
11
11
  module PDTP
12
- #maintains trust information for a single client
13
- class Trust
14
- #struct for storing a single trust relationship
15
- class Edge
16
- attr_accessor :trust, :success, :transfers
12
+ class Server
13
+ #maintains trust information for a single client
14
+ class Trust
15
+ #struct for storing a single trust relationship
16
+ class Edge
17
+ attr_accessor :trust, :success, :transfers
17
18
 
18
- def initialize(trust = 1.0, success = 1.0, transfers = 1.0)
19
- @trust = trust
20
- @success = success
21
- @transfers = transfers
19
+ def initialize(trust = 1.0, success = 1.0, transfers = 1.0)
20
+ @trust = trust
21
+ @success = success
22
+ @transfers = transfers
23
+ end
22
24
  end
23
- end
24
25
 
25
- attr_reader :outgoing, :implicit
26
+ attr_reader :outgoing, :implicit
26
27
 
27
- def initialize(incoming = {}, outgoing = {}, implicit = {})
28
- @incoming = incoming
29
- @outgoing = outgoing
30
- @implicit = implicit
31
- end
28
+ def initialize(incoming = {}, outgoing = {}, implicit = {})
29
+ @incoming = incoming
30
+ @outgoing = outgoing
31
+ @implicit = implicit
32
+ end
32
33
 
33
- #I have successfully downloaded a chunk from 'node'
34
- def success(node)
35
- if @outgoing[node].nil?
36
- @outgoing[node] = Edge.new
37
- else
38
- @outgoing[node].success += 1.0
39
- @outgoing[node].transfers += 1.0
34
+ #I have successfully downloaded a chunk from 'node'
35
+ def success(node)
36
+ if @outgoing[node].nil?
37
+ @outgoing[node] = Edge.new
38
+ else
39
+ @outgoing[node].success += 1.0
40
+ @outgoing[node].transfers += 1.0
41
+ end
42
+ normalize
40
43
  end
41
- normalize
42
- end
43
44
 
44
- #I have failed to download a chunk from 'node'
45
- def failure(node)
46
- @outgoing[node] = Edge.new if @outgoing[node].nil?
47
- @outgoing[node].transfers += 1.0
48
- normalize
49
- end
45
+ #I have failed to download a chunk from 'node'
46
+ def failure(node)
47
+ @outgoing[node] = Edge.new if @outgoing[node].nil?
48
+ @outgoing[node].transfers += 1.0
49
+ normalize
50
+ end
50
51
 
51
- #returns a number from 0 to 1 saying how much I trust 'node'
52
- def weight(node)
53
- return @outgoing[node].trust unless @outgoing[node].nil?
54
- return @implicit[node].trust unless @implicit[node].nil?
55
- 0
56
- end
52
+ #returns a number from 0 to 1 saying how much I trust 'node'
53
+ def weight(node)
54
+ return @outgoing[node].trust unless @outgoing[node].nil?
55
+ return @implicit[node].trust unless @implicit[node].nil?
56
+ 0
57
+ end
57
58
 
58
- # brings all trust values between 0 and 1
59
- def normalize
60
- total_success = 0
61
- total_transfers = 0
59
+ # brings all trust values between 0 and 1
60
+ def normalize
61
+ total_success = 0
62
+ total_transfers = 0
62
63
 
63
- @outgoing.each do |_, link|
64
- total_success += link.success
65
- total_transfers += link.transfers
66
- end
64
+ @outgoing.each do |_, link|
65
+ total_success += link.success
66
+ total_transfers += link.transfers
67
+ end
67
68
 
68
- @outgoing.each { |_, link| link.trust = link.success / total_transfers }
69
- @outgoing.each do |target, link|
70
- [target.outgoing, target.implicit].each do |links|
71
- links.each do |nextlinkedge|
72
- nextlinktarget = nextlinkedge[0]
73
- nextlink = nextlinkedge[1]
74
- next unless outgoing[nextlinktarget].nil?
69
+ @outgoing.each { |_, link| link.trust = link.success / total_transfers }
70
+ @outgoing.each do |target, link|
71
+ [target.outgoing, target.implicit].each do |links|
72
+ links.each do |nextlinkedge|
73
+ nextlinktarget = nextlinkedge[0]
74
+ nextlink = nextlinkedge[1]
75
+ next unless outgoing[nextlinktarget].nil?
75
76
 
76
- if implicit[nextlinktarget].nil? || implicit[nextlinktarget].trust < (link.trust * nextlink.trust)
77
- implicit[nextlinktarget] = Edge.new(
78
- link.trust * nextlink.trust,
79
- nextlink.success,
80
- nextlink.transfers
81
- )
77
+ if implicit[nextlinktarget].nil? || implicit[nextlinktarget].trust < (link.trust * nextlink.trust)
78
+ implicit[nextlinktarget] = Edge.new(
79
+ link.trust * nextlink.trust,
80
+ nextlink.success,
81
+ nextlink.transfers
82
+ )
83
+ end
82
84
  end
83
85
  end
84
86
  end