distribustream 0.2.1 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,11 +1,23 @@
1
1
  #--
2
2
  # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
- # All rights reserved. See COPYING for permissions.
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
4
16
  #
5
17
  # This source file is distributed as part of the
6
18
  # DistribuStream file transfer system.
7
19
  #
8
- # See http://distribustream.rubyforge.org/
20
+ # See http://distribustream.org/
9
21
  #++
10
22
 
11
23
  require 'rubygems'
@@ -1,16 +1,27 @@
1
1
  #--
2
2
  # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
- # All rights reserved. See COPYING for permissions.
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
4
16
  #
5
17
  # This source file is distributed as part of the
6
18
  # DistribuStream file transfer system.
7
19
  #
8
- # See http://distribustream.rubyforge.org/
20
+ # See http://distribustream.org/
9
21
  #++
10
22
 
11
23
  require 'rubygems'
12
24
  require 'eventmachine'
13
- require 'thread'
14
25
  require 'ipaddr'
15
26
 
16
27
  begin
@@ -40,9 +51,8 @@ module PDTP
40
51
  @connection_open
41
52
  end
42
53
 
43
- def initialize *args
54
+ def initialize(*args)
44
55
  user_data = nil
45
- @mutex = Mutex.new
46
56
  super
47
57
  end
48
58
 
@@ -85,7 +95,7 @@ module PDTP
85
95
  def receive_packet(packet)
86
96
  begin
87
97
  packet.chomp!
88
- @@log.debug "(#{remote_peer_id}) recv: " + packet
98
+ #@@log.debug "(#{remote_peer_id}) recv: " + packet
89
99
  message = JSON.parse(packet) rescue nil
90
100
  raise ProtocolError.new("JSON couldn't parse: #{packet}") if message.nil?
91
101
  Protocol.validate_message message
@@ -94,14 +104,16 @@ module PDTP
94
104
  hash_to_range command, options
95
105
  receive_message(command, options) if respond_to? :receive_message
96
106
  rescue ProtocolError => e
97
- @@log.warn "(#{remote_peer_id}) PROTOCOL ERROR: #{e.to_s}"
98
- @@log.debug e.backtrace.join("\n")
107
+ # FIXME Should likely be raised and handled higher
108
+ STDERR.write "(#{remote_peer_id}) PROTOCOL ERROR: #{e.to_s}\n"
109
+ STDERR.write e.backtrace.join("\n") + "\n"
99
110
  error_close_connection e.to_s
100
111
  rescue ProtocolWarn => e
101
112
  send_message :protocol_warn, :message => e.to_s
102
113
  rescue Exception => e
103
- puts "(#{remote_peer_id}) UNKNOWN EXCEPTION #{e.to_s}"
104
- puts e.backtrace.join("\n")
114
+ # FIXME Should likely be raised and handled higher
115
+ STDERR.write "(#{remote_peer_id}) UNKNOWN EXCEPTION #{e.to_s}\n"
116
+ STDERR.write e.backtrace.join("\n") + "\n"
105
117
  end
106
118
  end
107
119
 
@@ -155,12 +167,7 @@ module PDTP
155
167
  # Message format is a JSON array with the command (string) as the first entry
156
168
  # Second entry is an options hash/object
157
169
  message = [command.to_s, opts]
158
-
159
- #@mutex.synchronize do
160
- outstr = JSON.unparse(message) + "\n"
161
- @@log.debug "(#{remote_peer_id}) send: #{outstr.chomp}"
162
- send_packet outstr
163
- #end
170
+ send_packet JSON.unparse(message) + "\n"
164
171
  end
165
172
 
166
173
  #called by EventMachine when a connection is closed
@@ -1,68 +1,143 @@
1
1
  #--
2
2
  # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
- # All rights reserved. See COPYING for permissions.
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
4
16
  #
5
17
  # This source file is distributed as part of the
6
18
  # DistribuStream file transfer system.
7
19
  #
8
- # See http://distribustream.rubyforge.org/
20
+ # See http://distribustream.org/
9
21
  #++
10
22
 
11
23
  require 'rubygems'
12
24
  require 'eventmachine'
13
25
  require 'mongrel'
26
+ require 'logger'
27
+ require 'socket'
14
28
 
15
29
  require File.dirname(__FILE__) + '/common'
30
+ require File.dirname(__FILE__) + '/common/http_server'
16
31
  require File.dirname(__FILE__) + '/server/dispatcher'
17
32
  require File.dirname(__FILE__) + '/server/file_service'
33
+ require File.dirname(__FILE__) + '/server/file_service_protocol'
18
34
  require File.dirname(__FILE__) + '/server/connection'
19
- require File.dirname(__FILE__) + '/server/stats_handler'
35
+ require File.dirname(__FILE__) + '/server/status_handler'
20
36
 
21
37
  module PDTP
22
38
  # PDTP::Server provides an interface for creating a PDTP server
23
39
  class Server
40
+ attr_reader :addr, :port
41
+
24
42
  # Create a new PDTP::Server which will listen on the given address and port
25
- def initialize(addr, port = 6086)
26
- @addr, @port = addr, port
43
+ def initialize(addr, pdtp_port = DEFAULT_PORT, http_port = pdtp_port + 1)
44
+ @orig_addr, @port, @http_port = addr, pdtp_port, http_port
27
45
 
28
- @dispatcher = PDTP::Server::Dispatcher.new
29
- @dispatcher.file_service = PDTP::Server::FileService.new
46
+ @addr = IPSocket.getaddress(@orig_addr)
47
+ @file_service = FileService.new
48
+ @dispatcher = Dispatcher.new self, @file_service
30
49
  end
31
50
 
32
51
  # Run a web server to display statistics on the given address and port
33
- def enable_stats_service(addr = nil, port = 6087)
34
- # Use the same address as the main server unless a different one was specified
35
- addr ||= @addr
36
-
37
- @stats_server = Mongrel::HttpServer.new addr, port
38
- @@log.info "Mongrel server listening on port: #{port}"
39
- @stats_server.register '/', PDTP::Server::StatsHandler.new(@dispatcher)
40
- @stats_server.run
52
+ def enable_status_page(path = '/status')
53
+ log "status at http://#{@addr}:#{@http_port}#{path}"
54
+ http_server.register path, StatusHandler.new(@dispatcher)
41
55
  end
42
56
 
43
57
  # Serve files from the given directory
44
58
  def enable_file_service(path, options = {})
45
- opts = {
46
- :chunk_size => 100000
59
+ opts = {
60
+ :chunk_size => 100000,
61
+ :vhost => @orig_addr
47
62
  }.merge(options)
48
63
 
49
- @dispatcher.file_service.root = path
50
- @dispatcher.file_service.default_chunk_size = opts[:chunk_size]
64
+ @file_service.root = path
65
+ @file_service.default_chunk_size = opts[:chunk_size]
66
+ @vhost = opts[:vhost]
67
+ @file_service_enabled = true
68
+ end
69
+
70
+ # Is there an internal file service available for this server?
71
+ def file_service_enabled?
72
+ @file_service_enabled
73
+ end
74
+
75
+ # Enable logging
76
+ def enable_logging(logger = nil)
77
+ @log = logger || default_logger
51
78
  end
52
79
 
53
80
  # Run the PDTP server event loop
54
81
  def run
55
- EventMachine::run do
56
- EventMachine::start_server(@addr, @port, PDTP::Server::Connection) do |connection|
57
- connection.server = @dispatcher
82
+ EventMachine.run do
83
+ EventMachine.start_server(@addr, @port, Connection) do |connection|
84
+ connection.dispatcher = @dispatcher
58
85
  connection.connection_completed
59
86
  end
87
+
88
+ # Enable the file service if #enable_file_service has been called
89
+ if file_service_enabled?
90
+ EventMachine.connect(@addr, @port, PDTP::Server::FileService::Protocol) do |c|
91
+ # In future versions of EventMachine we'll be able to pass these as parameters to EM.connect
92
+ base_path, listen_port, vhost, server = @file_service.root, @http_port, @vhost, http_server
93
+
94
+ c.instance_eval do
95
+ @base_path, @listen_port, @vhost, @http_server = base_path, listen_port, vhost, server
96
+ end
97
+ end
98
+ end
60
99
 
61
- @@log.info "accepting connections with ev=#{EventMachine::VERSION}"
62
- @@log.info "host=#{@host} port=#{@port}"
100
+ log "accepting connections on #{@addr}:#{@port}"
101
+
102
+ # Start Mongrel in Evented mode if it's been requested
103
+ http_server.run_evented if @http_server
63
104
 
64
- EventMachine::add_periodic_timer(2) { @dispatcher.clear_all_stalled_transfers }
105
+ EventMachine.add_periodic_timer(2) { @dispatcher.clear_all_stalled_transfers }
65
106
  end
66
107
  end
108
+
109
+ # Write a message to the server log
110
+ def log(message, type = :info)
111
+ return unless @log
112
+ @log.send type, message
113
+ end
114
+
115
+ # Write a debug message to the server log (ignored in quiet mode)
116
+ def debug(message)
117
+ log message, :debug
118
+ end
119
+
120
+ #########
121
+ protected
122
+ #########
123
+
124
+ def default_logger
125
+ STDERR.sync = true
126
+ logger = Logger.new STDERR
127
+ logger.datetime_format = "%H:%M:%S "
128
+ logger
129
+ end
130
+
131
+ def http_server
132
+ # Start a mongrel server on the specified port. If it isnt available, keep trying higher ports
133
+ # FIXME: Evented Mongrel won't throw an exception if the port is unavailable. This needs to
134
+ # be dealt with through EventMachine
135
+ # begin
136
+ @http_server ||= Mongrel::HttpServer.new @addr, @http_port
137
+ # rescue
138
+ # @http_port += 1
139
+ # retry
140
+ # end
141
+ end
67
142
  end
68
- end
143
+ end
@@ -1,11 +1,23 @@
1
1
  #--
2
2
  # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
- # All rights reserved. See COPYING for permissions.
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
4
16
  #
5
17
  # This source file is distributed as part of the
6
18
  # DistribuStream file transfer system.
7
19
  #
8
- # See http://distribustream.rubyforge.org/
20
+ # See http://distribustream.org/
9
21
  #++
10
22
 
11
23
  require File.dirname(__FILE__) + '/trust'
@@ -1,11 +1,23 @@
1
1
  #--
2
2
  # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
- # All rights reserved. See COPYING for permissions.
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
4
16
  #
5
17
  # This source file is distributed as part of the
6
18
  # DistribuStream file transfer system.
7
19
  #
8
- # See http://distribustream.rubyforge.org/
20
+ # See http://distribustream.org/
9
21
  #++
10
22
 
11
23
  require File.dirname(__FILE__) + '/../common/protocol'
@@ -13,22 +25,32 @@ require File.dirname(__FILE__) + '/../common/protocol'
13
25
  module PDTP
14
26
  class Server
15
27
  class Connection < PDTP::Protocol
16
- attr_accessor :server
28
+ attr_accessor :dispatcher
17
29
  attr_accessor :user_data
18
30
 
31
+ # Is this connection the file service?
32
+ def file_service?
33
+ @file_service
34
+ end
35
+
36
+ # Mark this connection as being a file service
37
+ def mark_as_file_service
38
+ @file_service = true
39
+ end
40
+
19
41
  def connection_completed
20
- raise(RuntimeError, 'server was never initialized') unless @server
21
- @server.connection_created self
42
+ raise(RuntimeError, 'server was never initialized') unless @dispatcher
43
+ @dispatcher.connection_created self
22
44
  end
23
45
 
24
46
  def connection_destroyed
25
- raise(RuntimeError, 'server was never initialized') unless @server
26
- @server.connection_destroyed self
47
+ raise(RuntimeError, 'server was never initialized') unless @dispatcher
48
+ @dispatcher.connection_destroyed self
27
49
  end
28
50
 
29
51
  def receive_message(command, message)
30
- raise(RuntimeError, 'server was never initialized') unless @server
31
- @server.dispatch_message command, message, self
52
+ raise(RuntimeError, 'server was never initialized') unless @dispatcher
53
+ @dispatcher.dispatch_message command, message, self
32
54
  end
33
55
  end
34
56
  end
@@ -1,50 +1,66 @@
1
1
  #--
2
2
  # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
- # All rights reserved. See COPYING for permissions.
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU General Public License
15
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
4
16
  #
5
17
  # This source file is distributed as part of the
6
18
  # DistribuStream file transfer system.
7
19
  #
8
- # See http://distribustream.rubyforge.org/
20
+ # See http://distribustream.org/
9
21
  #++
10
22
 
11
23
  require File.dirname(__FILE__) + '/../common/protocol'
12
- require File.dirname(__FILE__) + '/../common/common_init'
13
24
  require File.dirname(__FILE__) + '/file_service'
14
25
  require File.dirname(__FILE__) + '/client_info'
15
26
  require File.dirname(__FILE__) + '/transfer'
16
27
 
17
- require 'thread'
18
- require 'erb'
19
-
20
28
  module PDTP
21
29
  class Server
22
30
  # Core dispatching and control logic for PDTP servers
23
31
  class Dispatcher
24
32
  attr_reader :connections
25
- attr_accessor :file_service
26
- def initialize
33
+
34
+ def initialize(server, file_service)
35
+ @server = server
36
+ @file_service = file_service
27
37
  @connections = []
28
38
  @used_client_ids = {} #keeps a list of client ids in use, they must be unique
29
39
  @updated_clients = {} #a set of clients that have been modified and need transfers spawned
30
- @stats_mutex=Mutex.new
31
40
  end
32
41
 
33
42
  #called by pdtp_protocol when a connection is created
34
43
  def connection_created(connection)
35
- @stats_mutex.synchronize do
36
- @@log.info "Client connected: #{connection.get_peer_info.inspect}"
37
- connection.user_data = ClientInfo.new
38
- @connections << connection
44
+ addr, port = connection.get_peer_info
45
+
46
+ if @server.file_service_enabled? and not @seen_file_service and addr == @server.addr
47
+ #display file service greeting when we see it connect
48
+ @server.log "file service running at #{addr}:#{@server.instance_eval { @http_port }}"
49
+ @seen_file_service = true
50
+ connection.mark_as_file_service
51
+ else
52
+ #display greeting for normal client
53
+ @server.log "client connected: #{connection.get_peer_info.inspect}"
39
54
  end
55
+
56
+ connection.user_data = ClientInfo.new
57
+ @connections << connection
40
58
  end
41
59
 
42
60
  #called by pdtp_protocol when a connection is destroyed
43
61
  def connection_destroyed(connection)
44
- @stats_mutex.synchronize do
45
- @@log.info "Client connection closed: #{connection.get_peer_info.inspect}"
46
- @connections.delete connection
47
- end
62
+ @server.log "client disconnected: #{connection.get_peer_info.inspect}"
63
+ @connections.delete connection
48
64
  end
49
65
 
50
66
  # returns the ClientInfo object associated with this connection
@@ -80,12 +96,6 @@ module PDTP
80
96
  ) if send_response
81
97
  end
82
98
 
83
- #outstr="#{@ids[transfer.giver]}->#{@ids[transfer.taker]} transfer completed: #{transfer}"
84
- #outstr=outstr+" t->g=#{c1.trust.weight(c2.trust)} g->t=#{c2.trust.weight(c1.trust)}"
85
- #outstr=outstr+"sent_by: "+ ( connection==transfer.taker ? "taker" : "giver" )
86
- #outstr=outstr+" success=#{success} "
87
- #@@log.debug(outstr)
88
-
89
99
  #remove this transfer from whoever sent it
90
100
  client_info(connection).transfers.delete(transfer.transfer_id)
91
101
  @updated_clients[connection]=true #flag this client for transfer creation
@@ -206,13 +216,6 @@ module PDTP
206
216
  false
207
217
  end
208
218
 
209
- #called by pdtp_protocol for each message that comes in from the wire
210
- def dispatch_message(command, message, connection)
211
- @stats_mutex.synchronize do
212
- dispatch_message_needslock command, message, connection
213
- end
214
- end
215
-
216
219
  #creates new transfers for all clients that have been updated
217
220
  def spawn_all_transfers
218
221
  while @updated_clients.size > 0 do
@@ -240,7 +243,7 @@ module PDTP
240
243
  end
241
244
 
242
245
  #handles all incoming messages from clients
243
- def dispatch_message_needslock(command, message, connection)
246
+ def dispatch_message(command, message, connection)
244
247
  # store the command in the message hash
245
248
  message["type"] = command
246
249
 
@@ -261,7 +264,7 @@ module PDTP
261
264
  client_info(connection).listen_port = message["listen_port"]
262
265
  client_info(connection).client_id = cid
263
266
  when "ask_info"
264
- info = file_service.get_info(message["url"])
267
+ info = @file_service.get_info(message["url"])
265
268
  response = { :url => message["url"] }
266
269
  unless info.nil?
267
270
  response[:size] = info.file_size
@@ -277,7 +280,7 @@ module PDTP
277
280
  transfer_id=Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
278
281
  ok = !!client_info(connection).transfers[transfer_id]
279
282
  client_info(connection).transfers[transfer_id].verification_asked=true if ok
280
- @@log.debug "AskVerify not ok: id=#{transfer_id}" unless ok
283
+ @server.debug "AskVerify not ok: id=#{transfer_id}" unless ok
281
284
  connection.send_message(:tell_verify,
282
285
  :url => message["url"],
283
286
  :peer_id => message["peer_id"],
@@ -293,7 +296,7 @@ module PDTP
293
296
  message["range"]
294
297
  )
295
298
  transfer=client_info(connection).transfers[transfer_id]
296
- @@log.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
299
+ @server.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
297
300
  if transfer
298
301
  transfer_completed(transfer,connection,message["hash"])
299
302
  else
@@ -312,59 +315,6 @@ module PDTP
312
315
  #return "#{get_id(c)}: #{host}:#{port}"
313
316
  client_info(c).client_id
314
317
  end
315
-
316
- def generate_html_stats
317
- @stats_mutex.synchronize { generate_html_stats_needslock }
318
- end
319
-
320
- #builds an html page with information about the server's internal workings
321
- def generate_html_stats_needslock
322
- s = ERB.new <<EOF
323
- <html><head><title>DistribuStream Statistics</title></head>
324
- <body>
325
- <h1>DistribuStream Statistics</h1>
326
- Time=<%= Time.new %><br> Connected Clients=<%= @connections.size %>
327
- <center><table border=1>
328
- <tr><th>Client</th><th>Transfers</th><th>Files</th></tr>
329
- <% @connections.each do |c| %>
330
- <tr><td>
331
- <% host, port = c.get_peer_info %>
332
- <%= connection_name(c) %><br><%= host %>:<%= port %>
333
- </td>
334
- <td>
335
- <%
336
- client_info(c).transfers.each do |key,t|
337
- if c==t.giver
338
- type="UP: "
339
- peer=t.taker
340
- else
341
- type="DOWN: "
342
- peer=t.giver
343
- end
344
- %>
345
- <%= type %> id=<%= t.transfer_id %><br>
346
- <%
347
- end
348
- %>
349
- </td>
350
- <td>
351
- <%
352
- client_info(c).chunk_info.get_file_stats.each do |fs|
353
- %>
354
- <%= fs.url %> size=<%= fs.file_chunks %> req=<%= fs.chunks_requested %>
355
- prov=<%= fs.chunks_provided %> transf=<%= fs.chunks_transferring %><br>
356
- <%
357
- end
358
- %>
359
- </td></tr>
360
- <%
361
- end
362
- %>
363
- </table>
364
- </body></html>
365
- EOF
366
- s.result binding
367
- end
368
318
  end
369
319
  end
370
320
  end