distribustream 0.2.1 → 0.3.0

data/lib/pdtp/common/length_prefix_protocol.rb

@@ -1,11 +1,23 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 require 'rubygems'

data/lib/pdtp/common/protocol.rb

@@ -1,16 +1,27 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 require 'rubygems'
 require 'eventmachine'
-require 'thread'
 require 'ipaddr'
 
 begin
@@ -40,9 +51,8 @@ module PDTP
       @connection_open
     end
 
-    def initialize *args
+    def initialize(*args)
       user_data = nil
-      @mutex = Mutex.new
       super
     end
 
@@ -85,7 +95,7 @@ module PDTP
     def receive_packet(packet)
       begin
         packet.chomp!
-        @@log.debug "(#{remote_peer_id}) recv: " + packet
+        #@@log.debug "(#{remote_peer_id}) recv: " + packet
         message = JSON.parse(packet) rescue nil
         raise ProtocolError.new("JSON couldn't parse: #{packet}") if message.nil?
         Protocol.validate_message message
@@ -94,14 +104,16 @@ module PDTP
         hash_to_range command, options
         receive_message(command, options) if respond_to? :receive_message
       rescue ProtocolError => e
-        @@log.warn "(#{remote_peer_id}) PROTOCOL ERROR: #{e.to_s}"
-        @@log.debug e.backtrace.join("\n")
+        # FIXME Should likely be raised and handled higher
+        STDERR.write "(#{remote_peer_id}) PROTOCOL ERROR: #{e.to_s}\n"
+        STDERR.write e.backtrace.join("\n") + "\n"
         error_close_connection e.to_s
       rescue ProtocolWarn => e
         send_message :protocol_warn, :message => e.to_s
       rescue Exception => e
-        puts "(#{remote_peer_id}) UNKNOWN EXCEPTION #{e.to_s}"
-        puts e.backtrace.join("\n")
+        # FIXME Should likely be raised and handled higher
+        STDERR.write "(#{remote_peer_id}) UNKNOWN EXCEPTION #{e.to_s}\n"
+        STDERR.write e.backtrace.join("\n") + "\n"
       end
     end
 
@@ -155,12 +167,7 @@ module PDTP
       # Message format is a JSON array with the command (string) as the first entry
       # Second entry is an options hash/object
       message = [command.to_s, opts]
-      
-      #@mutex.synchronize do
-        outstr = JSON.unparse(message) + "\n"
-        @@log.debug "(#{remote_peer_id}) send: #{outstr.chomp}"
-        send_packet outstr  
-      #end
+      send_packet JSON.unparse(message) + "\n"  
     end
 
     #called by EventMachine when a connection is closed

data/lib/pdtp/server.rb

@@ -1,68 +1,143 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 require 'rubygems'
 require 'eventmachine'
 require 'mongrel'
+require 'logger'
+require 'socket'
 
 require File.dirname(__FILE__) + '/common'
+require File.dirname(__FILE__) + '/common/http_server'
 require File.dirname(__FILE__) + '/server/dispatcher'
 require File.dirname(__FILE__) + '/server/file_service'
+require File.dirname(__FILE__) + '/server/file_service_protocol'
 require File.dirname(__FILE__) + '/server/connection'
-require File.dirname(__FILE__) + '/server/stats_handler'
+require File.dirname(__FILE__) + '/server/status_handler'
 
 module PDTP
   # PDTP::Server provides an interface for creating a PDTP server
   class Server
+    attr_reader :addr, :port
+    
     # Create a new PDTP::Server which will listen on the given address and port
-    def initialize(addr, port = 6086)
-      @addr, @port = addr, port
+    def initialize(addr, pdtp_port = DEFAULT_PORT, http_port = pdtp_port + 1)
+      @orig_addr, @port, @http_port = addr, pdtp_port, http_port
       
-      @dispatcher = PDTP::Server::Dispatcher.new
-      @dispatcher.file_service = PDTP::Server::FileService.new
+      @addr = IPSocket.getaddress(@orig_addr)
+      @file_service = FileService.new
+      @dispatcher = Dispatcher.new self, @file_service
     end
     
     # Run a web server to display statistics on the given address and port 
-    def enable_stats_service(addr = nil, port = 6087)
-      # Use the same address as the main server unless a different one was specified
-      addr ||= @addr
-      
-      @stats_server = Mongrel::HttpServer.new addr, port
-      @@log.info "Mongrel server listening on port: #{port}"
-      @stats_server.register '/', PDTP::Server::StatsHandler.new(@dispatcher)
-      @stats_server.run
+    def enable_status_page(path = '/status')
+      log "status at http://#{@addr}:#{@http_port}#{path}"
+      http_server.register path, StatusHandler.new(@dispatcher)
     end
     
     # Serve files from the given directory
     def enable_file_service(path, options = {})
-      opts = {
-        :chunk_size => 100000
+      opts = { 
+        :chunk_size => 100000,
+        :vhost => @orig_addr
       }.merge(options)
       
-      @dispatcher.file_service.root = path
-      @dispatcher.file_service.default_chunk_size = opts[:chunk_size]
+      @file_service.root = path
+      @file_service.default_chunk_size = opts[:chunk_size]
+      @vhost = opts[:vhost]
+      @file_service_enabled = true
+    end
+    
+    # Is there an internal file service available for this server?
+    def file_service_enabled?
+      @file_service_enabled
+    end
+    
+    # Enable logging
+    def enable_logging(logger = nil)
+      @log = logger || default_logger
     end
     
     # Run the PDTP server event loop
     def run
-      EventMachine::run do
-        EventMachine::start_server(@addr, @port, PDTP::Server::Connection) do |connection|
-          connection.server = @dispatcher    
+      EventMachine.run do
+        EventMachine.start_server(@addr, @port, Connection) do |connection|
+          connection.dispatcher = @dispatcher    
           connection.connection_completed
         end
+        
+        # Enable the file service if #enable_file_service has been called
+        if file_service_enabled?
+          EventMachine.connect(@addr, @port, PDTP::Server::FileService::Protocol) do |c|
+            # In future versions of EventMachine we'll be able to pass these as parameters to EM.connect
+            base_path, listen_port, vhost, server = @file_service.root, @http_port, @vhost, http_server
+            
+            c.instance_eval do
+              @base_path, @listen_port, @vhost, @http_server = base_path, listen_port, vhost, server
+            end
+          end
+        end
 
-        @@log.info "accepting connections with ev=#{EventMachine::VERSION}"
-        @@log.info "host=#{@host}  port=#{@port}"
+        log "accepting connections on #{@addr}:#{@port}"
+        
+        # Start Mongrel in Evented mode if it's been requested
+        http_server.run_evented if @http_server
 
-        EventMachine::add_periodic_timer(2) { @dispatcher.clear_all_stalled_transfers }
+        EventMachine.add_periodic_timer(2) { @dispatcher.clear_all_stalled_transfers }
       end
     end
+    
+    # Write a message to the server log
+    def log(message, type = :info)
+      return unless @log
+      @log.send type, message
+    end
+    
+    # Write a debug message to the server log (ignored in quiet mode)
+    def debug(message)
+      log message, :debug
+    end
+    
+    #########
+    protected
+    #########
+    
+    def default_logger
+      STDERR.sync = true
+      logger = Logger.new STDERR
+      logger.datetime_format = "%H:%M:%S "
+      logger
+    end
+    
+    def http_server
+      # Start a mongrel server on the specified port.  If it isnt available, keep trying higher ports
+      # FIXME: Evented Mongrel won't throw an exception if the port is unavailable.  This needs to
+      # be dealt with through EventMachine
+#      begin
+        @http_server ||= Mongrel::HttpServer.new @addr, @http_port
+#      rescue
+#        @http_port += 1
+#        retry
+#      end
+    end
   end
-end
+end

data/lib/pdtp/server/client_info.rb

@@ -1,11 +1,23 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 require File.dirname(__FILE__) + '/trust'

data/lib/pdtp/server/connection.rb

@@ -1,11 +1,23 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 require File.dirname(__FILE__) + '/../common/protocol'
@@ -13,22 +25,32 @@ require File.dirname(__FILE__) + '/../common/protocol'
 module PDTP
   class Server
     class Connection < PDTP::Protocol
-      attr_accessor :server
+      attr_accessor :dispatcher
       attr_accessor :user_data
       
+      # Is this connection the file service?
+      def file_service?
+        @file_service
+      end
+      
+      # Mark this connection as being a file service
+      def mark_as_file_service
+        @file_service = true
+      end
+      
       def connection_completed
-        raise(RuntimeError, 'server was never initialized') unless @server
-        @server.connection_created self
+        raise(RuntimeError, 'server was never initialized') unless @dispatcher
+        @dispatcher.connection_created self
       end
       
       def connection_destroyed
-        raise(RuntimeError, 'server was never initialized') unless @server
-        @server.connection_destroyed self
+        raise(RuntimeError, 'server was never initialized') unless @dispatcher
+        @dispatcher.connection_destroyed self
       end
       
       def receive_message(command, message)
-        raise(RuntimeError, 'server was never initialized') unless @server
-        @server.dispatch_message command, message, self
+        raise(RuntimeError, 'server was never initialized') unless @dispatcher
+        @dispatcher.dispatch_message command, message, self
       end
     end
   end

data/lib/pdtp/server/dispatcher.rb

@@ -1,50 +1,66 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 require File.dirname(__FILE__) + '/../common/protocol'
-require File.dirname(__FILE__) + '/../common/common_init'
 require File.dirname(__FILE__) + '/file_service'
 require File.dirname(__FILE__) + '/client_info'
 require File.dirname(__FILE__) + '/transfer'
 
-require 'thread'
-require 'erb'
-
 module PDTP
   class Server
     # Core dispatching and control logic for PDTP servers
     class Dispatcher
       attr_reader :connections
-      attr_accessor :file_service
-      def initialize
+      
+      def initialize(server, file_service)
+        @server = server
+        @file_service = file_service
         @connections = []
         @used_client_ids = {} #keeps a list of client ids in use, they must be unique
         @updated_clients = {} #a set of clients that have been modified and need transfers spawned
-        @stats_mutex=Mutex.new
       end
 
       #called by pdtp_protocol when a connection is created
       def connection_created(connection)
-        @stats_mutex.synchronize do
-          @@log.info "Client connected: #{connection.get_peer_info.inspect}"
-          connection.user_data = ClientInfo.new
-          @connections << connection 
+        addr, port = connection.get_peer_info
+        
+        if @server.file_service_enabled? and not @seen_file_service and addr == @server.addr
+          #display file service greeting when we see it connect
+          @server.log "file service running at #{addr}:#{@server.instance_eval { @http_port }}"
+          @seen_file_service = true
+          connection.mark_as_file_service
+        else
+          #display greeting for normal client
+          @server.log "client connected: #{connection.get_peer_info.inspect}"
         end
+        
+        connection.user_data = ClientInfo.new
+        @connections << connection 
       end  
 
       #called by pdtp_protocol when a connection is destroyed
       def connection_destroyed(connection)
-        @stats_mutex.synchronize do
-          @@log.info "Client connection closed: #{connection.get_peer_info.inspect}"
-          @connections.delete connection
-        end
+        @server.log "client disconnected: #{connection.get_peer_info.inspect}"
+        @connections.delete connection
       end
 
       # returns the ClientInfo object associated with this connection
@@ -80,12 +96,6 @@ module PDTP
           ) if send_response
         end
 
-        #outstr="#{@ids[transfer.giver]}->#{@ids[transfer.taker]} transfer completed: #{transfer}"
-        #outstr=outstr+" t->g=#{c1.trust.weight(c2.trust)} g->t=#{c2.trust.weight(c1.trust)}" 
-        #outstr=outstr+"sent_by: "+ ( connection==transfer.taker ? "taker" : "giver" )
-        #outstr=outstr+" success=#{success} "
-        #@@log.debug(outstr)
-
         #remove this transfer from whoever sent it
         client_info(connection).transfers.delete(transfer.transfer_id)
         @updated_clients[connection]=true #flag this client for transfer creation
@@ -206,13 +216,6 @@ module PDTP
         false
       end
 
-      #called by pdtp_protocol for each message that comes in from the wire
-      def dispatch_message(command, message, connection)
-        @stats_mutex.synchronize do
-          dispatch_message_needslock command, message, connection
-        end
-      end
-
       #creates new transfers for all clients that have been updated
       def spawn_all_transfers
         while @updated_clients.size > 0 do
@@ -240,7 +243,7 @@ module PDTP
       end
 
       #handles all incoming messages from clients
-      def dispatch_message_needslock(command, message, connection)
+      def dispatch_message(command, message, connection)
         # store the command in the message hash
         message["type"] = command
       
@@ -261,7 +264,7 @@ module PDTP
           client_info(connection).listen_port = message["listen_port"]
           client_info(connection).client_id = cid
         when "ask_info"
-          info = file_service.get_info(message["url"])
+          info = @file_service.get_info(message["url"])
           response = { :url => message["url"] }
           unless info.nil?
             response[:size] = info.file_size
@@ -277,7 +280,7 @@ module PDTP
           transfer_id=Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
           ok = !!client_info(connection).transfers[transfer_id]
           client_info(connection).transfers[transfer_id].verification_asked=true if ok
-          @@log.debug "AskVerify not ok: id=#{transfer_id}" unless ok
+          @server.debug "AskVerify not ok: id=#{transfer_id}" unless ok
           connection.send_message(:tell_verify,
             :url     => message["url"],
             :peer_id => message["peer_id"],
@@ -293,7 +296,7 @@ module PDTP
             message["range"]
           )
           transfer=client_info(connection).transfers[transfer_id]
-          @@log.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
+          @server.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
           if transfer
             transfer_completed(transfer,connection,message["hash"])
           else
@@ -312,59 +315,6 @@ module PDTP
         #return "#{get_id(c)}: #{host}:#{port}"
         client_info(c).client_id
       end
-
-      def generate_html_stats
-        @stats_mutex.synchronize { generate_html_stats_needslock }
-      end
-
-      #builds an html page with information about the server's internal workings
-      def generate_html_stats_needslock
-        s = ERB.new <<EOF
-<html><head><title>DistribuStream Statistics</title></head>
-<body>
-<h1>DistribuStream Statistics</h1>
-Time=<%= Time.new %><br> Connected Clients=<%= @connections.size %>
-<center><table border=1>
-<tr><th>Client</th><th>Transfers</th><th>Files</th></tr>
-<% @connections.each do |c| %>
-  <tr><td>
-  <% host, port = c.get_peer_info %>
-  <%= connection_name(c) %><br><%= host %>:<%= port %>
-  </td>
-  <td>
-  <%
-  client_info(c).transfers.each do |key,t|
-    if c==t.giver
-      type="UP: "
-      peer=t.taker
-    else
-      type="DOWN: "
-      peer=t.giver
-    end
-    %>
-    <%= type %> id=<%= t.transfer_id %><br>
-    <%
-  end
-  %>
-  </td>
-  <td>
-  <%
-  client_info(c).chunk_info.get_file_stats.each do |fs|
-    %>
-    <%= fs.url %> size=<%= fs.file_chunks %> req=<%= fs.chunks_requested %>
-    prov=<%= fs.chunks_provided %> transf=<%= fs.chunks_transferring %><br>    
-    <%
-  end      
-  %>
-  </td></tr>
-  <%
-end
-%>
-</table>
-</body></html>
-EOF
-        s.result binding
-      end
     end
   end
 end