distribustream 0.2.1 → 0.3.0

data/lib/pdtp/client/file_buffer.rb

@@ -1,11 +1,23 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 require 'rubygems'

data/lib/pdtp/client/file_service.rb

@@ -1,18 +1,25 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
-require 'rubygems'
-require 'mongrel'
-require 'uri'
-require 'pathname'
-
 require File.dirname(__FILE__) + '/../common/file_service.rb'
 require File.dirname(__FILE__) + '/file_buffer.rb'    
 
@@ -23,25 +30,24 @@ module PDTP
     class FileInfo < PDTP::FileInfo
       def initialize(filename, io = nil)
         @buffer = FileBuffer.new io || open(filename, 'w')
-        @lock = Mutex.new
       end
       
       # Write data into buffer starting at start_pos 
       def write(start_pos,data)
-        @lock.synchronize { @buffer.write start_pos, data }
+        @buffer.write start_pos, data
       end
 
       # Read a range of data out of buffer. Takes a ruby Range object
       def read(range)
         begin
-          @lock.synchronize { @buffer.read range }
+          @buffer.read range
         rescue nil
         end
       end
 
       # Return the number of bytes currently stored
       def bytes_downloaded
-        @lock.synchronize { @buffer.bytes_stored }
+        @buffer.bytes_stored
       end
     end
 

data/lib/pdtp/client/http_client.rb

@@ -1,34 +1,33 @@
-# $Id: httpclient.rb 518 2007-08-30 10:17:02Z blackhedd $
-#
-# Author:: Francis Cianfrocca (gmail: blackhedd)
-# Homepage::  http://rubyeventmachine.com
-# Date:: 16 July 2006
-# 
-# See EventMachine and EventMachine::Connection for documentation and
-# usage examples.
-#
-#----------------------------------------------------------------------------
-#
+#--
 # Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
 # Gmail: blackhedd
+# Modifications (C) 2007 ClickCaster, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
 # 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of either: 1) the GNU General Public License
-# as published by the Free Software Foundation; either version 2 of the
-# License, or (at your option) any later version; or 2) Ruby's License.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
-# See the file COPYING for complete licensing information.
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
 #
-#---------------------------------------------------------------------------
+# See http://distribustream.org/
 #
-# 
-
 # This version of HttpClient has been modified for use in DistribuStream
 # Notable changes:
 # - Moved into PDTP::Client namespace
 # - HTTP/1.1 switched to HTTP/1.0
 # - Introduced support for HTTP ranges
 # - Support for X headers
+#++
 
 module PDTP
   class Client

data/lib/pdtp/client/http_handler.rb

@@ -1,11 +1,23 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 require 'rubygems'
@@ -27,12 +39,12 @@ module PDTP
       # This method is called after a connection to the server
       # has been successfully established.
       def connection_created(connection)
-        @@log.debug("[mongrel] Opened connection...");
+        #@@log.debug("[mongrel] Opened connection...");
       end
 
       # This method is called when the server connection is destroyed
       def connection_destroyed(connection)
-        @@log.debug("[mongrel] Closed connection...")
+        #@@log.debug("[mongrel] Closed connection...")
       end
 
       # Returns a transfer object if the given connection is a peer associated with 
@@ -42,11 +54,10 @@ module PDTP
         nil
       end
 
-      # This method is called when an HTTP request is received. It is called in 
-      # a separate thread, one for each request.
-      def process(request,response)
+      # This method is called when an HTTP request is received.
+      def process(request,response)        
         begin
-          @@log.debug "Creating Transfer::Listener"
+          #@@log.debug "Creating Transfer::Listener"
           transfer = Transfer::Listener.new(
             @client.connection,
             request, 
@@ -54,9 +65,8 @@ module PDTP
             client.file_service
           )
 
-          #Needs to be locked because multiple threads could attempt to append a transfer at once
-          @client.lock.synchronize { @client.transfers << transfer }
-          transfer.handle_header 
+          @client.transfers << transfer
+          transfer.handle_header
         rescue Exception => e
           raise e if transfer.nil?
           transfer.write_http_exception(e)

data/lib/pdtp/client/transfer.rb

@@ -1,15 +1,25 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
-require "thread"
-#require "net/http"
 require "uri"
 require "digest/sha2"
 
@@ -92,20 +102,16 @@ module PDTP
               out.write(e.to_s + "\n\n" + e.backtrace.join("\n") )
             end
           else
-            @@log.info("MONGREL SERVER ERROR: exception:" + e.to_s+"\n\n"+e.backtrace.join("\n"))
+            STDERR.write "MONGREL SERVER ERROR: exception: " + e.to_s+"\n\n"+e.backtrace.join("\n")+"\n" 
             @response.start(500) do |head,out|
-              out.write("Server error, unknown exception:"+e.to_s + "\n\n" + e.backtrace.join("\n") )
+              out.write("Server error, unknown exception: "+e.to_s + "\n\n" + e.backtrace.join("\n") )
             end
           end
         end
 
         # Parse the HTTP header and ask for verification of transfer
-        # Thread is stopped after asking for verification and will
-        # be restarted when verification arrives
         def handle_header
-          @thread=Thread.current
-
-          @@log.debug "params=#{@request.params.inspect}"    
+          #@@log.debug "params=#{@request.params.inspect}"    
 
           @method=@request.params["REQUEST_METHOD"].downcase   
           @peer=@request.params["REMOTE_ADDR"]
@@ -124,33 +130,24 @@ module PDTP
           raise HTTPException.new(400, "Missing Range header") if @byte_range.nil?
 
           send_ask_verify_message
-          Thread.stop
-          after_verification
+          @response.pending = true
         end
 
         # Called after receiving verification message from the server
-        # Set the authorized status and restart the thread
-        # This throws us into after_verification
         def tell_verify(authorized)
-          @authorized=authorized
-          @thread.run
-        end
-
-        # Perform the transfer if verification was successful
-        def after_verification
           #check if the server authorized us
-          unless @authorized
+          unless authorized
             raise HTTPException.new(403,"Forbidden: the server did not authorize this transfer")  
           end
 
           info = @file_service.get_info(@url)
           if @method == "put"
             #we are the taker
-            @@log.debug("Body Downloaded: url=#{@url} range=#{@byte_range} peer=#{@peer}")      
+            #@@log.debug("Body Downloaded: url=#{@url} range=#{@byte_range} peer=#{@peer}")      
 
             @file_service.set_info(FileInfo.new) if info.nil? 
-            info.write(@byte_range.first, @request.body.read)
-            @hash=Digest::SHA256.hexdigest(res.body) rescue nil
+            info.write @byte_range.first, @request.body.read
+            @hash = Digest::SHA256.hexdigest(res.body) rescue nil
 
             # Stock HTTP OK response
             @response.start(200) do |head,out| 
@@ -158,7 +155,7 @@ module PDTP
           elsif @method=="get"
             #we are the giver
             raise HTTPException.new(404,"File not found: #{@url}") if info.nil?
-            data=info.read(@byte_range)
+            data = info.read @byte_range
             raise HTTPException.new(416,"Invalid range: #{@byte_range.inspect}") if data.nil?   
 
             #Request was GET, so now we need to send the data
@@ -167,8 +164,11 @@ module PDTP
               head['Content-Range'] = "bytes #{@byte_range.first}-#{@byte_range.last}/*"
               #FIXME must include a DATE header according to http
 
-              out.write(data)
+              out.write data
             end
+            
+            # Call response.finished ourselves since we left the response pending
+            @response.finished
           else
             raise HTTPException.new(405,"Invalid method: #{@method}")
           end
@@ -220,11 +220,11 @@ module PDTP
             hash = nil
             
             if @method == 'get' and response[:status] == 206
-              @@log.debug("Body Downloaded: url=#{@url} range=#{@byte_range} peer=#{@peer}:#{@port}")
+              #@@log.debug("Body Downloaded: url=#{@url} range=#{@byte_range} peer=#{@peer}:#{@port}")
               info.write(@byte_range.first,response[:content])
               hash = Digest::SHA256.hexdigest(response[:content]) rescue nil
             elsif @method == 'put' and response[:status] == 200
-              @@log.debug("Body Uploaded: url=#{@url} range=#{@byte_range} peer=#{@peer}:#{@port}")
+              #@@log.debug("Body Uploaded: url=#{@url} range=#{@byte_range} peer=#{@peer}:#{@port}")
             else
               raise RuntimeError, "Invalid method/status combination: #{@method} #{response[:status]}"
             end

data/lib/pdtp/common.rb

@@ -1,15 +1,30 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 # Namespace for all PDTP components
 module PDTP
-  PDTP::VERSION = '0.2.1' unless defined? PDTP::VERSION
+  PDTP::VERSION = '0.3.0' unless defined? PDTP::VERSION
   def self.version() VERSION end
-end
+
+  PDTP::DEFAULT_PORT = 6086 unless defined? PDTP::DEFAULT_PORT
+  def self.default_port() DEFAULT_PORT end
+end

data/lib/pdtp/common/file_service.rb

@@ -1,11 +1,23 @@
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # 
 # This source file is distributed as part of the 
 # DistribuStream file transfer system.
 #
-# See http://distribustream.rubyforge.org/
+# See http://distribustream.org/
 #++
 
 module PDTP

data/lib/pdtp/common/http_server.rb

@@ -0,0 +1,217 @@
+# This module rewrites pieces of the very good Mongrel web server in
+# order to change it from a threaded application to an event based
+# application running inside an EventMachine event loop.  It should
+# be compatible with the existing Mongrel handlers for Rails,
+# Camping, Nitro, etc....
+
+require 'rubygems'
+require 'eventmachine'
+require 'mongrel'
+
+module Mongrel
+  class Protocol < EventMachine::Connection
+    def post_init
+      @parser = HttpParser.new
+      @params = HttpParams.new
+      @nparsed = 0
+      @request = nil
+      @request_len = nil
+      @linebuffer = ''
+    end
+
+    def receive_data data
+      @linebuffer << data
+      @nparsed = @parser.execute(@params, @linebuffer, @nparsed) unless @parser.finished?
+      if @parser.finished?
+        if @request_len.nil?
+          @request_len = @params[::Mongrel::Const::CONTENT_LENGTH].to_i
+          script_name, path_info, handlers = ::Mongrel::HttpServer::Instance.classifier.resolve(@params[::Mongrel::Const::REQUEST_PATH])
+          if handlers
+            @params[::Mongrel::Const::PATH_INFO] = path_info
+            @params[::Mongrel::Const::SCRIPT_NAME] = script_name
+            @params[::Mongrel::Const::REMOTE_ADDR] = @params[::Mongrel::Const::HTTP_X_FORWARDED_FOR] || ::Socket.unpack_sockaddr_in(get_peername)[1]
+            @notifiers = handlers.select { |h| h.request_notify }
+          end
+          if @request_len > ::Mongrel::Const::MAX_BODY
+            new_buffer = Tempfile.new(::Mongrel::Const::MONGREL_TMP_BASE)
+            new_buffer.binmode
+            new_buffer << @linebuffer[@nparsed..-1]
+            @linebuffer = new_buffer
+          else
+            @linebuffer = StringIO.new(@linebuffer[@nparsed..-1])
+            @linebuffer.pos = @linebuffer.length
+          end
+        end
+        if @linebuffer.length >= @request_len
+          @linebuffer.rewind
+          ::Mongrel::HttpServer::Instance.process_http_request(@params,@linebuffer,self)
+          @linebuffer.delete if Tempfile === @linebuffer
+        end
+      elsif @linebuffer.length > ::Mongrel::Const::MAX_HEADER
+        close_connection
+        raise ::Mongrel::HttpParserError.new("HEADER is longer than allowed, aborting client early.")
+      end
+    rescue ::Mongrel::HttpParserError
+      if $mongrel_debug_client
+        STDERR.puts "#{Time.now}: BAD CLIENT (#{params[Const::HTTP_X_FORWARDED_FOR] || client.peeraddr.last}): #$!"
+        STDERR.puts "#{Time.now}: REQUEST DATA: #{data.inspect}\n---\nPARAMS: #{params.inspect}\n---\n"
+      end
+      close_connection
+    rescue Exception => e
+      close_connection
+      raise e
+    end
+
+    def write data
+      send_data data
+    end
+
+    def closed?
+      false
+    end
+
+  end
+
+  class HttpServer
+    def initialize(host, port, num_processors = 950, throttle = 0, timeout = 60)
+      @socket = nil
+      @classifier = Mongrel::URIClassifier.new
+      @host = host
+      @port = port
+      @workers = ThreadGroup.new
+      @timeout = timeout
+      @num_processors = num_processors
+      @death_time = 60
+      self.class.const_set(:Instance,self)
+    end
+
+    def run
+      trap('INT') { raise StopServer }
+      trap('TERM') { raise StopServer }
+      @acceptor = Thread.new do
+        EventMachine.run do
+          begin
+            run_evented
+          rescue StopServer
+            EventMachine.stop_event_loop
+          end
+        end
+      end
+    end
+    
+    def run_evented
+      EventMachine.start_server @host, @port, Mongrel::Protocol
+    end
+
+    def process_http_request(params,linebuffer,client)
+      if not params[Const::REQUEST_PATH]
+        uri = URI.parse(params[Const::REQUEST_URI])
+        params[Const::REQUEST_PATH] = uri.request_uri
+      end
+
+      raise "No REQUEST PATH" if not params[Const::REQUEST_PATH]
+
+      script_name, path_info, handlers = @classifier.resolve(params[Const::REQUEST_PATH])
+
+      if handlers
+        notifiers = handlers.select { |h| h.request_notify }
+        request = HttpRequest.new(params, linebuffer, notifiers)
+
+        # request is good so far, continue processing the response
+        response = HttpResponse.new(client)
+
+        # Process each handler in registered order until we run out or one finalizes the response.
+        dispatch_to_handlers(handlers,request,response)
+
+        # And finally, if nobody closed the response off, we finalize it.
+        unless response.done
+          response.finished unless response.pending
+        else
+          response.close_connection_after_writing
+        end
+      else
+        # Didn't find it, return a stock 404 response.
+        client.send_data(Const::ERROR_404_RESPONSE)
+        client.close_connection_after_writing
+      end
+    end
+
+    def dispatch_to_handlers(handlers,request,response)
+      handlers.each do |handler|
+        handler.process(request, response)
+        break if response.done
+      end
+    end
+  end
+
+  class HttpRequest
+    def initialize(params, linebuffer, dispatchers)
+      @params = params
+      @dispatchers = dispatchers
+      @body = linebuffer
+    end
+  end
+
+  class HttpResponse
+    # A flag to prevent response.finished from being called automatically
+    attr_accessor :pending
+    
+    def send_file(path, small_file = false)
+      File.open(path, "rb") do |f|
+        while chunk = f.read(Const::CHUNK_SIZE) and chunk.length > 0
+          begin
+            write(chunk)
+          rescue Object => exc
+            break
+          end
+        end
+      end
+      @body_sent = true
+    end
+
+    def write(data)
+      @socket.send_data data
+    end
+
+    def close_connection_after_writing
+      @socket.close_connection_after_writing
+    end
+
+    def socket_error(details)
+      @socket.close_connection
+      done = true
+      raise details
+    end
+
+    def finished
+      send_status
+      send_header
+      send_body
+      @socket.close_connection_after_writing
+    end
+  end
+
+  class Configurator
+    # This version fixes a bug in the regular Mongrel version by adding
+    # initialization of groups.
+    def change_privilege(user, group)
+      if user and group
+        log "Initializing groups for {#user}:{#group}."
+        Process.initgroups(user,Etc.getgrnam(group).gid)
+      end
+
+      if group
+        log "Changing group to #{group}."
+        Process::GID.change_privilege(Etc.getgrnam(group).gid)
+      end
+
+      if user
+        log "Changing user to #{user}."
+        Process::UID.change_privilege(Etc.getpwnam(user).uid)
+      end
+    rescue Errno::EPERM
+      log "FAILED to change user:group #{user}:#{group}: #$!"
+      exit 1
+    end
+  end
+end