distribustream 0.1.0

data/lib/pdtp/client/file_buffer.rb

@@ -0,0 +1,128 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+module PDTP
+  # Handle a file buffer, which may be written to and read from randomly
+  class FileBuffer
+    def initialize(io = nil)
+      @io = io
+      @written = 0
+      @entries = []
+    end
+
+    # Write data starting at start_pos. Overwrites any existing data in that block
+    def write(start_pos, data)
+      return if data.size == 0
+      
+      # create and entry and attempt to combine it with old entries
+      new_entry = Entry.new(start_pos,data)
+      
+      intersections = true
+      while intersections
+        intersections = false
+        @entries.each do |e|
+          if intersects?(new_entry, e)
+            new_entry = combine(e, new_entry)
+            @entries.delete(e)
+            intersections = true
+          end 
+        end
+      end
+
+      # Add entry to the local store
+      @entries << new_entry
+      
+      # Write contiguous blocks we receive to our internal IO cursor
+      if @io and start_pos == @written
+        data_begin = @written - new_entry.start_pos
+        bytes_written = @io.write(new_entry.data[data_begin..new_entry.data.length])
+        @written += bytes_written
+      else
+        bytes_written = data.size
+      end
+      
+      bytes_written
+    end
+
+    # Returns a string containing the desired data. 
+    # Returns nil if the data is not all there
+    def read(range)
+      return nil if range.first>range.last
+      current_byte=range.first
+
+      buffer = ''
+
+      while current_byte <= range.last do
+        # find an entry that contains this byte
+
+        found=false
+        @entries.each do |e|
+          if e.range.include?(current_byte)
+            internal_start=current_byte-e.start_pos #start position inside this entry's data
+            internal_end=(range.last<e.end_pos ? range.last : e.end_pos) - e.start_pos 
+            buffer << e.data[internal_start..internal_end]
+            current_byte+=internal_end-internal_start+1
+            found=true
+            break if current_byte>range.last
+          end
+        end
+        return nil if found==false
+      end   
+      
+      buffer
+    end
+
+    # Returns true if two entries intersect
+    def intersects?(entry1, entry2)
+      first, last = entry1.start_pos <= entry2.start_pos ? [entry1, entry2] : [entry2, entry1]
+      first.end_pos + 1 >= last.start_pos 
+    end
+
+    # Takes two Entries
+    # Returns nil if there is no intersection
+    # Returns the union if they intersect
+    def combine(old_entry, new_entry)
+      start = old_entry.start_pos < new_entry.start_pos ? old_entry.start_pos: new_entry.start_pos
+
+      stringio = StringIO.new
+      stringio.seek(old_entry.start_pos - start)
+      stringio.write(old_entry.data)
+      stringio.seek(new_entry.start_pos - start)
+      stringio.write(new_entry.data)
+      return Entry.new(start, stringio.string)    
+    end
+
+    # Return number of bytes currently in the buffer
+    def bytes_stored
+      bytes=0
+      @entries.each do |e|
+        bytes=bytes+e.data.size
+      end  
+      return bytes
+    end
+
+    # Container for an entry in the buffer
+    class Entry
+      def initialize(start_pos,data)
+        @start_pos,@data=start_pos,data
+      end
+
+      attr_accessor :start_pos, :data
+
+      def end_pos
+        @start_pos + data.length - 1
+      end
+
+      def range
+        Range.new(@start_pos,end_pos)
+      end
+    end
+  end
+end

data/lib/pdtp/client/file_buffer_spec.rb

@@ -0,0 +1,154 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require File.dirname(__FILE__) + '/file_buffer'
+
+describe PDTP::FileBuffer do
+  before(:each) do
+    @b = PDTP::FileBuffer.new
+  end
+
+  it "returns nil if read when empty" do
+    @b.bytes_stored.should == 0
+    @b.read(0..1).should == nil
+  end  
+end
+
+describe PDTP::FileBuffer, "with one entry" do
+  before(:each) do
+    @b = PDTP::FileBuffer.new
+    @b.write 0, 'hello'
+  end
+
+  it "calculates bytes stored correctly" do
+    @b.bytes_stored.should == 5
+  end
+
+  it "reads stored data correctly" do
+    @b.read(0..4).should == "hello"
+    @b.read(1..1).should == "e"
+    @b.read(-1..2).should == nil
+    @b.read(0..5).should == nil
+  end
+end
+
+describe PDTP::FileBuffer, "with two overlapping entries" do
+  before(:each) do
+    @b = PDTP::FileBuffer.new
+    @b.write(3,"hello")
+    @b.write(7,"World")
+  end
+  
+  it "calculates bytes stored correctly" do
+    @b.bytes_stored.should == 9
+  end
+
+  it "reads stored data correctly" do
+    @b.read(3..12).should == nil
+    @b.read(3..11).should == "hellWorld"
+    @b.read(3..1).should == nil
+    @b.read(2..4).should == nil
+  end
+
+end
+
+describe PDTP::FileBuffer, "with three overlapping entries" do
+  before(:each) do
+    @b = PDTP::FileBuffer.new
+    @b.write(3,"hello")
+    @b.write(7,"World")
+    @b.write(2,"123456789ABCDEF")
+  end
+
+  it "calculates bytes stored correctly" do
+    @b.bytes_stored.should == 15
+  end
+
+  it "reads stored data correctly" do
+    @b.read(2..16).should == "123456789ABCDEF"
+    @b.read(2..17).should == nil
+  end
+end
+
+describe PDTP::FileBuffer, "with two tangential entries" do
+  before(:each) do
+    @b = PDTP::FileBuffer.new
+    @b.write(3,"hello")
+    @b.write(8,"World")
+  end
+
+  it "calculates bytes stored correctly" do
+    @b.bytes_stored.should == 10
+  end
+  
+  it "reads stored data correctly" do
+    @b.read(3..12).should == "helloWorld"
+  end
+end
+
+describe PDTP::FileBuffer, "with a chain of overlapping entries" do
+  before(:each) do
+    @b = PDTP::FileBuffer.new
+    @b.write(3,"a123")
+    @b.write(4,"b4")
+    @b.write(0,"012c")
+  
+    #___a123
+    #___ab43
+    #012cb43
+
+  end
+
+  it "calculates bytes stored correctly" do
+    @b.bytes_stored.should == 7
+  end
+
+  it "reads stored data correctly" do
+    @b.read(0..6).should == "012cb43"
+    @b.read(3..6).should == "cb43"
+  end
+end
+
+describe PDTP::FileBuffer, "with an associated IO object" do
+  before(:each) do
+    @io = mock(:io)
+    @b = PDTP::FileBuffer.new @io
+  end
+  
+  it "writes received data to the IO object" do
+    @io.should_receive(:write).once.with('foo').and_return(3)
+    @b.write(0, "foo")
+  end
+  
+  it "writes successively received data to the IO object" do
+    @io.should_receive(:write).once.with('foo').and_return(3)
+    @b.write(0, "foo")
+    
+    @io.should_receive(:write).once.with('bar').and_return(3)
+    @b.write(3, "bar")
+    
+    @io.should_receive(:write).once.with('baz').and_return(3)
+    @b.write(6, "baz")
+  end
+  
+  it "reassembles single-byte out-of-order data and writes it to the IO object" do
+    @io.should_receive(:write).once.with('bar').and_return(3)
+    @b.write(1, 'a')
+    @b.write(2, 'r')
+    @b.write(0, 'b')
+  end
+  
+  it "reassembles multibyte out-of-order data and writes it to the IO object" do
+    @io.should_receive(:write).once.with('foobar').and_return(6)
+    @b.write(2, 'ob')
+    @b.write(4, 'ar')
+    @b.write(0, 'fo')
+  end
+end

data/lib/pdtp/client/file_service.rb

@@ -0,0 +1,60 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require 'uri'
+require 'pathname'
+require File.dirname(__FILE__) + '/../common/file_service.rb'
+require File.dirname(__FILE__) + '/file_buffer.rb'    
+
+module PDTP
+  class Client < Mongrel::HttpHandler
+    # The client specific file utilities. Most importantly, handling
+    # the data buffer.
+    class FileInfo < PDTP::FileInfo
+      def initialize(filename)
+        @buffer = FileBuffer.new open(filename, 'w')
+        @lock = Mutex.new
+      end
+      
+      # Write data into buffer starting at start_pos 
+      def write(start_pos,data)
+        @lock.synchronize { @buffer.write start_pos, data }
+      end
+
+      # Read a range of data out of buffer. Takes a ruby Range object
+      def read(range)
+        begin
+          @lock.synchronize { @buffer.read range }
+        rescue nil
+        end
+      end
+
+      # Return the number of bytes currently stored
+      def bytes_downloaded
+        @lock.synchronize { @buffer.bytes_stored }
+      end
+    end
+
+    # Container class for file data
+    class FileService < PDTP::FileService
+      def initialize
+        @files = {}
+      end
+
+      def get_info(url)
+        @files[url] rescue nil
+      end 
+
+      def set_info(url, info)
+        @files[url] = info
+      end
+    end
+  end
+end

data/lib/pdtp/client/protocol.rb

@@ -0,0 +1,66 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+module PDTP
+  class Client < Mongrel::HttpHandler
+    # Implementation of a ruby test client for the pdtp protocol
+    class Protocol < PDTP::Protocol
+      def initialize *args
+        super
+      end
+
+      # Called after a connection to the server has been established
+      def connection_completed
+        begin
+          listen_port = @@config[:listen_port]
+
+          #create the client
+          client = PDTP::Client.new
+          PDTP::Protocol.listener = client
+          client.server_connection = self
+          client.generate_client_id listen_port
+          client.file_service = PDTP::Client::FileService.new
+
+          # Start a mongrel server on the specified port.  If it isnt available, keep trying higher ports
+          begin
+            mongrel_server = Mongrel::HttpServer.new('0.0.0.0', listen_port)
+          rescue Exception => e
+            listen_port += 1
+            retry
+          end
+
+          @@log.info "listening on port #{listen_port}"
+          mongrel_server.register '/', client
+          mongrel_server.run
+
+          # Tell the server about ourself
+          send_message :client_info, :listen_port => listen_port, :client_id => client.my_id
+
+          # Ask the server for some information on the file we want
+          send_message :ask_info, :url => @@config[:request_url]      
+
+          # Request the file
+          send_message :request, :url => @@config[:request_url]
+
+          @@log.info "This client is requesting"
+        rescue Exception=>e
+          puts "Exception in connection_completed: #{e}"
+          puts e.backtrace.join("\n")
+          exit
+        end
+      end  
+
+      def unbind
+        super
+        puts 'Disconnected from PDTP server.'
+      end
+    end
+  end
+end

data/lib/pdtp/client/transfer.rb

@@ -0,0 +1,229 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require File.dirname(__FILE__) + '/file_service'
+require "thread"
+require "net/http"
+require "uri"
+require "digest/sha2"
+
+module PDTP
+  class Client < Mongrel::HttpHandler
+    module Transfer
+      # Generic HTTP Exception to be used on error
+      class HTTPException < Exception
+        attr_accessor :code
+        def initialize(code,message)
+          super(message)
+          @code = code
+        end
+      end
+
+      # The base information and methods needed by client transfers
+      class Base
+        attr_reader :peer, :peer_id, :url, :byte_range
+        attr_reader :server_connection, :file_service
+        attr_reader :method, :client, :hash
+
+        # Returns true if a server message matches this transfer
+        def matches_message?(message)
+          @peer == message["peer"] and
+          @url == message["url"] and
+          @byte_range == message["range"] and
+          @peer_id == message["peer_id"] 
+        end
+
+        # Takes an HTTP range and returns a ruby Range object
+        def parse_http_range(string)
+          begin
+            raise "Can't parse range string: #{string}" unless string =~ /bytes=([0-9]+)-([0-9]+)/
+            (($1).to_i)..(($2).to_i)
+          rescue nil
+          end
+        end
+
+        # Notify the server of transfer completion.
+        # Hash field is used to denote success or failure
+        def send_completed_message(hash)
+          @server_connection.send_message(:completed,
+            :url => @url,
+            :peer => @peer,
+            :range => @byte_range,
+            :peer_id => @peer_id,
+            :hash => hash
+          )
+        end
+
+        def send_ask_verify_message
+          @server_connection.send_message(:ask_verify,
+            :url => @url,
+            :peer => @peer,
+            :range => @byte_range,
+            :peer_id => @peer_id
+          )
+        end
+      end
+
+      # Implements the listening end (the server) of a peer to peer http connection
+      class Listener < Base
+        attr :request, :response
+
+        # Called with the request and response parameters given by Mongrel
+        def initialize(request,response,server_connection,file_service,client)
+          @request,@response=request,response
+          @server_connection,@file_service=server_connection,file_service
+          @authorized=false
+          @client=client
+        end
+
+        # Send an HTTP error response to requester
+        def write_http_exception(e)
+          if e.class == HTTPException
+            @response.start(e.code) do |head,out|
+              out.write(e.to_s + "\n\n" + e.backtrace.join("\n") )
+            end
+          else
+            @@log.info("MONGREL SERVER ERROR: exception:" + e.to_s+"\n\n"+e.backtrace.join("\n"))
+            @response.start(500) do |head,out|
+              out.write("Server error, unknown exception:"+e.to_s + "\n\n" + e.backtrace.join("\n") )
+            end
+          end
+        end
+
+        # Parse the HTTP header and ask for verification of transfer
+        # Thread is stopped after asking for verification and will
+        # be restarted when verification arrives
+        def handle_header
+          @thread=Thread.current
+
+          @@log.debug "params=#{@request.params.inspect}"    
+
+          @method=@request.params["REQUEST_METHOD"].downcase   
+          @peer=@request.params["REMOTE_ADDR"]
+
+          #here we construct the GUID for this file
+          path=@request.params["REQUEST_PATH"]
+          vhost=@request.params["HTTP_HOST"]
+          @url="http://"+vhost+path
+
+          @byte_range=parse_http_range(request.params["HTTP_RANGE"])
+          @peer_id=@request.params["HTTP_X_PDTP_PEER_ID"]
+
+          #sanity checking
+          raise HTTPException.new(400, "Missing X-PDTP-Peer-Id header") if @peer_id.nil?
+          raise HTTPException.new(400, "Missing Host header") if vhost.nil?
+          raise HTTPException.new(400, "Missing Range header") if @byte_range.nil?
+
+          send_ask_verify_message
+          Thread.stop
+          after_verification
+        end
+
+        # Called after receiving verification message from the server
+        # Set the authorized status and restart the thread
+        # This throws us into after_verification
+        def tell_verify(authorized)
+          @authorized=authorized
+          @thread.run
+        end
+
+        # Perform the transfer if verification was successful
+        def after_verification
+          #check if the server authorized us
+          unless @authorized
+            raise HTTPException.new(403,"Forbidden: the server did not authorize this transfer")  
+          end
+
+          info = @file_service.get_info(@url)
+          if @method == "put"
+            #we are the taker
+            @@log.debug("Body Downloaded: url=#{@url} range=#{@byte_range} peer=#{@peer}")      
+
+            @file_service.set_info(FileInfo.new) if info.nil? 
+            info.write(@byte_range.first, @request.body.read)
+            @hash=Digest::SHA256.hexdigest(res.body) rescue nil
+
+            # Stock HTTP OK response
+            @response.start(200) do |head,out| 
+            end
+          elsif @method=="get"
+            #we are the giver
+            raise HTTPException.new(404,"File not found: #{@url}") if info.nil?
+            data=info.read(@byte_range)
+            raise HTTPException.new(416,"Invalid range: #{@byte_range.inspect}") if data.nil?   
+
+            #Request was GET, so now we need to send the data
+            @response.start(206) do |head, out|
+              head['Content-Type'] = 'application/octet-stream'
+              head['Content-Range'] = "bytes #{@byte_range.first}-#{@byte_range.last}/*"
+              #FIXME must include a DATE header according to http
+
+              out.write(data)
+            end
+          else
+            raise HTTPException.new(405,"Invalid method: #{@method}")
+          end
+
+        end
+
+      end
+
+      # Implements http transfer between two peers from the connector's (client) perspective
+      class Connector < Base
+        def initialize(message,server_connection,file_service,client)
+          @server_connection,@file_service=server_connection,file_service
+          @peer,@port=message["host"],message["port"]
+          @method = message["method"]
+          @url=message["url"]
+          @byte_range=message["range"]
+          @peer_id=message["peer_id"]
+          @client=client
+        end
+
+        # Perform the transfer
+        def run
+          hash=nil
+
+          info=@file_service.get_info(@url)
+
+          #compute the vhost and path
+          #FIXME work with ports
+          uri=URI.split(@url)
+          path=uri[5]
+          vhost=uri[2]   
+
+          if @method == "get"
+            req = Net::HTTP::Get.new(path)
+            body = nil
+          elsif @method == "put"
+            req = Net::HTTP::Put.new(path)
+            body = info.read(@byte_range)
+          else
+            raise HTTPException.new(405,"Invalid method: #{@method}")
+          end
+
+          req.add_field("Range", "bytes=#{@byte_range.begin}-#{@byte_range.end}")
+          req.add_field("Host",vhost)
+          req.add_field("X-PDTP-Peer-Id",@client.my_id)
+          res = Net::HTTP.start(@peer,@port) {|http| http.request(req,body) }
+
+          if res.code == '206' and @method == 'get'
+            #we are the taker
+            @@log.debug("Body Downloaded: url=#{@url} range=#{@byte_range} peer=#{@peer}:#{@port}")
+            info.write(@byte_range.first,res.body)
+            @hash=Digest::SHA256.hexdigest(res.body) rescue nil
+          else
+            raise "HTTP RESPONSE: code=#{res.code} body=#{res.body}"
+          end  
+        end
+      end
+    end
+  end
+end