distribustream 0.1.0

data/lib/pdtp/common/common_init.rb

@@ -0,0 +1,122 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+#Provides functions used for initialization by both the client and server
+
+require 'optparse'
+require 'logger'
+
+require File.dirname(__FILE__) + '/protocol'
+
+STDOUT.sync=true
+STDERR.sync=true
+
+@@log=Logger.new(STDOUT)
+@@log.datetime_format=""
+
+CONFIG_TYPES = {
+  :host             => :string,
+  :vhost            => :string,
+  :port             => :int,
+  :listen_port      => :int,
+  :file_root        => :string,
+  :quiet            => :bool,
+  :chunk_size       => :int,
+  :request_url      => :string
+}
+
+#prints banner and loads config file
+def common_init(program_name, config = nil)
+  @@config = config || {
+    :host             => '0.0.0.0',
+    :port             => 6086, #server port
+    :listen_port      => 8000, #client listen port
+    :file_root        => '.',
+    :chunk_size       => 5000,
+    :quiet            => true
+  }
+  
+  config_filename=nil
+  
+  unless config
+    OptionParser.new do |opts|
+      opts.banner = "Usage: #{program_name} [options]"
+      opts.on("--config CONFIGFILE", "Load specified config file.") do |c|
+        config_filename=c
+      end
+      opts.on("--help", "Prints this usage info.") do
+        puts opts
+        exit
+      end
+    end.parse!      
+  end
+  
+  puts "#{program_name} starting. Run '#{program_name} --help' for more info."
+
+  load_config_file(config_filename) unless config
+
+  begin
+    @@config[:file_root]=File.expand_path(@@config[:file_root])
+  rescue 
+    puts "Invalid path specified for file_root"
+    return
+  end
+
+  puts "@@config=#{@@config.inspect}"
+  validate_config_options
+  handle_config_options
+end
+
+#loads a config file specified by config_filename
+def load_config_file(config_filename)
+  if config_filename.nil?
+    puts "No config file specified. Using defaults."
+    return
+  end
+
+  confstr=File.read(config_filename) rescue nil
+  if confstr.nil?
+    puts "Unable to open config file: #{config_filename}"
+    exit
+  end
+
+  begin
+    new_config = YAML.load confstr
+    @@config.merge!(new_config)
+    @@config[:vhost] ||= @@config[:host] # Use host as vhost unless specified
+  rescue Exception => e
+    puts "Error parsing config file: #{config_filename}"
+    puts e
+    exit
+  end
+
+  puts "Loaded config file: #{config_filename}"
+end
+
+#make sure all the config options are of the right type
+def validate_config_options
+  @@config.each do |key,val|
+    type=CONFIG_TYPES[key]
+    if type.nil?
+      puts "Unknown parameter: #{key}"
+      exit
+    end
+
+    unless PDTP::Protocol.obj_matches_type?(val,type)
+      puts "Parameter: #{key} is not of type: #{type}"
+      exit
+    end
+  end
+end
+
+#responds to config options that are used by both client and server
+def handle_config_options
+  @@log.level=Logger::INFO if @@config[:quiet]  
+end

data/lib/pdtp/common/file_service.rb

@@ -0,0 +1,69 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+module PDTP
+  #provides information about a single file on the network
+  class FileInfo
+    attr_accessor :file_size, :base_chunk_size, :streaming
+
+    #number of chunks in the file
+    def num_chunks
+      return 0 if @file_size==0
+      (@file_size - 1) / @base_chunk_size + 1  
+    end
+
+    #size of the specified chunk
+    def chunk_size(chunkid) 
+      raise "Invalid chunkid #{chunkid}" if chunkid<0 or chunkid>=num_chunks
+      chunkid == num_chunks - 1 ? @file_size - @base_chunk_size * chunkid : @base_chunk_size
+    end
+
+    #range of bytes taken up by this chunk in the entire file
+    def chunk_range(chunkid)
+      start_byte = chunkid * @base_chunk_size
+      end_byte = start_byte + chunk_size(chunkid) - 1
+      start_byte..end_byte
+    end
+
+    #returns the chunkid that contains the requested byte offset
+    def chunk_from_offset(offset)
+      raise "Invalid offset #{offset}" if offset < 0 or offset >= @file_size
+      offset / @base_chunk_size
+    end
+
+    #takes a byte_range in the file and returns an equivalent chunk range
+    #if exclude_partial is true, chunks that are not completely covered by the byte range are left out
+    def chunk_range_from_byte_range(byte_range,exclude_partial=true)
+      min=chunk_from_offset(byte_range.first)
+      min+=1 if exclude_partial and byte_range.first > min*@base_chunk_size 
+
+      max_byte=byte_range.last
+      max_byte=@file_size-1 if max_byte==-1 or max_byte>=@file_size 
+      max=chunk_from_offset(max_byte)
+      max-=1 if exclude_partial and max_byte<chunk_range(max).last
+      min..max
+    end
+
+    #returns a string containing the data for this chunk
+    #range specifies a range of bytes local to this chunk
+    #implemented in Client and Server file services
+    def chunk_data(chunkid,range=nil)
+    end
+  end
+
+  # base class for ClientFileService and ServerFileService.
+  # provides shared functionality
+
+  class FileService
+    #returns a FileInfo class associated with the url, or nil if the file isnt known
+    def get_info(url)
+    end
+  end
+end

data/lib/pdtp/common/file_service_spec.rb

@@ -0,0 +1,91 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require File.dirname(__FILE__) + '/file_service'
+
+describe "A FileInfo with chunk_size=1" do
+  before(:each) do
+    @fi=PDTP::FileInfo.new
+    @fi.file_size=5
+    @fi.base_chunk_size=1
+  end
+
+  it "chunk_size works" do
+    @fi.chunk_size(0).should == 1
+    @fi.chunk_size(3).should == 1
+    @fi.chunk_size(4).should == 1
+
+    proc{ @fi.chunk_size(-1)}.should raise_error
+    proc{ @fi.chunk_size(5)}.should raise_error        
+  end
+
+  it "num_chunks works" do
+    @fi.num_chunks.should == 5
+  end
+
+  it "chunk_from_offset works" do
+    @fi.chunk_from_offset(0).should == 0
+    @fi.chunk_from_offset(4).should == 4
+    proc{@fi.chunk_from_offset(5)}.should raise_error
+  end
+
+  it "chunk_range_from_byte_range works" do
+    @fi.chunk_range_from_byte_range(0..4,false).should == (0..4)
+    @fi.chunk_range_from_byte_range(0..4,true).should == (0..4)
+    proc{@fi.chunk_range_from_byte_range(-1..3,true)}.should raise_error  
+  end
+
+end
+
+describe "A FileInfo with chunk_size=256 and file_size=768" do
+  before(:each) do
+    @fi=PDTP::FileInfo.new
+    @fi.base_chunk_size=256
+    @fi.file_size=768
+  end  
+
+  it "chunk_size works" do
+    @fi.chunk_size(0).should == 256
+    @fi.chunk_size(2).should == 256
+    proc{@fi.chunk_size(3)}.should raise_error
+  end
+
+  it "num_chunks works" do
+    @fi.num_chunks.should == 3
+  end
+
+  it "chunk_from_offset works" do
+    @fi.chunk_from_offset(256).should == 1
+    @fi.chunk_from_offset(255).should == 0
+  end
+
+  it "chunk_range_from_byte_range works" do
+    @fi.chunk_range_from_byte_range(256..511,true).should == (1..1)
+    @fi.chunk_range_from_byte_range(256..511,false).should == (1..1)
+    @fi.chunk_range_from_byte_range(255..512,true).should == (1..1)
+    @fi.chunk_range_from_byte_range(255..512,false).should == (0..2)
+  end
+end
+
+describe "A FileInfo with chunk_size=256 and file_size=255" do
+  before(:each) do
+    @fi=PDTP::FileInfo.new
+    @fi.base_chunk_size=256
+    @fi.file_size=255
+  end
+
+  it "num_chunks works" do
+    @fi.num_chunks.should ==1
+  end
+
+  it "chunk_from_offset works" do
+    @fi.chunk_from_offset(254).should == 0
+  end
+end  

data/lib/pdtp/common/protocol.rb

@@ -0,0 +1,346 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require 'rubygems'
+require 'eventmachine'
+require 'thread'
+require 'uri'
+require 'ipaddr'
+
+begin
+  require 'fjson'
+rescue LoadError
+  require 'json'
+end
+
+module PDTP
+  PROTOCOL_DEBUG=true
+
+  class ProtocolError < Exception
+  end
+
+  class ProtocolWarn < Exception
+  end
+
+  # EventMachine handler class for the PDTP protocol
+  class Protocol < EventMachine::Protocols::LineAndTextProtocol
+    @@num_connections = 0
+    @@listener = nil
+    @@message_params = nil
+    @connection_open = false
+
+    def connection_open?
+      @connection_open
+    end
+
+    #sets the listener class (Server or Client)
+    def self.listener=(listener)
+      @@listener = listener
+    end
+
+    def initialize(*args)
+      user_data = nil
+      @mutex = Mutex.new
+      super
+    end
+
+    #called by EventMachine after a connection has been established
+    def post_init
+      # a cache of the peer info because eventmachine seems to drop it before we want
+      peername = get_peername
+      if peername.nil?
+        @cached_peer_info = ["<Peername nil!!!>", 91119] if peername.nil?
+      else
+        port, addr = Socket.unpack_sockaddr_in(peername)
+        @cached_peer_info = [addr.to_s, port.to_i]
+      end
+
+      @@num_connections += 1
+      @connection_open = true
+      @@listener.connection_created(self) if @@listener.respond_to?(:connection_created)
+    end
+
+    attr_accessor :user_data #users of this class may store arbitrary data here
+
+    #close a connection, but first send the specified error message
+    def error_close_connection(error) 
+      if PROTOCOL_DEBUG
+        send_message :protocol_error, :message => msg 
+        close_connection(true) # close after writing
+      else
+        close_connection
+      end
+    end
+
+    #override this in a child class to handle messages
+    def receive_message(command, message)
+      @@listener.dispatch_message command, message, self 
+    end
+
+    #debug routine: returns id of remote peer on this connection
+    def remote_peer_id
+      ret = user_data.client_id rescue nil
+      ret || 'NOID'
+    end
+
+    #called for each line of text received over the wire
+    #parses the JSON message and dispatches the message
+    def receive_line line
+      begin
+        line.chomp!
+        @@log.debug "(#{remote_peer_id}) recv: " + line
+        message = JSON.parse(line) rescue nil
+        raise ProtocolError.new("JSON couldn't parse: #{line}") if message.nil?
+        Protocol.validate_message message
+        
+        command, options = message
+        hash_to_range command, options
+        receive_message command, options
+      rescue ProtocolError => e
+        @@log.warn "(#{remote_peer_id}) PROTOCOL ERROR: #{e.to_s}"
+        @@log.debug e.backtrace.join("\n")
+        error_close_connection e.to_s
+      rescue ProtocolWarn => e
+        send_message :protocol_warn, :message => e.to_s
+      rescue Exception => e
+        puts "(#{remote_peer_id}) UNKNOWN EXCEPTION #{e.to_s}"
+        puts e.backtrace.join("\n")
+      end
+    end
+
+    RANGENAMES = %w{chunk_range range byte_range}
+
+    #converts Ruby Range classes in the message to PDTP protocol hashes with min and max
+    # 0..-1 => nil  (entire file)
+    # 10..-1 => {"min"=>10} (contents of file >= 10)
+    def range_to_hash(message)
+      message.each do |key,value|
+        if value.class==Range
+          if value==(0..-1)
+            message.delete(key)
+          elsif value.last==-1 
+            message[key]={"min"=>value.first}
+          else
+            message[key]={"min"=>value.first,"max"=>value.last}
+          end
+        end   
+      end
+    end
+
+    #converts a PDTP protocol min and max hash to a Ruby Range class
+    def hash_to_range(command, message)
+      key="range"
+      auto_types=["provide","request"] #these types assume a range if it isnt specified
+      auto_types.each do |type|
+        if command == type and message[key].nil?
+          message[key]={} # assume entire file if not specified
+        end
+      end
+
+      if message[key]
+        raise if message[key].class!=Hash
+        min=message[key]["min"] 
+        max=message[key]["max"]
+        message[key]= (min ? min : 0)..(max ? max : -1)
+      end
+    end
+
+    #sends a message, in the internal Hash format, over the wire
+    def send_message(command, opts = {})
+      #message = opts.merge(:type => command.to_s)
+
+      # Stringify all option keys
+      opts = opts.map { |k,v| [k.to_s, v] }.inject({}) { |h,(k,v)| h[k] = v; h }
+      
+      # Convert all Ruby ranges to JSON objects representing them
+      range_to_hash opts
+      
+      # Message format is a JSON array with the command (string) as the first entry
+      # Second entry is an options hash/object
+      message = [command.to_s, opts]
+      
+      @mutex.synchronize do
+        outstr = JSON.unparse(message) + "\n"
+        @@log.debug "(#{remote_peer_id}) send: #{outstr.chomp}"
+        send_data outstr  
+      end
+    end
+
+    #called by EventMachine when a connection is closed
+    def unbind
+      @@num_connections -= 1
+      @@listener.connection_destroyed(self) if @@listener.respond_to?(:connection_destroyed)
+      @connection_open = false
+    end
+
+    def self.print_info
+      puts "num_connections=#{@@num_connections}"
+    end
+
+    #returns the ip address and port in an array [ip, port]
+    def get_peer_info
+      @cached_peer_info
+    end
+
+    def to_s
+      addr,port = get_peer_info
+      "#{addr}:#{port}"
+    end
+
+    #makes sure that the message is valid.
+    #if not, throws a ProtocolError
+    def self.validate_message(message)
+      raise ProtocolError.new("Message is not a JSON array") unless message.is_a? Array
+      command, options = message
+      
+      @@message_params ||= define_message_params
+
+      params = @@message_params[command] rescue nil
+      raise ProtocolError.new("Invalid message type: #{command}") if params.nil?
+
+      params.each do |name,type|
+        if type.class == Optional
+          next if options[name].nil? #dont worry about it if they dont have this param
+          type = type.type #grab the real type from within the optional class
+        end
+
+        raise ProtocolError.new("required parameter: '#{name}' missing for message: '#{command}'") if options[name].nil?
+        unless obj_matches_type?(options[name], type)
+          raise ProtocolError.new("parameter: '#{name}' val='#{options[name]}' is not of type: '#{type}' for message: '#{command}' ")
+        end
+      end    
+    end
+
+    # an optional field of the specified type
+    class Optional
+      attr_accessor :type
+      def initialize(type)
+        @type=type
+      end
+    end
+
+    #returns whether or not a given ruby object matches the specified type
+    #available types:
+    # :url, :range, :ip, :int, :bool, :string
+    def self.obj_matches_type?(obj,type)
+      case type
+      when :url then obj.class == String
+      when :range then obj.class == Range or obj.class == Hash
+      when :int then obj.class == Fixnum
+      when :bool then obj == true or obj == false
+      when :string then obj.class == String
+      when :ip
+        ip = IPAddr.new(obj) rescue nil
+        !ip.nil?
+      else 
+        raise "Invalid type specified: #{type}"
+      end 
+    end
+
+    #this function defines the required fields for each message
+    def self.define_message_params
+      mp = {}
+
+      #must be the first message the client sends
+      mp["client_info"]={
+        "client_id"=>:string,
+        "listen_port"=>:int                  
+      }
+
+      mp["ask_info"]={
+        "url"=>:url
+      }
+
+      mp["tell_info"]={
+        "url"=>:url,
+        "size"=>Optional.new(:int),
+        "chunk_size"=>Optional.new(:int),
+        "streaming"=>Optional.new(:bool)
+      }
+
+      mp["ask_verify"]={
+        "peer"=>:ip,
+        "url"=>:url,
+        "range"=>:range,
+        "peer_id"=>:string
+      }
+
+      mp["tell_verify"]={
+        "peer"=>:ip,
+        "url"=>:url,
+        "range"=>:range,
+        "peer_id"=>:string,
+        "is_authorized"=>:bool
+      }
+
+      mp["request"]={
+        "url"=>:url,
+        "range"=>Optional.new(:range)
+      }
+
+      mp["provide"]={
+        "url"=>:url,
+        "range"=>Optional.new(:range)
+      }
+
+      mp["unrequest"]={
+        "url"=>:url,
+        "range"=>Optional.new(:range)
+      }
+
+      mp["unprovide"]={
+        "url"=>:url,
+        "range"=>Optional.new(:range)
+      }
+
+      #the taker sends this message when a transfer finishes
+      #if there is an error in the transfer, dont set a hash
+      #to signify failure
+      #when this is received from the taker, the connection is considered done for all parties
+      #
+      #The giver also sends this message when they are done transferring.
+      #this closes the connection on their side, allowing them to start other transfers
+      #It leaves the connection open on the taker side to allow them to decide if the transfer was successful
+      #the hash parameter is ignored when sent by the giver
+      mp["completed"]={
+        #"peer"=>:ip, no longer used
+        "url"=>:url,
+        "range"=>:range,
+        "peer_id"=>:string,
+        "hash"=>Optional.new(:string)
+      }
+
+      mp["hash_verify"]={
+        "url"=>:url,
+        "range"=>:range,
+        "hash_ok"=>:bool
+      }
+
+      mp["transfer"]={
+        "host"=>:string,
+        "port"=>:int,
+        "method"=>:string,
+        "url"=>:url,
+        "range"=>:range,
+        "peer_id"=>:string
+      }  
+
+      mp["protocol_error"]={
+        "message"=>Optional.new(:string)
+      }
+
+      mp["protocol_warn"]={
+        "message"=>Optional.new(:string)
+      }
+
+      mp
+    end
+  end
+end