distribustream 0.1.0 → 0.2.0

data/lib/pdtp/server.rb

@@ -8,361 +8,60 @@
 # See http://distribustream.rubyforge.org/
 #++
 
-require File.dirname(__FILE__) + '/common/protocol'
-require File.dirname(__FILE__) + '/common/common_init'
-require File.dirname(__FILE__) + '/server/file_service'
-require File.dirname(__FILE__) + '/server/client_info'
-require File.dirname(__FILE__) + '/server/transfer'
+require 'rubygems'
+require 'eventmachine'
+require 'mongrel'
 
-require 'thread'
-require 'erb'
+require File.dirname(__FILE__) + '/server/dispatcher'
+require File.dirname(__FILE__) + '/server/file_service'
+require File.dirname(__FILE__) + '/server/connection'
+require File.dirname(__FILE__) + '/server/stats_handler'
 
 module PDTP
-  # PDTP server implementation
+  # PDTP::Server provides an interface for creating a PDTP server
   class Server
-    attr_reader :connections
-    attr_accessor :file_service
-    def initialize
-      @connections = Array.new
-      @stats_mutex=Mutex.new
-      @used_client_ids=Hash.new #keeps a list of client ids in use, they must be unique
-      @updated_clients=Hash.new #a set of clients that have been modified and need transfers spawned
-    end
-
-    #called by pdtp_protocol when a connection is created
-    def connection_created(connection)
-      @stats_mutex.synchronize do
-        @@log.info "Client connected: #{connection.get_peer_info.inspect}"
-        connection.user_data = ClientInfo.new
-        @connections << connection 
-      end
-    end  
-
-    #called by pdtp_protocol when a connection is destroyed
-    def connection_destroyed(connection)
-      @stats_mutex.synchronize do
-        @@log.info "Client connection closed: #{connection.get_peer_info.inspect}"
-        @connections.delete connection
-      end
-    end
-
-    # returns the ClientInfo object associated with this connection
-    def client_info(connection)
-      connection.user_data ||= ClientInfo.new
-    end
-
-    # called when a transfer either finishes, successfully or not
-    def transfer_completed(transfer,connection,chunk_hash,send_response=true)      
-      # did the transfer complete successfully?
-      local_hash=@file_service.get_chunk_hash(transfer.url,transfer.chunkid)
-
-      c1=client_info(transfer.taker)
-      c2=client_info(transfer.giver)
-
-      if connection==transfer.taker
-        success= (chunk_hash==local_hash)
-
-        if success
-          #the taker now has the file, so he can provide it
-          client_info(transfer.taker).chunk_info.provide(transfer.url,transfer.chunkid..transfer.chunkid)
-          c1.trust.success(c2.trust)
-        else
-          #transfer failed, the client still wants the chunk
-          client_info(transfer.taker).chunk_info.request(transfer.url,transfer.chunkid..transfer.chunkid)
-          c1.trust.failure(c2.trust)
-        end 
-
-        transfer.taker.send_message(:hash_verify, 
-          :url => transfer.url, 
-          :range => transfer.byte_range, 
-          :hash_ok => success
-        ) if send_response
-      end
-
-      #outstr="#{@ids[transfer.giver]}->#{@ids[transfer.taker]} transfer completed: #{transfer}"
-      #outstr=outstr+" t->g=#{c1.trust.weight(c2.trust)} g->t=#{c2.trust.weight(c1.trust)}" 
-      #outstr=outstr+"sent_by: "+ ( connection==transfer.taker ? "taker" : "giver" )
-      #outstr=outstr+" success=#{success} "
-      #@@log.debug(outstr)
-
-      #remove this transfer from whoever sent it
-      client_info(connection).transfers.delete(transfer.transfer_id)
-      @updated_clients[connection]=true #flag this client for transfer creation
-    end
-
-    #Creates a new transfer between two peers
-    #returns true on success, or false if the specified transfer is already in progress
-    def begin_transfer(taker, giver, url, chunkid)
-      byte_range = @file_service.get_info(url).chunk_range(chunkid) 
-      t = Transfer.new(taker, giver, url, chunkid, byte_range)
-
-      #make sure this transfer doesnt already exist
-      t1 = client_info(taker).transfers[t.transfer_id]
-      t2 = client_info(giver).transfers[t.transfer_id]
-      return false unless t1.nil? and t2.nil?
-
-      client_info(taker).chunk_info.transfer(url, chunkid..chunkid) 
-      client_info(taker).transfers[t.transfer_id] = t
-      client_info(giver).transfers[t.transfer_id] = t
-
-      #send transfer message to the connector
-      addr, port = t.acceptor.get_peer_info
+    # Create a new PDTP::Server which will listen on the given address and port
+    def initialize(addr, port = 6086)
+      @addr, @port = addr, port
       
-      t.connector.send_message(:transfer,
-        :host => addr,
-        :port => t.acceptor.user_data.listen_port,
-        :method => t.connector == t.taker ? "get" : "put",
-        :url => url,
-        :range => byte_range,
-        :peer_id => client_info(t.acceptor).client_id
-      )
-      true
-    end
-
-    #this function removes all stalled transfers from the list
-    #and spawns new transfers as appropriate
-    #it must be called periodically by EventMachine
-    def clear_all_stalled_transfers
-      @connections.each { |connection| clear_stalled_transfers_for_client connection }  
-      spawn_all_transfers
-    end
-
-    #removes all stalled transfers that this client is a part of
-    def clear_stalled_transfers_for_client(client_connection)
-      client_info(client_connection).get_stalled_transfers.each do |transfer|
-        transfer_completed transfer, client_connection, nil, false
-      end  
-    end
-
-    #spawns uploads and downloads for this client.
-    #should be called every time there is a change that would affect 
-    #what this client has or wants
-    def spawn_transfers_for_client(client_connection)
-      info = client_info client_connection
-
-      while info.wants_download? do
-        break if spawn_download_for_client(client_connection) == false
-      end
-
-      while info.wants_upload? do
-        break if spawn_upload_for_client(client_connection) == false
-      end
-    end
-
-    #creates a single download for the specified client
-    #returns true on success, false on failure
-    def spawn_download_for_client(client_connection)
-      feasible_peers=[]
-
-      c1info=client_info(client_connection)
-      begin
-        url,chunkid=c1info.chunk_info.high_priority_chunk
-      rescue
-        return false
-      end
-
-      @connections.each do |c2|
-        next if client_connection==c2
-        next if client_info(c2).wants_upload? == false
-        if client_info(c2).chunk_info.provided?(url,chunkid)
-          feasible_peers << c2
-          break if feasible_peers.size > 5
-        end
-      end
-
-      # we now have a list of clients that have the requested chunk.
-      # pick one and start the transfer
-      if feasible_peers.size > 0
-        #FIXME base this on the trust model
-        giver=feasible_peers[rand(feasible_peers.size)]
-        return begin_transfer(client_connection,giver,url,chunkid)
-        #FIXME should we try again if begin_transfer fails?
-      end
-
-      false
-    end
-
-    #creates a single upload for the specified client
-    #returns true on success, false on failure
-    def spawn_upload_for_client(client_connection)
-      c1info=client_info(client_connection)
-
-      @connections.each do |c2|
-        next if client_connection==c2
-        next if client_info(c2).wants_download? == false
-
-        begin
-          url,chunkid=client_info(c2).chunk_info.high_priority_chunk
-        rescue
-          next
-        end
-
-        if c1info.chunk_info.provided?(url,chunkid)
-          return begin_transfer(c2,client_connection,url,chunkid)
-        end
-      end
-
-      false
-    end
-
-    #called by pdtp_protocol for each message that comes in from the wire
-    def dispatch_message(command, message, connection)
-      @stats_mutex.synchronize do
-        dispatch_message_needslock command, message, connection
-      end
-    end
-
-    #creates new transfers for all clients that have been updated
-    def spawn_all_transfers
-      while @updated_clients.size > 0 do
-        tmp=@updated_clients
-        @updated_clients=Hash.new
-        tmp.each do |client,true_key| 
-          spawn_transfers_for_client(client)
-        end
-      end    
-    end
-
-    #handles the request, provide, unrequest, unprovide messages
-    def handle_requestprovide(connection,message)
-      type=message["type"]
-      url=message["url"]
-      info=@file_service.get_info(url) rescue nil
-      raise ProtocolWarn.new("Requested URL: '#{url}' not found") if info.nil?
-
-      exclude_partial= (type=="provide") #only exclude partial chunks from provides
-      range=info.chunk_range_from_byte_range(message["range"],exclude_partial)
-
-      #call request, provide, unrequest, or unprovide
-      client_info(connection).chunk_info.send( type.to_sym, url, range)
-      @updated_clients[connection]=true #add to the list of client that need new transfers
-    end
-
-    #handles all incoming messages from clients
-    def dispatch_message_needslock(command, message, connection)
-      # store the command in the message hash
-      message["type"] = command
+      @dispatcher = PDTP::Server::Dispatcher.new
+      @dispatcher.file_service = PDTP::Server::FileService.new
+    end
+    
+    # Run a web server to display statistics on the given address and port 
+    def enable_stats_service(addr = nil, port = 6087)
+      # Use the same address as the main server unless a different one was specified
+      addr ||= @addr
       
-      #require the client to be logged in with a client id
-      if command != "client_info" and client_info(connection).client_id.nil?
-        raise ProtocolError.new("You need to send a 'client_info' message first")
-      end 
-
-      case command
-      when "client_info"
-        cid = message["client_id"]
-        #make sure this id isnt in use
-        if @used_client_ids[cid]
-          raise ProtocolError.new("Your client id: #{cid} is already in use.")   
-        end
-        
-        @used_client_ids[cid] = true 
-        client_info(connection).listen_port = message["listen_port"]
-        client_info(connection).client_id = cid
-      when "ask_info"
-        info = file_service.get_info(message["url"])
-        response = { :url => message["url"] }
-        unless info.nil?
-          response[:size] = info.file_size
-          response[:chunk_size] = info.base_chunk_size
-          response[:streaming] = info.streaming
-        end
-        connection.send_message :tell_info, response
-      when "request", "provide", "unrequest", "unprovide"
-        handle_requestprovide connection, message
-      when "ask_verify"
-        #check if the specified transfer is a real one
-        my_id = client_info(connection).client_id
-        transfer_id=Transfer.gen_transfer_id(my_id,message["peer_id"],message["url"],message["range"])
-        ok = !!client_info(connection).transfers[transfer_id]
-        client_info(connection).transfers[transfer_id].verification_asked=true if ok
-        @@log.debug "AskVerify not ok: id=#{transfer_id}" unless ok
-        connection.send_message(:tell_verify,
-          :url     => message["url"],
-          :peer_id => message["peer_id"],
-          :range   => message["range"],
-          :peer    => message["peer"],
-          :is_authorized=>ok
-        )
-      when "completed"
-        my_id = client_info(connection).client_id
-        transfer_id=  Transfer::gen_transfer_id(
-          my_id,message["peer_id"],
-          message["url"],
-          message["range"]
-        )
-        transfer=client_info(connection).transfers[transfer_id]
-        @@log.debug("Completed: id=#{transfer_id} ok=#{transfer != nil}" )
-        if transfer
-          transfer_completed(transfer,connection,message["hash"])
-        else
-          raise ProtocolWarn.new("You sent me a transfer completed message for unknown transfer: #{transfer_id}")
+      @stats_server = Mongrel::HttpServer.new addr, port
+      @@log.info "Mongrel server listening on port: #{port}"
+      @stats_server.register '/', PDTP::Server::StatsHandler.new(@dispatcher)
+      @stats_server.run
+    end
+    
+    # Serve files from the given directory
+    def enable_file_service(path, options = {})
+      opts = {
+        :chunk_size => 100000
+      }.merge(options)
+      
+      @dispatcher.file_service.root = path
+      @dispatcher.file_service.default_chunk_size = opts[:chunk_size]
+    end
+    
+    # Run the PDTP server event loop
+    def run
+      EventMachine::run do
+        EventMachine::start_server(@addr, @port, PDTP::Server::Connection) do |connection|
+          connection.server = @dispatcher    
+          connection.connection_completed
         end
-      when 'protocol_error', 'protocol_warn' #ignore
-      else raise ProtocolError.new("Unhandled message type: #{command}")
-      end
 
-      spawn_all_transfers
-    end
+        @@log.info "accepting connections with ev=#{EventMachine::VERSION}"
+        @@log.info "host=#{@host}  port=#{@port}"
 
-    #returns a string representing the specified connection
-    def connection_name(c)
-      #host,port=c.get_peer_info
-      #return "#{get_id(c)}: #{host}:#{port}"
-      client_info(c).client_id
-    end
-
-    def generate_html_stats
-      @stats_mutex.synchronize { generate_html_stats_needslock }
-    end
-
-    #builds an html page with information about the server's internal workings
-    def generate_html_stats_needslock
-      s = ERB.new <<EOF
-<html><head><title>DistribuStream Statistics</title></head>
-<body>
-<h1>DistribuStream Statistics</h1>
-Time=<%= Time.new %><br> Connected Clients=<%= @connections.size %>
-<center><table border=1>
-<tr><th>Client</th><th>Transfers</th><th>Files</th></tr>
-<% @connections.each do |c| %>
-  <tr><td>
-  <% host, port = c.get_peer_info %>
-  <%= connection_name(c) %><br><%= host %>:<%= port %>
-  </td>
-  <td>
-  <%
-  client_info(c).transfers.each do |key,t|
-    if c==t.giver
-      type="UP: "
-      peer=t.taker
-    else
-      type="DOWN: "
-      peer=t.giver
-    end
-    %>
-    <%= type %> id=<%= t.transfer_id %><br>
-    <%
-  end
-  %>
-  </td>
-  <td>
-  <%
-  client_info(c).chunk_info.get_file_stats.each do |fs|
-    %>
-    <%= fs.url %> size=<%= fs.file_chunks %> req=<%= fs.chunks_requested %>
-    prov=<%= fs.chunks_provided %> transf=<%= fs.chunks_transferring %><br>    
-    <%
-  end      
-  %>
-  </td></tr>
-  <%
-end
-%>
-</table>
-</body></html>
-EOF
-      s.result binding
+        EventMachine::add_periodic_timer(2) { @dispatcher.clear_all_stalled_transfers }
+      end
     end
   end
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: distribustream
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
-date: 2008-10-11 00:00:00 -06:00
+  version: 0.2.0
+date: 2008-10-22 00:00:00 -06:00
 summary: DistribuStream is a fully open peercasting system allowing on-demand or live streaming media to be delivered at a fraction of the normal cost
 require_paths: 
 - lib
@@ -33,30 +33,33 @@ authors:
 - James Sanders
 - Tom Stapleton
 files: 
-- bin/distribustream
 - bin/dsclient
 - bin/dsseed
+- bin/dstream
 - lib/pdtp
 - lib/pdtp/client
 - lib/pdtp/client.rb
 - lib/pdtp/common
 - lib/pdtp/server
 - lib/pdtp/server.rb
+- lib/pdtp/client/callbacks.rb
+- lib/pdtp/client/connection.rb
 - lib/pdtp/client/file_buffer.rb
-- lib/pdtp/client/file_buffer_spec.rb
 - lib/pdtp/client/file_service.rb
-- lib/pdtp/client/protocol.rb
+- lib/pdtp/client/http_handler.rb
 - lib/pdtp/client/transfer.rb
 - lib/pdtp/common/common_init.rb
 - lib/pdtp/common/file_service.rb
-- lib/pdtp/common/file_service_spec.rb
+- lib/pdtp/common/packet.rb
 - lib/pdtp/common/protocol.rb
-- lib/pdtp/common/protocol_spec.rb
 - lib/pdtp/server/client_info.rb
+- lib/pdtp/server/connection.rb
+- lib/pdtp/server/dispatcher.rb
 - lib/pdtp/server/file_service.rb
+- lib/pdtp/server/file_service_protocol.rb
+- lib/pdtp/server/stats_handler.rb
 - lib/pdtp/server/transfer.rb
 - lib/pdtp/server/trust.rb
-- lib/pdtp/server/trust_spec.rb
 - conf/bigchunk.yml
 - conf/debug.yml
 - conf/example.yml
@@ -77,7 +80,7 @@ extra_rdoc_files:
 - README
 - CHANGES
 executables: 
-- distribustream
+- dstream
 - dsseed
 - dsclient
 extensions: []

data/bin/distribustream

@@ -1,60 +0,0 @@
-#!/usr/bin/env ruby
-#--
-# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
-# 
-# This source file is distributed as part of the 
-# DistribuStream file transfer system.
-#
-# See http://distribustream.rubyforge.org/
-#++
-
-require 'rubygems'
-require 'eventmachine'
-require 'optparse'
-require 'logger'
-require 'mongrel'
-
-require File.dirname(__FILE__) + '/../lib/pdtp/server'
-
-common_init $0
-
-server = PDTP::Server.new
-server.file_service = PDTP::Server::FileService.new
-PDTP::Protocol.listener = server
-
-#set up the mongrel server for serving the stats page
-class MongrelServerHandler< Mongrel::HttpHandler
-  def initialize(server)
-    @server = server
-  end
-
-  def process(request,response)
-    response.start(200) do |head, out|
-      out.write begin
-        outstr = @server.generate_html_stats
-      rescue Exception=>e
-        outstr = "Exception: #{e}\n#{e.backtrace.join("\n")}"
-      end
-    end    
-  end
-end
-
-#run the mongrel server
-mongrel_server = Mongrel::HttpServer.new '0.0.0.0', @@config[:port] + 1
-@@log.info "Mongrel server listening on port: #{@@config[:port] + 1}"
-mongrel_server.register '/', MongrelServerHandler.new(server)
-mongrel_server.run
-
-#set root directory
-server.file_service.root = @@config[:file_root]
-server.file_service.default_chunk_size = @@config[:chunk_size]
-
-EventMachine::run do
-  host, port = '0.0.0.0', @@config[:port]
-  EventMachine::start_server host, port, PDTP::Protocol
-  @@log.info "accepting connections with ev=#{EventMachine::VERSION}"
-  @@log.info "host=#{host}  port=#{port}"
-
-  EventMachine::add_periodic_timer(2) { server.clear_all_stalled_transfers }
-end

data/lib/pdtp/client/file_buffer_spec.rb

@@ -1,154 +0,0 @@
-#--
-# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
-# All rights reserved.  See COPYING for permissions.
-# 
-# This source file is distributed as part of the 
-# DistribuStream file transfer system.
-#
-# See http://distribustream.rubyforge.org/
-#++
-
-require File.dirname(__FILE__) + '/file_buffer'
-
-describe PDTP::FileBuffer do
-  before(:each) do
-    @b = PDTP::FileBuffer.new
-  end
-
-  it "returns nil if read when empty" do
-    @b.bytes_stored.should == 0
-    @b.read(0..1).should == nil
-  end  
-end
-
-describe PDTP::FileBuffer, "with one entry" do
-  before(:each) do
-    @b = PDTP::FileBuffer.new
-    @b.write 0, 'hello'
-  end
-
-  it "calculates bytes stored correctly" do
-    @b.bytes_stored.should == 5
-  end
-
-  it "reads stored data correctly" do
-    @b.read(0..4).should == "hello"
-    @b.read(1..1).should == "e"
-    @b.read(-1..2).should == nil
-    @b.read(0..5).should == nil
-  end
-end
-
-describe PDTP::FileBuffer, "with two overlapping entries" do
-  before(:each) do
-    @b = PDTP::FileBuffer.new
-    @b.write(3,"hello")
-    @b.write(7,"World")
-  end
-  
-  it "calculates bytes stored correctly" do
-    @b.bytes_stored.should == 9
-  end
-
-  it "reads stored data correctly" do
-    @b.read(3..12).should == nil
-    @b.read(3..11).should == "hellWorld"
-    @b.read(3..1).should == nil
-    @b.read(2..4).should == nil
-  end
-
-end
-
-describe PDTP::FileBuffer, "with three overlapping entries" do
-  before(:each) do
-    @b = PDTP::FileBuffer.new
-    @b.write(3,"hello")
-    @b.write(7,"World")
-    @b.write(2,"123456789ABCDEF")
-  end
-
-  it "calculates bytes stored correctly" do
-    @b.bytes_stored.should == 15
-  end
-
-  it "reads stored data correctly" do
-    @b.read(2..16).should == "123456789ABCDEF"
-    @b.read(2..17).should == nil
-  end
-end
-
-describe PDTP::FileBuffer, "with two tangential entries" do
-  before(:each) do
-    @b = PDTP::FileBuffer.new
-    @b.write(3,"hello")
-    @b.write(8,"World")
-  end
-
-  it "calculates bytes stored correctly" do
-    @b.bytes_stored.should == 10
-  end
-  
-  it "reads stored data correctly" do
-    @b.read(3..12).should == "helloWorld"
-  end
-end
-
-describe PDTP::FileBuffer, "with a chain of overlapping entries" do
-  before(:each) do
-    @b = PDTP::FileBuffer.new
-    @b.write(3,"a123")
-    @b.write(4,"b4")
-    @b.write(0,"012c")
-  
-    #___a123
-    #___ab43
-    #012cb43
-
-  end
-
-  it "calculates bytes stored correctly" do
-    @b.bytes_stored.should == 7
-  end
-
-  it "reads stored data correctly" do
-    @b.read(0..6).should == "012cb43"
-    @b.read(3..6).should == "cb43"
-  end
-end
-
-describe PDTP::FileBuffer, "with an associated IO object" do
-  before(:each) do
-    @io = mock(:io)
-    @b = PDTP::FileBuffer.new @io
-  end
-  
-  it "writes received data to the IO object" do
-    @io.should_receive(:write).once.with('foo').and_return(3)
-    @b.write(0, "foo")
-  end
-  
-  it "writes successively received data to the IO object" do
-    @io.should_receive(:write).once.with('foo').and_return(3)
-    @b.write(0, "foo")
-    
-    @io.should_receive(:write).once.with('bar').and_return(3)
-    @b.write(3, "bar")
-    
-    @io.should_receive(:write).once.with('baz').and_return(3)
-    @b.write(6, "baz")
-  end
-  
-  it "reassembles single-byte out-of-order data and writes it to the IO object" do
-    @io.should_receive(:write).once.with('bar').and_return(3)
-    @b.write(1, 'a')
-    @b.write(2, 'r')
-    @b.write(0, 'b')
-  end
-  
-  it "reassembles multibyte out-of-order data and writes it to the IO object" do
-    @io.should_receive(:write).once.with('foobar').and_return(6)
-    @b.write(2, 'ob')
-    @b.write(4, 'ar')
-    @b.write(0, 'fo')
-  end
-end