distribustream 0.1.0 → 0.2.0

data/CHANGES

@@ -1,2 +1,13 @@
+Version 0.2.0
+* PDTP::Server and PDTP::Client now expose APIs intended for public consumption
+  They may change slightly down the road, but will remain largely the same
+
+* PDTP::Client now uses a Tempfile for its internal buffer, drastically 
+  improving its memory usage
+
+* The protocol now uses length prefix framing, rather than using CRLF encoding.
+  This breaks protocol compatibility with the 0.1.0 release but should represent
+  the last major change to the protocol format.
+
 Version 0.1.0
 * Initial release

data/Rakefile

@@ -25,20 +25,24 @@ end
 begin
 require 'spec/rake/spectask'
 
+SPECS = FileList['spec/**/*_spec.rb']
+
 Spec::Rake::SpecTask.new(:spec) do |task|
-  task.spec_files = FileList['**/*_spec.rb']
+  task.spec_files = SPECS
 end
 
-Spec::Rake::SpecTask.new(:specfs) do |task|
-  task.spec_files= FileList['**/*_spec.rb']
-  task.spec_opts="-f s".split
+namespace :spec do
+  Spec::Rake::SpecTask.new(:print) do |task|
+    task.spec_files = SPECS
+    task.spec_opts="-f s".split
+  end
+
+  Spec::Rake::SpecTask.new(:rcov) do |task|
+    task.spec_files = SPECS
+    task.rcov = true
+    task.rcov_opts = ['--exclude', 'spec']
+  end
 end
 
-Spec::Rake::SpecTask.new(:spec_coverage) do |task|
-  task.spec_files = FileList['**/*_spec.rb']
-  task.rcov = true
-end
 rescue LoadError
 end
-
-

data/bin/dsclient

@@ -1,4 +1,24 @@
 #!/usr/bin/env ruby
+
+# == Synopsis
+#
+# dsclient: DistribuStream client application
+#
+# == Usage
+#
+# dsclient [OPTION] ... URI
+#
+# -h, --help:
+#    Show help
+#
+# --output file, -o file:
+#    File to save to.  Use '-' for stdout  
+#
+# --listen_port port, -p port:
+#    Port for the local HTTP server to listen on
+#
+# URI: An address in the pdtp:// scheme to retrieve from the server
+
 #--
 # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
 # All rights reserved.  See COPYING for permissions.
@@ -9,35 +29,79 @@
 # See http://distribustream.rubyforge.org/
 #++
 
-require 'rubygems'
-require 'eventmachine'
-require 'mongrel'
-require 'optparse'
 require 'uri'
+require 'getoptlong'
+require 'rdoc/ri/ri_paths'
+require 'rdoc/usage'
+require 'logger'
 
 require File.dirname(__FILE__) + '/../lib/pdtp/client'
 
-uri = nil
-listen_port = 8000
+opts = GetoptLong.new(
+  [ '--help',   '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--output', '-o', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--listen', '-l', GetoptLong::OPTIONAL_ARGUMENT ]
+)
 
-OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options]"
-  opts.on("--url URL", "Fetch from the specified URL") do |u|
-    uri = URI.parse u
-  end
-  opts.on("--help", "Prints this usage info.") do
-    puts opts
-    exit
-  end
-  opts.on("--listen PORT", "Port to listen on") do |l|
-    listen_port = l.to_i
+filename = nil
+listen_port = 60860
+
+# Display usage information extraced from this file via RDoc
+def display_usage
+  File.open(__FILE__)  { |file| STDERR.write RDoc.find_comment(file).gsub(/^# ?/, '') }
+  exit
+end
+
+opts.each do |opt, arg|
+  case opt
+  when '--help'   then display_usage
+  when '--output' then filename = arg
+  when '--listen' then listen_port = arg.to_i
   end
-end.parse!
+end
+
+if ARGV.length != 1
+  puts "Missing URI argument (try --help)"
+  exit 0
+end
 
-raise "Please specify a URL in the form --url URL" unless uri
+uri = URI.parse ARGV.shift
+
+# Validate URI scheme
 raise "Only pdtp:// URLs are supported" unless uri.scheme == 'pdtp'
 
-options = { :listen_port => listen_port }
-options[:port] = uri.port unless uri.port.nil?
+STDERR.write "--#{Time.now.strftime("%H:%M:%S")}--  #{uri}\n"
+
+filename ||= File.basename(uri.path) 
+
+STDERR.write "           => '#{filename}'\n"
+
+# Open the output file
+output = case filename
+when '-' then STDOUT
+else open(filename, 'w')
+end
+
+class Callbacks < PDTP::Client::Callbacks
+  attr_accessor :uri
+  attr_accessor :output
+  
+  def connected
+    STDERR.write "Connected to #{uri.host}\n"
+    STDERR.write "Beginning transfer...\n"
+    
+    # Begin fetching the file once we've connected
+    client.get uri.path, output
+  end
+  
+  def disconnected
+    STDERR.write "Disconnected from #{uri.host}\n"
+    client.stop
+  end
+end
 
-PDTP::Client.get uri.host, uri.path, options
+@@log = Logger.new(STDERR)
+@@log.level=Logger::INFO
+  
+@client = PDTP::Client.new uri.host, uri.port || 6086, :listen_port => listen_port
+@client.connect(Callbacks) { |c| c.uri, c.output = uri, output }

data/bin/dsseed

@@ -14,90 +14,19 @@ require 'rubygems'
 require 'eventmachine'
 require 'mongrel'
 
-require File.dirname(__FILE__) + '/../lib/pdtp/client'
+require File.dirname(__FILE__) + '/../lib/pdtp/common/common_init'
+require File.dirname(__FILE__) + '/../lib/pdtp/server/file_service_protocol'
 
-common_init $0
-
-# Fine all suitable files in the give path
-def find_files(base_path)
-  require 'find'
-
-  found = []
-  excludes = %w{.svn CVS}
-  base_full = File.expand_path(base_path)
-
-  Find.find(base_full) do |path|
-    if FileTest.directory?(path)
-      next unless excludes.include?(File.basename(path))
-      Find.prune
-    else
-      filename = path[(base_path.size - path.size + 1)..-1] #the entire file path after the base_path
-      found << filename
-    end
-  end
-
-  found
-end
-
-# Implements the file service for the pdtp protocol
-class FileServiceProtocol < PDTP::Protocol
-  def initialize(*args)
-    super
-  end
-
-  # Called after a connection to the server has been established
-  def connection_completed
-    begin
-      listen_port = @@config[:listen_port]
-
-      #create the client
-      client = PDTP::Client.new
-      PDTP::Protocol.listener = client
-      client.server_connection = self
-      client.generate_client_id listen_port
-
-      # Start a mongrel server on the specified port.  If it isnt available, keep trying higher ports
-      begin
-        mongrel_server=Mongrel::HttpServer.new '0.0.0.0', listen_port
-      rescue Exception=>e
-        listen_port+=1
-        retry
-      end
-
-      @@log.info "listening on port #{listen_port}"
-      mongrel_server.register "/", client
-      mongrel_server.run
-
-      # Tell the server a little bit about ourself
-      send_message :client_info, :listen_port => listen_port, :client_id => client.my_id
-
-      @@log.info 'This client is providing'
-      sfs = PDTP::Server::FileService.new
-      sfs.root = @@config[:file_root]
-      client.file_service = sfs #give this client access to all data
-
-      hostname = @@config[:vhost]
-
-      # Provide all the files in the root directory
-      files = find_files @@config[:file_root]
-      files.each { |file| send_message :provide, :url => "http://#{hostname}/#{file}" }
-    rescue Exception => e
-      puts "Exception in connection_completed: #{e}"
-      puts e.backtrace.join("\n")
-      exit
-    end
-  end  
-
-  def unbind
-    super
-    puts "Disconnected from PDTP server."
-  end
-end
+common_init File.basename($0)
 
 # Run the EventMachine reactor loop
 EventMachine::run do
   host, port, listen_port = @@config[:host], @@config[:port], @@config[:listen_port]
-  connection = EventMachine::connect host, port, FileServiceProtocol
+  
+  connection = EventMachine::connect(host, port, PDTP::Server::FileService::Protocol) do |c|
+    c.instance_eval { @base_path = @@config[:file_root] }
+  end
+  
   @@log.info "connecting with ev=#{EventMachine::VERSION}"
   @@log.info "host= #{host}  port=#{port}"
 end

data/bin/dstream

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require File.dirname(__FILE__) + '/../lib/pdtp/common/common_init'
+require File.dirname(__FILE__) + '/../lib/pdtp/server'
+
+common_init File.basename($0)
+
+server = PDTP::Server.new @@config[:host], @@config[:port]
+server.enable_stats_service
+server.enable_file_service @@config[:file_root], :chunk_size => @@config[:chunk_size]
+server.run

data/conf/bigchunk.yml

@@ -15,4 +15,4 @@
 :file_root: /Users/tony/dstest/files
 
 # Size of segments to be distributed (in bytes)
-:chunk_size: 250000
+:chunk_size: 500000

data/conf/debug.yml

@@ -15,4 +15,4 @@
 :file_root: /Users/tony/dstest/files
 
 # Size of segments to be distributed (in bytes)
-:chunk_size: 5000
+:chunk_size: 100000

data/conf/example.yml

@@ -15,4 +15,4 @@
 :file_root: /Users/tony/dstest/files
 
 # Size of segments to be distributed (in bytes)
-:chunk_size: 5000
+:chunk_size: 100000

data/distribustream.gemspec

@@ -2,8 +2,8 @@ require 'rubygems'
 
 GEMSPEC = Gem::Specification.new do |s|
   s.name = "distribustream"
-  s.version = "0.1.0"
-  s.date = "2008-10-11"
+  s.version = "0.2.0"
+  s.date = "2008-10-22"
   s.summary = "DistribuStream is a fully open peercasting system allowing on-demand or live streaming media to be delivered at a fraction of the normal cost"
   s.email = "tony@clickcaster.com"
   s.homepage = "http://distribustream.rubyforge.org"
@@ -13,7 +13,7 @@ GEMSPEC = Gem::Specification.new do |s|
   s.extra_rdoc_files = ["COPYING", "README", "CHANGES"]
   s.authors = ["Tony Arcieri", "Ashvin Mysore", "Galen Pahlke", "James Sanders", "Tom Stapleton"]
   s.files = Dir.glob("{bin,lib,conf}/**/*") + ['Rakefile', 'distribustream.gemspec']
-  s.executables = %w{distribustream dsseed dsclient}
+  s.executables = %w{dstream dsseed dsclient}
   s.add_dependency("eventmachine", ">= 0.9.0")
   s.add_dependency("mongrel", ">= 1.0.1")
   s.add_dependency("json", ">= 1.1.0")

data/lib/pdtp/client/callbacks.rb

@@ -0,0 +1,29 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+module PDTP
+  class Client
+    # Callbacks 
+    class Callbacks
+      attr_accessor :client
+      
+      def initialize(connection)
+        @connection = connection
+      end
+      
+      def connected
+      end
+      
+      def disconnected
+        client.stop
+      end
+    end
+  end
+end

data/lib/pdtp/client/connection.rb

@@ -0,0 +1,114 @@
+#--
+# Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
+# All rights reserved.  See COPYING for permissions.
+# 
+# This source file is distributed as part of the 
+# DistribuStream file transfer system.
+#
+# See http://distribustream.rubyforge.org/
+#++
+
+require 'rubygems'
+require 'mongrel'
+
+require File.dirname(__FILE__) + '/../common/protocol.rb'
+
+module PDTP
+  class Client
+    # Implementation of the PDTP client protocol
+    class Connection < PDTP::Protocol
+      attr_accessor :client
+      attr_accessor :callbacks
+
+      # Called after a connection to the server has been established
+      def connection_completed
+        begin
+          #create the client
+          #PDTP::Protocol.listener = self
+
+          # Tell the server about ourself
+          send_message(:register,
+            :listen_port => @client.listen_port, 
+            :client_id => @client.client_id
+          )
+          
+          callbacks.connected
+        rescue Exception => e
+          puts "Exception in connection_completed: #{e}"
+          puts e.backtrace.join("\n")
+          exit
+        end
+      end      
+      
+      # Called when any server message is received. This is the brains of
+      # the client's protocol handling.
+      def receive_message(command, message)
+        case command
+        when "tell_info" # Receive and store information for this url
+          info = @client.file_service.get_info(message["url"])
+          #info = FileInfo.new message["url"].split('/').last
+          info.file_size = message["size"]
+          info.base_chunk_size = message["chunk_size"]
+          info.streaming = message["streaming"]
+          #@client.file_service.set_info(message["url"], info)   
+        when "transfer" # Begin a transfer as a connector
+          transfer = Transfer::Connector.new(
+            self,
+            message,
+            @client.file_service
+          )
+
+          @@log.debug "TRANSFER STARTING"
+
+          # Run each transfer in its own thread and notify the server upon completion
+          Thread.new(transfer) do |t|
+            begin
+              t.run
+            rescue Exception=>e
+              @@log.info("Exception in dispatch_message: " + e.exception + "\n" + e.backtrace.join("\n")) 
+            end
+            t.send_completed_message(t.hash)
+          end
+        when "tell_verify"
+          # We are a listener, and asked for verification of a transfer from a server.
+          # After asking for verification, we stopped running, and must be restarted
+          # if verification is successful
+
+          found=false
+          @client.transfers.each do |t|
+            if t.matches_message?(message)
+              finished(t)
+              t.tell_verify(message["authorized"])
+              found=true
+              break
+            end
+          end
+
+          unless found
+            puts "BUG: Tell verify sent for an unknown transfer"
+            exit!
+          end
+        when "hash_verify"
+          @@log.debug "Hash verified for url=#{message["url"]} range=#{message["range"]} hash_ok=#{message["hash_ok"]}"
+        when "protocol_error", "protocol_warn" #ignore
+        else raise "Server sent an unknown message type: #{command} "
+        end
+      end
+
+      def unbind
+        super
+        puts 'Disconnected from PDTP server.'
+      end
+      
+      #Prints the number of transfers associated with this client
+      def print_stats
+        @@log.debug "client:  num_transfers=#{@client.transfers.size}"
+      end
+
+      #Provides a threadsafe mechanism for transfers to report themselves finished
+      def finished(transfer)
+        @client.lock.synchronize { @client.transfers.delete(transfer) }
+      end
+    end
+  end
+end