snoopy 0.0.1

data/bin/snoopy

@@ -0,0 +1,97 @@
+#!/usr/local/bin/ruby
+#
+#  Created by me on 2007-12-24.
+#  Copyright (c) 2007. All pwnage reserved.
+
+require "rubygems"
+require "snoopy"
+
+##################################################################################
+
+opts = Trollop::options do
+  version "snoopy 0.0.4 (c) Ari Brown"
+  banner <<-EOS
+  snoopy is a multithreaded golden retriever for the internet
+  --> handles http, ftp (single threaded), and torrent
+
+  Usage:
+  snoopy <URI of file> [options]
+  Example:
+  snoopy http://www.lighttpd.net -t 5 option
+  where [options] are:
+  EOS
+
+  opt :verbose, "Be as loud as possible"
+  opt :clean, "Remove the out file after finishing (debugging)"
+  opt :out, "Where to save the file", :type => String
+  opt :threads, "Set number of threads to use", :default => 5
+  opt :dots, "Display a dot every X times a block comes in", :default => 10
+end
+Trollop::die "need a file to download" if ARGV.empty?
+
+##################################################################################
+
+opts[:in] = ARGV[0]
+
+url = URI.parse(opts[:in])
+
+ball = Source.new(url, opts)
+
+def start_report(opts)
+  puts Time.now
+  puts "Downloading #{opts[:in]}"
+  puts "with #{opts[:threads]} thread" + (opts[:threads] > 1 ? "s" : '')
+  puts
+end
+
+def determine_size(value)
+  if value >= 1073741824
+    end_value = value / 1073741824
+    unit = 'GB'
+  elsif value >= 1048576 and value < 1073741824
+    end_value = value / 1048576
+    unit = 'MB'
+  elsif value >= 1024 and value < 1048576
+    end_value = value / 1024
+    unit = 'KB'
+  else
+    end_value = value
+    unit = 'B'
+  end
+
+  [end_value, unit, value]
+end
+
+def end_report(source, opts)
+  puts
+  puts "Download file written to " + opts[:out] + (opts[:clean] ? ", and then deleted" : '')
+  puts "Elapsed time: " + $elapsed_time.to_s + " seconds"
+
+  end_value, unit, actual = determine_size(source.length)
+
+  if end_value.zero?
+    end_value, unit, actual = determine_size(File.size(opts[:out]))
+  end
+
+  puts "Total downloaded: " + end_value.to_s + " " + unit
+  res = determine_size(actual/$elapsed_time)
+  puts "At %.2f%s/s" % [res[0], res[1]]
+
+  source.finish
+
+
+end
+
+
+
+start_report(opts)
+##################################################################################
+$start_time = Time.now
+##################################################################################
+
+ball.roll
+
+##################################################################################
+$elapsed_time = Time.now - $start_time
+##################################################################################
+end_report(ball, opts)

data/lib/ftp.rb

@@ -0,0 +1,22 @@
+#!/usr/local/bin/ruby
+#
+#  Created by me on 2007-12-28.
+#  Copyright (c) 2007. All pwnage reserved.
+
+module FTP
+  
+  def setup(opts, uri)
+    @opts = opts
+  end
+  
+  def prep
+  end
+  
+  def fetch
+    Net::FTP.open(@opts[:in]) {|ftp| ftp.getbinaryfile(@opts[:in], @opts[:out]) }
+  end
+  
+  def clean
+  end
+  
+end

data/lib/http.rb

@@ -0,0 +1,94 @@
+#!/usr/local/bin/ruby
+#
+#  Created by me on 2007-12-28.
+#  Copyright (c) 2007. All pwnage reserved.
+
+module HTTP
+  class << self
+    def setup(opts, uri)
+      @opts = opts
+      @url = uri
+    end
+
+    def prep
+      req = Net::HTTP.new(@url.host, @url.port)
+      len = req.request_head(@url.page)['content-length']
+      @url.length = len.to_i
+
+      if len.nil? or @url.length == 0
+        @opts[:threads] = 1
+        puts "--> Uhoh! Server isn't cooperating..."
+        puts "--> Changing threadcount to 1"
+        alias_method :fetch_multi, :fetch
+        alias_method :fetch, :fetch_single
+      end
+    end
+
+    def fetch_single
+      p single
+      open(@opts[:out], "w") do |f|
+        http = Net::HTTP.new(@url.host, @url.port)
+        http.get(@url.page) do |str|
+          f.write str
+        end
+      end
+    end
+
+    def fetch
+      i = 0
+      array = []
+      start = 0
+      jump = @url.length / @opts[:threads]
+      finish = -1                     # -1 so that start becomes 0
+
+      @opts[:threads].times do |t|
+        start = finish + 1
+        finish = (start + jump) - 1
+        array << [start, finish, t + 1]
+      end
+
+      array.each_simultaneously { |d, b, w|
+        puts "[LOG] Setting up a new thread..." if @opts[:verbose]
+        open(@opts[:out], "a+") do |f|
+          begin
+            f.seek(d)
+            Net::HTTP.new(@url.host, @url.port).start do |http|
+              http.get(@url.page, {'range' => "bytes=#{d}-#{b}"}) do |str|
+                i += 1
+                print '.' if i % @opts[:dots] == 0
+                STDOUT.flush
+                f.write str if str
+              end
+            end
+          rescue Timeout::Error => e
+            print "\n[Error] Error in server response in thread #{w}. Retrying!\n"
+            puts e
+            STDOUT.flush
+            sleep 5
+            retry
+          rescue => e
+            print "\n[Error] Error in server response in thread #{w}. Retrying!\n"
+            puts e
+            STDOUT.flush
+            sleep 5
+            retry
+          end
+        end
+        puts "[LOG] Thread #{w} finished!" if @opts[:verbose]
+      }
+    end
+
+    def clean
+      remove_extra_files
+    end
+
+    def remove_extra_files
+      puts "[LOG] Writing #{@opts[:out]}" if @opts[:verbose]
+
+      if @opts[:clean]
+        File.delete(@opts[:out])
+        puts "[LOG] Removing #{@opts[:out]}" if @opts[:verbose]
+      end
+    end
+  end
+end

data/lib/https.rb

@@ -0,0 +1,113 @@
+#!/usr/local/bin/ruby
+#
+#  Created by me on 2007-12-28.
+#  Copyright (c) 2007. All pwnage reserved.
+
+module HTTPS
+  class << self
+    def setup(opts, uri)
+      @opts = opts
+      @url = uri
+    end
+
+    def prep
+      req = Net::HTTP.new(@url.host, @url.port)
+      req.use_ssl = true
+      len = req.request_head(@url.page)['content-length']
+      @url.length = len.to_i
+
+      if len.nil?
+        @opts[:threads] = 1
+        puts "--> Uhoh! Server isn't cooperating..."
+        puts "--> Changing threadcount to 1"
+      end
+    end
+
+    def fetch
+      array = []
+      start = 0
+      jump = @url.length / @opts[:threads]
+      finish = -1                     # -1 so that start becomes 0
+
+      @opts[:threads].times do |t|
+        start = finish + 1
+        finish = (start + jump) - 1
+        array << [start, finish, t + 1]
+      end
+
+      array.each_simultaneously { |d, b, w|
+        puts "[LOG] Setting up a new thread..." if @opts[:verbose]
+        p @opts[:out]
+        open(@opts[:out], "a+") do |f|
+          f.seek(d)
+          http = Net::HTTP.new(@url.host, @url.port)
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          c = http.start do |http|
+            a = Net::HTTP::Get.new(@url.page)
+            a.range = (d..b)
+            http.request(a)
+          end
+          f.write(c.body)
+        end
+      }
+    end
+=begin  
+    def fetch_fat
+      array = []
+      start = 0
+      jump = @url.length / @opts[:threads]
+      finish = -1                     # -1 so that start becomes 0
+
+      @opts[:threads].times do |t|
+        start = finish + 1
+        finish = (start + jump) - 1
+        array << [start, finish, t + 1]
+      end
+
+      array.each_simultaneously { |d, b, w|
+        puts "[LOG] Setting up a new thread..." if @opts[:verbose]
+        open("/tmp/part#{w}", "w") do |f|
+          http = Net::HTTP.new(@url.host, @url.port)
+          c = http.start do |http|
+            a = Net::HTTP::Get.new(@url.page)
+            a.range = (d..b)
+            http.request(a)
+          end
+          f.write(c.body)
+        end
+      }
+    end
+=end
+    def clean
+      mash_files
+      remove_extra_files
+    end
+
+    def remove_extra_files
+      puts "[LOG] Writing #{@opts[:out]}" if @opts[:verbose]
+
+      Dir["/tmp/part*"].each do |part|
+        File.delete(part)
+        puts "[LOG] Removing #{part}" if @opts[:verbose]
+      end
+
+      if @opts[:clean]
+        File.delete(@opts[:out])
+        puts "[LOG] Removing #{@opts[:out]}" if @opts[:verbose]
+      end
+    end
+
+    def mash_files
+      open(@opts[:out], "a") do |f|
+        files = Dir["/tmp/part*"].sort_by { |word|
+          word.scan(/\d+/)[0].to_i
+        }
+        files.each do |part|
+          puts "[LOG] Reading #{part}" if @opts[:verbose]
+          f.print open(part, "r").read
+        end
+      end
+    end
+  end
+end

data/lib/snoopy.rb

@@ -0,0 +1,22 @@
+require "rubygems"
+require "trollop"
+require "net/http"
+require "net/https"
+require "net/ftp"
+require "rubytorrent"
+require "tempfile"
+require "open-uri"
+
+require File.join(File.dirname(__FILE__), "source")
+require File.join(File.dirname(__FILE__), "http")
+require File.join(File.dirname(__FILE__), "https")
+require File.join(File.dirname(__FILE__), "ftp")
+
+
+module Enumerable
+  def each_simultaneously
+    threads = []
+    each { |e, f, i| threads << Thread.new { yield e, f, i } }
+    return threads.each {|t| t.join }
+  end
+end

data/lib/source.rb

@@ -0,0 +1,50 @@
+#!/usr/local/bin/ruby
+#
+#  Created by me on 2007-12-28.
+#  Copyright (c) 2007. All pwnage reserved.
+
+class Source
+  
+  attr_accessor :port, :scheme, :host, :path, :length, :page, :full
+  
+  def initialize(uri_obj, opts)
+    @uri = uri_obj
+    @host = uri_obj.host
+    @port = uri_obj.port
+    @scheme = uri_obj.scheme
+    @page = (uri_obj.path =~ /^$/ ? "/" : uri_obj.path)
+    @full = @scheme + '://' + @host + @page
+    @in = opts[:in]
+    @out = opts[:out]
+    opts[:out] = opts[:out] || (File.basename(@page) =~ /^\/$/ ? 'default' : File.basename(@page))
+    opts[:out] = (File.directory?(opts[:out]) ? File.join(opts[:out], File.basename(@page)) : opts[:out])
+    
+    case @scheme
+    when "http" then @prefix = HTTP
+    when "https" then @prefix = HTTPS
+    when "ftp" then @prefix = FTP
+    when "torrent" then @prefix = Torrent
+    else
+      abort "Unknown URL scheme!\nsnoopy --help for help and usage"
+    end
+    
+    prep(opts)
+    
+  end
+  
+  def start
+    @prefix.prep
+    @prefix.fetch
+  end
+  alias_method :roll, :start
+  
+  def finish
+    @prefix.clean
+  end
+  
+  private
+  def prep(opts)
+    @prefix.setup(opts, self)
+  end
+  
+end

data/lib/torrent.rb

@@ -0,0 +1,90 @@
+#!/usr/local/bin/ruby
+#
+#  Created by me on 2007-12-28.
+#  Copyright (c) 2007. All pwnage reserved.
+
+module Torrent
+  class << self
+
+    def setup(opts, uri)
+      @opts = opts
+      @url = uri
+    end
+
+    def prep
+      begin
+        mi = RubyTorrent::MetaInfo.from_location(File.join(@url.host, @url.path), ENV["http_proxy"])
+      rescue RubyTorrent::MetaInfoFormatError, RubyTorrent::BEncodingError => e
+        abort "[Error] Can't parse metainfo file '#{torrent}' -- maybe not a .torrent?"
+      rescue RubyTorrent::TypedStructError => e
+        raise e
+      rescue IOError, SystemCallError => e
+        abort "[Error] Can't read file '#{torrent}': #{e.message}"
+      end
+
+
+      if FileTest.directory?(@opts[:out]) && mi.info.single?
+        dest = File.join(@opts[:out], mi.info.name)
+      elsif FileTest.file?(@opts[:out]) && mi.info.multiple?
+        abort "[Error] .torrent contains multiple files, but '#{dest}' is a single file (must be a directory)"
+      end
+      dest ||= @opts[:out]
+
+      print "[LOG] Checking file status: " if @opts[:verbose]
+      package = RubyTorrent::Package.new(mi, dest) do |piece|
+        print(piece.complete? && piece.valid? ? "#" : ".") if @opts[:verbose]
+      end
+      puts " done" if @opts[:verbose]
+    end
+
+    def fetch
+      puts "[Log] Starting peer..." if @opts[:verbose]
+      bt = RubyTorrent::BitTorrent.new(mi, package, :http_proxy => proxy)
+
+      if @opts[:verbose]
+        bt.on_event(self, :added_peer) { |s, p| puts "[Log] Connected to peer #{p}" }
+        bt.on_event(self, :received_block) { |s, b, peer| puts "<-- got block #{b} from peer #{peer}, now #{package.pieces[b.pindex].percent_done.f}% done and #{package.pieces[b.pindex].percent_claimed.f}% claimed" }
+        bt.on_event(self, :sent_block) { |s, b, peer| puts "--> Sent block #{b} to peer #{peer}" }
+        bt.on_event(self, :discarded_piece) { |s, p| puts "[Log] Checksum error on piece #{p}, discarded" }
+        bt.on_event(self, :tracker_connected) { |s, url| puts "--> Connected to tracker #{url}" }
+        bt.on_event(self, :tracker_lost) { |s, url| puts "--> Couldn't connect to tracker #{url}" }
+      end
+
+      # Always register this
+      bt.on_event(self, :complete) do
+        @url.length = bt.dlamt
+        puts "--> Download complete!"
+      end
+
+      puts "[Log] Listening on #{bt.ip}:#{bt.port}" if @opts[:verbose]
+
+
+      thread = Thread.new do
+        until bt.complete?
+          previous = ''
+
+          if bt.tracker
+
+            current = '%s: %dk of %dk (%.2f%% complete)' % [Time.now,
+                                                            bt.bytes_completed / 1024,
+                                                            bt.total_bytes / 1024,
+                                                            bt.percent_completed]
+            print "\b" * previous.size
+            print curent
+            previous = ''
+
+            sleep(5)
+          else
+            sleep(5)
+          end
+        end
+      end
+
+      thread.join
+    end
+
+    def clean
+    end
+
+  end
+end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification 
+name: snoopy
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Ari Brown
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-01-14 00:00:00 -05:00
+default_executable: snoopy
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rubytorrent
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.3"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: trollop
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.0.1
+    version: 
+description: Segmented downloader for HTTP(S)/FTP/Bit Torrent.
+email: ari@aribrown.com
+executables: 
+- snoopy
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/ftp.rb
+- lib/http.rb
+- lib/https.rb
+- lib/snoopy.rb
+- lib/source.rb
+- lib/torrent.rb
+- bin/snoopy
+has_rdoc: false
+homepage: http://snoopy.rubyforge.org/
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.0.0
+signing_key: 
+specification_version: 2
+summary: Snoopy is a golden retriever for the internet. It's a great big world and there... And you need a great big dog to do your bidding.
+test_files: []
+