leecher 0.0.1 → 0.1.0

data/bin/leecher

@@ -6,6 +6,7 @@ module Leecher
     require "optparse"
     require "yaml"
     require "erubis"
+    require "fileutils"
     require "leecher/log"
     require "leecher/client"
 
@@ -17,7 +18,7 @@ module Leecher
 
     
     def initialize()
-      @config_fn = File.join(ENV["HOME"], ".leecher.yml")
+      @config_dirname = File.join(ENV["HOME"], ".leecher")
       @foreground = false
     end
 
@@ -29,10 +30,10 @@ module Leecher
         o.separator("")
         o.separator("options:")
         
-        o.on("--config=FILE",
-             "File containing settings",
-             "Default: #{@config_fn}") do |v|
-          @config_fn = v
+        o.on("--config=DIR",
+             "Dir containing settings",
+             "Default: #{@config_dirname}") do |v|
+          @config_dirname = v
         end
 
         o.on("-f", "--[no-]foreground",
@@ -41,12 +42,6 @@ module Leecher
           @foreground = v
         end
 
-        o.on("--getting-started",
-             "Run a tutorial which will give you a config") do
-          do_getting_started()
-          Kernel.exit!(RC_OK)
-        end
-
         o.separator("")
         o.on("-h", "--help", "You're reading it :-)") do
           puts o
@@ -59,28 +54,41 @@ module Leecher
     end
 
     def do_getting_started()
-      log.info("We'll ask you a few questions then write you a config in #{@config_fn}")
+      log.info("We'll ask you a few questions then write you a config in #{@config_dirname}")
       log.info(".. you can change where your config lives with -c/--config")
 
-      STDOUT.puts("\n\n")
-      config = Hash.new()
+      STDOUT.puts("\n")
+      config = { :config_dirname => @config_dirname }
       [
        [:username, "your username on the website?", :required],
        [:password, ".. and the password for that user?", :password],
-       [:log_fn, "filename to log to:", :disable],
+       [:log_fn, "filename to log to:", File.join(@config_dirname, "log")],
        [:host, "where does the amqp queue live?", "nzb.trim.za.net"],
        [:aria2_bin, "which aria2 must I use?", %x{which aria2c}.strip()],
+       [:aria2_dir, "what whould you like your default download directory to be?", Dir.pwd],
+       [:aria2_connections, "how many connections would you like to download with?", 5],
+       [:aria2_rpc_user, "what is your aria2 rpc username?", :required],
+       [:aria2_rpc_password, "what is your aria2 rpc password?", :password],
+       [:aria2_rpc_port, "what is your aria2 rpc port?", 6800],
       ].each do |setting, question, default|
         config[setting] = get_user_input(question, default)
       end
-      
+
+      FileUtils.mkdir_p(@config_dirname)
       erb = Erubis::Eruby.new(File.read(File.join(File.dirname(__FILE__),
                                                   "../config/client.yml")))
-      File.open(@config_fn, "w") do |f|
+      File.open(File.join(@config_dirname, "config.yml"), "w") do |f|
+        f.puts(erb.result(config))
+      end
+      
+      erb = Erubis::Eruby.new(File.read(File.join(File.dirname(__FILE__),
+                                                  "../config/aria2.conf")))
+      
+      File.open(File.join(@config_dirname, "aria2.conf"), "w") do |f|
         f.puts(erb.result(config))
       end
 
-      STDOUT.puts("\n\n")
+      STDOUT.puts("\n")
       log.info("Looking good, now take us for a real spin!")
     end
 
@@ -124,7 +132,7 @@ module Leecher
     end
 
     def run()
-      config = load_config(@config_fn)
+      config = load_config(@config_dirname)
       unless @foreground
         log.info("Fading into the ether ..")
         make_daemon(config[:daemon])
@@ -143,17 +151,20 @@ module Leecher
       @log ||= Leecher::Log.new()
     end
 
-    def load_config(config_fn)
-      config = YAML.load_file(config_fn)
+    def load_config(config_dirname)
+      unless File.exist?(config_dirname)
+        log.info("You don't have a leecher config setup")
+        do_getting_started()
+      end
+      
+      config = YAML.load_file(File.join(config_dirname, "config.yml"))
       if (log_conf = config[:log])
         @log = Leecher::Log.new(config[:log][:level],
                                 config[:log][:filename])
       end
       config
     rescue Errno::ENOENT
-      log.error("No such config file #{config_fn.inspect()}")
-      log.info("New user? Try #{Leecher::Log::ANSI_GREEN}"\
-               "--getting-started#{Leecher::Log::ANSI_NORMAL}")
+      log.error("No such config file #{config_dirname.inspect()}")
       Kernel.exit!(RC_CONF_MISSING)
     end
 

data/config/aria2.conf

@@ -0,0 +1,15 @@
+dir=<%= aria2_dir %>
+log=/tmp/aria.log
+check-integrity=true
+max-connection-per-server=<%= aria2_connections %>
+min-split-size=1M
+follow-torrent=mem
+max-overall-upload-limit=30K
+follow-metalink=mem
+metalink-servers=<%= aria2_connections %>
+rpc-listen-all=false
+rpc-listen-port=<%= aria2_rpc_port %>
+rpc-passwd=<%= aria2_rpc_password %>
+rpc-user=<%= aria2_rpc_user %>
+allow-piece-length-change=true
+save-session=<%= File.join(config_dirname, "session.aria2") %>

data/config/client.yml

@@ -19,11 +19,14 @@
   :heartbeat: 300
   :connect_timeout: 5
 :aria2_opts:
+  :config: <%= File.join(config_dirname, "aria2.conf") %>
   :bin: <%= aria2_bin %>
-  :args: [-x5, -V, --follow-metalink=mem]
+  :args: [-D, --enable-rpc]
+  :user: <%= aria2_rpc_user %>
+  :password: <%= aria2_rpc_password %>
+  :port: <%= aria2_rpc_port %>
 :daemon:
-  :chdir: nil
+  :chdir: <%= File.join(config_dirname) %>
   :umask: 0000
-  :stdin: nil
-  :stdout: nil
-  :stderr: nil
+  :stdout: <%= File.join(config_dirname, "stdout") %>
+  :stderr: <%= File.join(config_dirname, "stderr") %>

data/lib/leecher/client.rb

@@ -4,10 +4,25 @@ module Leecher
 
     require "bunny"
     require "leecher/shellout"
+    require 'xmlrpc/client'
     
     include Leecher::Shellout
 
     class ShutdownError < RuntimeError; end
+
+    # This is annoying: Bunny's errors don't inherit from a
+    # Bunny::Error parent, so we have to keep track of em :-(
+    BunnyErrors = 
+      [
+       Bunny::ConnectionError,            
+       Bunny::ForcedChannelCloseError,    
+       Bunny::ForcedConnectionCloseError, 
+       Bunny::MessageError,               
+       Bunny::ProtocolError,              
+       Bunny::ServerDownError,            
+       Bunny::UnsubscribeError,           
+       Bunny::AcknowledgementError,       
+      ]
     
     attr_accessor :username
     attr_accessor :password
@@ -15,6 +30,7 @@ module Leecher
     attr_accessor :bunny_opts
     attr_accessor :aria2_opts
     attr_accessor :log
+    attr_accessor :rpc_server
     
     def initialize(username,
                    password,
@@ -28,6 +44,7 @@ module Leecher
       self.bunny_opts = bunny_opts
       self.aria2_opts = aria2_opts
       self.log = log
+      self.rpc_server = make_rpc_client(aria2_opts)
     end
 
 
@@ -39,18 +56,45 @@ module Leecher
       @dont_kill_me = false
       [:INT, :TERM].each do |sig|
         trap(sig) do
+          log.info("Caught SIG#{sig}")
+
+          # Already asked to go away - impatient users :-(
+          if @graceful_shutdown
+            log.info("I'm tired of catching this so I'm going to (unsafely) go away.")
+            raise ShutdownError.new("SIG#{sig}")
+          end
+          
           @graceful_shutdown = true
-          log.info("Caught SIG{sig}")
           raise ShutdownError.new("SIG#{sig}") unless @dont_kill_me
         end
       end
       
-      q = get_queue(@bunny, username, metalink_queue_name)
       until @graceful_shutdown
         begin
+          q = get_queue(@bunny, username, metalink_queue_name)
           drain_metalink_queue(q)
         rescue ShutdownError => e
           log.info(["Going away because of", e.message].join(": "))
+        rescue *BunnyErrors => e
+          log.warn("Something has gone wrong with our "\
+                   "connection. We're going to try start again.")
+          e.backtrace.each do |line|
+            log.debug(line)
+          end
+
+          # Silently try throw away our existing bunny.
+          if @bunny.status == :connected
+            @bunny.stop() rescue Exception
+          end
+
+          log.debug("We've shot the old bunny - we'll make a new one in "\
+                    "a few seconds.")
+          Kernel.sleep(3)
+
+          # Start again.
+          log.debug("Reconnecting to rabbitmq")
+          @bunny = make_bunny(self.bunny_opts)
+          @bunny.start()
         rescue Exception => e
           log.error([e.class.name, e.message].join(": "))
           e.backtrace.each do |line|
@@ -67,35 +111,29 @@ module Leecher
     def get_queue(bunny, *queue_joinable_name)
       bunny.queue(File.join(*queue_joinable_name))
     end
-    
+
+    # Continuously get messages (metalinks) off the rabbitmq queue and
+    # invoke aria2c to download things.
     def drain_metalink_queue(q)
+      log.info("Subscribing to #{q.name.inspect()}")
+      
       next_queue_message(q) do |msg|
         @dont_kill_me = true
         
-        aria_exec = make_aria_exec(URI.parse(msg))
-        log.info("Will exec #{aria_exec.inspect()}")
-        stdout_rd, stdout_wr = IO.pipe()
-        stderr_rd, stderr_wr = IO.pipe()
-        rc = shellout(aria_exec, nil, stdout_wr, stderr_wr)
-        stdout_wr.close()
-        stderr_wr.close()
-
-        if rc == 0
-          log.info("aria2c exited happily")
+        log.info("Will call aria2.addUri #{msg.inspect()}")
+        
+        #FIXME is it possible something can go wrong but not throw an exception?
+        begin
+          status = call_rpc("aria2.addUri", [msg])
+          log.info("Added uri to aria with gid: #{status}")
           return true
-        else
-          log.error("aria2c exited unhappily (rc = #{rc})")
-          stdout_rd.each_line do |line|
-            log.error(line)
-          end
-          stderr_rd.each_line do |line|
-            log.debug(line)
-          end
+        rescue Exception => e
+          log.error("Failed to add uri to aria2c: #{e.message}")
           return false
         end
-        
-        @dont_kill_me = false
       end
+    ensure
+      @dont_kill_me = false
     end
 
     def next_queue_message(q, &block)
@@ -114,11 +152,52 @@ module Leecher
       end
     end
 
-    def make_aria_exec(uri)
+    def make_rpc_client(aria2_opts)
+      XMLRPC::Client.new2("http://#{aria2_opts[:user]}:#{aria2_opts[:password]}@127.0.0.1:#{aria2_opts[:port]}/rpc")
+    end
+    
+    def call_rpc(method, *args)
+      retried = false
+      begin
+        if args.nil? || args.empty?
+          rpc_server.call(method)
+        else
+          rpc_server.call(method, *args)
+        end
+      rescue Errno::ECONNREFUSED => e
+        if retried == true
+          raise e
+        else
+          retried = true
+          shellout(make_aria_exec, nil, nil, nil)
+          sleep 10
+          retry
+        end
+      rescue Errno::EPIPE => e
+        # The RPC client doesn't realise if the server closes the
+        # connection. So, we retry (which seems to remake the connection?).
+        if retried == true
+          raise e
+        else
+          retried = true
+          retry
+        end
+      end
+    end
+
+    def make_aria_exec
+      args = aria2_opts[:args]
+
+      if aria2_opts.has_key?(:config)
+        args.push([
+                   "--conf-path",
+                   aria2_opts[:config],
+                  ].join("="))
+      end
+      
       [
        aria2_opts[:bin],
-       aria2_opts[:args].join(" "),
-       uri.to_s(),
+       args.join(" "),
       ].join(" ")
     end
   end # class Client

data/lib/leecher/log.rb

@@ -61,6 +61,7 @@ module Leecher
       
       if @log_fd and not @log_fd.closed?
         @log_fd.puts("%s %s %s" % [iso8601_timestr, level.to_s.upcase, message])
+        @log_fd.flush()
       end
     end
 
@@ -82,8 +83,9 @@ module Leecher
     }
     def log_for_humans(iso8601_timestr, log_level, log_msg)
       level_str, level_colour = HUMAN_LOG_LEVELS[log_level]
-      puts("#{ANSI_BOLD_WHITE}#{iso8601_timestr}#{ANSI_NORMAL} "\
-           "#{level_colour}#{"%5.5s" % level_str}#{ANSI_NORMAL} #{log_msg}")
+      STDOUT.puts("#{ANSI_BOLD_WHITE}#{iso8601_timestr}#{ANSI_NORMAL} "\
+                  "#{level_colour}#{"%5.5s" % level_str}#{ANSI_NORMAL} #{log_msg}")
+      STDOUT.flush()
     end
   end
 end

data/lib/leecher/shellout.rb

@@ -8,7 +8,7 @@ module Leecher
 
       if child_pid
         # Parent process executes this
-        child_status = (Process.waitpid2(child_pid)[1]).to_i >> 8
+        child_pid, child_status = Process.waitpid2(child_pid)
       else
         Process.setsid()
         STDIN.reopen(stdin || "/dev/null")

data/lib/leecher/version.rb

@@ -1,3 +1,3 @@
 module Leecher
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/spec/client_spec.rb

@@ -41,34 +41,38 @@ describe Leecher::Client do
   end
 
   describe "#drain_metalink_queue" do
-    
-    it "should reject URIs that are not URIs" do
-      @c.stub(:next_queue_message).
-        and_yield(nil)
-      lambda { @c.drain_metalink_queue(double("q")) }.
-        should raise_error(URI::InvalidURIError)
+
+    before do
+      @q = double("q")
+      @q.should_receive(:name).
+        any_number_of_times.
+        and_return("qq")
     end
 
-    it "should shellout to aria2" do
+    
+    it "should rpc to aria2" do
       @c.stub(:next_queue_message).
         and_yield("http://some/url")
-      @c.should_receive(:shellout).
-        with("/usr/bin/aria2c -V http://some/url",
-             anything(), anything(), anything()).
-        and_return(0)
-      @c.drain_metalink_queue(double("q")).should be_true
+      rpc_server = double("rpc_server")
+      rpc_server.stub(:call).
+        and_return(true)
+      @c.stub(:rpc_server).
+        and_return(rpc_server)
+      @c.drain_metalink_queue(@q).should be_true
     end
 
-    it "should return according to the rc" do
+    it "should return according to the success of the rpc" do
       @c.stub(:next_queue_message).
         and_yield("http://some/url")
-      @c.stub(:shellout).
-        and_return(0)
-      q = double("q")
-      @c.drain_metalink_queue(q).should be_true
-      @c.stub(:shellout).
-        and_return(1)
-      @c.drain_metalink_queue(q).should be_false
+      rpc_server = double("rpc_server")
+      @c.stub(:rpc_server).
+        and_return(rpc_server)
+      rpc_server.stub(:call).
+        and_return(true)
+      @c.drain_metalink_queue(@q).should be_true
+      rpc_server.stub(:call).
+        and_raise(RuntimeError)
+      @c.drain_metalink_queue(@q).should be_false
     end
   end
 end

data/spec/shellout_spec.rb

@@ -0,0 +1,26 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+require 'leecher/shellout'
+
+include Leecher::Shellout
+
+describe Leecher::Shellout do
+  it "should return 0 for true" do
+    shellout("/bin/true").success?.should be_true
+  end
+
+  it "should return 1 for false" do
+    shellout("/bin/false").success?.should be_false
+  end
+
+  it "should work with pipes" do
+    stdin_rd, stdin_wr = IO.pipe()
+    stdout_rd, stdout_wr = IO.pipe()
+    stdin_wr.puts("foo")
+    stdin_wr.close()
+    child_status = shellout("cat", stdin_rd, stdout_wr)
+    stdout_wr.close()
+    stdout_rd.gets().should == "foo\n"
+    stdout_rd.close()
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: leecher
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: 
   segments: 
   - 0
-  - 0
   - 1
-  version: 0.0.1
+  - 0
+  version: 0.1.0
 platform: ruby
 authors: 
 - Marc Bowes
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-22 00:00:00 +02:00
+date: 2011-05-09 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -74,6 +74,7 @@ files:
 - Gemfile
 - Rakefile
 - bin/leecher
+- config/aria2.conf
 - config/client.yml
 - leecher.gemspec
 - lib/leecher.rb
@@ -82,6 +83,7 @@ files:
 - lib/leecher/shellout.rb
 - lib/leecher/version.rb
 - spec/client_spec.rb
+- spec/shellout_spec.rb
 - spec/spec_helper.rb
 has_rdoc: true
 homepage: ""