leecher 0.2.3 → 1.0.0

data/config/arguments.rb

@@ -0,0 +1,12 @@
+# Argument handling for your daemon is configured here.
+#
+# You have access to two variables when this file is
+# parsed. The first is +opts+, which is the object yielded from
+# +OptionParser.new+, the second is +@options+ which is a standard
+# Ruby hash that is later accessible through
+# DaemonKit.arguments.options and can be used in your daemon process.
+
+# Here is an example:
+# opts.on('-f', '--foo FOO', 'Set foo') do |foo|
+#  @options[:foo] = foo
+# end

data/config/aria2.example.yml

@@ -0,0 +1,6 @@
+config: /home/$USER/.leecher/aria2.conf
+bin: /usr/bin/aria2c
+args: [-D, --enable-rpc]
+user: aria2
+password: aria2
+port: 6800

data/config/boot.rb

@@ -0,0 +1,64 @@
+# Don't change this file!
+# Configure your daemon in config/environment.rb
+
+DAEMON_ROOT = "#{File.expand_path(File.dirname(__FILE__))}/.." unless defined?( DAEMON_ROOT )
+
+require "rubygems"
+require "bundler/setup"
+
+module DaemonKit
+  class << self
+    def boot!
+      unless booted?
+        pick_boot.run
+      end
+    end
+
+    def booted?
+      defined? DaemonKit::Initializer
+    end
+
+    def pick_boot
+      (vendor_kit? ? VendorBoot : GemBoot).new
+    end
+
+    def vendor_kit?
+      File.exists?( "#{DAEMON_ROOT}/vendor/daemon-kit" )
+    end
+  end
+
+  class Boot
+    def run
+      load_initializer
+      DaemonKit::Initializer.run
+    end
+  end
+
+  class VendorBoot < Boot
+    def load_initializer
+      require "#{DAEMON_ROOT}/vendor/daemon-kit/lib/daemon_kit/initializer"
+    end
+  end
+
+  class GemBoot < Boot
+    def load_initializer
+      begin
+        require 'rubygems' unless defined?( ::Gem )
+        gem 'daemon-kit'
+        require 'daemon_kit/initializer'
+      rescue ::Gem::LoadError => e
+        msg = <<EOF
+
+You are missing the daemon-kit gem. Please install the following gem:
+
+sudo gem install daemon-kit
+
+EOF
+        $stderr.puts msg
+        exit 1
+      end
+    end
+  end
+end
+
+DaemonKit.boot!

data/config/environment.rb

@@ -0,0 +1,22 @@
+# Be sure to restart your daemon when you modify this file
+
+# Uncomment below to force your daemon into production mode
+ENV['DAEMON_ENV'] ||= "production"
+
+# Boot up
+require File.join(File.dirname(__FILE__), 'boot')
+
+# Auto-require default libraries and those for the current Rails environment.
+Bundler.require :default, DaemonKit.env
+
+DaemonKit::Initializer.run do |config|
+
+  # The name of the daemon as reported by process monitoring tools
+  config.daemon_name = 'leecher'
+
+  # Force the daemon to be killed after X seconds from asking it to
+  # config.force_kill_wait = 30
+
+  # Log backraces when a thread/daemon dies (Recommended)
+  # config.backtraces = true
+end

data/config/environments/development.rb

@@ -0,0 +1,2 @@
+# This is the same context as the environment.rb file, it is only
+# loaded afterwards and only in the development environment

data/config/environments/production.rb

@@ -0,0 +1,5 @@
+# This is the same context as the environment.rb file, it is only
+# loaded afterwards and only in the production environment
+
+# Change the production log level to debug
+#config.log_level = :debug

data/config/environments/test.rb

@@ -0,0 +1,2 @@
+# This is the same context as the environment.rb file, it is only
+# loaded afterwards and only in the test environment

data/config/gheed.example.yml

@@ -0,0 +1,2 @@
+username: whoareyou
+password: proveit

data/config/leecher.example.yml

@@ -0,0 +1 @@
+work_dir: /home/$USER/.leecher

data/config/post-daemonize/readme

@@ -0,0 +1,5 @@
+# You can place files in here to be loaded after the code is daemonized.
+#
+# All the files placed here will just be required into the running
+# process. This is the correct place to open any IO objects, establish
+# database connections, etc.

data/config/pre-daemonize/amqp.rb

@@ -0,0 +1,6 @@
+begin
+  require 'amqp'
+rescue LoadError
+  $stderr.puts "Missing amqp gem. Please run 'bundle install'."
+  exit 1
+end

data/config/pre-daemonize/json.rb

@@ -0,0 +1,3 @@
+require "multi_json"
+require "yajl"
+MultiJson.engine = :yajl

data/config/pre-daemonize/readme

@@ -0,0 +1,12 @@
+# You can place files in here to be loaded before the code is daemonized.
+#
+# DaemonKit looks for a file named '<config.daemon_name>.rb' and loads
+# that file first, and inside a DaemonKit::Initializer block. The
+# remaning files then simply required into the running process.
+# 
+# These files are mostly useful for operations that should fail blatantly
+# before daemonizing, like loading gems.
+#
+# Be careful not to open any form of IO in here and expecting it to be
+# open inside the running daemon since all IO instances are closed when
+# daemonizing (including STDIN, STDOUT & STDERR).

data/config/pre-daemonize/safely.rb

@@ -0,0 +1,13 @@
+# Safely is responsible for providing exception reporting and the
+# logging of backtraces when your daemon dies unexpectedly. The full
+# documentation for safely can be found at
+# http://github.com/kennethkalmer/safely/wiki
+
+# By default Safely will use the daemon-kit's logger to log exceptions,
+# and will store backtraces in the "log" directory.
+
+# Comment out to enable Hoptoad support
+# Safely::Strategy::Hoptoad.hoptoad_key = ""
+
+# Comment out to use email exceptions
+# Safely::Strategy::Mail.recipient = "your.name@gmail.com"

data/leecher.gemspec

@@ -19,8 +19,17 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_development_dependency(%q<rspec>, [">=2"])
-  s.add_runtime_dependency(%q<bunny>)
-  s.add_runtime_dependency(%q<erubis>)
-  s.add_runtime_dependency(%q<yajl-ruby>)
+  s.add_runtime_dependency "daemon-kit"
+  s.add_runtime_dependency "safely"
+  s.add_runtime_dependency "multi_json"
+  s.add_runtime_dependency "yajl-ruby"
+  s.add_runtime_dependency "amqp"
+  s.add_runtime_dependency "erubis"
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+
+  if ENV["LEECHER_DEV"] && RUBY_VERSION == "1.9.3"
+    s.add_development_dependency "debugger"
+  end
 end

data/lib/leecher.rb

@@ -1,5 +1,27 @@
 module Leecher
-  
-  require "leecher/version"
+
   require "leecher/client"
+  require "leecher/gheed/client"
+  require "leecher/aria2/client"
+  
+  class << self
+
+    def config
+      DaemonKit::Config.load("leecher")
+    end
+
+    def gheed_client
+      config = DaemonKit::Config.load("gheed")
+      Leecher::Gheed::Client.new(config)
+    end
+
+    def aria2_client
+      config = DaemonKit::Config.load("aria2")
+      Leecher::Aria2::Client.new(config)
+    end
+
+    def new
+      Client.new(config, aria2_client, gheed_client)
+    end
+  end
 end

data/lib/leecher/aria2/client.rb

@@ -0,0 +1,65 @@
+module Leecher
+  module Aria2
+
+    require "xmlrpc/client"
+
+    class UnableToStartAriaError < RuntimeError; end
+    
+    class Client < XMLRPC::Client
+
+      attr_reader :config
+      
+      def initialize(config)
+        @config = config
+        super("127.0.0.1", "/rpc", config.port, nil, nil, config.user, config.password)
+      end
+      
+      def call(method, *args)
+        retried = false
+        super("aria2.#{method}", *args)
+      rescue Errno::ECONNREFUSED
+        start_aria2
+        Kernel.sleep(3) # Give it a chance to start.
+        retry unless retried
+      end
+
+      def add_uri(uris, options)
+        call("addUri", uris, options)
+      end
+
+      def tell_status(gid, keys = nil)
+        if keys.nil? or keys.empty?
+          call("tellStatus", gid.to_s)
+        else
+          call("tellStatus", gid.to_s, keys)
+        end
+      end
+
+      def make_aria_exec
+        args = config.args.dup
+        [
+         ["--conf-path", config.config],
+         ["--on-download-complete", callback_command],
+         ["--on-download-error", callback_command],
+         ["--on-download-pause", callback_command],
+         ["--on-download-start", callback_command],
+         ["--on-download-stop", callback_command],
+        ].each do |option , value|
+          args.push(option + "=" + value)
+        end
+
+        [config.bin, args.join(" ")].join(" ")
+      end
+
+      def start_aria2
+        command = make_aria_exec
+        IO.popen(command) {}
+        raise UnableToStartAriaError unless $?.success?
+      end
+
+      def callback_command
+        File.expand_path("../../../bin/leecher-aria2-event-hook", File.dirname(__FILE__))
+      end
+    end
+  end
+end

data/lib/leecher/client.rb

@@ -1,226 +1,46 @@
 module Leecher
-  
   class Client
 
-    require "bunny"
-    require "leecher/shellout"
-    require "xmlrpc/client"
-    require "yajl"
-    
-    include Leecher::Shellout
-
-    class ShutdownError < RuntimeError; end
-
-    # This is annoying: Bunny's errors don't inherit from a
-    # Bunny::Error parent, so we have to keep track of em :-(
-    BunnyErrors = 
-      [
-       Bunny::ConnectionError,            
-       Bunny::ForcedChannelCloseError,    
-       Bunny::ForcedConnectionCloseError, 
-       Bunny::MessageError,               
-       Bunny::ProtocolError,              
-       Bunny::ServerDownError,            
-       Bunny::UnsubscribeError,           
-       Bunny::AcknowledgementError,       
-      ]
-    
-    attr_accessor :username
-    attr_accessor :password
-    attr_accessor :metalink_queue_name
-    attr_accessor :bunny_opts
-    attr_accessor :aria2_opts
-    attr_accessor :log
-    attr_accessor :rpc_server
-    
-    def initialize(username,
-                   password,
-                   metalink_queue_name,
-                   bunny_opts,
-                   aria2_opts,
-                   log)
-      self.username = username
-      self.password = password
-      self.metalink_queue_name = metalink_queue_name
-      self.bunny_opts = bunny_opts
-      self.aria2_opts = aria2_opts
-      self.log = log
-      self.rpc_server = make_rpc_client(aria2_opts)
-    end
-
-
-    def run()
-      @graceful_shutdown = false
-      @dont_kill_me = false
-      [:INT, :TERM].each do |sig|
-        trap(sig) do
-          log.info("Caught SIG#{sig}")
-
-          # Already asked to go away - impatient users :-(
-          if @graceful_shutdown
-            log.info("I'm tired of catching this so I'm going to (unsafely) go away.")
-            raise ShutdownError.new("SIG#{sig}")
-          end
-          
-          @graceful_shutdown = true
-          raise ShutdownError.new("SIG#{sig}") unless @dont_kill_me
-        end
-      end
-      
-      until @graceful_shutdown
-        begin
-          b = make_bunny(self.bunny_opts)
-          q = get_queue(b, username, metalink_queue_name)
-          drain_metalink_queue(q)
-        rescue ShutdownError => e
-          log.info(["Going away because of", e.message].join(": "))
-        rescue *BunnyErrors => e
-          log.warn("Something has gone wrong with our "\
-                   "connection. We're going to try start again.")
-          e.backtrace.each do |line|
-            log.debug(line)
-          end
-
-          # Silently try throw away our existing bunny.
-          if b && b.status == :connected
-            b.stop() rescue Exception
-          end
-
-          log.debug("Will reconnect after 3s")
-        rescue Exception => e
-          log.error([e.class.name, e.message].join(": "))
-          e.backtrace.each do |line|
-            log.debug(line)
-          end
-        end
-      end
-    end
-
-    def make_bunny(bunny_opts)
-      b = Bunny.new(bunny_opts)
-      b.start()
-      b.qos()
-      b
-    end
+    require "fileutils"
+    require "multi_json"
+    require "leecher/metalink_queue"
 
-    def get_queue(bunny, *queue_joinable_name)
-      bunny.queue(File.join(*queue_joinable_name))
-    end
-
-    # Continuously get messages (metalinks) off the rabbitmq queue and
-    # invoke aria2c to download things.
-    def drain_metalink_queue(q)
-      log.info("Subscribing to #{q.name.inspect()}")
-      
-      next_queue_message(q) do |msg|
-        @dont_kill_me = true
+    attr_reader :config, :aria2_client, :gheed_client
 
-        # FIXME: is it possible something can go wrong but not throw an exception?
-        begin
-          uris, options = Yajl::Parser.parse(msg)
-          enqueue_with_aria(uris, options)
-          true
-        rescue Exception => e
-          log.error("Failed to add uri to aria2c: #{e.message}")
-          false
-        ensure
-          @dont_kill_me = false
-        end
-      end
+    def initialize(config, aria2_client, gheed_client)
+      @config = config
+      @aria2_client = aria2_client
+      @gheed_client = gheed_client
     end
 
-    def enqueue_with_aria(uris, options)
-      rewrite_aria_dir_opt(options)
-      log.info("Will call aria2.addUri(#{uris.inspect()}, #{options.inspect()})")
-      status = call_rpc("aria2.addUri", uris, options)
-      log.info("Added uri to aria with gid: #{status}")
-    end
 
-    def rewrite_aria_dir_opt(options)
-      # Attempt to rewrite +dir+ to a relative path
-      if (dir = options.delete("dir"))
-        begin
-          global_opts = call_rpc("aria2.getGlobalOption")
-          global_dir = global_opts.delete("dir")
-          options["dir"] = File.join(global_dir, dir)
-        rescue Exception => e
-          log.error("Couldn't get aria2 global options")
-        end
-      end
+    def queue
+      @queue ||= MetalinkQueue.new(filename("queue.yml"))
     end
-
-    def next_queue_message(q, &block)
-      # +:timeout+: Sometimes it happens that our connection is stale (e.g. DSL
-      # drops and comes back) but the connection never drops. Rabbitmq
-      # kicks us off after 15 min. Thus, use a timeout of 15 minutes
-      # so that, if lightning strikes, worst case is we're 15 minutes
-      # behind.
-      # +:ack+: Acks are sent by +queue.ack+, which is sent via the
-      # subscribe block automatically if the option is turned on.
-      q.subscribe(:timeout => 900,
-                  :ack => true) do |msg|
-        payload = msg[:payload]
-        case payload
-        when :queue_empty
-          next
-        else
-          if block_given?
-            yield(payload)
-          else
-            payload
-          end
-        end
-      end
-    end
-
-    def make_rpc_client(aria2_opts)
-      XMLRPC::Client.new2("http://#{aria2_opts[:user]}:#{aria2_opts[:password]}@127.0.0.1:#{aria2_opts[:port]}/rpc")
+    
+    def filename(relative)
+      File.join(@config.work_dir, relative)
     end
     
-    def call_rpc(method, *args)
-      retried = false
-      begin
-        if args.nil? || args.empty?
-          rpc_server.call(method)
-        else
-          rpc_server.call(method, *args)
-        end
-      rescue Errno::ECONNREFUSED => e
-        if retried == true
-          raise e
-        else
-          retried = true
-          log.debug("It appears aria isn't running. Attempting to start it up by calling: #{make_aria_exec}")
-          shellout(make_aria_exec, nil, nil, nil)
-          sleep 10
-          retry
-        end
-      rescue Errno::EPIPE => e
-        # The RPC client doesn't realise if the server closes the
-        # connection. So, we retry (which seems to remake the connection?).
-        if retried == true
-          raise e
-        else
-          retried = true
-          retry
-        end
-      end
+    def process_message_on_metalink_queue(msg)
+      # URIs will only ever be one (contract with gheed)
+      uris, options = MultiJson.decode(msg)
+      queue.add_uris(uris)
+      gid = aria2_client.add_uri(uris, options)
+      queue.set_store_entry(uris.first, :gid => gid.to_i, :state => :added_to_aria2)
     end
 
-    def make_aria_exec
-      args = aria2_opts[:args].dup
+    def state_change(gid)
+      response = aria2_client.tell_status(gid)
+      uri = response["files"].first["uris"].first["uri"]
+      state = response["status"].to_sym
+      queue.set_store_entry(uri, :gid => gid, :state => state)
+      notify_gheed(uri, state) if [:complete, :error].include?(state)
+    end
 
-      if aria2_opts.has_key?(:config)
-        args.push([
-                   "--conf-path",
-                   aria2_opts[:config],
-                  ].join("="))
-      end
-      
-      [
-       aria2_opts[:bin],
-       args.join(" "),
-      ].join(" ")
+    def notify_gheed(uri, state)
+      metalink, deliveries = gheed_client.state_change(uri, state)
+      queue.set_store_entry(metalink["metalink_url"], :state => :completed)
     end
-  end # class Client
-end # module Leecher
+  end
+end