leecher 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in leecher.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require "rspec/core/rake_task"
+desc "Run specs"
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = %w(-fs --color)
+  # t.ruby_opts = %w(-w)
+end
+

data/bin/leecher

@@ -0,0 +1,181 @@
+#!/usr/bin/env ruby
+
+module Leecher
+  class Cli
+
+    require "optparse"
+    require "yaml"
+    require "erubis"
+    require "leecher/log"
+    require "leecher/client"
+
+
+    RC_OK = 0
+    RC_USAGE = 1
+    RC_CONF_MISSING = 2
+    RC_CATASTROPHE = 99
+
+    
+    def initialize()
+      @config_fn = File.join(ENV["HOME"], ".leecher.yml")
+      @foreground = false
+    end
+
+
+    def parse_opts(argv)
+      optparser = OptionParser.new() do |o|
+        o.banner = "leecher [options]"
+
+        o.separator("")
+        o.separator("options:")
+        
+        o.on("--config=FILE",
+             "File containing settings",
+             "Default: #{@config_fn}") do |v|
+          @config_fn = v
+        end
+
+        o.on("-f", "--[no-]foreground",
+             "Run as a loop in a daemon?",
+             "Default #{@foreground}") do |v|
+          @foreground = v
+        end
+
+        o.on("--getting-started",
+             "Run a tutorial which will give you a config") do
+          do_getting_started()
+          Kernel.exit!(RC_OK)
+        end
+
+        o.separator("")
+        o.on("-h", "--help", "You're reading it :-)") do
+          puts o
+          Kernel.exit!(RC_USAGE)
+        end
+      end
+
+      optparser.parse(argv)
+      self
+    end
+
+    def do_getting_started()
+      log.info("We'll ask you a few questions then write you a config in #{@config_fn}")
+      log.info(".. you can change where your config lives with -c/--config")
+
+      STDOUT.puts("\n\n")
+      config = Hash.new()
+      [
+       [:username, "your username on the website?", :required],
+       [:password, ".. and the password for that user?", :password],
+       [:log_fn, "filename to log to:", :disable],
+       [:host, "where does the amqp queue live?", "nzb.trim.za.net"],
+       [:aria2_bin, "which aria2 must I use?", %x{which aria2c}.strip()],
+      ].each do |setting, question, default|
+        config[setting] = get_user_input(question, default)
+      end
+      
+      erb = Erubis::Eruby.new(File.read(File.join(File.dirname(__FILE__),
+                                                  "../config/client.yml")))
+      File.open(@config_fn, "w") do |f|
+        f.puts(erb.result(config))
+      end
+
+      STDOUT.puts("\n\n")
+      log.info("Looking good, now take us for a real spin!")
+    end
+
+    def get_user_input(question, default)
+      default_text = case default
+                     when :required, :password; nil
+                     when :disable; "blank to disable"
+                     else; "default: #{default}"
+                     end
+
+      response = ask([
+                      question,
+                      default_text ? "(#{default_text})" : nil,
+                     ].compact.join(" "),
+                     default == :password)
+      
+      if response.empty?
+        response = case default
+                   when :required, :password
+                     log.error("That option was required! Try again :-)")
+                     get_user_input(question, default)
+                   when :disable
+                     nil
+                   else
+                     default
+                   end
+      end
+      
+      return response
+    end
+
+    def ask(question, hide_input = false)
+      STDOUT.write(question + " ")
+      system("stty -echo") if hide_input
+      response = STDIN.gets().strip()
+      if hide_input
+        STDOUT.puts()
+        system("stty echo")
+      end
+      response
+    end
+
+    def run()
+      config = load_config(@config_fn)
+      unless @foreground
+        log.info("Fading into the ether ..")
+        make_daemon(config[:daemon])
+      end
+      client = Leecher::Client.new(config[:username],
+                                   config[:password],
+                                   config[:metalink_queue_name],
+                                   config[:bunny_opts],
+                                   config[:aria2_opts],
+                                   @log)
+      client.run()
+      Kernel.exit(RC_OK)
+    end
+
+    def log
+      @log ||= Leecher::Log.new()
+    end
+
+    def load_config(config_fn)
+      config = YAML.load_file(config_fn)
+      if (log_conf = config[:log])
+        @log = Leecher::Log.new(config[:log][:level],
+                                config[:log][:filename])
+      end
+      config
+    rescue Errno::ENOENT
+      log.error("No such config file #{config_fn.inspect()}")
+      log.info("New user? Try #{Leecher::Log::ANSI_GREEN}"\
+               "--getting-started#{Leecher::Log::ANSI_NORMAL}")
+      Kernel.exit!(RC_CONF_MISSING)
+    end
+
+    def make_daemon(config)
+      raise log.fatal("First fork failed") if (pid = Kernel.fork()) == -1
+      Kernel.exit!(RC_OK) unless pid.nil?
+      Process.setsid()
+      Dir.chdir(config[:chdir]) if config.has_key? :chdir
+      File.umask(config[:umask]) if config.has_key? :umask
+      [
+       [STDIN, :stdin],
+       [STDOUT, :stdout],
+       [STDERR, :stderr],
+      ].each do |io, key|
+         if config.has_key? key
+           io.reopen(*config[key])
+         else
+           io.reopen("/dev/null")
+         end
+      end
+    end
+  end
+end
+
+Leecher::Cli.new().parse_opts(ARGV).run()

data/config/client.yml

@@ -0,0 +1,29 @@
+:username: <%= username %>
+:password: <%= password %>
+:metalink_queue_name: metalink
+:log:
+  :level: :info
+  :filename: <%= log_fn %>
+:bunny_opts:
+  :host: <%= host %>
+  :port: 5672
+  :vhost: /
+  :user: <%= username %>
+  :pass: <%= password %>
+  :ssl: false
+  :verify_ssl: false
+  :logfile: false
+  :logfile: false
+  :frame_max: 131072
+  :channel_max: 0
+  :heartbeat: 300
+  :connect_timeout: 5
+:aria2_opts:
+  :bin: <%= aria2_bin %>
+  :args: [-x5, -V, --follow-metalink=mem]
+:daemon:
+  :chdir: nil
+  :umask: 0000
+  :stdin: nil
+  :stdout: nil
+  :stderr: nil

data/leecher.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "leecher/version"
+
+Gem::Specification.new do |s|
+  s.name        = "leecher"
+  s.version     = Leecher::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Marc Bowes"]
+  s.email       = ["marcbowes+leecher@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Download files off an AMQP queue}
+  s.description = %q{A server populates a queue, this client downloads those files}
+
+  s.rubyforge_project = "leecher"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency(%q<rspec>, [">=2"])
+  s.add_runtime_dependency(%q<bunny>)
+  s.add_runtime_dependency(%q<erubis>)
+end

data/lib/leecher.rb

@@ -0,0 +1,5 @@
+module Leecher
+  
+  require "leecher/version"
+  require "leecher/client"
+end

data/lib/leecher/client.rb

@@ -0,0 +1,125 @@
+module Leecher
+  
+  class Client
+
+    require "bunny"
+    require "leecher/shellout"
+    
+    include Leecher::Shellout
+
+    class ShutdownError < RuntimeError; end
+    
+    attr_accessor :username
+    attr_accessor :password
+    attr_accessor :metalink_queue_name
+    attr_accessor :bunny_opts
+    attr_accessor :aria2_opts
+    attr_accessor :log
+    
+    def initialize(username,
+                   password,
+                   metalink_queue_name,
+                   bunny_opts,
+                   aria2_opts,
+                   log)
+      self.username = username
+      self.password = password
+      self.metalink_queue_name = metalink_queue_name
+      self.bunny_opts = bunny_opts
+      self.aria2_opts = aria2_opts
+      self.log = log
+    end
+
+
+    def run()
+      @bunny = make_bunny(self.bunny_opts)
+      @bunny.start()
+
+      @graceful_shutdown = false
+      @dont_kill_me = false
+      [:INT, :TERM].each do |sig|
+        trap(sig) do
+          @graceful_shutdown = true
+          log.info("Caught SIG{sig}")
+          raise ShutdownError.new("SIG#{sig}") unless @dont_kill_me
+        end
+      end
+      
+      q = get_queue(@bunny, username, metalink_queue_name)
+      until @graceful_shutdown
+        begin
+          drain_metalink_queue(q)
+        rescue ShutdownError => e
+          log.info(["Going away because of", e.message].join(": "))
+        rescue Exception => e
+          log.error([e.class.name, e.message].join(": "))
+          e.backtrace.each do |line|
+            log.debug(line)
+          end
+        end
+      end
+    end
+
+    def make_bunny(bunny_opts)
+      Bunny.new(bunny_opts)
+    end
+
+    def get_queue(bunny, *queue_joinable_name)
+      bunny.queue(File.join(*queue_joinable_name))
+    end
+    
+    def drain_metalink_queue(q)
+      next_queue_message(q) do |msg|
+        @dont_kill_me = true
+        
+        aria_exec = make_aria_exec(URI.parse(msg))
+        log.info("Will exec #{aria_exec.inspect()}")
+        stdout_rd, stdout_wr = IO.pipe()
+        stderr_rd, stderr_wr = IO.pipe()
+        rc = shellout(aria_exec, nil, stdout_wr, stderr_wr)
+        stdout_wr.close()
+        stderr_wr.close()
+
+        if rc == 0
+          log.info("aria2c exited happily")
+          return true
+        else
+          log.error("aria2c exited unhappily (rc = #{rc})")
+          stdout_rd.each_line do |line|
+            log.error(line)
+          end
+          stderr_rd.each_line do |line|
+            log.debug(line)
+          end
+          return false
+        end
+        
+        @dont_kill_me = false
+      end
+    end
+
+    def next_queue_message(q, &block)
+      q.subscribe() do |msg|
+        payload = msg[:payload]
+        case payload
+        when :queue_empty
+          next
+        else
+          if block_given?
+            yield(payload)
+          else
+            payload
+          end
+        end
+      end
+    end
+
+    def make_aria_exec(uri)
+      [
+       aria2_opts[:bin],
+       aria2_opts[:args].join(" "),
+       uri.to_s(),
+      ].join(" ")
+    end
+  end # class Client
+end # module Leecher

data/lib/leecher/log.rb

@@ -0,0 +1,89 @@
+module Leecher
+  
+  class Log
+
+    require "time"
+    
+    LEVELS = {
+      :debug => 0,
+      :warn => 1,
+      :info  => 2,
+      :error => 3,
+      :fatal => 4,
+    }
+          
+    attr_accessor :log_fn
+    attr_reader :level
+    
+    def initialize(level = :debug,
+                   log_fn = nil)
+      self.level = level
+      self.log_fn = log_fn
+      open_log_fd()
+    end
+
+    def open_log_fd
+      @log_fd.close() if @log_fd and not @log_fd.closed?
+      if self.log_fn
+        @log_fd = File.open(self.log_fn, "a")
+      end
+    end
+
+    def close_log_fd
+      if @log_fd and not @log_fd.closed?
+        @log_fd.flush()
+        @log_fd.close()
+      end
+    end
+            
+    LEVELS.each_key do |level|
+      define_method(level) do |message|
+        write_to_log(level, message)
+      end
+    end
+
+    def level=(level)
+      @level = level
+      @_enum_level = LEVELS[level]
+    end
+    
+    def write_to_log(level, message)
+      if LEVELS[level] >= @_enum_level
+        do_write_to_log(level, message)
+      else
+        # Toss it
+      end
+    end
+
+    def do_write_to_log(level, message)
+      iso8601_timestr = Time.now.iso8601()
+      log_for_humans(iso8601_timestr, level, message)
+      
+      if @log_fd and not @log_fd.closed?
+        @log_fd.puts("%s %s %s" % [iso8601_timestr, level.to_s.upcase, message])
+      end
+    end
+
+    # Map the log level (LOG_LEVEL_*) to an array containing
+    # [human_name:String, colour:String(ANSI escape sequence)]
+    ANSI_RED = "\033[0;31m"
+    ANSI_RED_INVERTED = "\033[7;31m"
+    ANSI_BROWN = "\033[0;33m"
+    ANSI_MAGENTA = "\033[0;35m"
+    ANSI_GREEN = "\033[0;32m"
+    ANSI_BOLD_WHITE = "\033[0;37m"
+    ANSI_NORMAL = "\033[0m"
+    HUMAN_LOG_LEVELS = { 
+      :fatal => ["FATAL", ANSI_RED_INVERTED],
+      :error => ["ERROR", ANSI_RED],
+      :info => ["INFO", ANSI_GREEN],
+      :warn => ["WARN", ANSI_MAGENTA],
+      :debug => ["DEBUG", ANSI_BROWN],
+    }
+    def log_for_humans(iso8601_timestr, log_level, log_msg)
+      level_str, level_colour = HUMAN_LOG_LEVELS[log_level]
+      puts("#{ANSI_BOLD_WHITE}#{iso8601_timestr}#{ANSI_NORMAL} "\
+           "#{level_colour}#{"%5.5s" % level_str}#{ANSI_NORMAL} #{log_msg}")
+    end
+  end
+end

data/lib/leecher/shellout.rb

@@ -0,0 +1,27 @@
+module Leecher
+  module Shellout
+    
+    # FIXME: probably want to implement timeouts here..
+    def shellout(cmd, stdin = nil, stdout = nil, stderr = nil)
+      child_pid, child_status = nil
+      child_pid = Kernel.fork()
+
+      if child_pid
+        # Parent process executes this
+        child_status = (Process.waitpid2(child_pid)[1]).to_i >> 8
+      else
+        Process.setsid()
+        STDIN.reopen(stdin || "/dev/null")
+        STDOUT.reopen(stdout || "/dev/null")
+        STDERR.reopen(stderr || "/dev/null")
+        3.upto(256) { |fd| IO.new(fd).close rescue nil }
+
+        Kernel.exec(cmd)
+        # Never reached.
+        Kernel.exit!
+      end
+
+      child_status
+    end
+  end
+end

data/lib/leecher/version.rb

@@ -0,0 +1,3 @@
+module Leecher
+  VERSION = "0.0.1"
+end

data/spec/client_spec.rb

@@ -0,0 +1,74 @@
+require File.dirname(__FILE__) + "/spec_helper"
+
+require "lib/leecher/client"
+
+describe Leecher::Client do
+
+  before do
+    mock_log = double("log")
+    %w(debug warn info error fatal).each do |log_level|
+      mock_log.should_receive(log_level).any_number_of_times
+    end
+    @c = Leecher::Client.new("username",
+                             "password",
+                             "metalink",
+                             double("bunny_opts"),
+                             {
+                               :bin => "/usr/bin/aria2c",
+                               :args => ["-V"],
+                             },
+                             mock_log)
+  end
+
+  
+  it "should initialize" do
+    @c.username.should == "username"
+    @c.password.should == "password"
+    @c.metalink_queue_name.should == "metalink"
+  end
+
+  describe "#next_queue_message" do
+    
+    it "should only return when something is on the queue" do
+      bunny = double("bunny")
+      @c.stub(:make_bunny) { bunny }
+      q = double("q")
+      q.should_receive(:subscribe).
+        and_yield({:payload => :queue_empty}).
+        and_yield({:payload => "message"})
+      @c.next_queue_message(q).should == "message"
+    end
+  end
+
+  describe "#drain_metalink_queue" do
+    
+    it "should reject URIs that are not URIs" do
+      @c.stub(:next_queue_message).
+        and_yield(nil)
+      lambda { @c.drain_metalink_queue(double("q")) }.
+        should raise_error(URI::InvalidURIError)
+    end
+
+    it "should shellout to aria2" do
+      @c.stub(:next_queue_message).
+        and_yield("http://some/url")
+      @c.should_receive(:shellout).
+        with("/usr/bin/aria2c -V http://some/url",
+             anything(), anything(), anything()).
+        and_return(0)
+      @c.drain_metalink_queue(double("q")).should be_true
+    end
+
+    it "should return according to the rc" do
+      @c.stub(:next_queue_message).
+        and_yield("http://some/url")
+      @c.stub(:shellout).
+        and_return(0)
+      q = double("q")
+      @c.drain_metalink_queue(q).should be_true
+      @c.stub(:shellout).
+        and_return(1)
+      @c.drain_metalink_queue(q).should be_false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__) + "../lib")

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification 
+name: leecher
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Marc Bowes
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-22 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 2
+        version: "2"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: erubis
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: A server populates a queue, this client downloads those files
+email: 
+- marcbowes+leecher@gmail.com
+executables: 
+- leecher
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Rakefile
+- bin/leecher
+- config/client.yml
+- leecher.gemspec
+- lib/leecher.rb
+- lib/leecher/client.rb
+- lib/leecher/log.rb
+- lib/leecher/shellout.rb
+- lib/leecher/version.rb
+- spec/client_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: leecher
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Download files off an AMQP queue
+test_files: []
+