leecher 0.2.3 → 1.0.0

data/lib/leecher/gheed/client.rb

@@ -0,0 +1,35 @@
+module Leecher
+  module Gheed
+
+    class Client
+
+      require "uri"
+      require "net/http"
+      require "net/https"
+
+      attr_reader :config
+      
+      def initialize(config)
+        @config = config
+      end
+
+      
+      def state_change(download, state)
+        uri = URI.parse(config.uri)
+        req = Net::HTTP::Put.new("/leecher/state_change")
+        req.set_form_data("uri" => download, "state" => state)
+        req.basic_auth(config.username, config.password)
+        res = Net::HTTP.start(uri.hostname, uri.port) do |http|
+          http.request(req)
+        end
+
+        case res
+        when Net::HTTPSuccess
+          MultiJson.decode(res.body)
+        else
+          res.error!
+        end
+      end
+    end
+  end
+end

data/lib/leecher/metalink_queue.rb

@@ -0,0 +1,68 @@
+module Leecher
+  class MetalinkQueue
+
+    require "yaml/store"
+    require "time"
+    
+    def initialize(filename)
+      @store = YAML::Store.new(filename)
+      @transaction_depth = 0
+    end
+
+
+    def get_queue
+      transaction do
+        @store[:queue] ||= {}
+      end
+    end
+    
+    def transaction
+      @transaction_depth += 1
+      
+      if @transaction_depth == 1
+        @store.transaction do
+          begin
+            yield
+          ensure
+            @transaction_depth = 0
+          end
+        end
+      else
+        begin
+          yield
+        ensure
+          @transaction_depth -= 1
+        end
+      end
+    end
+    
+    def set_store_entry(uri, properties = {})
+      transaction do
+        @store[:queue] ||= {}
+        @store[:queue][uri] ||= new_uri_entry
+        @store[:queue][uri].merge!(properties)
+      end
+    end
+
+    def add_uris(uris)
+      transaction do
+        uris.each do |uri|
+          set_store_entry(uri, :state => :brand_new)
+        end
+      end
+    end
+
+    def new_uri_entry
+      {
+        :created_at => Time.now.iso8601,
+        :state => :brand_new,
+      }
+    end
+
+    def update_uri_state(uri, state)
+      transaction do
+        set_store_entry(uri, :state => state)
+      end
+    end
+  end
+end

data/lib/leecher/version.rb

@@ -1,3 +1,3 @@
 module Leecher
-  VERSION = "0.2.3"
+  VERSION = "1.0.0"
 end

data/libexec/leecher-daemon.rb

@@ -0,0 +1,34 @@
+# Generated amqp daemon
+
+# Do your post daemonization configuration here
+# At minimum you need just the first line (without the block), or a lot
+# of strange things might start happening...
+DaemonKit::Application.running! do |config|
+  # Trap signals with blocks or procs
+  # config.trap( 'INT' ) do
+  #   # do something clever
+  # end
+  # config.trap( 'TERM', Proc.new { puts 'Going down' } )
+end
+
+leecher = Leecher.new
+
+# Run an event-loop for processing
+DaemonKit::AMQP.run do |connection|
+  # Inside this block we're running inside the reactor setup by the
+  # amqp gem. Any code in the examples (from the gem) would work just
+  # fine here.
+
+  connection.on_tcp_connection_loss do |client, settings|
+    DaemonKit.logger.debug("AMQP connection status changed: #{status}")
+    client.reconnect(false, 1)
+  end
+
+  amq = AMQP::Channel.new
+  amq.queue(leecher.config.metalink_queue_name).subscribe(:ack => true) do |metadata, payload|
+    DaemonKit.logger.debug "Received [#{leecher.config.metalink_queue_name}]: #{payload.inspect}"
+    DaemonKit.logger.debug leecher.process_message_on_metalink_queue(payload)
+    metadata.ack
+    DaemonKit.logger.debug "ack"
+  end
+end

data/script/console

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+#!/usr/bin/env ruby
+require File.expand_path('../../config/boot',  __FILE__)
+require 'daemon_kit/commands/console'

data/script/destroy

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+#!/usr/bin/env ruby
+require File.expand_path( '../../config/environment', __FILE__ )
+require 'daemon_kit/commands/generate'

data/script/generate

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+#!/usr/bin/env ruby
+require File.expand_path( '../../config/environment', __FILE__ )
+require 'daemon_kit/commands/generate'

data/spec/aria2/client_spec.rb

@@ -0,0 +1,26 @@
+require "leecher/aria2/client"
+require "ostruct"
+
+module Leecher::Aria2
+  describe Client do
+
+    let(:config) { OpenStruct.new(:port => 9000) }
+    let(:client) { Client.new(config) }
+
+    it "prefixes calls with 'aria2.'" do
+      client.should_receive(:call2).with("aria2.method").and_return([true, :param])
+      client.call("method")
+    end
+    
+    it "responds to #add_uri by delegating to 'aria2.addUri'" do
+      client.should_receive(:call).with("addUri", ["uri"], { option: "value" })
+      client.add_uri(["uri"], { option: "value" })
+    end
+
+    it "knows where the callback command is" do
+      expected = File.expand_path("../../bin/leecher-aria2-event-hook", File.dirname(__FILE__))
+      client.callback_command.should eq(expected)
+      File.exist?(client.callback_command).should be_true
+    end
+  end
+end

data/spec/client_spec.rb

@@ -1,98 +1,57 @@
-require File.dirname(__FILE__) + "/spec_helper"
+require "leecher/client"
+require "ostruct"
 
-require "lib/leecher/client"
+module Leecher
+  describe Client do
 
-describe Leecher::Client do
+    let(:work_dir) { "/tmp/client-spec-#{Process.pid}" }
+    let(:config) { OpenStruct.new(:work_dir => work_dir) }
+    let(:aria2_client) { double("aria2-client") }
+    let(:gheed_client) { double("gheed-client") }
+    let(:leecher) { Client.new(config, aria2_client, gheed_client) }
 
-  before do
-    mock_log = double("log")
-    %w(debug warn info error fatal).each do |log_level|
-      mock_log.should_receive(log_level).any_number_of_times
+    before do
+      FileUtils.mkdir_p(work_dir)
     end
-    @c = Leecher::Client.new("username",
-                             "password",
-                             "metalink",
-                             double("bunny_opts"),
-                             {
-                               :bin => "/usr/bin/aria2c",
-                               :args => ["-V"],
-                             },
-                             mock_log)
-  end
-
-  
-  it "should initialize" do
-    @c.username.should == "username"
-    @c.password.should == "password"
-    @c.metalink_queue_name.should == "metalink"
-  end
-
-  context "next_queue_message" do
     
-    it "should only return when something is on the queue" do
-      bunny = double("bunny")
-      @c.stub(:make_bunny) { bunny }
-      q = double("q")
-      q.should_receive(:subscribe).
-        and_yield({:payload => :queue_empty}).
-        and_yield({:payload => "message"})
-      @c.next_queue_message(q).should == "message"
+    after do
+      FileUtils.rm_r(work_dir) if File.exist?(work_dir)
     end
-  end
+    
+    context "#process_message_on_metalink_queue" do
 
-  context "drain_metalink_queue" do
 
-    before do
-      @q = double("q")
-      @q.should_receive(:name).
-        any_number_of_times.
-        and_return("qq")
-    end
+      let(:uri) { "http://gheed/some.metalink" }
+      let(:options) { { "dir" => "/download/here" } }
+      let(:json_instruction) { MultiJson.encode([[uri], options]) }
 
-    
-    it "should rpc to aria2" do
-      uris = ["http://some/url"]
-      options = {}
-      @c.stub(:next_queue_message).
-        and_yield(Yajl::Encoder.encode([uris, options]))
-      rpc_server = double("rpc_server")
-      rpc_server.should_receive(:call).
-        with("aria2.addUri", uris, options).
-        and_return(true)
-      @c.stub(:rpc_server).
-        and_return(rpc_server)
-      @c.drain_metalink_queue(@q).should be_true
-    end
+      before do
+        leecher.stub(:aria2_client).and_return(aria2_client)
+      end
+
+      it "consumes aria instructions encoded in JSON and forwards them onto aria" do
+        aria2_client.should_receive(:add_uri).with([uri], options).and_return(1)
+        leecher.process_message_on_metalink_queue(json_instruction)
+      end
 
-    it "should return according to the success of the rpc" do
-      @c.stub(:next_queue_message).
-        and_yield(Yajl::Encoder.encode([[], {}]))
-      rpc_server = double("rpc_server")
-      @c.stub(:rpc_server).
-        and_return(rpc_server)
-      rpc_server.stub(:call).
-        and_return(true)
-      @c.drain_metalink_queue(@q).should be_true
-      rpc_server.stub(:call).
-        and_raise(RuntimeError)
-      @c.drain_metalink_queue(@q).should be_false
+      it "keeps a list of what is in-flight" do
+        aria2_client.stub(:add_uri)
+        leecher.process_message_on_metalink_queue(json_instruction)
+        leecher.queue.get_queue.should have_key(uri)
+      end
     end
 
-    it "should rewrite the dir option" do
-      uris = ["http://some/url"]
-      options = { "dir" => "foo" }
-      @c.stub(:next_queue_message).
-        and_yield(Yajl::Encoder.encode([uris, options]))
-      rpc_server = double("rpc_server")
-      rpc_server.should_receive(:call).
-        with("aria2.getGlobalOption").
-        and_return({"dir" => "/tmp"})
-      rpc_server.should_receive(:call).
-        with("aria2.addUri", uris, {"dir" => "/tmp/foo"}).
-        and_return(true)
-      @c.stub(:rpc_server).
-        and_return(rpc_server)
-      @c.drain_metalink_queue(@q).should be_true
+    context "#status_change" do
+
+      it "updates the store status" do
+        response = {"bitfield"=>"80", "completedLength"=>"726", "connections"=>"0", "dir"=>"movies/720p/example", "downloadSpeed"=>"0", "errorCode"=>"0", "files"=>[{"completedLength"=>"726", "index"=>"1", "length"=>"726", "path"=>"movies/720p/example/nfo.metalink", "selected"=>"true", "uris"=>[{"status"=>"used", "uri"=>"http://example/download/metalinks/nfo.metalink"}, {"status"=>"waiting", "uri"=>"http://example/download/metalinks/nfo.metalink"}, {"status"=>"waiting", "uri"=>"http://example/download/metalinks/nfo.metalink"}, {"status"=>"waiting", "uri"=>"http://example/download/metalinks/nfo.metalink"}, {"status"=>"waiting", "uri"=>"http://example/download/metalinks/nfo.metalink"}]}], "followedBy"=>["2"], "gid"=>"1", "numPieces"=>"1", "pieceLength"=>"1048576", "status"=>"complete", "totalLength"=>"726", "uploadLength"=>"0", "uploadSpeed"=>"0"}
+        aria2_client.should_receive(:tell_status).with(1).and_return(response)
+        leecher.status_change(1)
+        entry = leecher.queue.get_queue["http://example/download/metalinks/nfo.metalink"]
+        entry.should_not be_nil
+        entry[:gid].should eq(1)
+        entry[:state].should eq(:complete)
+      end
     end
   end
 end

data/spec/metalink_queue_spec.rb

@@ -0,0 +1,25 @@
+require "leecher/metalink_queue"
+
+module Leecher
+  describe MetalinkQueue do
+
+    require "fileutils"
+    
+    let(:filename) { "/tmp/metalink-queue-spec-#{Process.pid}" }
+    let(:queue) { MetalinkQueue.new(filename) }
+    
+    after do
+      FileUtils.rm(filename) if File.exist?(filename)
+    end
+
+    it "stores the queue to disk" do
+      File.size?(filename).should be_nil
+      queue.get_queue.should be_empty
+      queue.add_uris(["a", "b"])
+      File.size(filename).should_not be_zero
+      queue.get_queue.should have_key("a")
+      queue.get_queue.should have_key("b")
+      MetalinkQueue.new(filename).get_queue.keys.should eq(["a", "b"])
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/spec/spec_helper.rb

@@ -1,2 +0,0 @@
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-$LOAD_PATH.unshift(File.dirname(__FILE__) + "../lib")

data/tasks/rspec.rake

@@ -0,0 +1,6 @@
+require 'rspec/core/rake_task'
+
+desc "Run the specs under spec/"
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = ['--options', "spec/spec.opts"]
+end

metadata

@@ -1,137 +1,181 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: leecher
-version: !ruby/object:Gem::Version 
-  hash: 17
+version: !ruby/object:Gem::Version
+  version: 1.0.0
   prerelease: 
-  segments: 
-  - 0
-  - 2
-  - 3
-  version: 0.2.3
 platform: ruby
-authors: 
+authors:
 - Marc Bowes
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-08-22 00:00:00 +02:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: rspec
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+date: 2012-07-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: daemon-kit
+  requirement: &7773340 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 2
-        version: "2"
-  type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: bunny
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *7773340
+- !ruby/object:Gem::Dependency
+  name: safely
+  requirement: &7788340 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: erubis
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *7788340
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: &7786200 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: *7786200
+- !ruby/object:Gem::Dependency
   name: yajl-ruby
+  requirement: &7784880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7784880
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: &7784000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *7784000
+- !ruby/object:Gem::Dependency
+  name: erubis
+  requirement: &7782780 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id004
+  prerelease: false
+  version_requirements: *7782780
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &7781680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *7781680
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &7780840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *7780840
 description: A server populates a queue, this client downloads those files
-email: 
+email:
 - marcbowes+leecher@gmail.com
-executables: 
+executables:
 - leecher
+- leecher-aria2-event-hook
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
+- .travis.yml
 - Gemfile
+- Gemfile.lock
+- README
 - Rakefile
 - bin/leecher
-- config/aria2.conf
-- config/client.yml
+- bin/leecher-aria2-event-hook
+- config/amqp.example.yml
+- config/arguments.rb
+- config/aria2.example.yml
+- config/boot.rb
+- config/environment.rb
+- config/environments/development.rb
+- config/environments/production.rb
+- config/environments/test.rb
+- config/gheed.example.yml
+- config/leecher.example.yml
+- config/post-daemonize/readme
+- config/pre-daemonize/amqp.rb
+- config/pre-daemonize/json.rb
+- config/pre-daemonize/readme
+- config/pre-daemonize/safely.rb
 - leecher.gemspec
 - lib/leecher.rb
+- lib/leecher/aria2/client.rb
 - lib/leecher/client.rb
-- lib/leecher/log.rb
-- lib/leecher/shellout.rb
+- lib/leecher/gheed/client.rb
+- lib/leecher/metalink_queue.rb
 - lib/leecher/version.rb
+- libexec/leecher-daemon.rb
+- script/console
+- script/destroy
+- script/generate
+- spec/aria2/client_spec.rb
 - spec/client_spec.rb
-- spec/shellout_spec.rb
+- spec/metalink_queue_spec.rb
+- spec/spec.opts
 - spec/spec_helper.rb
-has_rdoc: true
-homepage: ""
+- tasks/rspec.rake
+homepage: ''
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: -3150552957577293375
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
+      hash: -3150552957577293375
 requirements: []
-
 rubyforge_project: leecher
-rubygems_version: 1.6.2
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: Download files off an AMQP queue
 test_files: []
-