leecher 1.0.0 → 1.1.0

data/Gemfile.lock

@@ -1,12 +1,14 @@
 PATH
   remote: .
   specs:
-    leecher (1.0.0)
+    leecher (1.1.0)
       amqp
       daemon-kit
       erubis
       multi_json
+      rest-client
       safely
+      sys-filesystem
       yajl-ruby
 
 GEM
@@ -26,8 +28,12 @@ GEM
     diff-lcs (1.1.3)
     erubis (2.7.0)
     eventmachine (0.12.10)
+    ffi (1.1.0)
+    mime-types (1.19)
     multi_json (1.3.6)
     rake (0.9.2.2)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     rspec (2.11.0)
       rspec-core (~> 2.11.0)
       rspec-expectations (~> 2.11.0)
@@ -37,6 +43,8 @@ GEM
       diff-lcs (~> 1.1.3)
     rspec-mocks (2.11.1)
     safely (0.3.2)
+    sys-filesystem (1.0.0)
+      ffi (>= 1.0.0)
     yajl-ruby (1.1.0)
 
 PLATFORMS

data/leecher.gemspec

@@ -25,6 +25,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "yajl-ruby"
   s.add_runtime_dependency "amqp"
   s.add_runtime_dependency "erubis"
+  s.add_runtime_dependency "sys-filesystem"
+  s.add_runtime_dependency "rest-client"
 
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"

data/lib/leecher/aria2/client.rb

@@ -16,17 +16,35 @@ module Leecher
       
       def call(method, *args)
         retried = false
-        super("aria2.#{method}", *args)
+        response = super("aria2.#{method}", *args)
+        @http.finish if @http.started?
+        response
       rescue Errno::ECONNREFUSED
+        retried = true
         start_aria2
         Kernel.sleep(3) # Give it a chance to start.
         retry unless retried
+        raise
       end
 
       def add_uri(uris, options)
+        rewrite_aria_dir_opt(options)
         call("addUri", uris, options)
       end
 
+      # WEIRD: sometimes we get nil back here.
+      def global_opts
+        call("getGlobalOption")
+      end
+
+      def rewrite_aria_dir_opt(options)
+        # Attempt to rewrite +dir+ to a relative path
+        if (dir = options.delete("dir"))
+          global_dir = global_opts.delete("dir")
+          options["dir"] = File.join(global_dir, dir)
+        end
+      end
+      
       def tell_status(gid, keys = nil)
         if keys.nil? or keys.empty?
           call("tellStatus", gid.to_s)
@@ -35,6 +53,18 @@ module Leecher
         end
       end
 
+      def get_version
+        call("getVersion")
+      end
+
+      def tell_active
+        call("tellActive")
+      end
+
+      def tell_waiting(offset, limit)
+        call("tellWaiting", offset, limit)
+      end
+
       def make_aria_exec
         args = config.args.dup
         [

data/lib/leecher/client.rb

@@ -3,7 +3,9 @@ module Leecher
 
     require "fileutils"
     require "multi_json"
+    require "sys/filesystem"
     require "leecher/metalink_queue"
+    require "leecher/status"
 
     attr_reader :config, :aria2_client, :gheed_client
 
@@ -40,7 +42,17 @@ module Leecher
 
     def notify_gheed(uri, state)
       metalink, deliveries = gheed_client.state_change(uri, state)
-      queue.set_store_entry(metalink["metalink_url"], :state => :completed)
+      queue.set_store_entry(metalink["metalink_url"], :state => :complete)
+    end
+
+    def status_update
+      status = Status.new(aria2_client)
+      if status.changed?(@status)
+        @status = status
+        gheed_client.status_update(@status.to_params)
+      else
+        false
+      end
     end
   end
 end

data/lib/leecher/gheed/client.rb

@@ -3,9 +3,7 @@ module Leecher
 
     class Client
 
-      require "uri"
-      require "net/http"
-      require "net/https"
+      require "rest_client"
 
       attr_reader :config
       
@@ -13,22 +11,22 @@ module Leecher
         @config = config
       end
 
+
+      def new_resource(path)
+        uri = URI.join(config.uri, path).to_s
+        RestClient::Resource.new(uri, config.username, config.password)
+      end
       
       def state_change(download, state)
-        uri = URI.parse(config.uri)
-        req = Net::HTTP::Put.new("/leecher/state_change")
-        req.set_form_data("uri" => download, "state" => state)
-        req.basic_auth(config.username, config.password)
-        res = Net::HTTP.start(uri.hostname, uri.port) do |http|
-          http.request(req)
-        end
+        req = new_resource("/leecher/state_change")
+        res = req.put({ "uri" => download, "state" => state })
+        MultiJson.decode(res)
+      end
 
-        case res
-        when Net::HTTPSuccess
-          MultiJson.decode(res.body)
-        else
-          res.error!
-        end
+      def status_update(params)
+        req = new_resource("/leecher/status_update")
+        res = req.post({ "status" => params })
+        MultiJson.decode(res)
       end
     end
   end

data/lib/leecher/status.rb

@@ -0,0 +1,81 @@
+module Leecher
+  class Status
+
+    attr_accessor :aria2_client
+    
+    def initialize(aria2_client)
+      @aria2_client = aria2_client
+    end
+
+    
+    def aria2_version
+      @aria2_version ||= aria2_client.get_version["version"]
+    end
+    
+    def active
+      @active ||= aria2_client.tell_active
+    end
+
+    def waiting
+      @waiting ||= aria2_client.tell_waiting(0, 100)
+    end
+
+    def aria2_status
+      return "downloading" if active.any?
+      if waiting.empty?
+        "idle"
+      else
+        "paused"
+      end
+    end
+
+    def downspeed
+      @downspeed ||= calc_downspeed
+    end
+
+    def calc_downspeed
+      active.inject(0) do |sum, download|
+        sum += Integer(download["downloadSpeed"])
+      end
+    end
+
+    def diskfree
+      @diskfree ||= calc_diskfree
+    end
+
+    def calc_diskfree
+      dir = aria2_client.global_opts["dir"]
+      stat = Sys::Filesystem.stat(dir)
+      stat.block_size * stat.blocks_available
+    end
+
+    def to_params
+      {
+        "leecher_version" => Leecher::VERSION,
+        "aria2_version" => aria2_version,
+        "aria2_status" => aria2_status,
+        "downspeed" => downspeed,
+        "n_downloads" => active.size + waiting.size,
+        "n_active_downloads" => active.size,
+        "diskfree" => diskfree,
+      }
+    end
+
+    def changed?(previous, diskchange_threshold = 1048576)
+      return true if previous.nil?
+
+      # compare params, ignoring diskfree
+      now = self.to_params
+      now_disk = now.delete("diskfree")
+      before = previous.to_params
+      before_disk = before.delete("diskfree")
+      return true unless now == before
+
+      significant_diskchange?(now_disk, before_disk, diskchange_threshold)
+    end
+
+    def significant_diskchange?(now, before, diskchange_threshold = 1048576)
+      (now - before).abs > diskchange_threshold
+    end
+  end
+end

data/lib/leecher/version.rb

@@ -1,3 +1,3 @@
 module Leecher
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/libexec/leecher-daemon.rb

@@ -15,18 +15,27 @@ leecher = Leecher.new
 
 # Run an event-loop for processing
 DaemonKit::AMQP.run do |connection|
-  # Inside this block we're running inside the reactor setup by the
-  # amqp gem. Any code in the examples (from the gem) would work just
-  # fine here.
-
   connection.on_tcp_connection_loss do |client, settings|
     DaemonKit.logger.debug("AMQP connection status changed: #{status}")
     client.reconnect(false, 1)
   end
 
+  EventMachine.add_periodic_timer(60) do
+    DaemonKit.logger.debug "Checking status update"
+    begin
+      if (update = leecher.status_update)
+        DaemonKit.logger.info "Status updated: #{update.inspect}"
+      else
+        DaemonKit.logger.debug "No status change"
+      end
+    rescue => e
+      DaemonKit.logger.error "Unable to post status update: #{e.message}"
+    end
+  end
+
   amq = AMQP::Channel.new
   amq.queue(leecher.config.metalink_queue_name).subscribe(:ack => true) do |metadata, payload|
-    DaemonKit.logger.debug "Received [#{leecher.config.metalink_queue_name}]: #{payload.inspect}"
+    DaemonKit.logger.info "Received [#{leecher.config.metalink_queue_name}]: #{payload.inspect}"
     DaemonKit.logger.debug leecher.process_message_on_metalink_queue(payload)
     metadata.ack
     DaemonKit.logger.debug "ack"

data/spec/aria2/client_spec.rb

@@ -11,10 +11,20 @@ module Leecher::Aria2
       client.should_receive(:call2).with("aria2.method").and_return([true, :param])
       client.call("method")
     end
-    
-    it "responds to #add_uri by delegating to 'aria2.addUri'" do
-      client.should_receive(:call).with("addUri", ["uri"], { option: "value" })
-      client.add_uri(["uri"], { option: "value" })
+
+    context "#addUri" do
+      
+      it "responds to #add_uri by delegating to 'aria2.addUri'" do
+        client.should_receive(:call).with("addUri", ["uri"], { option: "value" })
+        client.add_uri(["uri"], { option: "value" })
+      end
+
+      it "rewrites the 'dir' option to be relative to the global dir" do
+        client.should_receive(:call).with("getGlobalOption").
+          and_return({"dir"=>"/tmp/aria2"})
+        client.should_receive(:call).with("addUri", ["uri"], { "dir" => "/tmp/aria2/foo" })
+        client.add_uri(["uri"], { "dir" => "foo" })
+      end
     end
 
     it "knows where the callback command is" do

data/spec/client_spec.rb

@@ -46,7 +46,9 @@ module Leecher
       it "updates the store status" do
         response = {"bitfield"=>"80", "completedLength"=>"726", "connections"=>"0", "dir"=>"movies/720p/example", "downloadSpeed"=>"0", "errorCode"=>"0", "files"=>[{"completedLength"=>"726", "index"=>"1", "length"=>"726", "path"=>"movies/720p/example/nfo.metalink", "selected"=>"true", "uris"=>[{"status"=>"used", "uri"=>"http://example/download/metalinks/nfo.metalink"}, {"status"=>"waiting", "uri"=>"http://example/download/metalinks/nfo.metalink"}, {"status"=>"waiting", "uri"=>"http://example/download/metalinks/nfo.metalink"}, {"status"=>"waiting", "uri"=>"http://example/download/metalinks/nfo.metalink"}, {"status"=>"waiting", "uri"=>"http://example/download/metalinks/nfo.metalink"}]}], "followedBy"=>["2"], "gid"=>"1", "numPieces"=>"1", "pieceLength"=>"1048576", "status"=>"complete", "totalLength"=>"726", "uploadLength"=>"0", "uploadSpeed"=>"0"}
         aria2_client.should_receive(:tell_status).with(1).and_return(response)
-        leecher.status_change(1)
+        gheed_client.should_receive(:state_change).with("http://example/download/metalinks/nfo.metalink", :complete).
+          and_return([{"metalink_url" => "http://example/download/metalinks/nfo.metalink"}, []])
+        leecher.state_change(1)
         entry = leecher.queue.get_queue["http://example/download/metalinks/nfo.metalink"]
         entry.should_not be_nil
         entry[:gid].should eq(1)

data/spec/status_spec.rb

@@ -0,0 +1,34 @@
+require "leecher/status"
+
+module Leecher
+  describe Status do
+
+    let(:aria2_client) { double("aria2_client") }
+    let(:status) { Status.new(aria2_client) }
+    
+    context "changed?" do
+
+      it "finds changes if there was no previous status" do
+        status.changed?(nil).should be_true
+      end
+
+      it "finds changes if the params have changed" do
+        other = status.dup
+        status.stub(:to_params).and_return("foo" => "bar")
+        other.stub(:to_params).and_return("foo" => "changed")
+        status.changed?(other).should be_true
+      end
+    end
+
+    context "disk space changes" do
+      
+      it "ignores disk free changes under a threshold" do
+        status.significant_diskchange?(100, 200).should be_false
+      end
+
+      it "detects significant changes" do
+        status.significant_diskchange?(100, 200, 1).should be_true
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: leecher
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-15 00:00:00.000000000 Z
+date: 2012-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemon-kit
-  requirement: &7773340 !ruby/object:Gem::Requirement
+  requirement: &13644920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *7773340
+  version_requirements: *13644920
 - !ruby/object:Gem::Dependency
   name: safely
-  requirement: &7788340 !ruby/object:Gem::Requirement
+  requirement: &13644100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *7788340
+  version_requirements: *13644100
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &7786200 !ruby/object:Gem::Requirement
+  requirement: &13643400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *7786200
+  version_requirements: *13643400
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &7784880 !ruby/object:Gem::Requirement
+  requirement: &13642640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *7784880
+  version_requirements: *13642640
 - !ruby/object:Gem::Dependency
   name: amqp
-  requirement: &7784000 !ruby/object:Gem::Requirement
+  requirement: &13641980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *7784000
+  version_requirements: *13641980
 - !ruby/object:Gem::Dependency
   name: erubis
-  requirement: &7782780 !ruby/object:Gem::Requirement
+  requirement: &13639820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,32 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *7782780
+  version_requirements: *13639820
+- !ruby/object:Gem::Dependency
+  name: sys-filesystem
+  requirement: &13638180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *13638180
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: &13637180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *13637180
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &7781680 !ruby/object:Gem::Requirement
+  requirement: &13690700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +109,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *7781680
+  version_requirements: *13690700
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &7780840 !ruby/object:Gem::Requirement
+  requirement: &13689740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,7 +120,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *7780840
+  version_requirements: *13689740
 description: A server populates a queue, this client downloads those files
 email:
 - marcbowes+leecher@gmail.com
@@ -137,6 +159,7 @@ files:
 - lib/leecher/client.rb
 - lib/leecher/gheed/client.rb
 - lib/leecher/metalink_queue.rb
+- lib/leecher/status.rb
 - lib/leecher/version.rb
 - libexec/leecher-daemon.rb
 - script/console
@@ -147,6 +170,7 @@ files:
 - spec/metalink_queue_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- spec/status_spec.rb
 - tasks/rspec.rake
 homepage: ''
 licenses: []
@@ -162,7 +186,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3150552957577293375
+      hash: -4395843245282034706
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -171,7 +195,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3150552957577293375
+      hash: -4395843245282034706
 requirements: []
 rubyforge_project: leecher
 rubygems_version: 1.8.11