junkie 0.0.6 → 0.0.7

data/bin/junkie

@@ -5,6 +5,14 @@
 $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'junkie'
-include Junkie
+require 'logging'
+
+Logging.appenders.stdout(
+    'stdout',
+    :level  => :debug,
+    :layout => Logging.layouts.pattern(:pattern => '[%d] %-5l: %m\n')
+)
+Logging.logger.root.appenders = Logging.appenders.stdout
 
+include Junkie
 start_reactor

data/lib/junkie/errors.rb

@@ -7,6 +7,8 @@ module Junkie
   class HTTP404Error < Exception;  end
   class HTTP500Error < Exception;  end
 
+  class InvalidResponse < Exception; end
+
   class InvalidConfigError < Exception;  end
 
   class InvalidStateError < Exception;  end

data/lib/junkie/pyload/api.rb

@@ -30,8 +30,8 @@ module Junkie
         protocol: 'http',
         host: 'localhost',
         port: 8000,
-        api_user: '',
-        api_password: ''
+        api_user: 'user',
+        api_password: 'password'
       }
 
       def initialize()

data/lib/junkie/pyload/observer.rb

@@ -7,6 +7,8 @@ module Junkie
     class Observer
       include Log, Helper, Config
 
+      attr_accessor :api
+
       DEFAULT_CONFIG = {
           watchdog_refresh: 10,          # interval the watchdog_timer is fired
       }
@@ -25,7 +27,6 @@ module Junkie
         @ready_for_new_links = true
         @watchdog_enabled = false
         @skipped_timer_at_first_complete_detection = false
-        @should_send_it_on_channel = false
 
         @channels[:episodes].subscribe do |episode|
           next unless episode.status == :found
@@ -45,24 +46,13 @@ module Junkie
         }
 
         EM.add_periodic_timer(@config[:watchdog_refresh]) do
-
-          # temporary fix for the SystemStackError
-          if @should_send_it_on_channel
-            log.info("Sending complete episode out on the channel")
-            @channels[:episodes].push(@active_episode)
-            @active_episode = nil
-
-            @should_send_it_on_channel = false
-          end
-
           monitor_progress if @watchdog_enabled
 
-          in_fiber {
-            add_next_episode_to_pyload
-          }
+          in_fiber { add_next_episode_to_pyload }
         end
       end
 
+      private
 
       # Add next episode to Pyload which downloads the episode and extracts it
       #
@@ -99,81 +89,132 @@ module Junkie
         @ready_for_new_links
       end
 
-      private
 
-      # This method is called from the watchdog timer periodically
+      # Utility method that fetches the QueueData, validates it and returns
+      # the JSON Object as a real Ruby Object
       #
-      # It monitors the download process and reacts depending on the results
+      # @raise [Junkie::InvalidResponse] if the response is faulty
+      # @return [Array] the JSON response as an object
+      def get_queue_data
+        data = @api.call(:getQueueData)
+
+        if data && data.is_a?(Array)
+          return data
+        end
+        raise Junkie::InvalidResponse,
+            "'#{data}' is not a valid response from Pyload"
+      end
+
+
+      #########################################################################
+      # This method is called from the watchdog timer periodically            #
+      #                                                                       #
+      # It monitors the download process and reacts depending on the results  #
       def monitor_progress
         log.debug("Watchdog timer has been fired")
         in_fiber {
           catch(:break) {
-            queue_data = @api.call(:getQueueData)
+            queue_data = get_queue_data
 
-            if queue_data.empty? and @active_episode.nil?
-              log.info("Empty Pyload queue, I cancel the watchdog timer")
-              @watchdog_enabled = false
-              throw :break
-            end
+            # check for empty queue and disable watchdog if needed
+            check_for_empty_queue(queue_data)
 
             # extract package IDs and map them to current downloads
             update_package_ids(queue_data)
 
-            if has_queue_any_failed_links?(queue_data)
-              log.info("There are failed links in the Queue, will fix this")
-              @api.call(:restartFailed)
-              throw :break
-            end
-
-            # look for complete downloads
-            pids = get_complete_downloads(queue_data)
-
-            if not pids.empty?
-              if @skipped_timer_at_first_complete_detection
-
-                # post process complete active download and send it out on the
-                # channel
-                if pids.include? @active_episode.pid
-                  log.info("'#{@active_episode}' is extracted completely")
-                  @active_episode.pid = nil
-                  @active_episode.status = :extracted
-
-                  @should_send_it_on_channel = true
-                end
-
-                # remove all complete packages
-                @api.call(:deletePackages, {pids: pids})
-                log.info("Complete packages are removed from Pyload Queue")
-
-                @skipped_timer_at_first_complete_detection = false
-                @ready_for_new_links = true
-              else
-                # If a package is complete, we are sometimes so fast to remove
-                # it, before the extracting can be started, so we will skip it
-                # the first time
-                log.info("Complete package detected, skip the first time")
-                @skipped_timer_at_first_complete_detection = true
-              end
-            end
+            check_for_failed_links(queue_data)
+
+            check_for_complete_packages(queue_data)
           }
         }
+      rescue Junkie::InvalidResponse => e
+        log.error("Pyload returns InvalidResponse (#{e}), I will ignore this")
       end
 
 
-      # Searches for failed links in the pyload queue
+      # Checks if the Pyload Queue is empty and disables the watchdog if
+      # it is really real
       #
       # @param [Array] queue_data returned from :getQueueData api method
+      def check_for_empty_queue(queue_data)
+        if queue_data.empty?
+
+          unless @active_episode.nil?
+            log.info("Empty queue detected but a download is set as active")
+            return
+          end
+
+          log.info("Empty Pyload queue detected, will double check")
+
+          # check twice if the queue is definitely empty
+          check_data = get_queue_data
+          if check_data.empty?
+            log.info("Double check is successful, I will disable the watchdog")
+            @watchdog_enabled = false
+            throw :break
+          else
+            log.info("Double check failed, Queue is not empty")
+          end
+        end
+      end
+
+      # Searches for failed links in the pyload queue and reacts on this
+      # through issuing a restartFailed call
       #
-      # @return [Boolean] true if there are any failed links, false otherwise
-      def has_queue_any_failed_links?(queue_data)
+      # @param [Array] queue_data returned from :getQueueData api method
+      def check_for_failed_links(queue_data)
+        failed = 0
         queue_data.each do |package|
           package['links'].each do |link|
             status = Pyload::Api::FILE_STATUS[link['status']]
-            return true if status == :failed
+            (failed += 1) if status == :failed
           end
         end
 
-        false
+        if failed > 0
+          log.info("There are failed links in the Queue, will fix this")
+          @api.call(:restartFailed)
+          throw :break
+        end
+      end
+
+
+      # Looks for complete packages and does some post-processing
+      #
+      # @param [Array] queue_data returned from :getQueueData api method
+      def check_for_complete_packages(queue_data)
+        pids = get_complete_downloads(queue_data)
+
+        throw :break if pids.empty?
+
+        if @skipped_timer_at_first_complete_detection
+
+          # post process complete active download and send it out on the
+          # channel
+          if pids.include? @active_episode.pid
+            log.info("'#{@active_episode}' has been extracted completely")
+            @active_episode.pid = nil
+            @active_episode.status = :extracted
+
+            log.info("Sending complete episode out on the channel")
+            @channels[:episodes].push(@active_episode)
+            @active_episode = nil
+          end
+
+          # remove all complete packages
+          @api.call(:deletePackages, { pids: pids })
+          log.info("Complete packages are removed from Pyload Queue #{ pids }")
+
+          @skipped_timer_at_first_complete_detection = false
+          @ready_for_new_links = true
+
+        else
+          # If a package is complete, we are sometimes so fast to remove
+          # it, before the extracting can be started, so we will skip it
+          # the first time
+          log.info("Complete package detected, skip the first time")
+          @skipped_timer_at_first_complete_detection = true
+        end
       end
 
 
@@ -189,46 +230,45 @@ module Junkie
 
           next if package['linksdone'] == 0
 
-          # When extracting is in progress the status of the first link is set
-          # to :extracting
-          extracting = package['links'].select do |link|
-            Pyload::Api::FILE_STATUS[link['status']] == :extracting
+          invalid_links = package['links'].reject do |e|
+            [:done, :skipped].include?(
+              Junkie::Pyload::Api::FILE_STATUS[e['status']])
           end
-          next unless extracting.empty?
+          next unless invalid_links.empty?
 
           sizetotal = package['sizetotal'].to_i
           sizedone = package['sizedone'].to_i
 
-          if sizetotal > 0 && sizedone > 0
-
-            if sizetotal == sizedone
+          if sizetotal > 0 && sizedone > 0 && sizetotal == sizedone
               pids << package['pid']
-            end
           end
         end
 
         pids
       end
 
+
       # extract package IDs and map them to current downloads
       #
       # @param [Array] queue_data returned from :getQueueData api method
       def update_package_ids(queue_data)
-        queue_data.each do |package|
-          pid = package['pid']
-          next if @active_episode.pid == pid
 
-          if @active_episode.pid == pid-1
-            log.info("Package ID has been changed, I will correct this")
+        if queue_data.size == 1
+          pid = queue_data[0]['pid']
+          if @active_episode && @active_episode.pid != pid
+            log.info("Package ID has been changed, New PID is #{pid}")
             @active_episode.pid = pid
-            next
           end
 
+        elsif queue_data.size > 1
           raise InvalidStateError,
-            "PackageID #{pid} can't be mapped to active Download"
+            "There is more than 1 Package in the Queue"
         end
       end
 
+
+      # Removes existing packages from the Pyload Queue and creates a valid
+      # state to begin communication
       def cleanup
         log.debug("Made a cleanup of Pyload's Queue")
         @api.call(:stopAllDownloads)

data/lib/junkie/version.rb

@@ -1,3 +1,3 @@
 module Junkie
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/lib/junkie.rb

@@ -9,14 +9,6 @@ require 'junkie/patched/sjunkieex'
 require 'junkie/pyload/api'
 require 'junkie/pyload/observer'
 require 'junkie/notification/twitter'
-require 'logging'
-
-Logging.appenders.stdout(
-    'stdout',
-    :level  => :debug,
-    :layout => Logging.layouts.pattern(:pattern => '[%d] %-5l: %m\n')
-)
-Logging.logger.root.appenders = Logging.appenders.stdout
 
 module Junkie
 

data/spec/fixtures/pyload_queue_data.rb

@@ -0,0 +1,94 @@
+require 'json'
+
+module Junkie::Pyload::Observer::TestData
+  def self.get_fixture
+    json = <<-eos
+      [
+      {
+        "name": "onetreehill_sl_s09e04",
+        "links": [
+          {
+            "status": 12,
+            "format_size": "102.00 MiB",
+            "name": "onetreehill_sl_s09e04.part1.rar",
+            "plugin": "UploadedTo",
+            "url": "http:\/\/uploaded.net\/file\/kigscay8",
+            "packageID": 11,
+            "fid": 827,
+            "error": "",
+            "statusmsg": "downloade",
+            "order": 0,
+            "size": 106954752
+          },
+          {
+            "status": 12,
+            "format_size": "102.00 MiB",
+            "name": "onetreehill_sl_s09e04.part2.rar",
+            "plugin": "UploadedTo",
+            "url": "http:\/\/uploaded.net\/file\/pm6m7v3d",
+            "packageID": 11,
+            "fid": 828,
+            "error": "",
+            "statusmsg": "downloade",
+            "order": 1,
+            "size": 106954752
+          },
+          {
+            "status": 12,
+            "format_size": "102.00 MiB",
+            "name": "onetreehill_sl_s09e04.part3.rar",
+            "plugin": "UploadedTo",
+            "url": "http:\/\/uploaded.net\/file\/3tzo52uh",
+            "packageID": 11,
+            "fid": 829,
+            "error": "",
+            "statusmsg": "downloade",
+            "order": 2,
+            "size": 106954752
+          },
+          {
+            "status": 2,
+            "format_size": "102.00 MiB",
+            "name": "onetreehill_sl_s09e04.part4.rar",
+            "plugin": "UploadedTo",
+            "url": "http:\/\/uploaded.net\/file\/0e367cl3\/onetreehill_sl_s09e04.part4.rar",
+            "packageID": 11,
+            "fid": 830,
+            "error": "",
+            "statusmsg": "online",
+            "order": 3,
+            "size": 106954752
+          },
+          {
+            "status": 2,
+            "format_size": "78.96 MiB",
+            "name": "onetreehill_sl_s09e04.part5.rar",
+            "plugin": "UploadedTo",
+            "url": "http:\/\/uploaded.net\/file\/9iuhwfjk\/onetreehill_sl_s09e04.part5.rar",
+            "packageID": 11,
+            "fid": 831,
+            "error": "",
+            "statusmsg": "online",
+            "order": 4,
+            "size": 82795518
+          }
+        ],
+        "dest": 1,
+        "pid": 11,
+        "site": "",
+        "linksdone": 0,
+        "fids": null,
+        "sizetotal": 510614526,
+        "sizedone": 0,
+        "linkstotal": null,
+        "folder": "onetreehill_sl_s09e04",
+        "password": "",
+        "order": 0
+      }
+    ]
+    eos
+
+    JSON.load(json)
+  end
+
+end

data/spec/pyload_observer_spec.rb

@@ -0,0 +1,216 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require File.expand_path(File.dirname(__FILE__) + '/fixtures/pyload_queue_data')
+require 'yaml'
+
+describe Junkie::Pyload::Observer do
+  include EMHelper
+
+  # remove the Config module behaviour
+  before(:each) do
+    Junkie::Config.stub!(:get_config) do |clas|
+      clas.class::DEFAULT_CONFIG
+    end
+
+    stub_const("Junkie::CONFIG_FILE", '/dev/null')
+  end
+
+  let(:channels) { {episodes: EM::Channel.new} }
+  let(:epi_channel) { channels[:episodes]}
+  let(:episode) do
+    epi = Junkie::Episode.new( "One Tree Hill", "http://local.host/12345",
+                               "One.Tree.Hill.S09E01.Testepisode")
+    epi.status = :found
+    epi
+  end
+
+  context "monitor the pyload progress" do
+
+    let(:observer) do
+      ob = Junkie::Pyload::Observer.new(channels)
+      ob.api.stub(:call).and_return(nil)
+      ob.instance_variable_set(:@found_episodes, [episode])
+      ob.instance_variable_set(:@ready_for_new_links, true)
+      ob.send(:add_next_episode_to_pyload)
+
+      episode.pid = 10
+      ob
+    end
+
+    context "#get_queue_data" do
+
+      it "should throw InvalidResponse on empty string" do
+        observer.api.should_receive(:call).and_return("")
+        expect {
+          observer.send(:get_queue_data)
+        }.to raise_error(Junkie::InvalidResponse)
+      end
+
+      it "should throw InvalidResponse on empty response" do
+        observer.api.should_receive(:call).and_return(nil)
+        expect {
+          observer.send(:get_queue_data)
+        }.to raise_error(Junkie::InvalidResponse)
+      end
+
+      it "should return the valid response" do
+        response = [ {pid: 13} ]
+        observer.api.should_receive(:call).and_return(response)
+        observer.send(:get_queue_data).should eq response
+      end
+    end
+
+    context "#monitor_progress" do
+
+      let(:response) do
+        Junkie::Pyload::Observer::TestData::get_fixture
+      end
+
+      it "should catch InvalidResponse raised from #get_queue_data" do
+        observer.api.should_receive(:call).and_return("")
+
+        expect {
+          observer.send(:monitor_progress)
+        }.to_not raise_error(Junkie::InvalidResponse)
+      end
+
+      context "#check_for_empty_queue" do
+
+        it "should be an active watchdog timer" do
+          observer.instance_variable_get(:@watchdog_enabled).should be true
+        end
+
+        it "should disable the watchdog if there isn't an active download" do
+          observer.instance_variable_set(:@active_episode, nil)
+          # it double checks for emptiness
+          observer.should_receive(:get_queue_data).and_return([])
+
+          expect {
+            observer.send(:check_for_empty_queue, [])
+          }.to throw_symbol(:break)
+
+          observer.instance_variable_get(:@watchdog_enabled).should be false
+        end
+
+        it "shouldn't disable the watchdog if double check fails" do
+          observer.instance_variable_set(:@active_episode, nil)
+          observer.should_receive(:get_queue_data).and_return(response)
+
+          observer.send(:check_for_empty_queue, [])
+          observer.instance_variable_get(:@watchdog_enabled).should be true
+        end
+
+        it "shouldn't disable the watchdog if there is an active download" do
+          observer.send(:check_for_empty_queue, [])
+          observer.instance_variable_get(:@watchdog_enabled).should be true
+        end
+      end
+
+      context "#update_package_ids" do
+
+        it "should update the package id" do
+          observer.send(:update_package_ids, response)
+          episode.pid.should eq 11
+        end
+
+        it "should update the pid also if it is not incremented by 1" do
+          response[0]['pid'] = 12
+
+          expect {
+            observer.send(:update_package_ids, response)
+          }.to_not raise_error(Junkie::InvalidStateError)
+          episode.pid.should eq 12
+        end
+      end
+
+      context "#react_on_failed_links" do
+
+        let(:failed_id){ Junkie::Pyload::Api::FILE_STATUS.key(:failed) }
+
+        it "should call restartFailed on the Api if there are failed links" do
+          response[0]['links'][0]['status'] = failed_id
+
+          observer.api.should_receive(:call).with(:restartFailed)
+          expect {
+            observer.send(:check_for_failed_links, response)
+          }.to throw_symbol(:break)
+        end
+      end
+
+      context "#check_for_complete_packages" do
+
+        let(:file_status){ Junkie::Pyload::Api::FILE_STATUS }
+        let(:complete_response) do
+          copy = response.clone
+          new_links = copy[0]['links'].map do |e|
+            e['status'] = file_status.key(:done)
+            e
+          end
+          copy[0]['links'] = new_links
+          copy[0]['linksdone'] = new_links.size
+          copy[0]['sizedone'] = 12345
+          copy[0]['sizetotal'] = 12345
+          copy[0]['pid'] = 12
+          copy
+        end
+
+        context "#get_complete_downloads" do
+          it "should return pids where alle links are done and size is right" do
+            pids = observer.send(:get_complete_downloads, complete_response)
+            pids.should include(12)
+          end
+
+          it "should treat :skipped and :done links as complete" do
+            complete_response[0]['links'][1]['status'] = file_status.key(:skipped)
+            complete_response[0]['links'][3]['status'] = file_status.key(:skipped)
+
+            pids = observer.send(:get_complete_downloads, complete_response)
+            pids.should include(12)
+          end
+
+          it "should take file status more seriously than sizedone" do
+            complete_response[0]['links'][1]['status'] = file_status.key(:failed)
+
+            pids = observer.send(:get_complete_downloads, complete_response)
+            pids.should_not include(12)
+          end
+        end
+
+
+        context "detected complete package the first time" do
+
+          it "should set a flag if a complete package has been detected" do
+            observer.send(:check_for_complete_packages, complete_response)
+            observer.instance_variable_get(
+              :@skipped_timer_at_first_complete_detection).should eq true
+          end
+        end
+
+        context "detected complete package second time" do
+
+          it "should remove the complete packages from Queue" do
+            observer.api.should_receive(:call).with(:deletePackages, {pids: [12]})
+            observer.instance_variable_set(
+              :@skipped_timer_at_first_complete_detection, true)
+
+            em {
+              observer.send(:check_for_complete_packages, complete_response)
+            }
+          end
+
+          it "should send out the active episode out on the channel" do
+            observer.instance_variable_set(
+              :@skipped_timer_at_first_complete_detection, true)
+            episode.pid = 12
+
+            observer.send(:check_for_complete_packages, complete_response)
+            observer.instance_variable_get(:@active_episode).should be_nil
+            observer.send(:is_ready?).should be true
+          end
+
+        end
+      end
+
+    end
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: junkie
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-11 00:00:00.000000000 Z
+date: 2012-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
@@ -202,7 +202,9 @@ files:
 - lib/junkie/version.rb
 - spec/config_spec.rb
 - spec/environment_spec.rb
+- spec/fixtures/pyload_queue_data.rb
 - spec/pyload_api_spec.rb
+- spec/pyload_observer_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/pboehm/junkie
 licenses: []
@@ -231,6 +233,8 @@ summary: TV series managament tool that does all the work you have with your ser
 test_files:
 - spec/config_spec.rb
 - spec/environment_spec.rb
+- spec/fixtures/pyload_queue_data.rb
 - spec/pyload_api_spec.rb
+- spec/pyload_observer_spec.rb
 - spec/spec_helper.rb
 has_rdoc: