get_them_all 1.0.0 → 1.0.1

data/bin/gta

@@ -32,14 +32,9 @@ class GtaRunner < Thor
     fail("file #{script_path} should define class #{class_name} !") unless Object.const_defined?( class_name.to_sym )
 
     info("Started with config file #{File.basename(script_path)}")
-    
-    # CTRL+C
-    trap("INT") do
-      EM::stop_event_loop()
-    end
-    
+        
     # create the instance (and start download)
-    class_name.constantize.new(
+    crawler = class_name.constantize.new(
         :storage => {
             :type => 'file',
             :params => {
@@ -47,7 +42,14 @@ class GtaRunner < Thor
               }
           },
         :extensions => [GetThemAll::ActionLogger.new]
-      ).start()
+      )
+    
+    # CTRL+C
+    trap("INT") do
+      crawler.stop()
+    end    
+    
+    crawler.start()
   end
 end
 

data/lib/get_them_all/action.rb

@@ -7,7 +7,7 @@ module GetThemAll
   
     include EM::Deferrable
   
-    def initialize(downloader, h, params = {})
+    def initialize(downloader, h = {}, params = {})
       @downloader = downloader
     
       @storage = @downloader.storage

data/lib/get_them_all/extensions/action_logger.rb

@@ -6,11 +6,33 @@ module GetThemAll
   # 
   class ActionLogger < Extension
     def initialize
+      
+      #
+      # reactor events
+      #
       register_handler('downloader.started') do |name, downloader|
         @skipped_files = 0
         @download_files = 0
       end
-    
+      
+      register_handler('downloader.stopping') do |name, downloader|
+        log("Engine stopping...")
+      end
+      
+      register_handler('downloader.stopped') do |name, downloader|
+        log("Engine stopped")
+      end
+      
+      register_handler('downloader.completed') do |name, worker, downloader|
+        log ""
+        log "Downloaded #{@download_files} files"
+        log "Skipped: #{@skipped_files}"
+      end
+      
+      
+      #
+      # examine events
+      #
       register_handler('action.examine.started') do |name, worker, action|
         log("Examining[#{action.level}] #{action.url}")
       end
@@ -23,8 +45,11 @@ module GetThemAll
       register_handler('action.examine.success') do |name, worker, action|
         # do nothing
       end
-    
-    
+      
+      
+      #
+      # download events
+      #
       register_handler('action.download.started') do |name, worker, action|
         log("Downloading #{action.url}")
       end
@@ -41,13 +66,7 @@ module GetThemAll
         @download_files += 1
         log("File downloaded: #{destpath}")
       end
-    
-      register_handler('downloader.completed') do |name, worker, downloader|
-        log ""
-        log "Downloaded #{@download_files} files"
-        log "Skipped: #{@skipped_files}"
-      end
-    
+      
     end
   
     def log(str)

data/lib/get_them_all/site_downloader.rb

@@ -1,4 +1,6 @@
 
+require 'fiber'
+
 require 'addressable/uri'
 require 'active_support/hash_with_indifferent_access'
 
@@ -14,9 +16,20 @@ module GetThemAll
   class SiteDownloader
     include Notifier
     
+    # number of worker for each tasks
     class_attribute :examiners_count, :downloaders_count
-    class_attribute :config
-  
+    
+    # delay between each action for one worker
+    class_attribute :examiners_delay, :downloaders_delay
+    
+    
+    self.examiners_count = 1
+    self.downloaders_count = 1
+    
+    # default: 100 to 200ms between actions
+    self.downloaders_delay = [100, 200]
+    self.examiners_delay = [100, 200]
+    
     ##
     # Determine what will be stored in the history file,
     # the default is to store the last url before the download
@@ -28,9 +41,6 @@ module GetThemAll
     # 
     class_attribute :history_tracking
   
-    self.examiners_count = 1
-    self.downloaders_count = 1
-  
     self.history_tracking = :default
   
     attr_reader :base_url, :storage, :history
@@ -62,6 +72,7 @@ module GetThemAll
       @base_url= args.delete(:base_url)
       @start_url = args.delete(:start_url) || '/'
       @folder_name= args.delete(:folder_name)
+      @login_request = args.delete(:login_request)
       
       # keep a pointer to each extension
       @extensions = args.delete(:extensions) || [ActionLogger]
@@ -105,10 +116,10 @@ module GetThemAll
       notify('downloader.started', self)
     
       EM::run do
-        EM::add_periodic_timer(5) do
-          if (EM::connection_count() == 0) && !@storage.working?
-            debug("no connections, exiting")
-            EM::stop_event_loop()
+        @exit_timer = EM::add_periodic_timer(2) do
+          # if all workers are idle
+          if @examiners.all?(&:idle?) && @downloaders.all?(&:idle?)
+            self.stop()
           end
         end
       
@@ -122,37 +133,93 @@ module GetThemAll
             end
           end
         end
-     
         
-        # queue the first action to start crawling
-        #  
-        @examine_queue.push(ExamineAction.new(self,
-            :url => @start_url,
-            :destination_folder => '/',
-            :level => 0,
-          ), 0)
+        # authenticate connection if required
+        if @login_request
+          open_url(*@login_request) do |req, doc|
+            after_login()
+          end
+        else
+          after_login()
+        end
         
+      end
       
-        # now that actions are queued, start handling them
-        # start each "worker"
-        # dequeuing is priority based, the download actions
-        # first and then the higher the level the higher the
-        # priority for examine actions, this is done this way
-        # to give work to the download workers asap.
-        # 
-        1.upto(self.class.examiners_count) do |n|
-          Worker.new(:examiner, n - 1, self, @examine_queue)
-        end
+      notify('downloader.completed', self)
+    end
+    
+    def after_login
+      # queue the first action to start crawling
+      #  
+      @examine_queue.push(ExamineAction.new(self,
+          :url => @start_url,
+          :destination_folder => '/',
+          :level => 0,
+        ), 0)
       
-        1.upto(self.class.downloaders_count) do |n|
-          Worker.new(:downloader, n - 1, self, @download_queue)
-        end
-
+    
+      # now that actions are queued, start handling them
+      # start each "worker"
+      # dequeuing is priority based, the download actions
+      # first and then the higher the level the higher the
+      # priority for examine actions, this is done this way
+      # to give work to the download workers asap.
+      # 
+      
+      @examiners = []
+      @downloaders = []
+      
+      1.upto(self.class.examiners_count) do |n|
+        @examiners << Worker.new(:examiner, n - 1, @examine_queue, self.class.examiners_delay)
       end
-   
-      save_history()
     
-      notify('downloader.completed', self)
+      1.upto(self.class.downloaders_count) do |n|
+        @downloaders << Worker.new(:downloader, n - 1, @download_queue, self.class.downloaders_delay)
+      end
+    end
+    
+    ##
+    # Cleanly stop the engine and ensure the history file is
+    # written.
+    # 
+    def stop
+      return if @stopping
+      
+      # first stop the exit timer, no longer needed once we are here
+      @exit_timer.cancel()
+      @stopping = true
+      
+      Fiber.new do
+        fiber = Fiber.current
+      
+        notify('downloader.stopping', self)
+      
+        # first ask every workers to stop their work
+        # starting with examiners
+        @examiners.each do |worker|
+          debug "Stopping Examiner #{worker.index}..."
+          worker.request_stop { fiber.resume }
+          Fiber.yield
+          debug "Stopped Examiner #{worker.index}"
+        end
+      
+        @downloaders.each do |worker|
+          debug "Stopping Downloader #{worker.index}..."
+          worker.request_stop { fiber.resume }
+          Fiber.yield
+          debug "Stopped Downloader #{worker.index}"
+        end
+        
+        # now that every worker is stopped, write the history
+        deferrable = save_history()
+        deferrable.callback{ fiber.resume }
+        Fiber.yield
+        
+        notify('downloader.stopped', self)
+        
+        # and stop the reactor
+        EM::stop_event_loop()
+      end.resume
     end
     
     class AssertionFailed < RuntimeError; end

data/lib/get_them_all/storage/file_storage.rb

@@ -18,8 +18,11 @@ module GetThemAll
       open(destpath, "wb") do |f|
         f.write( data )
       end
-    
-      deferrable.succeed
+      
+      # allow time to caller to schedule
+      # callbacks on the deferrable
+      
+      EM::next_tick{ deferrable.succeed }
       deferrable
     
     rescue Errno::EINVAL

data/lib/get_them_all/version.rb

@@ -1,3 +1,3 @@
 module GetThemAll
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

data/lib/get_them_all/worker.rb

@@ -4,45 +4,133 @@ module GetThemAll
   # each time an action is put in the queue it will handle it
   #
   class Worker
+    include Notifier
+    
     attr_reader :type, :index
-  
-    def initialize(type, index, downloader, queue)
+    
+    ##
+    # Create a worker.
+    # 
+    # @param [StringSymbol] type Name assigned to the worker
+    #   the only real use is to identify the worker.
+    # @param [Integer] index additional way to indetify the worker.
+    # @param [EM::Queue] queue the queue from which this worker
+    #   will take its jobs.
+    # @param [Integer,Array] delay Number of milliseconds between two
+    #   actions, if an array is provided the value will be randomized
+    #   between the two first values in the array.
+    # 
+    def initialize(type, index, queue, delay = 0)
       @type = type
       @index = index
+      @delay = delay
+      
+      # ensure delay is valid
+      unless @delay.is_a?(Integer) || (@delay.is_a?(Array) && @delay.size >= 2)
+        raise "invalid value for delay: #{@delay}"
+      end
     
-      @downloader = downloader
       @queue = queue
-    
+      @idle = true
+      
+
+      @stop_requested = false
+      
+      notify('worker.started', self)
+      
       @queue.pop do |action|
         handle_action(action)
       end
     
     end
-  
+    
+    ##
+    # when called the worker will
+    # finish the current job and then stop taking
+    # new jobs.
+    #
+    # if a block is given it will be called when
+    # the worker is no longer taking actions.
+    # 
+    def request_stop(&block)
+      @stop_requested = true
+      notify('worker.stop_requested', self)
+      
+      if @idle
+        # we are already stopped, just call the block
+        EM::next_tick{ block.call }
+      else
+        @stop_requested_block = block
+      end
+    end
+    
+    def idle?
+      @idle
+    end
+    
     ##
     # Take the next action in queue
     # 
     def take_next_job
-      EM::next_tick do
-        @queue.pop{|act| handle_action(act) }
+      @idle = true
+      
+      if @stop_requested
+        # do not take new jobs and call
+        # the passed block is any
+        @stop_requested_block.call if @stop_requested_block
+        notify('worker.stopped', self)
+      else
+        delay = delay_before_next_action()
+        EM::add_timer(delay / 1000) do
+          @queue.pop do |act|
+            handle_action(act)
+          end
+        end
+        
       end
     end
   
   
     def handle_action(action)
-      action.callback( &method(:take_next_job) )
-      action.errback do
-        # in case of failure, try again later (with slightly lower priority)
-        EM::add_timer(50) do
-          @queue.push(action, [action.level - 1, 0].max)
-        end
+      @idle = false
+      @current_action = action
       
-        # and take the next job
-        take_next_job()
-      end
-    
+      # register callbacks
+      action.callback( &method(:action_succeeded) )
+      action.errback( &method(:action_failed) )
+      
+      # and start the action
       action.do_action(self)
     end
   
+  private
+    def action_failed
+      # in case of failure, try again later (with slightly lower priority)
+      @queue.push(@current_action, [@current_action.level - 1, 0].max)
+    
+      # and take the next job
+      take_next_job()
+    end
+    
+    def action_succeeded
+      take_next_job()
+    end
+    
+    
+    ##
+    # Compute the delay before the next action can
+    # take place.
+    # 
+    # @return [Integer] Number of milliseconds to wait
+    # 
+    def delay_before_next_action
+      case @delay
+      when Integer then  @delay
+      when Array  then  rand(@delay[1] - @delay[0]) + @delay[0]
+      else
+        0
+      end
+    end
+    
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: get_them_all
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-10-16 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &70256844459680 !ruby/object:Gem::Requirement
+  requirement: &70096048287860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70256844459680
+  version_requirements: *70096048287860
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &70256844459180 !ruby/object:Gem::Requirement
+  requirement: &70096048287240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70256844459180
+  version_requirements: *70096048287240
 - !ruby/object:Gem::Dependency
   name: em-priority-queue
-  requirement: &70256844458680 !ruby/object:Gem::Requirement
+  requirement: &70096048286640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70256844458680
+  version_requirements: *70096048286640
 - !ruby/object:Gem::Dependency
   name: hpricot
-  requirement: &70256844458220 !ruby/object:Gem::Requirement
+  requirement: &70096048286060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.8.1
   type: :runtime
   prerelease: false
-  version_requirements: *70256844458220
+  version_requirements: *70096048286060
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &70256844457840 !ruby/object:Gem::Requirement
+  requirement: &70096048285560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70256844457840
+  version_requirements: *70096048285560
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70256844457300 !ruby/object:Gem::Requirement
+  requirement: &70096048284900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 3.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70256844457300
+  version_requirements: *70096048284900
 - !ruby/object:Gem::Dependency
   name: therubyracer
-  requirement: &70256844456800 !ruby/object:Gem::Requirement
+  requirement: &70096048284280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,10 @@ dependencies:
         version: 0.9.8
   type: :runtime
   prerelease: false
-  version_requirements: *70256844456800
+  version_requirements: *70096048284280
 - !ruby/object:Gem::Dependency
   name: dropbox
-  requirement: &70256844456420 !ruby/object:Gem::Requirement
+  requirement: &70096048283860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70256844456420
+  version_requirements: *70096048283860
 - !ruby/object:Gem::Dependency
   name: girl_friday
-  requirement: &70256844455800 !ruby/object:Gem::Requirement
+  requirement: &70096048283380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70256844455800
+  version_requirements: *70096048283380
 description: Mass downloader useable as standalone or as a library
 email: []
 executables:
@@ -153,7 +153,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3600791884245333025
+      hash: 2337480508786894492
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -162,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3600791884245333025
+      hash: 2337480508786894492
 requirements: []
 rubyforge_project: get_them_all
 rubygems_version: 1.8.11