get_them_all 1.0.3 → 1.0.4

data/bin/gta

@@ -49,7 +49,10 @@ class GtaRunner < Thor
       crawler.stop()
     end    
     
-    crawler.start()
+    deferrable = crawler.start()
+    deferrable.callback do
+      EM::stop_event_loop()
+    end
   end
 end
 

data/lib/get_them_all/actions/download_action.rb

@@ -14,20 +14,43 @@ module GetThemAll
     
         req = @downloader.open_url(@url, "GET", nil, @referer)
         req.callback do |req|
-      
-          destpath = compute_filename(worker)
-          download = @storage.write(destpath, req.response)
-      
-          download.callback do      
+          
+          destpath= @downloader.get_file_destpath_from_action(self)
+          
+          file_exists = @storage.exist?(destpath)
+          
+          # if the destination path does not already exists
+          # or if it does exists but we were asked to rename
+          # the file.
+          # 
+          if !file_exists || @downloader.rename_duplicates
+            
+            if file_exists
+              destpath = compute_unique_filename(destpath)
+              notify('action.download.renamed', worker, self, destpath)
+            end
+            
+            download = @storage.write(destpath, req.response)
+
+            download.callback do      
+              add_to_history()
+              set_deferred_status(:succeeded)
+
+              notify('action.download.success', worker, self, destpath)
+            end
+
+            download.errback do
+              notify('action.download.failure', worker, self)
+            end
+            
+          else
+            # if we arrive here the destination exists
+            # and we don't want to rename it
             add_to_history()
             set_deferred_status(:succeeded)
-      
-            notify('action.download.success', worker, self, destpath)
+            notify('action.download.already_exists', worker, self, destpath)
           end
-      
-          download.errback do
-            notify('action.download.failure', worker, self)
-          end      
+          
         end
     
         req.timeout(5)
@@ -63,16 +86,13 @@ module GetThemAll
       end
     end
   
-    def compute_filename(worker)
-      destpath= @downloader.get_file_destpath_from_action(self)
-        
+    def compute_unique_filename(destpath)
       # find an unused filename
-      while @storage.exist?(destpath)        
+      begin
         path, filename= File.dirname(destpath), File.basename(destpath).split(".")
         filename= "#{filename[0]}_#{random_string(2)}.#{filename[1]}"
         destpath= File.join(path, filename)
-        notify('action.download.renamed', worker, self, destpath)
-      end
+      end while @storage.exist?(destpath)
     
       destpath
     end

data/lib/get_them_all/extensions/action_logger.rb

@@ -67,6 +67,11 @@ module GetThemAll
         log("File downloaded: #{destpath}")
       end
       
+      register_handler('action.download.already_exists') do |name, worker, action, destpath|
+        @download_files += 1
+        log("oo File already exists: #{destpath}")
+      end
+      
     end
   
     def log(str)

data/lib/get_them_all/extensions/gauge_display.rb

@@ -51,6 +51,8 @@ module GetThemAll
       register_handler('action.download.success', &method(:work_completed))
       register_handler('action.download.failure', &method(:work_failed))
       register_handler('action.download.skipped', &method(:work_skipped))
+      register_handler('action.download.already_exists', &method(:work_skipped))
+      
     end
   
 

data/lib/get_them_all/site_downloader.rb

@@ -45,6 +45,11 @@ module GetThemAll
   
     attr_reader :base_url, :storage, :history
     
+    ##
+    # If true a new filename will be generated for every file
+    # for which the destination already exists
+    attr_reader :rename_duplicates
+    
     ##
     # Create and start the crawler.
     # 
@@ -56,6 +61,8 @@ module GetThemAll
     # @option args [String] :folder_name The root path where
     #   downloaded files will be saved (appended to the storage root).
     # @option args [Array] :extensions Array of Extension object.
+    # @option args [Boolean] :rename_duplicates If true a new name will be
+    #   generated if the file exists.
     # 
     # @option args [Hash] :storage Configure storage backend
     #   :type is the backend name
@@ -73,6 +80,7 @@ module GetThemAll
       @start_url = args.delete(:start_url) || '/'
       @folder_name= args.delete(:folder_name)
       @login_request = args.delete(:login_request)
+      @rename_duplicates = args.delete(:rename_duplicates)
       
       # keep a pointer to each extension
       @extensions = args.delete(:extensions) || [ActionLogger]
@@ -186,7 +194,7 @@ module GetThemAll
     # Cleanly stop the engine and ensure the history file is
     # written.
     # 
-    def stop
+    def stop(&block)
       return if @stopping
       
       # first stop the exit timer, no longer needed once we are here
@@ -221,9 +229,9 @@ module GetThemAll
         
         notify('downloader.stopped', self)
         
-        # and stop the reactor
-        EM::stop_event_loop()
+        block.call if block
       end.resume
+      
     end
     
     class AssertionFailed < RuntimeError; end

data/lib/get_them_all/storage/dropbox_storage.rb

@@ -60,8 +60,12 @@ module GetThemAll
       else
         true
       end
+      
     rescue Dropbox::FileNotFoundError
       false
+    rescue => err
+      show_error(err)
+      false
     end
   
     ##
@@ -85,6 +89,7 @@ module GetThemAll
       
       deferrable
     rescue => err
+      show_error(err)
       if retries < 4
         # puts "[#{retries}] Upload error, retrying: #{err}"
         retries += 1
@@ -104,6 +109,7 @@ module GetThemAll
       destpath = build_destpath(path)
       @session.download(destpath)
     rescue => err
+      show_error(err)
       raise ReadError, "cannot read file: #{err}"
     end
   

data/lib/get_them_all/storage.rb

@@ -23,5 +23,13 @@ module GetThemAll
       File.join(@root, path)
     end
   
+  private
+    def show_error(err)
+      error(err.inspect)
+      err.backtrace.each do |line|
+        error(line)
+      end
+    end
+    
   end
 end

data/lib/get_them_all/version.rb

@@ -1,3 +1,3 @@
 module GetThemAll
-  VERSION = "1.0.3"
+  VERSION = "1.0.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: get_them_all
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-10-16 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &70225302446980 !ruby/object:Gem::Requirement
+  requirement: &70163898376540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70225302446980
+  version_requirements: *70163898376540
 - !ruby/object:Gem::Dependency
   name: em-http-request
-  requirement: &70225302446460 !ruby/object:Gem::Requirement
+  requirement: &70163898375880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70225302446460
+  version_requirements: *70163898375880
 - !ruby/object:Gem::Dependency
   name: em-priority-queue
-  requirement: &70225302445960 !ruby/object:Gem::Requirement
+  requirement: &70163898375200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70225302445960
+  version_requirements: *70163898375200
 - !ruby/object:Gem::Dependency
   name: hpricot
-  requirement: &70225302445480 !ruby/object:Gem::Requirement
+  requirement: &70163898374580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.8.1
   type: :runtime
   prerelease: false
-  version_requirements: *70225302445480
+  version_requirements: *70163898374580
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &70225302445100 !ruby/object:Gem::Requirement
+  requirement: &70163898374060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70225302445100
+  version_requirements: *70163898374060
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70225302444480 !ruby/object:Gem::Requirement
+  requirement: &70163898373340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 3.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70225302444480
+  version_requirements: *70163898373340
 - !ruby/object:Gem::Dependency
   name: therubyracer
-  requirement: &70225302443860 !ruby/object:Gem::Requirement
+  requirement: &70163898372680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,10 @@ dependencies:
         version: 0.9.8
   type: :runtime
   prerelease: false
-  version_requirements: *70225302443860
+  version_requirements: *70163898372680
 - !ruby/object:Gem::Dependency
   name: dropbox
-  requirement: &70225302443460 !ruby/object:Gem::Requirement
+  requirement: &70163898372160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70225302443460
+  version_requirements: *70163898372160
 - !ruby/object:Gem::Dependency
   name: girl_friday
-  requirement: &70225302443000 !ruby/object:Gem::Requirement
+  requirement: &70163898371540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70225302443000
+  version_requirements: *70163898371540
 description: Mass downloader useable as standalone or as a library
 email: []
 executables:
@@ -153,7 +153,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3586238315157463383
+      hash: -592874061905408396
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -162,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3586238315157463383
+      hash: -592874061905408396
 requirements: []
 rubyforge_project: get_them_all
 rubygems_version: 1.8.11