mir 0.1.3 → 0.1.4

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+# Mir Changelog
+
+### 0.1.4
+
+* Fixed bug that occurred during pull operations when the connection with Amazon S3 is interrupted mid-transmission. Pull operations will now continue to download an asset from S3 until the checksum matches what has been stored in the local index
+* Added configuration option max_download_attempts to allow the user to set the maximum number of pull attempts per file

data/README.md

@@ -16,7 +16,8 @@ The inspiration for this tool is to provide similar functionality to the classic
 Mir uses a YAML file for configuration settings. Unless you specify otherwise, Mir will look for the file  'mir_settings.yml' in the HOME and /etc/mir directories.
 
     settings:
-      max_upload_retries: 5
+      max_upload_attempts: 5
+      max_download_attempts: 5
       max_threads: 5
       cloud_provider:
         type: s3

data/lib/mir/application.rb

@@ -7,7 +7,7 @@ module Mir
   class Application
     
     DEFAULT_SETTINGS_FILE_NAME = "mir_settings.yml"
-    DEFAULT_BATCH_SIZE = 20
+    DEFAULT_BATCH_SIZE = 100
         
     # Creates a new Mir instance
     def self.start
@@ -146,37 +146,53 @@ module Mir
       
       # Copy the remote disk contents into the specified directory
       def pull(target)
-        Utils.try_create_dir(target)
-        write_dir = Dir.new(target)
+        write_dir = Utils.try_create_dir(target)
         Mir.logger.info "Copying remote disk to #{write_dir.path} using #{config.max_threads} threads"
+        failed_downloads = []
         
         time = Benchmark.measure do 
           queue = WorkQueue.new(config.max_threads)
-        
-          # loop through each resource. If the resource is a directory, create the path
-          # otherwise download the file
+          
           Models::Resource.ordered_groups(DEFAULT_BATCH_SIZE) do |resources|
-            resources.each do |resource|
-              dest = File.join(write_dir.path, resource.filename)
-              if resource.is_directory?  
-                Utils.try_create_dir(dest)
-              elsif !resource.synchronized?(dest)
-                queue.enqueue_b do 
-                  disk.copy(resource.abs_path, dest)
-                  if resource.synchronized?(dest)
-                    Mir.logger.info "Successful download #{dest}"
-                    puts "Pulled #{dest}"
-                  else
-                    Mir.logger.error "Incomplete download #{dest}"
+            # Track the number of download attempts made per resource
+            batch = resources.inject({}) { |set, resource| set[resource] = 0; set }
+            
+            while !batch.empty? do
+              batch.each do |resource, attempts|
+                dest = File.join(write_dir.path, resource.filename)
+                if resource.is_directory?
+                  Utils.try_create_dir(dest)
+                  batch.delete(resource)
+                elsif attempts >= config.max_download_attempts
+                  Mir.logger.info "Resource #{resource.abs_path} failed to download"
+                  failed_downloads << resource
+                  batch.delete(resource)
+                elsif resource.synchronized? dest
+                  Mir.logger.debug "Skipping already downloaded file #{resource.abs_path}"
+                  batch.delete(resource)
+                else
+                  batch[resource] += 1
+                  queue.enqueue_b do
+                    Mir.logger.debug "Beginning download of #{resource.abs_path}"
+                    disk.copy(resource.abs_path, dest)
+                    if resource.synchronized? dest
+                      FileUtils.chmod_R 0755, dest # allow binaries to execute
+                      puts "Pulled #{dest}"
+                      batch.delete(resource)
+                    end
                   end
                 end
               end
+              queue.join
             end
-            queue.join
           end
         end
         Mir.logger.info time
         puts "Completed pull operation #{time}"
+        unless failed_downloads.empty?
+          puts "The following files failed to download after several attempts:\n}"
+          puts failed_downloads.join("\n")
+        end
       end
           
   end

data/lib/mir/config.rb

@@ -1,5 +1,3 @@
-
-
 # The config class is used for storage of user settings and preferences releated to 
 # S3 storage
 module Mir
@@ -10,9 +8,20 @@ module Mir
     def initialize(config_file = nil)
       @config_file = config_file
       @settings, @database = nil, nil
+      
+      self.max_upload_attempts = 10
+      self.max_download_attempts = 10
+      self.max_threads = 5
+      
+      yield self if block_given?
     end
     
+    attr_accessor :max_upload_attempts, :max_download_attempts, :max_threads
+    
     # Validates configuration settings
+    # TODO: move this into a class method responsible for parsing the YAML file. We
+    # shouldn't have to explicitly check for a valid object unless the user has provided
+    # a settings file
     def valid?
       if @config_file.nil? or !File.exist?(@config_file)
         Mir.logger.error("Configuration file not found")

data/lib/mir/models/resource.rb

@@ -101,7 +101,7 @@ module Mir
       
       def update_failure
         num_times_failed = times_failed + 1
-        will_requeue = (num_times_failed < Mir::Application.config.max_upload_retries)
+        will_requeue = (num_times_failed < Mir::Application.config.max_upload_attempts)
         update_attributes :times_failed => num_times_failed, :in_progress => false, :queued => will_requeue
       end
       

data/lib/mir/utils.rb

@@ -11,6 +11,7 @@ module Mir
     
     def self.try_create_dir(path)
       Dir.mkdir(path) unless Dir.exist?(path)
+      Dir.new(path)
     end
     
     # Splits a file into pieces that may be reassembled later

data/lib/mir/version.rb

@@ -1,5 +1,5 @@
 module Mir
-  VERSION = [0,1,3]
+  VERSION = [0,1,4]
   
   def self.version
     VERSION.join('.')

metadata

@@ -2,7 +2,7 @@
 name: mir
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors: 
 - Nate Miller
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-22 00:00:00 -07:00
+date: 2011-09-23 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -90,6 +90,7 @@ extra_rdoc_files: []
 
 files: 
 - .gitignore
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md