s3_meta_sync 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e68ff7b84c212d7cc29205287b35c9fa181303c1
-  data.tar.gz: ed8e7d21833f827d6e10e092d6a257fb3b03e29e
+  metadata.gz: '09f0ddb82b6163f8a53ff0ba9912bbd8225c3553'
+  data.tar.gz: 7e3062e0a16d5f1c76166bfdacd86621a4b0c48f
 SHA512:
-  metadata.gz: b0e3edc1923c29beed3247808673e97d805f674564ab2ff1bf4133d2035f04d5af6b7e7a6ea374d8191342a327b8c55414e09a4f5ce01a2fc0282a5a6d174cf4
-  data.tar.gz: 0b2ebf35c1eb061fc6d3ff6bb47ce9b8818cba8622142e739bdc6f8601946262a74b77213dc5c9c10b8fbc8928cc75d53a607cfccc8ba8953123171a89bc853e
+  metadata.gz: 44432cf00d405f9ddaf94ea3376d0b8024237a43ce8fc0f97ad18c04e42ffe5387649dbecf22fd925625d42c9b477dd92aefe5c300872af91843fffb46dcc606
+  data.tar.gz: 8865b2d1ec3214bbcf3ac7e409aeaf7114b91d2a338eb40c3b10e06c96f0261fb159e6e84c34be582f39299911ed92cd93ff97b8f7bf526092027c8848c6ace5

data/lib/s3_meta_sync.rb

@@ -39,6 +39,8 @@ module S3MetaSync
         opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
         opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
         opts.on("-p", "--parallel COUNT", Integer, "Use COUNT threads for download/upload default: 10") { |c| options[:parallel] = c }
+        opts.on("-o", "--open-timeout TIMEOUT", Integer, "Net::HTTP open timeout in seconds default: 5") { |c| options[:open_timeout] = c }
+        opts.on("-t", "--read-timeout TIMEOUT", Integer, "Net::HTTP read timeout in seconds default: 10") { |c| options[:read_timeout] = c }
         opts.on("--ssl-none", "Do not verify ssl certs") { options[:ssl_none] = true }
         opts.on("-z", "--zip", "Zip when uploading to save bandwidth") { options[:zip] = true }
         opts.on("--no-local-changes", "Do not md5 all the local files, they did not change") { options[:no_local_changes] = true }

data/lib/s3_meta_sync/syncer.rb

@@ -81,8 +81,10 @@ module S3MetaSync
           store_meta(staging_area, remote_meta)
 
           verify_integrity!(staging_area, destination, download, remote_meta[:files])
+          log "Swapping in directories #{destination} and #{staging_area}"
           self.class.swap_in_directory(destination, staging_area)
           FileUtils.mkdir(staging_area) # mktmpdir tries to remove this directory
+          log "Download finished"
         end
       end
     end
@@ -129,6 +131,7 @@ module S3MetaSync
     end
 
     def verify_integrity!(staging_area, destination, changed, remote)
+      log "Verifying integrity of #{changed.size} files" if changed.size > 0
       local = md5_hash(staging_area, changed)
       corrupted = local.select { |file, md5| remote[file] != md5 }.map(&:first)
       return if corrupted.empty?
@@ -180,6 +183,7 @@ module S3MetaSync
     end
 
     def delete_local_files(local, paths)
+      log "Delete #{paths.size} local files" if paths.size > 0
       paths = paths.map { |path| "#{local}/#{path}" }
       paths.each { |path| log "Deleting #{path}" }
       FileUtils.rm_f(paths)
@@ -198,6 +202,7 @@ module S3MetaSync
     end
 
     def store_meta(source, meta)
+      log "Storing meta file"
       file = "#{source}/#{META_FILE}"
       FileUtils.mkdir_p(File.dirname(file))
       File.write(file, meta.to_yaml)
@@ -262,12 +267,14 @@ module S3MetaSync
           "https://s3#{"-#{region}" if region}.amazonaws.com/#{@bucket}/#{path}"
         end
       options = (@config[:ssl_none] ? {:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE} : {})
+      options[:open_timeout] = @config.fetch(:open_timeout, 5) # 5 seconds
+      options[:read_timeout] = @config.fetch(:read_timeout, 10) # 10 seconds
       retry_downloads(url: url) { open(url, options) }
     end
 
     def retry_downloads(url:)
       yield
-    rescue OpenURI::HTTPError, Errno::ECONNRESET => e
+    rescue OpenURI::HTTPError, Errno::ECONNRESET, Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout => e
       max_retries = @config[:max_retries] || 2
       http_error_retries ||= 0
       http_error_retries += 1
@@ -290,6 +297,7 @@ module S3MetaSync
     end
 
     def delete_empty_folders(destination)
+      log "Deleting empty folders"
       `find #{destination} -depth -empty -delete`
     end
 

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.11.0"
+  VERSION = "0.12.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-11 00:00:00.000000000 Z
+date: 2017-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core