s3_meta_sync 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6235bd32a6abcd45233ba1051b3069b051989565
-  data.tar.gz: eaf9169f2d31c82d4da45ca5f277aac62ecbc179
+  metadata.gz: e68ff7b84c212d7cc29205287b35c9fa181303c1
+  data.tar.gz: ed8e7d21833f827d6e10e092d6a257fb3b03e29e
 SHA512:
-  metadata.gz: 992e315158c7980790e148a1ce34d2152fb57b4b25d66f66f79b6245323f80ddf729586bfd67a81d067aee840d2d6a107d0f4897c3df59f13c0dbe554d902a2f
-  data.tar.gz: 5e3326f634a9865bd9e840f262f793ef055c869e6c8f76b8a9c868ae943dcef5834be84e2ede43eafad7aeb93298a37d5641071224f6583137195db6e203a79f
+  metadata.gz: b0e3edc1923c29beed3247808673e97d805f674564ab2ff1bf4133d2035f04d5af6b7e7a6ea374d8191342a327b8c55414e09a4f5ce01a2fc0282a5a6d174cf4
+  data.tar.gz: 0b2ebf35c1eb061fc6d3ff6bb47ce9b8818cba8622142e739bdc6f8601946262a74b77213dc5c9c10b8fbc8928cc75d53a607cfccc8ba8953123171a89bc853e

data/lib/s3_meta_sync.rb

@@ -42,6 +42,7 @@ module S3MetaSync
         opts.on("--ssl-none", "Do not verify ssl certs") { options[:ssl_none] = true }
         opts.on("-z", "--zip", "Zip when uploading to save bandwidth") { options[:zip] = true }
         opts.on("--no-local-changes", "Do not md5 all the local files, they did not change") { options[:no_local_changes] = true }
+        opts.on("--retries MAX", Integer, "MAX number of times retrying failed http requests default: 2") { |c| options[:max_retries] = c }
         opts.on("-V", "--verbose", "Verbose mode"){ options[:verbose] = true }
         opts.on("-h", "--help", "Show this.") { puts opts; exit }
         opts.on("-v", "--version", "Show Version") { puts VERSION; exit}

data/lib/s3_meta_sync/syncer.rb

@@ -11,6 +11,7 @@ require "s3_meta_sync/zip"
 module S3MetaSync
   class Syncer
     DEFAULT_REGION = 'us-east-1'
+    STAGING_AREA_PREFIX = "s3ms_"
 
     def initialize(config)
       @config = config
@@ -59,6 +60,8 @@ module S3MetaSync
     end
 
     def download(source, destination)
+      delete_old_temp_folders
+
       remote_meta = download_meta(source)
       local_files = ((@config[:no_local_changes] && read_meta(destination)) || meta_data(destination))[:files]
 
@@ -68,7 +71,8 @@ module S3MetaSync
       log "Downloading: #{download.size} Deleting: #{delete.size}", true
 
       if download.any? || delete.any?
-        Dir.mktmpdir do |staging_area|
+        Dir.mktmpdir(STAGING_AREA_PREFIX) do |staging_area|
+          log "Staging area: #{staging_area}"
           FileUtils.mkdir_p(destination)
           copy_content(destination, staging_area)
           download_files(source, staging_area, download, remote_meta[:zip])
@@ -83,7 +87,21 @@ module S3MetaSync
       end
     end
 
+    # Sometimes SIGTERM causes Dir.mktmpdir to not properly delete the temp folder
+    # Remove 1 day old folders
+    def delete_old_temp_folders
+      path = File.join(Dir.tmpdir, STAGING_AREA_PREFIX + '*')
+
+      day = 24 * 60 * 60
+      dirs = Dir.glob(path)
+      dirs.select! { |dir| Time.now.utc - File.ctime(dir).utc > day } # only stale ones
+      removed = dirs.each { |dir| FileUtils.rm_rf(dir) }
+
+      log "Removed #{removed} old temp folder(s)" if removed.count > 0
+    end
+
     def copy_content(destination, dir)
+      log "Copying content from #{destination} to #{dir}"
       system "cp -R #{destination}/* #{dir} 2>/dev/null"
     end
 
@@ -209,13 +227,13 @@ module S3MetaSync
       content = download_content("#{destination}/#{META_FILE}") { |io| io.read }
       parse_yaml_content(content)
     rescue OpenURI::HTTPError
-      retries ||= 1
-      if retries == 2
-        raise RemoteWithoutMeta
-      else
-        retries += 1
+      retries ||= 0
+      retries += 1
+      if retries <= 1
         sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
         retry
+      else
+        raise RemoteWithoutMeta
       end
     end
 
@@ -244,22 +262,27 @@ module S3MetaSync
           "https://s3#{"-#{region}" if region}.amazonaws.com/#{@bucket}/#{path}"
         end
       options = (@config[:ssl_none] ? {:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE} : {})
-      open(url, options)
-    rescue OpenURI::HTTPError
-      retries ||= 0
-      retries += 1
-      if retries >= 3
+      retry_downloads(url: url) { open(url, options) }
+    end
+
+    def retry_downloads(url:)
+      yield
+    rescue OpenURI::HTTPError, Errno::ECONNRESET => e
+      max_retries = @config[:max_retries] || 2
+      http_error_retries ||= 0
+      http_error_retries += 1
+      if http_error_retries <= max_retries
+        log "#{e.class} error downloading #{url}, retrying #{http_error_retries}/#{max_retries}"
+        retry
+      else
         $!.message << " -- while trying to download #{url}"
         raise
-      else
-        log "HTTP Error downloading #{path}, retrying"
-        retry
       end
     rescue OpenSSL::SSL::SSLError
-      retries ||= 0
-      retries += 1
-      if retries == 1
-        log "SSL error downloading #{path}, retrying"
+      ssl_error_retries ||= 0
+      ssl_error_retries += 1
+      if ssl_error_retries == 1
+        log "SSL error downloading #{url}, retrying"
         retry
       else
         raise

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.10.0"
+  VERSION = "0.11.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-20 00:00:00.000000000 Z
+date: 2017-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -56,7 +56,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Sync folders with s3 using a metadata file and md5 diffs