s3_meta_sync 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5656a323ca7eaad2a794b645ff158819a2d1edd6
-  data.tar.gz: 47323041dbc94b89b34aa8dc821eb54dd9be3880
+  metadata.gz: 7e7e8df15d87392103c01db3a50028655bfaeee1
+  data.tar.gz: eb8535f78ac0c43e5446f24d940c40f927548e97
 SHA512:
-  metadata.gz: 3b8037f17c549849ab87b37c4587c68cdaaafe70d0532f556dbfd7e30a6f6d9a42ab1d28ffa6668d401e07ecfd2328282b94b366c54e94f3e0d740536e480ce5
-  data.tar.gz: 37130dbe101aa047ad4fdd30163a5cd20754985368eb2db6b4db98d84e72ae2a84e4fad4b99ea64d62d637d04cb89c0c20744ea6aec5c59eb4569079e2a16431
+  metadata.gz: 16603033e2b2c25c91df705cb4ae1ecefcd94bf79f2cb5859986a828e980ae882a99c6bbb9643bdd219d335bf67a4caaea95d6e150bf87a6d85cc211c19dfa1b
+  data.tar.gz: 25154a729785baf27091469c2a08b735b80105d829f2e01368a6ad539212539a155ba6ca4edb6410f9cb1582c95ee3c2e3a9e672b42a4c868637b54e6b1660d8

data/lib/s3_meta_sync.rb

@@ -19,6 +19,7 @@ module S3MetaSync
   RemoteWithoutMeta = Class.new(Exception)
   RemoteCorrupt = Class.new(Exception)
   META_FILE = ".s3-meta-sync"
+  CORRUPT_FILES_LOG = "s3-meta-sync-corrupted.log"
 
   class Syncer
     def initialize(config)
@@ -40,15 +41,16 @@ module S3MetaSync
     private
 
     def upload(source, destination)
+      corrupted = consume_corrupted_files(source)
       remote_info = begin
         download_meta(destination)
       rescue RemoteWithoutMeta
-        log "Remote has no .s3-meta-sync, uploading everything"
+        log "Remote has no .s3-meta-sync, uploading everything", true
         {}
       end
       generate_meta(source)
       local_info = read_meta(source)
-      upload = local_info.select { |path, md5| remote_info[path] != md5 }.map(&:first)
+      upload = local_info.select { |path, md5| remote_info[path] != md5 }.map(&:first) | corrupted
       delete = remote_info.keys - local_info.keys
       log "Uploading: #{upload.size} Deleting: #{delete.size}", true
 
@@ -67,14 +69,14 @@ module S3MetaSync
       log "Downloading: #{download.size} Deleting: #{delete.size}", true
 
       unless download.empty? && delete.empty?
-        Dir.mktmpdir do |dir|
-          copy_content(destination, dir)
-          download_files(source, dir, download)
-          delete_local_files(dir, delete)
-          download_file(source, META_FILE, dir)
-          verify_integrity!(dir)
-          delete_empty_folders(dir)
-          swap_in_directory(destination, dir)
+        Dir.mktmpdir do |staging_area|
+          copy_content(destination, staging_area)
+          download_files(source, staging_area, download)
+          delete_local_files(staging_area, delete)
+          download_file(source, META_FILE, staging_area)
+          verify_integrity!(staging_area, destination)
+          delete_empty_folders(staging_area)
+          swap_in_directory(destination, staging_area)
         end
       end
     end
@@ -89,17 +91,29 @@ module S3MetaSync
       FileUtils.mkdir(dir) # make ensure in outside mktmpdir not blow up
     end
 
-    def verify_integrity!(source)
-      file = "#{source}/#{META_FILE}"
-      old = File.read(file)
-      generate_meta(source)
-      new = File.read(file)
-      if new != old
-        log "old meta:\n#{old}\n\nnew meta:\n#{new}", true
+    def verify_integrity!(staging_area, destination)
+      file = "#{staging_area}/#{META_FILE}"
+      downloaded = YAML.load_file(file)
+      actual = meta_data(staging_area)
+
+      if downloaded != actual
+        corrupted = actual.select { |file, md5| downloaded[file] != md5 }.map(&:first)
+        File.write("#{destination}/#{CORRUPT_FILES_LOG}", corrupted.join("\n"))
+        log "corrupted files downloaded:\n#{corrupted.join("\n")}", true
         raise RemoteCorrupt
       end
-    ensure
-      File.write(file, old) if old
+    end
+
+    def consume_corrupted_files(source)
+      log = "#{source}/#{CORRUPT_FILES_LOG}"
+      if File.exist?(log)
+        corrupted = File.read(log).split("\n")
+        log "force uploading #{corrupted.size} corrupted files", true
+        File.unlink log
+        corrupted
+      else
+        []
+      end
     end
 
     def upload_file(source, path, destination)
@@ -236,6 +250,7 @@ module S3MetaSync
         opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
         opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
         opts.on("-p", "--parallel COUNT", Integer, "Use COUNT threads for download/upload default: 10") { |c| options[:parallel] = c }
+        opts.on("--log-corrupted-files", "Log corrupted files, they will be force uploaded on the next run") { |c| options[:log_corrupted_files] = true }
         opts.on("--ssl-none", "Do not verify ssl certs") { |c| options[:ssl_none] = true }
         opts.on("-V", "--verbose", "Verbose mode"){ options[:verbose] = true }
         opts.on("-h", "--help", "Show this.") { puts opts; exit }

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Michael Grosser
@@ -30,7 +30,7 @@ cert_chain:
   F5etKHZg0j3eHO31/i2HnswY04lqGImUu6aM5EnijFTB7PPW2KwKKM4+kKDYFdlw
   /0WV1Ng2/Y6qsHwmqGg2VlYj2h4=
   -----END CERTIFICATE-----
-date: 2014-02-27 00:00:00.000000000 Z
+date: 2014-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk