s3_meta_sync 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4f29dd0019e474ba1364be76ebfd587faff17782
-  data.tar.gz: a849c0d899cc4165741cda9c222ff8c817bd54c2
+  metadata.gz: b2ae988de36b8fabaa5dfa6f19726fc971b46133
+  data.tar.gz: 50e7f0a149cb5ed3bd3d6c09c9b32e2998118996
 SHA512:
-  metadata.gz: a0e544f47de061d2acb397f36474d2c91c69688ec1588b818a93c16b592c0862f906f9cb4f749c348008f15a1eacb534d8d549c6d30fee077983296f5fde0cd3
-  data.tar.gz: 317dd76fb942a58c8f60ccd41f2d27fe2a2bb2cd7acf54b7dca3ac21c7a60fa738dbbaa3162efdfb2368de3423f215134fc2c1275f825828463e4784c35fa157
+  metadata.gz: fa22acee1c5ecac7af983b52fa856948fb7c98f8b66bd374f5f869b24d08d725f6c083ebd89c6c440df3cd14ec50cfa0603546c8bfe899cf3bf1e5ddd09428fb
+  data.tar.gz: c6e7d3927654ec728757431865ced2cd2a08db8f5b78e9fe9a306f55a9b4c3527af8d114dffd5c2d3aa1fb92d5efdef22682787597bbf3a62270dcc71d3b42de

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

data/lib/s3_meta_sync.rb

@@ -34,31 +34,33 @@ module S3MetaSync
       remote_info = begin
         download_meta(destination)
       rescue RemoteWithoutMeta
+        log "Remote has no .s3-meta-sync, uploading everything"
         {}
       end
       generate_meta(source)
       local_info = read_meta(source)
+      upload = local_info.select { |path, md5| remote_info[path] != md5 }.map(&:first)
+      delete = remote_info.keys - local_info.keys
 
-      in_multiple_processes(local_info) do |path, md5|
-        next if remote_info[path] == md5
-        upload_file(source, path, destination)
-      end
-
-      delete_remote_files(destination, remote_info.keys - local_info.keys)
-
+      upload_files(source, destination, upload)
+      delete_remote_files(destination, delete)
       upload_file(source, META_FILE, destination)
     end
 
     def upload_file(source, path, destination)
+      log "Uploading #{path}"
       s3.objects["#{destination}/#{path}"].write File.read("#{source}/#{path}"), :acl => :public_read
     end
 
     def delete_remote_files(remote, paths)
+      paths.each { |path| log "Deleting #{@bucket}:#{remote}/#{path}" }
       s3.objects.delete paths.map { |path| "#{remote}/#{path}" }
     end
 
     def delete_local_files(local, paths)
-      File.delete(*paths.map { |path| "#{local}/#{path}" })
+      paths = paths.map { |path| "#{local}/#{path}" }
+      paths.each { |path| log "Deleting #{path}" }
+      File.delete(*paths)
     end
 
     def s3
@@ -94,6 +96,7 @@ module S3MetaSync
     end
 
     def download_content(path)
+      log "Downloading #{path}"
       url = "https://s3#{"-#{region}" if region}.amazonaws.com/#{@bucket}/#{path}"
       open(url).read
     rescue OpenURI::HTTPError
@@ -103,27 +106,39 @@ module S3MetaSync
     def download(source, destination)
       remote_info = download_meta(source)
       generate_meta(destination)
-      local_info = read_meta(destination) # TODO maybe generate !?
-
-      in_multiple_processes(remote_info) do |path, md5|
-        next if local_info[path] == md5
-        download_file(source, path, destination)
-      end
-
-      delete_local_files(destination, local_info.keys - remote_info.keys)
+      local_info = read_meta(destination)
+      download = remote_info.select { |path, md5| local_info[path] != md5 }.map(&:first)
+      delete = local_info.keys - remote_info.keys
 
+      download_files(source, destination, download)
+      delete_local_files(destination, delete)
       download_file(source, META_FILE, destination)
+      delete_empty_folders(destination)
+    end
 
+    def delete_empty_folders(destination)
       `find #{destination} -depth -empty -delete`
     end
 
+    def download_files(source, destination, paths)
+      in_parallel(:threads, paths) { |path| download_file(source, path, destination) }
+    end
+
+    def upload_files(source, destination, paths)
+      in_parallel(:processes, paths) { |path| upload_file(source, path, destination) }
+    end
+
     def region
       @config[:region] unless @config[:region].to_s.empty?
     end
 
-    def in_multiple_processes(data, &block)
+    def in_parallel(way, data, &block)
       processes = [@config[:parallel] || 10, data.size].min
-      Parallel.each(data, :in_processes => processes, &block) # we tried threads but that blew up with weird errors when having lot's of uploads :/
+      Parallel.each(data, :"in_#{way}" => processes, &block) # we tried threads but that blew up with weird errors when having lot's of uploads :/
+    end
+
+    def log(text)
+      $stderr.puts text if @config[:verbose]
     end
   end
 
@@ -157,6 +172,7 @@ module S3MetaSync
         opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
         opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
         opts.on("-p", "--parallel COUNT", Integer, "Use COUNT processes for download/upload default: 10") { |c| options[:parallel] = c }
+        opts.on("-V", "--verbose", "Verbose mode"){ options[:verbose] = true }
         opts.on("-h", "--help", "Show this.") { puts opts; exit }
         opts.on("-v", "--version", "Show Version"){ puts VERSION; exit}
       end.parse!(argv)

data.tar.gz.sig

@@ -1,2 +1,2 @@
-f�1]�a��Q�׭I������u}��X��fƍ[$iG�(D=�e�`J,ږN]
-#�:��v�N���Ci�u(����Qx_Ǹ��s%�S���/�$���K`+�у�I�4
+�
+l�.N�!�hD��j���������N���7$u+�D��=��ݤ&Hl��$6���^���/v�q<�n�L����rj�0�^����6*:2�\��Ѿe��An�{��-f^<��������uUe��f�0����V�g�_�&(w˦���(JGU���,�(�
=O������6��!�*3_+rr��.�������}GT�Ԯ0����w��R���z!�cB��%G�M����ɿ��r�2a�heV���8�

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Michael Grosser
@@ -29,7 +29,7 @@ cert_chain:
   y0kCSWmK6D+x/SbfS6r7Ke07MRqziJdB9GuE1+0cIRuFh8EQ+LN6HXCKM5pon/GU
   ycwMXfl0
   -----END CERTIFICATE-----
-date: 2013-09-26 00:00:00.000000000 Z
+date: 2013-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk