s3_meta_sync 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 37a24cec2c4aebc13b052d615ef89e90680f2382
-  data.tar.gz: 27acabc2a3fa0a110dad04c1567c78fc37d7ed07
+  metadata.gz: 2e0d479fa242abc69a677504e15d1e3907da0d40
+  data.tar.gz: 351802613cb38c566476317eaa4cbcff2ce5f936
 SHA512:
-  metadata.gz: 003a7fe86ed7329f5352540790edb64168bbe6fe7e6d51e6e4539ee8238744c4fd3e9bfa6e96aa81fe7c23a8b4d762ed94595453c2860930c9cd36a5ddaeccde
-  data.tar.gz: 2b6d7b57bb944b4ad31316a400323223ee7a32d86df96d622b3bb0d14f941e90618399d8ade1d9faa6ea87f931b4828070a2a67ce03eb1054d51a590ab64c462
+  metadata.gz: 25283e6b0bb280cd37cf24050322b4802b407f7587ab982b0890b00680c0e1ac4d0137d44f353e3f5f1cc46aa363fb1c2ef47a58f11bdeebae0e44517cba36fd
+  data.tar.gz: 96cf366f80bb9340350a8147d158ab9969fa6e734e8d32a694526eeae54b64c65b033dbe3eefb0b6440a59bc4a62e56a7b4e639d44c94a02e5cec9431c5af70b

data/lib/s3_meta_sync.rb

@@ -4,6 +4,7 @@ require "yaml"
 require "digest/md5"
 require "aws/s3"
 require "optparse"
+require "parallel"
 
 module S3MetaSync
   RemoteWithoutMeta = Class.new(Exception)
@@ -37,7 +38,7 @@ module S3MetaSync
       generate_meta(source)
       local_info = read_meta(source)
 
-      in_multiple_threads(local_info) do |path, md5|
+      in_multiple_processes(local_info) do |path, md5|
         next if remote_info[path] == md5
         upload_file(source, path, destination)
       end
@@ -105,7 +106,7 @@ module S3MetaSync
       generate_meta(destination)
       local_info = read_meta(destination) # TODO maybe generate !?
 
-      in_multiple_threads(remote_info) do |path, md5|
+      in_multiple_processes(remote_info) do |path, md5|
         next if local_info[path] == md5
         download_file(source, path, destination)
       end
@@ -123,16 +124,9 @@ module S3MetaSync
       @config[:region] unless @config[:region].to_s.empty?
     end
 
-    def in_multiple_threads(data, &block)
-      threads = [@config[:parallel] || 10, data.size].min
-      data = data.to_a
-      (1..threads).to_a.map do
-        Thread.new do
-          while chunk = data.pop
-            yield chunk
-          end
-        end
-      end.map(&:join)
+    def in_multiple_processes(data, &block)
+      processes = [@config[:parallel] || 10, data.size].min
+      Parallel.each(data, :in_processes => processes, &block) # we tried threads but that blew up with weird errors when having lot's of uploads :/
     end
   end
 
@@ -161,7 +155,7 @@ module S3MetaSync
         opts.on("-k", "--key KEY", "AWS access key") { |c| options[:key] = c }
         opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
         opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
-        opts.on("-p", "--parallel COUNT", Integer, "Use COUNT threads for download/upload default: 10") { |c| options[:parallel] = c }
+        opts.on("-p", "--parallel COUNT", Integer, "Use COUNT processes for download/upload default: 10") { |c| options[:parallel] = c }
         opts.on("-h", "--help", "Show this.") { puts opts; exit }
         opts.on("-v", "--version", "Show Version"){ puts VERSION; exit}
       end.parse!(argv)

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Michael Grosser
@@ -45,6 +45,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email: michael@grosser.it
 executables: