s3_meta_sync 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d378c2048e347b69b8163925f00afe0be9fc81e1
-  data.tar.gz: da855bf2005d23ec2b707f12193c208fee1a3054
+  metadata.gz: b85122789e5f9115e5e5feae13687496fe84afc6
+  data.tar.gz: 934425dbaa4f5555e418f6ccaade64144d888b99
 SHA512:
-  metadata.gz: 9a75bf8dbb10772fd13b5eeac525a461a070a2416ee4e49ca608e70154ba33fc9a6aecc81c212d67f92f2b26e1c4dde0696ffdb06eff711d8f08973028d910ab
-  data.tar.gz: 24254882c7a9db8bcfee7a5f9f452ee73dc7cda15f2ea5c99c36b175ec540e17f7d510ecd646c0231ddb5e5c25cbf4c65094bc8b60d59e89b5ce8005eb09b4d7
+  metadata.gz: 3ea26c334b2aceda1df5f55cf17e66c5cb1f3a5cfb2d5246a09059f6bfc9246f17b7eb24680f492a461a40ebb9b98644b5fdb40cccf027fdf3ef93544219e6d8
+  data.tar.gz: 1067bb5ff41eae7fed797b291bdeb936cb19bf7060bf4b7d36ea46ea5ef5994653c54d1715f8e764a7ba821e40aed7e05c1c22ea3e8c3d1c89fe1c5cba672558

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.1.7"
+  VERSION = "0.2.0"
 end

data/lib/s3_meta_sync.rb

@@ -2,11 +2,15 @@ require "s3_meta_sync/version"
 require "open-uri"
 require "yaml"
 require "digest/md5"
-require "aws/s3"
 require "optparse"
-require "parallel"
 require "fileutils"
 
+# need to require these or upload in multiple threads will fail on systems with high load
+require "aws/s3"
+require "aws/s3/s3_object"
+require "aws/core/response"
+require "aws/s3/object_collection"
+
 module S3MetaSync
   RemoteWithoutMeta = Class.new(Exception)
   META_FILE = ".s3-meta-sync"
@@ -124,20 +128,27 @@ module S3MetaSync
     end
 
     def download_files(source, destination, paths)
-      in_parallel(:threads, paths) { |path| download_file(source, path, destination) }
+      in_multiple_threads(paths) { |path| download_file(source, path, destination) }
     end
 
     def upload_files(source, destination, paths)
-      in_parallel(:processes, paths) { |path| upload_file(source, path, destination) }
+      in_multiple_threads(paths) { |path| upload_file(source, path, destination) }
     end
 
     def region
       @config[:region] unless @config[:region].to_s.empty?
     end
 
-    def in_parallel(way, data, &block)
-      processes = [@config[:parallel] || 10, data.size].min
-      Parallel.each(data, :"in_#{way}" => processes, &block) # we tried threads but that blew up with weird errors when having lot's of uploads :/
+    def in_multiple_threads(data)
+      threads = [@config[:parallel] || 10, data.size].min
+      data = data.dup
+      threads.times do
+        Thread.new do
+          while slice = data.shift
+            yield slice
+          end
+        end
+      end
     end
 
     def log(text, important=false)
@@ -174,7 +185,7 @@ module S3MetaSync
         opts.on("-k", "--key KEY", "AWS access key") { |c| options[:key] = c }
         opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
         opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
-        opts.on("-p", "--parallel COUNT", Integer, "Use COUNT processes for download/upload default: 10") { |c| options[:parallel] = c }
+        opts.on("-p", "--parallel COUNT", Integer, "Use COUNT threads for download/upload default: 10") { |c| options[:parallel] = c }
         opts.on("-V", "--verbose", "Verbose mode"){ options[:verbose] = true }
         opts.on("-h", "--help", "Show this.") { puts opts; exit }
         opts.on("-v", "--version", "Show Version"){ puts VERSION; exit}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - Michael Grosser
@@ -45,20 +45,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: 
 email: michael@grosser.it
 executables: