s3_meta_sync 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc3bcfa388ab131fa0a8cd4117dac7c2e21460e7
-  data.tar.gz: 311333996d21e3ee022dd1beb8727b5298b2b5d6
+  metadata.gz: 37a24cec2c4aebc13b052d615ef89e90680f2382
+  data.tar.gz: 27acabc2a3fa0a110dad04c1567c78fc37d7ed07
 SHA512:
-  metadata.gz: 1d6d33a5fbfac5e447a0aa76470d35ab31e691bca5a5464af9d767d2a2aac569d61669a75d14e32d177f18cb2bf8ec68dabd1b0adae3c755e910c1f5d02fd5f1
-  data.tar.gz: 3b2adea548763266b7c439148d629c6ac521567aea458158c15be01bf7c81cf481de867790998dd2c7695b4a603d98e9344e4e79a3a6cd09c3e603c0ba2ca65b
+  metadata.gz: 003a7fe86ed7329f5352540790edb64168bbe6fe7e6d51e6e4539ee8238744c4fd3e9bfa6e96aa81fe7c23a8b4d762ed94595453c2860930c9cd36a5ddaeccde
+  data.tar.gz: 2b6d7b57bb944b4ad31316a400323223ee7a32d86df96d622b3bb0d14f941e90618399d8ade1d9faa6ea87f931b4828070a2a67ce03eb1054d51a590ab64c462

data/lib/s3_meta_sync.rb

@@ -37,7 +37,7 @@ module S3MetaSync
       generate_meta(source)
       local_info = read_meta(source)
 
-      local_info.each do |path, md5|
+      in_multiple_threads(local_info) do |path, md5|
         next if remote_info[path] == md5
         upload_file(source, path, destination)
       end
@@ -90,7 +90,7 @@ module S3MetaSync
       content = download_content("#{source}/#{path}")
       file = "#{destination}/#{path}"
       FileUtils.mkdir_p(File.dirname(file))
-      File.write(file, content)
+      File.write(file, content, :encoding => content.encoding)
     end
 
     def download_content(path)
@@ -105,7 +105,7 @@ module S3MetaSync
       generate_meta(destination)
       local_info = read_meta(destination) # TODO maybe generate !?
 
-      remote_info.each do |path, md5|
+      in_multiple_threads(remote_info) do |path, md5|
         next if local_info[path] == md5
         download_file(source, path, destination)
       end
@@ -122,6 +122,18 @@ module S3MetaSync
     def region
       @config[:region] unless @config[:region].to_s.empty?
     end
+
+    def in_multiple_threads(data, &block)
+      threads = [@config[:parallel] || 10, data.size].min
+      data = data.to_a
+      (1..threads).to_a.map do
+        Thread.new do
+          while chunk = data.pop
+            yield chunk
+          end
+        end
+      end.map(&:join)
+    end
   end
 
   class << self
@@ -149,6 +161,7 @@ module S3MetaSync
         opts.on("-k", "--key KEY", "AWS access key") { |c| options[:key] = c }
         opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
         opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
+        opts.on("-p", "--parallel COUNT", Integer, "Use COUNT threads for download/upload default: 10") { |c| options[:parallel] = c }
         opts.on("-h", "--help", "Show this.") { puts opts; exit }
         opts.on("-v", "--version", "Show Version"){ puts VERSION; exit}
       end.parse!(argv)

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Michael Grosser