filey-diff 1.2.1 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ba4ae040f41db6cdd1961ba07104e6d2a06934da
-  data.tar.gz: a27e219df60ba06eec12835610baac8d181f210d
+  metadata.gz: 47c72a8f409e6259b8e477bfab6367164f433feb
+  data.tar.gz: 11a0a59733aabaed1c83d1bbc2d59965f257b919
 SHA512:
-  metadata.gz: b515c0d55fd854cbbfe55260a3359c2293e01215d7bc50aa134659b03085b1934ac626b771dc924234cd31a80a4bce49aaa5870eed7c9b62e1fb227b916416e9
-  data.tar.gz: 4cf9a66141a590496a286ed3aa8875f65e005d170dc249287782e545eb75f2f50281e081aadf1fc49d4f6493c98da49000d7cd81949897e7c6e48873ca4c5f5c
+  metadata.gz: 1c68b67b49f876f3e532834fc765f9abe323c515fff0862dd4280d45e98805ef4c5cc1fd161f92fb72a48b7c63d9d01aac3461e2a1a9193136629e9dd869e59e
+  data.tar.gz: 95ac9a4c7c740c3b824176a4951f6d2f2f1ff68840c3f446334860801ea45e9ffc0543e805f8223df56541631380403322bcd25746a823bcbca1c7771345170b

data/changelog.md

@@ -2,6 +2,11 @@
 
 This project is [Semantically Versioned](http://semver.org).
 
+## 1.2.2
+
+* AwsSdkS3 data source: load S3 objects in slices of 100 when concurrency is
+  enabled
+
 ## 1.2.1
 
 * Allow disabling of parallel processing with `ENV['disable_parallel_processing'] = 'true'`

data/filey-diff.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name              = 'filey-diff'
-  s.version           = '1.2.1'
+  s.version           = '1.2.2'
 
   s.summary     = "Compare two data sources that contain file-like objects"
   s.description =

data/lib/filey-diff/data-sources/aws_sdk_s3.rb

@@ -17,17 +17,22 @@ module Filey
       end
 
       def in_parallel_or_sequentially(map_to_filey)
-        if ENV['disable_parallel_processing']
-          @s3_bucket.objects.each do |s3_object|
+        jobs = @s3_bucket.objects.map { |s3_object|
+          lambda {
             map_to_filey.call s3_object
-          end
+          }
+        }
+        if ENV['disable_parallel_processing']
+          jobs.each(&:call)
         else
-          threads = @s3_bucket.objects.map { |s3_object|
-            Thread.new(s3_object) {
-              map_to_filey.call s3_object
+          jobs.each_slice(100) { |jobs|
+            threads = jobs.map { |job|
+              Thread.new {
+                job.call
+              }
             }
+            threads.each(&:join)
           }
-          threads.each { |thread| thread.join }
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filey-diff
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-22 00:00:00.000000000 Z
+date: 2013-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake