filey-diff 1.2.3 → 1.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fdba17c6c8afd37f9538cb948c05cb2d7c50c1a
-  data.tar.gz: 1a8f55161f7fb8df68efc11b19165b5b01976ac2
+  metadata.gz: ed51d42ff41aad4b13ab034de0da688e6c76ae59
+  data.tar.gz: 990b5d2fefb0fc6d019e2de83b694ce2ff27ae24
 SHA512:
-  metadata.gz: 2812fc594872cc2005683f4647e26773794a1a4f421fde220b3eba010697dc54559d22e96150593bafdc034456bbb9f370e5efb156f5c5200a23d55dfa006abc
-  data.tar.gz: c297b4d930fe9a5b4e52d626cbbe7a06fe4b171a0261da66c68713927c37e75dcb9aa37cdba065bf9b84231fa65fbeb30dc966b1f56e81865a949e1a01e3a041
+  metadata.gz: 9f214eea6234bc96138f0508b17cc9966bb6ef605b9b23c2f846017ec9fac11b9bfd28c512f38da93bc7ca25257f692c11f0f38a94e7b235b0828323b429995e
+  data.tar.gz: e41c9a33c420db5c103a3711416fa01831389c1562e85123427ec588f0fb0e43b3a0859f283809bbd2c55f10dbcd35b9ad4400ed6bada2d3f9dbd2cfc2ef7931

data/changelog.md

@@ -2,6 +2,10 @@
 
 This project is [Semantically Versioned](http://semver.org).
 
+## 1.2.4
+
+* Prepare for future development by refactoring the concurrency implementation.
+
 ## 1.2.3
 
 * Mention the license in the gemspec file

data/filey-diff.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name              = 'filey-diff'
-  s.version           = '1.2.3'
+  s.version           = '1.2.4'
 
   s.summary     = "Compare two data sources that contain file-like objects"
   s.description =

data/lib/filey-diff/data-sources/aws_sdk_s3.rb

@@ -7,6 +7,8 @@ module Filey
 
       private
 
+      DEFAULT_CONCURRENCY_LEVEL = 100
+
       def do_internal_load
         fileys = []
         map_to_filey = lambda { |s3_object|
@@ -16,16 +18,16 @@ module Filey
         fileys
       end
 
-      def in_parallel_or_sequentially(map_to_filey)
+      def in_parallel_or_sequentially(operation_on_s3_object)
         jobs = @s3_bucket.objects.map { |s3_object|
           lambda {
-            map_to_filey.call s3_object
+            operation_on_s3_object.call s3_object
           }
         }
         if ENV['disable_parallel_processing']
           jobs.each(&:call)
         else
-          jobs.each_slice(100) { |jobs|
+          jobs.each_slice(DEFAULT_CONCURRENCY_LEVEL) { |jobs|
             threads = jobs.map { |job|
               Thread.new {
                 job.call

data/spec/data_sources_spec.rb

@@ -61,6 +61,46 @@ describe Filey::DataSources::AwsSdkS3 do
 
   it_should_behave_like "a data source", s3_bucket
 
+  context 'when parallelism is disabled' do
+    before(:each) {
+      @disable_parallel_processing_before = ENV['disable_parallel_processing']
+      ENV['disable_parallel_processing'] = 'true'
+    }
+
+    after(:each) {
+      unless @disable_parallel_processing_before == nil
+        ENV['disable_parallel_processing'] = @disable_parallel_processing_before
+      end
+    }
+
+    it_should_behave_like "a data source", s3_bucket
+  end
+
+  context '#in_parallel_or_sequentially' do
+    describe Filey::DataSources::AwsSdkS3::DEFAULT_CONCURRENCY_LEVEL do
+      concurrency_level = Filey::DataSources::AwsSdkS3::DEFAULT_CONCURRENCY_LEVEL
+
+      before(:each) {
+        @ints = []
+        operation_on_s3_object = lambda do |s3_object|
+          @ints << s3_object
+        end
+        s3_bucket = double('s3_bucket', :objects => (1..(concurrency_level * 2)).map do |int| int end)
+        data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket)
+        data_source.send(:in_parallel_or_sequentially, operation_on_s3_object)
+      }
+
+      it "does at most #{concurrency_level} ops in parallel" do
+        @ints.take(concurrency_level).all? { |int|
+          int <= concurrency_level
+        }.should be true
+        @ints.drop(concurrency_level).take(concurrency_level).all? { |int|
+          int > concurrency_level
+        }.should be true
+      end
+    end
+  end
+
   context 'gzip' do
     let(:gzip_tempfile_and_path) {
       original_object = objects.first

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filey-diff
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.4
 platform: ruby
 authors:
 - Lauri Lehmijoki