filey-diff 1.2.4 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed51d42ff41aad4b13ab034de0da688e6c76ae59
-  data.tar.gz: 990b5d2fefb0fc6d019e2de83b694ce2ff27ae24
+  metadata.gz: 1fe9b2c5268647baab2e70424019f87a308c45d4
+  data.tar.gz: 76c1429b9a18f19dae15856372fefa39edb7b45a
 SHA512:
-  metadata.gz: 9f214eea6234bc96138f0508b17cc9966bb6ef605b9b23c2f846017ec9fac11b9bfd28c512f38da93bc7ca25257f692c11f0f38a94e7b235b0828323b429995e
-  data.tar.gz: e41c9a33c420db5c103a3711416fa01831389c1562e85123427ec588f0fb0e43b3a0859f283809bbd2c55f10dbcd35b9ad4400ed6bada2d3f9dbd2cfc2ef7931
+  metadata.gz: ba377502534f9b43ba5d1abf50b1bb15f43dc975863856414e1dc3ea8642f01616869b0cd0578168165fba51fe93df54ca1f662aafe818418482505105709886
+  data.tar.gz: 88c98411ed8cafa1a598b2aa869831eb93d3c6bc20ed707ad3ec9a4d0056f83a5064f17ec3e8e962d871ca88a683cebe6333161c69ed254244d04beff839ab94

data/README.md

@@ -61,6 +61,18 @@ Filey::Comparison.list_outdated(fs_data_source, s3_data_source).each { |filey|
 }
 ```
 
+## AWS SDK data source
+
+### Specifing custom concurrency level
+
+The concurrency level determines the amount of parallel operations that the
+`AwsSdkS3` data source performs against the S3 API.
+
+```ruby
+config = { :concurrency_level => 1000 }
+s3_data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket, config)
+```
+
 ## Example use cases
 
 Arnie has a blog on AWS S3. He has just finished a new post and wants to upload

data/changelog.md

@@ -2,6 +2,10 @@
 
 This project is [Semantically Versioned](http://semver.org).
 
+## 1.3.0
+
+* Support custom concurrency level in AwsSdkS3
+
 ## 1.2.4
 
 * Prepare for future development by refactoring the concurrency implementation.

data/filey-diff.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name              = 'filey-diff'
-  s.version           = '1.2.4'
+  s.version           = '1.3.0'
 
   s.summary     = "Compare two data sources that contain file-like objects"
   s.description =

data/lib/filey-diff/data-sources/aws_sdk_s3.rb

@@ -1,8 +1,9 @@
 module Filey
   module DataSources
     class AwsSdkS3 < DataSource
-      def initialize(s3_bucket)
+      def initialize(s3_bucket, config = { :concurrency_level => DEFAULT_CONCURRENCY_LEVEL })
         @s3_bucket = s3_bucket
+        @config = config
       end
 
       private
@@ -27,7 +28,7 @@ module Filey
         if ENV['disable_parallel_processing']
           jobs.each(&:call)
         else
-          jobs.each_slice(DEFAULT_CONCURRENCY_LEVEL) { |jobs|
+          jobs.each_slice(slice_size) { |jobs|
             threads = jobs.map { |job|
               Thread.new {
                 job.call
@@ -38,6 +39,11 @@ module Filey
         end
       end
 
+      def slice_size
+        slice_size_from_cfg = @config[:concurrency_level] || @config['concurrency_level']
+        slice_size_from_cfg || DEFAULT_CONCURRENCY_LEVEL
+      end
+
       def map_s3_object_to_filey(s3_object)
         if (s3_object.key.include?'/')
           path = s3_object.key.scan(/(.*\/).*/).first.first

data/spec/data_sources_spec.rb

@@ -77,28 +77,55 @@ describe Filey::DataSources::AwsSdkS3 do
   end
 
   context '#in_parallel_or_sequentially' do
-    describe Filey::DataSources::AwsSdkS3::DEFAULT_CONCURRENCY_LEVEL do
-      concurrency_level = Filey::DataSources::AwsSdkS3::DEFAULT_CONCURRENCY_LEVEL
 
-      before(:each) {
+    shared_examples 'concurrent processing' do |concurrency_level, config|
+      let(:operation_on_s3_object) {
         @ints = []
         operation_on_s3_object = lambda do |s3_object|
           @ints << s3_object
         end
-        s3_bucket = double('s3_bucket', :objects => (1..(concurrency_level * 2)).map do |int| int end)
-        data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket)
+      }
+
+      before(:each) {
+        s3_bucket = double(
+          's3_bucket',
+          :objects => (0..200).map { |int| int }
+      )
+        data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket, config)
         data_source.send(:in_parallel_or_sequentially, operation_on_s3_object)
       }
 
-      it "does at most #{concurrency_level} ops in parallel" do
+      it "honors the concurrency level by processing the first #{concurrency_level} items first" do
         @ints.take(concurrency_level).all? { |int|
           int <= concurrency_level
         }.should be true
+      end
+
+      it "honors the concurrency level by processing the second patch of #{concurrency_level} items second" do
         @ints.drop(concurrency_level).take(concurrency_level).all? { |int|
-          int > concurrency_level
+          int >= concurrency_level && int < (2 * concurrency_level)
         }.should be true
       end
     end
+
+    describe 'default concurrency level' do
+      concurrency_level = Filey::DataSources::AwsSdkS3::DEFAULT_CONCURRENCY_LEVEL
+
+      include_examples(
+        'concurrent processing',
+        concurrency_level,
+        nil
+      )
+    end
+
+    describe 'specifying custom concurrency level' do
+      concurrency_level = 20
+      include_examples(
+        'concurrent processing',
+        concurrency_level,
+        { :concurrency_level => concurrency_level }
+      )
+    end
   end
 
   context 'gzip' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filey-diff
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.3.0
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-12 00:00:00.000000000 Z
+date: 2013-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake