filey-diff 1.2.3 → 1.2.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/changelog.md +4 -0
- data/filey-diff.gemspec +1 -1
- data/lib/filey-diff/data-sources/aws_sdk_s3.rb +5 -3
- data/spec/data_sources_spec.rb +40 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ed51d42ff41aad4b13ab034de0da688e6c76ae59
|
4
|
+
data.tar.gz: 990b5d2fefb0fc6d019e2de83b694ce2ff27ae24
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9f214eea6234bc96138f0508b17cc9966bb6ef605b9b23c2f846017ec9fac11b9bfd28c512f38da93bc7ca25257f692c11f0f38a94e7b235b0828323b429995e
|
7
|
+
data.tar.gz: e41c9a33c420db5c103a3711416fa01831389c1562e85123427ec588f0fb0e43b3a0859f283809bbd2c55f10dbcd35b9ad4400ed6bada2d3f9dbd2cfc2ef7931
|
data/changelog.md
CHANGED
data/filey-diff.gemspec
CHANGED
@@ -7,6 +7,8 @@ module Filey
|
|
7
7
|
|
8
8
|
private
|
9
9
|
|
10
|
+
DEFAULT_CONCURRENCY_LEVEL = 100
|
11
|
+
|
10
12
|
def do_internal_load
|
11
13
|
fileys = []
|
12
14
|
map_to_filey = lambda { |s3_object|
|
@@ -16,16 +18,16 @@ module Filey
|
|
16
18
|
fileys
|
17
19
|
end
|
18
20
|
|
19
|
-
def in_parallel_or_sequentially(
|
21
|
+
def in_parallel_or_sequentially(operation_on_s3_object)
|
20
22
|
jobs = @s3_bucket.objects.map { |s3_object|
|
21
23
|
lambda {
|
22
|
-
|
24
|
+
operation_on_s3_object.call s3_object
|
23
25
|
}
|
24
26
|
}
|
25
27
|
if ENV['disable_parallel_processing']
|
26
28
|
jobs.each(&:call)
|
27
29
|
else
|
28
|
-
jobs.each_slice(
|
30
|
+
jobs.each_slice(DEFAULT_CONCURRENCY_LEVEL) { |jobs|
|
29
31
|
threads = jobs.map { |job|
|
30
32
|
Thread.new {
|
31
33
|
job.call
|
data/spec/data_sources_spec.rb
CHANGED
@@ -61,6 +61,46 @@ describe Filey::DataSources::AwsSdkS3 do
|
|
61
61
|
|
62
62
|
it_should_behave_like "a data source", s3_bucket
|
63
63
|
|
64
|
+
context 'when parallelism is disabled' do
|
65
|
+
before(:each) {
|
66
|
+
@disable_parallel_processing_before = ENV['disable_parallel_processing']
|
67
|
+
ENV['disable_parallel_processing'] = 'true'
|
68
|
+
}
|
69
|
+
|
70
|
+
after(:each) {
|
71
|
+
unless @disable_parallel_processing_before == nil
|
72
|
+
ENV['disable_parallel_processing'] = @disable_parallel_processing_before
|
73
|
+
end
|
74
|
+
}
|
75
|
+
|
76
|
+
it_should_behave_like "a data source", s3_bucket
|
77
|
+
end
|
78
|
+
|
79
|
+
context '#in_parallel_or_sequentially' do
|
80
|
+
describe Filey::DataSources::AwsSdkS3::DEFAULT_CONCURRENCY_LEVEL do
|
81
|
+
concurrency_level = Filey::DataSources::AwsSdkS3::DEFAULT_CONCURRENCY_LEVEL
|
82
|
+
|
83
|
+
before(:each) {
|
84
|
+
@ints = []
|
85
|
+
operation_on_s3_object = lambda do |s3_object|
|
86
|
+
@ints << s3_object
|
87
|
+
end
|
88
|
+
s3_bucket = double('s3_bucket', :objects => (1..(concurrency_level * 2)).map do |int| int end)
|
89
|
+
data_source = Filey::DataSources::AwsSdkS3.new(s3_bucket)
|
90
|
+
data_source.send(:in_parallel_or_sequentially, operation_on_s3_object)
|
91
|
+
}
|
92
|
+
|
93
|
+
it "does at most #{concurrency_level} ops in parallel" do
|
94
|
+
@ints.take(concurrency_level).all? { |int|
|
95
|
+
int <= concurrency_level
|
96
|
+
}.should be true
|
97
|
+
@ints.drop(concurrency_level).take(concurrency_level).all? { |int|
|
98
|
+
int > concurrency_level
|
99
|
+
}.should be true
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
64
104
|
context 'gzip' do
|
65
105
|
let(:gzip_tempfile_and_path) {
|
66
106
|
original_object = objects.first
|