mapahead 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +34 -0
- data/README.md +13 -0
- data/lib/mapahead.rb +31 -0
- data/mapahead.gemspec +20 -0
- data/spec/mapahead_spec.rb +112 -0
- metadata +93 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a9c357cb99922fd9b4e140274be6664dfdfd7806
|
4
|
+
data.tar.gz: 42f33eef97ff5d6a9c9063f652e9c15ce438292f
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3e0fd5d3b4250755510f6a124d9ff1efd1c349a80fd32a1a26f81f106e0173431291ac239247d5308f57189506aaa586b70fc7509f3bcdefe654e725e09ad6bd
|
7
|
+
data.tar.gz: e54d17ec479ad5c8fafa00b212dc745173c8e55d4543ab0009804420c7165c73fd8fefcc0c92bc084b6391be726f1c19bd41ae9b51312fa57f4db1969cfdcaf6
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
GEM
|
2
|
+
remote: https://rubygems.org/
|
3
|
+
specs:
|
4
|
+
coderay (1.1.2)
|
5
|
+
concurrent-ruby (1.0.5)
|
6
|
+
diff-lcs (1.3)
|
7
|
+
method_source (0.9.0)
|
8
|
+
pry (0.11.3)
|
9
|
+
coderay (~> 1.1.0)
|
10
|
+
method_source (~> 0.9.0)
|
11
|
+
rspec (3.7.0)
|
12
|
+
rspec-core (~> 3.7.0)
|
13
|
+
rspec-expectations (~> 3.7.0)
|
14
|
+
rspec-mocks (~> 3.7.0)
|
15
|
+
rspec-core (3.7.0)
|
16
|
+
rspec-support (~> 3.7.0)
|
17
|
+
rspec-expectations (3.7.0)
|
18
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
19
|
+
rspec-support (~> 3.7.0)
|
20
|
+
rspec-mocks (3.7.0)
|
21
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
22
|
+
rspec-support (~> 3.7.0)
|
23
|
+
rspec-support (3.7.0)
|
24
|
+
|
25
|
+
PLATFORMS
|
26
|
+
ruby
|
27
|
+
|
28
|
+
DEPENDENCIES
|
29
|
+
concurrent-ruby
|
30
|
+
pry
|
31
|
+
rspec
|
32
|
+
|
33
|
+
BUNDLED WITH
|
34
|
+
1.14.6
|
data/README.md
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
playing around with mapping a stream (enumerable)
|
2
|
+
while working items ahead of current read point
|
3
|
+
|
4
|
+
this example will end up calling sleep in parallel x2
|
5
|
+
```
|
6
|
+
enum = Enumerator.new { |y| 10.times { |i| y << i } }
|
7
|
+
enum = MapAhead.stream_map(enum, 2) { sleep 1 }
|
8
|
+
enum.each do |r|
|
9
|
+
puts r
|
10
|
+
end
|
11
|
+
```
|
12
|
+
|
13
|
+
see specs
|
data/lib/mapahead.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'concurrent'
|
3
|
+
|
4
|
+
class MapAhead
|
5
|
+
def stream_map enumerable, work_ahead=1, &blk
|
6
|
+
return enumerable if blk.nil?
|
7
|
+
Enumerator.new do |y|
|
8
|
+
pool = Concurrent::FixedThreadPool.new(work_ahead)
|
9
|
+
results = Queue.new
|
10
|
+
work_ahead.times do
|
11
|
+
work = enumerable.next
|
12
|
+
pool.post do
|
13
|
+
results << blk.call(work)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
while !pool.shutdown? || !results.empty?
|
17
|
+
begin
|
18
|
+
y << results.pop(true)
|
19
|
+
work = enumerable.next
|
20
|
+
pool.post do
|
21
|
+
results << blk.call(work)
|
22
|
+
end
|
23
|
+
rescue ThreadError
|
24
|
+
nil
|
25
|
+
rescue StopIteration
|
26
|
+
pool.shutdown
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/mapahead.gemspec
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
Gem::Specification.new do |spec|
|
4
|
+
spec.name = "mapahead"
|
5
|
+
spec.version = "0.0.1"
|
6
|
+
spec.authors = ["Robby"]
|
7
|
+
spec.email = ["robby.ranshous@coxautoinc.com"]
|
8
|
+
spec.summary = "work ahead in the stream"
|
9
|
+
spec.description = "work ahead in the stream"
|
10
|
+
spec.homepage = ""
|
11
|
+
spec.license = ""
|
12
|
+
|
13
|
+
spec.files = `git ls-files -z`.split("\x0")
|
14
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
15
|
+
spec.require_paths = ["lib"]
|
16
|
+
|
17
|
+
spec.add_dependency "concurrent-ruby"
|
18
|
+
spec.add_development_dependency "rspec"
|
19
|
+
spec.add_development_dependency "pry"
|
20
|
+
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require_relative '../lib/mapahead'
|
3
|
+
require 'pry'
|
4
|
+
|
5
|
+
Thread.abort_on_exception = true
|
6
|
+
|
7
|
+
describe MapAhead do
|
8
|
+
|
9
|
+
describe 'map against stream (:stream_map)' do
|
10
|
+
|
11
|
+
let(:enum_length) { rand(10..20) }
|
12
|
+
let(:source_enum) { get_enumerable(enum_length) }
|
13
|
+
|
14
|
+
it 'calls block for each element in enumerable' do
|
15
|
+
calls = 0
|
16
|
+
subject.stream_map(source_enum) { calls += 1 }.to_a
|
17
|
+
expect(calls).to eq enum_length
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'returns enumerable with same count as source enum' do
|
21
|
+
expect(subject.stream_map(source_enum){}.count).to eq enum_length
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'returns enum which contains result of calling block for each element in source enum' do
|
25
|
+
random_array = Array.new(enum_length) { rand(100) }
|
26
|
+
expected_array = random_array.map { |i| i + 1 }
|
27
|
+
counting_enum = Enumerator.new do |y|
|
28
|
+
while r = random_array.shift
|
29
|
+
y << r
|
30
|
+
end
|
31
|
+
end
|
32
|
+
r = subject.stream_map(counting_enum) { |i| i += 1 }
|
33
|
+
expect(r.to_a).to include(*expected_array)
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'work ahead argument provided' do
|
37
|
+
it 'does work in parallel' do
|
38
|
+
working = false
|
39
|
+
collisions = []
|
40
|
+
subject.stream_map(source_enum, 2) do
|
41
|
+
collisions << 1 if working
|
42
|
+
working = true
|
43
|
+
sleep 0.1
|
44
|
+
working = false
|
45
|
+
end.to_a
|
46
|
+
expect(collisions.length).to be > 0
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'has more than 1 block called in parallel' do
|
50
|
+
lock = Mutex.new
|
51
|
+
max_in_flight = 0
|
52
|
+
in_flight = 0
|
53
|
+
subject.stream_map(source_enum, 2) do
|
54
|
+
lock.synchronize { in_flight += 1 }
|
55
|
+
sleep 0.1
|
56
|
+
lock.synchronize {
|
57
|
+
max_in_flight = [in_flight, max_in_flight].max
|
58
|
+
in_flight -= 1
|
59
|
+
}
|
60
|
+
end.to_a
|
61
|
+
expect(max_in_flight).to be > 1
|
62
|
+
end
|
63
|
+
|
64
|
+
it 'never has more blocks called in parallel than work ahead arg' do
|
65
|
+
max_in_flight = 0
|
66
|
+
in_flight = 0
|
67
|
+
subject.stream_map(source_enum, 2) do
|
68
|
+
in_flight += 1
|
69
|
+
sleep 0.1
|
70
|
+
max_in_flight = [in_flight, max_in_flight].max
|
71
|
+
in_flight -= 1
|
72
|
+
end
|
73
|
+
expect(max_in_flight).to be <= 2
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'does not read ahead more than passed work ahead' do
|
77
|
+
count_read = 0
|
78
|
+
count_worked = 0
|
79
|
+
diff_max = 0
|
80
|
+
custom_enum = Enumerator.new do |y|
|
81
|
+
enum_length.times do
|
82
|
+
y << count_read += 1
|
83
|
+
end
|
84
|
+
end
|
85
|
+
subject.stream_map(custom_enum, 2) do |i|
|
86
|
+
sleep 0.1
|
87
|
+
count_worked += 1
|
88
|
+
diff_max = [diff_max, count_read - count_worked].max
|
89
|
+
end.to_a
|
90
|
+
expect(diff_max).to be <= 2
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
context 'no block is provided' do
|
95
|
+
it 'returns source enumerable' do
|
96
|
+
expect(subject.stream_map(source_enum)).to eq source_enum
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
context 'given an enumerable with zero elements' do
|
101
|
+
let(:enum_length) { 0 }
|
102
|
+
it 'returns an enumerable without elements' do
|
103
|
+
expect(subject.stream_map(source_enum).count).to eq 0
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
|
110
|
+
def get_enumerable(length=0)
|
111
|
+
Enumerator.new { |y| length.times { |i| y << i } }
|
112
|
+
end
|
metadata
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mapahead
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Robby
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-11-21 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: concurrent-ruby
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: pry
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: work ahead in the stream
|
56
|
+
email:
|
57
|
+
- robby.ranshous@coxautoinc.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- Gemfile
|
63
|
+
- Gemfile.lock
|
64
|
+
- README.md
|
65
|
+
- lib/mapahead.rb
|
66
|
+
- mapahead.gemspec
|
67
|
+
- spec/mapahead_spec.rb
|
68
|
+
homepage: ''
|
69
|
+
licenses:
|
70
|
+
- ''
|
71
|
+
metadata: {}
|
72
|
+
post_install_message:
|
73
|
+
rdoc_options: []
|
74
|
+
require_paths:
|
75
|
+
- lib
|
76
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '0'
|
81
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
requirements: []
|
87
|
+
rubyforge_project:
|
88
|
+
rubygems_version: 2.5.2
|
89
|
+
signing_key:
|
90
|
+
specification_version: 4
|
91
|
+
summary: work ahead in the stream
|
92
|
+
test_files:
|
93
|
+
- spec/mapahead_spec.rb
|