slow_enumerator_tools 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/NEWS.md +12 -1
- data/README.md +28 -0
- data/lib/slow_enumerator_tools.rb +8 -0
- data/lib/slow_enumerator_tools/batcher.rb +12 -13
- data/lib/slow_enumerator_tools/bufferer.rb +28 -0
- data/lib/slow_enumerator_tools/merger.rb +11 -13
- data/lib/slow_enumerator_tools/util.rb +20 -0
- data/lib/slow_enumerator_tools/version.rb +1 -1
- data/spec/batcher_spec.rb +83 -0
- data/spec/bufferer_spec.rb +101 -0
- data/spec/merger_spec.rb +54 -0
- data/spec/slow_enumerator_tools_spec.rb +5 -0
- data/spec/spec_helper.rb +21 -0
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 7dde65640ab4d6809c1a3c77466e156f78cbf416e4eeb2349810a8ae4a0849a3
|
4
|
+
data.tar.gz: 0f11581710842aa47b6f28a79ccc5e4c9445cbea3daa2e02b78ee9943549751c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 324bf9924f0fe0900ee38708aa62186c75badaef9f5dbbb8aee2de85116a336cc17290ca489868cb78747c51dcc76ef102156fd98fbf6cff60975f282377bfc1
|
7
|
+
data.tar.gz: 7b1193a0691ae9b6fde6637f77efeda5c9bd46acd6e420e968a102086110d38cbf370010ae180d1c64b304a0b7e67b6e0e1358c6137b3ca42f53fa87a4a7dcf6
|
data/NEWS.md
CHANGED
@@ -1,10 +1,21 @@
|
|
1
1
|
# SlowEnumeratorTools news
|
2
2
|
|
3
|
+
## 1.1.0 (2017-11-08)
|
4
|
+
|
5
|
+
Fixes:
|
6
|
+
|
7
|
+
* Handled errors in `.merge` (#3)
|
8
|
+
* Handled errors in `.batch` (#2)
|
9
|
+
|
10
|
+
Features:
|
11
|
+
|
12
|
+
* Added `.buffer` (#1)
|
13
|
+
|
3
14
|
## 1.0.0 (2017-10-21)
|
4
15
|
|
5
16
|
Changes:
|
6
17
|
|
7
|
-
* Renamed buffer to batch
|
18
|
+
* Renamed `.buffer` to `.batch`
|
8
19
|
|
9
20
|
## 1.0.0a1 (2017-10-08)
|
10
21
|
|
data/README.md
CHANGED
@@ -12,6 +12,8 @@ _SlowEnumeratorTools_ provides tools for transforming Ruby enumerators that prod
|
|
12
12
|
|
13
13
|
* `SlowEnumeratorTools.batch`: given an enumerable, creates a new enumerable that yields batches containing all elements currently available.
|
14
14
|
|
15
|
+
* `SlowEnumeratorTools.buffer`: given an enumerable and a number, will create a buffer of that number of elements and try to fill it up with as many elements from that enumerable, so that they can be yielded immediately.
|
16
|
+
|
15
17
|
## Installation
|
16
18
|
|
17
19
|
Add this line to your application's Gemfile:
|
@@ -95,6 +97,32 @@ p batch_enum.next
|
|
95
97
|
p batch_enum.next
|
96
98
|
```
|
97
99
|
|
100
|
+
### `SlowEnumeratorTools.buffer`
|
101
|
+
|
102
|
+
Given an enumerable and a number, will create a buffer of that number of elements and try to fill it up with as many elements from that enumerable.
|
103
|
+
|
104
|
+
This is particularly useful when reading from a slow source and writing to a slow sink, because the two will be able to work concurrently.
|
105
|
+
|
106
|
+
```ruby
|
107
|
+
# Create (fake) articles enumerator
|
108
|
+
articles =
|
109
|
+
Enumerator.new do |y|
|
110
|
+
5.times do |i|
|
111
|
+
sleep 1
|
112
|
+
y << "Article #{i}"
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
# Buffer
|
117
|
+
articles = SlowEnumeratorTools.buffer(articles, 5)
|
118
|
+
|
119
|
+
# Print each article
|
120
|
+
# This takes 6 seconds, rather than 10!
|
121
|
+
articles.each do |a|
|
122
|
+
sleep 1
|
123
|
+
end
|
124
|
+
```
|
125
|
+
|
98
126
|
## Development
|
99
127
|
|
100
128
|
Install dependencies:
|
@@ -8,8 +8,16 @@ module SlowEnumeratorTools
|
|
8
8
|
def self.batch(es)
|
9
9
|
SlowEnumeratorTools::Batcher.batch(es)
|
10
10
|
end
|
11
|
+
|
12
|
+
def self.buffer(es, size)
|
13
|
+
SlowEnumeratorTools::Bufferer.buffer(es, size)
|
14
|
+
end
|
11
15
|
end
|
12
16
|
|
13
17
|
require_relative 'slow_enumerator_tools/version'
|
18
|
+
|
19
|
+
require_relative 'slow_enumerator_tools/util'
|
20
|
+
|
14
21
|
require_relative 'slow_enumerator_tools/batcher'
|
22
|
+
require_relative 'slow_enumerator_tools/bufferer'
|
15
23
|
require_relative 'slow_enumerator_tools/merger'
|
@@ -3,16 +3,9 @@
|
|
3
3
|
module SlowEnumeratorTools
|
4
4
|
module Batcher
|
5
5
|
def self.batch(enum)
|
6
|
-
|
7
|
-
stop = Object.new
|
6
|
+
queue = Queue.new
|
8
7
|
|
9
|
-
t =
|
10
|
-
Thread.new do
|
11
|
-
enum.each do |e|
|
12
|
-
q << e
|
13
|
-
end
|
14
|
-
q << stop
|
15
|
-
end
|
8
|
+
t = SlowEnumeratorTools::Util.gen_collector_thread(enum, queue)
|
16
9
|
|
17
10
|
Enumerator.new do |y|
|
18
11
|
loop do
|
@@ -20,20 +13,26 @@ module SlowEnumeratorTools
|
|
20
13
|
ended = false
|
21
14
|
|
22
15
|
# pop first
|
23
|
-
elem =
|
24
|
-
|
16
|
+
elem = queue.pop
|
17
|
+
if SlowEnumeratorTools::Util::STOP_OK.equal?(elem)
|
18
|
+
break
|
19
|
+
elsif SlowEnumeratorTools::Util::STOP_ERR.equal?(elem)
|
20
|
+
raise queue.pop
|
21
|
+
end
|
25
22
|
res << elem
|
26
23
|
|
27
24
|
loop do
|
28
25
|
# pop remaining
|
29
26
|
begin
|
30
|
-
elem =
|
27
|
+
elem = queue.pop(true)
|
31
28
|
rescue ThreadError
|
32
29
|
break
|
33
30
|
end
|
34
|
-
if
|
31
|
+
if SlowEnumeratorTools::Util::STOP_OK.equal?(elem)
|
35
32
|
ended = true
|
36
33
|
break
|
34
|
+
elsif SlowEnumeratorTools::Util::STOP_ERR.equal?(elem)
|
35
|
+
raise queue.pop
|
37
36
|
end
|
38
37
|
res << elem
|
39
38
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SlowEnumeratorTools
|
4
|
+
module Bufferer
|
5
|
+
def self.buffer(enum, size)
|
6
|
+
queue = SizedQueue.new(size)
|
7
|
+
thread = SlowEnumeratorTools::Util.gen_collector_thread(enum, queue)
|
8
|
+
gen_enumerator(queue, thread)
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.gen_enumerator(queue, collector_thread)
|
12
|
+
Enumerator.new do |y|
|
13
|
+
loop do
|
14
|
+
elem = queue.pop
|
15
|
+
|
16
|
+
if SlowEnumeratorTools::Util::STOP_OK.equal?(elem)
|
17
|
+
break
|
18
|
+
elsif SlowEnumeratorTools::Util::STOP_ERR.equal?(elem)
|
19
|
+
raise queue.pop
|
20
|
+
end
|
21
|
+
|
22
|
+
y << elem
|
23
|
+
end
|
24
|
+
collector_thread.join
|
25
|
+
end.lazy
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -11,8 +11,6 @@ module SlowEnumeratorTools
|
|
11
11
|
end
|
12
12
|
|
13
13
|
class Iterator
|
14
|
-
DONE = Object.new
|
15
|
-
|
16
14
|
def initialize(enums)
|
17
15
|
@enums = enums
|
18
16
|
@q = SizedQueue.new(5)
|
@@ -23,9 +21,11 @@ module SlowEnumeratorTools
|
|
23
21
|
raise StopIteration if @done
|
24
22
|
|
25
23
|
nxt = @q.pop
|
26
|
-
if
|
24
|
+
if SlowEnumeratorTools::Util::STOP_OK.equal?(nxt)
|
27
25
|
@done = true
|
28
26
|
raise StopIteration
|
27
|
+
elsif SlowEnumeratorTools::Util::STOP_ERR.equal?(nxt)
|
28
|
+
raise @q.pop
|
29
29
|
else
|
30
30
|
nxt
|
31
31
|
end
|
@@ -34,24 +34,22 @@ module SlowEnumeratorTools
|
|
34
34
|
def start
|
35
35
|
threads = @enums.map { |enum| spawn_empty_into(enum, @q) }
|
36
36
|
|
37
|
-
|
37
|
+
Thread.new do
|
38
38
|
threads.each(&:join)
|
39
|
-
@q <<
|
39
|
+
@q << SlowEnumeratorTools::Util::STOP_OK
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
43
|
protected
|
44
44
|
|
45
45
|
def spawn_empty_into(enum, queue)
|
46
|
-
spawn do
|
47
|
-
enum.each { |e| queue << e }
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def spawn
|
52
46
|
Thread.new do
|
53
|
-
|
54
|
-
|
47
|
+
begin
|
48
|
+
enum.each { |e| queue << e }
|
49
|
+
rescue StandardError => e
|
50
|
+
queue << SlowEnumeratorTools::Util::STOP_ERR
|
51
|
+
queue << e
|
52
|
+
end
|
55
53
|
end
|
56
54
|
end
|
57
55
|
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SlowEnumeratorTools
|
4
|
+
module Util
|
5
|
+
STOP_OK = Object.new
|
6
|
+
STOP_ERR = Object.new
|
7
|
+
|
8
|
+
def self.gen_collector_thread(enum, queue)
|
9
|
+
Thread.new do
|
10
|
+
begin
|
11
|
+
enum.each { |e| queue << e }
|
12
|
+
queue << STOP_OK
|
13
|
+
rescue StandardError => e
|
14
|
+
queue << STOP_ERR
|
15
|
+
queue << e
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/spec/batcher_spec.rb
CHANGED
@@ -34,4 +34,87 @@ describe SlowEnumeratorTools::Batcher do
|
|
34
34
|
sleep 0.45
|
35
35
|
expect(subject.next).to eq([0, 1, 2])
|
36
36
|
end
|
37
|
+
|
38
|
+
context 'empty enumerable' do
|
39
|
+
let(:wrapped) do
|
40
|
+
Enumerator.new do |y|
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'returns nothing' do
|
45
|
+
expect { subject.next }.to raise_error(StopIteration)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
context 'instant-erroring enumerable' do
|
50
|
+
let(:wrapped) do
|
51
|
+
Enumerator.new do |_y|
|
52
|
+
raise 'boom'
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'returns nothing' do
|
57
|
+
expect { subject.next }.to raise_error(RuntimeError, 'boom')
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'error in taken elements' do
|
62
|
+
let(:wrapped) do
|
63
|
+
Enumerator.new do |y|
|
64
|
+
y << 1
|
65
|
+
y << 2
|
66
|
+
y << 3
|
67
|
+
raise 'boom'
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'does not raise right away' do
|
72
|
+
subject
|
73
|
+
sleep 0.01
|
74
|
+
subject
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'does not raise when only taken' do
|
78
|
+
subject
|
79
|
+
sleep 0.01
|
80
|
+
subject.take(1)
|
81
|
+
end
|
82
|
+
|
83
|
+
it 'raises when evaluated' do
|
84
|
+
subject
|
85
|
+
sleep 0.01
|
86
|
+
expect { subject.take(1).first }
|
87
|
+
.to raise_error(RuntimeError, 'boom')
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
context 'error past taken elements' do
|
92
|
+
let(:wrapped) do
|
93
|
+
Enumerator.new do |y|
|
94
|
+
y << 1
|
95
|
+
y << 2
|
96
|
+
y << 3
|
97
|
+
sleep 0.1
|
98
|
+
raise 'boom'
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
it 'does not raise right away' do
|
103
|
+
subject
|
104
|
+
sleep 0.01
|
105
|
+
subject
|
106
|
+
end
|
107
|
+
|
108
|
+
it 'does not raise when only taken' do
|
109
|
+
subject
|
110
|
+
sleep 0.01
|
111
|
+
subject.take(1)
|
112
|
+
end
|
113
|
+
|
114
|
+
it 'does not raise when evaluated' do
|
115
|
+
subject
|
116
|
+
sleep 0.01
|
117
|
+
expect(subject.take(1).first).to eq([1, 2, 3])
|
118
|
+
end
|
119
|
+
end
|
37
120
|
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
describe SlowEnumeratorTools::Bufferer do
|
4
|
+
describe '.new + #call' do
|
5
|
+
subject { described_class.buffer(enum, size) }
|
6
|
+
|
7
|
+
let(:enum) { [] }
|
8
|
+
let(:size) { 10 }
|
9
|
+
|
10
|
+
context 'empty array' do
|
11
|
+
example do
|
12
|
+
expect(subject.to_a).to eq([])
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'small array' do
|
17
|
+
let(:enum) { [1, 2, 3] }
|
18
|
+
|
19
|
+
example do
|
20
|
+
expect(subject.to_a).to eq([1, 2, 3])
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'infinite array' do
|
25
|
+
let(:enum) { (1..(1.0 / 0)) }
|
26
|
+
|
27
|
+
example do
|
28
|
+
expect(subject.take(3).to_a).to eq([1, 2, 3])
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
context 'error in taken elements' do
|
33
|
+
let(:enum) do
|
34
|
+
Enumerator.new do |y|
|
35
|
+
y << 1
|
36
|
+
y << 2
|
37
|
+
raise 'boom'
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'does not raise right away' do
|
42
|
+
subject
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'does not raise when only taken' do
|
46
|
+
subject.take(3)
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'raises when evaluated' do
|
50
|
+
expect { subject.take(3).to_a }
|
51
|
+
.to raise_error(RuntimeError, 'boom')
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'error past taken elements' do
|
56
|
+
let(:enum) do
|
57
|
+
Enumerator.new do |y|
|
58
|
+
y << 1
|
59
|
+
y << 2
|
60
|
+
y << 3
|
61
|
+
raise 'boom'
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'does not raise right away' do
|
66
|
+
subject
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'does not raise when only taken' do
|
70
|
+
subject.take(3)
|
71
|
+
end
|
72
|
+
|
73
|
+
it 'does not raise when evaluated' do
|
74
|
+
expect(subject.take(3).to_a).to eq([1, 2, 3])
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
context 'slow source' do
|
79
|
+
let(:enum) do
|
80
|
+
Enumerator.new do |y|
|
81
|
+
20.times do |i|
|
82
|
+
y << i
|
83
|
+
sleep 0.1
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'takes a while to take elements when unbuffered' do
|
89
|
+
expect { subject.take(5).to_a }
|
90
|
+
.to finish_in(0.5, 0.1)
|
91
|
+
end
|
92
|
+
|
93
|
+
it 'takes no time to take elements when buffered' do
|
94
|
+
subject
|
95
|
+
sleep 0.5
|
96
|
+
expect { subject.take(5).to_a }
|
97
|
+
.to finish_in(0.0, 0.1)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
data/spec/merger_spec.rb
CHANGED
@@ -67,4 +67,58 @@ describe SlowEnumeratorTools::Merger do
|
|
67
67
|
let(:enums) { [[1, 2], [3, 4]] }
|
68
68
|
it { is_expected.to match_array([1, 2, 3, 4]) }
|
69
69
|
end
|
70
|
+
|
71
|
+
context 'error in taken elements' do
|
72
|
+
subject { described_class.merge(enums) }
|
73
|
+
|
74
|
+
let(:enum) do
|
75
|
+
Enumerator.new do |y|
|
76
|
+
y << 1
|
77
|
+
y << 2
|
78
|
+
raise 'boom'
|
79
|
+
end.lazy
|
80
|
+
end
|
81
|
+
|
82
|
+
let(:enums) { [enum] }
|
83
|
+
|
84
|
+
it 'does not raise right away' do
|
85
|
+
subject
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'does not raise when only taken' do
|
89
|
+
subject.take(3)
|
90
|
+
end
|
91
|
+
|
92
|
+
it 'raises when evaluated' do
|
93
|
+
expect { subject.take(3).to_a }
|
94
|
+
.to raise_error(RuntimeError, 'boom')
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
context 'error past taken elements' do
|
99
|
+
subject { described_class.merge(enums) }
|
100
|
+
|
101
|
+
let(:enum) do
|
102
|
+
Enumerator.new do |y|
|
103
|
+
y << 1
|
104
|
+
y << 2
|
105
|
+
y << 3
|
106
|
+
raise 'boom'
|
107
|
+
end.lazy
|
108
|
+
end
|
109
|
+
|
110
|
+
let(:enums) { [enum] }
|
111
|
+
|
112
|
+
it 'does not raise right away' do
|
113
|
+
subject
|
114
|
+
end
|
115
|
+
|
116
|
+
it 'does not raise when only taken' do
|
117
|
+
subject.take(3)
|
118
|
+
end
|
119
|
+
|
120
|
+
it 'does not raise when evaluated' do
|
121
|
+
expect(subject.take(3).to_a).to eq([1, 2, 3])
|
122
|
+
end
|
123
|
+
end
|
70
124
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -15,3 +15,24 @@ RSpec.configure do |c|
|
|
15
15
|
format: '%c/%C |<%b>%i| %p%%',
|
16
16
|
}
|
17
17
|
end
|
18
|
+
|
19
|
+
RSpec::Matchers.define :finish_in do |expected, delta|
|
20
|
+
supports_block_expectations
|
21
|
+
|
22
|
+
match do |actual|
|
23
|
+
before = Time.now
|
24
|
+
actual.call
|
25
|
+
after = Time.now
|
26
|
+
@actual_duration = after - before
|
27
|
+
range = (expected - delta..expected + delta)
|
28
|
+
range.include?(@actual_duration)
|
29
|
+
end
|
30
|
+
|
31
|
+
failure_message do |_actual|
|
32
|
+
"expected that proc would finish in #{expected}s (±#{delta}s), but took #{format '%0.1fs', @actual_duration}"
|
33
|
+
end
|
34
|
+
|
35
|
+
failure_message_when_negated do |_actual|
|
36
|
+
"expected that proc would not finish in #{expected}s (±#{delta}s), but took #{format '%0.1fs', @actual_duration}"
|
37
|
+
end
|
38
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: slow_enumerator_tools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Denis Defreyne
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -43,11 +43,14 @@ files:
|
|
43
43
|
- Rakefile
|
44
44
|
- lib/slow_enumerator_tools.rb
|
45
45
|
- lib/slow_enumerator_tools/batcher.rb
|
46
|
+
- lib/slow_enumerator_tools/bufferer.rb
|
46
47
|
- lib/slow_enumerator_tools/merger.rb
|
48
|
+
- lib/slow_enumerator_tools/util.rb
|
47
49
|
- lib/slow_enumerator_tools/version.rb
|
48
50
|
- scripts/release
|
49
51
|
- slow_enumerator_tools.gemspec
|
50
52
|
- spec/batcher_spec.rb
|
53
|
+
- spec/bufferer_spec.rb
|
51
54
|
- spec/merger_spec.rb
|
52
55
|
- spec/slow_enumerator_tools_spec.rb
|
53
56
|
- spec/spec_helper.rb
|
@@ -71,7 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
71
74
|
version: '0'
|
72
75
|
requirements: []
|
73
76
|
rubyforge_project:
|
74
|
-
rubygems_version: 2.
|
77
|
+
rubygems_version: 2.7.1
|
75
78
|
signing_key:
|
76
79
|
specification_version: 4
|
77
80
|
summary: provides tools for transforming Ruby enumerators that produce data slowly
|