slow_enumerator_tools 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5cecaee8660e212274a61072f434341d20318afd
-  data.tar.gz: 953e06c5e5beb1bca606f7a6c95ab399d7967a46
+SHA256:
+  metadata.gz: 7dde65640ab4d6809c1a3c77466e156f78cbf416e4eeb2349810a8ae4a0849a3
+  data.tar.gz: 0f11581710842aa47b6f28a79ccc5e4c9445cbea3daa2e02b78ee9943549751c
 SHA512:
-  metadata.gz: e41bd66e2621550a7132d7b8e4f092a713cc6cab2cae3d29e4cd9b0f230d03bd914e5396bc5cfd070b446c06a37dafaa29dab322175234ee0221f68769171542
-  data.tar.gz: ed7b14c899dba6e8c9ccc5b1142017f69b37bc5271867299ea06822a453fd0c4b111ca8141ae7b6db5c1844d87eb91a472b84f9c080e970e85a4f068fbb8ee55
+  metadata.gz: 324bf9924f0fe0900ee38708aa62186c75badaef9f5dbbb8aee2de85116a336cc17290ca489868cb78747c51dcc76ef102156fd98fbf6cff60975f282377bfc1
+  data.tar.gz: 7b1193a0691ae9b6fde6637f77efeda5c9bd46acd6e420e968a102086110d38cbf370010ae180d1c64b304a0b7e67b6e0e1358c6137b3ca42f53fa87a4a7dcf6

data/NEWS.md

@@ -1,10 +1,21 @@
 # SlowEnumeratorTools news
 
+## 1.1.0 (2017-11-08)
+
+Fixes:
+
+* Handled errors in `.merge` (#3)
+* Handled errors in `.batch` (#2)
+
+Features:
+
+* Added `.buffer` (#1)
+
 ## 1.0.0 (2017-10-21)
 
 Changes:
 
-* Renamed buffer to batch
+* Renamed `.buffer` to `.batch`
 
 ## 1.0.0a1 (2017-10-08)
 

data/README.md

@@ -12,6 +12,8 @@ _SlowEnumeratorTools_ provides tools for transforming Ruby enumerators that prod
 
 * `SlowEnumeratorTools.batch`: given an enumerable, creates a new enumerable that yields batches containing all elements currently available.
 
+* `SlowEnumeratorTools.buffer`: given an enumerable and a number, will create a buffer of that number of elements and try to fill it up with as many elements from that enumerable, so that they can be yielded immediately.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -95,6 +97,32 @@ p batch_enum.next
 p batch_enum.next
 ```
 
+### `SlowEnumeratorTools.buffer`
+
+Given an enumerable and a number, will create a buffer of that number of elements and try to fill it up with as many elements from that enumerable.
+
+This is particularly useful when reading from a slow source and writing to a slow sink, because the two will be able to work concurrently.
+
+```ruby
+# Create (fake) articles enumerator
+articles =
+  Enumerator.new do |y|
+    5.times do |i|
+      sleep 1
+      y << "Article #{i}"
+    end
+  end
+
+# Buffer
+articles = SlowEnumeratorTools.buffer(articles, 5)
+
+# Print each article
+# This takes 6 seconds, rather than 10!
+articles.each do |a|
+  sleep 1
+end
+```
+
 ## Development
 
 Install dependencies:

data/lib/slow_enumerator_tools.rb

@@ -8,8 +8,16 @@ module SlowEnumeratorTools
   def self.batch(es)
     SlowEnumeratorTools::Batcher.batch(es)
   end
+
+  def self.buffer(es, size)
+    SlowEnumeratorTools::Bufferer.buffer(es, size)
+  end
 end
 
 require_relative 'slow_enumerator_tools/version'
+
+require_relative 'slow_enumerator_tools/util'
+
 require_relative 'slow_enumerator_tools/batcher'
+require_relative 'slow_enumerator_tools/bufferer'
 require_relative 'slow_enumerator_tools/merger'

data/lib/slow_enumerator_tools/batcher.rb

@@ -3,16 +3,9 @@
 module SlowEnumeratorTools
   module Batcher
     def self.batch(enum)
-      q = Queue.new
-      stop = Object.new
+      queue = Queue.new
 
-      t =
-        Thread.new do
-          enum.each do |e|
-            q << e
-          end
-          q << stop
-        end
+      t = SlowEnumeratorTools::Util.gen_collector_thread(enum, queue)
 
       Enumerator.new do |y|
         loop do
@@ -20,20 +13,26 @@ module SlowEnumeratorTools
           ended = false
 
           # pop first
-          elem = q.pop
-          break if stop.equal?(elem)
+          elem = queue.pop
+          if SlowEnumeratorTools::Util::STOP_OK.equal?(elem)
+            break
+          elsif SlowEnumeratorTools::Util::STOP_ERR.equal?(elem)
+            raise queue.pop
+          end
           res << elem
 
           loop do
             # pop remaining
             begin
-              elem = q.pop(true)
+              elem = queue.pop(true)
             rescue ThreadError
               break
             end
-            if stop.equal?(elem)
+            if SlowEnumeratorTools::Util::STOP_OK.equal?(elem)
               ended = true
               break
+            elsif SlowEnumeratorTools::Util::STOP_ERR.equal?(elem)
+              raise queue.pop
             end
             res << elem
           end

data/lib/slow_enumerator_tools/bufferer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SlowEnumeratorTools
+  module Bufferer
+    def self.buffer(enum, size)
+      queue = SizedQueue.new(size)
+      thread = SlowEnumeratorTools::Util.gen_collector_thread(enum, queue)
+      gen_enumerator(queue, thread)
+    end
+
+    def self.gen_enumerator(queue, collector_thread)
+      Enumerator.new do |y|
+        loop do
+          elem = queue.pop
+
+          if SlowEnumeratorTools::Util::STOP_OK.equal?(elem)
+            break
+          elsif SlowEnumeratorTools::Util::STOP_ERR.equal?(elem)
+            raise queue.pop
+          end
+
+          y << elem
+        end
+        collector_thread.join
+      end.lazy
+    end
+  end
+end

data/lib/slow_enumerator_tools/merger.rb

@@ -11,8 +11,6 @@ module SlowEnumeratorTools
     end
 
     class Iterator
-      DONE = Object.new
-
       def initialize(enums)
         @enums = enums
         @q = SizedQueue.new(5)
@@ -23,9 +21,11 @@ module SlowEnumeratorTools
         raise StopIteration if @done
 
         nxt = @q.pop
-        if DONE.equal?(nxt)
+        if SlowEnumeratorTools::Util::STOP_OK.equal?(nxt)
           @done = true
           raise StopIteration
+        elsif SlowEnumeratorTools::Util::STOP_ERR.equal?(nxt)
+          raise @q.pop
         else
           nxt
         end
@@ -34,24 +34,22 @@ module SlowEnumeratorTools
       def start
         threads = @enums.map { |enum| spawn_empty_into(enum, @q) }
 
-        spawn do
+        Thread.new do
           threads.each(&:join)
-          @q << DONE
+          @q << SlowEnumeratorTools::Util::STOP_OK
         end
       end
 
       protected
 
       def spawn_empty_into(enum, queue)
-        spawn do
-          enum.each { |e| queue << e }
-        end
-      end
-
-      def spawn
         Thread.new do
-          Thread.current.abort_on_exception = true
-          yield
+          begin
+            enum.each { |e| queue << e }
+          rescue StandardError => e
+            queue << SlowEnumeratorTools::Util::STOP_ERR
+            queue << e
+          end
         end
       end
     end

data/lib/slow_enumerator_tools/util.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SlowEnumeratorTools
+  module Util
+    STOP_OK = Object.new
+    STOP_ERR = Object.new
+
+    def self.gen_collector_thread(enum, queue)
+      Thread.new do
+        begin
+          enum.each { |e| queue << e }
+          queue << STOP_OK
+        rescue StandardError => e
+          queue << STOP_ERR
+          queue << e
+        end
+      end
+    end
+  end
+end

data/lib/slow_enumerator_tools/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SlowEnumeratorTools
-  VERSION = '1.0.0'
+  VERSION = '1.1.0'
 end

data/spec/batcher_spec.rb

@@ -34,4 +34,87 @@ describe SlowEnumeratorTools::Batcher do
     sleep 0.45
     expect(subject.next).to eq([0, 1, 2])
   end
+
+  context 'empty enumerable' do
+    let(:wrapped) do
+      Enumerator.new do |y|
+      end
+    end
+
+    it 'returns nothing' do
+      expect { subject.next }.to raise_error(StopIteration)
+    end
+  end
+
+  context 'instant-erroring enumerable' do
+    let(:wrapped) do
+      Enumerator.new do |_y|
+        raise 'boom'
+      end
+    end
+
+    it 'returns nothing' do
+      expect { subject.next }.to raise_error(RuntimeError, 'boom')
+    end
+  end
+
+  context 'error in taken elements' do
+    let(:wrapped) do
+      Enumerator.new do |y|
+        y << 1
+        y << 2
+        y << 3
+        raise 'boom'
+      end
+    end
+
+    it 'does not raise right away' do
+      subject
+      sleep 0.01
+      subject
+    end
+
+    it 'does not raise when only taken' do
+      subject
+      sleep 0.01
+      subject.take(1)
+    end
+
+    it 'raises when evaluated' do
+      subject
+      sleep 0.01
+      expect { subject.take(1).first }
+        .to raise_error(RuntimeError, 'boom')
+    end
+  end
+
+  context 'error past taken elements' do
+    let(:wrapped) do
+      Enumerator.new do |y|
+        y << 1
+        y << 2
+        y << 3
+        sleep 0.1
+        raise 'boom'
+      end
+    end
+
+    it 'does not raise right away' do
+      subject
+      sleep 0.01
+      subject
+    end
+
+    it 'does not raise when only taken' do
+      subject
+      sleep 0.01
+      subject.take(1)
+    end
+
+    it 'does not raise when evaluated' do
+      subject
+      sleep 0.01
+      expect(subject.take(1).first).to eq([1, 2, 3])
+    end
+  end
 end

data/spec/bufferer_spec.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+describe SlowEnumeratorTools::Bufferer do
+  describe '.new + #call' do
+    subject { described_class.buffer(enum, size) }
+
+    let(:enum) { [] }
+    let(:size) { 10 }
+
+    context 'empty array' do
+      example do
+        expect(subject.to_a).to eq([])
+      end
+    end
+
+    context 'small array' do
+      let(:enum) { [1, 2, 3] }
+
+      example do
+        expect(subject.to_a).to eq([1, 2, 3])
+      end
+    end
+
+    context 'infinite array' do
+      let(:enum) { (1..(1.0 / 0)) }
+
+      example do
+        expect(subject.take(3).to_a).to eq([1, 2, 3])
+      end
+    end
+
+    context 'error in taken elements' do
+      let(:enum) do
+        Enumerator.new do |y|
+          y << 1
+          y << 2
+          raise 'boom'
+        end
+      end
+
+      it 'does not raise right away' do
+        subject
+      end
+
+      it 'does not raise when only taken' do
+        subject.take(3)
+      end
+
+      it 'raises when evaluated' do
+        expect { subject.take(3).to_a }
+          .to raise_error(RuntimeError, 'boom')
+      end
+    end
+
+    context 'error past taken elements' do
+      let(:enum) do
+        Enumerator.new do |y|
+          y << 1
+          y << 2
+          y << 3
+          raise 'boom'
+        end
+      end
+
+      it 'does not raise right away' do
+        subject
+      end
+
+      it 'does not raise when only taken' do
+        subject.take(3)
+      end
+
+      it 'does not raise when evaluated' do
+        expect(subject.take(3).to_a).to eq([1, 2, 3])
+      end
+    end
+
+    context 'slow source' do
+      let(:enum) do
+        Enumerator.new do |y|
+          20.times do |i|
+            y << i
+            sleep 0.1
+          end
+        end
+      end
+
+      it 'takes a while to take elements when unbuffered' do
+        expect { subject.take(5).to_a }
+          .to finish_in(0.5, 0.1)
+      end
+
+      it 'takes no time to take elements when buffered' do
+        subject
+        sleep 0.5
+        expect { subject.take(5).to_a }
+          .to finish_in(0.0, 0.1)
+      end
+    end
+  end
+end

data/spec/merger_spec.rb

@@ -67,4 +67,58 @@ describe SlowEnumeratorTools::Merger do
     let(:enums) { [[1, 2], [3, 4]] }
     it { is_expected.to match_array([1, 2, 3, 4]) }
   end
+
+  context 'error in taken elements' do
+    subject { described_class.merge(enums) }
+
+    let(:enum) do
+      Enumerator.new do |y|
+        y << 1
+        y << 2
+        raise 'boom'
+      end.lazy
+    end
+
+    let(:enums) { [enum] }
+
+    it 'does not raise right away' do
+      subject
+    end
+
+    it 'does not raise when only taken' do
+      subject.take(3)
+    end
+
+    it 'raises when evaluated' do
+      expect { subject.take(3).to_a }
+        .to raise_error(RuntimeError, 'boom')
+    end
+  end
+
+  context 'error past taken elements' do
+    subject { described_class.merge(enums) }
+
+    let(:enum) do
+      Enumerator.new do |y|
+        y << 1
+        y << 2
+        y << 3
+        raise 'boom'
+      end.lazy
+    end
+
+    let(:enums) { [enum] }
+
+    it 'does not raise right away' do
+      subject
+    end
+
+    it 'does not raise when only taken' do
+      subject.take(3)
+    end
+
+    it 'does not raise when evaluated' do
+      expect(subject.take(3).to_a).to eq([1, 2, 3])
+    end
+  end
 end

data/spec/slow_enumerator_tools_spec.rb

@@ -14,4 +14,9 @@ describe SlowEnumeratorTools do
     expect(SlowEnumeratorTools.batch([1, 2]).to_a)
       .to match_array([[1, 2]])
   end
+
+  it 'supports .buffer shorthand' do
+    expect(SlowEnumeratorTools.buffer([1, 2], 5).to_a)
+      .to match_array([1, 2])
+  end
 end

data/spec/spec_helper.rb

@@ -15,3 +15,24 @@ RSpec.configure do |c|
     format: '%c/%C |<%b>%i| %p%%',
   }
 end
+
+RSpec::Matchers.define :finish_in do |expected, delta|
+  supports_block_expectations
+
+  match do |actual|
+    before = Time.now
+    actual.call
+    after = Time.now
+    @actual_duration = after - before
+    range = (expected - delta..expected + delta)
+    range.include?(@actual_duration)
+  end
+
+  failure_message do |_actual|
+    "expected that proc would finish in #{expected}s (±#{delta}s), but took #{format '%0.1fs', @actual_duration}"
+  end
+
+  failure_message_when_negated do |_actual|
+    "expected that proc would not finish in #{expected}s (±#{delta}s), but took #{format '%0.1fs', @actual_duration}"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: slow_enumerator_tools
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Denis Defreyne
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-21 00:00:00.000000000 Z
+date: 2017-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -43,11 +43,14 @@ files:
 - Rakefile
 - lib/slow_enumerator_tools.rb
 - lib/slow_enumerator_tools/batcher.rb
+- lib/slow_enumerator_tools/bufferer.rb
 - lib/slow_enumerator_tools/merger.rb
+- lib/slow_enumerator_tools/util.rb
 - lib/slow_enumerator_tools/version.rb
 - scripts/release
 - slow_enumerator_tools.gemspec
 - spec/batcher_spec.rb
+- spec/bufferer_spec.rb
 - spec/merger_spec.rb
 - spec/slow_enumerator_tools_spec.rb
 - spec/spec_helper.rb
@@ -71,7 +74,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.1
 signing_key: 
 specification_version: 4
 summary: provides tools for transforming Ruby enumerators that produce data slowly