mapahead 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a9c357cb99922fd9b4e140274be6664dfdfd7806
+  data.tar.gz: 42f33eef97ff5d6a9c9063f652e9c15ce438292f
+SHA512:
+  metadata.gz: 3e0fd5d3b4250755510f6a124d9ff1efd1c349a80fd32a1a26f81f106e0173431291ac239247d5308f57189506aaa586b70fc7509f3bcdefe654e725e09ad6bd
+  data.tar.gz: e54d17ec479ad5c8fafa00b212dc745173c8e55d4543ab0009804420c7165c73fd8fefcc0c92bc084b6391be726f1c19bd41ae9b51312fa57f4db1969cfdcaf6

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'pry'
+gem 'rspec'
+gem 'concurrent-ruby'

data/Gemfile.lock

@@ -0,0 +1,34 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    concurrent-ruby (1.0.5)
+    diff-lcs (1.3)
+    method_source (0.9.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  concurrent-ruby
+  pry
+  rspec
+
+BUNDLED WITH
+   1.14.6

data/README.md

@@ -0,0 +1,13 @@
+playing around with mapping a stream (enumerable)
+while working items ahead of current read point
+
+this example will end up calling sleep in parallel x2
+```
+enum = Enumerator.new { |y| 10.times { |i| y << i } }
+enum = MapAhead.stream_map(enum, 2) { sleep 1 }
+enum.each do |r|
+  puts r
+end
+```
+
+see specs

data/lib/mapahead.rb

@@ -0,0 +1,31 @@
+require 'thread'
+require 'concurrent'
+
+class MapAhead
+  def stream_map enumerable, work_ahead=1, &blk
+    return enumerable if blk.nil?
+    Enumerator.new do |y|
+      pool = Concurrent::FixedThreadPool.new(work_ahead)
+      results = Queue.new
+      work_ahead.times do
+        work = enumerable.next
+        pool.post do
+          results << blk.call(work)
+        end
+      end
+      while !pool.shutdown? || !results.empty?
+        begin
+          y << results.pop(true)
+          work = enumerable.next
+          pool.post do
+            results << blk.call(work)
+          end
+        rescue ThreadError
+          nil
+        rescue StopIteration
+          pool.shutdown
+        end
+      end
+    end
+  end
+end

data/mapahead.gemspec

@@ -0,0 +1,20 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = "mapahead"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Robby"]
+  spec.email         = ["robby.ranshous@coxautoinc.com"]
+  spec.summary       = "work ahead in the stream"
+  spec.description   = "work ahead in the stream"
+  spec.homepage      = ""
+  spec.license       = ""
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "concurrent-ruby"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+end

data/spec/mapahead_spec.rb

@@ -0,0 +1,112 @@
+require 'rspec'
+require_relative '../lib/mapahead'
+require 'pry'
+
+Thread.abort_on_exception = true
+
+describe MapAhead do
+
+  describe 'map against stream (:stream_map)' do
+
+    let(:enum_length) { rand(10..20) }
+    let(:source_enum) { get_enumerable(enum_length) }
+
+    it 'calls block for each element in enumerable' do
+      calls = 0
+      subject.stream_map(source_enum) { calls += 1 }.to_a
+      expect(calls).to eq enum_length
+    end
+
+    it 'returns enumerable with same count as source enum' do
+      expect(subject.stream_map(source_enum){}.count).to eq enum_length
+    end
+
+    it 'returns enum which contains result of calling block for each element in source enum' do
+      random_array = Array.new(enum_length) { rand(100) }
+      expected_array = random_array.map { |i| i + 1 }
+      counting_enum = Enumerator.new do |y|
+        while r = random_array.shift
+          y << r
+        end
+      end
+      r = subject.stream_map(counting_enum) { |i| i += 1 }
+      expect(r.to_a).to include(*expected_array)
+    end
+
+    context 'work ahead argument provided' do
+      it 'does work in parallel' do
+        working = false
+        collisions = []
+        subject.stream_map(source_enum, 2) do
+          collisions << 1 if working
+          working = true
+          sleep 0.1
+          working = false
+        end.to_a
+        expect(collisions.length).to be > 0
+      end
+
+      it 'has more than 1 block called in parallel' do
+        lock = Mutex.new
+        max_in_flight = 0
+        in_flight = 0
+        subject.stream_map(source_enum, 2) do
+          lock.synchronize {  in_flight += 1 }
+          sleep 0.1
+          lock.synchronize {
+            max_in_flight = [in_flight, max_in_flight].max
+            in_flight -= 1
+          }
+        end.to_a
+        expect(max_in_flight).to be > 1
+      end
+
+      it 'never has more blocks called in parallel than work ahead arg' do
+        max_in_flight = 0
+        in_flight = 0
+        subject.stream_map(source_enum, 2) do
+          in_flight += 1
+          sleep 0.1
+          max_in_flight = [in_flight, max_in_flight].max
+          in_flight -= 1
+        end
+        expect(max_in_flight).to be <= 2
+      end
+
+      it 'does not read ahead more than passed work ahead' do
+        count_read = 0
+        count_worked = 0
+        diff_max = 0
+        custom_enum = Enumerator.new do |y|
+          enum_length.times do
+            y << count_read += 1
+          end
+        end
+        subject.stream_map(custom_enum, 2) do |i|
+          sleep 0.1
+          count_worked += 1
+          diff_max = [diff_max, count_read - count_worked].max
+        end.to_a
+        expect(diff_max).to be <= 2
+      end
+    end
+
+    context 'no block is provided' do
+      it 'returns source enumerable' do
+        expect(subject.stream_map(source_enum)).to eq source_enum
+      end
+    end
+
+    context 'given an enumerable with zero elements' do
+      let(:enum_length) { 0 }
+      it 'returns an enumerable without elements' do
+        expect(subject.stream_map(source_enum).count).to eq 0
+      end
+    end
+  end
+
+end
+
+def get_enumerable(length=0)
+  Enumerator.new { |y| length.times { |i| y << i } }
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: mapahead
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Robby
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-11-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: work ahead in the stream
+email:
+- robby.ranshous@coxautoinc.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- lib/mapahead.rb
+- mapahead.gemspec
+- spec/mapahead_spec.rb
+homepage: ''
+licenses:
+- ''
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: work ahead in the stream
+test_files:
+- spec/mapahead_spec.rb