enumerating 1.1.1 → 1.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MGI2YTQ2Y2Y0ZmFjOTljNmEzMjU2ZDc4YTFkNWE3ZjY0MWY4Mjg1Ng==
+  data.tar.gz: !binary |-
+    ODUwMjlhZmYzN2M1ZTgxNTkwNmI4N2E4YTg5YzI5ZmY2NTJiNTgyZQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MWYzMjdhYTNlYjE3ZjRkODI0NDQzMmQ0YTA3NDIwNjFjMDViMWYwNjZhMmVk
+    OTBiZWEwMTJjODY2NjFiYzhlYzMwMGI4ODUwNTU3MWViNjYyYzNhNzIwNTdj
+    ZWEwNGRkMWZjY2ZkYjM1YzUwZjJhNzc2YzZmNDcxNDBjYTg0MTE=
+  data.tar.gz: !binary |-
+    M2U3ZWJjYjY0NjlkOGQyOTg2MjVhOTYzMTk1NjgzYjg2OGFkM2E5MzBiMGYz
+    ZjU2ZmZmNTJjMjIwYmRiNGNjYTk4Yzg3YTBkNjVmMTQzYjNhMjUxMzRlNDE1
+    OGQyNjUxMTg2NjE1ZGEyODJhZDczNDA1Nzc4OTEyY2ZmNjBiOGE=

data/.rspec

@@ -0,0 +1 @@
+--color

data/CHANGES.md

@@ -0,0 +1,10 @@
+# Changelog
+
+## 1.2.0 (2013-04-16)
+
+* Add `#threading`.
+* Add `#prefetching`.
+
+## Previously
+
+* Stuff happened

data/Gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source "https://rubygems.org"
 
 gemspec
 

data/{README.markdown → README.md}

@@ -13,6 +13,7 @@ Enumerating extends Enumerable with "lazy" versions of various common operations
 * `#mapping` is an alias for `#collecting` (like `Enumerable#map`)
 * `#uniqing` discards duplicates (like `Enumerable#uniq`)
 * `#taking`, `#taking_while`, `#dropping` and `#dropping_while` also do what you expect
+* '#[]' gets the nth element of a collection
 
 We say the "...ing" variants are "lazy", because they defer per-element processing until the result is used.  They return Enumerable "result proxy" objects, rather than Arrays, and only perform the actual filtering (or transformation) as the result proxy is enumerated.
 
@@ -60,7 +61,22 @@ By combining two or more of the lazy operations provided by Enumerating, you can
     # resolve
     companies.to_a              #=> ["Disney"]
 
-Because each processing step proceeds in parallel, without creation of intermediate collections (Arrays), you can efficiently operate on large (or even infinite) Enumerable collections.
+Because the steps in the pipeline operate in parallel, without creation of intermediate collections (Arrays), you can efficiently operate on large (or even infinite) Enumerable collections.
+
+Multi-threaded processing
+-------------------------
+
+`Enumerable#threading` is a multi-threaded version of `Enumerable#collecting`, allowing multiple Ruby Thread's to be used to process elements of an collection.  It requires a numeric argument specifying the maximum number of Threads to use. 
+
+    start = Time.now
+    [5,6,7,8].threading(4) do |delay|
+      sleep(delay)
+    end.to_a
+    (Time.now - start).to_i     #=> 8
+
+Outputs will be yielded in the expected order, making it a drop-in replacement for `#collecting`.  
+
+Unlike some other "parallel map" implementations, the output of `#threading` is lazy, though it does need to pre-fetch elements from the source collection as required to start Threads.
 
 Lazy combination of Enumerables
 -------------------------------
@@ -78,7 +94,7 @@ Enumerating also provides some interesting ways to combine several Enumerable co
     array1 = [1,3,6]
     array2 = [2,4,7]
     Enumerating.concatenating(array1, array2)
-                                          # generates: [1,3,6,2,4,7]
+                                           # generates: [1,3,6,2,4,7]
 
 `Enumerating.merging` merges multiple collections, preserving sort-order.  The inputs are assumed to be sorted already.
 

data/benchmarks/pipeline_bench.rb

@@ -2,9 +2,7 @@ require 'benchmark'
 
 $: << File.expand_path("../../lib", __FILE__)
 
-RUBY19 = RUBY_VERSION =~ /^1\.9/
-
-if RUBY19
+if defined?(Fiber)
   require "lazing"
   module Enumerable
     alias :lazing_select :selecting
@@ -52,20 +50,26 @@ def benchmark(description, control_result = nil)
   result
 end
 
-@control = benchmark "conventional" do
+@control = benchmark "conventional (eager)" do
   array.select { |x| x.even? }.collect { |x| x*x }
 end
 
-benchmark "enumerating", @control do 
+benchmark "enumerating", @control do
   array.selecting { |x| x.even? }.collecting { |x| x*x }
 end
 
-if RUBY19
+if defined?(Fiber)
   benchmark "lazing", @control do
     array.lazing_select { |x| x.even? }.lazing_collect { |x| x*x }
   end
 end
 
+if array.respond_to?(:lazy)
+  benchmark "ruby2 Enumerable#lazy", @control do
+    array.lazy.select { |x| x.even? }.lazy.collect { |x| x*x }
+  end
+end
+
 benchmark "facets Enumerable#defer", @control do
   array.defer.select { |x| x.even? }.collect { |x| x*x }
 end

data/lib/enumerating.rb

@@ -1,2 +1,4 @@
 require 'enumerating/filtering'
 require 'enumerating/mixing'
+require 'enumerating/prefetching'
+require 'enumerating/threading'

data/lib/enumerating/filtering.rb

@@ -107,5 +107,8 @@ module Enumerable
     end
   end
 
-end
+  def [](n)
+    dropping(n).first
+  end
 
+end

data/lib/enumerating/prefetching.rb

@@ -0,0 +1,41 @@
+module Enumerating
+
+  class Prefetcher
+
+    include Enumerable
+
+    def initialize(source, buffer_size)
+      @source = source.to_enum
+      @buffer_size = buffer_size
+    end
+
+    def each(&block)
+      return @source.each(&block) if @buffer_size <= 0
+      buffered_elements = []
+      i = 0
+      @source.each do |element|
+        slot = i % @buffer_size
+        if i >= @buffer_size
+          yield buffered_elements[slot]
+        end
+        buffered_elements[slot] = element
+        i += 1
+      end
+      buffered_elements.size.times do
+        slot = i % buffered_elements.size
+        yield buffered_elements[slot]
+        i += 1
+      end
+    end
+
+  end
+
+end
+
+module Enumerable
+
+  def prefetching(size)
+    Enumerating::Prefetcher.new(self, size)
+  end
+
+end

data/lib/enumerating/threading.rb

@@ -0,0 +1,51 @@
+require 'enumerating/prefetching'
+
+module Enumerating
+
+  class ThreadStarter
+
+    include Enumerable
+
+    def initialize(source, block)
+      @source = source
+      @block = block
+    end
+
+    def each
+      @source.each do |source_item|
+        thread = Thread.new do
+          @block.call(source_item)
+        end
+        yield thread
+      end
+    end
+
+  end
+
+  class ThreadJoiner
+
+    include Enumerable
+
+    def initialize(threads)
+      @threads = threads
+    end
+
+    def each
+      @threads.each do |thread|
+        thread.join
+        yield thread.value
+      end
+    end
+
+  end
+
+end
+
+module Enumerable
+
+  def threading(max_threads, &block)
+    threads = Enumerating::ThreadStarter.new(self, block)
+    Enumerating::ThreadJoiner.new(threads.prefetching(max_threads - 1))
+  end
+
+end

data/lib/enumerating/version.rb

@@ -1,3 +1,3 @@
 module Enumerating
-  VERSION = "1.1.1".freeze
+  VERSION = "1.2.0".freeze
 end

data/spec/enumerating/filtering_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 module Enumerable
-  
+
   unless method_defined?(:first)
     def first
       each do |first_item|
@@ -9,7 +9,7 @@ module Enumerable
       end
     end
   end
-  
+
 end
 
 describe Enumerable do
@@ -70,61 +70,77 @@ describe Enumerable do
     end
 
   end
-  
+
   describe "#taking" do
-    
+
     it "includes the specified number" do
       @array = [1,2,3,4]
       @array.taking(3).to_a.should == [1,2,3]
     end
-    
+
     it "is lazy" do
       [1,2].with_time_bomb.taking(2).to_a.should == [1,2]
     end
-    
+
     it "copes with 0" do
       [].with_time_bomb.taking(0).to_a.should == []
     end
-    
+
   end
-  
+
   describe "#taking_while" do
-    
+
     it "takes elements as long as the predicate is true" do
       @array = [2,4,6,3]
       @array.taking_while(&:even?).to_a.should == [2,4,6]
     end
-    
+
     it "is lazy" do
       [2,3].with_time_bomb.taking_while(&:even?).to_a.should == [2]
     end
-    
+
   end
-  
+
   describe "#dropping" do
-    
+
     it "excludes the specified number" do
       @array = [1,2,3,4]
       @array.dropping(2).to_a.should == [3,4]
     end
-    
+
     it "is lazy" do
       [1,2,3,4].with_time_bomb.dropping(2).taking(1).to_a.should == [3]
     end
-    
+
   end
-  
+
   describe "#dropping_while" do
-    
+
     it "drops elements as long as the predicate is true" do
       @array = [2,4,6,3,4]
       @array.dropping_while(&:even?).to_a.should == [3,4]
     end
-    
+
     it "is lazy" do
       [2,3].with_time_bomb.dropping_while(&:even?).taking(1).to_a.should == [3]
     end
-    
+
+  end
+
+  describe "#[]" do
+
+    before do
+      @evens = [1,2,3,4,5].collecting { |x| x * 2 }
+    end
+
+    it "finds the specified element" do
+      @evens[2].should == 6
+    end
+
+    it "is lazy" do
+      @evens.with_time_bomb[4].should == 10
+    end
+
   end
 
 end

data/spec/enumerating/prefetching_spec.rb

@@ -0,0 +1,69 @@
+require "spec_helper"
+
+describe Enumerable do
+
+  Counter = Class.new do
+
+    include Enumerable
+
+    def initialize(source)
+      @source = source
+      @number_yielded = 0
+    end
+
+    def each
+      @source.each do |item|
+        @number_yielded += 1
+        yield item
+      end
+    end
+
+    attr_reader :number_yielded
+
+  end
+
+  describe "#prefetching" do
+
+    let(:source) { [1, 2, 3, 4, nil, false, 7] }
+
+    it "yields all items" do
+      source.prefetching(2).to_a.should eq(source)
+      source.prefetching(3).to_a.should eq(source)
+    end
+
+    it "is stateless" do
+      source.prefetching(2).first.should eq(source.first)
+      source.prefetching(2).first.should eq(source.first)
+    end
+
+    it "is lazy" do
+      source.with_time_bomb.prefetching(2).first.should eq(source.first)
+    end
+
+    it "pre-computes the specified number of elements" do
+      counter = Counter.new(source)
+      counter.prefetching(2).take(1)
+      counter.number_yielded.should eq(3)
+    end
+
+    context "with a buffer size of zero" do
+
+      it "does not pre-fetch anything" do
+        counter = Counter.new(source)
+        counter.prefetching(0).take(1)
+        counter.number_yielded.should eq(1)
+      end
+
+    end
+
+    context "with a buffer bigger than the source Enumerable" do
+
+      it "yields all items" do
+        source.prefetching(20).to_a.should eq(source)
+      end
+
+    end
+
+  end
+
+end

data/spec/enumerating/threading_spec.rb

@@ -0,0 +1,46 @@
+require "spec_helper"
+
+describe Enumerable do
+
+  describe "#threading" do
+
+    it "acts like #collect" do
+      [1,2,3].threading(5) { |x| x * 2 }.to_a.should == [2,4,6]
+    end
+
+    it "runs things in separate threads" do
+      [1,2,3].threading(5) { Thread.current.object_id }.to_a.uniq.size.should eq(3)
+    end
+
+    it "is lazy" do
+      [1,2,3].with_time_bomb.threading(2) { |x| x * 2 }.first.should == 2
+    end
+
+    it "runs the specified number of threads in parallel" do
+      delays = [0.01, 0.01, 0.01]
+      start = Time.now
+      delays.threading(2) do |delay|
+        sleep(delay)
+      end.to_a
+      (Time.now - start).round(2).should eq(0.01 * 2)
+    end
+
+    it "acts as a sliding window" do
+      delays = [0.05, 0.04, 0.03, 0.02, 0.01]
+      start = Time.now
+      elapsed_times = delays.threading(3) do |delay|
+        sleep(delay)
+        (Time.now - start).round(2)
+      end
+      elapsed_times.to_a.should eq([0.05, 0.04, 0.03, 0.07, 0.06])
+    end
+
+    it "surfaces exceptions" do
+      lambda do
+        [1,2,3].threading(5) { raise "hell" }.to_a
+      end.should raise_error(RuntimeError, "hell")
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -26,12 +26,12 @@ class WithTimeBomb
 end
 
 module Enumerable
-  
+
   # extend an Enumerable to throw an exception after last element
   def with_time_bomb
     WithTimeBomb.new(self)
   end
-  
+
 end
 
 require "enumerating"

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: enumerating
 version: !ruby/object:Gem::Version
-  version: 1.1.1
-  prerelease: 
+  version: 1.2.0
 platform: ruby
 authors:
 - Mike Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-25 00:00:00.000000000Z
+date: 2013-04-15 00:00:00.000000000 Z
 dependencies: []
 description: Enumerating extends Enumerable with "lazy" versions of various operations,
   allowing streamed processing of large (or even infinite) collections.  Even in Ruby
@@ -20,8 +19,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
+- CHANGES.md
 - Gemfile
-- README.markdown
+- README.md
 - Rakefile
 - benchmarks/pipeline_bench.rb
 - enumerating.gemspec
@@ -30,46 +31,45 @@ files:
 - lib/enumerating/filtering.rb
 - lib/enumerating/merging.rb
 - lib/enumerating/mixing.rb
+- lib/enumerating/prefetching.rb
+- lib/enumerating/threading.rb
 - lib/enumerating/version.rb
 - lib/enumerating/zipping.rb
 - spec/enumerating/concatenating_spec.rb
 - spec/enumerating/filtering_spec.rb
 - spec/enumerating/merging_spec.rb
+- spec/enumerating/prefetching_spec.rb
+- spec/enumerating/threading_spec.rb
 - spec/enumerating/zipping_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/mdub/enumerating
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -2370472515613642162
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -2370472515613642162
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.6
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Lazy filtering/transforming of Enumerable collections
 test_files:
 - spec/enumerating/concatenating_spec.rb
 - spec/enumerating/filtering_spec.rb
 - spec/enumerating/merging_spec.rb
+- spec/enumerating/prefetching_spec.rb
+- spec/enumerating/threading_spec.rb
 - spec/enumerating/zipping_spec.rb
 - spec/spec_helper.rb