stages 0.3.4 → 0.4.0

data/README.md

@@ -45,6 +45,20 @@ If you are writing a generator, you probably want to subclass Stage and implemen
 Stern Warnings
 --------------
 
-Returning nil from handle_value kills a pipeline.  We may change this behavior in the future, but for now, it makes life easy.
+There are BREAKING CHANGES in 0.4.0.  Nil and false no longer kill pipelines, there is a special value that does that.  If you are overriding process and have a construct like this:
+```ruby
+while v = input
+  do_things v
+end
+```
+
+You will need to replace it with something like this:
+```ruby
+while !source_empty?
+  do_things input
+end
+```
+But, your pipelines now treat nil and false as perfectly valid values.  It's breaking, but it's probably an improvement.
+
 
 

data/VERSION

@@ -1 +1 @@
-0.3.4
+0.4.0

data/lib/stage_base.rb

@@ -8,36 +8,51 @@ module Stages
     end
 
     def initialize_loop
-       @fiber_delegate = Fiber.new do
+      @done = false
+      @cached_value = :stages_empty_cache
+      @fiber_delegate = Fiber.new do
         process
+        @done = true
         die
       end
     end
 
     def run
+      if @cached_value != :stages_empty_cache
+        v = @cached_value
+        @cached_value = :stages_empty_cache
+        return v
+      end
       @fiber_delegate.resume
     end
 
-    #seperate from reset! for restrict/resume purposes.
-    def reset
-      initialize_loop
-      @source.reset if @source
+    def done?
+      return true if @done
+      return false if @cached_value != :stages_empty_cache
+      next_value = @fiber_delegate.resume
+      if next_value == :stages_eos
+        @done = true
+        @cached_value = :stages_empty_cache
+        return true
+      end
+      @cached_value = next_value
+      return false
     end
 
-    def reset!
+    def reset
       initialize_loop
-      @source.reset! if @source
+      @source.reset if @source
     end
 
     def die
       loop do
-        output nil
+        output :stages_eos
       end
     end
 
     def process
-      while value = input
-        handle_value value
+      while !source_empty?
+        handle_value input
       end
     end
 
@@ -45,8 +60,12 @@ module Stages
       output value
     end
 
+    def source_empty?
+      (source.nil? || source.done?)
+    end
+
     def input
-      source.nil? ? nil : source.run
+      source.nil? ? :stages_eos : source.run
     end
 
     def output(value)
@@ -59,6 +78,9 @@ module Stages
       other
     end
 
+    #root_source lets you add to existing pipelines
+    #the result is always the rightmost stage
+    #so adding things to it is problematic
     def root_source
       source.nil? ? self : source.root_source
     end

data/lib/stages/cache.rb

@@ -1,13 +1,9 @@
 module Stages
   class Cache < Stage
-    def initialize()
-      super()
-    end
-
     def process
       cache = []
-      while i = input
-        cache << i
+      while !source_empty?
+        cache << input
       end
       cache.each{ |x| output x}
     end

data/lib/stages/count.rb

@@ -1,15 +1,16 @@
 module Stages
   class Count < Stage
-    def input
+    def process
       result = Hash.new{ |h, k| h[k] = 0 }
-      while v = source.run
+      while !source_empty?
+        v = source.run
         if v.is_a? Hash
           v.each_pair{ |key, value| result[key] += (value || 0) }
         else
           result[v] += 1
         end
       end
-      result.empty? ? nil : result
+      handle_value result
     end
   end
 end

data/lib/stages/each.rb

@@ -5,29 +5,29 @@ module Stages
       @block = block
       super()
     end
-        
-    def process 
+
+    def process
       if @things
         process_things
       else
         process_inputs
       end
     end
-    
+
     def process_inputs
-      while v = input
+      while !source_empty?
+        v = input
         v = @block.call(v) if @block
         v.each do |v|
           output v
         end
       end
     end
-    
+
     def process_things
       @things = @block.call(@things) if @block
-      output nil if @things.nil?
       @things.each do |thing|
-        output thing
+        handle_value thing
       end
     end
   end

data/lib/stages/emit.rb

@@ -4,8 +4,8 @@ module Stages
       @thing = thing
       super()
     end
-        
-    def process 
+
+    def process
       output @thing
     end
   end

data/lib/stages/exhaust.rb

@@ -2,8 +2,8 @@ module Stages
   class Exhaust < Stage
     def process
       results = []
-      while value = input
-        results << value
+      while !source_empty?
+        results << input
       end
       output results
     end

data/lib/stages/exhaust_count.rb

@@ -2,7 +2,8 @@ module Stages
   class ExhaustCount < Stage
     def process
       results = 0
-      while value = input
+      while !source_empty?
+        input
         results += 1
       end
       output results

data/lib/stages/map.rb

@@ -1,7 +1,7 @@
 module Stages
-  class Map < Stage    
+  class Map < Stage
     def handle_value(value)
       output @block.call(value)
     end
-  end  
+  end
 end

data/lib/stages/unique.rb

@@ -9,11 +9,12 @@ module Stages
 
     def process
       set = Set.new
-      while i = input
+      while !source_empty?
+        i = input
         added = set.add? i
         handle_value i if added && !@prefetch
       end
-      set.each{ |x| output x} if @prefetch
+      set.each{ |x| handle_value x} if @prefetch
       set = nil
     end
   end

data/lib/stages/wrap.rb

@@ -16,30 +16,26 @@ module Stages
 
     def reset
       initialize_loop
-      @pipeline.reset!
+      @pipeline.reset
       @source.reset if @source
     end
 
-    def reset!
-      initialize_loop
-      @pipeline.reset!
-      @source.reset! if @source
-    end
-
     def process
-      while value = input
+      while !source_empty?
+        value = input
         subpipe = Emit.new(value) | @pipeline
         results = []
-        while v = subpipe.run
+        while !subpipe.done?
+          v = subpipe.run
           @output_style == :each ? output(v) : results << v
         end
         results = results.first if @aggregated
         output results if @output_style == :array
         output({ value => results}) if @output_style == :hash
         @pipeline.drop_leftmost!
-        @pipeline.reset!
+        @pipeline.reset
       end
-      @pipeline.reset!
+      @pipeline.reset
     end
   end
 end

data/lib/sugar.rb

@@ -28,14 +28,6 @@ module Stages
       ExhaustCount.new(*args, &block)
     end
 
-    def restrict(*args, &block)
-      Restrict.new(*args, &block)
-    end
-
-    def resume(*args, &block)
-      Resume.new(*args, &block)
-    end
-
     def group(*args, &block)
       Count.new
     end

data/stages.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "stages"
-  s.version = "0.3.4"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["The Justice Eight"]
-  s.date = "2012-04-02"
+  s.date = "2012-04-12"
   s.description = "pipeline builder"
   s.email = "support@igodigital.com"
   s.extra_rdoc_files = [
@@ -22,7 +22,6 @@ Gem::Specification.new do |s|
     "README.md",
     "Rakefile",
     "VERSION",
-    "examples/sing.rb",
     "examples/sing_custom_stages.rb",
     "examples/sing_subpipes.rb",
     "lib/stage_base.rb",
@@ -34,9 +33,6 @@ Gem::Specification.new do |s|
     "lib/stages/exhaust.rb",
     "lib/stages/exhaust_count.rb",
     "lib/stages/map.rb",
-    "lib/stages/restrict.rb",
-    "lib/stages/resume.rb",
-    "lib/stages/resume_count.rb",
     "lib/stages/select.rb",
     "lib/stages/unique.rb",
     "lib/stages/wrap.rb",

data/test/helper.rb

@@ -46,7 +46,6 @@ class MiniTest::Unit
 
 
         @broken = nil
-
         @@out.print(case run_one inst
                     when :pass
                       @broken = false

data/test/test_pipeline.rb

@@ -53,10 +53,10 @@ class TestPipeline < MiniTest::Unit::TestCase
     assert_equal({ 'bar' => { :b => 1, :a => 1, :r => 1}}, result)
   end
 
-  test 'reset! resets everything' do
+  test 'reset resets everything' do
     pipeline = Each.new([1, 2, 3])
     assert_equal(1, pipeline.run)
-    pipeline.reset!
+    pipeline.reset
     assert_equal(1, pipeline.run)
     assert_equal(2, pipeline.run)
   end

data/test/test_stages.rb

@@ -32,36 +32,6 @@ class TestStages < MiniTest::Unit::TestCase
     assert_equal(['doe a deer a female deer', 'ray a drop of golden sun', 'me a name I call myself'], result)
   end
 
-  test 'restrict' do
-    pipeline = Evens.new | Restrict.new | Map.new{ |x| x * 2}
-    result = []
-    while v = pipeline.run
-      result << v
-    end
-    assert_equal([0], result)
-    pipeline.reset
-    while v = pipeline.run
-      result << v
-    end
-    assert_equal([0, 4], result)
-  end
-
-  test 'resume' do
-    pipeline = Each.new(%w(foo bar)) | Restrict.new | Each.new{ |x| x.chars} | Resume.new
-    result = []
-    while v = pipeline.run
-      result << v
-    end
-    assert_equal([{ 'foo' => %w(f o o)}, {'bar' => %w(b a r)}], result)
-  end
-
-  test 'resume with count' do
-    resume = ResumeCount.new
-    pipeline = Each.new(%w(foo bar)) | Restrict.new | Each.new{ |x| x.chars} | Map.new{ |x| x.to_sym } | resume
-    result = pipeline.run
-    assert_equal({ 'foo' => { :f => 1, :o => 2}}, result)
-  end
-
   test 'hash mode wrap' do
     pipeline = Each.new(%w(foo bar)) | Wrap.new(Each.new{ |x| x.chars})
     result = pipeline.run
@@ -90,20 +60,25 @@ class TestStages < MiniTest::Unit::TestCase
   test 'each mode wrap' do
     pipeline = Each.new(%w(foo bar)) | Wrap.new(Each.new{ |x| x.chars}, :each)
     expected = %w(r a b o o f)
-    while r = pipeline.run
-      assert_equal(expected.pop, r)
+    while !pipeline.done?
+      assert_equal(expected.pop, pipeline.run)
     end
   end
 
+  test 'nil makes it through' do
+    pipeline = Each.new([1, 2, nil, 4]) | Map.new{ |x| x.nil? ? x : x*2} | Exhaust.new
+    result = pipeline.run
+    assert_equal([2, 4, nil, 8], result)
+  end
 
   test 'unique-jit' do
     order = []
     pipeline = Each.new('abcadefbega'){ |x| x.chars} |
-      Map.new{ |x| order << 'a'; x} | Unique.new|
+      Map.new{ |x| order << 'a'; x} | Unique.new |
       Map.new{ |x| order << 'b'; x}
     results = []
-    while r = pipeline.run
-      results << r
+    while !pipeline.done?
+      results << pipeline.run
     end
     assert_equal(%w(a b c d e f g), results)
     assert_equal(%w(a b a b a b a a b a b a b a a a b a), order)
@@ -115,8 +90,8 @@ class TestStages < MiniTest::Unit::TestCase
       Map.new{ |x| order << 'a'; x} | Unique.new(prefetch: true) |
       Map.new{ |x| order << 'b'; x}
     results = []
-    while r = pipeline.run
-      results << r
+    while !pipeline.done?
+      results << pipeline.run
     end
     assert_equal(%w(a b c d e f g), results)
     assert_equal(['a']*11 + ['b']*7, order)
@@ -149,9 +124,9 @@ class TestStages < MiniTest::Unit::TestCase
     assert_equal({ a: 2}, pipeline.run)
   end
 
-  test 'trying to pull from nil returns nil, not an excemption' do
+  test 'trying to pull from nil returns nil, not an exception' do
     pipeline = Unique.new
-    assert_equal(nil, pipeline.run)
+    assert pipeline.done?
   end
 
   def sing

data/test/test_syntax.rb

@@ -14,20 +14,11 @@ class TestSyntax < MiniTest::Unit::TestCase
     assert_equal({ 1 => 6}, pipeline.run)
   end
 
-  test 'restrict and resume' do
-    pipeline = each(%w(foo bar)) | restrict | each{ |x| x.chars} | resume
-    result = []
-    while v = pipeline.run
-      result << v
-    end
-    assert_equal([{ 'foo' => %w(f o o)}, {'bar' => %w(b a r)}], result)
-  end
-
   test 'wrap' do
     pipeline = each(%w(foo bar)) | wrap(each{ |x| x.chars}, :each)
     expected = %w(r a b o o f)
-    while r = pipeline.run
-      assert_equal(expected.pop, r)
+    while !pipeline.done?
+      assert_equal(expected.pop, pipeline.run)
     end
   end
 

metadata

@@ -2,7 +2,7 @@
 name: stages
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors: 
 - The Justice Eight
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-04-02 00:00:00 Z
+date: 2012-04-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rake
@@ -38,7 +38,6 @@ files:
 - README.md
 - Rakefile
 - VERSION
-- examples/sing.rb
 - examples/sing_custom_stages.rb
 - examples/sing_subpipes.rb
 - lib/stage_base.rb
@@ -50,9 +49,6 @@ files:
 - lib/stages/exhaust.rb
 - lib/stages/exhaust_count.rb
 - lib/stages/map.rb
-- lib/stages/restrict.rb
-- lib/stages/resume.rb
-- lib/stages/resume_count.rb
 - lib/stages/select.rb
 - lib/stages/unique.rb
 - lib/stages/wrap.rb
@@ -75,7 +71,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 2855126924498756477
+      hash: -3184955072304030691
       segments: 
       - 0
       version: "0"

data/examples/sing.rb

@@ -1,48 +0,0 @@
-require "#{File.dirname(__FILE__)}/../lib/stages"
-
-include Stages
-
-#count the occurance of each letter in these song lyrics
-def sing
-  { :do => 'doe a deer a female deer',
-    :re => 'ray a drop of golden sun',
-    :mi => 'me a name I call myself',
-    :fa => 'far a long long way to run',
-    :so => 'a needle pulling thread',
-    :la => 'a note to follow so',
-    :ti => 'a drink with jam and bread'}
-end
-
-def setup_pipeline
-  generator = Each.new sing.keys
-  loop = Restrict.new
-  get_lyric = Map.new{ |x| sing[x]}
-  each_character = Each.new{ |x| x.chars }
-  whitespace = Select.new{ |x| x != ' '}
-  pool = ResumeCount.new
-  subtotals = Map.new { |x| x.values.first }
-  iterator = Each.new
-  aggregator = SuperAggregator.new
-  
-  generator | loop | get_lyric | each_character | whitespace | pool | subtotals | iterator  |  aggregator
-end
-
-class SuperAggregator < Stage
-  def initialize
-    @accumulator = Hash.new{ |h,k| h[k] = 0}
-    super()
-  end
-  
-  def handle_value(value)
-    @accumulator[value[0]] += value[1]
-    while v = input
-      @accumulator[v[0]] += v[1]
-    end
-    output @accumulator
-  end
-end
-
-puts setup_pipeline.run.inspect
-
-
-

data/lib/stages/restrict.rb

@@ -1,27 +0,0 @@
-module Stages
-  class Restrict < Stage
-    
-    def initialize
-      @open = true
-      super()
-    end
-    
-    def reset
-      @open = true
-      r = @last_value
-      @last_value = nil
-      r
-    end
-    
-    def process
-      while value = input
-        while !@open
-          handle_value nil
-        end
-        @open = false
-        @last_value = value
-        handle_value value
-      end
-    end
-  end
-end

data/lib/stages/resume.rb

@@ -1,16 +0,0 @@
-module Stages
-  class Resume < Stage    
-    def input
-      result = []
-      while v = source.run
-        result << v
-      end
-      continued = @source.reset
-      if continued.nil?
-        nil
-      else
-        { continued => result}
-      end
-    end
-  end
-end

data/lib/stages/resume_count.rb

@@ -1,16 +0,0 @@
-module Stages
-  class ResumeCount < Stage    
-    def input
-      result = Hash.new{ |h, k| h[k] = 0 }
-      while v = source.run
-        result[v] += 1
-      end
-      continued = @source.reset
-      if continued.nil?
-        nil
-      else
-        { continued => result}
-      end
-    end
-  end
-end