stages 0.0.1 → 0.1.0

data/.autotest

@@ -3,7 +3,7 @@ Autotest.add_hook(:initialize)  do |at|
     at.add_exception(exception)
   end
 
-  at.add_mapping(%r%^dataprocessing/pipeline(/stages)?/(.*).rb$%, true) do |filename, _|
+  at.add_mapping(%r%^lib(/stages)?/(.*).rb$%, true) do |filename, _|
     ['test/test_pipeline.rb', 'test/test_stages.rb']
   end   
 

data/{README → README.md}

@@ -19,7 +19,7 @@ We have included some general purpose stages, map and select, which can accompli
 
 ```ruby
 pipeline = Evens.new | Map.new{ |x| x * 3} | Select.new{ |x| x % 7 == 0}
-(0..2).map{ |x| pipeline.run } #[0, 42, 84]}
+3.times{ pipeline.run } #[0, 42, 84]}
 ```
 
 Writing New Stages

data/Rakefile

@@ -25,7 +25,7 @@ Jeweler::Tasks.new do |gem|
   gem.summary = "pipeline builder"
   gem.description = "pipeline builder"
   gem.email = "support@igodigital.com"
-  gem.authors = ["Nathan Acuff", "Justin Hill", "Matt Brown", "Kyle Prifogle"]
+  gem.authors = ["The Justice Eight"]
   # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.1.0

data/examples/sing.rb

@@ -0,0 +1,48 @@
+require "#{File.dirname(__FILE__)}/../lib/stages"
+
+include Stages
+
+#count the occurance of each letter in these song lyrics
+def sing
+  { :do => 'doe a deer a female deer',
+    :re => 'ray a drop of golden sun',
+    :mi => 'me a name I call myself',
+    :fa => 'far a long long way to run',
+    :so => 'a needle pulling thread',
+    :la => 'a note to follow so',
+    :ti => 'a drink with jam and bread'}
+end
+
+def setup_pipeline
+  generator = EachElement.new sing.keys
+  loop = Restrict.new
+  get_lyric = HashLookup.new sing
+  each_character = EachInput.new{ |x| x.chars }
+  whitespace = Select.new{ |x| x != ' '}
+  pool = ResumeCount.new
+  subtotals = Map.new { |x| x.values.first }
+  iterator = EachInput.new
+  aggregator = SuperAggregator.new
+  
+  generator | loop | get_lyric | each_character | whitespace | pool | subtotals | iterator  |  aggregator
+end
+
+class SuperAggregator < Stage
+  def initialize
+    @accumulator = Hash.new{ |h,k| h[k] = 0}
+    super()
+  end
+  
+  def handle_value(value)
+    @accumulator[value[0]] += value[1]
+    while v = input
+      @accumulator[v[0]] += v[1]
+    end
+    output @accumulator
+  end
+end
+
+puts setup_pipeline.run.inspect
+
+
+

data/examples/sing_subpipes.rb

@@ -0,0 +1,34 @@
+require "#{File.dirname(__FILE__)}/../lib/stages"
+
+include Stages
+
+#count the occurance of each letter in these song lyrics
+def sing
+  { :do => 'doe a deer a female deer',
+    :re => 'ray a drop of golden sun',
+    :mi => 'me a name I call myself',
+    :fa => 'far a long long way to run',
+    :so => 'a needle pulling thread',
+    :la => 'a note to follow so',
+    :ti => 'a drink with jam and bread'}
+end
+
+def setup_pipeline
+  get_lyric = HashLookup.new sing
+  each_character = EachInput.new{ |x| x.chars }
+  whitespace = Select.new{ |x| x != ' '}
+  sub_pipeline = get_lyric | each_character | whitespace 
+  process_elements = SubStage.new(sub_pipeline)
+  
+  generator = EachElement.new sing.keys
+  subtotals = Map.new { |x| x.values.first }
+  iterator = EachInput.new
+  count = Count.new
+  
+  generator | process_elements | subtotals | iterator | count
+end
+
+puts setup_pipeline.run.inspect
+
+
+

data/lib/stage_base.rb

@@ -4,14 +4,23 @@ module Stages
     
     def initialize(&block)
       @block = block
-      @fiber_delegate = Fiber.new do
+      initialize_loop
+    end
+    
+    def initialize_loop
+       @fiber_delegate = Fiber.new do
         process
         die
       end
     end
     
     def run
-        @fiber_delegate.resume    
+      @fiber_delegate.resume    
+    end
+    
+    def continue
+      initialize_loop
+      @source.continue if @source
     end
     
     def die
@@ -38,10 +47,34 @@ module Stages
       Fiber.yield value
     end
       
-    def |(other=nil)
-      other.source = self
+    def |(other)
+      other.root_source.source = self
       other
     end
+    
+    def root_source
+      source.nil? ? self : source.root_source
+    end
+    
+    def drop_leftmost!
+      if @source.end?
+        @source = nil
+      else
+        @source.drop_leftmost!
+      end
+    end
+    
+    def end?
+      @source.nil?
+    end
+    
+    def length
+      if source
+        source.length + 1
+      else
+        1
+      end
+    end
   end  
 end 
  

data/lib/stages.rb

@@ -1,3 +1,3 @@
-require 'stage_base'
+require "#{File.dirname(__FILE__)}/stage_base"
 Dir["#{File.dirname(__FILE__)}/stages/*.rb"].each { |file| require file.gsub(".rb", "")}
 

data/lib/stages/count.rb

@@ -0,0 +1,11 @@
+module Stages
+  class Count < Stage    
+    def input
+      result = Hash.new{ |h, k| h[k] = 0 }
+      while v = source.run
+        result[v] += 1
+      end
+      result.empty? ? nil : result
+    end
+  end
+end

data/lib/stages/each_element.rb

@@ -4,7 +4,7 @@ module Stages
       @things = things
       super()
     end
-    
+        
     def process 
       @things.each do |thing|
         output thing 

data/lib/stages/each_input.rb

@@ -0,0 +1,10 @@
+module Stages
+  class EachInput < Stage
+    def handle_value(value)
+      value = @block.call(value) if @block
+      value.each do |i|
+        output i
+      end
+    end
+  end
+end

data/lib/stages/restrict.rb

@@ -0,0 +1,28 @@
+module Stages
+  class Restrict < Stage
+    alias :super_continue :continue
+    
+    def initialize
+      @open = true
+      super()
+    end
+    
+    def continue
+      @open = true
+      r = @last_value
+      @last_value = nil
+      r
+    end
+    
+    def process
+      while value = input
+        while !@open
+          handle_value nil
+        end
+        @open = false
+        @last_value = value
+        handle_value value
+      end
+    end
+  end
+end

data/lib/stages/resume.rb

@@ -0,0 +1,16 @@
+module Stages
+  class Resume < Stage    
+    def input
+      result = []
+      while v = source.run
+        result << v
+      end
+      continued = @source.continue
+      if continued.nil?
+        nil
+      else
+        { continued => result}
+      end
+    end
+  end
+end

data/lib/stages/resume_count.rb

@@ -0,0 +1,16 @@
+module Stages
+  class ResumeCount < Stage    
+    def input
+      result = Hash.new{ |h, k| h[k] = 0 }
+      while v = source.run
+        result[v] += 1
+      end
+      continued = @source.continue
+      if continued.nil?
+        nil
+      else
+        { continued => result}
+      end
+    end
+  end
+end

data/lib/stages/sub_stage.rb

@@ -0,0 +1,20 @@
+module Stages
+  class SubStage < Stage    
+    def initialize(pipeline)
+      @pipeline = pipeline
+      super()
+    end
+    
+    def handle_value(value)
+      results = []
+      subpipe = (EachElement.new([value]) | @pipeline)
+      while v = subpipe.run
+        results << v
+      end
+      @pipeline.drop_leftmost!
+      @pipeline.continue
+      results = results.first if results.length == 1
+      output ({ value => results })
+    end
+  end  
+end

data/stages.gemspec

@@ -5,31 +5,39 @@
 
 Gem::Specification.new do |s|
   s.name = "stages"
-  s.version = "0.0.1"
+  s.version = "0.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Nathan Acuff", "Justin Hill", "Matt Brown", "Kyle Prifogle"]
-  s.date = "2012-01-13"
+  s.authors = ["The Justice Eight"]
+  s.date = "2012-01-14"
   s.description = "pipeline builder"
   s.email = "support@igodigital.com"
   s.extra_rdoc_files = [
-    "README"
+    "README.md"
   ]
   s.files = [
     ".autotest",
     "Gemfile",
     "Gemfile.lock",
-    "README",
+    "README.md",
     "Rakefile",
     "VERSION",
+    "examples/sing.rb",
+    "examples/sing_subpipes.rb",
     "lib/stage_base.rb",
     "lib/stages.rb",
+    "lib/stages/count.rb",
     "lib/stages/each_element.rb",
+    "lib/stages/each_input.rb",
     "lib/stages/evens.rb",
     "lib/stages/hash_lookup.rb",
     "lib/stages/map.rb",
     "lib/stages/multiples_of.rb",
+    "lib/stages/restrict.rb",
+    "lib/stages/resume.rb",
+    "lib/stages/resume_count.rb",
     "lib/stages/select.rb",
+    "lib/stages/sub_stage.rb",
     "stages.gemspec",
     "test/helper.rb",
     "test/test_pipeline.rb",

data/test/test_pipeline.rb

@@ -10,19 +10,19 @@ class TestPipeline < MiniTest::Unit::TestCase
     mx3.source = evens 
     mx7.source = mx3
     
-    result = (0..2).map{ |x| mx7.run }
+    result = (0..2).map{ mx7.run }
     assert_equal([0, 42, 84], result)
   end
   
   test 'pipeline pipe syntax works' do
     pipeline = Evens.new | MultiplesOf.new(3) | MultiplesOf.new(7)
-    result = (0..2).map{ |x| pipeline.run }
+    result = (0..2).map{ pipeline.run }
     assert_equal([0, 42, 84], result)
   end
   
   test 'block stages work' do
     pipeline = Evens.new | Map.new{ |x| x * 3} | Select.new{ |x| x % 7 == 0}
-    result = (0..2).map{ |x| pipeline.run }
+    result = (0..2).map{ pipeline.run }
     assert_equal([0, 42, 84], result)
   end
   
@@ -43,14 +43,5 @@ class TestPipeline < MiniTest::Unit::TestCase
     end
     assert_equal([1, 2], result)    
   end  
-  
-  def sing
-    { :do => 'doe a deer a female deer',
-      :re => 'ray a drop of golden sun',
-      :mi => 'me a name I call myself',
-      :fa => 'far a long long way to run',
-      :so => 'A needle pulling thread',
-      :la => 'a note to follow so',
-      :ti => 'a drink with jam and bread'}
-  end
+
 end

data/test/test_stages.rb

@@ -11,34 +11,82 @@ class TestStages < MiniTest::Unit::TestCase
     
   test 'select' do
     pipeline = Evens.new | Select.new{ |val| val > 6}
-    result = (0..2).map{ |x| pipeline.run }
+    result = (0..2).map{ pipeline.run }
     assert_equal([8, 10, 12], result)
   end
   
   test 'map' do
     pipeline = Evens.new | Map.new{ |val| val * 3}
-    result = (0..2).map{ |x| pipeline.run }
+    result = (0..2).map{ pipeline.run }
     assert_equal([0, 6, 12], result)
   end
   
   test 'multiples_of' do
     pipeline = Evens.new | MultiplesOf.new(3)
-    result = (0..3).map{ |x| pipeline.run }
+    result = (0..3).map{ pipeline.run }
     assert_equal([0, 6, 12, 18], result)
   end
   
   test 'each_element' do
     pipeline = EachElement.new([1, 2, 3])
-    result = (0..2).map{ |x| pipeline.run }
+    result = (0..2).map{ pipeline.run }
     assert_equal([1, 2, 3], result)
   end
   
   test 'hash_lookup' do
     pipeline = EachElement.new([:do, :re, :mi]) | HashLookup.new(sing)
-    result = (0..2).map{ |x| pipeline.run }
+    result = (0..2).map { pipeline.run }
     assert_equal(['doe a deer a female deer', 'ray a drop of golden sun', 'me a name I call myself'], result)
   end   
   
+  test 'restrict' do
+    pipeline = Evens.new | Restrict.new | Map.new{ |x| x * 2}
+    result = []
+    while v = pipeline.run
+      result << v
+    end
+    assert_equal([0], result)
+    pipeline.continue
+    while v = pipeline.run
+      result << v
+    end
+    assert_equal([0, 4], result)
+  end
+  
+  test 'each_input' do
+    pipeline = EachElement.new([[1, 2], [3, 4]]) | EachInput.new 
+    result = []
+    while v = pipeline.run
+      result << v
+    end
+    assert_equal([1, 2, 3, 4], result)
+  end
+  
+  test 'resume' do
+    pipeline = EachElement.new(%w(foo bar)) | Restrict.new | EachInput.new{ |x| x.chars} | Resume.new
+    result = []
+    while v = pipeline.run
+      result << v
+    end
+    assert_equal([{ 'foo' => %w(f o o)}, {'bar' => %w(b a r)}], result)
+  end
+  
+  test 'resume with count' do
+    resume = ResumeCount.new
+    pipeline = EachElement.new(%w(foo bar)) | Restrict.new | EachInput.new{ |x| x.chars} | Map.new{ |x| x.to_sym } | resume
+    result = pipeline.run
+    assert_equal({ 'foo' => { :f => 1, :o => 2}}, result)
+  end
+  
+  test 'substage instead of resume' do
+    sub = EachInput.new{ |x| x.chars } | Map.new{ |x| x.to_sym} | Count.new
+    pipeline = EachElement.new(%w(foo bar)) | SubStage.new(sub)
+    result = pipeline.run
+    assert_equal({ 'foo' => { :f => 1, :o => 2}}, result)
+    result = pipeline.run
+    assert_equal({ 'bar' => { :b => 1, :a => 1, :r => 1}}, result)
+  end
+  
   def sing
     { :do => 'doe a deer a female deer',
       :re => 'ray a drop of golden sun',

metadata

@@ -2,18 +2,15 @@
 name: stages
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors: 
-- Nathan Acuff
-- Justin Hill
-- Matt Brown
-- Kyle Prifogle
+- The Justice Eight
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2012-01-13 00:00:00 Z
+date: 2012-01-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rake
@@ -33,22 +30,30 @@ executables: []
 extensions: []
 
 extra_rdoc_files: 
-- README
+- README.md
 files: 
 - .autotest
 - Gemfile
 - Gemfile.lock
-- README
+- README.md
 - Rakefile
 - VERSION
+- examples/sing.rb
+- examples/sing_subpipes.rb
 - lib/stage_base.rb
 - lib/stages.rb
+- lib/stages/count.rb
 - lib/stages/each_element.rb
+- lib/stages/each_input.rb
 - lib/stages/evens.rb
 - lib/stages/hash_lookup.rb
 - lib/stages/map.rb
 - lib/stages/multiples_of.rb
+- lib/stages/restrict.rb
+- lib/stages/resume.rb
+- lib/stages/resume_count.rb
 - lib/stages/select.rb
+- lib/stages/sub_stage.rb
 - stages.gemspec
 - test/helper.rb
 - test/test_pipeline.rb
@@ -66,7 +71,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 4101338441187827743
+      hash: 1745323606775845054
       segments: 
       - 0
       version: "0"