piece_pipe 0.1.0 → 0.2.0

data/documentation_todo.txt

@@ -1,5 +1,3 @@
-MethodStep
-MethodAssemblyStep
 TapStep
 MapStep
 HashedAggregator

data/lib/piece_pipe.rb

@@ -6,6 +6,7 @@ require 'piece_pipe/collector'
 require 'piece_pipe/debug_step'
 require 'piece_pipe/hashed_aggregator'
 require 'piece_pipe/map_step'
+require 'piece_pipe/group_by_step'
 require 'piece_pipe/method_assembly_step'
 require 'piece_pipe/method_step'
 require 'piece_pipe/tap_step'

data/lib/piece_pipe/assembly_step.rb

@@ -1,5 +1,33 @@
 
 module PiecePipe
+  #
+  # A specialization of Step, designed to operate on subsets of input and add information
+  # to the flow.  Inputs arrive as Hashes, and outputs are generated by (effectively)
+  # merging new key/value pairs on the Hash and producing the result.
+  #
+  # Instead of implementing #process, AssemblySteps should implement #receive.
+  #
+  # Instead of generating output with #produce, safely generate compound output
+  # by calling #install with a hash of one or more interesting values.  As with
+  # process/produce, you may call #install zero or more times, depending on your
+  # need to filter, transform or expand the stream of objects.
+  #
+  # The input to #receive is a duplicate of the actual Hash object that was
+  # produced by the previous step, so modifying the inputs directly will
+  # neither help you, nor harm subsequent steps.
+  #
+  # AssemblySteps insist on consuming Hash-like objects; though the preceding step
+  # in the pipeline needn't actually be an AssemblyStep, it must produce Hash-like 
+  # objects as output.
+  #
+  # #install starts with a fresh copy of the inputs each time, so multiple 
+  # invocations will not share the installed values.
+  #
+  # IF YOUR STEP DECIDES NOT TO INSTALL ANYTHING: be sure to call #noop
+  # (equiv to calling #install({})) which will pass all inputs on to the next
+  # step of the pipeline unmodified.  If you DON'T call #install, your 
+  # step effectively becomes a filter, and the flow of objects will cease.
+  #
   class AssemblyStep < Step
     def process(item)
       ensure_hash_like_object item
@@ -11,6 +39,7 @@ module PiecePipe
       end
     end
 
+    # Default implementation is a noop
     def receive(assembly)
       noop
     end

data/lib/piece_pipe/group_by_step.rb

@@ -0,0 +1,34 @@
+module PiecePipe
+
+  # does a map / aggregate for you
+  class GroupByStep < Step
+    def initialize(*keys)
+      @hash = {}
+      @keys = keys
+    end
+
+    def process(item)
+      key = nil
+      if @keys.size > 1
+        key = []
+        @keys.each do |k|
+          key << item[k]
+        end
+      else
+        key = item[@keys.first]
+      end
+
+      val = item
+      @hash[key] ||= []
+      @hash[key] << item
+    end
+
+    def generate_sequence
+      super
+      @hash.each do |key,values|
+        produce key => values
+      end
+    end
+  end
+
+end

data/lib/piece_pipe/hashed_aggregator.rb

@@ -1,4 +1,16 @@
 module PiecePipe
+  #
+  # This Step acts like a sink: it will process ALL available inputs until the stream ends,
+  # and THEN call #aggregrate multiple times to transform accumulated data and produce output.
+  #
+  # For every key-value pair that arrives as an input (see MapStep for generating key-value pairs),
+  # an array of values is accumulated per key.
+  #
+  # When all inputs have been collected, #aggregate is called for each key that has been seen to-date.
+  #
+  # Your job is to implement #aggregate, perform some calculation on the key and its accumulated
+  # values, and generate output via zero or more calls to #produce.
+  #
   class HashedAggregator < Step
     def initialize
       @hash = {}
@@ -19,6 +31,8 @@ module PiecePipe
       @hash[item[:key]] << item[:value]
     end
 
+    # Accepts a key and an array of all values that have been collected for that key.
+    # Default impl simply produces { :key => key, :value => values } (essentially a noop).
     def aggregate(key, values)
       produce key: key, values: values 
     end

data/lib/piece_pipe/map_step.rb

@@ -1,4 +1,23 @@
 module PiecePipe
+  #
+  # This type of Step is designed to produce key-value pairs in anticipation of
+  # a following aggregation step (see HashedAggregator).
+  #
+  # Subclasses implement #map and can produce output via #emit(key, value).
+  # Zero-or-more pairs may be generted per #map invocation.
+  #
+  # The output structure is in fact a Hash with two keys: :key and :value, eg
+  #
+  #   class CalorieCounter < PiecePipe::MapStep
+  #     # Expects inputs to be a Hash with :person, :date and :calories.
+  #     # Output will look like: { :key => ["Dave","2012-05-10"], :value => 6000 }
+  #     def map(inputs)
+  #       key = [inputs[:person], inputs[:date]]
+  #       val = inputs[:calories]
+  #       emit key, val
+  #     end
+  #   end
+  # 
   class MapStep < Step
     def process(item)
       map item

data/lib/piece_pipe/method_assembly_step.rb

@@ -1,4 +1,16 @@
 module PiecePipe
+  # Essentially the same thing as MethodStep with AssemblyStep semantics.
+  #
+  # Because we don't have a clever way to determine that you might want to create
+  # an assembly line out of wrapped methods, you need to use the conveneince method Pipeline#assembly_step, eg.
+  #
+  #   PiecePipe::Pipeline.new.assembly_step(:add_seatbelts)
+  #
+  # Currently only supports arity-1 methods: the parameter will be the inputs Hash,
+  # the return is expected to be a Hash that will be produced via AssemblyStep#install.
+  #
+  # TODO; add support for arity-2 method wrapping to allow for zero-or-more #intall calls
+  #
   class MethodAssemblyStep < AssemblyStep
     def initialize(meth)
       @method = meth

data/lib/piece_pipe/method_step.rb

@@ -1,4 +1,25 @@
 module PiecePipe
+  # MethodStep makes it possible to define a pipeline step by wrapping a Method
+  # object. Eg,
+  #   PiecePipe::Pipeline.new.step( method(:do_something) )
+  #
+  # In this contrived example, whatever class is building the Pipeline has
+  # a #do_something method, which is accessed as a Method object from Ruby's
+  # reflection API via Kernel#method.
+  #
+  # If the wrapped method accepts a single parameter (arity 1), the inputs
+  # to your MethodStep will be passed in via this parameter, and the return
+  # value of your method will be automatically #produced, even if you 
+  # return nil.
+  #
+  # If the wrapped method accepts TWO parameters (arity 2), the inputs will
+  # still arrive via the first paramter, and the second parameter will
+  # be a "producer" object that responds to #produce.  This gives you the
+  # flexibility to #produce zero-or-many items, as in normal Step 
+  # subclasses.  In the case of arity-2 methods, the return value
+  # is disregarded.  Failing to call producer#produce will result
+  # in a filtering-out of objects.
+  #
   class MethodStep < Step
     def initialize(meth)
       raise "method cannot be nil" if meth.nil?

data/lib/piece_pipe/pipeline.rb

@@ -35,6 +35,12 @@ module PiecePipe
       add_step step
     end
 
+    # like rails group by, takes the entire row and keys it based on
+    # the fields
+    def group_by(*keys)
+      add_step(GroupByStep.new(*keys))
+    end
+
     def add_step(step)
       step.source = @latest_source
       @latest_source = step

data/lib/piece_pipe/version.rb

@@ -1,3 +1,3 @@
 module PiecePipe
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/piece_pipe.gemspec

@@ -17,5 +17,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "mocha"
   gem.add_development_dependency "rspec"
   gem.add_development_dependency "rake"
+  gem.add_development_dependency "pry"
     
 end

data/spec/group_by_step_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe PiecePipe::GroupByStep do
+  
+  context "grouping by a subset" do
+
+    it "groups by a single key" do
+      subject = PiecePipe::GroupByStep.new(:last)
+      inputs = [ { first: 'David', last: 'Crosby', age: 65 },
+                 { first: 'Bill', last: 'Crosby', age: 67 } ]
+
+      PiecePipe::Pipeline.new.
+        source(inputs).
+        step(subject).to_a.first.should == {
+        "Crosby" => inputs
+      }
+
+      # ezpipe(subject,inputs)
+    end
+
+    it "groups by multiple keys" do
+      subject = PiecePipe::GroupByStep.new(:last, :first)
+      inputs = { first: 'David', last: 'Crosby', age: 65 }
+      ezpipe(subject,inputs).to_a.first.should == {
+        ["Crosby", 'David'] => [inputs]
+      }
+
+    end
+  end
+end

data/spec/pipeline_spec.rb

@@ -87,4 +87,23 @@ describe PiecePipe::Pipeline do
     end
   end
 
+  describe "#group_by" do
+    let(:project_info) {
+      [
+        {name: "Project 1", project_health_summary: "Summary 1"},
+        {name: "Project 2", project_health_summary: "Summary 2"},
+      ]
+    }
+
+    it "maps the pipeline items down to the values indicated by the given key" do
+      subject.
+        source(project_info).
+        group_by(:name)
+
+      p1 = {"Project 1" => [ {name: "Project 1", project_health_summary: "Summary 1"} ]}
+      p2 = {"Project 2" => [ {name: "Project 2", project_health_summary: "Summary 2"} ]}
+      subject.to_enum.to_a.should == [ p1, p2]
+    end
+  end
+
 end

data/spec/spec_helper.rb

@@ -1,4 +1,5 @@
 require 'piece_pipe'
+require 'pry'
 
 module SpecHelpers
   def ezpipe(single_step, inputz)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: piece_pipe
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-07 00:00:00.000000000Z
+date: 2012-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &2152338560 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +22,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152338560
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152337340 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +38,31 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152337340
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2152321200 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,7 +70,12 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152321200
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! 'PiecePipe is about breaking your problem into its smallest, most interesting
   pieces, solving those pieces and not spending time on the glue code between them. '
 email:
@@ -65,6 +96,7 @@ files:
 - lib/piece_pipe/assembly_step.rb
 - lib/piece_pipe/collector.rb
 - lib/piece_pipe/debug_step.rb
+- lib/piece_pipe/group_by_step.rb
 - lib/piece_pipe/hashed_aggregator.rb
 - lib/piece_pipe/map_step.rb
 - lib/piece_pipe/method_assembly_step.rb
@@ -76,6 +108,7 @@ files:
 - piece_pipe.gemspec
 - spec/assembly_step_spec.rb
 - spec/collector_spec.rb
+- spec/group_by_step_spec.rb
 - spec/hashed_aggregator_spec.rb
 - spec/map_step_spec.rb
 - spec/method_assembly_step_spec.rb
@@ -95,15 +128,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2537402222991668823
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 2537402222991668823
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: PiecePipe helps you break your code into small interesting pieces and provides
@@ -111,6 +150,7 @@ summary: PiecePipe helps you break your code into small interesting pieces and p
 test_files:
 - spec/assembly_step_spec.rb
 - spec/collector_spec.rb
+- spec/group_by_step_spec.rb
 - spec/hashed_aggregator_spec.rb
 - spec/map_step_spec.rb
 - spec/method_assembly_step_spec.rb