chicago-etl 0.1.3 → 0.1.4

data/VERSION

@@ -1 +1 @@
-0.1.3
+0.1.4

data/chicago-etl.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.1.3"
+  s.version = "0.1.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
@@ -32,10 +32,11 @@ Gem::Specification.new do |s|
     "lib/chicago/etl/core_extensions.rb",
     "lib/chicago/etl/counter.rb",
     "lib/chicago/etl/dataset_batch_stage.rb",
+    "lib/chicago/etl/dataset_builder.rb",
     "lib/chicago/etl/key_builder.rb",
     "lib/chicago/etl/load_dataset_builder.rb",
-    "lib/chicago/etl/load_pipeline_stage_builder.rb",
     "lib/chicago/etl/pipeline.rb",
+    "lib/chicago/etl/schema_sinks_and_transformations_builder.rb",
     "lib/chicago/etl/schema_table_sink_factory.rb",
     "lib/chicago/etl/screens/column_screen.rb",
     "lib/chicago/etl/screens/invalid_element.rb",
@@ -62,7 +63,6 @@ Gem::Specification.new do |s|
     "lib/chicago/flow/mysql_file_sink.rb",
     "lib/chicago/flow/null_sink.rb",
     "lib/chicago/flow/pipeline_endpoint.rb",
-    "lib/chicago/flow/pipeline_stage.rb",
     "lib/chicago/flow/sink.rb",
     "lib/chicago/flow/transformation.rb",
     "lib/chicago/flow/transformation_chain.rb",
@@ -70,7 +70,7 @@ Gem::Specification.new do |s|
     "spec/etl/batch_spec.rb",
     "spec/etl/core_extensions_spec.rb",
     "spec/etl/counter_spec.rb",
-    "spec/etl/dataset_batch_stage_spec.rb",
+    "spec/etl/define_dimension_stage_spec.rb",
     "spec/etl/define_stage_spec.rb",
     "spec/etl/etl_batch_id_dataset_filter.rb",
     "spec/etl/key_builder_spec.rb",
@@ -82,6 +82,7 @@ Gem::Specification.new do |s|
     "spec/etl/screens/out_of_bounds_spec.rb",
     "spec/etl/sequel/dependant_tables_spec.rb",
     "spec/etl/sequel/filter_to_etl_batch_spec.rb",
+    "spec/etl/stage_spec.rb",
     "spec/etl/table_builder_spec.rb",
     "spec/etl/task_spec.rb",
     "spec/etl/transformations/deduplicate_rows_spec.rb",
@@ -95,7 +96,6 @@ Gem::Specification.new do |s|
     "spec/flow/mysql_file_serializer_spec.rb",
     "spec/flow/mysql_file_sink_spec.rb",
     "spec/flow/mysql_integration_spec.rb",
-    "spec/flow/pipeline_stage_spec.rb",
     "spec/flow/transformation_chain_spec.rb",
     "spec/flow/transformation_spec.rb",
     "spec/spec_helper.rb"

data/lib/chicago/etl/dataset_batch_stage.rb

@@ -4,48 +4,28 @@ module Chicago
     #
     # Allows deferring constructing a DatasetSource until extract
     # time, so that it can be filtered to an ETL batch appropriately.
-    class DatasetBatchStage
+    class DatasetBatchStage < Stage
       attr_reader :name
 
-      def initialize(name, dataset, pipeline_stage, options={})
-        @name = name
-        @dataset = dataset
-        @pipeline_stage = pipeline_stage
-        @filter_strategy = options[:filter_strategy] || lambda {|dataset, etl_batch|
-          dataset.filter_to_etl_batch(etl_batch)
-        }
+      def initialize(name, options={})
+        super
+        @filter_strategy = options[:filter_strategy] ||
+          lambda { |dataset, etl_batch| @source.filter_to_etl_batch(etl_batch)}
         @truncate_pre_load = !!options[:truncate_pre_load]
-      end
+     end
 
       # Executes this ETL stage.
       #
       # Configures the dataset and flows rows into the pipeline.
       def execute(etl_batch, reextract=false)
         if @truncate_pre_load
-          pipeline_stage.sinks.each {|sink| sink.truncate }
-        elsif reextract && pipeline_stage.sink(:error)
-          pipeline_stage.sink(:error).truncate
-        end
-
-        pipeline_stage.execute(source(etl_batch, reextract))
-      end
-
-      # Returns the pipeline for this stage.
-      def pipeline_stage
-        @pipeline_stage.sink(:default).
-          set_constant_values(:_inserted_at => Time.now)
-        @pipeline_stage
-      end
-
-      # Returns a DatasetSource for the provided dataset filtered to
-      # the ETL batch as appropriate.
-      def source(etl_batch, reextract=false)
-        if reextract
-          filtered_dataset = @dataset
-        else
-          filtered_dataset = @filter_strategy.call(@dataset, etl_batch)
+          sinks.each {|sink| sink.truncate }
+        elsif reextract && sink(:error)
+          sink(:error).truncate
         end
-        Chicago::Flow::DatasetSource.new(filtered_dataset)
+        
+        sink(:default).set_constant_values(:_inserted_at => Time.now)
+        super
       end
     end
   end

data/lib/chicago/etl/dataset_builder.rb

@@ -0,0 +1,60 @@
+module Chicago
+  module ETL
+    # Provides convenience methods for defining source datasets.
+    class DatasetBuilder
+      attr_reader :db
+
+      # @api private
+      def initialize(db)
+        @db = db
+      end
+
+      # @api private
+      def build(&block)
+        instance_eval(&block)
+      end
+
+      protected
+      
+      def key_field(field, name)
+        :if[{field => nil}, 1, field].as(name)
+      end
+      
+      # Returns a column for use in a Sequel::Dataset#select method to
+      # return a dimension key.
+      #
+      # Takes care of using the key tables correctly, and dealing with
+      # missing dimension values.
+      def dimension_key(name)
+        key_field("keys_dimension_#{name}__dimension_id".to_sym,
+                  "#{name}_dimension_id".to_sym)
+      end
+
+      # Returns a column for use in a Sequel::Dataset#select method to
+      # return a date dimension key.
+      def date_dimension_column(dimension)
+        :if.sql_function({:id.qualify(dimension) => nil},
+                         1, 
+                         :id.qualify(dimension)).
+          as("#{dimension}_dimension_id".to_sym)
+      end
+
+      # Rounds a monetary value to 2 decimal places.
+      #
+      # By default, natural rounding is used, you can specify either
+      # :up or :down as the direction.
+      #
+      # @deprecated
+      def round(stmt, direction = :none)
+        case direction
+        when :none
+          :round.sql_function(stmt, 2)
+        when :up
+          :ceil.sql_function(stmt * 100) / 100
+        when :down
+          :floor.sql_function(stmt * 100) / 100
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/pipeline.rb

@@ -61,13 +61,17 @@ module Chicago
       # @api private
       def build(name, &block)
         instance_eval &block
-        unless defined? @pipeline_stage
+        unless defined? @sinks_and_transformations
           pipeline do
           end
         end
-        DatasetBatchStage.new(name, @dataset, @pipeline_stage,
-                              :filter_strategy => @filter_strategy,
-                              :truncate_pre_load => @truncate_pre_load)
+         DatasetBatchStage.new(name,
+                               :source => @dataset, 
+                               :transformations => @sinks_and_transformations[:transformations],
+                               :sinks => @sinks_and_transformations[:sinks],
+                               :filter_strategy => @filter_strategy,
+                               :truncate_pre_load => @truncate_pre_load)
+
       end
 
       protected
@@ -88,7 +92,7 @@ module Chicago
       # for details.
       # TODO: rename pipeline => transforms below this method
       def pipeline(&block)
-        @pipeline_stage = LoadPipelineStageBuilder.new(@db, @schema_table).
+        @sinks_and_transformations = SchemaSinksAndTransformationsBuilder.new(@db, @schema_table).
           build(&block)
       end
 
@@ -106,62 +110,5 @@ module Chicago
         @filter_strategy = block
       end
     end
-
-    # Provides convenience methods for defining source datasets.
-    class DatasetBuilder
-      attr_reader :db
-
-      # @api private
-      def initialize(db)
-        @db = db
-      end
-
-      # @api private
-      def build(&block)
-        instance_eval(&block)
-      end
-
-      protected
-      
-      def key_field(field, name)
-        :if[{field => nil}, 1, field].as(name)
-      end
-      
-      # Returns a column for use in a Sequel::Dataset#select method to
-      # return a dimension key.
-      #
-      # Takes care of using the key tables correctly, and dealing with
-      # missing dimension values.
-      def dimension_key(name)
-        key_field("keys_dimension_#{name}__dimension_id".to_sym,
-                  "#{name}_dimension_id".to_sym)
-      end
-
-      # Returns a column for use in a Sequel::Dataset#select method to
-      # return a date dimension key.
-      def date_dimension_column(dimension)
-        :if.sql_function({:id.qualify(dimension) => nil},
-                         1, 
-                         :id.qualify(dimension)).
-          as("#{dimension}_dimension_id".to_sym)
-      end
-
-      # Rounds a monetary value to 2 decimal places.
-      #
-      # By default, natural rounding is used, you can specify either
-      # :up or :down as the direction.
-      #
-      # @deprecated
-      def round(stmt, direction = :none)
-        case direction
-        when :none
-          :round.sql_function(stmt, 2)
-        when :up
-          :ceil.sql_function(stmt * 100) / 100
-        when :down
-          :floor.sql_function(stmt * 100) / 100
-        end
-      end
-    end
   end
 end

data/lib/chicago/etl/{load_pipeline_stage_builder.rb → schema_sinks_and_transformations_builder.rb}

@@ -5,7 +5,7 @@ module Chicago
     #
     # Clients will not normally instantiate this themselves but use it
     # in the context of defining an ETL stage.
-    class LoadPipelineStageBuilder
+    class SchemaSinksAndTransformationsBuilder
       # @api private
       KeyMapping = Struct.new(:table, :field)
 
@@ -41,9 +41,9 @@ module Chicago
         add_screens
         add_key_transforms
         add_final_transforms
-        pipeline_stage = create_pipeline_stage
-        register_additional_sinks(pipeline_stage)
-        pipeline_stage
+        sinks_and_transformations = create_sinks_and_transformations
+        register_additional_sinks(sinks_and_transformations)
+        sinks_and_transformations
       end
 
       protected
@@ -81,7 +81,7 @@ module Chicago
 
       private
 
-      def create_pipeline_stage
+      def create_sinks_and_transformations
         default = @sink_factory.sink(:ignore => @ignore_present_rows,
                                      :exclude => @load_separately)
         key_sink = if @schema_table.kind_of?(Chicago::Schema::Dimension)
@@ -90,24 +90,26 @@ module Chicago
                      # Facts have no key table to write to.
                      Flow::NullSink.new
                    end
-        
-        Flow::PipelineStage.
-          new(:transformations => concat_transformations,
-              :sinks => {
-                :default => default,
-                :dimension_key => key_sink,
-                :error => @sink_factory.error_sink
-              })
+
+        {
+          :transformations => concat_transformations,
+          :sinks => {
+            :default => default,
+            :dimension_key => key_sink,
+            :error => @sink_factory.error_sink
+          }
+        }
       end
 
       def concat_transformations
         TRANSFORMATION_ORDER.map {|k| @transformations[k] }.flatten
       end
 
-      def register_additional_sinks(pipeline_stage)
+      def register_additional_sinks(sinks_and_transformations)
+        sinks = sinks_and_transformations[:sinks]
         @key_mappings.each do |mapping|
           sink = @sink_factory.key_sink(:table => mapping.table)
-          pipeline_stage.register_sink(mapping.table, sink)
+          sinks[mapping.table] = sink
         end
       end
 

data/lib/chicago/etl/stage.rb

@@ -1,42 +1,30 @@
 module Chicago
   module ETL
+    # A Stage in the ETL pipeline.
+    #
+    # A Stage wires together a Source, 0 or more Transformations and 1
+    # or more Sinks.
     class Stage
+      # Returns the source for this stage.
+      attr_reader :source
+      
+      # Returns the name of this stage.
       attr_reader :name
 
       def initialize(name, options={})
         @name = name
-        @source = options.fetch(:source)
-        raise ArgumentError, "Stage #{name} requires a source" unless @source
-
-        @sinks = options.fetch(:sinks)
-        raise ArgumentError, "Stage #{name} requires at least one sink" if @sinks.empty?
-
-        @transformations = options.fetch(:transformations)
-        @transformation_chain = Chicago::Flow::TransformationChain.
-          new(*@transformations)
-
+        @source = options[:source]
+        @sinks = options[:sinks]
+        @transformations = options[:transformations] || []
         @filter_strategy = options[:filter_strategy] || 
           lambda {|source, _| source }
-      end
 
-      def execute(etl_batch, reextract)
-        modified_source = reextract_and_filter_source(@source, etl_batch, reextract)
-        transform_and_load_from(modified_source)
-      end
-      
-      def transform_and_load_from(source)
+        validate_arguments
       end
 
-      def reextract_and_filter_source(source, etl_batch, reextract=false)
-        if reextract
-          filtered_dataset = source
-        else
-          filtered_dataset = @filter_strategy.call(source, etl_batch)
-        end
-        Chicago::Flow::DatasetSource.new(filtered_dataset)
+      def execute(etl_batch, reextract=false)
+        transform_and_load filtered_source(etl_batch, reextract)
       end
-
-          attr_reader :transformation_chain
       
       # Returns the named sink, if it exists
       def sink(name)
@@ -46,20 +34,22 @@ module Chicago
       def sinks
         @sinks.values
       end
+      
+      def filtered_source(etl_batch, reextract=false)
+        filtered_dataset = reextract ? source : 
+          @filter_strategy.call(source, etl_batch)
 
-      def register_sink(name, sink)
-        @sinks[name.to_sym] = sink
-        self
+        Chicago::Flow::DatasetSource.new(filtered_dataset)
       end
-            
-      def transform_and_load_from(source)
+
+      private
+
+      def transform_and_load(source)
         sinks.each(&:open)
         pipe_rows_to_sinks_from(source)
         sinks.each(&:close)
       end
-      
-      private
-      
+
       def pipe_rows_to_sinks_from(source)
         source.each do |row|
           transformation_chain.process(row).each {|row| process_row(row) }
@@ -67,10 +57,25 @@ module Chicago
         transformation_chain.flush.each {|row| process_row(row) }
       end
 
+      def transformation_chain
+        @transformation_chain ||= Chicago::Flow::TransformationChain.
+          new(*@transformations)
+      end
+
       def process_row(row)
         stream = row.delete(:_stream) || :default
         @sinks[stream] << row
       end
+
+      def validate_arguments
+        if @source.nil?
+          raise ArgumentError, "Stage #{@name} requires a source"
+        end
+
+        if @sinks.blank?
+          raise ArgumentError, "Stage #{@name} requires at least one sink"
+        end
+      end
     end
   end
 end

data/lib/chicago/etl/stage_builder.rb

@@ -1,8 +1,6 @@
 module Chicago
   module ETL
     class StageBuilder
-      attr_reader :sink_factory
-
       def initialize(db)
         @db = db
       end
@@ -20,15 +18,17 @@ module Chicago
                   :filter_strategy => @filter_strategy)
       end
 
+      protected
+
       def source(&block)
         @dataset = DatasetBuilder.new(@db).build(&block)
       end
       
-      def transformations(klass=TransformationBuilder, &block)
-        @transformations = klass.new.build(&block)
+      def transformations(&block)
+        @transformations = TransformationBuilder.new.build(&block)
       end
 
-      def sinks(options={}, &block)
+      def sinks(&block)
         @sinks = SinkBuilder.new.build(&block)
       end
 

data/lib/chicago/etl.rb

@@ -10,7 +10,6 @@ require 'chicago/flow/errors'
 require 'chicago/flow/transformation'
 require 'chicago/flow/filter'
 require 'chicago/flow/transformation_chain'
-require 'chicago/flow/pipeline_stage'
 require 'chicago/flow/pipeline_endpoint'
 require 'chicago/flow/array_source'
 require 'chicago/flow/dataset_source'
@@ -25,12 +24,12 @@ require 'chicago/etl/key_builder'
 require 'chicago/etl/schema_table_sink_factory'
 require 'chicago/etl/transformations'
 require 'chicago/etl/load_dataset_builder'
-require 'chicago/etl/dataset_batch_stage'
-require 'chicago/etl/load_pipeline_stage_builder'
-require 'chicago/etl/pipeline'
-
+require 'chicago/etl/dataset_builder'
 require 'chicago/etl/stage'
 require 'chicago/etl/stage_builder'
+require 'chicago/etl/dataset_batch_stage'
+require 'chicago/etl/schema_sinks_and_transformations_builder'
+require 'chicago/etl/pipeline'
 
 # Sequel Extensions
 require 'chicago/etl/sequel/filter_to_etl_batch'

data/spec/etl/define_dimension_stage_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe "creating and running a dimension stage" do
+  let(:rows) { [{:some_field => "value"}] } 
+  let(:db) { double(:db).as_null_object }
+  let(:schema) { 
+    schema = Chicago::StarSchema.new
+
+    schema.define_dimension(:test) do
+      columns do
+        string :foo
+      end
+    end
+
+    schema
+  }
+
+  let(:pipeline) { Chicago::ETL::Pipeline.new(db, schema)}
+
+  it "glues the source, transformations, and sink correctly" do
+    pipeline.define_dimension_load(:test) do
+      dataset do
+        db.test_dataset_method
+      end
+    end
+
+    pipeline.stages.each do |stage|
+      stage.execute(double, true)
+    end
+  end
+
+  it "should set the inserted at time on the dimension"
+
+  it "truncates the dimension if specified"
+end

data/spec/etl/define_stage_spec.rb

@@ -39,26 +39,6 @@ describe "defining and executing a stage" do
     stage.sink(:another_stream).data.length.should == 0
   end
 
-  it "requires sinks" do
-    expect {
-      pipeline.define_stage(:test_stage) do
-        source do
-          db.test_dataset_method
-        end
-      end
-    }.to raise_error(ArgumentError)
-  end
-  
-  it "requires sources" do
-    expect {
-      pipeline.define_stage(:test_stage) do
-        sinks do
-          add Chicago::Flow::ArraySink.new(:test)
-        end
-      end
-    }.to raise_error(ArgumentError)
-  end
-
   it "glues the source, transformations, and sink correctly" do
     pipeline.define_stage(:test_stage) do
       source do
@@ -90,8 +70,8 @@ describe "defining and executing a stage" do
   it "allows the source to be filtered via a filter strategy" do
     etl_batch_double = double
     fake_source = []
+    fake_source.should_receive(:another_dataset_method).and_return([])
     
-    fake_source.should_receive(:another_dataset_method).and_return([])    
     pipeline.define_stage(:test_stage) do
       source do
         fake_source

data/spec/etl/pipeline_stage_builder_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Chicago::ETL::LoadPipelineStageBuilder do
+describe Chicago::ETL::SchemaSinksAndTransformationsBuilder do
   let(:dimension) { stub(:dimension).as_null_object }
   let(:db) { stub(:db).as_null_object }
   let(:sink_factory) { stub(:sink_factory).as_null_object }
@@ -34,6 +34,6 @@ describe Chicago::ETL::LoadPipelineStageBuilder do
       key_mapping :bar, :original_id
     end
 
-    stage.sink(:bar).should_not be_nil
+    stage[:sinks][:bar].should_not be_nil
   end
 end

data/spec/etl/stage_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Chicago::ETL::Stage do
+  it "requires a source" do
+    expect {
+      described_class.new(:test,
+                          :source => nil,
+                          :sinks => {:default => stub(:sink)})
+    }.to raise_error(ArgumentError)
+  end
+
+  it "requires sinks" do
+    expect {
+      described_class.new(:test,
+                          :source => stub(:source),
+                          :sinks => nil)
+    }.to raise_error(ArgumentError)
+  end
+
+  it "does not filter the dataset if re-extracting" do
+    stage = described_class.new(:test,
+                                :source => stub(:source),
+                                :sinks => {:default => stub(:sink)},
+                                :filter_strategy => lambda { fail })
+
+    stage.filtered_source(stub(:etl_batch), true)
+  end
+
+  it "opens sinks before writing and closes them afterwards" do
+    sink = mock(:sink)
+    sink.should_receive(:open)
+    sink.should_receive(:close)
+
+    stage = described_class.new(:test,
+                                :source => [],
+                                :sinks => {:default => sink})
+
+    stage.execute(stub(:etl_batch), true)
+  end
+end

data/spec/flow/mysql_integration_spec.rb

@@ -44,29 +44,33 @@ describe "Mysql -> Mysql through transformation chain" do
 
   it "copies data from source to destination" do
     TEST_DB[:source].multi_insert([{:foo => nil, :bin => :unhex.sql_function("1F")},
-                                  {:foo => "Hello", :bin => :unhex.sql_function("1F")}])
-
+                                   {:foo => "Hello", :bin => :unhex.sql_function("1F")}])
+    
     source = Chicago::Flow::DatasetSource.
       new(TEST_DB[:source].
           select(:id, :foo, :hex.sql_function(:bin).as(:bin)))
+
+    transformations = [dup_row.new(:onto => :other)]
+
     sink_1 = Chicago::Flow::MysqlFileSink.
       new(TEST_DB, :destination, [:id, :foo, :bin])
     sink_2 = Chicago::Flow::ArraySink.new([:id, :foo, :bin])
 
-    stage = Chicago::Flow::PipelineStage.
-      new(:transformations => [dup_row.new(:onto => :other)])
-
-    expect { stage.execute(source) }.to raise_error
-
-    stage.register_sink(:default, sink_1)
-    stage.register_sink(:other, sink_2)
+    stage = Chicago::ETL::Stage.new(:test, 
+                                    :source => source, 
+                                    :transformations => transformations, 
+                                    :sinks => {
+                                      :default => sink_1, 
+                                      :other => sink_2
+                                    })
 
-    stage.execute(source)
+    stage.execute(stub(:etl_batch), true)
 
     expected = [{:id => 1, :foo => nil, :bin => "1F"},
                 {:id => 2, :foo => "Hello", :bin => "1F"}]
 
     sink_2.data.should == expected
-    TEST_DB[:destination].select(:id, :foo, :hex.sql_function(:bin).as(:bin)).all.should == expected
+    TEST_DB[:destination].select(:id, :foo, :hex.sql_function(:bin).as(:bin)).
+      all.should == expected
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: chicago-etl
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 19
   prerelease: 
   segments: 
   - 0
   - 1
-  - 3
-  version: 0.1.3
+  - 4
+  version: 0.1.4
 platform: ruby
 authors: 
 - Roland Swingler
@@ -243,10 +243,11 @@ files:
 - lib/chicago/etl/core_extensions.rb
 - lib/chicago/etl/counter.rb
 - lib/chicago/etl/dataset_batch_stage.rb
+- lib/chicago/etl/dataset_builder.rb
 - lib/chicago/etl/key_builder.rb
 - lib/chicago/etl/load_dataset_builder.rb
-- lib/chicago/etl/load_pipeline_stage_builder.rb
 - lib/chicago/etl/pipeline.rb
+- lib/chicago/etl/schema_sinks_and_transformations_builder.rb
 - lib/chicago/etl/schema_table_sink_factory.rb
 - lib/chicago/etl/screens/column_screen.rb
 - lib/chicago/etl/screens/invalid_element.rb
@@ -273,7 +274,6 @@ files:
 - lib/chicago/flow/mysql_file_sink.rb
 - lib/chicago/flow/null_sink.rb
 - lib/chicago/flow/pipeline_endpoint.rb
-- lib/chicago/flow/pipeline_stage.rb
 - lib/chicago/flow/sink.rb
 - lib/chicago/flow/transformation.rb
 - lib/chicago/flow/transformation_chain.rb
@@ -281,7 +281,7 @@ files:
 - spec/etl/batch_spec.rb
 - spec/etl/core_extensions_spec.rb
 - spec/etl/counter_spec.rb
-- spec/etl/dataset_batch_stage_spec.rb
+- spec/etl/define_dimension_stage_spec.rb
 - spec/etl/define_stage_spec.rb
 - spec/etl/etl_batch_id_dataset_filter.rb
 - spec/etl/key_builder_spec.rb
@@ -293,6 +293,7 @@ files:
 - spec/etl/screens/out_of_bounds_spec.rb
 - spec/etl/sequel/dependant_tables_spec.rb
 - spec/etl/sequel/filter_to_etl_batch_spec.rb
+- spec/etl/stage_spec.rb
 - spec/etl/table_builder_spec.rb
 - spec/etl/task_spec.rb
 - spec/etl/transformations/deduplicate_rows_spec.rb
@@ -306,7 +307,6 @@ files:
 - spec/flow/mysql_file_serializer_spec.rb
 - spec/flow/mysql_file_sink_spec.rb
 - spec/flow/mysql_integration_spec.rb
-- spec/flow/pipeline_stage_spec.rb
 - spec/flow/transformation_chain_spec.rb
 - spec/flow/transformation_spec.rb
 - spec/spec_helper.rb

data/lib/chicago/flow/pipeline_stage.rb

@@ -1,68 +0,0 @@
-module Chicago
-  module Flow
-    # Co-ordinates iterating over rows provided by a source, passing
-    # them through a transformation chain before writing them to
-    # sink(s).
-    #
-    # @api public
-    class PipelineStage
-      attr_reader :transformation_chain
-      
-      def initialize(options={})
-        @sinks  = options[:sinks] || {}
-        @transformations = options[:transformations] || []
-        @error_handler = options[:error_handler] || RaisingErrorHandler.new
-        @transformation_chain = TransformationChain.new(*@transformations)
-      end
-
-      # Returns the named sink, if it exists
-      def sink(name)
-        @sinks[name.to_sym]
-      end
-
-      def sinks
-        @sinks.values
-      end
-
-      def register_sink(name, sink)
-        @sinks[name.to_sym] = sink
-        self
-      end
-      
-      def validate_pipeline
-        unless unregistered_sinks.empty?
-          @error_handler.unregistered_sinks(unregistered_sinks)
-        end
-      end
-      
-      def execute(source)
-        validate_pipeline
-        sinks.each(&:open)
-        pipe_rows_to_sinks_from(source)
-        sinks.each(&:close)
-      end
-
-      def required_sinks
-        transformation_chain.output_streams | [:default]
-      end
-
-      def unregistered_sinks
-        required_sinks - @sinks.keys
-      end
-      
-      private
-      
-      def pipe_rows_to_sinks_from(source)
-        source.each do |row|
-          transformation_chain.process(row).each {|row| process_row(row) }
-        end
-        transformation_chain.flush.each {|row| process_row(row) }
-      end
-
-      def process_row(row)
-        stream = row.delete(:_stream) || :default
-        @sinks[stream] << row
-      end
-    end
-  end
-end

data/spec/etl/dataset_batch_stage_spec.rb

@@ -1,55 +0,0 @@
-require 'spec_helper'
-
-describe Chicago::ETL::DatasetBatchStage do
-  let(:pipeline_stage) { mock(:pipeline_stage).as_null_object }
-  let(:dataset) { mock(:dataset).as_null_object }
-  let(:stage) { described_class.new(:foo, dataset, pipeline_stage) }
-  let(:etl_batch) { stub(:etl_batch) }
-
-  it "has a name" do
-    stage.name.should == :foo
-  end
-
-  it "should set the inserted at time on the default sink" do
-    sink = Chicago::Flow::ArraySink.new(:foo)
-    pipeline_stage.stub(:sink).with(:default).and_return(sink)
-    stage.pipeline_stage.should == pipeline_stage
-
-    sink.constant_values[:_inserted_at].should_not be_nil
-  end
-
-  it "filters the dataset to the batch" do
-    dataset.should_recieve(:filter_to_etl_batch).with(etl_batch)
-    stage.source(etl_batch)
-  end
-
-  it "does not filter the dataset if re-extracting" do
-    dataset.should_not_recieve(:filter_to_etl_batch)
-    stage.source(etl_batch, true)
-  end
-
-  it "can filter via a custom strategy" do
-    dataset.should_not_recieve(:filter_to_etl_batch)
-
-    filter_strategy = lambda {|ds, batch| ds }
-    described_class.new(:foo, dataset, pipeline_stage, :filter_strategy => filter_strategy).
-      source(etl_batch)
-  end
-
-  it "executes the pipeline stage using a DatasetSource" do
-    pipeline_stage.should_receive(:execute).
-      with(kind_of(Chicago::Flow::DatasetSource))
-    stage.execute(etl_batch, true)
-  end
-
-  it "truncates any sinks if truncate_pre_load has been set" do
-    stage = described_class.new(:foo, dataset, pipeline_stage,
-                                :truncate_pre_load => true)
-
-    sink = Chicago::Flow::ArraySink.new(:output)
-    sink << {:foo => "foo"}
-    pipeline_stage.stub(:sinks).and_return([sink])
-    stage.execute(etl_batch)
-    sink.data.should == []
-  end
-end

data/spec/flow/pipeline_stage_spec.rb

@@ -1,89 +0,0 @@
-require 'spec_helper'
-
-describe Chicago::Flow::PipelineStage do
-  let(:transform) {
-    Class.new(Chicago::Flow::Transformation) {
-      def process_row(row)
-        row[:a] += 1
-        row
-      end
-    }
-  }
-  
-  let(:add_error) {
-    Class.new(Chicago::Flow::Transformation) {
-      # add_output_stream :error
-      def output_streams
-        [:default, :error]
-      end
-      
-      def process_row(row)
-        [row, {Chicago::Flow::STREAM => :error, :message => "error"}]
-      end
-    }
-  }
-
-  let(:sink) { Chicago::Flow::ArraySink.new(:test) }
-  let(:source) { Chicago::Flow::ArraySource.new([{:a => 1}]) }
-
-  it "returns all sinks" do
-    stage = described_class.new.register_sink(:default, sink)
-    stage.sinks.should == [sink]
-  end
-
-  it "returns a sink by name" do
-    stage = described_class.new.register_sink(:default, sink)
-    stage.sink(:default).should == sink
-  end
-
-  it "reads from source to sink" do
-    pipeline = described_class.new.register_sink(:default, sink)
-    pipeline.execute(source)
-    sink.data.should == [{:a => 1}]
-  end
-
-  it "passes rows through transforms" do
-    pipeline = described_class.new(:transformations => [transform.new]).
-      register_sink(:default, sink)
-                                   
-    pipeline.execute(source)
-    sink.data.should == [{:a => 2}]
-  end
-
-  it "writes rows to the appropriate sink for their stream, and strips the stream tag" do
-    error_sink = Chicago::Flow::ArraySink.new(:test)
-
-    pipeline = described_class.new(:transformations => [add_error.new]).
-      register_sink(:default, sink).
-      register_sink(:error, error_sink)
-
-    pipeline.execute(source)
-    sink.data.should == [{:a => 1}]
-    error_sink.data.should == [{:message => "error"}]
-  end
-
-  it "calls an error handler if sinks are not registered" do
-    error_handler = mock(:error_handler)
-    error_handler.should_receive(:unregistered_sinks).
-      with([:default, :error])
-
-    pipeline = described_class.new(:transformations => [add_error.new],
-                                   :error_handler => error_handler)
-
-    pipeline.validate_pipeline
-  end
-
-  it "by default raises an exception if the pipeline is not valid when executed" do
-    pipeline = described_class.new(:transformations => [add_error.new])
-    expect { pipeline.execute(source) }.to raise_error(Chicago::Flow::Error)
-  end
-
-  it "opens sinks before writing and closes them afterwards" do
-    sink = mock(:sink)
-    pipeline = described_class.new.register_sink(:default, sink)
-    sink.should_receive(:open)
-    sink.stub(:<<)
-    sink.should_receive(:close)
-    pipeline.execute(source)
-  end
-end