chicago-etl 0.2.4 → 0.2.5

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MTdkYTQ0NmIxNGQ0ODU2NTAzMWU4NThjYzhjZWMwOTVjNmU4NjRhNg==
+  data.tar.gz: !binary |-
+    NDA3N2Y0Y2Y3ZTk1ZGQ2Mzg0NDdkMjA4YzRjODMwMDlmZTRjOWZkMw==
+SHA512:
+  metadata.gz: !binary |-
+    MTJiYzg5MGFlMjcwNTBlOTc5N2RkOGM3ZmVmMzIwNTMyN2FmYWQ2ODgwMmYz
+    YTI2MGYwZGQ5ZGNkZWU0MzNlOWNmNTlhODg0NTRkNzU5NDhlYTZjNzcxOWRi
+    ODE3NGNiNWZjMjg3NzJkNzc4OWRhYThlOWUwMjBkNzkzMmYwMWQ=
+  data.tar.gz: !binary |-
+    MDg3ZDQ0ZjhjZTYyZjMzYmFhNWUwOWJlNTAzMzBkOTdjNGFlMzU2NTZkMzA5
+    ZjcwNDBmYTg0YTUyZDE5MThhY2IxYzQwYzM1NzE1MzcwYWEyM2FhYTllYWU3
+    OTkwZmJlMDEwMTQzNjhjMmNjMTAxYWRjYmY3ZjNjZGY5ZDIwYjg=

data/Gemfile

@@ -13,7 +13,6 @@ group :development do
   gem "yard"
   gem "flog"
   gem "simplecov", :platforms => :mri_19, :require => false
-  gem "ZenTest"
   gem "mysql2"
   gem "bundler", "~> 1"
   gem "jeweler"

data/VERSION

@@ -1 +1 @@
-0.2.4
+0.2.5

data/chicago-etl.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: chicago-etl 0.2.5 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.2.4"
+  s.version = "0.2.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Roland Swingler"]
-  s.date = "2014-01-07"
+  s.date = "2014-08-07"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [
@@ -45,6 +47,7 @@ Gem::Specification.new do |s|
     "lib/chicago/etl/null_sink.rb",
     "lib/chicago/etl/pipeline.rb",
     "lib/chicago/etl/pipeline_endpoint.rb",
+    "lib/chicago/etl/row_transformation_stage.rb",
     "lib/chicago/etl/schema_sinks_and_transformations_builder.rb",
     "lib/chicago/etl/schema_table_sink_factory.rb",
     "lib/chicago/etl/schema_table_stage_builder.rb",
@@ -85,6 +88,7 @@ Gem::Specification.new do |s|
     "spec/etl/mysql_file_sink_spec.rb",
     "spec/etl/mysql_integration_spec.rb",
     "spec/etl/pipeline_stage_builder_spec.rb",
+    "spec/etl/row_transformation_stage_spec.rb",
     "spec/etl/schema_table_sink_factory_spec.rb",
     "spec/etl/screens/invalid_element_spec.rb",
     "spec/etl/screens/missing_value_spec.rb",
@@ -92,7 +96,6 @@ Gem::Specification.new do |s|
     "spec/etl/sequel/dependant_tables_spec.rb",
     "spec/etl/sequel/filter_to_etl_batch_spec.rb",
     "spec/etl/stage_name_spec.rb",
-    "spec/etl/stage_spec.rb",
     "spec/etl/table_builder_spec.rb",
     "spec/etl/task_spec.rb",
     "spec/etl/transformation_chain_spec.rb",
@@ -105,12 +108,11 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/notonthehighstreet/chicago-etl"
   s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.25"
+  s.rubygems_version = "2.2.1"
   s.summary = "Chicago ETL"
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<chicagowarehouse>, [">= 0.4.6", "~> 0.4"])
@@ -122,7 +124,6 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<yard>, [">= 0"])
       s.add_development_dependency(%q<flog>, [">= 0"])
       s.add_development_dependency(%q<simplecov>, [">= 0"])
-      s.add_development_dependency(%q<ZenTest>, [">= 0"])
       s.add_development_dependency(%q<mysql2>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
@@ -136,7 +137,6 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<yard>, [">= 0"])
       s.add_dependency(%q<flog>, [">= 0"])
       s.add_dependency(%q<simplecov>, [">= 0"])
-      s.add_dependency(%q<ZenTest>, [">= 0"])
       s.add_dependency(%q<mysql2>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1"])
       s.add_dependency(%q<jeweler>, [">= 0"])
@@ -151,7 +151,6 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<yard>, [">= 0"])
     s.add_dependency(%q<flog>, [">= 0"])
     s.add_dependency(%q<simplecov>, [">= 0"])
-    s.add_dependency(%q<ZenTest>, [">= 0"])
     s.add_dependency(%q<mysql2>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1"])
     s.add_dependency(%q<jeweler>, [">= 0"])

data/lib/chicago/etl.rb

@@ -28,6 +28,7 @@ require 'chicago/etl/transformations'
 require 'chicago/etl/load_dataset_builder'
 require 'chicago/etl/dataset_builder'
 require 'chicago/etl/stage'
+require 'chicago/etl/row_transformation_stage'
 require 'chicago/etl/stage_builder'
 require 'chicago/etl/schema_sinks_and_transformations_builder'
 require 'chicago/etl/pipeline'

data/lib/chicago/etl/pipeline.rb

@@ -6,40 +6,37 @@ module Chicago
       attr_reader :stages
 
       # Creates a pipeline for a Schema.
-      def initialize(db, schema)
+      def initialize(db, schema, &block)
         @schema, @db = schema, db
         @stages = Chicago::Schema::NamedElementCollection.new
+        @builder_class_factory = block || lambda {|name, options| StageBuilder }
       end
 
       # Defines a generic stage in the pipeline.
       def define_stage(*args, &block)
         options = args.last.kind_of?(Hash) ? args.pop : {}
 
-        name = StageName.new(args)
-        
-        if name =~ [:load, :dimensions]
-          @stages << build_dimension_load_stage(name, options, &block)
-        elsif name =~ [:load, :facts]
-          @stages << build_fact_load_stage(name, options, &block)
+        if args.last.kind_of?(Stage)
+          stage = args.pop
+          name = StageName.new(args)
         else
-          @stages << StageBuilder.new(@db).build(name, &block)
+          name = StageName.new(args)
+          stage = build_stage(name, options, &block)
         end
-      end
 
-      def build_stage(name, schema_table, &block)
-        SchemaTableStageBuilder.new(@db, schema_table).build(name, &block)
-      end
+        stage.name = StageName.new(args)
 
-      private
+        @stages << stage
+      end
 
-      def build_dimension_load_stage(name, options, &block)
-        dimension_name = options[:dimension] || name.name
-        build_stage(name, @schema.dimension(dimension_name), &block)
+      def build_stage(name, options, &block)
+        builder(name, options).build(name, options, &block)
       end
 
-      def build_fact_load_stage(name, options, &block)
-        fact_name = options[:fact] || name.name
-        build_stage(name, @schema.fact(fact_name), &block)
+      private
+      
+      def builder(name, options)
+        @builder_class_factory.call(name, options).new(@db, @schema)
       end
     end
   end

data/lib/chicago/etl/row_transformation_stage.rb

@@ -0,0 +1,78 @@
+module Chicago
+  module ETL
+    # A Stage that passes source rows through a transformation chain.
+    #
+    # All rows are read into Ruby and then written to sinks after
+    # passing through 0 or more Transformations.
+    class RowTransformationStage < Stage
+      # Returns the source for this stage.
+      attr_reader :source
+      
+      def initialize(options={})
+        super
+        @source = options[:source]
+        @sinks = options[:sinks]
+        @transformations = options[:transformations] || []
+        @filter_strategy = options[:filter_strategy] || lambda {|s, _| s }
+
+        validate_arguments
+      end
+            
+      # Executes this stage in the context of an ETL::Batch
+      def perform_execution(etl_batch)
+        transform_and_load filtered_source(etl_batch)
+      end
+      
+      # Returns the named sink, if it exists
+      def sink(name)
+        @sinks[name.to_sym]
+      end
+
+      def sinks
+        @sinks.values
+      end
+      
+      # @api private
+      def filtered_source(etl_batch)
+        filtered_dataset = etl_batch.reextracting? ? source : 
+          @filter_strategy.call(source, etl_batch)
+
+        DatasetSource.new(filtered_dataset)
+      end
+
+      private
+
+      def transform_and_load(source)
+        sinks.each(&:open)
+        pipe_rows_to_sinks_from(source)
+        sinks.each(&:close)
+      end
+
+      def pipe_rows_to_sinks_from(source)
+        source.each do |row|
+          transformation_chain.process(row).each {|row| process_row(row) }
+        end
+        transformation_chain.flush.each {|row| process_row(row) }
+      end
+
+      def transformation_chain
+        @transformation_chain ||= TransformationChain.new(*@transformations)
+      end
+
+      def process_row(row)
+        stream = row.delete(:_stream) || :default
+        @sinks[stream] << row
+      end
+
+      def validate_arguments
+        if @source.nil?
+          raise ArgumentError, "Stage #{@name} requires a source"
+        end
+
+        if @sinks.blank?
+          raise ArgumentError, "Stage #{@name} requires at least one sink"
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/schema_table_stage_builder.rb

@@ -7,11 +7,11 @@ module Chicago
     # Clients shouldn't need to instantiate this directly, but instead
     # call the protected methods in the context of defining a Pipeline
     class SchemaTableStageBuilder < StageBuilder
-      # @api private
-      def initialize(db, schema_table)
-        super(db)
+      def build(name, options, &block)
         @wrapped_builder = SchemaSinksAndTransformationsBuilder.
-          new(@db, schema_table)
+          new(@db, schema_table(name, options))
+
+        super
       end
 
       protected
@@ -47,5 +47,19 @@ module Chicago
         }
       end
     end
+
+    class LoadDimensionStageBuilder < SchemaTableStageBuilder
+      def schema_table(name, options)
+        dimension_name = options[:dimension] || name.name
+        @schema.dimension(dimension_name)
+      end
+    end
+
+    class LoadFactStageBuilder < SchemaTableStageBuilder
+      def schema_table(name, options)
+        fact_name = options[:dimension] || name.name
+        @schema.fact(fact_name)
+      end
+    end
   end
 end

data/lib/chicago/etl/stage.rb

@@ -1,30 +1,24 @@
 module Chicago
   module ETL
-    # A Stage in the ETL pipeline.
+    # A Stage in the ETL Pipeline.
     #
-    # A Stage wires together a Source, 0 or more Transformations and 1
-    # or more Sinks.
+    # Stage subclasses vary in how they perform their execution - some
+    # stages may pipe rows from a source to sinks, others may perform
+    # direct in-database updates.
+    #
+    # @abstract
     class Stage
-      # Returns the source for this stage.
-      attr_reader :source
-      
-      # Returns the name of this stage.
-      attr_reader :name
+      # The name of this stage.
+      attr_accessor :name
 
-      def initialize(name, options={})
-        @name = name
-        @source = options[:source]
-        @sinks = options[:sinks]
-        @transformations = options[:transformations] || []
-        @filter_strategy = options[:filter_strategy] || lambda {|s, _| s }
-        @pre_execution_strategies = options[:pre_execution_strategies] || []
+      def initialize(options={})
         @executable = options.has_key?(:executable) ? options[:executable] : true
-
-        validate_arguments
+        @pre_execution_strategies = options[:pre_execution_strategies] || []
       end
-      
+
       # Returns the unqualified name of this stage.
       def task_name
+        raise "This Stage has not been bound to a name" if @name.nil?
         name.name
       end
       
@@ -32,28 +26,25 @@ module Chicago
       def executable?
         @executable
       end
-      
-      # Executes this stage in the context of an ETL::Batch
+
+      # Executes this stage in the context of an ETL::Batch.
+      #
+      # This should not be overridden by subclasses; perform_execution
+      # should be changed instead.
       def execute(etl_batch)
         prepare_stage(etl_batch)
-        transform_and_load filtered_source(etl_batch)
-      end
-      
-      # Returns the named sink, if it exists
-      def sink(name)
-        @sinks[name.to_sym]
+        perform_execution(etl_batch)
       end
 
-      def sinks
-        @sinks.values
-      end
-      
-      # @api private
-      def filtered_source(etl_batch)
-        filtered_dataset = etl_batch.reextracting? ? source : 
-          @filter_strategy.call(source, etl_batch)
+      protected
 
-        DatasetSource.new(filtered_dataset)
+      # Does the actual work involved in executing this stage.
+      #
+      # This should be overridden by subclasses.
+      #
+      # @abstract
+      def perform_execution(etl_batch)
+        raise "perform_execution method has not been overridden."
       end
 
       private
@@ -63,38 +54,6 @@ module Chicago
           strategy.call(self, etl_batch)
         end
       end
-
-      def transform_and_load(source)
-        sinks.each(&:open)
-        pipe_rows_to_sinks_from(source)
-        sinks.each(&:close)
-      end
-
-      def pipe_rows_to_sinks_from(source)
-        source.each do |row|
-          transformation_chain.process(row).each {|row| process_row(row) }
-        end
-        transformation_chain.flush.each {|row| process_row(row) }
-      end
-
-      def transformation_chain
-        @transformation_chain ||= TransformationChain.new(*@transformations)
-      end
-
-      def process_row(row)
-        stream = row.delete(:_stream) || :default
-        @sinks[stream] << row
-      end
-
-      def validate_arguments
-        if @source.nil?
-          raise ArgumentError, "Stage #{@name} requires a source"
-        end
-
-        if @sinks.blank?
-          raise ArgumentError, "Stage #{@name} requires at least one sink"
-        end
-      end
     end
   end
 end

data/lib/chicago/etl/stage_builder.rb

@@ -1,24 +1,25 @@
 module Chicago
   module ETL
     class StageBuilder
-      def initialize(db)
+      def initialize(db, schema)
         @db = db
+        @schema = schema
       end
 
-      def build(name, &block)
+      def build(name, options, &block)
         @pre_execution_strategies = []
         @executable = true
 
         instance_eval &block
         set_default_stage_values
 
-        Stage.new(name,
-                  :source => @dataset, 
-                  :sinks => @sinks, 
-                  :transformations => @transformations, 
-                  :filter_strategy => @filter_strategy,
-                  :pre_execution_strategies => @pre_execution_strategies,
-                  :executable => @executable)
+        RowTransformationStage.
+          new(:source => @dataset, 
+              :sinks => @sinks, 
+              :transformations => @transformations, 
+              :filter_strategy => @filter_strategy,
+              :pre_execution_strategies => @pre_execution_strategies,
+              :executable => @executable)
       end
 
       protected

data/spec/etl/define_dimension_stage_spec.rb

@@ -15,8 +15,12 @@ describe "creating and running a dimension stage" do
     schema
   }
 
-  let(:pipeline) { Chicago::ETL::Pipeline.new(db, schema)}
-
+  let(:pipeline) { 
+    Chicago::ETL::Pipeline.new(db, schema) do |name, options|
+      Chicago::ETL::LoadDimensionStageBuilder
+    end
+  }
+  
   it "glues the source, transformations, and sink correctly" do
     pipeline.define_stage(:load, :dimensions, :test) do
       source do

data/spec/etl/mysql_integration_spec.rb

@@ -56,13 +56,13 @@ describe "Mysql -> Mysql through transformation chain" do
       new(TEST_DB, :destination, [:id, :foo, :bin])
     sink_2 = Chicago::ETL::ArraySink.new([:id, :foo, :bin])
 
-    stage = Chicago::ETL::Stage.new(:test, 
-                                    :source => source, 
-                                    :transformations => transformations, 
-                                    :sinks => {
-                                      :default => sink_1, 
-                                      :other => sink_2
-                                    })
+    stage = Chicago::ETL::RowTransformationStage.
+      new(:source => source, 
+          :transformations => transformations, 
+          :sinks => {
+            :default => sink_1, 
+            :other => sink_2
+          })
 
     stage.execute(double(:etl_batch, :reextracting? => true))
 

data/spec/etl/{stage_spec.rb → row_transformation_stage_spec.rb}

@@ -1,27 +1,24 @@
 require 'spec_helper'
 
-describe Chicago::ETL::Stage do
+describe Chicago::ETL::RowTransformationStage do
   let(:etl_batch) { double(:etl_batch, :reextracting? => true) }
 
   it "requires a source" do
     expect {
-      described_class.new(:test,
-                          :source => nil,
+      described_class.new(:source => nil,
                           :sinks => {:default => double(:sink)})
     }.to raise_error(ArgumentError)
   end
 
   it "requires sinks" do
     expect {
-      described_class.new(:test,
-                          :source => double(:source),
+      described_class.new(:source => double(:source),
                           :sinks => nil)
     }.to raise_error(ArgumentError)
   end
 
   it "does not filter the dataset if re-extracting" do
-    stage = described_class.new(:test,
-                                :source => double(:source),
+    stage = described_class.new(:source => double(:source),
                                 :sinks => {:default => double(:sink)},
                                 :filter_strategy => lambda { fail })
 
@@ -33,8 +30,7 @@ describe Chicago::ETL::Stage do
     sink.should_receive(:open)
     sink.should_receive(:close)
 
-    stage = described_class.new(:test,
-                                :source => [],
+    stage = described_class.new(:source => [],
                                 :sinks => {:default => sink})
 
     stage.execute(etl_batch)

metadata

@@ -1,225 +1,197 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: chicago-etl
-version: !ruby/object:Gem::Version 
-  hash: 31
-  prerelease: 
-  segments: 
-  - 0
-  - 2
-  - 4
-  version: 0.2.4
+version: !ruby/object:Gem::Version
+  version: 0.2.5
 platform: ruby
-authors: 
+authors:
 - Roland Swingler
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2014-01-07 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        - 4
-        - 6
+date: 2014-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chicagowarehouse
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.4.6
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        - 4
-        version: "0.4"
-  requirement: *id001
-  type: :runtime
-  prerelease: false
-  name: chicagowarehouse
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id002
+      - !ruby/object:Gem::Version
+        version: '0.4'
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.6
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
   name: fastercsv
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 0
-        - 0
-        - 2
-        version: 0.0.2
-  requirement: *id003
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: sequel_load_data_infile
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id004
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
   name: sequel_fast_columns
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 2
-        version: "2"
-  requirement: *id005
-  type: :development
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: rspec
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id006
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
   name: timecop
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id007
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: yard
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id008
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: flog
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id009 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id009
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: simplecov
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id010 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id010
-  type: :development
-  prerelease: false
-  name: ZenTest
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id011 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id011
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: mysql2
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id012 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 1
-        version: "1"
-  requirement: *id012
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: bundler
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id013 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id013
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
   name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ETL tools for Chicago
 email: roland.swingler@gmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.txt
 - README.rdoc
-files: 
+files:
 - .document
 - .rspec
 - Gemfile
@@ -248,6 +220,7 @@ files:
 - lib/chicago/etl/null_sink.rb
 - lib/chicago/etl/pipeline.rb
 - lib/chicago/etl/pipeline_endpoint.rb
+- lib/chicago/etl/row_transformation_stage.rb
 - lib/chicago/etl/schema_sinks_and_transformations_builder.rb
 - lib/chicago/etl/schema_table_sink_factory.rb
 - lib/chicago/etl/schema_table_stage_builder.rb
@@ -288,6 +261,7 @@ files:
 - spec/etl/mysql_file_sink_spec.rb
 - spec/etl/mysql_integration_spec.rb
 - spec/etl/pipeline_stage_builder_spec.rb
+- spec/etl/row_transformation_stage_spec.rb
 - spec/etl/schema_table_sink_factory_spec.rb
 - spec/etl/screens/invalid_element_spec.rb
 - spec/etl/screens/missing_value_spec.rb
@@ -295,7 +269,6 @@ files:
 - spec/etl/sequel/dependant_tables_spec.rb
 - spec/etl/sequel/filter_to_etl_batch_spec.rb
 - spec/etl/stage_name_spec.rb
-- spec/etl/stage_spec.rb
 - spec/etl/table_builder_spec.rb
 - spec/etl/task_spec.rb
 - spec/etl/transformation_chain_spec.rb
@@ -306,37 +279,28 @@ files:
 - spec/etl/transformations_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/notonthehighstreet/chicago-etl
-licenses: 
+licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.2.1
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Chicago ETL
 test_files: []
-
+has_rdoc: