chicago-etl 0.1.0 → 0.1.1

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/chicago-etl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
-  s.date = "2013-09-05"
+  s.date = "2013-11-07"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [
@@ -43,10 +43,13 @@ Gem::Specification.new do |s|
     "lib/chicago/etl/screens/out_of_bounds.rb",
     "lib/chicago/etl/sequel/dependant_tables.rb",
     "lib/chicago/etl/sequel/filter_to_etl_batch.rb",
+    "lib/chicago/etl/stage.rb",
+    "lib/chicago/etl/stage_builder.rb",
     "lib/chicago/etl/table_builder.rb",
     "lib/chicago/etl/task_invocation.rb",
     "lib/chicago/etl/tasks.rb",
     "lib/chicago/etl/transformations.rb",
+    "lib/chicago/etl/transformations/deduplicate_rows.rb",
     "lib/chicago/etl/transformations/uk_post_code.rb",
     "lib/chicago/etl/transformations/uk_post_code_field.rb",
     "lib/chicago/flow/array_sink.rb",
@@ -68,6 +71,7 @@ Gem::Specification.new do |s|
     "spec/etl/core_extensions_spec.rb",
     "spec/etl/counter_spec.rb",
     "spec/etl/dataset_batch_stage_spec.rb",
+    "spec/etl/define_stage_spec.rb",
     "spec/etl/etl_batch_id_dataset_filter.rb",
     "spec/etl/key_builder_spec.rb",
     "spec/etl/load_dataset_builder_spec.rb",
@@ -80,6 +84,7 @@ Gem::Specification.new do |s|
     "spec/etl/sequel/filter_to_etl_batch_spec.rb",
     "spec/etl/table_builder_spec.rb",
     "spec/etl/task_spec.rb",
+    "spec/etl/transformations/deduplicate_rows_spec.rb",
     "spec/etl/transformations/uk_post_code_field_spec.rb",
     "spec/etl/transformations/uk_post_code_spec.rb",
     "spec/etl/transformations_spec.rb",

data/lib/chicago/etl/batch.rb

@@ -31,27 +31,6 @@ module Chicago
         end
       end
 
-      # Deprecated.
-      #
-      # @deprecated Use perform_task instead
-      def load(task_name, &block)
-        perform_task(:load, task_name, &block)
-      end
-
-      # Deprecated.
-      #
-      # @deprecated Use perform_task instead
-      def transform(task_name, &block)
-        perform_task(:extract, task_name, &block)
-      end
-
-      # Deprecated.
-      #
-      # @deprecated Use perform_task instead
-      def extract(task_name, &block)
-        perform_task(:extract, task_name, &block)
-      end
-
       # Performs a named task if it hasn't already run successfully in
       # this batch.
       def perform_task(stage, task_name, &block)

data/lib/chicago/etl/pipeline.rb

@@ -8,11 +8,24 @@ module Chicago
       # Returns all defined fact load tasks
       attr_reader :load_facts
 
+      # Returns all the defined generic stages.
+      attr_reader :stages
+
       # Creates a pipeline for a Schema.
       def initialize(db, schema)
         @schema, @db = schema, db
         @load_dimensions = Chicago::Schema::NamedElementCollection.new
         @load_facts = Chicago::Schema::NamedElementCollection.new
+        @stages = Chicago::Schema::NamedElementCollection.new
+      end
+
+      # Defines a generic stage in the pipeline.
+      def define_stage(name, &block)
+        @stages << build_schemaless_stage(name, &block)
+      end
+
+      def build_schemaless_stage(name, &block)
+        StageBuilder.new(@db).build(name, &block)
       end
 
       # Defines a dimension load stage
@@ -73,6 +86,7 @@ module Chicago
 
       # Define elements of the pipeline. See LoadPipelineStageBuilder
       # for details.
+      # TODO: rename pipeline => transforms below this method
       def pipeline(&block)
         @pipeline_stage = LoadPipelineStageBuilder.new(@db, @schema_table).
           build(&block)
@@ -81,9 +95,11 @@ module Chicago
       # Defines the dataset, see DatasetBuilder .
       #
       # The block must return a Sequel::Dataset.
-      def dataset(&block)
+      # TODO: rename dataset => source below this method, make generic
+      def source(&block)
         @dataset = DatasetBuilder.new(@db).build(&block)
       end
+      alias :dataset :source
 
       # Define a custom filter strategy for filtering to an ETL batch.
       def filter_strategy(&block)

data/lib/chicago/etl/stage.rb

@@ -0,0 +1,76 @@
+module Chicago
+  module ETL
+    class Stage
+      attr_reader :name
+
+      def initialize(name, options={})
+        @name = name
+        @source = options.fetch(:source)
+        raise ArgumentError, "Stage #{name} requires a source" unless @source
+
+        @sinks = options.fetch(:sinks)
+        raise ArgumentError, "Stage #{name} requires at least one sink" if @sinks.empty?
+
+        @transformations = options.fetch(:transformations)
+        @transformation_chain = Chicago::Flow::TransformationChain.
+          new(*@transformations)
+
+        @filter_strategy = options[:filter_strategy] || 
+          lambda {|source, _| source }
+      end
+
+      def execute(etl_batch, reextract)
+        modified_source = reextract_and_filter_source(@source, etl_batch, reextract)
+        transform_and_load_from(modified_source)
+      end
+      
+      def transform_and_load_from(source)
+      end
+
+      def reextract_and_filter_source(source, etl_batch, reextract=false)
+        if reextract
+          filtered_dataset = source
+        else
+          filtered_dataset = @filter_strategy.call(source, etl_batch)
+        end
+        Chicago::Flow::DatasetSource.new(filtered_dataset)
+      end
+
+          attr_reader :transformation_chain
+      
+      # Returns the named sink, if it exists
+      def sink(name)
+        @sinks[name.to_sym]
+      end
+
+      def sinks
+        @sinks.values
+      end
+
+      def register_sink(name, sink)
+        @sinks[name.to_sym] = sink
+        self
+      end
+            
+      def transform_and_load_from(source)
+        sinks.each(&:open)
+        pipe_rows_to_sinks_from(source)
+        sinks.each(&:close)
+      end
+      
+      private
+      
+      def pipe_rows_to_sinks_from(source)
+        source.each do |row|
+          transformation_chain.process(row).each {|row| process_row(row) }
+        end
+        transformation_chain.flush.each {|row| process_row(row) }
+      end
+
+      def process_row(row)
+        stream = row.delete(:_stream) || :default
+        @sinks[stream] << row
+      end
+    end
+  end
+end

data/lib/chicago/etl/stage_builder.rb

@@ -0,0 +1,69 @@
+module Chicago
+  module ETL
+    class StageBuilder
+      attr_reader :sink_factory
+
+      def initialize(db)
+        @db = db
+      end
+
+      def build(name, &block)
+        @sinks = {}
+        @transformations = []
+
+        instance_eval &block
+        
+        Stage.new(name,
+                  :source => @dataset, 
+                  :sinks => @sinks, 
+                  :transformations => @transformations, 
+                  :filter_strategy => @filter_strategy)
+      end
+
+      def source(&block)
+        @dataset = DatasetBuilder.new(@db).build(&block)
+      end
+      
+      def transformations(klass=TransformationBuilder, &block)
+        @transformations = klass.new.build(&block)
+      end
+
+      def sinks(options={}, &block)
+        @sinks = SinkBuilder.new.build(&block)
+      end
+
+      # TODO: think of potentially better ways of dealig with this
+      # problem.
+      def filter_strategy(&block)
+        @filter_strategy = block
+      end
+
+      class TransformationBuilder
+        def build(&block)
+          @transformations = []
+          instance_eval(&block)
+          @transformations
+        end
+        
+        def add(transformation)
+          @transformations << transformation
+        end
+      end
+
+      class SinkBuilder
+        def build(&block)
+          @sinks = {}
+          instance_eval(&block)
+          @sinks
+        end
+
+        protected
+
+        def add(sink, options={})
+          stream = options[:stream] || :default
+          @sinks[stream] = sink
+        end
+      end
+    end
+  end
+end

data/lib/chicago/etl/transformations/deduplicate_rows.rb

@@ -0,0 +1,40 @@
+module Chicago
+  module ETL
+    class DeduplicateRows < Chicago::Flow::Transformation
+      def process_row(row)
+        if @working_row.nil?
+          @working_row = row
+          return
+        elsif same_row?(row)
+          @working_row = merge_rows(row)
+          return
+        else
+          assign_new_row_and_return_old_row(row)
+        end
+      end
+
+      def flush
+        @working_row.nil? ? [] : [@working_row]
+      end
+
+      protected
+
+      attr_reader :working_row
+
+      # This should be implemented by clients
+      def merge_rows(row)
+      end
+      
+      # This should be implemented by clients
+      def same_row?(row)
+      end
+
+      private
+
+      def assign_new_row_and_return_old_row(row)
+        row, @working_row = @working_row, row
+        row
+      end
+    end
+  end
+end

data/lib/chicago/etl.rb

@@ -29,6 +29,9 @@ require 'chicago/etl/dataset_batch_stage'
 require 'chicago/etl/load_pipeline_stage_builder'
 require 'chicago/etl/pipeline'
 
+require 'chicago/etl/stage'
+require 'chicago/etl/stage_builder'
+
 # Sequel Extensions
 require 'chicago/etl/sequel/filter_to_etl_batch'
 require 'chicago/etl/sequel/dependant_tables'
@@ -40,6 +43,7 @@ require 'chicago/etl/screens/invalid_element'
 require 'chicago/etl/screens/out_of_bounds'
 
 # Transformations
+require 'chicago/etl/transformations/deduplicate_rows'
 require 'chicago/etl/transformations/uk_post_code'
 require 'chicago/etl/transformations/uk_post_code_field'
 

data/spec/etl/define_stage_spec.rb

@@ -0,0 +1,114 @@
+require "spec_helper"
+
+class TestTransformation < Chicago::Flow::Transformation
+  def output_streams
+    [:another_stream]
+  end
+
+  def process_row(row)
+    [row, assign_stream({:some_field => "has an error value"}, :another_stream)]
+  end
+end
+
+describe "defining and executing a stage" do
+  let(:rows) { [{:some_field => "value"}] } 
+  let(:db) { double(:test_dataset_method => rows) }
+  let(:schema) { double }
+  let(:pipeline) { Chicago::ETL::Pipeline.new(db, schema)}
+
+  it "allows no tranformations" do
+    pipeline.define_stage(:test_stage) do
+      source do
+        db.test_dataset_method
+      end
+
+      sinks do
+        add Chicago::Flow::ArraySink.new(:test)
+        add Chicago::Flow::ArraySink.new(:test), :stream => :another_stream
+      end
+    end
+
+    pipeline.stages.each do |stage|
+      stage.execute(double, true)
+    end
+
+    stage = pipeline.stages.first
+    stage.sink(:default).data.length.should == 1
+    stage.sink(:default).data.first.should == {:some_field => "value"}
+
+    stage.sink(:another_stream).data.length.should == 0
+  end
+
+  it "requires sinks" do
+    expect {
+      pipeline.define_stage(:test_stage) do
+        source do
+          db.test_dataset_method
+        end
+      end
+    }.to raise_error(ArgumentError)
+  end
+  
+  it "requires sources" do
+    expect {
+      pipeline.define_stage(:test_stage) do
+        sinks do
+          add Chicago::Flow::ArraySink.new(:test)
+        end
+      end
+    }.to raise_error(ArgumentError)
+  end
+
+  it "glues the source, transformations, and sink correctly" do
+    pipeline.define_stage(:test_stage) do
+      source do
+        db.test_dataset_method
+      end
+
+      transformations do
+        add TestTransformation.new
+      end
+
+      sinks do
+        add Chicago::Flow::ArraySink.new(:test)
+        add Chicago::Flow::ArraySink.new(:test), :stream => :another_stream
+      end
+    end
+
+    pipeline.stages.each do |stage|
+      stage.execute(double, true)
+    end
+
+    stage = pipeline.stages.first
+    stage.sink(:default).data.length.should == 1
+    stage.sink(:default).data.first.should == {:some_field => "value"}
+
+    stage.sink(:another_stream).data.length.should == 1
+    stage.sink(:another_stream).data.first.should == {:some_field => "has an error value"}
+  end
+
+  it "allows the source to be filtered via a filter strategy" do
+    etl_batch_double = double
+    fake_source = []
+    
+    fake_source.should_receive(:another_dataset_method).and_return([])    
+    pipeline.define_stage(:test_stage) do
+      source do
+        fake_source
+      end
+
+      sinks do
+        add Chicago::Flow::ArraySink.new(:test)
+      end
+
+      filter_strategy do |source, etl_batch|
+        etl_batch.should == etl_batch_double
+        source.another_dataset_method
+      end
+    end
+    
+    pipeline.stages.each do |stage|
+      stage.execute(etl_batch_double, false)
+    end
+  end
+end

data/spec/etl/transformations/deduplicate_rows_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe Chicago::ETL::DeduplicateRows do
+  it "deduplicates rows" do
+    class TestTransform < described_class
+      def merge_rows(row)
+        working_row.merge(row)
+      end
+
+      def same_row?(row)
+        working_row[:id] == row[:id]
+      end
+    end
+
+    transform = TestTransform.new
+
+    transform.process({:id => 1, :foo => :bar}).should be_blank
+    transform.process({:id => 1, :bar => :baz}).should be_blank
+    transform.process({:id => 2, :foo => :quux}).should == {:id => 1, :foo => :bar, :bar => :baz}
+
+    transform.flush.should == [{:id => 2, :foo => :quux}]
+  end
+end

metadata

@@ -1,233 +1,248 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: chicago-etl
-version: !ruby/object:Gem::Version 
-  hash: 27
+version: !ruby/object:Gem::Version
+  version: 0.1.1
   prerelease: 
-  segments: 
-  - 0
-  - 1
-  - 0
-  version: 0.1.0
 platform: ruby
-authors: 
+authors:
 - Roland Swingler
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-09-05 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
+date: 2013-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chicagowarehouse
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        - 4
-        version: "0.4"
-  requirement: *id001
+      - !ruby/object:Gem::Version
+        version: '0.4'
   type: :runtime
   prerelease: false
-  name: chicagowarehouse
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id002
-  type: :runtime
-  prerelease: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
   name: fastercsv
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id003
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: sequel
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id004 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 0
-        - 0
-        - 2
-        version: 0.0.2
-  requirement: *id004
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: sequel_load_data_infile
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id005 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id005
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
   name: sequel_fast_columns
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id006 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 2
-        version: "2"
-  requirement: *id006
-  type: :development
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: rspec
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id007 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id007
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
   name: timecop
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id008 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id008
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: yard
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id009 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id009
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: flog
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id010 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id010
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: simplecov
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id011 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id011
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: ZenTest
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id012 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        hash: 45
-        segments: 
-        - 2
-        - 8
-        - 1
-        version: 2.8.1
-  requirement: *id012
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: mysql
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id013 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 1
-        version: "1"
-  requirement: *id013
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.1
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+- !ruby/object:Gem::Dependency
   name: bundler
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id014 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id014
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
   name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ETL tools for Chicago
 email: roland.swingler@gmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.txt
 - README.rdoc
-files: 
+files:
 - .document
 - .rspec
 - Gemfile
@@ -254,10 +269,13 @@ files:
 - lib/chicago/etl/screens/out_of_bounds.rb
 - lib/chicago/etl/sequel/dependant_tables.rb
 - lib/chicago/etl/sequel/filter_to_etl_batch.rb
+- lib/chicago/etl/stage.rb
+- lib/chicago/etl/stage_builder.rb
 - lib/chicago/etl/table_builder.rb
 - lib/chicago/etl/task_invocation.rb
 - lib/chicago/etl/tasks.rb
 - lib/chicago/etl/transformations.rb
+- lib/chicago/etl/transformations/deduplicate_rows.rb
 - lib/chicago/etl/transformations/uk_post_code.rb
 - lib/chicago/etl/transformations/uk_post_code_field.rb
 - lib/chicago/flow/array_sink.rb
@@ -279,6 +297,7 @@ files:
 - spec/etl/core_extensions_spec.rb
 - spec/etl/counter_spec.rb
 - spec/etl/dataset_batch_stage_spec.rb
+- spec/etl/define_stage_spec.rb
 - spec/etl/etl_batch_id_dataset_filter.rb
 - spec/etl/key_builder_spec.rb
 - spec/etl/load_dataset_builder_spec.rb
@@ -291,6 +310,7 @@ files:
 - spec/etl/sequel/filter_to_etl_batch_spec.rb
 - spec/etl/table_builder_spec.rb
 - spec/etl/task_spec.rb
+- spec/etl/transformations/deduplicate_rows_spec.rb
 - spec/etl/transformations/uk_post_code_field_spec.rb
 - spec/etl/transformations/uk_post_code_spec.rb
 - spec/etl/transformations_spec.rb
@@ -306,37 +326,31 @@ files:
 - spec/flow/transformation_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/notonthehighstreet/chicago-etl
-licenses: 
+licenses:
 - MIT
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: -2054734000096616506
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Chicago ETL
 test_files: []
-