chicago-etl 0.2.1 → 0.2.2

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.2.2

data/chicago-etl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.2.1"
+  s.version = "0.2.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
-  s.date = "2013-11-19"
+  s.date = "2013-11-26"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [
@@ -33,7 +33,6 @@ Gem::Specification.new do |s|
     "lib/chicago/etl/batch.rb",
     "lib/chicago/etl/core_extensions.rb",
     "lib/chicago/etl/counter.rb",
-    "lib/chicago/etl/dataset_batch_stage.rb",
     "lib/chicago/etl/dataset_builder.rb",
     "lib/chicago/etl/dataset_source.rb",
     "lib/chicago/etl/errors.rb",
@@ -48,6 +47,7 @@ Gem::Specification.new do |s|
     "lib/chicago/etl/pipeline_endpoint.rb",
     "lib/chicago/etl/schema_sinks_and_transformations_builder.rb",
     "lib/chicago/etl/schema_table_sink_factory.rb",
+    "lib/chicago/etl/schema_table_stage_builder.rb",
     "lib/chicago/etl/screens/column_screen.rb",
     "lib/chicago/etl/screens/invalid_element.rb",
     "lib/chicago/etl/screens/missing_value.rb",
@@ -57,6 +57,7 @@ Gem::Specification.new do |s|
     "lib/chicago/etl/sink.rb",
     "lib/chicago/etl/stage.rb",
     "lib/chicago/etl/stage_builder.rb",
+    "lib/chicago/etl/stage_name.rb",
     "lib/chicago/etl/table_builder.rb",
     "lib/chicago/etl/task_invocation.rb",
     "lib/chicago/etl/tasks.rb",
@@ -76,6 +77,7 @@ Gem::Specification.new do |s|
     "spec/etl/define_dimension_stage_spec.rb",
     "spec/etl/define_stage_spec.rb",
     "spec/etl/etl_batch_id_dataset_filter.rb",
+    "spec/etl/execution_wrapper_spec.rb",
     "spec/etl/filter_spec.rb",
     "spec/etl/key_builder_spec.rb",
     "spec/etl/load_dataset_builder_spec.rb",
@@ -89,6 +91,7 @@ Gem::Specification.new do |s|
     "spec/etl/screens/out_of_bounds_spec.rb",
     "spec/etl/sequel/dependant_tables_spec.rb",
     "spec/etl/sequel/filter_to_etl_batch_spec.rb",
+    "spec/etl/stage_name_spec.rb",
     "spec/etl/stage_spec.rb",
     "spec/etl/table_builder_spec.rb",
     "spec/etl/task_spec.rb",

data/lib/chicago/etl/pipeline.rb

@@ -2,116 +2,44 @@ module Chicago
   module ETL
     # An ETL pipeline.
     class Pipeline
-      # Returns all defined dimension load tasks
-      attr_reader :load_dimensions
-
-      # Returns all defined fact load tasks
-      attr_reader :load_facts
-
-      # Returns all the defined generic stages.
+      # Returns all the defined stages.
       attr_reader :stages
 
       # Creates a pipeline for a Schema.
       def initialize(db, schema)
         @schema, @db = schema, db
-        @load_dimensions = Chicago::Schema::NamedElementCollection.new
-        @load_facts = Chicago::Schema::NamedElementCollection.new
         @stages = Chicago::Schema::NamedElementCollection.new
       end
 
       # Defines a generic stage in the pipeline.
-      def define_stage(name, &block)
-        @stages << build_schemaless_stage(name, &block)
-      end
-
-      def build_schemaless_stage(name, &block)
-        StageBuilder.new(@db).build(name, &block)
-      end
-
-      # Defines a dimension load stage
-      def define_dimension_load(name, options={}, &block)
-        dimension_name = options[:dimension] || name
-        @load_dimensions << build_stage(name, 
-                                        @schema.dimension(dimension_name),
-                                        &block)
-      end
-
-      # Defines a fact load stage
-      def define_fact_load(name, options={}, &block)
-        fact_name = options[:fact] || name
-        @load_facts << build_stage(name, @schema.fact(fact_name), &block)
-      end
-
-      # Builds a stage, but does not define it.
-      def build_stage(name, schema_table, &block)
-        DatasetBatchStageBuilder.new(@db, schema_table).build(name, &block)
-      end
-    end
-
-    # Provides DSL methods for building a DataSetBatchStage.
-    #
-    # Clients shouldn't need to instantiate this directly, but instead
-    # call the protected methods in the context of defining a Pipeline
-    class DatasetBatchStageBuilder
-      # @api private
-      def initialize(db, schema_table)
-        @db, @schema_table = db, schema_table
-      end
-
-      # @api private
-      def build(name, &block)
-        instance_eval &block
-        unless defined? @sinks_and_transformations
-          pipeline do
-          end
-        end
+      def define_stage(*args, &block)
+        options = args.last.kind_of?(Hash) ? args.pop : {}
 
-        @filter_strategy ||= lambda {|dataset, etl_batch| 
-          dataset.filter_to_etl_batch(etl_batch)
-        }
+        name = StageName.new(args)
         
-        DatasetBatchStage.new(name,
-                              :source => @dataset, 
-                              :transformations => @sinks_and_transformations[:transformations],
-                              :sinks => @sinks_and_transformations[:sinks],
-                              :filter_strategy => @filter_strategy,
-                              :truncate_pre_load => @truncate_pre_load)
-      end
-
-      protected
-      
-      # Specifies that the sinks should be truncated before loading
-      # data.
-      def truncate_pre_load
-        @truncate_pre_load = true
+        if name =~ [:load, :dimensions]
+          @stages << build_dimension_load_stage(name, options, &block)
+        elsif name =~ [:load, :facts]
+          @stages << build_fact_load_stage(name, options, &block)
+        else
+          @stages << StageBuilder.new(@db).build(name, &block)
+        end
       end
 
-      # Specifies that the dataset should never be filtered to the ETL
-      # batch - i.e. it should behave as if reextract was always true
-      def full_reload
-        @filter_strategy = lambda {|dataset, etl_batch| dataset }
+      def build_stage(name, schema_table, &block)
+        SchemaTableStageBuilder.new(@db, schema_table).build(name, &block)
       end
 
-      # Define elements of the pipeline. See LoadPipelineStageBuilder
-      # for details.
-      # TODO: rename pipeline => transforms below this method
-      def pipeline(&block)
-        @sinks_and_transformations = SchemaSinksAndTransformationsBuilder.new(@db, @schema_table).
-          build(&block)
-      end
+      private
 
-      # Defines the dataset, see DatasetBuilder .
-      #
-      # The block must return a Sequel::Dataset.
-      # TODO: rename dataset => source below this method, make generic
-      def source(&block)
-        @dataset = DatasetBuilder.new(@db).build(&block)
+      def build_dimension_load_stage(name, options, &block)
+        dimension_name = options[:dimension] || name.name
+        build_stage(name, @schema.dimension(dimension_name), &block)
       end
-      alias :dataset :source
 
-      # Define a custom filter strategy for filtering to an ETL batch.
-      def filter_strategy(&block)
-        @filter_strategy = block
+      def build_fact_load_stage(name, options, &block)
+        fact_name = options[:fact] || name.name
+        build_stage(name, @schema.fact(fact_name), &block)
       end
     end
   end

data/lib/chicago/etl/schema_table_stage_builder.rb

@@ -0,0 +1,48 @@
+require 'chicago/etl/stage_builder'
+
+module Chicago
+  module ETL
+    # Provides DSL methods for building a DataSetBatchStage.
+    #
+    # Clients shouldn't need to instantiate this directly, but instead
+    # call the protected methods in the context of defining a Pipeline
+    class SchemaTableStageBuilder < StageBuilder
+      # @api private
+      def initialize(db, schema_table)
+        super(db)
+        @wrapped_builder = SchemaSinksAndTransformationsBuilder.
+          new(@db, schema_table)
+      end
+
+      protected
+      
+      # Define elements of the pipeline. See LoadPipelineStageBuilder
+      # for details.
+      #
+      # @deprecated
+      def pipeline(&block)
+        sinks_and_transformations = @wrapped_builder.build(&block)
+        @sinks = sinks_and_transformations[:sinks]
+        @transformations = sinks_and_transformations[:transformations] || []
+      end
+
+      # @api private
+      def set_default_stage_values
+        unless defined? @sinks
+          pipeline do
+          end
+        end
+
+        @pre_execution_strategies << lambda {|stage, etl_batch, reextract|
+          stage.sink(:error).truncate if reextract && stage.sink(:error)
+          stage.sink(:default).
+            set_constant_values(:_inserted_at => Time.now)
+        }
+
+        @filter_strategy ||= lambda {|dataset, etl_batch| 
+          dataset.filter_to_etl_batch(etl_batch)
+        }
+      end
+    end
+  end
+end

data/lib/chicago/etl/stage.rb

@@ -16,20 +16,26 @@ module Chicago
         @source = options[:source]
         @sinks = options[:sinks]
         @transformations = options[:transformations] || []
-        @filter_strategy = options[:filter_strategy] || 
-          lambda {|source, _| source }
-        @truncate_pre_load = !!options[:truncate_pre_load]
+        @filter_strategy = options[:filter_strategy] || lambda {|s, _| s }
+        @pre_execution_strategies = options[:pre_execution_strategies] || []
+        @executable = options.has_key?(:executable) ? options[:executable] : true
 
         validate_arguments
       end
       
-      # Returns true if the sinks should be truncated pre-load.
-      def truncate_pre_load?
-        @truncate_pre_load
+      # Returns the unqualified name of this stage.
+      def task_name
+        name.name
       end
-
+      
+      # Returns true if this stage should be executed.
+      def executable?
+        @executable
+      end
+      
+      # Executes this stage in the context of an ETL::Batch
       def execute(etl_batch, reextract=false)
-        sinks.each {|sink| sink.truncate } if truncate_pre_load?
+        prepare_stage(etl_batch, reextract)
         transform_and_load filtered_source(etl_batch, reextract)
       end
       
@@ -42,6 +48,7 @@ module Chicago
         @sinks.values
       end
       
+      # @api private
       def filtered_source(etl_batch, reextract=false)
         filtered_dataset = reextract ? source : 
           @filter_strategy.call(source, etl_batch)
@@ -51,6 +58,12 @@ module Chicago
 
       private
 
+      def prepare_stage(etl_batch, reextract)
+        @pre_execution_strategies.each do |strategy| 
+          strategy.call(self, etl_batch, reextract)
+        end
+      end
+
       def transform_and_load(source)
         sinks.each(&:open)
         pipe_rows_to_sinks_from(source)

data/lib/chicago/etl/stage_builder.rb

@@ -6,20 +6,42 @@ module Chicago
       end
 
       def build(name, &block)
-        @sinks = {}
-        @transformations = []
+        @pre_execution_strategies = []
+        @executable = true
 
         instance_eval &block
-        
+        set_default_stage_values
+
         Stage.new(name,
                   :source => @dataset, 
                   :sinks => @sinks, 
                   :transformations => @transformations, 
-                  :filter_strategy => @filter_strategy)
+                  :filter_strategy => @filter_strategy,
+                  :pre_execution_strategies => @pre_execution_strategies,
+                  :executable => @executable)
       end
 
       protected
 
+      # Specifies that the sinks should be truncated before loading
+      # data.
+      def truncate_pre_load
+        @pre_execution_strategies << lambda {|stage, etl_batch, reextract|
+          stage.sinks.each {|sink| sink.truncate }
+        }
+      end
+
+      # Specifies that the dataset should never be filtered to the ETL
+      # batch - i.e. it should behave as if reextract was always true
+      def full_reload
+        @filter_strategy = lambda {|dataset, etl_batch| dataset }
+      end
+
+      # Mark this stage as executable or non-executable.
+      def executable(value=true)
+        @executable = value
+      end
+
       def source(&block)
         @dataset = DatasetBuilder.new(@db).build(&block)
       end
@@ -32,12 +54,18 @@ module Chicago
         @sinks = SinkBuilder.new.build(&block)
       end
 
-      # TODO: think of potentially better ways of dealig with this
+      # TODO: think of potentially better ways of dealing with this
       # problem.
       def filter_strategy(&block)
         @filter_strategy = block
       end
 
+      # @api private
+      def set_default_stage_values
+        @sinks ||= sinks {}
+        @transformations ||= transformations {}
+      end
+
       class TransformationBuilder
         def build(&block)
           @transformations = []

data/lib/chicago/etl/stage_name.rb

@@ -0,0 +1,51 @@
+module Chicago
+  module ETL
+    # A namespaced name for an ETL stage.
+    #
+    # @api private
+    class StageName
+      def initialize(*names)
+        if names.size == 1 && names.first.kind_of?(String)
+          @names = names.first.split(".").map(&:to_sym).freeze
+        else
+          @names = names.flatten.map(&:to_sym).freeze
+        end
+      end
+
+      def name
+        @names.last
+      end
+
+      def match?(*pattern)
+        pattern.flatten!
+        return false if pattern.size > @names.size
+
+        pattern.each_with_index.all? do |part, i|
+          part == :* || @names[i] == part
+        end
+      end
+      alias :=~ :match?
+
+      def namespace
+        @names[0...(@names.size - 1)]
+      end
+
+      def eql?(other)
+        to_s == other.to_s
+      end
+      alias :== :eql?
+
+      def hash
+        to_s.hash
+      end
+
+      def to_a
+        @names.dup
+      end
+
+      def to_s
+        @string_representation ||= @names.join('.')
+      end
+    end
+  end
+end

data/lib/chicago/etl.rb

@@ -19,15 +19,16 @@ require 'chicago/etl/null_sink'
 require 'chicago/etl/mysql'
 
 require 'chicago/etl/core_extensions'
+require 'chicago/etl/stage_name'
 require 'chicago/etl/counter'
 require 'chicago/etl/key_builder'
 require 'chicago/etl/schema_table_sink_factory'
+require 'chicago/etl/schema_table_stage_builder'
 require 'chicago/etl/transformations'
 require 'chicago/etl/load_dataset_builder'
 require 'chicago/etl/dataset_builder'
 require 'chicago/etl/stage'
 require 'chicago/etl/stage_builder'
-require 'chicago/etl/dataset_batch_stage'
 require 'chicago/etl/schema_sinks_and_transformations_builder'
 require 'chicago/etl/pipeline'
 
@@ -60,9 +61,13 @@ module Chicago
     # within a batch.
     def self.execute(stage, etl_batch, reextract, logger)
       etl_batch.perform_task(:load, stage.name) do
-        logger.debug "Starting loading #{stage.name}"
-        stage.execute(etl_batch, reextract)
-        logger.debug "Finished loading #{stage.name}"
+        if stage.executable?
+          logger.debug "Starting loading #{stage.name}"
+          stage.execute(etl_batch, reextract)
+          logger.info "Finished loading #{stage.name}"
+        else
+          logger.info "Skipping stage #{stage.name}"
+        end
       end
     end
   end

data/spec/etl/define_dimension_stage_spec.rb

@@ -18,8 +18,8 @@ describe "creating and running a dimension stage" do
   let(:pipeline) { Chicago::ETL::Pipeline.new(db, schema)}
 
   it "glues the source, transformations, and sink correctly" do
-    pipeline.define_dimension_load(:test) do
-      dataset do
+    pipeline.define_stage(:load, :dimensions, :test) do
+      source do
         db.test_dataset_method
       end
     end

data/spec/etl/execution_wrapper_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe "Chicago::ETL Execution method" do
+  class StubBatch
+    def perform_task(*args)
+      yield
+    end
+  end
+
+  let(:logger) { mock(:logger).as_null_object }
+  let(:batch) { StubBatch.new }
+
+  it "only logs skipping the stage if the stage is not executable" do
+    stage = stub(:stage, :executable? => false, :name => "test")
+    stage.should_not_receive(:execute)
+    logger.should_receive(:info).with("Skipping stage test")
+
+    Chicago::ETL.execute(stage, batch, false, logger)
+  end
+
+  it "executes the stage" do
+    stage = stub(:stage, :executable? => true, :name => "test")
+    stage.should_receive(:execute).with(batch, false)
+
+    Chicago::ETL.execute(stage, batch, false, logger)
+  end
+end

data/spec/etl/stage_name_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+require 'set'
+
+describe Chicago::ETL::StageName do
+  it "can be consturcted with variable args" do
+    described_class.new(:a, :b).to_a.should == [:a, :b]
+  end
+
+  it "can be constructed with an array of symbols" do
+    described_class.new([:a, :b]).to_a.should == [:a, :b]
+  end
+
+  it "can be constructed with a dot seaprated string" do
+    described_class.new("foo.bar").to_a.should == [:foo, :bar]
+  end
+
+  it "has a name" do
+    described_class.new("foo.bar.baz").name.should == :baz
+  end
+
+  it "has a namespace" do
+    described_class.new("foo.bar.baz").namespace.should == [:foo, :bar]
+  end
+
+  it "supports equality" do
+    described_class.new(:a, :b).should == described_class.new(:a, :b)
+    set = Set.new
+    set << described_class.new(:a, :b)
+    set.should include(described_class.new(:a, :b))
+  end
+
+  it "has a dotted string representation" do
+    described_class.new(:a, :b).to_s.should == "a.b"
+  end
+
+  it "matches an exact pattern" do
+    described_class.new(:a, :b).match?(:a, :b).should be_true
+    described_class.new(:a, :b).match?(:a, :c).should be_false
+  end
+
+  it "matches a left-anchored partial pattern" do
+    described_class.new(:a, :b).match?(:a).should be_true
+    described_class.new(:a, :b).match?(:b).should be_false
+  end
+
+  it "allows wildcards matching" do
+    described_class.new(:a, :b).match?(:*, :b).should be_true
+    described_class.new(:a, :b).match?(:*, :*).should be_true
+    described_class.new(:a, :b).match?(:*, :*, :*).should be_false
+  end
+
+  it "can use the =~ operator" do
+    (described_class.new(:a, :b) =~ [:*, :b]).should be_true
+  end
+end

metadata

@@ -1,233 +1,248 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: chicago-etl
-version: !ruby/object:Gem::Version 
-  hash: 21
+version: !ruby/object:Gem::Version
+  version: 0.2.2
   prerelease: 
-  segments: 
-  - 0
-  - 2
-  - 1
-  version: 0.2.1
 platform: ruby
-authors: 
+authors:
 - Roland Swingler
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-11-19 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
+date: 2013-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chicagowarehouse
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        - 4
-        version: "0.4"
-  requirement: *id001
+      - !ruby/object:Gem::Version
+        version: '0.4'
   type: :runtime
   prerelease: false
-  name: chicagowarehouse
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id002
-  type: :runtime
-  prerelease: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
   name: fastercsv
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id003
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: sequel
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id004 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 0
-        - 0
-        - 2
-        version: 0.0.2
-  requirement: *id004
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: sequel_load_data_infile
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id005 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id005
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
   name: sequel_fast_columns
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id006 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 2
-        version: "2"
-  requirement: *id006
-  type: :development
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: rspec
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id007 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id007
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
   name: timecop
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id008 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id008
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: yard
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id009 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id009
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: flog
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id010 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id010
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: simplecov
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id011 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id011
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: ZenTest
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id012 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        hash: 45
-        segments: 
-        - 2
-        - 8
-        - 1
-        version: 2.8.1
-  requirement: *id012
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: mysql
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id013 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 1
-        version: "1"
-  requirement: *id013
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.1
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+- !ruby/object:Gem::Dependency
   name: bundler
-- !ruby/object:Gem::Dependency 
-  version_requirements: &id014 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  requirement: *id014
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
   type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
   name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ETL tools for Chicago
 email: roland.swingler@gmail.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.txt
 - README.rdoc
-files: 
+files:
 - .document
 - .rspec
 - Gemfile
@@ -244,7 +259,6 @@ files:
 - lib/chicago/etl/batch.rb
 - lib/chicago/etl/core_extensions.rb
 - lib/chicago/etl/counter.rb
-- lib/chicago/etl/dataset_batch_stage.rb
 - lib/chicago/etl/dataset_builder.rb
 - lib/chicago/etl/dataset_source.rb
 - lib/chicago/etl/errors.rb
@@ -259,6 +273,7 @@ files:
 - lib/chicago/etl/pipeline_endpoint.rb
 - lib/chicago/etl/schema_sinks_and_transformations_builder.rb
 - lib/chicago/etl/schema_table_sink_factory.rb
+- lib/chicago/etl/schema_table_stage_builder.rb
 - lib/chicago/etl/screens/column_screen.rb
 - lib/chicago/etl/screens/invalid_element.rb
 - lib/chicago/etl/screens/missing_value.rb
@@ -268,6 +283,7 @@ files:
 - lib/chicago/etl/sink.rb
 - lib/chicago/etl/stage.rb
 - lib/chicago/etl/stage_builder.rb
+- lib/chicago/etl/stage_name.rb
 - lib/chicago/etl/table_builder.rb
 - lib/chicago/etl/task_invocation.rb
 - lib/chicago/etl/tasks.rb
@@ -287,6 +303,7 @@ files:
 - spec/etl/define_dimension_stage_spec.rb
 - spec/etl/define_stage_spec.rb
 - spec/etl/etl_batch_id_dataset_filter.rb
+- spec/etl/execution_wrapper_spec.rb
 - spec/etl/filter_spec.rb
 - spec/etl/key_builder_spec.rb
 - spec/etl/load_dataset_builder_spec.rb
@@ -300,6 +317,7 @@ files:
 - spec/etl/screens/out_of_bounds_spec.rb
 - spec/etl/sequel/dependant_tables_spec.rb
 - spec/etl/sequel/filter_to_etl_batch_spec.rb
+- spec/etl/stage_name_spec.rb
 - spec/etl/stage_spec.rb
 - spec/etl/table_builder_spec.rb
 - spec/etl/task_spec.rb
@@ -311,37 +329,31 @@ files:
 - spec/etl/transformations_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/notonthehighstreet/chicago-etl
-licenses: 
+licenses:
 - MIT
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: -2294158408277347233
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Chicago ETL
 test_files: []
-

data/lib/chicago/etl/dataset_batch_stage.rb

@@ -1,19 +0,0 @@
-module Chicago
-  module ETL
-    # Allows deferring constructing a DatasetSource until extract
-    # time, so that it can be filtered to an ETL batch appropriately.
-    class DatasetBatchStage < Stage
-      # Executes this ETL stage.
-      #
-      # Configures the dataset and flows rows into the pipeline.
-      def execute(etl_batch, reextract=false)
-        if reextract && sink(:error) && !truncate_pre_load?
-          sink(:error).truncate
-        end
-        
-        sink(:default).set_constant_values(:_inserted_at => Time.now)
-        super
-      end
-    end
-  end
-end