chicago-etl 0.2.3 → 0.2.4

data/VERSION

@@ -1 +1 @@
-0.2.3
+0.2.4

data/chicago-etl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.2.3"
+  s.version = "0.2.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
-  s.date = "2014-01-06"
+  s.date = "2014-01-07"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [

data/lib/chicago/etl/batch.rb

@@ -45,6 +45,18 @@ module Chicago
         perform_task(:extract, task_name, &block)
       end
 
+      # Marks this batch for re-extraction.
+      def reextract
+        @reextract = true
+        self
+      end
+
+      # Returns true when this batch should re-extract - i.e. load
+      # records without regard to creation/update times.
+      def reextracting?
+        !!@reextract
+      end
+
       # Deprecated.
       #
       # @deprecated Use perform_task instead

data/lib/chicago/etl/schema_table_stage_builder.rb

@@ -33,8 +33,11 @@ module Chicago
           end
         end
 
-        @pre_execution_strategies << lambda {|stage, etl_batch, reextract|
-          stage.sink(:error).truncate if reextract && stage.sink(:error)
+        @pre_execution_strategies << lambda {|stage, etl_batch|
+          if etl_batch.reextracting? && stage.sink(:error)
+            stage.sink(:error).truncate
+          end
+
           stage.sink(:default).
             set_constant_values(:_inserted_at => Time.now)
         }

data/lib/chicago/etl/stage.rb

@@ -34,9 +34,9 @@ module Chicago
       end
       
       # Executes this stage in the context of an ETL::Batch
-      def execute(etl_batch, reextract=false)
-        prepare_stage(etl_batch, reextract)
-        transform_and_load filtered_source(etl_batch, reextract)
+      def execute(etl_batch)
+        prepare_stage(etl_batch)
+        transform_and_load filtered_source(etl_batch)
       end
       
       # Returns the named sink, if it exists
@@ -49,8 +49,8 @@ module Chicago
       end
       
       # @api private
-      def filtered_source(etl_batch, reextract=false)
-        filtered_dataset = reextract ? source : 
+      def filtered_source(etl_batch)
+        filtered_dataset = etl_batch.reextracting? ? source : 
           @filter_strategy.call(source, etl_batch)
 
         DatasetSource.new(filtered_dataset)
@@ -58,9 +58,9 @@ module Chicago
 
       private
 
-      def prepare_stage(etl_batch, reextract)
+      def prepare_stage(etl_batch)
         @pre_execution_strategies.each do |strategy| 
-          strategy.call(self, etl_batch, reextract)
+          strategy.call(self, etl_batch)
         end
       end
 

data/lib/chicago/etl/stage_builder.rb

@@ -26,13 +26,13 @@ module Chicago
       # Specifies that the sinks should be truncated before loading
       # data.
       def truncate_pre_load
-        @pre_execution_strategies << lambda {|stage, etl_batch, reextract|
+        @pre_execution_strategies << lambda {|stage, etl_batch|
           stage.sinks.each {|sink| sink.truncate }
         }
       end
 
       # Specifies that the dataset should never be filtered to the ETL
-      # batch - i.e. it should behave as if reextract was always true
+      # batch - i.e. it should behave as if the batch is reextracting
       def full_reload
         @filter_strategy = lambda {|dataset, etl_batch| dataset }
       end

data/lib/chicago/etl.rb

@@ -59,12 +59,12 @@ module Chicago
     # Tasks execution status is stored in a database etl task
     # invocations table - this ensures tasks aren't run more than once
     # within a batch.
-    def self.execute(stage, etl_batch, reextract, logger)
+    def self.execute(stage, etl_batch, logger)
       etl_batch.perform_task(:load, stage.name) do
         if stage.executable?
-          logger.debug "Starting loading #{stage.name}"
-          stage.execute(etl_batch, reextract)
-          logger.info "Finished loading #{stage.name}"
+          logger.debug "Starting executing stage: #{stage.name}"
+          stage.execute etl_batch
+          logger.info "Finished executing stage: #{stage.name}"
         else
           logger.info "Skipping stage #{stage.name}"
         end

data/spec/etl/batch_spec.rb

@@ -84,4 +84,9 @@ describe Chicago::ETL::Batch do
     batch = ETL::Batch.instance.start
     lambda { batch.perform_task(:transform, "Test") {} }.should_not raise_error
   end
+
+  it "can be marked as re-extracting" do
+    ETL::Batch.instance.reextract.should be_reextracting
+    ETL::Batch.instance.should_not be_reextracting
+  end
 end

data/spec/etl/define_dimension_stage_spec.rb

@@ -25,7 +25,7 @@ describe "creating and running a dimension stage" do
     end
 
     pipeline.stages.each do |stage|
-      stage.execute(double, true)
+      stage.execute(double(:reextracting? => true))
     end
   end
 

data/spec/etl/define_stage_spec.rb

@@ -29,7 +29,7 @@ describe "defining and executing a stage" do
     end
 
     pipeline.stages.each do |stage|
-      stage.execute(double, true)
+      stage.execute(double(:reextracting? => true))
     end
 
     stage = pipeline.stages.first
@@ -56,7 +56,7 @@ describe "defining and executing a stage" do
     end
 
     pipeline.stages.each do |stage|
-      stage.execute(double, true)
+      stage.execute(double(:reextracting? => true))
     end
 
     stage = pipeline.stages.first
@@ -68,7 +68,7 @@ describe "defining and executing a stage" do
   end
 
   it "allows the source to be filtered via a filter strategy" do
-    etl_batch_double = double
+    etl_batch_double = double(:reextracting? => false)
     fake_source = []
     fake_source.should_receive(:another_dataset_method).and_return([])
     
@@ -88,7 +88,7 @@ describe "defining and executing a stage" do
     end
     
     pipeline.stages.each do |stage|
-      stage.execute(etl_batch_double, false)
+      stage.execute(etl_batch_double)
     end
   end
 end

data/spec/etl/execution_wrapper_spec.rb

@@ -15,13 +15,13 @@ describe "Chicago::ETL Execution method" do
     stage.should_not_receive(:execute)
     logger.should_receive(:info).with("Skipping stage test")
 
-    Chicago::ETL.execute(stage, batch, false, logger)
+    Chicago::ETL.execute(stage, batch, logger)
   end
 
   it "executes the stage" do
     stage = double(:stage, :executable? => true, :name => "test")
-    stage.should_receive(:execute).with(batch, false)
+    stage.should_receive(:execute).with(batch)
 
-    Chicago::ETL.execute(stage, batch, false, logger)
+    Chicago::ETL.execute(stage, batch, logger)
   end
 end

data/spec/etl/mysql_integration_spec.rb

@@ -64,7 +64,7 @@ describe "Mysql -> Mysql through transformation chain" do
                                       :other => sink_2
                                     })
 
-    stage.execute(double(:etl_batch), true)
+    stage.execute(double(:etl_batch, :reextracting? => true))
 
     expected = [{:id => 1, :foo => nil, :bin => "1F"},
                 {:id => 2, :foo => "Hello", :bin => "1F"}]

data/spec/etl/stage_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
 
 describe Chicago::ETL::Stage do
+  let(:etl_batch) { double(:etl_batch, :reextracting? => true) }
+
   it "requires a source" do
     expect {
       described_class.new(:test,
@@ -23,7 +25,7 @@ describe Chicago::ETL::Stage do
                                 :sinks => {:default => double(:sink)},
                                 :filter_strategy => lambda { fail })
 
-    stage.filtered_source(double(:etl_batch), true)
+    stage.filtered_source(etl_batch)
   end
 
   it "opens sinks before writing and closes them afterwards" do
@@ -35,6 +37,6 @@ describe Chicago::ETL::Stage do
                                 :source => [],
                                 :sinks => {:default => sink})
 
-    stage.execute(double(:etl_batch), true)
+    stage.execute(etl_batch)
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: chicago-etl
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 31
   prerelease: 
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - Roland Swingler
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-01-06 00:00:00 Z
+date: 2014-01-07 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement