chicago-etl 0.2.0 → 0.2.1

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.2.1

data/chicago-etl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "chicago-etl"
-  s.version = "0.2.0"
+  s.version = "0.2.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Roland Swingler"]
-  s.date = "2013-11-18"
+  s.date = "2013-11-19"
   s.description = "ETL tools for Chicago"
   s.email = "roland.swingler@gmail.com"
   s.extra_rdoc_files = [

data/lib/chicago/etl/dataset_batch_stage.rb

@@ -1,25 +1,13 @@
 module Chicago
   module ETL
-    # Links a PipelineStage to a Dataset.
-    #
     # Allows deferring constructing a DatasetSource until extract
     # time, so that it can be filtered to an ETL batch appropriately.
     class DatasetBatchStage < Stage
-      attr_reader :name
-
-      def initialize(name, options={})
-        super
-        @filter_strategy = options[:filter_strategy] ||
-          lambda { |dataset, etl_batch| @source.filter_to_etl_batch(etl_batch)}
-     end
-
       # Executes this ETL stage.
       #
       # Configures the dataset and flows rows into the pipeline.
       def execute(etl_batch, reextract=false)
-        if @truncate_pre_load
-          sinks.each {|sink| sink.truncate }
-        elsif reextract && sink(:error)
+        if reextract && sink(:error) && !truncate_pre_load?
           sink(:error).truncate
         end
         

data/lib/chicago/etl/pipeline.rb

@@ -65,13 +65,17 @@ module Chicago
           pipeline do
           end
         end
-         DatasetBatchStage.new(name,
-                               :source => @dataset, 
-                               :transformations => @sinks_and_transformations[:transformations],
-                               :sinks => @sinks_and_transformations[:sinks],
-                               :filter_strategy => @filter_strategy,
-                               :truncate_pre_load => @truncate_pre_load)
 
+        @filter_strategy ||= lambda {|dataset, etl_batch| 
+          dataset.filter_to_etl_batch(etl_batch)
+        }
+        
+        DatasetBatchStage.new(name,
+                              :source => @dataset, 
+                              :transformations => @sinks_and_transformations[:transformations],
+                              :sinks => @sinks_and_transformations[:sinks],
+                              :filter_strategy => @filter_strategy,
+                              :truncate_pre_load => @truncate_pre_load)
       end
 
       protected

data/lib/chicago/etl/stage.rb

@@ -18,11 +18,18 @@ module Chicago
         @transformations = options[:transformations] || []
         @filter_strategy = options[:filter_strategy] || 
           lambda {|source, _| source }
+        @truncate_pre_load = !!options[:truncate_pre_load]
 
         validate_arguments
       end
+      
+      # Returns true if the sinks should be truncated pre-load.
+      def truncate_pre_load?
+        @truncate_pre_load
+      end
 
       def execute(etl_batch, reextract=false)
+        sinks.each {|sink| sink.truncate } if truncate_pre_load?
         transform_and_load filtered_source(etl_batch, reextract)
       end
       

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: chicago-etl
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 21
   prerelease: 
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - Roland Swingler
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-11-18 00:00:00 Z
+date: 2013-11-19 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement