activewarehouse-etl 0.5.1 → 0.5.2

data/CHANGELOG

@@ -31,4 +31,8 @@
 
 0.5.1 - Feb 18, 2007
 * Fixed up truncate processor
-* Updated HOW_TO_RELEASE doc
+* Updated HOW_TO_RELEASE doc
+
+0.5.2 - Feb 19, 2007
+* Added error threshold
+* Fixed problem with transform error handling

data/lib/etl/control/control.rb

@@ -9,14 +9,20 @@ module ETL #:nodoc:
         end
       end
       
+      # Initialize the context
       def initialize(control)
         @control = control
       end
       
+      # Get the control file
       def file
         @control.file
       end
       
+      def set_error_threshold(error_threshold)
+        @control.error_threshold = error_threshold
+      end
+      
       # Define a source.
       def source(name, configuration={}, definition={})
         source_types = [:file, :db]
@@ -48,6 +54,7 @@ module ETL #:nodoc:
         @control.destinations
       end
       
+      # Define a transform
       def transform(name, transformer=nil, configuration={}, &block)
         transforms[name] ||= []
         if transformer
@@ -65,10 +72,12 @@ module ETL #:nodoc:
         end
       end
       
+      # Get the defined transforms
       def transforms
         @control.transforms
       end
       
+      # Define a pre-processor
       def pre_process(name, configuration={})
         class_name = "#{name.to_s.classify}Processor"
         begin
@@ -79,10 +88,12 @@ module ETL #:nodoc:
         end
       end
       
+      # Get the defined pre-processors
       def pre_processors
         @control.pre_processors
       end
       
+      # Define a post-processor
       def post_process(name, configuration={})
         class_name = "#{name.to_s.classify}Processor"
         begin
@@ -93,10 +104,12 @@ module ETL #:nodoc:
         end
       end
       
+      # Get the defined post-processors
       def post_processors
         @control.post_processors
       end
       
+      # Get the binding object
       def get_binding
         binding
       end
@@ -119,6 +132,9 @@ module ETL #:nodoc:
       # The File object
       attr_reader :file
       
+      # The error threshold
+      attr_accessor :error_threshold
+      
       class << self
         # Parse a control file and return a Control instance
         def parse(control_file)
@@ -187,6 +203,11 @@ module ETL #:nodoc:
         @transforms ||= {}
       end
       
+      # Get the error threshold. Defaults to 100.
+      def error_threshold
+        @error_threshold ||= 100
+      end
+      
       # Validate the control file
       def validate
         unless sources.length > 0

data/lib/etl/engine.rb

@@ -51,6 +51,10 @@ module ETL #:nodoc:
     def say_on_own_line(message)
       say("\n" + message)
     end
+    
+    def errors
+      @errors ||= []
+    end
   
     # Process a control file or object. Acceptable values for control are:
     # * Path to a file
@@ -73,7 +77,7 @@ module ETL #:nodoc:
         say "Source: #{source}"
         source.each_with_index do |row, index|
           Engine.current_source_row = index + 1
-          if Engine.realtime_activity && index % 1000 == 0
+          if Engine.realtime_activity && index > 0 && index % 1000 == 0
             say_without_newline "."
           end
           
@@ -84,8 +88,9 @@ module ETL #:nodoc:
             end
           rescue => e
             msg = "Error transforming from #{source} on line #{index}: #{e}"
-            source.errors << msg
+            errors << msg
             Engine.logger.error msg
+            break if exceeded_error_threshold?(control)
           end
           
           begin
@@ -96,11 +101,17 @@ module ETL #:nodoc:
             end
           rescue
             msg = "Error writing to #{destination} on line #{index}"
-            destination.errors << msg
+            errors << msg
             Engine.logger.error msg
+            break if exceeded_error_threshold?(control)
           end
         end
-        say_on_own_line "Processed #{Engine.current_source_row} rows in #{distance_of_time_in_words(start_time)}"
+        if exceeded_error_threshold?(control)
+          say_on_own_line "Exiting due to exceeding error threshold: #{control.error_threshold}"
+        else
+          say_on_own_line "Processed #{Engine.current_source_row} rows in #{distance_of_time_in_words(start_time)} with #{errors} errors."
+        end
+        
         destinations.each do |destination|
           destination.close
         end
@@ -112,6 +123,10 @@ module ETL #:nodoc:
     end
     
     private
+    def exceeded_error_threshold?(control)
+      errors.length > control.error_threshold
+    end
+    
     # Execute all preprocessors
     def pre_process(control)
       control.pre_processors.each do |processor|

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -2,6 +2,9 @@ module ETL #:nodoc:
   module Transform #:nodoc:
     # Transform which looks up the value and replaces it with a foriegn key reference
     class ForeignKeyLookupTransform < ETL::Transform::Transform
+      # The resolver to use if the foreign key is not found in the collection
+      attr_accessor :resolver
+      
       # Initialize the foreign key lookup transform.
       #
       # Configuration options:
@@ -21,20 +24,28 @@ module ETL #:nodoc:
       def transform(value)
         fk = @collection[value]
         unless fk
-          raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless @resolver
-          raise ResolverError, "Resolver does not appear to respond to resolve method" unless @resolver.respond_to?(:resolve)
-          fk = @resolver.resolve(value)
+          raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless resolver
+          raise ResolverError, "Resolver does not appear to respond to resolve method" unless resolver.respond_to?(:resolve)
+          fk = resolver.resolve(value)
           raise ResolverError, "Unable to resolve #{value} to foreign key" unless fk
           @collection[value] = fk
         end
         fk
       end
     end
+    # Alias class name for the ForeignKeyLookupTransform.
+    class FkLookupTransform < ForeignKeyLookupTransform; end
   end
 end
 
 # Resolver which resolves using ActiveRecord.
 class ActiveRecordResolver
+  # The ActiveRecord class to use
+  attr_accessor :ar_class
+  
+  # The find method to use (as a symbol)
+  attr_accessor :find_method
+  
   # Initialize the resolver. The ar_class argument should extend from ActiveRecord::Base. The find_method argument
   # must be a symbol for the finder method used. For example:
   # 
@@ -47,7 +58,7 @@ class ActiveRecordResolver
   end
   # Resolve the value
   def resolve(value)
-    rec = @ar_class.__send__(@find_method, value)
+    rec = ar_class.__send__(find_method, value)
     rec.nil? ? nil : rec.id
   end
 end

data/lib/etl/transform/transform.rb

@@ -23,17 +23,13 @@ module ETL#:nodoc:
         def transform(name, value, transforms)
           # logger.debug "Transforming field #{name}" if transforms.length > 0
           transforms.each do |transform|
-            begin
-              case transform
-              when Proc
-                value = transform.call(value)
-              when Transform
-                value = transform.transform(value)
-              else
-                raise ControlError, "Unsupported transform configuration type: #{transform}"
-              end
-            rescue
-              raise TransformError, "Error transforming #{value} with #{transform}"
+            case transform
+            when Proc
+              value = transform.call(value)
+            when Transform
+              value = transform.transform(value)
+            else
+              raise ControlError, "Unsupported transform configuration type: #{transform}"
             end
           end
           value

data/lib/etl/version.rb

@@ -2,7 +2,7 @@ module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 5
-    TINY  = 1
+    TINY  = 2
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0.10
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.5.1
-date: 2007-02-18 00:00:00 -05:00
+  version: 0.5.2
+date: 2007-02-19 00:00:00 -05:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib