activewarehouse-etl 0.6.1 → 0.7.0

data/lib/etl/control/destination/database_destination.rb

@@ -19,13 +19,13 @@ module ETL #:nodoc:
       #
       # Configuration options:
       # * <tt>:database</tt>: The database name (REQUIRED)
-      # * <tt>:table<tt>: The table to write to (REQUIRED)
+      # * <tt>:table</tt>: The table to write to (REQUIRED)
       # * <tt>:truncate</tt>: Set to true to truncate before writing (defaults to false)
       # * <tt>:unique</tt>: Set to true to only insert unique records (defaults to false)
       # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
       # * <tt>:username</tt>: The database username (defaults to 'root')
       # * <tt>:password</tt>: The password to the database (defaults to nothing)
-      # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
+      # * <tt>:host</tt>: The host for the database (defaults to 'localhost')
       # * <tt>:append_rows</tt>: Array of rows to append
       #
       # Mapping options:

data/lib/etl/control/destination/file_destination.rb

@@ -1,3 +1,5 @@
+# This source file contains the ETL::Control::FileDestination
+
 module ETL #:nodoc:
   module Control #:nodoc:
     # File as the final destination.
@@ -60,6 +62,7 @@ module ETL #:nodoc:
       def flush
         #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
         buffer.flatten.each do |row|
+          #puts "row change type: #{row.change_type}"
           # check to see if this row's compound key constraint already exists
           # note that the compound key constraint may not utilize virtual fields
           next unless row_allowed?(row)
@@ -68,11 +71,22 @@ module ETL #:nodoc:
           add_virtuals!(row)
           
           # collect all of the values using the order designated in the configuration
-          values = order.collect { |name| row[name] }
+          values = order.collect do |name|
+            value = row[name]
+            case value
+            when Date, Time, DateTime
+              value.to_s(:db)
+            else
+              value.to_s
+            end
+          end
+          
+          values.collect! { |v| v.gsub(separator, "\\#{separator}")}
+          values.collect! { |v| v.gsub(/\n|\r/, '')}
           
           # enclose the value if required
           if !enclose.nil?
-            values.collect! { |v| enclose + v.to_s.gsub(/(#{enclose})/, '\\\\\1') + enclose }
+            values.collect! { |v| enclose + v.gsub(/(#{enclose})/, '\\\\\1') + enclose }
           end
           
           # write the values joined by the separator defined in the configuration
@@ -81,6 +95,7 @@ module ETL #:nodoc:
           # write the end-of-line
           f.write(eol)
         end
+        f.flush
         buffer.clear
         #puts "After flush there are #{buffer.length} rows"
       end
@@ -91,6 +106,14 @@ module ETL #:nodoc:
         @f ||= open(file, mode)
       end
       
+      def options
+        @options ||= {
+          :col_sep => separator,
+          :row_sep => eol,
+          :force_quotes => !enclose.nil?
+        }
+      end
+      
       # Get the appropriate mode to open the file stream
       def mode
         append ? 'a' : 'w'

data/lib/etl/control/source.rb

@@ -14,12 +14,14 @@ module ETL #:nodoc:
       attr_accessor :definition
       
       # Returns true if the source data should be stored locally for archival
+      # Default behavior will return true.
       attr_accessor :store_locally
       
       class << self
         # Convert the name to a Source class.
         # 
-        # For example if name is :database then this will return a DatabaseSource class
+        # For example if name is :database then this will return a 
+        # DatabaseSource class
         def class_for_name(name)
           ETL::Control.const_get("#{name.to_s.classify}Source")
         end
@@ -31,7 +33,8 @@ module ETL #:nodoc:
       # * <tt>definition</tt>: The source layout definition
       #
       # Configuration options:
-      # * <tt>:store_locally</tt>: Set to false to not store source data locally (defaults to true)
+      # * <tt>:store_locally</tt>: Set to false to not store source data
+      #   locally (defaults to true)
       def initialize(control, configuration, definition)
         @control = control
         @configuration = configuration
@@ -40,6 +43,7 @@ module ETL #:nodoc:
         @store_locally = configuration[:store_locally] || true
       end
       
+      # Get an array of errors that occur during reading from the source
       def errors
         @errors ||= []
       end
@@ -57,18 +61,35 @@ module ETL #:nodoc:
         @local_base ||= 'source_data'
       end
       
-      # The local directory for storing. This method must be overriden by subclasses
+      # The local directory for storing. This method must be overriden by 
+      # subclasses
       def local_directory
         raise "local_directory method is abstract"
       end
       
-      # Return the local file for storing the raw source data. Each call to this method will
-      # result in a timestamped file, so you cannot expect to call it multiple times and reference
-      # the same file
-      def local_file
+      # Return the local file for storing the raw source data. Each call to 
+      # this method will result in a timestamped file, so you cannot expect 
+      # to call it multiple times and reference the same file
+      # 
+      # Optional sequence can be specified if there are multiple source files
+      def local_file(sequence=nil)
+        filename = timestamp.to_s
+        filename += sequence.to_s if sequence
+        
         local_dir = local_directory
         FileUtils.mkdir_p(local_dir)
-        File.join(local_dir, "#{timestamp}.csv")
+        File.join(local_dir, "#{filename}.csv")
+      end
+      
+      # Get the local trigger file that is used to indicate that the file has
+      # been completely written
+      def local_file_trigger(file)
+        Pathname.new(file.to_s + '.trig')
+      end
+      
+      # Return true if the source should read locally.
+      def read_locally
+        Engine.read_locally
       end
       
     end

data/lib/etl/control/source/database_source.rb

@@ -22,10 +22,18 @@ module ETL #:nodoc:
       # Other options:
       # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
       # * <tt>:username</tt>: The database username (defaults to 'root')
-      # * <tt>:password</tt>: The password to the database (defaults to nothing)
-      # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
-      # * <tt>:store_locally</tt>: Set to false to not store a copy of the source data locally
-      #   in a flat file (defaults to true)
+      # * <tt>:password</tt>: The password to the database (defaults to 
+      #   nothing)
+      # * <tt>:host</tt>: The host for the database (defaults to 
+      #   'localhost')
+      # * <tt>:join</tt>: Optional join part for the query (ignored unless 
+      #   specified)
+      # * <tt>:select</tt>: Optional select part for the query (defaults to 
+      #   '*')
+      # * <tt>:order</tt>: Optional order part for the query (ignored unless 
+      #   specified)
+      # * <tt>:store_locally</tt>: Set to false to not store a copy of the 
+      #   source data locally in a flat file (defaults to true)
       def initialize(control, configuration, definition)
         super
         connect
@@ -36,41 +44,117 @@ module ETL #:nodoc:
         "#{host}/#{configuration[:database]}/#{configuration[:table]}"
       end
       
-      # Get the local directory to use, which is a combination of the local_base, the db hostname
-      # the db database name and the db table.
+      # Get the local directory to use, which is a combination of the 
+      # local_base, the db hostname the db database name and the db table.
       def local_directory
         File.join(local_base, host, configuration[:database], configuration[:table])
       end
       
-      # Returns each row from the source
-      def each
-        if store_locally
-          file = local_file
-          columns = connection.columns(configuration[:table].to_s)
-          FasterCSV.open(file, 'w') do |f|
-            f << columns.collect { |column| column.name }
-            connection.select_all("SELECT * FROM #{configuration[:table]}").each do |row|
-              values = columns.collect { |column| row[column.name] }
-              #puts "row: #{values.inspect}"
-              f << values
+      # Get the join part of the query, defaults to nil
+      def join
+        configuration[:join]
+      end
+      
+      # Get the select part of the query, defaults to '*'
+      def select
+        configuration[:select] || '*'
+      end
+      
+      # Get the order for the query, defaults to nil
+      def order
+        configuration[:order]
+      end
+      
+      # Get the list of columns to read. This is defined in the source
+      # definition as either an Array or Hash
+      def columns
+        case definition
+        when Array
+          definition.collect(&:to_sym)
+        when Hash
+          definition.keys.collect(&:to_sym)
+        else
+          raise "Definition must be either an Array or a Hash"
+        end
+      end
+      
+      # Returns each row from the source. If read_locally is specified then
+      # this method will attempt to read from the last stored local file. 
+      # If no locally stored file exists or if the trigger file for the last
+      # locally stored file does not exist then this method will raise an
+      # error.
+      def each(&block)
+        if read_locally # Read from the last stored source
+          read_rows(&block)
+        else # Read from the original source
+          if store_locally
+            write_local
+            read_rows(&block)
+          else
+            connection.select_all(query).each do |row|
+              row = Row.new(row.symbolize_keys)
+              yield row
             end
           end
+        end
+      end
+      
+      private
+      # Read rows from the local cache
+      def read_rows
+        file = local_file
+        
+        File.exists?(file) or raise "Local cache file not found"
+        File.exists?(local_file_trigger(file)) or raise "Local cache trigger file not found"
+        
+        t = Benchmark.realtime do
           FasterCSV.open(file, :headers => true).each do |row|
-            result_row = {}
+            result_row = ETL::Row.new
             row.each do |header, field|
               result_row[header.to_sym] = field
             end
-            #puts "yielding #{result_row.inspect}"
             yield result_row
           end
-        else
-          connection.select_all("SELECT * FROM #{configuration[:table]}").each do |row|
-            yield HashWithIndifferentAccess.new(row)
+        end
+        ETL::Engine.average_rows_per_second = ETL::Engine.rows_read / t
+      end
+      
+      # Write rows to the local cache
+      def write_local
+        file = local_file
+      
+        lines = 0
+        t = Benchmark.realtime do
+          FasterCSV.open(file, 'w') do |f|
+            f << columns
+            connection.select_all(query).each do |row|
+              f << columns.collect { |column| row[column.to_s] }
+              lines += 1
+            end
           end
+          File.open(local_file_trigger(file), 'w') {|f| }
         end
+        ETL::Engine.logger.info "Stored locally in #{t}s (avg: #{lines/t} lines/sec)"
+      end
+      
+      # Get the query to use
+      def query
+        return @query if @query
+        q = "SELECT #{select} FROM #{configuration[:table]}"
+        q << " #{join}" if join
+        q << " ORDER BY #{order}" if order
+        if ETL::Engine.limit || ETL::Engine.offset
+          options = {}
+          options[:limit] = ETL::Engine.limit if ETL::Engine.limit
+          options[:offset] = ETL::Engine.offset if ETL::Engine.offset
+          connection.add_limit_offset!(q, options)
+        end
+        #q << " LIMIT #{ETL::Engine.limit}" unless ETL::Engine.limit.nil?
+        q = q.gsub(/\n/,' ')
+        ETL::Engine.logger.info "Query: #{q}"
+        @query = q
       end
       
-      private
       # Get the database connection to use
       def connection
         ETL::Source.connection
@@ -99,7 +183,8 @@ module ETL #:nodoc:
       # Options:
       # * <tt>:adapter</tt>: The adapter to use (defaults to :mysql)
       # * <tt>:username</tt>: The database username (defaults to 'root')
-      # * <tt>:password</tt>: The password to the database (defaults to nothing)
+      # * <tt>:password</tt>: The password to the database (defaults 
+      #   to nothing)
       # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
       def connect
         ETL::Source.establish_connection(

data/lib/etl/control/source/file_source.rb

@@ -8,16 +8,21 @@ module ETL #:nodoc:
       # Accessor for the underlying parser
       attr_accessor :parser
       
+      # The source file
+      attr_accessor :file
+      
       # Initialize the source
       #
       # Configuration options:
       # * <tt>:file</tt>: The source file
-      # * <tt>:parser</tt>: One of the following: a parser name as a String or symbol, a class which extends 
-      #   from Parser, a Hash with :name and optionally an :options key. Whether or not the parser uses the 
-      #   options is dependent on which parser is used. See the documentation for each parser for information 
-      #   on what options it accepts.
-      # * <tt>:skip_lines<tt>: The number of lines to skip (defaults to 0)
-      # * <tt>:store_locally</tt>: Set to false to not store a copy of the source data locally for archival
+      # * <tt>:parser</tt>: One of the following: a parser name as a String or
+      #   symbol, a class which extends from Parser, a Hash with :name and 
+      #   optionally an :options key. Whether or not the parser uses the 
+      #   options is dependent on which parser is used. See the documentation 
+      #   for each parser for information on what options it accepts.
+      # * <tt>:skip_lines</tt>: The number of lines to skip (defaults to 0)
+      # * <tt>:store_locally</tt>: Set to false to not store a copy of the 
+      #   source data locally for archival
       def initialize(control, configuration, definition)
         super
         configure
@@ -25,18 +30,21 @@ module ETL #:nodoc:
       
       # Get a String identifier for the source
       def to_s
-        configuration[:file]
+        file
       end
       
       # Get the local storage directory
       def local_directory
-        File.join(local_base, File.basename(configuration[:file], File.extname(configuration[:file])))
+        File.join(local_base, File.basename(file, File.extname(file)))
       end
       
       # Returns each row from the source
       def each
         copy_sources if store_locally
         @parser.each do |row|
+          # TODO skip rows if offset is defined
+          # TODO stop processing if limit is reached
+          row = ETL::Row[row]
           yield row
         end
       end
@@ -44,29 +52,34 @@ module ETL #:nodoc:
       private
       # Copy source data to a local directory structure
       def copy_sources
-        path = Pathname.new(configuration[:file])
+        sequence = 0
+        path = Pathname.new(file)
         path = path.absolute? ? path : Pathname.new(File.dirname(control.file)) + path
         Pathname.glob(path).each do |f|
           next if f.directory?
-          FileUtils.cp(f, local_file)
+          lf = local_file(sequence)
+          FileUtils.cp(f, lf)
+          File.open(local_file_trigger(lf), 'w') {|f| }
+          sequence += 1
         end
       end
       
       # Configure the source
       def configure
-        case @configuration[:parser]
+        @file = configuration[:file]
+        case configuration[:parser]
         when Class
-          @parser = @configuration[:parser].new(self)
+          @parser = configuration[:parser].new(self)
         when String, Symbol
-          @parser = ETL::Parser::Parser.class_for_name(@configuration[:parser]).new(self)
+          @parser = ETL::Parser::Parser.class_for_name(configuration[:parser]).new(self)
         when Hash
-          name = @configuration[:parser][:name]
-          options = @configuration[:parser][:options]
+          name = configuration[:parser][:name]
+          options = configuration[:parser][:options]
           @parser = ETL::Parser::Parser.class_for_name(name).new(self, options)
         else
           raise ControlError, "Configuration option :parser must be a Class, String or Symbol"
         end
-        @skip_lines = @configuration[:skip_lines] ||= 0
+        @skip_lines = configuration[:skip_lines] ||= 0
       end
     end
   end

data/lib/etl/engine.rb

@@ -10,6 +10,24 @@ module ETL #:nodoc:
   # The main ETL engine clas
   class Engine
     class << self
+      # Initialization that is run when a job is executed.
+      def init(options={})
+        unless @initialized
+          @limit = options[:limit]
+          @offset = options[:offset]
+          @log_write_mode = 'w' if options[:newlog]
+          @skip_bulk_import = options[:skip_bulk_import]
+          @read_locally = options[:read_locally]
+          options[:config] ||= 'database.yml'
+          database_configuration = YAML::load(ERB.new(IO.read(options[:config])).result + "\n")
+          ActiveRecord::Base.configurations = database_configuration
+          require 'etl/execution'
+          ETL::Execution::Base.establish_connection :etl_execution
+          ETL::Execution::Execution.migrate
+          @initialized = true
+        end
+      end
+      
       # Process the specified control file. Acceptable values for control_file are
       # * Path to a file
       # * File object
@@ -20,6 +38,12 @@ module ETL #:nodoc:
       
       attr_accessor :timestamped_log
       
+      # Accessor for the log write mode. Default is 'a' for append.
+      attr_accessor :log_write_mode
+      def log_write_mode
+        @log_write_mode ||= 'a'
+      end
+      
       # A logger for the engine
       attr_accessor :logger
       
@@ -28,7 +52,7 @@ module ETL #:nodoc:
           if timestamped_log
             @logger = Logger.new("etl_#{timestamp}.log")
           else
-            @logger = Logger.new(File.open('etl.log', 'a'))
+            @logger = Logger.new(File.open('etl.log', log_write_mode))
           end
           @logger.level = Logger::ERROR
           @logger.formatter = Logger::Formatter.new
@@ -50,21 +74,43 @@ module ETL #:nodoc:
       # The current destination
       attr_accessor :current_destination
       
-      # Set to true to activate realtime activity. This will cause certain information messages
-      # to be printed to STDOUT
+      # Set to true to activate realtime activity. This will cause certain 
+      # information messages to be printed to STDOUT
       attr_accessor :realtime_activity
       
+      # Accessor for the total number of rows read from sources
       attr_accessor :rows_read
-      
       def rows_read
         @rows_read ||= 0
       end
       
+      # Accessor for the total number of rows processed
       attr_accessor :rows_written
-      
       def rows_written
         @rows_written ||= 0
       end
+      
+      # Access the current ETL::Execution::Job instance
+      attr_accessor :job
+      
+      # The limit on rows to load from the source, useful for testing the ETL
+      # process prior to executing the entire batch. Default value is nil and 
+      # indicates that there is no limit
+      attr_accessor :limit
+      
+      # The offset for the source to begin at, useful for testing the ETL
+      # process prior to executing the entire batch. Default value is nil and
+      # indicates that there is no offset
+      attr_accessor :offset
+      
+      # Set to true to skip all bulk importing
+      attr_accessor :skip_bulk_import
+      
+      # Set to true to read locally from the last source cache files
+      attr_accessor :read_locally
+      
+      # Accessor for the average rows per second processed
+      attr_accessor :average_rows_per_second
     end
     
     # Say the specified message, with a newline
@@ -89,6 +135,22 @@ module ETL #:nodoc:
     def errors
       @errors ||= []
     end
+    
+    # Get a Hash of benchmark values where each value represents the total
+    # amount of time in seconds spent processing in that portion of the ETL
+    # pipeline. Keys include:
+    # * <tt>:transforms</tt>
+    # * <tt>:after_reads</tt>
+    # * <tt>:before_writes</tt>
+    # * <tt>:writes</tt>
+    def benchmarks
+      @benchmarks ||= {
+        :transforms => 0,
+        :after_reads => 0,
+        :before_writes => 0,
+        :writes => 0,
+      }
+    end
   
     # Process a control file or object. Acceptable values for control are:
     # * Path to a file
@@ -97,6 +159,11 @@ module ETL #:nodoc:
     def process(control)
       control = ETL::Control::Control.resolve(control)
       
+      ETL::Engine.job = ETL::Execution::Job.create!(
+        :control_file => control.file, 
+        :status => 'executing'
+      )
+      
       execute_dependencies(control)
       
       start_time = Time.now
@@ -108,11 +175,22 @@ module ETL #:nodoc:
       sources = control.sources
       destinations = control.destinations
       
+      say "Skipping bulk import" if Engine.skip_bulk_import
+      
       sources.each do |source|
         Engine.current_source = source
         Engine.logger.debug "Processing source #{source}"
         say "Source: #{source}"
+        say "Limiting enabled: #{Engine.limit}" if Engine.limit != nil
+        say "Offset enabled: #{Engine.offset}" if Engine.offset != nil
         source.each_with_index do |row, index|
+          # Break out of the row loop if the +Engine.limit+ is specified and 
+          # the number of rows read exceeds that value.
+          if Engine.limit != nil && Engine.rows_read >= Engine.limit
+            puts "Reached limit of #{Engine.limit}"
+            break
+          end
+          
           Engine.logger.debug "Row #{index}: #{row.inspect}"
           Engine.rows_read += 1
           Engine.current_source_row = index + 1
@@ -120,80 +198,85 @@ module ETL #:nodoc:
             say_without_newline "."
           end
           
-          # At this point a single row may be turned into multiple rows via row processors
-          # all code after this line should work with the array of rows rather than the
-          # single row
+          # At this point a single row may be turned into multiple rows via row 
+          # processors all code after this line should work with the array of 
+          # rows rather than the single row
           rows = [row]
           
-          begin
-            Engine.logger.debug "Processing after read"
-            control.after_read_processors.each do |processor|
-              processed_rows = []
-              rows.each do |row|
-                processed_rows << processor.process(row)
+          benchmarks[:after_reads] += Benchmark.realtime do
+            begin
+              Engine.logger.debug "Processing after read"
+              control.after_read_processors.each do |processor|
+                processed_rows = []
+                rows.each do |row|
+                  processed_rows << processor.process(row)
+                end
+                rows = processed_rows.flatten
               end
-              rows = processed_rows.flatten
+            rescue => e
+              msg = "Error processing rows after read from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
+              errors << msg
+              Engine.logger.error(msg)
+              exceeded_error_threshold?(control) ? break : next
             end
-          rescue => e
-            msg = "Error processing rows after read from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
-            errors << msg
-            Engine.logger.error(msg)
-            exceeded_error_threshold?(control) ? break : next
           end
           
-          begin
-            # execute transforms
-            Engine.logger.debug "Executing transforms"
-            rows.each do |row|
-              row.each do |name, value|
-                name = name.to_sym
-                transformers = control.transform(name)
-                #Engine.logger.debug "Transforms for #{name}: #{transformers.inspect}"
-                row[name] = ETL::Transform::Transform.transform(name, value, row, transformers)
+          benchmarks[:transforms] += Benchmark.realtime do
+            begin
+              # execute transforms
+              Engine.logger.debug "Executing transforms"
+              rows.each do |row|
+                control.transforms.each do |transform|
+                  name = transform.name.to_sym
+                  row[name] = transform.transform(name, row[name], row)
+                end
               end
+            rescue => e
+              msg = "Error transforming from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
+              errors << msg
+              Engine.logger.error(msg)
+              e.backtrace.each { |line| Engine.logger.error(line) }
+              exceeded_error_threshold?(control) ? break : next
             end
-          rescue => e
-            msg = "Error transforming from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
-            errors << msg
-            Engine.logger.error(msg)
-            e.backtrace.each { |line| Engine.logger.error(line) }
-            exceeded_error_threshold?(control) ? break : next
           end
           
-          
-          begin
-            # execute row-level "before write" processing
-            Engine.logger.debug "Processing before write"
-            control.before_write_processors.each do |processor|
-              processed_rows = []
-              rows.each do |row|
-                processed_rows << processor.process(row)
+          benchmarks[:before_writes] += Benchmark.realtime do
+            begin
+              # execute row-level "before write" processing
+              Engine.logger.debug "Processing before write"
+              control.before_write_processors.each do |processor|
+                processed_rows = []
+                rows.each do |row|
+                  processed_rows << processor.process(row)
+                end
+                rows = processed_rows.flatten.compact
               end
-              rows = processed_rows.flatten
+            rescue => e
+              msg = "Error processing rows before write from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
+              errors << msg
+              Engine.logger.error(msg)
+              e.backtrace.each { |line| Engine.logger.error(line) }
+              exceeded_error_threshold?(control) ? break : next
             end
-          rescue => e
-            msg = "Error processing rows before write from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
-            errors << msg
-            Engine.logger.error(msg)
-            e.backtrace.each { |line| Engine.logger.error(line) }
-            exceeded_error_threshold?(control) ? break : next
           end
           
-          begin
-            # write the row to the destination
-            destinations.each_with_index do |destination, index|
-              Engine.current_destination = destination
-              rows.each do |row|
-                destination.write(row)
-                Engine.rows_written += 1 if index == 0
+          benchmarks[:writes] += Benchmark.realtime do
+            begin
+              # write the row to the destination
+              destinations.each_with_index do |destination, index|
+                Engine.current_destination = destination
+                rows.each do |row|
+                  destination.write(row)
+                  Engine.rows_written += 1 if index == 0
+                end
               end
+            rescue => e
+              msg = "Error writing to #{Engine.current_destination}: #{e}"
+              errors << msg
+              Engine.logger.error msg
+              e.backtrace.each { |line| Engine.logger.error(line) }
+              exceeded_error_threshold?(control) ? break : next
             end
-          rescue => e
-            msg = "Error writing to #{Engine.current_destination}: #{e}"
-            errors << msg
-            Engine.logger.error msg
-            e.backtrace.each { |line| Engine.logger.error(line) }
-            break if exceeded_error_threshold?(control)
           end
         end
         
@@ -220,6 +303,20 @@ module ETL #:nodoc:
         say "Wrote #{Engine.rows_written} lines to destinations"
       end
       say "Completed #{control.file} in #{distance_of_time_in_words(start_time)} with #{errors.length} errors."
+      say "Processing average: #{Engine.average_rows_per_second} rows/sec)"
+      
+      say "Avg after_reads: #{Engine.rows_read/benchmarks[:after_reads]} rows/sec" if benchmarks[:after_reads] > 0
+      say "Avg before_writes: #{Engine.rows_read/benchmarks[:before_writes]} rows/sec" if benchmarks[:before_writes] > 0
+      say "Avg transforms: #{Engine.rows_read/benchmarks[:transforms]} rows/sec" if benchmarks[:transforms] > 0
+      say "Avg writes: #{Engine.rows_read/benchmarks[:writes]} rows/sec" if benchmarks[:writes] > 0
+      
+      # ETL::Transform::Transform.benchmarks.each do |klass, t|
+#         say "Avg #{klass}: #{Engine.rows_read/t} rows/sec"
+#       end
+      
+      ETL::Engine.job.completed_at = Time.now
+      ETL::Engine.job.status = (errors.length > 0 ? 'completed with errors' : 'completed')
+      ETL::Engine.job.save!
     end
     
     private
@@ -285,6 +382,10 @@ module ETL #:nodoc:
       s
     end
     
+    # Get the approximate disntance of time in words from the given from_time
+    # to the the given to_time. If to_time is not specified then it is set
+    # to Time.now. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
     def approximate_distance_of_time_in_words(from_time, to_time=Time.now, include_seconds=true)
       from_time = from_time.to_time if from_time.respond_to?(:to_time)
       to_time = to_time.to_time if to_time.respond_to?(:to_time)