activewarehouse-etl 0.5.2 → 0.6.0

data/lib/etl/control/destination/database_destination.rb

@@ -26,6 +26,7 @@ module ETL #:nodoc:
       # * <tt>:username</tt>: The database username (defaults to 'root')
       # * <tt>:password</tt>: The password to the database (defaults to nothing)
       # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
+      # * <tt>:append_rows</tt>: Array of rows to append
       #
       # Mapping options:
       # * <tt>:order</tt>: The order of fields to write (REQUIRED)
@@ -44,7 +45,7 @@ module ETL #:nodoc:
         conn.transaction do
           conn.truncate(configuration[:table]) if truncate
           
-          buffer.each do |row|
+          buffer.flatten.each do |row|
             # check to see if this row's compound key constraint already exists
             # note that the compound key constraint may not utilize virtual fields
             next unless row_allowed?(row)
@@ -69,6 +70,7 @@ module ETL #:nodoc:
       
       # Close the connection
       def close
+        buffer << append_rows if append_rows
         flush
         ETL::ActiveRecord::Base.connection.disconnect!
       end

data/lib/etl/control/destination/file_destination.rb

@@ -32,6 +32,7 @@ module ETL #:nodoc:
       # * <tt>:eol</tt>: End of line marker (default is \n)
       # * <tt>:enclose</tt>: Enclosure character (default is none)
       # * <tt>:unique</tt>: Set to true to only write unique records
+      # * <tt>:append_rows</tt>: Array of rows to append
       # 
       # Mapping options:
       # * <tt>:order</tt>: The order array
@@ -50,14 +51,15 @@ module ETL #:nodoc:
       
       # Close the destination. This will flush the buffer and close the underlying stream or connection.
       def close
+        buffer << append_rows if append_rows
         flush
         f.close
       end
       
       # Flush the destination buffer
       def flush
-        # Engine.logger.debug "Flushing buffer #{buffer}"
-        buffer.each do |row|
+        #puts "Flushing buffer (#{file}) with #{buffer.length} rows"
+        buffer.flatten.each do |row|
           # check to see if this row's compound key constraint already exists
           # note that the compound key constraint may not utilize virtual fields
           next unless row_allowed?(row)
@@ -80,6 +82,7 @@ module ETL #:nodoc:
           f.write(eol)
         end
         buffer.clear
+        #puts "After flush there are #{buffer.length} rows"
       end
       
       private

data/lib/etl/control/source.rb

@@ -13,6 +13,9 @@ module ETL #:nodoc:
       # The definition Hash
       attr_accessor :definition
       
+      # Returns true if the source data should be stored locally for archival
+      attr_accessor :store_locally
+      
       class << self
         # Convert the name to a Source class.
         # 
@@ -26,15 +29,48 @@ module ETL #:nodoc:
       # * <tt>control</tt>: The control object
       # * <tt>configuration</tt>: The configuration hash
       # * <tt>definition</tt>: The source layout definition
+      #
+      # Configuration options:
+      # * <tt>:store_locally</tt>: Set to false to not store source data locally (defaults to true)
       def initialize(control, configuration, definition)
         @control = control
         @configuration = configuration
         @definition = definition
+        
+        @store_locally = configuration[:store_locally] || true
       end
       
       def errors
         @errors ||= []
       end
+      
+      # Get a timestamp value as a string
+      def timestamp
+        Engine.timestamp
+      end
+      
+      # The base directory where local files are stored.
+      attr_accessor :local_base
+      
+      # Get the local base, defaults to 'source_data'
+      def local_base
+        @local_base ||= 'source_data'
+      end
+      
+      # The local directory for storing. This method must be overriden by subclasses
+      def local_directory
+        raise "local_directory method is abstract"
+      end
+      
+      # Return the local file for storing the raw source data. Each call to this method will
+      # result in a timestamped file, so you cannot expect to call it multiple times and reference
+      # the same file
+      def local_file
+        local_dir = local_directory
+        FileUtils.mkdir_p(local_dir)
+        File.join(local_dir, "#{timestamp}.csv")
+      end
+      
     end
   end
 end

data/lib/etl/control/source/database_source.rb

@@ -1,4 +1,10 @@
+require 'fileutils'
+
 module ETL #:nodoc:
+  class Source < ::ActiveRecord::Base #:nodoc:
+    # Connection for database sources
+  end
+  
   module Control #:nodoc:
     # Source object which extracts data from a database using ActiveRecord.
     class DatabaseSource < Source
@@ -18,24 +24,73 @@ module ETL #:nodoc:
       # * <tt>:username</tt>: The database username (defaults to 'root')
       # * <tt>:password</tt>: The password to the database (defaults to nothing)
       # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
+      # * <tt>:store_locally</tt>: Set to false to not store a copy of the source data locally
+      #   in a flat file (defaults to true)
       def initialize(control, configuration, definition)
         super
         connect
       end
       
+      # Get a String identifier for the source
       def to_s
-        "#{configuration[:host]}/#{configuration[:database]}"
+        "#{host}/#{configuration[:database]}/#{configuration[:table]}"
+      end
+      
+      # Get the local directory to use, which is a combination of the local_base, the db hostname
+      # the db database name and the db table.
+      def local_directory
+        File.join(local_base, host, configuration[:database], configuration[:table])
       end
       
       # Returns each row from the source
       def each
-        conn = ETL::ActiveRecord::Base.connection
-        conn.select_all("SELECT * FROM #{configuration[:table]}").each do |row|
-          yield row
+        if store_locally
+          file = local_file
+          columns = connection.columns(configuration[:table].to_s)
+          FasterCSV.open(file, 'w') do |f|
+            f << columns.collect { |column| column.name }
+            connection.select_all("SELECT * FROM #{configuration[:table]}").each do |row|
+              values = columns.collect { |column| row[column.name] }
+              #puts "row: #{values.inspect}"
+              f << values
+            end
+          end
+          FasterCSV.open(file, :headers => true).each do |row|
+            result_row = {}
+            row.each do |header, field|
+              result_row[header.to_sym] = field
+            end
+            #puts "yielding #{result_row.inspect}"
+            yield result_row
+          end
+        else
+          connection.select_all("SELECT * FROM #{configuration[:table]}").each do |row|
+            yield HashWithIndifferentAccess.new(row)
+          end
         end
       end
       
       private
+      # Get the database connection to use
+      def connection
+        ETL::Source.connection
+      end
+      
+      # Get the adapter name, defaults to :mysql
+      def adapter
+        configuration[:adapter] || :mysql
+      end
+      
+      # Get the host, defaults to 'localhost'
+      def host
+        configuration[:host] || "localhost"
+      end
+      
+      # Get the username, defaults to 'root'
+      def username
+        configuration[:username] || 'root'
+      end
+      
       # Connect to the database. 
       #
       # Required options:
@@ -47,10 +102,10 @@ module ETL #:nodoc:
       # * <tt>:password</tt>: The password to the database (defaults to nothing)
       # * <tt>:host<tt>: The host for the database (defaults to 'localhost')
       def connect
-        ETL::ActiveRecord::Base.establish_connection(
-            :adapter  => (configuration[:adapter] || :mysql),
-            :username => (configuration[:username] || 'root'),
-            :host     => (configuration[:host] || 'localhost'),
+        ETL::Source.establish_connection(
+            :adapter  => adapter,
+            :username => username,
+            :host     => host,
             :password => configuration[:password],
             :database => configuration[:database] 
         )

data/lib/etl/control/source/file_source.rb

@@ -11,25 +11,46 @@ module ETL #:nodoc:
       # Initialize the source
       #
       # Configuration options:
-      # * <tt>:parser</tt>: One of the following: a parser name as a String or symbol, a class which extends from Parser, 
-      #   a Hash with :name and optionally an :options key. Whether or not the parser uses the options is dependent on 
-      #   which parser is used. See the documentation for each parser for information on what options it accepts.
+      # * <tt>:file</tt>: The source file
+      # * <tt>:parser</tt>: One of the following: a parser name as a String or symbol, a class which extends 
+      #   from Parser, a Hash with :name and optionally an :options key. Whether or not the parser uses the 
+      #   options is dependent on which parser is used. See the documentation for each parser for information 
+      #   on what options it accepts.
       # * <tt>:skip_lines<tt>: The number of lines to skip (defaults to 0)
+      # * <tt>:store_locally</tt>: Set to false to not store a copy of the source data locally for archival
       def initialize(control, configuration, definition)
         super
         configure
       end
       
+      # Get a String identifier for the source
       def to_s
         configuration[:file]
       end
       
+      # Get the local storage directory
+      def local_directory
+        File.join(local_base, File.basename(configuration[:file], File.extname(configuration[:file])))
+      end
+      
       # Returns each row from the source
       def each
-        @parser.each { |row| yield row }
+        copy_sources if store_locally
+        @parser.each do |row|
+          yield row
+        end
       end
       
       private
+      # Copy source data to a local directory structure
+      def copy_sources
+        source_file = File.join(File.dirname(control.file), configuration[:file])
+        Dir.glob(source_file).each do |f|
+          next if File.directory?(f)
+          FileUtils.cp(f, local_file)
+        end
+      end
+      
       # Configure the source
       def configure
         case @configuration[:parser]

data/lib/etl/engine.rb

@@ -9,7 +9,6 @@ module ETL #:nodoc:
   
   # The main ETL engine clas
   class Engine
-  
     class << self
       # Process the specified control file. Acceptable values for control_file are
       # * Path to a file
@@ -24,23 +23,49 @@ module ETL #:nodoc:
       
       def logger #:nodoc:
         unless @logger
-          @logger = Logger.new('etl.log')
+          @logger = Logger.new("etl_#{timestamp}.log")
           @logger.level = Logger::WARN
         end
         @logger
       end
       
+      # Get a timestamp value as a string
+      def timestamp
+        Time.now.strftime("%Y%m%d%H%M%S")
+      end
+
+      # The current source
       attr_accessor :current_source
+
+      # The current source row
       attr_accessor :current_source_row
+      
+      # The current destination
       attr_accessor :current_destination
       
+      # Set to true to activate realtime activity. This will cause certain information messages
+      # to be printed to STDOUT
       attr_accessor :realtime_activity
+      
+      attr_accessor :rows_read
+      
+      def rows_read
+        @rows_read ||= 0
+      end
+      
+      attr_accessor :rows_written
+      
+      def rows_written
+        @rows_written ||= 0
+      end
     end
     
+    # Say the specified message, with a newline
     def say(message)
       say_without_newline(message + "\n")
     end
     
+    # Say the specified message without a newline
     def say_without_newline(message)
       if Engine.realtime_activity
         $stdout.print message
@@ -48,10 +73,12 @@ module ETL #:nodoc:
       end
     end
     
+    # Say the message on its own line
     def say_on_own_line(message)
       say("\n" + message)
     end
     
+    # Array of errors encountered during execution of the ETL process
     def errors
       @errors ||= []
     end
@@ -61,9 +88,12 @@ module ETL #:nodoc:
     # * File object
     # * ETL::Control::Control instance
     def process(control)
-      start_time = Time.now
       control = ETL::Control::Control.resolve(control)
       
+      execute_dependencies(control)
+      
+      start_time = Time.now
+      
       Engine.logger.debug "Pre-processing #{control.file}"
       pre_process(control)
       Engine.logger.debug "Pre-processing complete"
@@ -76,40 +106,93 @@ module ETL #:nodoc:
         Engine.logger.debug "Processing source #{source}"
         say "Source: #{source}"
         source.each_with_index do |row, index|
+          Engine.logger.debug "Row #{index}: #{row.inspect}"
+          Engine.rows_read += 1
           Engine.current_source_row = index + 1
           if Engine.realtime_activity && index > 0 && index % 1000 == 0
             say_without_newline "."
           end
           
+          # At this point a single row may be turned into multiple rows via row processors
+          # all code after this line should work with the array of rows rather than the
+          # single row
+          rows = [row]
+          
+          begin
+            Engine.logger.debug "Processing after read"
+            control.after_read_processors.each do |processor|
+              processed_rows = []
+              rows.each do |row|
+                processed_rows << processor.process(row)
+              end
+              rows = processed_rows.flatten
+            end
+          rescue => e
+            msg = "Error processing rows after read from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
+            errors << msg
+            Engine.logger.error(msg)
+            exceeded_error_threshold?(control) ? break : next
+          end
+          
           begin
             # execute transforms
-            row.each do |name, value|
-              row[name] = ETL::Transform::Transform.transform(name, value, control.transform(name))
+            Engine.logger.debug "Executing transforms"
+            rows.each do |row|
+              row.each do |name, value|
+                name = name.to_sym
+                transformers = control.transform(name)
+                #Engine.logger.debug "Transforms for #{name}: #{transformers.inspect}"
+                row[name] = ETL::Transform::Transform.transform(name, value, row, transformers)
+              end
             end
           rescue => e
-            msg = "Error transforming from #{source} on line #{index}: #{e}"
+            msg = "Error transforming from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
             errors << msg
-            Engine.logger.error msg
-            break if exceeded_error_threshold?(control)
+            Engine.logger.error(msg)
+            e.backtrace.each { |line| Engine.logger.error(line) }
+            exceeded_error_threshold?(control) ? break : next
+          end
+          
+          
+          begin
+            # execute row-level "before write" processing
+            Engine.logger.debug "Processing before write"
+            control.before_write_processors.each do |processor|
+              processed_rows = []
+              rows.each do |row|
+                processed_rows << processor.process(row)
+              end
+              rows = processed_rows.flatten
+            end
+          rescue => e
+            msg = "Error processing rows before write from #{Engine.current_source} on line #{Engine.current_source_row}: #{e}"
+            errors << msg
+            Engine.logger.error(msg)
+            e.backtrace.each { |line| Engine.logger.error(line) }
+            exceeded_error_threshold?(control) ? break : next
           end
           
           begin
             # write the row to the destination
-            destinations.each do |destination|
+            destinations.each_with_index do |destination, index|
               Engine.current_destination = destination
-              destination.write(row)
+              rows.each do |row|
+                destination.write(row)
+                Engine.rows_written += 1 if index == 0
+              end
             end
-          rescue
-            msg = "Error writing to #{destination} on line #{index}"
+          rescue => e
+            msg = "Error writing to #{Engine.current_destination}: #{e}"
             errors << msg
             Engine.logger.error msg
+            e.backtrace.each { |line| Engine.logger.error(line) }
             break if exceeded_error_threshold?(control)
           end
         end
+        
         if exceeded_error_threshold?(control)
           say_on_own_line "Exiting due to exceeding error threshold: #{control.error_threshold}"
-        else
-          say_on_own_line "Processed #{Engine.current_source_row} rows in #{distance_of_time_in_words(start_time)} with #{errors} errors."
+          return
         end
         
         destinations.each do |destination|
@@ -117,12 +200,23 @@ module ETL #:nodoc:
         end
       end
       
+      say_on_own_line "Executing post processes"
       Engine.logger.debug "Post-processing #{control.file}"
       post_process(control)
       Engine.logger.debug "Post-processing complete"
+      say "Post-processing complete"
+      
+      if sources.length > 0
+        say_on_own_line "Read #{Engine.rows_read} lines from sources"
+      end
+      if destinations.length > 0
+        say "Wrote #{Engine.rows_written} lines to destinations"
+      end
+      say "Completed #{control.file} in #{distance_of_time_in_words(start_time)} with #{errors.length} errors."
     end
     
     private
+    # Return true if the error threshold is exceeded
     def exceeded_error_threshold?(control)
       errors.length > control.error_threshold
     end
@@ -141,6 +235,26 @@ module ETL #:nodoc:
       end
     end
     
+    # Execute all dependencies
+    def execute_dependencies(control)
+      Engine.logger.debug "Executing dependencies"
+      control.dependencies.flatten.each do |dependency|
+        case dependency
+        when Symbol
+          f = dependency.to_s + '.ctl'
+          Engine.logger.debug "Executing dependency: #{f}"
+          say "Executing dependency: #{f}"
+          process(f)
+        when String
+          Engine.logger.debug "Executing dependency: #{f}"
+          say "Executing dependency: #{f}"
+          process(dependency)
+        else
+          raise "Invalid dependency type: #{dependency.class}"
+        end
+      end
+    end
+    
     # Return the distance of time in words from the given from_time to the specified to_time. If to_time
     # is not specified then Time.now is used. By default seconds are included...set the include_seconds
     # argument to false to disable the seconds.