aeden-activewarehouse-etl 0.9.1

data/etl/lib/etl/control.rb

@@ -0,0 +1,3 @@
+require 'etl/control/control'
+require 'etl/control/source'
+require 'etl/control/destination'

data/etl/lib/etl/control/control.rb

@@ -0,0 +1,403 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # The Context is passed to eval. 
+    class Context
+      require 'test/unit/assertions'
+      include Test::Unit::Assertions
+      attr_reader :control
+      
+      class << self
+        # Create a Context instance
+        def create(control)
+          Context.new(control).get_binding
+        end
+      end
+      
+      # Initialize the context
+      def initialize(control)
+        @control = control
+      end
+      
+      # Get the control file
+      def file
+        control.file
+      end
+      
+      # Set the allowed error threshold
+      def set_error_threshold(error_threshold)
+        control.error_threshold = error_threshold
+      end
+      
+      # Define a list of control files that this file depends on. Those control
+      # files will be executed prior to this control file. The list may 
+      # contain symbols that will be converted to file names by calling 
+      # to_s + '.ctl', or they may be strings in which case they will be used 
+      # as is
+      def depends_on(*args)
+        (dependencies << args).flatten!
+      end
+      
+      # Get the defined dependencies
+      def dependencies
+        control.dependencies
+      end
+      
+      # Define a source.
+      def source(name, configuration={}, definition={})
+        if configuration[:type]
+          case configuration[:type]
+          when Class
+            source_class = configuration[:type]
+            sources << source_class.new(self, configuration, definition)
+          when String, Symbol
+            source_class = ETL::Control::Source.class_for_name(configuration[:type])
+            sources << source_class.new(self, configuration, definition)
+          else
+            if configuration[:type].is_a?(ETL::Control::Source)
+              sources << configuration[:type]
+            else
+              raise ControlError, "Type must be a Class, String, Symbol or object extending ETL::Control::Source"
+            end
+          end
+        else
+          source_types.each do |source_type|
+            if configuration[source_type]
+              source_class = ETL::Control::Source.class_for_name(source_type)
+              sources << source_class.new(self, configuration, definition)
+              break
+            end
+            raise ControlError, "A source was specified but no matching type was found"
+          end
+        end
+      end
+      
+      # Get the defined source
+      def sources
+        control.sources
+      end
+      
+      # Define a destination
+      def destination(name, configuration={}, mapping={})
+        if configuration[:type]
+          case configuration[:type]
+          when Class
+            dest_class = configuration[:type]
+            destinations << dest_class.new(self, configuration, mapping)
+          when String, Symbol
+            dest_class = ETL::Control::Destination.class_for_name(configuration[:type])
+            destinations << dest_class.new(self, configuration, mapping)
+          else
+            if configuration[:type].is_a?(ETL::Control::Destination)
+              destinations << configuration[:type]
+            else
+              raise ControlError, "Type must be a Class, String, Symbol or object extending ETL::Control::Destination"
+            end
+          end
+        else
+          destination_types.each do |dest_type|
+            if configuration[dest_type]
+              dest_class = ETL::Control::Destination.class_for_name(dest_type)
+              destinations << dest_class.new(self, configuration, mapping)
+              break
+            end
+            raise ControlError, "A destination was specified but no matching destination type was found"
+          end
+        end
+      end
+      
+      # Get the defined destinations
+      def destinations
+        control.destinations
+      end
+      
+      # Define a transform
+      def transform(name, transformer=nil, configuration={}, &block)
+        if transformer
+          case transformer
+          when String, Symbol
+            class_name = "#{transformer.to_s.camelize}Transform"
+            begin
+              transform_class = ETL::Transform.const_get(class_name)
+              transforms << transform_class.new(self, name, configuration)
+            rescue NameError => e
+              raise ControlError, "Unable to find transformer #{class_name}: #{e}"
+            end
+          else
+            #transformer.class.inspect
+            if transformer.is_a?(ETL::Transform::Transform)
+              Engine.logger.debug "Adding transformer #{transformer.inspect} for field #{name}"
+              t = transformer.dup
+              t.name = name
+              transforms << t 
+            else
+              raise ControlError, "Transformer must be a String, Symbol or Transform instance"
+            end
+          end
+        elsif block_given?
+          transforms << ETL::Transform::BlockTransform.new(self, name, :block => block)
+        else
+          raise ControlError, "Either a transformer or a block must be specified"
+        end
+      end
+      
+      # Get the defined transforms
+      def transforms
+        control.transforms
+      end
+      
+      # Define a before post-process screen block. The type argument must be
+      # one of :fatal, :error or :warn
+      def screen(type, &block)
+        screens[type] << block
+      end
+      
+      # Get the before post-process screen blocks
+      def screens
+        control.screens
+      end
+      
+      # Define an after post-proces screen block. The type argument must be
+      # one of :fatal, :error or :warn
+      def after_post_process_screen(type, &block)
+        after_post_process_screens[type] << block
+      end
+      
+      # Get the after post-process screen blocks
+      def after_post_process_screens
+        control.after_post_process_screens
+      end
+      
+      # Rename the source field to the destination field
+      def rename(source, destination)
+        after_read :rename, :source => source, :dest => destination
+      end
+      
+      # Copy the source field to the destination field
+      def copy(source, destination)
+        after_read :copy_field, :source => source, :dest => destination
+      end
+      
+      protected
+      # This method is used to define a processor and insert into the specified processor
+      # collection.
+      def define_processor(name, processor_collection, configuration, proc)
+        case name
+        when String, Symbol, nil
+          name ||= 'block'
+          class_name = "#{name.to_s.camelize}Processor"
+          begin
+            processor_class = ETL::Processor.const_get(class_name)
+            if name == 'block'
+              raise ControlError, "A block must be passed for block processor" if proc.nil?
+              configuration[:block] = proc
+            end
+            processor_collection << processor_class.new(self, configuration)
+          rescue NameError => e
+            raise ControlError, "Unable to find processor #{class_name}: #{e}"
+          end
+        when Class
+          processor_collection << name.new(self, configuration)
+        else
+          raise ControlError, "The process declaration requires a String, Symbol or Class, or a Block to be passed"
+        end
+      end
+      
+      public
+      # Define an "after read" processor. This must be a row-level processor.
+      def after_read(name='block', configuration={}, &block)
+        define_processor(name, after_read_processors, configuration, block)
+      end
+      
+      # Get the defined "after read" processors
+      def after_read_processors
+        control.after_read_processors
+      end
+      
+      # Define a "before write" processor. This must be a row-level processor.
+      def before_write(name='block', configuration={}, &block)
+        define_processor(name, before_write_processors, configuration, block)
+      end
+      
+      # Get the defined "before write" processors
+      def before_write_processors
+        control.before_write_processors
+      end
+      
+      # Define a pre-processor
+      def pre_process(name='block', configuration={}, &block)
+        define_processor(name, pre_processors, configuration, block)
+      end
+      
+      # Get the defined pre-processors
+      def pre_processors
+        control.pre_processors
+      end
+      
+      # Define a post-processor
+      def post_process(name='block', configuration={}, &block)
+        define_processor(name, post_processors, configuration, block)
+      end
+      
+      # Get the defined post-processors
+      def post_processors
+        control.post_processors
+      end
+      
+      # Get the binding object
+      def get_binding
+        binding
+      end
+      
+      protected
+      # Get an array of supported source types
+      def source_types
+        control.source_types
+      end
+      
+      # Get an array of supported destination types
+      def destination_types
+        control.destination_types
+      end
+      
+    end
+    
+    # Object representation of a control file
+    class Control
+      # The File object
+      attr_reader :file
+      
+      # The error threshold
+      attr_accessor :error_threshold
+      
+      class << self
+        # Parse a control file and return a Control instance
+        def parse(control_file)
+          control_file = control_file.path if control_file.instance_of?(File)
+          control = ETL::Control::Control.new(control_file)
+          # TODO: better handling of parser errors. Return the line in the control file where the error occurs.
+          eval(IO.readlines(control_file).join("\n"), Context.create(control), control_file)
+          control.validate
+          control
+        end
+        
+        def parse_text(text)
+          control = ETL::Control::Control.new(nil)
+          eval(text, Context.create(control), 'inline')
+          control.validate
+          control
+        end
+        
+        # Resolve the given object to an ETL::Control::Control instance. Acceptable arguments
+        # are:
+        # * The path to a control file as a String
+        # * A File object referencing the control file
+        # * The ETL::Control::Control object (which will just be returned)
+        #
+        # Raises a ControlError if any other type is given
+        def resolve(control)
+          case control
+          when String
+            ETL::Control::Control.parse(File.new(control))
+          when File
+            ETL::Control::Control.parse(control)
+          when ETL::Control::Control
+            control
+          else
+            raise ControlError, "Control must be a String, File or Control object"
+          end
+        end
+      end
+      
+      # Initialize the instance with the given File object
+      def initialize(file)
+        @file = file
+      end
+      
+      # Get a list of dependencies
+      def dependencies
+        @dependencies ||= []
+      end
+      
+      # Get the defined source
+      def sources
+        @sources ||= []
+      end
+      
+      # Get the defined destinations
+      def destinations
+        @destinations ||= []
+      end
+      
+      # Get the transforms with the specified name
+      # def transform(name)
+#         transforms[name] ||= []
+#       end
+      
+      def after_read_processors
+        @after_read_processors ||= []
+      end
+      
+      # Get all of the "before write" processors
+      def before_write_processors
+        @before_write_processors ||= []
+      end
+      
+      # Get an Array of preprocessors
+      def pre_processors
+        @pre_processors ||= []
+      end
+      
+      # Get an Array of post processors
+      def post_processors
+        @post_processors ||= []
+      end
+      
+      # Get an Array of all transforms for this control
+      def transforms
+        @transforms ||= []
+      end
+      
+      # A hash of the screens executed before post-process
+      def screens
+        @screens ||= {
+          :fatal => [],
+          :error => [],
+          :warn => []
+        }
+      end
+      
+      # A hash of the screens executed after post-process
+      def after_post_process_screens
+        @after_post_process_screens ||= {
+          :fatal => [],
+          :error => [],
+          :warn => []
+        }
+      end
+      
+      # Get the error threshold. Defaults to 100.
+      def error_threshold
+        @error_threshold ||= 100
+      end
+      
+      # Validate the control file
+      def validate
+        #unless sources.length > 0
+        #  raise ControlError, "Configuration must include one of the following for the source: #{source_types.join(',')}"
+        #end
+        #unless destinations.length > 0
+        #  raise ControlError, "Configuration must include one of the following for the destination: #{destination_types.join(',')}"
+        #end
+      end
+      
+      def source_types
+        [:file, :database]
+      end
+      
+      def destination_types
+        [:file, :database]
+      end
+      
+    end
+  end
+end

data/etl/lib/etl/control/destination.rb

@@ -0,0 +1,420 @@
+module ETL #:nodoc:
+  module Control #:nodoc:
+    # Base class for destinations.
+    class Destination
+      # Read-only accessor for the ETL::Control::Control instance
+      attr_reader :control
+      
+      # Read-only accessor for the configuration Hash
+      attr_reader :configuration
+      
+      # Read-only accessor for the destination mapping Hash
+      attr_reader :mapping
+      
+      # Accessor to the buffer size
+      attr_accessor :buffer_size
+      
+      # Unique flag.
+      attr_accessor :unique
+      
+      # A condition for writing
+      attr_accessor :condition
+      
+      # An array of rows to append to the destination
+      attr_accessor :append_rows
+      
+      class << self
+        # Get the destination class for the specified name.
+        # 
+        # For example if name is :database or 'database' then the 
+        # DatabaseDestination class is returned
+        def class_for_name(name)
+          ETL::Control.const_get("#{name.to_s.camelize}Destination")
+        end
+      end
+      
+      # Initialize the destination
+      #
+      # Arguments:
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping Hash
+      #
+      # Options:
+      # * <tt>:buffer_size</tt>: The output buffer size (default 1000 records)
+      # * <tt>:condition</tt>: A conditional proc that must return true for the 
+      #   row to be written
+      # * <tt>:append_rows</tt>: An array of rows to append
+      def initialize(control, configuration, mapping)
+        @control = control
+        @configuration = configuration
+        @mapping = mapping
+        @buffer_size = configuration[:buffer_size] ||= 100
+        @condition = configuration[:condition]
+        @append_rows = configuration[:append_rows]
+      end
+      
+      # Get the current row number
+      def current_row
+        @current_row ||= 1
+      end
+      
+      # Write the given row
+      def write(row)
+        if @condition.nil? || @condition.call(row)
+          process_change(row)
+        end
+        flush if buffer.length >= buffer_size
+      end
+      
+      # Abstract method
+      def flush
+        raise NotImplementedError, "flush method must be implemented by subclasses"
+      end
+      
+      # Abstract method
+      def close
+        raise NotImplementedError, "close method must be implemented by subclasses"
+      end
+      
+      def errors
+        @errors ||= []
+      end
+      
+      protected
+      # Access the buffer
+      def buffer
+        @buffer ||= []
+      end
+      
+      # Access the generators map
+      def generators
+        @generators ||= {}
+      end
+      
+      # Get the order of elements from the source order
+      def order_from_source
+        order = []
+        control.sources.first.definition.each do |item|
+          case item
+          when Hash
+            order << item[:name]
+          else
+            order << item
+          end
+        end
+        order
+      end
+      
+      # Return true if the row is allowed. The row will not be allowed if the
+      # :unique option is specified in the configuration and the compound key 
+      # already exists
+      def row_allowed?(row)
+        if unique
+          key = (unique.collect { |k| row[k] }).join('|')
+          return false if compound_key_constraints[key]
+          compound_key_constraints[key] = 1
+        end
+        return true
+      end
+      
+      # Get a hash of compound key contraints. This is used to determine if a
+      # row can be written when the unique option is specified
+      def compound_key_constraints
+        @compound_key_constraints ||= {}
+      end
+      
+      # Return fields which are Slowly Changing Dimension fields. 
+      # Uses the scd_fields specified in the configuration.  If that's
+      # missing, uses all of the row's fields.
+      def scd_fields(row)
+        @scd_fields ||= configuration[:scd_fields] || row.keys
+      end
+      
+      def non_scd_fields(row)
+        @non_csd_fields ||= row.keys - natural_key - scd_fields(row) -
+          [primary_key, scd_effective_date_field, scd_end_date_field, scd_latest_version_field]
+      end
+      
+      def non_evolving_fields
+        (Array(configuration[:scd][:non_evolving_fields]) << primary_key).uniq
+      end
+      
+      def scd?
+        !configuration[:scd].nil?
+      end
+      
+      def scd_type
+        scd? ? configuration[:scd][:type] : nil
+      end
+      
+      # Get the Slowly Changing Dimension effective date field. Defaults to
+      # 'effective_date'.
+      def scd_effective_date_field
+        configuration[:scd][:effective_date_field] || :effective_date if scd?
+      end
+      
+      # Get the Slowly Changing Dimension end date field. Defaults to 
+      # 'end_date'.
+      def scd_end_date_field
+        configuration[:scd][:end_date_field] || :end_date if scd?
+      end
+      
+      # Get the Slowly Changing Dimension latest version field. Defaults to
+      # 'latest_version'.
+      def scd_latest_version_field
+        configuration[:scd][:latest_version_field] || :latest_version if scd?
+      end
+      
+      # Return the natural key field names, defaults to []
+      def natural_key
+        @natural_key ||= determine_natural_key
+      end
+      
+      # Get the dimension table if specified
+      def dimension_table
+        @dimension_table ||= if scd?
+          ETL::Engine.table(configuration[:scd][:dimension_table], dimension_target) or raise ConfigurationError, "dimension_table setting required" 
+        end
+      end
+      
+      # Get the dimension target if specified
+      def dimension_target
+        @dimension_target ||= if scd?
+          configuration[:scd][:dimension_target] or raise ConfigurationError, "dimension_target setting required"
+        end
+      end
+      
+      # Process a row to determine the change type
+      def process_change(row)
+        ETL::Engine.logger.debug "Processing row: #{row.inspect}"
+        return unless row
+        
+        # Change processing can only occur if the natural key exists in the row 
+        ETL::Engine.logger.debug "Checking for natural key existence"
+        unless has_natural_key?(row)
+          buffer << row
+          return
+        end
+        
+        @timestamp = Time.now
+
+        # See if the scd_fields of the current record have changed
+        # from the last time this record was loaded into the data
+        # warehouse. If they match then throw away this row (no need
+        # to process). If they do not match then the record is an
+        # 'update'. If the record doesn't exist then it is an 'insert'
+        ETL::Engine.logger.debug "Checking record for SCD change"
+        if @existing_row = preexisting_row(row)
+          if has_scd_field_changes?(row)
+            process_scd_change(row)
+          else
+            process_scd_match(row)
+          end
+        else
+          schedule_new_record(row)
+        end
+      end
+      
+      # Add any virtual fields to the row. Virtual rows will get their value 
+      # from one of the following:
+      # * If the mapping is a Class, then an object which implements the next 
+      #   method
+      # * If the mapping is a Symbol, then the XGenerator where X is the 
+      #   classified symbol
+      # * If the mapping is a Proc, then it will be called with the row
+      # * Otherwise the value itself will be assigned to the field
+      def add_virtuals!(row)
+        if mapping[:virtual]
+          mapping[:virtual].each do |key,value|
+            # If the row already has the virtual set, assume that's correct
+            next if row[key]
+            # Engine.logger.debug "Mapping virtual #{key}/#{value} for row #{row}"
+            case value
+            when Class
+              generator = generators[key] ||= value.new
+              row[key] = generator.next
+            when Symbol
+              generator = generators[key] ||= ETL::Generator::Generator.class_for_name(value).new(options)
+              row[key] = generator.next
+            when Proc
+              row[key] = value.call(row)
+            else
+              if value.is_a?(ETL::Generator::Generator)
+                row[key] = value.next
+              else
+                row[key] = value
+              end
+            end
+          end
+        end
+      end
+      
+      private
+      
+      # Determine the natural key. This method will always return an array
+      # of symbols. The default value is [].
+      def determine_natural_key
+        Array(configuration[:natural_key]).collect(&:to_sym)
+      end
+      
+      # Check whether a natural key has been defined, and if so, whether
+      # this row has enough information to do searches based on that natural
+      # key.
+      # 
+      # TODO: This should be factored out into
+      # ETL::Row#has_all_fields?(field_array) But that's not possible
+      # until *all* sources cast to ETL::Row, instead of sometimes
+      # using Hash
+      def has_natural_key?(row)
+        natural_key.any? && natural_key.all? { |key| row.has_key?(key) }
+      end
+      
+      # Helper for generating the SQL where clause that allows searching
+      # by a natural key
+      def natural_key_equality_for_row(row)
+        statement = []
+        values = []
+        natural_key.each do |nk|
+          statement << "#{nk} = ?"
+          values << row[nk]
+        end
+        statement = statement.join(" AND ")
+        ActiveRecord::Base.send(:sanitize_sql, [statement, *values])
+      end
+      
+      # Do all the steps required when a SCD *has* changed.  Exact steps
+      # depend on what type of SCD we're handling.
+      def process_scd_change(row)
+        ETL::Engine.logger.debug "SCD fields do not match"
+        
+        if scd_type == 2
+          # SCD Type 2: new row should be added and old row should be updated
+          ETL::Engine.logger.debug "type 2 SCD"
+          
+          # To update the old row, we delete the version in the database
+          # and insert a new expired version
+          
+          # If there is no truncate then the row will exist twice in the database
+          delete_outdated_record
+          
+          ETL::Engine.logger.debug "expiring original record"
+          @existing_row[scd_end_date_field] = @timestamp
+          @existing_row[scd_latest_version_field] = false
+          
+          buffer << @existing_row
+
+        elsif scd_type == 1
+          # SCD Type 1: only the new row should be added
+          ETL::Engine.logger.debug "type 1 SCD"
+
+          # Copy primary key, and other non-evolving fields over from
+          # original version of record
+          non_evolving_fields.each do |non_evolving_field|            
+            row[non_evolving_field] = @existing_row[non_evolving_field]
+          end
+          
+          # If there is no truncate then the row will exist twice in the database
+          delete_outdated_record
+        else
+          # SCD Type 3: not supported
+          ETL::Engine.logger.debug "SCD type #{scd_type} not supported"
+        end
+        
+        # In all cases, the latest, greatest version of the record
+        # should go into the load
+        schedule_new_record(row)
+      end
+      
+      # Do all the steps required when a SCD has *not* changed.  Exact
+      # steps depend on what type of SCD we're handling.
+      def process_scd_match(row)
+        ETL::Engine.logger.debug "SCD fields match"
+        
+        if scd_type == 2 && has_non_scd_field_changes?(row)
+          ETL::Engine.logger.debug "Non-SCD field changes"
+          # Copy important data over from original version of record
+          row[primary_key]              = @existing_row[primary_key]
+          row[scd_end_date_field]       = @existing_row[scd_end_date_field]
+          row[scd_effective_date_field] = @existing_row[scd_effective_date_field]
+          row[scd_latest_version_field] = @existing_row[scd_latest_version_field]
+
+          # If there is no truncate then the row will exist twice in the database
+          delete_outdated_record
+          
+          buffer << row
+        else
+          # The record is totally the same, so skip it
+        end
+      end
+      
+      # Find the version of this row that already exists in the datawarehouse.
+      def preexisting_row(row)
+        q = "SELECT * FROM #{dimension_table} WHERE #{natural_key_equality_for_row(row)}"
+        q << " AND #{scd_latest_version_field}" if scd_type == 2
+        
+        #puts "looking for original record"
+        result = connection.select_one(q)
+        
+        #puts "Result: #{result.inspect}"
+        
+        result ? ETL::Row[result.symbolize_keys!] : nil
+      end
+      
+      # Check whether non-scd fields have changed since the last
+      # load of this record.
+      def has_scd_field_changes?(row)
+        scd_fields(row).any? { |csd_field| row[csd_field].to_s != @existing_row[csd_field].to_s }
+      end
+      
+      # Check whether non-scd fields have changed since the last
+      # load of this record.
+      def has_non_scd_field_changes?(row)
+        non_scd_fields(row).any? { |non_csd_field| row[non_csd_field].to_s != @existing_row[non_csd_field].to_s }
+      end
+      
+      # Grab, or re-use, a database connection for running queries directly
+      # during the destination processing.
+      def connection
+        @conn ||= ETL::Engine.connection(dimension_target)
+      end
+      
+      # Utility for removing a row that has outdated information.  Note
+      # that this deletes directly from the database, even if this is a file
+      # destination.  It needs to do this because you can't do deletes in a 
+      # bulk load.
+      def delete_outdated_record
+        ETL::Engine.logger.debug "deleting old row"
+        
+        q = "DELETE FROM #{dimension_table} WHERE #{primary_key} = #{@existing_row[primary_key]}"
+        connection.delete(q)
+      end
+      
+      # Schedule the latest, greatest version of the row for insertion
+      # into the database
+      def schedule_new_record(row)
+        ETL::Engine.logger.debug "writing new record"
+        if scd_type == 2
+          row[scd_effective_date_field] = @timestamp
+          row[scd_end_date_field] = '9999-12-31 00:00:00'
+          row[scd_latest_version_field] = true
+        end
+        buffer << row
+      end
+      
+      # Get the name of the primary key for this table.  Asks the dimension
+      # model class for this information, but if that class hasn't been 
+      # defined, just defaults to :id.
+      def primary_key
+        return @primary_key if @primary_key
+        @primary_key = dimension_table.to_s.camelize.constantize.primary_key.to_sym
+      rescue NameError => e
+        ETL::Engine.logger.debug "couldn't get primary_key from dimension model class, using default :id"
+        @primary_key = :id
+      end
+
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/destination/*.rb"].each { |file| require(file) }