aeden-activewarehouse-etl 0.9.1

data/etl/lib/etl/processor/truncate_processor.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A processor which will truncate a table. Use as a pre-processor for cleaning out a table
+    # prior to loading
+    class TruncateProcessor < ETL::Processor::Processor
+      # Defines the table to truncate
+      attr_reader :table
+      
+      # Defines the database connection to use
+      attr_reader :target
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The table name
+      def initialize(control, configuration)
+        super
+        #@file = File.join(File.dirname(control.file), configuration[:file])
+        @target = configuration[:target] || {}
+        @table = configuration[:table]
+      end
+      
+      def process
+        conn = ETL::Engine.connection(target)
+        conn.truncate(table_name)
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/etl/lib/etl/row.rb

@@ -0,0 +1,20 @@
+# This source file contains the ETL::Row class.
+
+module ETL #:nodoc:
+  # This class represents a single row currently passing through the ETL pipeline
+  class Row < Hash
+    # Accessor for the originating source
+    attr_accessor :source
+    
+    # All change types
+    CHANGE_TYPES = [:insert, :update, :delete]
+    
+    # Accessor for the row's change type
+    attr_accessor :change_type
+    
+    # Get the change type, defaults to :insert
+    def change_type
+      @change_type ||= :insert
+    end
+  end
+end

data/etl/lib/etl/screen.rb

@@ -0,0 +1,14 @@
+# This source file contains the ETL::Screen module and requires all of the 
+# screens
+
+module ETL #:nodoc:
+  # The ETL::Screen module contains pre-built screens useful for checking the 
+  # ETL state during execution. Screens may be fatal, which will result in 
+  # termination of the ETL process, errors, which will result in the
+  # termination of just the current ETL control file, or warnings, which will
+  # result in a warning message.
+  module Screen
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/screen/*.rb"].each { |file| require(file) }

data/etl/lib/etl/screen/row_count_screen.rb

@@ -0,0 +1,20 @@
+module ETL
+  module Screen
+    # This screen validates the number of rows which will be bulk loaded
+    # against the results from some sort of a row count query. If there 
+    # is a difference then the screen will not pass
+    class RowCountScreen
+      attr_accessor :control, :configuration
+      def initialize(control, configuration={})
+        @control = control
+        @configuration = configuration
+        execute
+      end
+      def execute
+        unless Engine.rows_written == configuration[:rows]
+          raise "Rows written (#{Engine.rows_written}) does not match expected rows (#{configuration[:rows]})"
+        end
+      end
+    end
+  end
+end

data/etl/lib/etl/transform.rb

@@ -0,0 +1,2 @@
+require 'etl/transform/transform'
+Dir[File.dirname(__FILE__) + "/transform/*.rb"].each { |file| require(file) }

data/etl/lib/etl/transform/block_transform.rb

@@ -0,0 +1,13 @@
+module ETL
+  module Transform
+    class BlockTransform < ETL::Transform::Transform
+      def initialize(control, name, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def transform(name, value, row)
+        @block.call(name, value, row)
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/date_to_string_transform.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a Date or Time to a formatted string instance
+    class DateToStringTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      #
+      # Configuration options:
+      # * <tt>:format</tt>: A format passed to strftime. Defaults to %Y-%m-%d
+      def initialize(control, name, configuration={})
+        super
+        @format = configuration[:format] || "%Y-%m-%d"
+      end
+      # Transform the value using strftime
+      def transform(name, value, row)
+        return value unless value.respond_to?(:strftime)
+        value.strftime(@format)
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/decode_transform.rb

@@ -0,0 +1,51 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which decodes coded values
+    class DecodeTransform < ETL::Transform::Transform
+      attr_accessor :decode_table_path
+      
+      attr_accessor :decode_table_delimiter
+      
+      attr_accessor :default_value
+      
+      # Initialize the transformer
+      #
+      # Configuration options:
+      # * <tt>:decode_table_path</tt>: The path to the decode table (defaults to 'decode.txt')
+      # * <tt>:decode_table_delimiter</tt>: The decode table delimiter (defaults to ':')
+      # * <tt>:default_value</tt>: The default value to use (defaults to 'No Value')
+      def initialize(control, name, configuration={})
+        super
+        
+        if configuration[:decode_table_path]
+          configuration[:decode_table_path] = File.join(File.dirname(control.file), configuration[:decode_table_path])
+        end
+        
+        @decode_table_path = (configuration[:decode_table_path] || 'decode.txt')
+        @decode_table_delimiter = (configuration[:decode_table_delimiter] || ':')
+        @default_value = (configuration[:default_value] || 'No Value')
+      end
+      
+      # Transform the value
+      def transform(name, value, row)
+        decode_table[value] || default_value
+      end
+      
+      # Get the decode table
+      def decode_table
+        unless @decode_table
+          @decode_table = {}
+          open(decode_table_path).each do |line|
+            code, value = line.strip.split(decode_table_delimiter)
+            if code && code.length > 0
+              @decode_table[code] = value
+            else
+              @default_value = value
+            end
+          end
+        end
+        @decode_table
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/default_transform.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which will replace nil or empty values with a specified value.
+    class DefaultTransform < Transform
+      attr_accessor :default_value
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:default_value</tt>: The default value to use if the incoming value is blank
+      def initialize(control, name, configuration)
+        super
+        @default_value = configuration[:default_value]
+      end
+      # Transform the value
+      def transform(name, value, row)
+        value.blank? ? default_value : value
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -0,0 +1,122 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which looks up the value and replaces it with a foriegn key reference
+    class ForeignKeyLookupTransform < ETL::Transform::Transform
+      # The resolver to use if the foreign key is not found in the collection
+      attr_accessor :resolver
+      
+      # Initialize the foreign key lookup transform.
+      #
+      # Configuration options:
+      # *<tt>:collection</tt>: A Hash of natural keys mapped to surrogate keys. If this is not specified then
+      #  an empty Hash will be used. This Hash will be used to cache values that have been resolved already
+      #  for future use.
+      # *<tt>:resolver</tt>: Object or Class which implements the method resolve(value)
+      def initialize(control, name, configuration={})
+        super
+        
+        @collection = (configuration[:collection] || {})
+        @resolver = configuration[:resolver]
+        @resolver = @resolver.new if @resolver.is_a?(Class)
+      end
+      
+      # Transform the value by resolving it to a foriegn key
+      def transform(name, value, row)
+        fk = @collection[value]
+        unless fk
+          raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless resolver
+          raise ResolverError, "Resolver does not appear to respond to resolve method" unless resolver.respond_to?(:resolve)
+          fk = resolver.resolve(value)
+          raise ResolverError, "Unable to resolve #{value} to foreign key for #{name} in row #{ETL::Engine.rows_read}" unless fk
+          @collection[value] = fk
+        end
+        fk
+      end
+    end
+    # Alias class name for the ForeignKeyLookupTransform.
+    class FkLookupTransform < ForeignKeyLookupTransform; end
+  end
+end
+
+# Resolver which resolves using ActiveRecord.
+class ActiveRecordResolver
+  # The ActiveRecord class to use
+  attr_accessor :ar_class
+  
+  # The find method to use (as a symbol)
+  attr_accessor :find_method
+  
+  # Initialize the resolver. The ar_class argument should extend from 
+  # ActiveRecord::Base. The find_method argument must be a symbol for the 
+  # finder method used. For example:
+  # 
+  # ActiveRecordResolver.new(Person, :find_by_name)
+  #
+  # Note that the find method defined must only take a single argument.
+  def initialize(ar_class, find_method)
+    @ar_class = ar_class
+    @find_method = find_method
+  end
+  
+  # Resolve the value
+  def resolve(value)
+    rec = ar_class.__send__(find_method, value)
+    rec.nil? ? nil : rec.id
+  end
+end
+
+class SQLResolver
+  # Initialize the SQL resolver. Use the given table and field name to search
+  # for the appropriate foreign key. The field should be the name of a natural
+  # key that is used to locate the surrogate key for the record.
+  #
+  # The connection argument is optional. If specified it can be either a symbol
+  # referencing a connection defined in the ETL database.yml file or an actual
+  # ActiveRecord connection instance. If the connection is not specified then
+  # the ActiveRecord::Base.connection will be used.
+  def initialize(table, field, connection=nil)
+    @table = table
+    @field = field
+    @connection = (connection.respond_to?(:quote) ? connection : ETL::Engine.connection(connection)) if connection
+    @connection ||= ActiveRecord::Base.connection
+  end
+  def resolve(value)
+    @connection.select_value("SELECT id FROM #{table_name} WHERE #{@field} = #{@connection.quote(value)}")
+  end
+  def table_name
+    ETL::Engine.table(@table, @connection)
+  end
+end
+
+class FlatFileResolver
+  # Initialize the flat file resolver. Expects to open a comma-delimited file. 
+  # Returns the column with the given result_field_index.
+  #
+  # The matches argument is a Hash with the key as the column index to search and
+  # the value of the Hash as a String to match exactly. It will only match the first
+  # result.
+  def initialize(file, match_index, result_field_index)
+    @file = file
+    @match_index = match_index
+    @result_field_index = result_field_index
+  end
+  
+  # Get the rows from the file specified in the initializer.
+  def rows
+    @rows ||= FasterCSV.read(@file)
+  end
+  protected :rows
+  
+  # Match the row field from the column indicated by the match_index with the given 
+  # value and return the field value from the column identified by the result_field_index.
+  def resolve(value)
+    rows.each do |row|
+      #puts "checking #{row.inspect} for #{value}"
+      if row[@match_index] == value
+        #puts "match found!, returning #{row[@result_field_index]}"
+        return row[@result_field_index]
+      end
+    end
+    nil
+  end
+end

data/etl/lib/etl/transform/hierarchy_lookup_transform.rb

@@ -0,0 +1,49 @@
+module ETL #:nodoc: 
+  module Transform #:nodoc:
+    # Transform which walks up the hierarchy tree to find a value of the current level's value
+    # is nil.
+    #
+    # TODO: Let the resolver be implemented in a class so different resolution methods are
+    # possible.
+    class HierarchyLookupTransform < ETL::Transform::Transform
+      # The name of the field to use for the parent ID
+      attr_accessor :parent_id_field
+      
+      # The target connection name
+      attr_accessor :target
+      
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:target</tt>: The target connection name (required)
+      # * <tt>:parent_id_field</tt>: The name of the field to use for the parent ID (defaults to :parent_id)
+      def initialize(control, name, configuration={})
+        super
+        @parent_id_field = configuration[:parent_id_field] || :parent_id
+        @target = configuration[:target]
+      end
+      
+      # Transform the value.
+      def transform(name, value, row)
+        if parent_id = row[parent_id_field]
+          # TODO: should use more than just the first source out of the control
+          parent_id, value = lookup(name, 
+            control.sources.first.configuration[:table], parent_id, parent_id_field)        
+          until value || parent_id.nil?
+            # TODO: should use more than just the first source out of the control
+            parent_id, value = lookup(name, 
+              control.sources.first.configuration[:table], parent_id, parent_id_field)
+          end
+        end
+        value
+      end
+      
+      # Lookup the parent value.
+      def lookup(field, table, parent_id, parent_id_field)
+        q = "SELECT #{parent_id_field}, #{field} FROM #{table} WHERE id = #{parent_id}"
+        row = ETL::Engine.connection(target).select_one(q)
+        return row[parent_id_field.to_s], row[field.to_s]
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/ordinalize_transform.rb

@@ -0,0 +1,12 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a number to an ordinalized version using the ActiveSupport ordinalize
+    # core extension
+    class OrdinalizeTransform < ETL::Transform::Transform
+      # Transform the value from a number to an ordinalized number
+      def transform(name, value, row)
+        value.ordinalize
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/sha1_transform.rb

@@ -0,0 +1,13 @@
+require 'digest/sha1'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a SHA-1 hash algorithm
+    class Sha1Transform < ETL::Transform::Transform
+      # Transform the value with a SHA1 digest algorithm.
+      def transform(name, value, row)
+        Digest::SHA1.hexdigest(value)
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/string_to_date_transform.rb

@@ -0,0 +1,16 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Date instance
+    class StringToDateTransform < ETL::Transform::Transform
+      # Transform the value using Date.parse
+      def transform(name, value, row)
+        return value if value.nil?
+        begin
+          Date.parse(value)
+        rescue => e
+          return value
+        end
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/string_to_datetime_transform.rb

@@ -0,0 +1,14 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a DateTime instance
+    class StringToDateTimeTransform < ETL::Transform::Transform
+      # Transform the value using DateTime.parse.
+      #
+      # WARNING: This transform is slow (due to the Ruby implementation), but if you need to 
+      # parse timestamps before or after the values supported by the Time.parse.
+      def transform(name, value, row)
+        DateTime.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/etl/lib/etl/transform/string_to_time_transform.rb

@@ -0,0 +1,11 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Time instance
+    class StringToTimeTransform < ETL::Transform::Transform
+      # Transform the value using Time.parse
+      def transform(name, value, row)
+        Time.parse(value) unless value.nil?
+      end
+    end
+  end
+end