activewarehouse-etl-sgonyea 0.9.6

data/lib/etl/transform.rb

@@ -0,0 +1,2 @@
+require 'etl/transform/transform'
+Dir[File.dirname(__FILE__) + "/transform/*.rb"].each { |file| require(file) }

data/lib/etl/transform/block_transform.rb

@@ -0,0 +1,13 @@
+module ETL
+  module Transform
+    class BlockTransform < ETL::Transform::Transform
+      def initialize(control, name, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def transform(name, value, row)
+        @block.call(name, value, row)
+      end
+    end
+  end
+end

data/lib/etl/transform/calculation_transform.rb

@@ -0,0 +1,71 @@
+module ETL
+  module Transform
+    class CalculationTransform < ETL::Transform::Transform
+      attr_reader :function
+      attr_reader :fields
+
+      def initialize(control, name, configuration)
+        @function = configuration[:function]
+        @fields = configuration[:fields]
+        super
+      end
+      
+      def transform(name, value, row)
+        return nil if row.nil?
+        return nil if row[@fields[0]].nil?
+
+        if (@function.eql? "A + B")
+          result = ""
+          @fields.each do |field|
+            next if field.nil?
+
+            string = ""
+            if field.to_s.eql? field
+              string = field
+              begin
+                string = eval('"' + field + '"')
+              rescue
+              end
+            else
+              string = row[field]
+            end
+            next if string.nil?
+
+            result = result + string
+          end
+
+          row[name] = result
+        end
+
+        if (@function.eql? "date A")
+          first = row[@fields[0]]
+          row[name] = Time.parse(first)
+        end
+
+        if (@function.eql? "trim A")
+          first = row[@fields[0]]
+          row[name] = first.strip
+        end
+
+        if (@function.eql? "lower A")
+          first = row[@fields[0]]
+          row[name] = first.downcase
+        end
+
+        if (@function.eql? "upper A")
+          first = row[@fields[0]]
+          row[name] = first.upcase
+        end
+
+        if (@function.eql? "encoding A")
+          # Bug from ruby 1.8 http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
+          first = row[@fields[0]]
+          row[name] = Iconv.conv(@fields[1], @fields[2], first + ' ')[0..-2]
+        end
+
+        row[name]
+      end
+
+    end
+  end
+end

data/lib/etl/transform/date_to_string_transform.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a Date or Time to a formatted string instance
+    class DateToStringTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      #
+      # Configuration options:
+      # * <tt>:format</tt>: A format passed to strftime. Defaults to %Y-%m-%d
+      def initialize(control, name, configuration={})
+        super
+        @format = configuration[:format] || "%Y-%m-%d"
+      end
+      # Transform the value using strftime
+      def transform(name, value, row)
+        return value unless value.respond_to?(:strftime)
+        value.strftime(@format)
+      end
+    end
+  end
+end

data/lib/etl/transform/decode_transform.rb

@@ -0,0 +1,51 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which decodes coded values
+    class DecodeTransform < ETL::Transform::Transform
+      attr_accessor :decode_table_path
+      
+      attr_accessor :decode_table_delimiter
+      
+      attr_accessor :default_value
+      
+      # Initialize the transformer
+      #
+      # Configuration options:
+      # * <tt>:decode_table_path</tt>: The path to the decode table (defaults to 'decode.txt')
+      # * <tt>:decode_table_delimiter</tt>: The decode table delimiter (defaults to ':')
+      # * <tt>:default_value</tt>: The default value to use (defaults to 'No Value')
+      def initialize(control, name, configuration={})
+        super
+        
+        if configuration[:decode_table_path]
+          configuration[:decode_table_path] = File.join(File.dirname(control.file), configuration[:decode_table_path])
+        end
+        
+        @decode_table_path = (configuration[:decode_table_path] || 'decode.txt')
+        @decode_table_delimiter = (configuration[:decode_table_delimiter] || ':')
+        @default_value = (configuration[:default_value] || 'No Value')
+      end
+      
+      # Transform the value
+      def transform(name, value, row)
+        decode_table[value] || default_value
+      end
+      
+      # Get the decode table
+      def decode_table
+        unless @decode_table
+          @decode_table = {}
+          open(decode_table_path).each do |line|
+            code, value = line.strip.split(decode_table_delimiter)
+            if code && code.length > 0
+              @decode_table[code] = value
+            else
+              @default_value = value
+            end
+          end
+        end
+        @decode_table
+      end
+    end
+  end
+end

data/lib/etl/transform/default_transform.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which will replace nil or empty values with a specified value.
+    class DefaultTransform < Transform
+      attr_accessor :default_value
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:default_value</tt>: The default value to use if the incoming value is blank
+      def initialize(control, name, configuration)
+        super
+        @default_value = configuration[:default_value]
+      end
+      # Transform the value
+      def transform(name, value, row)
+        value.blank? ? default_value : value
+      end
+    end
+  end
+end

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -0,0 +1,211 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which looks up the value and replaces it with a foriegn key reference
+    class ForeignKeyLookupTransform < ETL::Transform::Transform
+      # The resolver to use if the foreign key is not found in the collection
+      attr_accessor :resolver
+      
+      # The default foreign key to use if none is found.
+      attr_accessor :default
+      
+      # Initialize the foreign key lookup transform.
+      #
+      # Configuration options:
+      # *<tt>:collection</tt>: A Hash of natural keys mapped to surrogate keys. If this is not specified then
+      #  an empty Hash will be used. This Hash will be used to cache values that have been resolved already
+      #  for future use.
+      # *<tt>:resolver</tt>: Object or Class which implements the method resolve(value)
+      # *<tt>:default</tt>: A default foreign key to use if no foreign key is found
+      # *<tt>:cache</tt>: If true and the resolver responds to load_cache, load_cache will be called
+      def initialize(control, name, configuration={})
+        super
+        
+        @collection = (configuration[:collection] || {})
+        @resolver = configuration[:resolver]
+        @resolver = @resolver.new if @resolver.is_a?(Class)
+        @default = configuration[:default]
+        
+        configuration[:cache] = true if configuration[:cache].nil?
+        
+        if configuration[:cache]
+          if resolver.respond_to?(:load_cache)
+            resolver.load_cache
+          else
+            ETL::Engine.logger.info "#{resolver.class.name} does not support caching"
+          end
+        end
+      end
+      
+      # Transform the value by resolving it to a foriegn key
+      def transform(name, value, row)
+        fk = @collection[value]
+        unless fk
+          raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless resolver
+          raise ResolverError, "Resolver does not appear to respond to resolve method" unless resolver.respond_to?(:resolve)
+          fk = resolver.resolve(value)
+          fk ||= @default
+          raise ResolverError, "Unable to resolve #{value} to foreign key for #{name} in row #{ETL::Engine.rows_read}. You may want to specify a :default value." unless fk
+          @collection[value] = fk
+        end
+        fk
+      end
+    end
+    # Alias class name for the ForeignKeyLookupTransform.
+    class FkLookupTransform < ForeignKeyLookupTransform; end
+  end
+end
+
+# Resolver which resolves using ActiveRecord.
+class ActiveRecordResolver
+  # The ActiveRecord class to use
+  attr_accessor :ar_class
+  
+  # The find method to use (as a symbol)
+  attr_accessor :find_method
+  
+  # Initialize the resolver. The ar_class argument should extend from 
+  # ActiveRecord::Base. The find_method argument must be a symbol for the 
+  # finder method used. For example:
+  # 
+  # ActiveRecordResolver.new(Person, :find_by_name)
+  #
+  # Note that the find method defined must only take a single argument.
+  def initialize(ar_class, find_method)
+    @ar_class = ar_class
+    @find_method = find_method
+  end
+  
+  # Resolve the value
+  def resolve(value)
+    rec = ar_class.__send__(find_method, value)
+    rec.nil? ? nil : rec.id
+  end
+end
+
+class SQLResolver
+  # Initialize the SQL resolver. Use the given table and field name to search
+  # for the appropriate foreign key. The field should be the name of a natural
+  # key that is used to locate the surrogate key for the record.
+  #
+  # The connection argument is optional. If specified it can be either a symbol
+  # referencing a connection defined in the ETL database.yml file or an actual
+  # ActiveRecord connection instance. If the connection is not specified then
+  # the ActiveRecord::Base.connection will be used.
+  def initialize(atable, afield, connection=nil)
+    # puts "table: #{atable.inspect} field:#{afield.inspect}"
+    @table = atable
+    @field = afield
+    @connection = (connection.respond_to?(:quote) ? connection : ETL::Engine.connection(connection)) if connection
+    @connection ||= ActiveRecord::Base.connection
+  end
+  
+  def resolve(value)
+    return nil if value.nil?
+    r = nil
+    if @use_cache
+      r = cache[value]
+      # puts "resolve failed: #{value.class.name}:#{value.inspect} from: #{@table}.#{@field}" unless r
+    else
+      q = "SELECT id FROM #{table_name} WHERE #{wheres(value)}"
+      # puts q
+      r = @connection.select_value(q)
+    end
+    r
+  end
+  
+  def table_name
+    ETL::Engine.table(@table, @connection)
+  end
+  
+  def cache
+    @cache ||= {}
+  end
+  
+  def load_cache
+    q = "SELECT id, #{field.join(', ')} FROM #{table_name}"
+    # puts q
+    @connection.select_all(q).each do |record|
+      ck = @field.kind_of?(Array) ? record.values_at(*@field) : record[@field]
+      # puts "load_cache key: #{ck.class.name}:#{ck.inspect}"
+      # puts "  #{@field.class.name}:#{@field.inspect}"
+      # puts "  #{record[@field].class.name}:#{record[@field].inspect}"
+      cache[ck] = record['id']
+    end
+    @use_cache = true
+  end
+
+  private
+
+  def field
+    if @field.kind_of?(Array)
+      @field
+    else
+      [ @field ]
+    end
+  end
+
+  def wheres(value)
+    value  = [ value ]  unless value.kind_of?(Array)
+    field.zip(value).collect { |a|
+      "#{a[0]} = #{@connection.quote(a[1])}"
+    }.join(' AND ')
+  end
+end
+
+class IncrementalCacheSQLResolver < SQLResolver
+
+  def initialize(atable, afield, connection=nil)
+    super
+  end
+  
+  def resolve(value)
+    return nil if value.nil?
+    r = cache[value]
+    unless r
+      q = "SELECT id FROM #{table_name} WHERE #{wheres(value)}"
+      r = @connection.select_value(q)
+      if r
+        cache[value] = r
+      end
+    end
+    r
+  end
+
+  def load_cache
+    @cache = {}
+  end
+
+end
+
+class FlatFileResolver
+  # Initialize the flat file resolver. Expects to open a comma-delimited file. 
+  # Returns the column with the given result_field_index.
+  #
+  # The matches argument is a Hash with the key as the column index to search and
+  # the value of the Hash as a String to match exactly. It will only match the first
+  # result.
+  def initialize(file, match_index, result_field_index)
+    @file = file
+    @match_index = match_index
+    @result_field_index = result_field_index
+  end
+  
+  # Get the rows from the file specified in the initializer.
+  def rows
+    @rows ||= CSV.read(@file)
+  end
+  protected :rows
+  
+  # Match the row field from the column indicated by the match_index with the given 
+  # value and return the field value from the column identified by the result_field_index.
+  def resolve(value)
+    rows.each do |row|
+      #puts "checking #{row.inspect} for #{value}"
+      if row[@match_index] == value
+        #puts "match found!, returning #{row[@result_field_index]}"
+        return row[@result_field_index]
+      end
+    end
+    nil
+  end
+end

data/lib/etl/transform/hierarchy_lookup_transform.rb

@@ -0,0 +1,49 @@
+module ETL #:nodoc: 
+  module Transform #:nodoc:
+    # Transform which walks up the hierarchy tree to find a value of the current level's value
+    # is nil.
+    #
+    # TODO: Let the resolver be implemented in a class so different resolution methods are
+    # possible.
+    class HierarchyLookupTransform < ETL::Transform::Transform
+      # The name of the field to use for the parent ID
+      attr_accessor :parent_id_field
+      
+      # The target connection name
+      attr_accessor :target
+      
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:target</tt>: The target connection name (required)
+      # * <tt>:parent_id_field</tt>: The name of the field to use for the parent ID (defaults to :parent_id)
+      def initialize(control, name, configuration={})
+        super
+        @parent_id_field = configuration[:parent_id_field] || :parent_id
+        @target = configuration[:target]
+      end
+      
+      # Transform the value.
+      def transform(name, value, row)
+        if parent_id = row[parent_id_field]
+          # TODO: should use more than just the first source out of the control
+          parent_id, value = lookup(name, 
+            control.sources.first.configuration[:table], parent_id, parent_id_field)        
+          until value || parent_id.nil?
+            # TODO: should use more than just the first source out of the control
+            parent_id, value = lookup(name, 
+              control.sources.first.configuration[:table], parent_id, parent_id_field)
+          end
+        end
+        value
+      end
+      
+      # Lookup the parent value.
+      def lookup(field, table, parent_id, parent_id_field)
+        q = "SELECT #{parent_id_field}, #{field} FROM #{table} WHERE id = #{parent_id}"
+        row = ETL::Engine.connection(target).select_one(q)
+        return row[parent_id_field.to_s], row[field.to_s]
+      end
+    end
+  end
+end

data/lib/etl/transform/md5_transform.rb

@@ -0,0 +1,13 @@
+require 'digest/md5'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a MD5 hash algorithm
+    class Md5Transform < ETL::Transform::Transform
+      # Transform the value with a MD5 digest algorithm.
+      def transform(name, value, row)
+        Digest::MD5.hexdigest(value)
+      end
+    end
+  end
+end