darrell-activewarehouse-etl 0.9.1.4

data/lib/etl/row.rb

@@ -0,0 +1,20 @@
+# This source file contains the ETL::Row class.
+
+module ETL #:nodoc:
+  # This class represents a single row currently passing through the ETL pipeline
+  class Row < Hash
+    # Accessor for the originating source
+    attr_accessor :source
+    
+    # All change types
+    CHANGE_TYPES = [:insert, :update, :delete]
+    
+    # Accessor for the row's change type
+    attr_accessor :change_type
+    
+    # Get the change type, defaults to :insert
+    def change_type
+      @change_type ||= :insert
+    end
+  end
+end

data/lib/etl/screen/row_count_screen.rb

@@ -0,0 +1,20 @@
+module ETL
+  module Screen
+    # This screen validates the number of rows which will be bulk loaded
+    # against the results from some sort of a row count query. If there 
+    # is a difference then the screen will not pass
+    class RowCountScreen
+      attr_accessor :control, :configuration
+      def initialize(control, configuration={})
+        @control = control
+        @configuration = configuration
+        execute
+      end
+      def execute
+        unless Engine.rows_written == configuration[:rows]
+          raise "Rows written (#{Engine.rows_written}) does not match expected rows (#{configuration[:rows]})"
+        end
+      end
+    end
+  end
+end

data/lib/etl/screen.rb

@@ -0,0 +1,14 @@
+# This source file contains the ETL::Screen module and requires all of the 
+# screens
+
+module ETL #:nodoc:
+  # The ETL::Screen module contains pre-built screens useful for checking the 
+  # ETL state during execution. Screens may be fatal, which will result in 
+  # termination of the ETL process, errors, which will result in the
+  # termination of just the current ETL control file, or warnings, which will
+  # result in a warning message.
+  module Screen
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/screen/*.rb"].each { |file| require(file) }

data/lib/etl/transform/block_transform.rb

@@ -0,0 +1,13 @@
+module ETL
+  module Transform
+    class BlockTransform < ETL::Transform::Transform
+      def initialize(control, name, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def transform(name, value, row)
+        @block.call(name, value, row)
+      end
+    end
+  end
+end

data/lib/etl/transform/date_to_string_transform.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a Date or Time to a formatted string instance
+    class DateToStringTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      #
+      # Configuration options:
+      # * <tt>:format</tt>: A format passed to strftime. Defaults to %Y-%m-%d
+      def initialize(control, name, configuration={})
+        super
+        @format = configuration[:format] || "%Y-%m-%d"
+      end
+      # Transform the value using strftime
+      def transform(name, value, row)
+        return value unless value.respond_to?(:strftime)
+        value.strftime(@format)
+      end
+    end
+  end
+end

data/lib/etl/transform/decode_transform.rb

@@ -0,0 +1,51 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which decodes coded values
+    class DecodeTransform < ETL::Transform::Transform
+      attr_accessor :decode_table_path
+      
+      attr_accessor :decode_table_delimiter
+      
+      attr_accessor :default_value
+      
+      # Initialize the transformer
+      #
+      # Configuration options:
+      # * <tt>:decode_table_path</tt>: The path to the decode table (defaults to 'decode.txt')
+      # * <tt>:decode_table_delimiter</tt>: The decode table delimiter (defaults to ':')
+      # * <tt>:default_value</tt>: The default value to use (defaults to 'No Value')
+      def initialize(control, name, configuration={})
+        super
+        
+        if configuration[:decode_table_path]
+          configuration[:decode_table_path] = File.join(File.dirname(control.file), configuration[:decode_table_path])
+        end
+        
+        @decode_table_path = (configuration[:decode_table_path] || 'decode.txt')
+        @decode_table_delimiter = (configuration[:decode_table_delimiter] || ':')
+        @default_value = (configuration[:default_value] || 'No Value')
+      end
+      
+      # Transform the value
+      def transform(name, value, row)
+        decode_table[value] || default_value
+      end
+      
+      # Get the decode table
+      def decode_table
+        unless @decode_table
+          @decode_table = {}
+          open(decode_table_path).each do |line|
+            code, value = line.strip.split(decode_table_delimiter)
+            if code && code.length > 0
+              @decode_table[code] = value
+            else
+              @default_value = value
+            end
+          end
+        end
+        @decode_table
+      end
+    end
+  end
+end

data/lib/etl/transform/default_transform.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which will replace nil or empty values with a specified value.
+    class DefaultTransform < Transform
+      attr_accessor :default_value
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:default_value</tt>: The default value to use if the incoming value is blank
+      def initialize(control, name, configuration)
+        super
+        @default_value = configuration[:default_value]
+      end
+      # Transform the value
+      def transform(name, value, row)
+        value.blank? ? default_value : value
+      end
+    end
+  end
+end

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -0,0 +1,169 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which looks up the value and replaces it with a foriegn key reference
+    class ForeignKeyLookupTransform < ETL::Transform::Transform
+      # The resolver to use if the foreign key is not found in the collection
+      attr_accessor :resolver
+      
+      # The default foreign key to use if none is found.
+      attr_accessor :default
+      
+      # Initialize the foreign key lookup transform.
+      #
+      # Configuration options:
+      # *<tt>:collection</tt>: A Hash of natural keys mapped to surrogate keys. If this is not specified then
+      #  an empty Hash will be used. This Hash will be used to cache values that have been resolved already
+      #  for future use.
+      # *<tt>:resolver</tt>: Object or Class which implements the method resolve(value)
+      # *<tt>:default</tt>: A default foreign key to use if no foreign key is found
+      def initialize(control, name, configuration={})
+        super
+        
+        @collection = (configuration[:collection] || {})
+        @resolver = configuration[:resolver]
+        @resolver = @resolver.new if @resolver.is_a?(Class)
+        @default = configuration[:default]
+        if configuration[:cache] ||= true
+          if resolver.respond_to?(:load_cache)
+            resolver.load_cache
+          else
+            ETL::Engine.logger.info "#{resolver.class.name} does not support caching"
+          end
+        end
+      end
+      
+      # Transform the value by resolving it to a foriegn key
+      def transform(name, value, row)
+        fk = @collection[value]
+        unless fk
+          raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless resolver
+          raise ResolverError, "Resolver does not appear to respond to resolve method" unless resolver.respond_to?(:resolve)
+          fk = resolver.resolve(value)
+          fk ||= @default
+          raise ResolverError, "Unable to resolve #{value} to foreign key for #{name} in row #{ETL::Engine.rows_read}. You may want to specify a :default value." unless fk
+          @collection[value] = fk
+        end
+        fk
+      end
+    end
+    # Alias class name for the ForeignKeyLookupTransform.
+    class FkLookupTransform < ForeignKeyLookupTransform; end
+  end
+end
+
+# Resolver which resolves using ActiveRecord.
+class ActiveRecordResolver
+  # The ActiveRecord class to use
+  attr_accessor :ar_class
+  
+  # The find method to use (as a symbol)
+  attr_accessor :find_method
+  
+  # Initialize the resolver. The ar_class argument should extend from 
+  # ActiveRecord::Base. The find_method argument must be a symbol for the 
+  # finder method used. For example:
+  # 
+  # ActiveRecordResolver.new(Person, :find_by_name)
+  #
+  # Note that the find method defined must only take a single argument.
+  def initialize(ar_class, find_method)
+    @ar_class = ar_class
+    @find_method = find_method
+  end
+  
+  # Resolve the value
+  def resolve(value)
+    rec = ar_class.__send__(find_method, value)
+    rec.nil? ? nil : rec.id
+  end
+end
+
+class SQLResolver
+  # Initialize the SQL resolver. Use the given table and field name to search
+  # for the appropriate foreign key. The field should be the name of a natural
+  # key that is used to locate the surrogate key for the record.
+  #
+  # The connection argument is optional. If specified it can be either a symbol
+  # referencing a connection defined in the ETL database.yml file or an actual
+  # ActiveRecord connection instance. If the connection is not specified then
+  # the ActiveRecord::Base.connection will be used.
+  def initialize(table, field, connection=nil)
+    @table = table
+    @field = field
+    @connection = (connection.respond_to?(:quote) ? connection : ETL::Engine.connection(connection)) if connection
+    @connection ||= ActiveRecord::Base.connection
+  end
+  def resolve(value)
+    if @use_cache
+      cache[cache_key(value)]
+    else
+      q = "SELECT id FROM #{table_name} WHERE #{wheres(value)}"
+      @connection.select_value(q)
+    end
+  end
+  def table_name
+    ETL::Engine.table(@table, @connection)
+  end
+  def cache
+    @cache ||= {}
+  end
+  def load_cache
+    @use_cache = true
+    q = "SELECT id, #{field.join(', ')} FROM #{table_name}"
+    @connection.select_all(q).each do |record|
+      cache[cache_key(record.values_at(*field))] = record['id']
+    end
+  end
+
+  private
+  def field
+    unless @field.kind_of?(Array)
+      @field = [ @field ]
+    end
+    @field
+  end
+
+  def cache_key(value)
+    value.hash
+  end
+
+  def wheres(value)
+    value  = [ value ]  unless value.kind_of?(Array)
+    field.zip(value).collect { |a|
+      "#{a[0]} = #{@connection.quote(a[1])}"
+    }.join(' AND ')
+  end
+end
+
+class FlatFileResolver
+  # Initialize the flat file resolver. Expects to open a comma-delimited file. 
+  # Returns the column with the given result_field_index.
+  #
+  # The matches argument is a Hash with the key as the column index to search and
+  # the value of the Hash as a String to match exactly. It will only match the first
+  # result.
+  def initialize(file, match_index, result_field_index)
+    @file = file
+    @match_index = match_index
+    @result_field_index = result_field_index
+  end
+  
+  # Get the rows from the file specified in the initializer.
+  def rows
+    @rows ||= FasterCSV.read(@file)
+  end
+  protected :rows
+  
+  # Match the row field from the column indicated by the match_index with the given 
+  # value and return the field value from the column identified by the result_field_index.
+  def resolve(value)
+    rows.each do |row|
+      #puts "checking #{row.inspect} for #{value}"
+      if row[@match_index] == value
+        #puts "match found!, returning #{row[@result_field_index]}"
+        return row[@result_field_index]
+      end
+    end
+    nil
+  end
+end

data/lib/etl/transform/hierarchy_lookup_transform.rb

@@ -0,0 +1,49 @@
+module ETL #:nodoc: 
+  module Transform #:nodoc:
+    # Transform which walks up the hierarchy tree to find a value of the current level's value
+    # is nil.
+    #
+    # TODO: Let the resolver be implemented in a class so different resolution methods are
+    # possible.
+    class HierarchyLookupTransform < ETL::Transform::Transform
+      # The name of the field to use for the parent ID
+      attr_accessor :parent_id_field
+      
+      # The target connection name
+      attr_accessor :target
+      
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:target</tt>: The target connection name (required)
+      # * <tt>:parent_id_field</tt>: The name of the field to use for the parent ID (defaults to :parent_id)
+      def initialize(control, name, configuration={})
+        super
+        @parent_id_field = configuration[:parent_id_field] || :parent_id
+        @target = configuration[:target]
+      end
+      
+      # Transform the value.
+      def transform(name, value, row)
+        if parent_id = row[parent_id_field]
+          # TODO: should use more than just the first source out of the control
+          parent_id, value = lookup(name, 
+            control.sources.first.configuration[:table], parent_id, parent_id_field)        
+          until value || parent_id.nil?
+            # TODO: should use more than just the first source out of the control
+            parent_id, value = lookup(name, 
+              control.sources.first.configuration[:table], parent_id, parent_id_field)
+          end
+        end
+        value
+      end
+      
+      # Lookup the parent value.
+      def lookup(field, table, parent_id, parent_id_field)
+        q = "SELECT #{parent_id_field}, #{field} FROM #{table} WHERE id = #{parent_id}"
+        row = ETL::Engine.connection(target).select_one(q)
+        return row[parent_id_field.to_s], row[field.to_s]
+      end
+    end
+  end
+end

data/lib/etl/transform/ordinalize_transform.rb

@@ -0,0 +1,12 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a number to an ordinalized version using the ActiveSupport ordinalize
+    # core extension
+    class OrdinalizeTransform < ETL::Transform::Transform
+      # Transform the value from a number to an ordinalized number
+      def transform(name, value, row)
+        value.ordinalize
+      end
+    end
+  end
+end

data/lib/etl/transform/sha1_transform.rb

@@ -0,0 +1,13 @@
+require 'digest/sha1'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a SHA-1 hash algorithm
+    class Sha1Transform < ETL::Transform::Transform
+      # Transform the value with a SHA1 digest algorithm.
+      def transform(name, value, row)
+        Digest::SHA1.hexdigest(value)
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_date_transform.rb

@@ -0,0 +1,16 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Date instance
+    class StringToDateTransform < ETL::Transform::Transform
+      # Transform the value using Date.parse
+      def transform(name, value, row)
+        return value if value.nil?
+        begin
+          Date.parse(value)
+        rescue => e
+          return value
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_datetime_transform.rb

@@ -0,0 +1,14 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a DateTime instance
+    class StringToDateTimeTransform < ETL::Transform::Transform
+      # Transform the value using DateTime.parse.
+      #
+      # WARNING: This transform is slow (due to the Ruby implementation), but if you need to 
+      # parse timestamps before or after the values supported by the Time.parse.
+      def transform(name, value, row)
+        DateTime.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_time_transform.rb

@@ -0,0 +1,11 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Time instance
+    class StringToTimeTransform < ETL::Transform::Transform
+      # Transform the value using Time.parse
+      def transform(name, value, row)
+        Time.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/lib/etl/transform/transform.rb

@@ -0,0 +1,61 @@
+module ETL#:nodoc:
+  module Transform#:nodoc:
+    # Base class for transforms.
+    #
+    # A transform converts one value to another value using some sort of algorithm.
+    #
+    # A simple transform has two arguments, the field to transform and the name of the transform:
+    #
+    #   transform :ssn, :sha1
+    # 
+    # Transforms can also be blocks:
+    #
+    #   transform(:ssn){ |v| v[0,24] }
+    #
+    # Finally, a transform can include a configuration hash:
+    #
+    #   transform :sex, :decode, {:decode_table_path => 'delimited_decode.txt'}
+    class Transform
+      class << self
+        # Transform the specified value using the given transforms. The transforms can either be
+        # Proc objects or objects which extend from Transform and implement the method <tt>transform(value)</tt>.
+        # Any other object will result in a ControlError being raised.
+        def transform(name, value, row, transforms)
+          transforms.each do |transform|
+            benchmarks[transform.class] ||= 0
+            benchmarks[transform.class] += Benchmark.realtime do
+              Engine.logger.debug "Transforming field #{name} with #{transform.inspect}"
+              case transform
+              when Proc
+                value = transform.call([name, value, row])
+              when Transform
+                value = transform.transform(name, value, row)
+              else
+                raise ControlError, "Unsupported transform configuration type: #{transform}"
+              end
+            end
+          end
+          value
+        end
+        
+        def benchmarks
+          @benchmarks ||= {}
+        end
+      end
+      
+      attr_reader :control, :name, :configuration
+      
+      # Initialize the transform object with the given control object, field name and 
+      # configuration hash
+      def initialize(control, name, configuration={})
+        @control = control
+        @name = name
+        @configuration = configuration
+      end
+      
+      def transform(name, value, row)
+        raise "transform is an abstract method"
+      end
+    end
+  end
+end

data/lib/etl/transform/trim_transform.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform to trim string
+    class TrimTransform < ETL::Transform::Transform
+      # Configuration options:
+      # * <tt>:type</tt>: :left, :right or :both. Default is :both
+      def initialize(control, name, configuration={})
+        super
+        @type = (configuration[:type] || :both).to_sym
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :left
+          value.lstrip
+        when :right
+          value.rstrip
+        when :both
+          value.strip
+        else
+          raise "Trim type, if specified, must be :left, :right or :both"
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/type_transform.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform from one type to another
+    class TypeTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      # 
+      # Configuration options:
+      # * <tt>:type</tt>: The type to convert to. Supported types:
+      # ** :string
+      # ** :number,:integer
+      # ** :float
+      # ** :decimal
+      def initialize(control, name, configuration={})
+        super
+        @type = configuration[:type]
+        @significant = configuration[:significant] ||= 0
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :string
+          value.to_s
+        when :number, :integer
+          value.to_i
+        when :float
+          value.to_f
+        when :decimal
+          BigDecimal.new(value.to_s, @significant)
+        else
+          raise "Unsupported type: #{@type}"
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform.rb

@@ -0,0 +1,2 @@
+require 'etl/transform/transform'
+Dir[File.dirname(__FILE__) + "/transform/*.rb"].each { |file| require(file) }

data/lib/etl/util.rb

@@ -0,0 +1,59 @@
+module ETL
+  module Util
+    # Return the distance of time in words from the given from_time to the specified to_time. If to_time
+    # is not specified then Time.now is used. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def distance_of_time_in_words(from_time, to_time=Time.now)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      seconds = (to_time - from_time).round
+      distance_in_days = (seconds/(60*60*24)).round
+      seconds = seconds % (60*60*24)
+      distance_in_hours = (seconds/(60*60)).round
+      seconds = seconds % (60*60)
+      distance_in_minutes = (seconds/60).round
+      seconds = seconds % 60
+      distance_in_seconds = seconds
+    
+      s = ''
+      s << "#{distance_in_days} days," if distance_in_days > 0
+      s << "#{distance_in_hours} hours, " if distance_in_hours > 0
+      s << "#{distance_in_minutes} minutes, " if distance_in_minutes > 0
+      s << "#{distance_in_seconds} seconds"
+      s
+    end
+  
+    # Get the approximate disntance of time in words from the given from_time
+    # to the the given to_time. If to_time is not specified then it is set
+    # to Time.now. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def approximate_distance_of_time_in_words(from_time, to_time=Time.now, include_seconds=true)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      distance_in_minutes = (((to_time - from_time).abs)/60).round
+      distance_in_seconds = ((to_time - from_time).abs).round
+    
+      case distance_in_minutes
+        when 0..1
+          return (distance_in_minutes == 0) ? 'less than a minute' : '1 minute' unless include_seconds
+        case distance_in_seconds
+          when 0..4   then 'less than 5 seconds'
+          when 5..9   then 'less than 10 seconds'
+          when 10..19 then 'less than 20 seconds'
+          when 20..39 then 'half a minute'
+          when 40..59 then 'less than a minute'
+          else             '1 minute'
+        end
+        when 2..44           then "#{distance_in_minutes} minutes"
+        when 45..89          then 'about 1 hour'
+        when 90..1439        then "about #{(distance_in_minutes.to_f / 60.0).round} hours"
+        when 1440..2879      then '1 day'
+        when 2880..43199     then "#{(distance_in_minutes / 1440).round} days"
+        when 43200..86399    then 'about 1 month'
+        when 86400..525959   then "#{(distance_in_minutes / 43200).round} months"
+        when 525960..1051919 then 'about 1 year'
+        else                      "over #{(distance_in_minutes / 525960).round} years"
+      end
+    end
+  end
+end

data/lib/etl/version.rb

@@ -0,0 +1,9 @@
+module ETL#:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 9
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end