colincasey-activewarehouse-etl 0.9.4

data/lib/etl/processor/row_processor.rb

@@ -0,0 +1,17 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Processor which processes a specific row. Unlike a transformer, which deals with a specific
+    # value in the row, row processors can process an entire row at once, which can be used to 
+    # explode a single row into multiple rows (for example)
+    class RowProcessor < Processor
+      # Initialize the processor
+      def initialize(control, configuration)
+        super
+      end
+      # Process the specified row. This method must return the row.
+      def process(row)
+        raise "process_row is an abstract method"
+      end
+    end
+  end
+end

data/lib/etl/processor/sequence_processor.rb

@@ -0,0 +1,23 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to generate a sequence.
+    #
+    # Configuration options:
+    # * <tt>:context</tt>: A context name, if none is specified then the context will be
+    #   the current ETL run
+    # * <tt>:dest</tt>: The destination field name
+    class SequenceProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        sequences[configuration[:context]] ||= 0
+        row[configuration[:dest]] = sequences[configuration[:context]] += 1
+        row
+      end
+      
+      protected
+      # Get a Hash of sequences
+      def sequences
+        @sequences ||= {}
+      end
+    end
+  end
+end

data/lib/etl/processor/surrogate_key_processor.rb

@@ -0,0 +1,53 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row level processor that provides surrogate keys
+    class SurrogateKeyProcessor < ETL::Processor::RowProcessor
+      attr_accessor :destination
+      attr_accessor :table
+      attr_accessor :column
+      attr_accessor :target
+      
+      # Initialize the surrogate key generator
+      #
+      # Configuration options
+      # * <tt>:query</tt>: If specified it contains a query to be used to 
+      #   locate the last surrogate key. If this is specified then :target
+      #   must also be specified.
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:destination</tt>: The destination column name (defaults to :id)
+      def initialize(control, configuration)
+        super
+        @table = configuration[:table]
+        @column = configuration[:column] || 'id'
+        @target = configuration[:target]
+        if configuration[:query]
+          raise ControlError, "Query option is no longer value, use :column and :table instead"
+        end
+        if table
+          @surrogate_key = ETL::Engine.connection(target).select_value("SELECT max(#{column}) FROM #{table_name}")
+        end
+        #puts "initial surrogate key: #{@surrogate_key}"
+        @surrogate_key = 0 if @surrogate_key.blank?
+        @surrogate_key = @surrogate_key.to_i
+        #puts "surrogate key: #{@surrogate_key}"
+        @destination = configuration[:destination] || :id
+      end
+      
+      # Add a surrogate key to the row
+      def process(row)
+        if row
+          #puts "processing row #{row.inspect}"
+          @surrogate_key += 1
+          #puts "adding surrogate key to row: #{@surrogate_key}"
+          row[destination] = @surrogate_key
+          row
+        end
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/truncate_processor.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A processor which will truncate a table. Use as a pre-processor for cleaning out a table
+    # prior to loading
+    class TruncateProcessor < ETL::Processor::Processor
+      # Defines the table to truncate
+      attr_reader :table
+      
+      # Defines the database connection to use
+      attr_reader :target
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The table name
+      def initialize(control, configuration)
+        super
+        #@file = File.join(File.dirname(control.file), configuration[:file])
+        @target = configuration[:target] || {}
+        @table = configuration[:table]
+      end
+      
+      def process
+        conn = ETL::Engine.connection(target)
+        conn.truncate(table_name)
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/row.rb

@@ -0,0 +1,20 @@
+# This source file contains the ETL::Row class.
+
+module ETL #:nodoc:
+  # This class represents a single row currently passing through the ETL pipeline
+  class Row < Hash
+    # Accessor for the originating source
+    attr_accessor :source
+    
+    # All change types
+    CHANGE_TYPES = [:insert, :update, :delete]
+    
+    # Accessor for the row's change type
+    attr_accessor :change_type
+    
+    # Get the change type, defaults to :insert
+    def change_type
+      @change_type ||= :insert
+    end
+  end
+end

data/lib/etl/screen.rb

@@ -0,0 +1,14 @@
+# This source file contains the ETL::Screen module and requires all of the 
+# screens
+
+module ETL #:nodoc:
+  # The ETL::Screen module contains pre-built screens useful for checking the 
+  # ETL state during execution. Screens may be fatal, which will result in 
+  # termination of the ETL process, errors, which will result in the
+  # termination of just the current ETL control file, or warnings, which will
+  # result in a warning message.
+  module Screen
+  end
+end
+
+Dir[File.dirname(__FILE__) + "/screen/*.rb"].each { |file| require(file) }

data/lib/etl/screen/row_count_screen.rb

@@ -0,0 +1,20 @@
+module ETL
+  module Screen
+    # This screen validates the number of rows which will be bulk loaded
+    # against the results from some sort of a row count query. If there 
+    # is a difference then the screen will not pass
+    class RowCountScreen
+      attr_accessor :control, :configuration
+      def initialize(control, configuration={})
+        @control = control
+        @configuration = configuration
+        execute
+      end
+      def execute
+        unless Engine.rows_written == configuration[:rows]
+          raise "Rows written (#{Engine.rows_written}) does not match expected rows (#{configuration[:rows]})"
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform.rb

@@ -0,0 +1,2 @@
+require 'etl/transform/transform'
+Dir[File.dirname(__FILE__) + "/transform/*.rb"].each { |file| require(file) }

data/lib/etl/transform/block_transform.rb

@@ -0,0 +1,13 @@
+module ETL
+  module Transform
+    class BlockTransform < ETL::Transform::Transform
+      def initialize(control, name, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def transform(name, value, row)
+        @block.call(name, value, row)
+      end
+    end
+  end
+end

data/lib/etl/transform/date_to_string_transform.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a Date or Time to a formatted string instance
+    class DateToStringTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      #
+      # Configuration options:
+      # * <tt>:format</tt>: A format passed to strftime. Defaults to %Y-%m-%d
+      def initialize(control, name, configuration={})
+        super
+        @format = configuration[:format] || "%Y-%m-%d"
+      end
+      # Transform the value using strftime
+      def transform(name, value, row)
+        return value unless value.respond_to?(:strftime)
+        value.strftime(@format)
+      end
+    end
+  end
+end

data/lib/etl/transform/decode_transform.rb

@@ -0,0 +1,51 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which decodes coded values
+    class DecodeTransform < ETL::Transform::Transform
+      attr_accessor :decode_table_path
+      
+      attr_accessor :decode_table_delimiter
+      
+      attr_accessor :default_value
+      
+      # Initialize the transformer
+      #
+      # Configuration options:
+      # * <tt>:decode_table_path</tt>: The path to the decode table (defaults to 'decode.txt')
+      # * <tt>:decode_table_delimiter</tt>: The decode table delimiter (defaults to ':')
+      # * <tt>:default_value</tt>: The default value to use (defaults to 'No Value')
+      def initialize(control, name, configuration={})
+        super
+        
+        if configuration[:decode_table_path]
+          configuration[:decode_table_path] = File.join(File.dirname(control.file), configuration[:decode_table_path])
+        end
+        
+        @decode_table_path = (configuration[:decode_table_path] || 'decode.txt')
+        @decode_table_delimiter = (configuration[:decode_table_delimiter] || ':')
+        @default_value = (configuration[:default_value] || 'No Value')
+      end
+      
+      # Transform the value
+      def transform(name, value, row)
+        decode_table[value] || default_value
+      end
+      
+      # Get the decode table
+      def decode_table
+        unless @decode_table
+          @decode_table = {}
+          open(decode_table_path).each do |line|
+            code, value = line.strip.split(decode_table_delimiter)
+            if code && code.length > 0
+              @decode_table[code] = value
+            else
+              @default_value = value
+            end
+          end
+        end
+        @decode_table
+      end
+    end
+  end
+end

data/lib/etl/transform/default_transform.rb

@@ -0,0 +1,20 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which will replace nil or empty values with a specified value.
+    class DefaultTransform < Transform
+      attr_accessor :default_value
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:default_value</tt>: The default value to use if the incoming value is blank
+      def initialize(control, name, configuration)
+        super
+        @default_value = configuration[:default_value]
+      end
+      # Transform the value
+      def transform(name, value, row)
+        value.blank? ? default_value : value
+      end
+    end
+  end
+end

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -0,0 +1,151 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which looks up the value and replaces it with a foriegn key reference
+    class ForeignKeyLookupTransform < ETL::Transform::Transform
+      # The resolver to use if the foreign key is not found in the collection
+      attr_accessor :resolver
+      
+      # The default foreign key to use if none is found.
+      attr_accessor :default
+      
+      # Initialize the foreign key lookup transform.
+      #
+      # Configuration options:
+      # *<tt>:collection</tt>: A Hash of natural keys mapped to surrogate keys. If this is not specified then
+      #  an empty Hash will be used. This Hash will be used to cache values that have been resolved already
+      #  for future use.
+      # *<tt>:resolver</tt>: Object or Class which implements the method resolve(value)
+      # *<tt>:default</tt>: A default foreign key to use if no foreign key is found
+      def initialize(control, name, configuration={})
+        super
+        
+        @collection = (configuration[:collection] || {})
+        @resolver = configuration[:resolver]
+        @resolver = @resolver.new if @resolver.is_a?(Class)
+        @default = configuration[:default]
+        if configuration[:cache] ||= true
+          if resolver.respond_to?(:load_cache)
+            resolver.load_cache
+          else
+            ETL::Engine.logger.info "#{resolver.class.name} does not support caching"
+          end
+        end
+      end
+      
+      # Transform the value by resolving it to a foriegn key
+      def transform(name, value, row)
+        fk = @collection[value]
+        unless fk
+          raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless resolver
+          raise ResolverError, "Resolver does not appear to respond to resolve method" unless resolver.respond_to?(:resolve)
+          fk = resolver.resolve(value)
+          fk ||= @default
+          raise ResolverError, "Unable to resolve #{value} to foreign key for #{name} in row #{ETL::Engine.rows_read}. You may want to specify a :default value." unless fk
+          @collection[value] = fk
+        end
+        fk
+      end
+    end
+    # Alias class name for the ForeignKeyLookupTransform.
+    class FkLookupTransform < ForeignKeyLookupTransform; end
+  end
+end
+
+# Resolver which resolves using ActiveRecord.
+class ActiveRecordResolver
+  # The ActiveRecord class to use
+  attr_accessor :ar_class
+  
+  # The find method to use (as a symbol)
+  attr_accessor :find_method
+  
+  # Initialize the resolver. The ar_class argument should extend from 
+  # ActiveRecord::Base. The find_method argument must be a symbol for the 
+  # finder method used. For example:
+  # 
+  # ActiveRecordResolver.new(Person, :find_by_name)
+  #
+  # Note that the find method defined must only take a single argument.
+  def initialize(ar_class, find_method)
+    @ar_class = ar_class
+    @find_method = find_method
+  end
+  
+  # Resolve the value
+  def resolve(value)
+    rec = ar_class.__send__(find_method, value)
+    rec.nil? ? nil : rec.id
+  end
+end
+
+class SQLResolver
+  # Initialize the SQL resolver. Use the given table and field name to search
+  # for the appropriate foreign key. The field should be the name of a natural
+  # key that is used to locate the surrogate key for the record.
+  #
+  # The connection argument is optional. If specified it can be either a symbol
+  # referencing a connection defined in the ETL database.yml file or an actual
+  # ActiveRecord connection instance. If the connection is not specified then
+  # the ActiveRecord::Base.connection will be used.
+  def initialize(table, field, connection=nil)
+    @table = table
+    @field = field
+    @connection = (connection.respond_to?(:quote) ? connection : ETL::Engine.connection(connection)) if connection
+    @connection ||= ActiveRecord::Base.connection
+  end
+  def resolve(value)
+    if @use_cache
+      cache[value]
+    else
+      q = "SELECT id FROM #{table_name} WHERE #{@field} = #{@connection.quote(value)}"
+      ETL::Engine.logger.debug("Executing query: #{q}")
+      @connection.select_value(q)
+    end
+  end
+  def table_name
+    ETL::Engine.table(@table, @connection)
+  end
+  def cache
+    @cache ||= {}
+  end
+  def load_cache
+    @use_cache = true
+    q = "SELECT id, #{@field} FROM #{table_name}"
+    @connection.select_all(q).each do |record|
+      cache[record[@field]] = record['id']
+    end
+  end
+end
+
+class FlatFileResolver
+  # Initialize the flat file resolver. Expects to open a comma-delimited file. 
+  # Returns the column with the given result_field_index.
+  #
+  # The matches argument is a Hash with the key as the column index to search and
+  # the value of the Hash as a String to match exactly. It will only match the first
+  # result.
+  def initialize(file, match_index, result_field_index)
+    @file = file
+    @match_index = match_index
+    @result_field_index = result_field_index
+  end
+  
+  # Get the rows from the file specified in the initializer.
+  def rows
+    @rows ||= FasterCSV.read(@file)
+  end
+  protected :rows
+  
+  # Match the row field from the column indicated by the match_index with the given 
+  # value and return the field value from the column identified by the result_field_index.
+  def resolve(value)
+    rows.each do |row|
+      #puts "checking #{row.inspect} for #{value}"
+      if row[@match_index] == value
+        #puts "match found!, returning #{row[@result_field_index]}"
+        return row[@result_field_index]
+      end
+    end
+    nil
+  end
+end