factorylabs-activewarehouse-etl 0.9.1.2

data/lib/etl/processor/check_exist_processor.rb

@@ -0,0 +1,80 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row-level processor that checks if the row already exists in the 
+    # target table
+    class CheckExistProcessor < ETL::Processor::RowProcessor
+      # A symbol or array of symbols representing keys that should be skipped
+      attr_accessor :skip
+      
+      # The target database
+      attr_accessor :target
+      
+      # The name of the table to check against
+      attr_accessor :table
+      
+      # An array of columns representing the natural key
+      attr_accessor :columns
+      
+      # Is set to true if the processor should execute the check. If there are
+      # no rows in the target table then this should return false.
+      attr_accessor :should_check
+      
+      # Initialize the processor
+      # Configuration options:
+      # * <tt>:skip</tt>: A symbol or array of column names that should not 
+      #   be checked
+      # * <tt>:table</tt>: The table name
+      # * <tt>:columns</tt>: An array of columns which represent the natural
+      #   key
+      def initialize(control, configuration)
+        super
+        @skip = configuration[:skip] || []
+        @target = configuration[:target] || raise(ETL::ControlError, "target must be specified")
+        @table = configuration[:table] || raise(ETL::ControlError, "table must be specified")
+        @columns = configuration[:columns]
+        
+        q = "SELECT COUNT(*) FROM #{table_name}"
+        @should_check = ETL::Engine.connection(target).select_value(q).to_i > 0 
+      end
+      
+      # Return true if the given key should be skipped
+      def skip?(key)
+        case skip
+        when Array
+          skip.include?(key)
+        else
+          skip.to_sym == key.to_sym
+        end
+      end
+      
+      # Return true if the row should be checked
+      def should_check?
+        @should_check ? true : false
+      end
+      
+      # Process the row
+      def process(row)
+        return row unless should_check?
+        conn = ETL::Engine.connection(target)
+        q = "SELECT * FROM #{table_name} WHERE "
+        conditions = []
+        row.each do |k,v| 
+          if columns.nil? || columns.include?(k.to_sym)
+            conditions << "#{k} = #{conn.quote(v)}" unless skip?(k.to_sym)
+          end
+        end
+        q << conditions.join(" AND ")
+        q << " LIMIT 1"
+      
+        #puts "query: #{q}"
+        result = conn.select_one(q)
+        return row if result.nil?
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/check_unique_processor.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row processor that checks whether or not the row has already passed 
+    # through the ETL processor, using the key fields provided as the keys
+    # to check.
+    class CheckUniqueProcessor < ETL::Processor::RowProcessor
+
+      # The keys to check
+      attr_accessor :keys
+      
+      # Initialize the processor
+      # Configuration options:
+      # * <tt>:keys</tt>: An array of keys to check against
+      def initialize(control, configuration)
+        super
+        @keys = configuration[:keys]
+      end
+
+      # A Hash of keys that have already been processed.
+      def compound_key_constraints
+        @compound_key_constraints ||= {}
+      end
+      
+      # Process the row. This implementation will only return a row if it
+      # it's key combination has not already been seen.
+      def process(row)
+        key = (keys.collect { |k| row[k] }).join('|')
+        unless compound_key_constraints[key]
+          compound_key_constraints[key] = 1
+          return row
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/copy_field_processor.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row processor that will copy one field to another
+    #
+    # Configuration options:
+    # * <tt>:destination</tt>: The destination field
+    # * <tt>:dest</tt>: Alias for :destination
+    # * <tt>:source</tt>: The source field
+    class CopyFieldProcessor < ETL::Processor::RowProcessor
+      # Process the given row
+      def process(row)
+        destination = (configuration[:destination] || configuration[:dest])
+        source_value = row[configuration[:source]]
+        case source_value
+        when Numeric
+          row[destination] = source_value
+        when nil
+          row[destination] = nil
+        else
+          row[destination] = source_value.dup
+        end
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/encode_processor.rb

@@ -0,0 +1,55 @@
+require 'iconv'
+
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # The encode processor uses Iconv to convert a file from one encoding (eg: utf-8) to another (eg: latin1), line by line.
+    class EncodeProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :source_file
+      # The file to write to
+      attr_reader :target_file
+      # The source file encoding
+      attr_reader :source_encoding
+      # The target file encoding
+      attr_reader :target_encoding
+      
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:source_file</tt>: The file to load data from
+      # * <tt>:source_encoding</tt>: The source file encoding (eg: 'latin1','utf-8'), as supported by Iconv
+      # * <tt>:target_file</tt>: The file to write data to
+      # * <tt>:target_encoding</tt>: The target file encoding
+      def initialize(control, configuration)
+        super
+        raise ControlError, "Source file must be specified" if configuration[:source_file].nil?
+        raise ControlError, "Target file must be specified" if configuration[:target_file].nil?
+        @source_file = File.join(File.dirname(control.file), configuration[:source_file])
+        @source_encoding = configuration[:source_encoding]
+        @target_file = File.join(File.dirname(control.file), configuration[:target_file])
+        @target_encoding = configuration[:target_encoding]
+        raise ControlError, "Source and target file cannot currently point to the same file" if source_file == target_file
+        begin
+          @iconv = Iconv.new(target_encoding,source_encoding)
+        rescue Iconv::InvalidEncoding
+          raise ControlError, "Either the source encoding '#{source_encoding}' or the target encoding '#{target_encoding}' is not supported"
+        end
+      end
+      
+      # Execute the processor
+      def process
+        # operate line by line to handle large files without loading them in-memory
+        # could be replaced by a system iconv call when available, for greater performance
+        File.open(source_file) do |source|
+          #puts "Opening #{target_file}"
+          File.open(target_file,'w') do |target|
+            source.each_line do |line|
+              target << @iconv.iconv(line)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/hierarchy_exploder_processor.rb

@@ -0,0 +1,55 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row-level processor that will convert a single row into multiple rows designed to be inserted
+    # into a hierarchy bridge table.
+    class HierarchyExploderProcessor < ETL::Processor::RowProcessor
+      attr_accessor :id_field
+      attr_accessor :parent_id_field
+      
+      # Initialize the processor
+      #
+      # Configuration options:
+      # * <tt>:connection</tt>: The ActiveRecord adapter connection
+      # * <tt>:id_field</tt>: The name of the id field (defaults to 'id')
+      # * <tt>:parent_id_field</tt>: The name of the parent id field (defaults to 'parent_id')
+      #
+      # TODO: Allow resolver to be implemented in a customizable fashion, i.e. don't rely
+      # on AR as the only resolution method.
+      def initialize(control, configuration={})
+        @id_field = configuration[:id_field] || 'id'
+        @parent_id_field = configuration[:parent_id_field] || 'parent_id'
+        super
+      end
+  
+      # Process the row expanding it into hierarchy values
+      def process(row)
+        rows = []
+        target = configuration[:target]
+        table = configuration[:table]
+        conn = ETL::Engine.connection(target)
+        build_rows([row[:id]], row[:id], row[:id], row[:parent_id].nil?, 0, rows, table, conn)
+        rows
+      end
+  
+      protected
+      # Recursive function that will add a row for the current level and then call build_rows
+      # for all of the children of the current level
+      def build_rows(ids, parent_id, row_id, root, level, rows, table, conn)
+        ids.each do |id|
+          child_ids = conn.select_values("SELECT #{id_field} FROM #{table} WHERE #{parent_id_field} = #{id}")
+      
+          row = {
+            :parent_id => row_id, 
+            :child_id => id, 
+            :num_levels_from_parent => level, 
+            :is_bottom => (child_ids.empty? ? 1 : 0),
+            :is_top => (root ? 1 : 0),
+          }
+          rows << row
+      
+          build_rows(child_ids, id, row_id, false, level + 1, rows, table, conn)
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/print_row_processor.rb

@@ -0,0 +1,12 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Debugging processor for printing the current row
+    class PrintRowProcessor < ETL::Processor::RowProcessor
+      # Process the row
+      def process(row)
+        puts row.inspect
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/processor.rb

@@ -0,0 +1,25 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Base class for pre and post processors. Subclasses must implement the +process+ method.
+    class Processor
+      def initialize(control, configuration)
+        @control = control
+        @configuration = configuration
+        after_initialize if respond_to?(:after_initialize)
+      end
+      protected
+      # Get the control object
+      def control
+        @control
+      end
+      # Get the configuration Hash
+      def configuration
+        @configuration
+      end
+      # Get the engine logger
+      def log
+        Engine.logger
+      end
+    end
+  end
+end

data/lib/etl/processor/rename_processor.rb

@@ -0,0 +1,24 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to rename a field in the row.
+    #
+    # Configuration options:
+    # * <tt>:source</tt>: the source field name
+    # * <tt>:dest</tt>: The destination field name
+    class RenameProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        source_value = row[configuration[:source]]
+        case source_value
+        when Numeric
+          row[configuration[:dest]] = source_value
+        when nil
+          row[configuration[:dest]] = nil
+        else
+          row[configuration[:dest]] = source_value.dup
+        end
+        row.delete(configuration[:source])
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/require_non_blank_processor.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A processor which requires that the particular fields are non-blank in
+    # order for the row to be retained.
+    class RequireNonBlankProcessor < ETL::Processor::RowProcessor
+      # An array of fields to check
+      attr_reader :fields
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:fields</tt>: An array of fields to check, for example:
+      #   [:first_name,:last_name]
+      def initialize(control, configuration)
+        super
+        @fields = configuration[:fields] || []
+      end
+      
+      # Process the row.
+      def process(row)
+        fields.each { |field| return if row[field].blank? }
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/row_processor.rb

@@ -0,0 +1,17 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Processor which processes a specific row. Unlike a transformer, which deals with a specific
+    # value in the row, row processors can process an entire row at once, which can be used to 
+    # explode a single row into multiple rows (for example)
+    class RowProcessor < Processor
+      # Initialize the processor
+      def initialize(control, configuration)
+        super
+      end
+      # Process the specified row. This method must return the row.
+      def process(row)
+        raise "process_row is an abstract method"
+      end
+    end
+  end
+end

data/lib/etl/processor/sequence_processor.rb

@@ -0,0 +1,23 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to generate a sequence.
+    #
+    # Configuration options:
+    # * <tt>:context</tt>: A context name, if none is specified then the context will be
+    #   the current ETL run
+    # * <tt>:dest</tt>: The destination field name
+    class SequenceProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        sequences[configuration[:context]] ||= 0
+        row[configuration[:dest]] = sequences[configuration[:context]] += 1
+        row
+      end
+      
+      protected
+      # Get a Hash of sequences
+      def sequences
+        @sequences ||= {}
+      end
+    end
+  end
+end

data/lib/etl/processor/surrogate_key_processor.rb

@@ -0,0 +1,53 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row level processor that provides surrogate keys
+    class SurrogateKeyProcessor < ETL::Processor::RowProcessor
+      attr_accessor :destination
+      attr_accessor :table
+      attr_accessor :column
+      attr_accessor :target
+      
+      # Initialize the surrogate key generator
+      #
+      # Configuration options
+      # * <tt>:query</tt>: If specified it contains a query to be used to 
+      #   locate the last surrogate key. If this is specified then :target
+      #   must also be specified.
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:destination</tt>: The destination column name (defaults to :id)
+      def initialize(control, configuration)
+        super
+        @table = configuration[:table]
+        @column = configuration[:column] || 'id'
+        @target = configuration[:target]
+        if configuration[:query]
+          raise ControlError, "Query option is no longer value, use :column and :table instead"
+        end
+        if table
+          @surrogate_key = ETL::Engine.connection(target).select_value("SELECT max(#{column}) FROM #{table_name}")
+        end
+        #puts "initial surrogate key: #{@surrogate_key}"
+        @surrogate_key = 0 if @surrogate_key.blank?
+        @surrogate_key = @surrogate_key.to_i
+        #puts "surrogate key: #{@surrogate_key}"
+        @destination = configuration[:destination] || :id
+      end
+      
+      # Add a surrogate key to the row
+      def process(row)
+        if row
+          #puts "processing row #{row.inspect}"
+          @surrogate_key += 1
+          #puts "adding surrogate key to row: #{@surrogate_key}"
+          row[destination] = @surrogate_key
+          row
+        end
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end