darrell-activewarehouse-etl 0.9.1.4

data/lib/etl/parser/xml_parser.rb

@@ -0,0 +1,65 @@
+require 'rexml/document'
+
+module ETL
+  module Parser
+    class XmlParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each
+        Dir.glob(file).each do |file|
+          doc = nil
+          t = Benchmark.realtime do
+            doc = REXML::Document.new(File.new(file))
+          end
+          Engine.logger.info "XML #{file} parsed in #{t}s"
+          doc.elements.each(@collection_xpath) do |element|
+            row = {}
+            fields.each do |f|
+              value = element.text(f.xpath)
+              row[f.name] = value
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        @collection_xpath = source.definition[:collection]
+        raise "Collection XPath is required" if @collection_xpath.nil?
+        
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options, options.to_s)
+          when Hash
+            options[:xpath] ||= options[:name]
+            fields << Field.new(options[:name], options[:xpath].to_s)
+          else
+            raise DefinitionError, "Each field definition must either be an symbol or a hash of options for the field"
+          end
+        end
+      end
+      
+      class Field
+        attr_reader :name, :xpath
+        def initialize(name, xpath)
+          @name = name
+          @xpath = xpath
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser.rb

@@ -0,0 +1,11 @@
+# This source file contains the ETL::Parser module and requires all of the files
+# in the parser directory ending with .rb
+
+module ETL #:nodoc:
+  # The ETL::Parser module provides various text parsers.
+  module Parser
+  end
+end
+
+require 'etl/parser/parser'
+Dir[File.dirname(__FILE__) + "/parser/*.rb"].each { |file| require(file) }

data/lib/etl/processor/block_processor.rb

@@ -0,0 +1,14 @@
+module ETL
+  module Processor
+    # This processor is both a valid RowProcessor (called on each row with after_read) or a Processor (called once on pre_process or post_process)
+    class BlockProcessor < ETL::Processor::RowProcessor
+      def initialize(control, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def process(row=nil)
+        @block.call(row)
+      end
+    end
+  end
+end

data/lib/etl/processor/bulk_import_processor.rb

@@ -0,0 +1,83 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Processor which is used to bulk import data into a target database. The
+    # underlying database driver from ActiveRecord must support the methods
+    # +bulk_load+ method.
+    class BulkImportProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :file
+      # The target database
+      attr_reader :target
+      # The table name
+      attr_reader :table
+      # Set to true to truncate
+      attr_reader :truncate
+      # Array of symbols representing the column load order
+      attr_reader :columns
+      # The field separator (defaults to a comma)
+      attr_accessor :field_separator
+      # The field enclosure (defaults to nil)
+      attr_accessor :field_enclosure
+      # The line separator (defaults to a newline)
+      attr_accessor :line_separator
+      # The string that indicates a NULL (defaults to an empty string)
+      attr_accessor :null_string
+       
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:file</tt>: The file to load data from
+      # * <tt>:target</tt>: The target database
+      # * <tt>:table</tt>: The table name
+      # * <tt>:truncate</tt>: Set to true to truncate before loading
+      # * <tt>:columns</tt>: The columns to load in the order they appear in
+      #   the bulk data file
+      # * <tt>:field_separator</tt>: The field separator. Defaults to a comma
+      # * <tt>:line_separator</tt>: The line separator. Defaults to a newline
+      # * <tt>:field_enclosure</tt>: The field enclosure charcaters
+      def initialize(control, configuration)
+        super
+        @target = configuration[:target]
+        path = Pathname.new(configuration[:file])
+        @file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(control.file))) + path
+
+        @table = configuration[:table]
+        @truncate = configuration[:truncate] ||= false
+        @columns = configuration[:columns]
+        @field_separator = (configuration[:field_separator] || ',')
+        @line_separator = (configuration[:line_separator] || "\n")
+        @null_string = (configuration[:null_string] || "")
+        @field_enclosure = configuration[:field_enclosure]
+        
+        raise ControlError, "Target must be specified" unless @target
+        raise ControlError, "Table must be specified" unless @table
+      end
+      
+      # Execute the processor
+      def process
+        return if ETL::Engine.skip_bulk_import
+        return if File.size(file) == 0
+        
+        conn = ETL::Engine.connection(target)
+        conn.transaction do
+          conn.truncate(table_name) if truncate
+          options = {}
+          options[:columns] = columns
+          if field_separator || field_enclosure || line_separator || null_string
+            options[:fields] = {}
+            options[:fields][:null_string] = null_string if null_string
+            options[:fields][:delimited_by] = field_separator if field_separator
+            options[:fields][:enclosed_by] = field_enclosure if field_enclosure
+            options[:fields][:terminated_by] = line_separator if line_separator
+          end
+          conn.bulk_load(file, table_name, options)
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/check_exist_processor.rb

@@ -0,0 +1,80 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row-level processor that checks if the row already exists in the 
+    # target table
+    class CheckExistProcessor < ETL::Processor::RowProcessor
+      # A symbol or array of symbols representing keys that should be skipped
+      attr_accessor :skip
+      
+      # The target database
+      attr_accessor :target
+      
+      # The name of the table to check against
+      attr_accessor :table
+      
+      # An array of columns representing the natural key
+      attr_accessor :columns
+      
+      # Is set to true if the processor should execute the check. If there are
+      # no rows in the target table then this should return false.
+      attr_accessor :should_check
+      
+      # Initialize the processor
+      # Configuration options:
+      # * <tt>:skip</tt>: A symbol or array of column names that should not 
+      #   be checked
+      # * <tt>:table</tt>: The table name
+      # * <tt>:columns</tt>: An array of columns which represent the natural
+      #   key
+      def initialize(control, configuration)
+        super
+        @skip = configuration[:skip] || []
+        @target = configuration[:target] || raise(ETL::ControlError, "target must be specified")
+        @table = configuration[:table] || raise(ETL::ControlError, "table must be specified")
+        @columns = configuration[:columns]
+        
+        q = "SELECT COUNT(*) FROM #{table_name}"
+        @should_check = ETL::Engine.connection(target).select_value(q).to_i > 0 
+      end
+      
+      # Return true if the given key should be skipped
+      def skip?(key)
+        case skip
+        when Array
+          skip.include?(key)
+        else
+          skip.to_sym == key.to_sym
+        end
+      end
+      
+      # Return true if the row should be checked
+      def should_check?
+        @should_check ? true : false
+      end
+      
+      # Process the row
+      def process(row)
+        return row unless should_check?
+        conn = ETL::Engine.connection(target)
+        q = "SELECT * FROM #{table_name} WHERE "
+        conditions = []
+        row.each do |k,v| 
+          if columns.nil? || columns.include?(k.to_sym)
+            conditions << "#{k} = #{conn.quote(v)}" unless skip?(k.to_sym)
+          end
+        end
+        q << conditions.join(" AND ")
+        q << " LIMIT 1"
+      
+        #puts "query: #{q}"
+        result = conn.select_one(q)
+        return row if result.nil?
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/check_unique_processor.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row processor that checks whether or not the row has already passed 
+    # through the ETL processor, using the key fields provided as the keys
+    # to check.
+    class CheckUniqueProcessor < ETL::Processor::RowProcessor
+
+      # The keys to check
+      attr_accessor :keys
+      
+      # Initialize the processor
+      # Configuration options:
+      # * <tt>:keys</tt>: An array of keys to check against
+      def initialize(control, configuration)
+        super
+        @keys = configuration[:keys]
+      end
+
+      # A Hash of keys that have already been processed.
+      def compound_key_constraints
+        @compound_key_constraints ||= {}
+      end
+      
+      # Process the row. This implementation will only return a row if it
+      # it's key combination has not already been seen.
+      def process(row)
+        key = (keys.collect { |k| row[k] }).join('|')
+        unless compound_key_constraints[key]
+          compound_key_constraints[key] = 1
+          return row
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/copy_field_processor.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row processor that will copy one field to another
+    #
+    # Configuration options:
+    # * <tt>:destination</tt>: The destination field
+    # * <tt>:dest</tt>: Alias for :destination
+    # * <tt>:source</tt>: The source field
+    class CopyFieldProcessor < ETL::Processor::RowProcessor
+      # Process the given row
+      def process(row)
+        destination = (configuration[:destination] || configuration[:dest])
+        source_value = row[configuration[:source]]
+        case source_value
+        when Numeric
+          row[destination] = source_value
+        when nil
+          row[destination] = nil
+        else
+          row[destination] = source_value.dup
+        end
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/encode_processor.rb

@@ -0,0 +1,55 @@
+require 'iconv'
+
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # The encode processor uses Iconv to convert a file from one encoding (eg: utf-8) to another (eg: latin1), line by line.
+    class EncodeProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :source_file
+      # The file to write to
+      attr_reader :target_file
+      # The source file encoding
+      attr_reader :source_encoding
+      # The target file encoding
+      attr_reader :target_encoding
+      
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:source_file</tt>: The file to load data from
+      # * <tt>:source_encoding</tt>: The source file encoding (eg: 'latin1','utf-8'), as supported by Iconv
+      # * <tt>:target_file</tt>: The file to write data to
+      # * <tt>:target_encoding</tt>: The target file encoding
+      def initialize(control, configuration)
+        super
+        raise ControlError, "Source file must be specified" if configuration[:source_file].nil?
+        raise ControlError, "Target file must be specified" if configuration[:target_file].nil?
+        @source_file = File.join(File.dirname(control.file), configuration[:source_file])
+        @source_encoding = configuration[:source_encoding]
+        @target_file = File.join(File.dirname(control.file), configuration[:target_file])
+        @target_encoding = configuration[:target_encoding]
+        raise ControlError, "Source and target file cannot currently point to the same file" if source_file == target_file
+        begin
+          @iconv = Iconv.new(target_encoding,source_encoding)
+        rescue Iconv::InvalidEncoding
+          raise ControlError, "Either the source encoding '#{source_encoding}' or the target encoding '#{target_encoding}' is not supported"
+        end
+      end
+      
+      # Execute the processor
+      def process
+        # operate line by line to handle large files without loading them in-memory
+        # could be replaced by a system iconv call when available, for greater performance
+        File.open(source_file) do |source|
+          #puts "Opening #{target_file}"
+          File.open(target_file,'w') do |target|
+            source.each_line do |line|
+              target << @iconv.iconv(line)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/hierarchy_exploder_processor.rb

@@ -0,0 +1,55 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row-level processor that will convert a single row into multiple rows designed to be inserted
+    # into a hierarchy bridge table.
+    class HierarchyExploderProcessor < ETL::Processor::RowProcessor
+      attr_accessor :id_field
+      attr_accessor :parent_id_field
+      
+      # Initialize the processor
+      #
+      # Configuration options:
+      # * <tt>:connection</tt>: The ActiveRecord adapter connection
+      # * <tt>:id_field</tt>: The name of the id field (defaults to 'id')
+      # * <tt>:parent_id_field</tt>: The name of the parent id field (defaults to 'parent_id')
+      #
+      # TODO: Allow resolver to be implemented in a customizable fashion, i.e. don't rely
+      # on AR as the only resolution method.
+      def initialize(control, configuration={})
+        @id_field = configuration[:id_field] || 'id'
+        @parent_id_field = configuration[:parent_id_field] || 'parent_id'
+        super
+      end
+  
+      # Process the row expanding it into hierarchy values
+      def process(row)
+        rows = []
+        target = configuration[:target]
+        table = configuration[:table]
+        conn = ETL::Engine.connection(target)
+        build_rows([row[:id]], row[:id], row[:id], row[:parent_id].nil?, 0, rows, table, conn)
+        rows
+      end
+  
+      protected
+      # Recursive function that will add a row for the current level and then call build_rows
+      # for all of the children of the current level
+      def build_rows(ids, parent_id, row_id, root, level, rows, table, conn)
+        ids.each do |id|
+          child_ids = conn.select_values("SELECT #{id_field} FROM #{table} WHERE #{parent_id_field} = #{id}")
+      
+          row = {
+            :parent_id => row_id, 
+            :child_id => id, 
+            :num_levels_from_parent => level, 
+            :is_bottom => (child_ids.empty? ? 1 : 0),
+            :is_top => (root ? 1 : 0),
+          }
+          rows << row
+      
+          build_rows(child_ids, id, row_id, false, level + 1, rows, table, conn)
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/print_row_processor.rb

@@ -0,0 +1,12 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Debugging processor for printing the current row
+    class PrintRowProcessor < ETL::Processor::RowProcessor
+      # Process the row
+      def process(row)
+        puts row.inspect
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/processor.rb

@@ -0,0 +1,25 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Base class for pre and post processors. Subclasses must implement the +process+ method.
+    class Processor
+      def initialize(control, configuration)
+        @control = control
+        @configuration = configuration
+        after_initialize if respond_to?(:after_initialize)
+      end
+      protected
+      # Get the control object
+      def control
+        @control
+      end
+      # Get the configuration Hash
+      def configuration
+        @configuration
+      end
+      # Get the engine logger
+      def log
+        Engine.logger
+      end
+    end
+  end
+end

data/lib/etl/processor/rename_processor.rb

@@ -0,0 +1,24 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to rename a field in the row.
+    #
+    # Configuration options:
+    # * <tt>:source</tt>: the source field name
+    # * <tt>:dest</tt>: The destination field name
+    class RenameProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        source_value = row[configuration[:source]]
+        case source_value
+        when Numeric
+          row[configuration[:dest]] = source_value
+        when nil
+          row[configuration[:dest]] = nil
+        else
+          row[configuration[:dest]] = source_value.dup
+        end
+        row.delete(configuration[:source])
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/require_non_blank_processor.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A processor which requires that the particular fields are non-blank in
+    # order for the row to be retained.
+    class RequireNonBlankProcessor < ETL::Processor::RowProcessor
+      # An array of fields to check
+      attr_reader :fields
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:fields</tt>: An array of fields to check, for example:
+      #   [:first_name,:last_name]
+      def initialize(control, configuration)
+        super
+        @fields = configuration[:fields] || []
+      end
+      
+      # Process the row.
+      def process(row)
+        fields.each { |field| return if row[field].blank? }
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/row_processor.rb

@@ -0,0 +1,17 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Processor which processes a specific row. Unlike a transformer, which deals with a specific
+    # value in the row, row processors can process an entire row at once, which can be used to 
+    # explode a single row into multiple rows (for example)
+    class RowProcessor < Processor
+      # Initialize the processor
+      def initialize(control, configuration)
+        super
+      end
+      # Process the specified row. This method must return the row.
+      def process(row)
+        raise "process_row is an abstract method"
+      end
+    end
+  end
+end

data/lib/etl/processor/sequence_processor.rb

@@ -0,0 +1,23 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to generate a sequence.
+    #
+    # Configuration options:
+    # * <tt>:context</tt>: A context name, if none is specified then the context will be
+    #   the current ETL run
+    # * <tt>:dest</tt>: The destination field name
+    class SequenceProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        sequences[configuration[:context]] ||= 0
+        row[configuration[:dest]] = sequences[configuration[:context]] += 1
+        row
+      end
+      
+      protected
+      # Get a Hash of sequences
+      def sequences
+        @sequences ||= {}
+      end
+    end
+  end
+end

data/lib/etl/processor/surrogate_key_processor.rb

@@ -0,0 +1,53 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row level processor that provides surrogate keys
+    class SurrogateKeyProcessor < ETL::Processor::RowProcessor
+      attr_accessor :destination
+      attr_accessor :table
+      attr_accessor :column
+      attr_accessor :target
+      
+      # Initialize the surrogate key generator
+      #
+      # Configuration options
+      # * <tt>:query</tt>: If specified it contains a query to be used to 
+      #   locate the last surrogate key. If this is specified then :target
+      #   must also be specified.
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:destination</tt>: The destination column name (defaults to :id)
+      def initialize(control, configuration)
+        super
+        @table = configuration[:table]
+        @column = configuration[:column] || 'id'
+        @target = configuration[:target]
+        if configuration[:query]
+          raise ControlError, "Query option is no longer value, use :column and :table instead"
+        end
+        if table
+          @surrogate_key = ETL::Engine.connection(target).select_value("SELECT max(#{column}) FROM #{table_name}")
+        end
+        #puts "initial surrogate key: #{@surrogate_key}"
+        @surrogate_key = 0 if @surrogate_key.blank?
+        @surrogate_key = @surrogate_key.to_i
+        #puts "surrogate key: #{@surrogate_key}"
+        @destination = configuration[:destination] || :id
+      end
+      
+      # Add a surrogate key to the row
+      def process(row)
+        if row
+          #puts "processing row #{row.inspect}"
+          @surrogate_key += 1
+          #puts "adding surrogate key to row: #{@surrogate_key}"
+          row[destination] = @surrogate_key
+          row
+        end
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/truncate_processor.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A processor which will truncate a table. Use as a pre-processor for cleaning out a table
+    # prior to loading
+    class TruncateProcessor < ETL::Processor::Processor
+      # Defines the table to truncate
+      attr_reader :table
+      
+      # Defines the database connection to use
+      attr_reader :target
+      
+      # Initialize the processor
+      #
+      # Options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The table name
+      def initialize(control, configuration)
+        super
+        #@file = File.join(File.dirname(control.file), configuration[:file])
+        @target = configuration[:target] || {}
+        @table = configuration[:table]
+      end
+      
+      def process
+        conn = ETL::Engine.connection(target)
+        conn.truncate(table_name)
+      end
+      
+      private
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor.rb

@@ -0,0 +1,11 @@
+# This source file contains the ETL::Processor module and requires all of the processors
+
+module ETL #:nodoc:
+  # The ETL::Processor module contains row-level and bulk processors
+  module Processor
+  end
+end
+
+require 'etl/processor/processor'
+require 'etl/processor/row_processor'
+Dir[File.dirname(__FILE__) + "/processor/*.rb"].each { |file| require(file) }