etl 0.9.5.rc1

data/lib/etl/parser/xml_parser.rb

@@ -0,0 +1,65 @@
+require 'rexml/document'
+
+module ETL
+  module Parser
+    class XmlParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each
+        Dir.glob(file).each do |file|
+          doc = nil
+          t = Benchmark.realtime do
+            doc = REXML::Document.new(File.new(file))
+          end
+          Engine.logger.info "XML #{file} parsed in #{t}s"
+          doc.elements.each(@collection_xpath) do |element|
+            row = {}
+            fields.each do |f|
+              value = element.text(f.xpath)
+              row[f.name] = value
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        @collection_xpath = source.definition[:collection]
+        raise "Collection XPath is required" if @collection_xpath.nil?
+        
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options, options.to_s)
+          when Hash
+            options[:xpath] ||= options[:name]
+            fields << Field.new(options[:name], options[:xpath].to_s)
+          else
+            raise DefinitionError, "Each field definition must either be an symbol or a hash of options for the field"
+          end
+        end
+      end
+      
+      class Field
+        attr_reader :name, :xpath
+        def initialize(name, xpath)
+          @name = name
+          @xpath = xpath
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor.rb

@@ -0,0 +1,11 @@
+# This source file contains the ETL::Processor module and requires all of the processors
+
+module ETL #:nodoc:
+  # The ETL::Processor module contains row-level and bulk processors
+  module Processor
+  end
+end
+
+require 'etl/processor/processor'
+require 'etl/processor/row_processor'
+Dir[File.dirname(__FILE__) + "/processor/*.rb"].each { |file| require(file) }

data/lib/etl/processor/block_processor.rb

@@ -0,0 +1,14 @@
+module ETL
+  module Processor
+    # This processor is both a valid RowProcessor (called on each row with after_read) or a Processor (called once on pre_process or post_process)
+    class BlockProcessor < ETL::Processor::RowProcessor
+      def initialize(control, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def process(row=nil)
+        @block.call(row)
+      end
+    end
+  end
+end

data/lib/etl/processor/bulk_import_processor.rb

@@ -0,0 +1,94 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Processor which is used to bulk import data into a target database. The
+    # underlying database driver from ActiveRecord must support the methods
+    # +bulk_load+ method.
+    class BulkImportProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :file
+      # The target database
+      attr_reader :target
+      # The table name
+      attr_reader :table
+      # Set to true to truncate
+      attr_reader :truncate
+      # Array of symbols representing the column load order
+      attr_reader :columns
+      # The field separator (defaults to a comma)
+      attr_accessor :field_separator
+      # The field enclosure (defaults to nil)
+      attr_accessor :field_enclosure
+      # The line separator (defaults to a newline)
+      attr_accessor :line_separator
+      # The string that indicates a NULL (defaults to an empty string)
+      attr_accessor :null_string
+      # boolean that indicates disable keys before, then enable after load (MySql only optimization)
+      attr_accessor :disable_keys
+      # replace existing records, not just insert
+      attr_accessor :replace
+       
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:file</tt>: The file to load data from
+      # * <tt>:target</tt>: The target database
+      # * <tt>:table</tt>: The table name
+      # * <tt>:truncate</tt>: Set to true to truncate before loading
+      # * <tt>:columns</tt>: The columns to load in the order they appear in
+      #   the bulk data file
+      # * <tt>:field_separator</tt>: The field separator. Defaults to a comma
+      # * <tt>:line_separator</tt>: The line separator. Defaults to a newline
+      # * <tt>:field_enclosure</tt>: The field enclosure charcaters
+      # * <tt>:disable_keys</tt>: Set to true to disable keys before, then enable after load (MySql only optimization)
+      def initialize(control, configuration)
+        super
+        @target = configuration[:target]
+        path = Pathname.new(configuration[:file])
+        @file = path.absolute? ? path : Pathname.new(File.dirname(File.expand_path(control.file))) + path
+
+        @table = configuration[:table]
+        @truncate = configuration[:truncate] ||= false
+        @columns = configuration[:columns]
+        @field_separator = (configuration[:field_separator] || ',')
+        @line_separator = (configuration[:line_separator] || "\n")
+        @null_string = (configuration[:null_string] || "")
+        @field_enclosure = configuration[:field_enclosure]
+        @disable_keys = configuration[:disable_keys] || false
+        @replace = configuration[:replace] || false
+        
+        raise ControlError, "Target must be specified" unless @target
+        raise ControlError, "Table must be specified" unless @table
+      end
+      
+      # Execute the processor
+      def process
+        return if ETL::Engine.skip_bulk_import
+        return if File.size(file) == 0
+        
+        conn = ETL::Engine.connection(target)
+        conn.transaction do
+          conn.truncate(table_name) if truncate
+          options = {}
+          options[:columns] = columns
+          
+          options[:disable_keys] = true if disable_keys
+          options[:replace] = true if replace
+          
+          if field_separator || field_enclosure || line_separator || null_string
+            options[:fields] = {}
+            options[:fields][:null_string] = null_string if null_string
+            options[:fields][:delimited_by] = field_separator if field_separator
+            options[:fields][:enclosed_by] = field_enclosure if field_enclosure
+            options[:fields][:terminated_by] = line_separator if line_separator
+          end
+          conn.bulk_load(file, table_name, options)
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/check_exist_processor.rb

@@ -0,0 +1,80 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row-level processor that checks if the row already exists in the 
+    # target table
+    class CheckExistProcessor < ETL::Processor::RowProcessor
+      # A symbol or array of symbols representing keys that should be skipped
+      attr_accessor :skip
+      
+      # The target database
+      attr_accessor :target
+      
+      # The name of the table to check against
+      attr_accessor :table
+      
+      # An array of columns representing the natural key
+      attr_accessor :columns
+      
+      # Is set to true if the processor should execute the check. If there are
+      # no rows in the target table then this should return false.
+      attr_accessor :should_check
+      
+      # Initialize the processor
+      # Configuration options:
+      # * <tt>:columns</tt>: An array of symbols for columns that should be included in the query conditions. If this option is not specified then all of the columns in the row will be included in the conditions (unless :skip is specified).
+      # * <tt>:skip</tt>: A symbol or array of symbols that should not be included in the existence check. If this option is not specified then all of the columns will be included in the existence check (unless :columns is specified).
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:table</tt>: The table name
+      def initialize(control, configuration)
+        super
+        @skip = configuration[:skip] || []
+        @target = configuration[:target] || raise(ETL::ControlError, "target must be specified")
+        @table = configuration[:table] || raise(ETL::ControlError, "table must be specified")
+        @columns = configuration[:columns]
+        
+        q = "SELECT COUNT(*) FROM #{table_name}"
+        @should_check = ETL::Engine.connection(target).select_value(q).to_i > 0 
+      end
+      
+      # Return true if the given key should be skipped
+      def skip?(key)
+        case skip
+        when Array
+          skip.include?(key)
+        else
+          skip.to_sym == key.to_sym
+        end
+      end
+      
+      # Return true if the row should be checked
+      def should_check?
+        @should_check ? true : false
+      end
+      
+      # Process the row
+      def process(row)
+        return row unless should_check?
+        conn = ETL::Engine.connection(target)
+        q = "SELECT * FROM #{table_name} WHERE "
+        conditions = []
+        ensure_columns_available_in_row!(row, columns, 'for existence check')
+        row.each do |k,v| 
+          if columns.nil? || columns.include?(k.to_sym)
+            conditions << "#{k} = #{conn.quote(v)}" unless skip?(k.to_sym)
+          end
+        end
+        q << conditions.join(" AND ")
+        q << " LIMIT 1"
+      
+        result = conn.select_one(q)
+        return row if result.nil?
+      end
+      
+private
+
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end

data/lib/etl/processor/check_unique_processor.rb

@@ -0,0 +1,39 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row processor that checks whether or not the row has already passed 
+    # through the ETL processor, using the key fields provided as the keys
+    # to check.
+    class CheckUniqueProcessor < ETL::Processor::RowProcessor
+
+      # The keys to check
+      attr_accessor :keys
+      
+      # Initialize the processor
+      # Configuration options:
+      # * <tt>:keys</tt>: An array of keys to check against
+      def initialize(control, configuration)
+        super
+        @keys = configuration[:keys]
+      end
+
+      # A Hash of keys that have already been processed.
+      def compound_key_constraints
+        @compound_key_constraints ||= {}
+      end
+      
+      # Process the row. This implementation will only return a row if it
+      # it's key combination has not already been seen.
+      #
+      # An error will be raised if the row doesn't include the keys.
+      def process(row)
+        ensure_columns_available_in_row!(row, keys, 'for unicity check')
+        
+        key = (keys.collect { |k| row[k] }).join('|')
+        unless compound_key_constraints[key]
+          compound_key_constraints[key] = 1
+          return row
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/copy_field_processor.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row processor that will copy one field to another
+    #
+    # Configuration options:
+    # * <tt>:destination</tt>: The destination field
+    # * <tt>:dest</tt>: Alias for :destination
+    # * <tt>:source</tt>: The source field
+    class CopyFieldProcessor < ETL::Processor::RowProcessor
+      # Process the given row
+      def process(row)
+        destination = (configuration[:destination] || configuration[:dest])
+        source_value = row[configuration[:source]]
+        case source_value
+        when Numeric
+          row[destination] = source_value
+        when nil
+          row[destination] = nil
+        else
+          row[destination] = source_value.dup
+        end
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/database_join_processor.rb

@@ -0,0 +1,82 @@
+module ETL
+  module Processor
+    class DatabaseJoinProcessor < ETL::Processor::RowProcessor
+      attr_reader :target
+      attr_reader :query
+      attr_reader :fields
+
+      # Initialize the procesor.
+      #
+      # Arguments:
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>definition</tt>: The source definition
+      #
+      # Required configuration options:
+      # * <tt>:target</tt>: The target connection
+      # * <tt>:query</tt>: The join query
+      # * <tt>:fields</tt>: The fields to add to the row
+      def initialize(control, configuration)
+        super
+        @target = configuration[:target]
+        @query = configuration[:query]
+        @fields = configuration[:fields]
+        raise ControlError, ":target must be specified" unless @target
+        raise ControlError, ":query must be specified" unless @query
+        raise ControlError, ":fields must be specified" unless @fields
+      end
+      
+      # Get a String identifier for the source
+      def to_s
+        "#{host}/#{database}"
+      end
+      
+      def process(row)
+        return nil if row.nil?
+
+        q = @query
+        begin
+          q = eval('"' + @query + '"')
+        rescue
+        end
+
+        ETL::Engine.logger.debug("Executing select: #{q}")
+        res = connection.execute(q)
+
+        case connection
+          when ActiveRecord::ConnectionAdapters::PostgreSQLAdapter;
+            res.each do |r|
+              @fields.each do |field|
+                row[field.to_sym] = r[field.to_s]
+              end
+            end
+          when ActiveRecord::ConnectionAdapters::MysqlAdapter;
+            res.each_hash do |r|
+              @fields.each do |field|
+                row[field.to_sym] = r[field.to_s]
+              end
+            end
+            res.free
+          else raise "Unsupported adapter #{connection.class} for this destination"
+        end
+
+        return row
+      end
+
+      private
+      # Get the database connection to use
+      def connection
+        ETL::Engine.connection(target)
+      end
+      
+      # Get the host, defaults to 'localhost'
+      def host
+        ETL::Base.configurations[target.to_s]['host'] || 'localhost'
+      end
+      
+      def database
+        ETL::Base.configurations[target.to_s]['database']
+      end
+    end
+  end
+end

data/lib/etl/processor/encode_processor.rb

@@ -0,0 +1,55 @@
+require 'iconv'
+
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # The encode processor uses Iconv to convert a file from one encoding (eg: utf-8) to another (eg: latin1), line by line.
+    class EncodeProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :source_file
+      # The file to write to
+      attr_reader :target_file
+      # The source file encoding
+      attr_reader :source_encoding
+      # The target file encoding
+      attr_reader :target_encoding
+      
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:source_file</tt>: The file to load data from
+      # * <tt>:source_encoding</tt>: The source file encoding (eg: 'latin1','utf-8'), as supported by Iconv
+      # * <tt>:target_file</tt>: The file to write data to
+      # * <tt>:target_encoding</tt>: The target file encoding
+      def initialize(control, configuration)
+        super
+        raise ControlError, "Source file must be specified" if configuration[:source_file].nil?
+        raise ControlError, "Target file must be specified" if configuration[:target_file].nil?
+        @source_file = File.join(File.dirname(control.file), configuration[:source_file])
+        @source_encoding = configuration[:source_encoding]
+        @target_file = File.join(File.dirname(control.file), configuration[:target_file])
+        @target_encoding = configuration[:target_encoding]
+        raise ControlError, "Source and target file cannot currently point to the same file" if source_file == target_file
+        begin
+          @iconv = Iconv.new(target_encoding,source_encoding)
+        rescue Iconv::InvalidEncoding
+          raise ControlError, "Either the source encoding '#{source_encoding}' or the target encoding '#{target_encoding}' is not supported"
+        end
+      end
+      
+      # Execute the processor
+      def process
+        # operate line by line to handle large files without loading them in-memory
+        # could be replaced by a system iconv call when available, for greater performance
+        File.open(source_file) do |source|
+          #puts "Opening #{target_file}"
+          File.open(target_file,'w') do |target|
+            source.each_line do |line|
+              target << @iconv.iconv(line)
+            end
+          end
+        end
+      end
+    end
+  end
+end