activewarehouse-etl 0.6.1 → 0.7.0

data/lib/etl/execution.rb

@@ -0,0 +1,19 @@
+module ETL #:nodoc
+  # Classes which store information about ETL execution
+  module Execution
+    # Execution management
+    class Execution
+      class << self
+        # Migrate the data store
+        def migrate
+          ETL::Execution::Migration.migrate
+        end
+      end
+    end
+  end
+end
+
+require 'etl/execution/base'
+require 'etl/execution/job'
+require 'etl/execution/record'
+require 'etl/execution/migration'

data/lib/etl/execution/base.rb

@@ -0,0 +1,9 @@
+module ETL #:nodoc:
+  module Execution #:nodoc:
+    # Base class for ETL execution information
+    class Base < ActiveRecord::Base
+      self.abstract_class = true
+      establish_connection :etl_execution
+    end
+  end
+end

data/lib/etl/execution/job.rb

@@ -0,0 +1,7 @@
+module ETL #:nodoc:
+  module Execution #:nodoc:
+    # Persistent class representing an ETL job
+    class Job < Base
+    end
+  end
+end

data/lib/etl/execution/migration.rb

@@ -0,0 +1,54 @@
+module ETL #:nodoc:
+  module Execution #:nodoc
+    # Handles migration of tables required for persistent storage of meta data 
+    # for the ETL engine
+    class Migration
+      class << self
+        # Execute the migrations
+        def migrate
+          connection.initialize_schema_information
+          v = connection.select_value("SELECT version FROM #{schema_info_table_name}").to_i
+          v.upto(target - 1) { |i| __send__("migration_#{i+1}".to_sym) }
+        end
+        protected
+        # Get the schema info table name
+        def schema_info_table_name
+          ETL::Execution::Base.table_name_prefix + "schema_info" + 
+            ETL::Execution::Base.table_name_suffix
+        end
+        
+        # Get the connection to use during migration
+        def connection
+          @connection ||= ETL::Execution::Base.connection
+        end
+        
+        # Get the final target version number
+        def target
+          1
+        end
+        
+        private
+        def migration_1 #:nodoc:
+          connection.create_table :jobs do |t|
+            t.column :control_file, :string, :null => false
+            t.column :created_at, :datetime, :null => false
+            t.column :completed_at, :datetime
+            t.column :status, :string
+          end
+          connection.create_table :records do |t|
+            t.column :control_file, :string, :null => false
+            t.column :natural_key, :string, :null => false
+            t.column :crc, :string, :null => false
+            t.column :job_id, :integer, :null => false
+          end
+          update_schema_info(1)
+        end
+      
+        # Update the schema info table, setting the version value
+        def update_schema_info(version)
+          connection.update("UPDATE #{schema_info_table_name} SET version = #{version}")
+        end
+      end
+    end
+  end
+end

data/lib/etl/execution/record.rb

@@ -0,0 +1,8 @@
+module ETL #:nodoc:
+  module Execution #:nodoc:
+    # Represents a single record
+    class Record < ETL::Execution::Base
+      belongs_to :table
+    end
+  end
+end

data/lib/etl/generator/surrogate_key_generator.rb

@@ -1,3 +1,5 @@
+# This source file contains code for a basic sequential surrogate key generator
+
 module ETL #:nodoc:
   module Generator #:nodoc:
     # Surrogate key generator.

data/lib/etl/parser.rb

@@ -1,2 +1,11 @@
+# This source file contains the ETL::Parser module and requires all of the files
+# in the parser directory ending with .rb
+
+module ETL #:nodoc:
+  # The ETL::Parser module provides various text parsers.
+  module Parser
+  end
+end
+
 require 'etl/parser/parser'
 Dir[File.dirname(__FILE__) + "/parser/*.rb"].each { |file| require(file) }

data/lib/etl/parser/parser.rb

@@ -1,5 +1,8 @@
-module ETL
-  module Parser
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Base parser class. Implementation classes must extend this class and implement
+    # the each method. The each method should return each row of the source data as 
+    # a Hash.
     class Parser
       include Enumerable
       class << self

data/lib/etl/parser/sax_parser.rb

@@ -29,6 +29,7 @@ module ETL #:nodoc:
         end
       end
       
+      # Get an array of Field objects
       def fields
         @fields ||= []
       end
@@ -44,16 +45,21 @@ module ETL #:nodoc:
         end
       end
       
+      # Class representing a field to be loaded from the source
       class Field
-        attr_reader :name, :path
-        def initialize(name, path)
+        # The name of the field
+        attr_reader :name
+        # The XPath-like path to the field in the XML document
+        attr_reader :path
+        
+        def initialize(name, path) #:nodoc
           @name = name
           @path = path
         end
       end
     end
     
-    class Listener
+    class Listener #:nodoc:
       include REXML::SAX2Listener
       def initialize(parser, &block)
         @parser = parser
@@ -120,19 +126,28 @@ module ETL #:nodoc:
       end
     end
 
-    module XPath
-      class Path
+    # Module which contains classes that are used for XPath-like filtering
+    # on the SAX parser
+    module XPath #:nodoc:
+      class Path #:nodoc:
+        # Get the elements in the path
         attr_accessor :elements
+        
+        # Initialize
         def initialize
           @elements = []
         end
+        
+        # Convert to a string representation
         def to_s
           @elements.map{ |e| e.to_s }.join("/")
         end
+        
         # Returns true if the last part of the path refers to an attribute
         def is_attribute?
           elements.last.attributes.length > 0
         end
+        
         # Return the name of the attribute referenced by the last element in this path. Returns nil if the last element
         # does not reference an attribute.
         #
@@ -142,6 +157,7 @@ module ETL #:nodoc:
           return nil unless is_attribute?
           elements.last.attributes.keys.first
         end
+        
         # Return true if this XPath::Path matches the given path string. This is a fail-fast match, so the first mismatch
         # will cause the method to return false.
         def match?(s)
@@ -178,7 +194,7 @@ module ETL #:nodoc:
           path
         end
       end
-      class Element
+      class Element #:nodoc
         attr_reader :name
         attr_reader :attributes
         def initialize(name, attributes={})

data/lib/etl/processor.rb

@@ -1,3 +1,11 @@
+# This source file contains the ETL::Processor module and requires all of the processors
+
+module ETL #:nodoc:
+  # The ETL::Processor module contains row-level and bulk processors
+  module Processor
+  end
+end
+
 require 'etl/processor/processor'
 require 'etl/processor/row_processor'
 Dir[File.dirname(__FILE__) + "/processor/*.rb"].each { |file| require(file) }

data/lib/etl/processor/bulk_import_processor.rb

@@ -1,11 +1,35 @@
 module ETL #:nodoc:
   module Processor #:nodoc:
-    # Processor which is used to bulk import data into a target database
+    # Processor which is used to bulk import data into a target database. The
+    # underlying database driver from ActiveRecord must support the methods
+    # +bulk_load+ method.
     class BulkImportProcessor < ETL::Processor::Processor
-      attr_reader :file, :target, :truncate, :columns
+      # The file to load from
+      attr_reader :file
+      # The target database information (see +initialize+)
+      attr_reader :target
+      # Set to true to truncate
+      attr_reader :truncate
+      # Array of symbols representing the column load order
+      attr_reader :columns
+      # The field separator (defaults to a comma)
       attr_accessor :field_separator
+      # The field enclosure (defaults to nil)
       attr_accessor :field_enclosure
+      # The line separator (defaults to a newline)
       attr_accessor :line_separator
+       
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:file</tt>: The file to load data from
+      # * <tt>:target</tt>: The target connection information
+      # * <tt>:truncate</tt>: Set to true to truncate before loading
+      # * <tt>:columns</tt>: The columns to load in the order they appear in
+      #   the bulk data file
+      # * <tt>:field_separator</tt>: The field separator. Defaults to a comma
+      # * <tt>:line_separator</tt>: The line separator. Defaults to a newline
+      # * <tt>:field_enclosure</tt>: The field enclosure charcaters
       def initialize(control, configuration)
         super
         @file = File.join(File.dirname(control.file), configuration[:file])
@@ -13,12 +37,15 @@ module ETL #:nodoc:
         @truncate = configuration[:truncate] ||= false
         @columns = configuration[:columns]
         @field_separator = (configuration[:field_separator] || ',')
-        @line_separator = configuration[:line_separator]
+        @line_separator = (configuration[:line_separator] || "\n")
         @field_enclosure = configuration[:field_enclosure]
         connect
       end
+      
+      # Execute the processor
       def process
-        # columns = control.destinations.first.order.join(',') # TODO: support multiple destinations?
+        return if ETL::Engine.skip_bulk_import
+        
         conn = ETL::ActiveRecord::Base.connection
         conn.transaction do
           # TODO: Support all database types
@@ -30,6 +57,7 @@ module ETL #:nodoc:
             options[:fields] = {}
             options[:fields][:delimited_by] = field_separator if field_separator
             options[:fields][:enclosed_by] = field_enclosure if field_enclosure
+            options[:fields][:terminated_by] = line_separator if line_separator
           end
           conn.bulk_load(file, target[:table], options)
         end

data/lib/etl/processor/check_exist_processor.rb

@@ -0,0 +1,69 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row-level processor that checks if the row already exists in the 
+    # target table
+    class CheckExistProcessor < ETL::Processor::RowProcessor
+      # A symbol or array of symbols representing keys that should be skipped
+      attr_accessor :skip
+      
+      # The name of the table to check against
+      attr_accessor :table
+      
+      # An array of columns representing the natural key
+      attr_accessor :columns
+      
+      # Is set to true if the processor should execute the check. If there are
+      # no rows in the target table then this should return false.
+      attr_accessor :should_check
+      
+      # Initialize the processor
+      # Configuration options:
+      # * <tt>:skip</tt>: A symbol or array of column names that should not 
+      #   be checked
+      # * <tt>:table</tt>: The table name
+      # * <tt>:columns</tt>: An array of columns which represent the natural
+      #   key
+      def initialize(control, configuration)
+        super
+        @skip = configuration[:skip]
+        @table = configuration[:table]
+        @columns = configuration[:columns]
+        
+        q = "SELECT COUNT(*) FROM #{table}"
+        @should_check = ActiveRecord::Base.connection.select_value(q).to_i > 0 
+      end
+      
+      # Return true if the given key should be skipped
+      def skip?(key)
+        case skip
+        when Array
+          skip.include?(key)
+        else
+          skip.to_sym == key.to_sym
+        end
+      end
+      
+      # Return true if the row should be checked
+      def should_check?
+        @should_check ? true : false
+      end
+      
+      # Process the row
+      def process(row)
+        return row unless should_check?
+        q = "SELECT * FROM #{table} WHERE "
+        conditions = []
+        row.each do |k,v| 
+          if columns.nil? || columns.include?(k.to_sym)
+            conditions << "#{k} = '#{v}'" unless skip?(k)
+          end
+        end
+        q << conditions.join(" AND ")
+      
+        #puts "query: #{q}"
+        result = ActiveRecord::Base.connection.select_one(q)
+        return row if result.nil?
+      end
+    end
+  end
+end

data/lib/etl/processor/check_unique_processor.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row processor that checks whether or not the row has already passed 
+    # through the ETL processor, using the key fields provided as the keys
+    # to check.
+    class CheckUniqueProcessor < ETL::Processor::RowProcessor
+
+      # The keys to check
+      attr_accessor :keys
+      
+      # Initialize the processor
+      # Configuration options:
+      # * <tt>:keys</tt>: An array of keys to check against
+      def initialize(control, configuration)
+        super
+        @keys = configuration[:keys]
+      end
+
+      # A Hash of keys that have already been processed.
+      def compound_key_constraints
+        @compound_key_constraints ||= {}
+      end
+      
+      # Process the row. This implementation will only return a row if it
+      # it's key combination has not already been seen.
+      def process(row)
+        key = (keys.collect { |k| row[k] }).join('|')
+        unless compound_key_constraints[key]
+          compound_key_constraints[key] = 1
+          return row
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/copy_field_processor.rb

@@ -1,8 +1,24 @@
-module ETL
-  module Processor
-    class CopyField < ETL::Processor::RowProcessor
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row processor that will copy one field to another
+    #
+    # Configuration options:
+    # * <tt>:destination</tt>: The destination field
+    # * <tt>:dest</tt>: Alias for :destination
+    # * <tt>:source</tt>: The source field
+    class CopyFieldProcessor < ETL::Processor::RowProcessor
+      # Process the given row
       def process(row)
-        row[configuration[:destination]] = row[configuration[:source]].dup
+        destination = (configuration[:destination] || configuration[:dest])
+        source_value = row[configuration[:source]]
+        case source_value
+        when Numeric
+          row[destination] = source_value
+        when nil
+          row[destination] = nil
+        else
+          row[destination] = source_value.dup
+        end
         row
       end
     end

data/lib/etl/processor/processor.rb

@@ -7,12 +7,15 @@ module ETL #:nodoc:
         @configuration = configuration
       end
       protected
+      # Get the control object
       def control
         @control
       end
+      # Get the configuration Hash
       def configuration
         @configuration
       end
+      # Get the engine logger
       def log
         Engine.logger
       end