activewarehouse-etl 0.4.0 → 0.5.0

data/CHANGELOG

@@ -1,18 +1,30 @@
 0.1.0 - Dec 6, 2006
- * Initial release
- 
+* Initial release
+
 0.2.0 - Dec 7, 2006
- * Added an XML parser for source parsing
- * Added support for compound key constraints in destinations via the :unique => [] option
- * Added ability to declare explicit columns in bulk import
- * Added support for generators in destinations
- * Added a SurrogateKeyGenerator for cases where the database doesn't support auto generated surrogate keys
- 
+* Added an XML parser for source parsing
+* Added support for compound key constraints in destinations via the :unique => [] option
+* Added ability to declare explicit columns in bulk import
+* Added support for generators in destinations
+* Added a SurrogateKeyGenerator for cases where the database doesn't support auto generated surrogate keys
+
 0.3.0 - Dec 19, 2006
- * Added support for calculated values in virtual fields with Proc
- 
+* Added support for calculated values in virtual fields with Proc
+
 0.4.0 - Jan 11, 2006
- * Added :skip_lines option to file source configurations, which can be used to skip the first n lines in the source data file
- * Added better error handling in delimited parser - an error is now raised if the expected and actual field lengths do not match
- * Added :truncate option for database destination. Set to true to truncate before importing data.
- * Added support for :unique => [] option and virtual fields for the database destination
+* Added :skip_lines option to file source configurations, which can be used to skip the first n lines in the source data file
+* Added better error handling in delimited parser - an error is now raised if the expected and actual field lengths do not match
+* Added :truncate option for database destination. Set to true to truncate before importing data.
+* Added support for :unique => [] option and virtual fields for the database destination
+
+0.5.0 - 
+* Changed require_gem to gem and added alias to allow for older versions of rubygems
+* Added support for Hash in the source configuration where :name => :parser_name defines the parser to use and
+  :options => {} defines options to pass to the parser
+* Added support for passing a custom Parser class in the source configuration
+* Removed the need to include Enumerable in each parser implementation
+* Added new date_to_string and string_to_date transformers
+* Implemented foreign_key_lookup transform including an ActiveRecordResolver
+* Added real time activity logging which is called when the etl bin script is invoked
+* Improved error handling
+* Default logger level is now WARN

data/TODO

@@ -2,8 +2,10 @@ TODO
 
 * Add build-in support for audit_dimension
 * Do not rerun the processing if it isn't needed, i.e. the source and control files have not been modified (allow forced override)
-* Add built-in FK lookup
 * Provide greater control in error handling
 ** Allow a error threshold
 ** Don't die completely if a parse error, just stop processing that specific file if error threshold is reached
-** Allow mismatch row length error in delimited parser to be ignored
+** Allow mismatch row length error in delimited parser to be ignored
+* Improve error messages throughout, but especially in problems with the control files
+* Add support for determining if a row should be added, updated or removed vs. just blindly inserting
+* Add support for paritioned views during the insert process. Use specifiable columns as the trigger columns for determining the data output destination.

data/lib/etl.rb

@@ -30,17 +30,22 @@ unless defined?(Logger)
 end
 
 require 'rubygems'
+
+unless Kernel.respond_to?(:gem)
+  Kernel.send :alias_method, :gem, :require_gem
+end
+
 unless defined?(ActiveSupport)
-  require_gem 'activesupport'
+  gem 'activesupport'
   require 'active_support'
 end
 
 unless defined?(ActiveRecord)
-  require_gem 'activerecord'
+  gem 'activerecord'
   require 'active_record'
 end
 
-require_gem 'fastercsv'
+gem 'fastercsv'
 require 'faster_csv'
 
 $:.unshift(File.dirname(__FILE__))
@@ -64,4 +69,6 @@ module ETL #:nodoc:
   end
   class MismatchError < ETLError #:nodoc:
   end
+  class ResolverError < ETLError #:nodoc:
+  end
 end

data/lib/etl/active_record_ext/connection_adapters/mysql_adapter.rb

@@ -1,7 +1,7 @@
 require 'active_record/connection_adapters/abstract_adapter'
 
-module ActiveRecord
-  module ConnectionAdapters
+module ActiveRecord #:nodoc:
+  module ConnectionAdapters #:nodoc:
     class MysqlAdapter < AbstractAdapter
       # Execute a truncate statement on the table. Note that in MySQL a truncate will *NOT* reset
       # the auto_increment

data/lib/etl/commands/etl.rb

@@ -34,12 +34,11 @@ if ARGV.length < 1
 else
   puts "Starting ETL process"
 
-  t = Benchmark.realtime do
-    ARGV.each do |f|
-      puts "Processing #{f}"
-      ETL::Engine.process(f)
-    end
+  ARGV.each do |f|
+    puts "Processing #{f}"
+    ETL::Engine.realtime_activity = true
+    ETL::Engine.process(f)
   end
-
-  puts "ETL process complete in #{sprintf('%.3f', t)} seconds"
+  
+  puts "ETL process complete"
 end

data/lib/etl/control/control.rb

@@ -1,40 +1,23 @@
 module ETL #:nodoc:
   module Control #:nodoc:
-    # Object representation of a control file
-    class Control
-      attr_reader :file
-      
+    # The Context is passed to eval. 
+    class Context
       class << self
-        # Parse a control file and return a Control instance
-        def parse(control_file)
-          control_file = control_file.path if control_file.instance_of?(File)
-          # logger.debug "Parsing control file #{control_file.path}"
-          control = ETL::Control::Control.new(control_file)
-          # TODO: better handling of parser errors. Return the line in the control file where the error occurs.
-          eval(IO.readlines(control_file).join("\n"), control.get_binding)
-          control.validate
-          control
-        end
-        
-        def resolve(control)
-          case control
-          when String
-            ETL::Control::Control.parse(File.new(control))
-          when File
-            ETL::Control::Control.parse(control)
-          when ETL::Control::Control
-            control
-          else
-            raise ControlError, "Control must be a String, File or Control object"
-          end
+        # Create a Context instance
+        def create(control)
+          Context.new(control).get_binding
         end
       end
       
-      def initialize(file)
-        @file = file
+      def initialize(control)
+        @control = control
+      end
+      
+      def file
+        @control.file
       end
       
-      # Define a source
+      # Define a source.
       def source(name, configuration={}, definition={})
         source_types = [:file, :db]
         source_types.each do |source_type|
@@ -47,7 +30,7 @@ module ETL #:nodoc:
       
       # Get the defined source
       def sources
-        @sources ||= []
+        @control.sources
       end
       
       # Define a destination
@@ -62,14 +45,19 @@ module ETL #:nodoc:
       
       # Get the defined destinations
       def destinations
-        @destinations ||= []
+        @control.destinations
       end
       
       def transform(name, transformer=nil, configuration={}, &block)
         transforms[name] ||= []
         if transformer
-          transform_class = ETL::Transform.const_get("#{transformer.to_s.classify}Transform")
-          transforms[name] << transform_class.new(self, configuration)
+          class_name = "#{transformer.to_s.classify}Transform"
+          begin
+            transform_class = ETL::Transform.const_get(class_name)
+            transforms[name] << transform_class.new(self, configuration)
+          rescue NameError => e
+            raise ControlError, "Unable to find transformer #{class_name}: #{e}"
+          end
         elsif block_given?
           transforms[name] << block
         else
@@ -77,32 +65,123 @@ module ETL #:nodoc:
         end
       end
       
-      def get_transform(name)
-        transforms[name] ||= []
+      def transforms
+        @control.transforms
       end
       
       def pre_process(name, configuration={})
-        processor_class = ETL::Processor.const_get("#{name.to_s.classify}Processor")
-        pre_processors << processor_class.new(self, configuration)
+        class_name = "#{name.to_s.classify}Processor"
+        begin
+          processor_class = ETL::Processor.const_get(class_name)
+          pre_processors << processor_class.new(self, configuration)
+        rescue NameError
+          raise ControlError, "Unable to find preprocessor #{class_name}"
+        end
       end
       
       def pre_processors
-        @pre_processors ||= []
+        @control.pre_processors
       end
       
       def post_process(name, configuration={})
-        processor_class = ETL::Processor.const_get("#{name.to_s.classify}Processor")
-        post_processors << processor_class.new(self, configuration)
+        class_name = "#{name.to_s.classify}Processor"
+        begin
+          processor_class = ETL::Processor.const_get(class_name)
+          post_processors << processor_class.new(self, configuration)
+        rescue NameError
+          raise ControlError, "Unable to find postprocessor #{class_name}"
+        end
       end
       
       def post_processors
-        @post_processors ||= []
+        @control.post_processors
       end
       
       def get_binding
         binding
       end
       
+      protected
+      # Get an array of supported source types
+      def source_types
+        [:file, :database]
+      end
+      
+      # Get an array of supported destination types
+      def destination_types
+        [:file, :database]
+      end
+      
+    end
+    
+    # Object representation of a control file
+    class Control
+      # The File object
+      attr_reader :file
+      
+      class << self
+        # Parse a control file and return a Control instance
+        def parse(control_file)
+          control_file = control_file.path if control_file.instance_of?(File)
+          # logger.debug "Parsing control file #{control_file.path}"
+          control = ETL::Control::Control.new(control_file)
+          # TODO: better handling of parser errors. Return the line in the control file where the error occurs.
+          eval(IO.readlines(control_file).join("\n"), Context.create(control), control_file)
+          control.validate
+          control
+        end
+        
+        # Resolve the given object to an ETL::Control::Control instance. Acceptable arguments
+        # are:
+        # * The path to a control file as a String
+        # * A File object referencing the control file
+        # * The ETL::Control::Control object (which will just be returned)
+        #
+        # Raises a ControlError if any other type is given
+        def resolve(control)
+          case control
+          when String
+            ETL::Control::Control.parse(File.new(control))
+          when File
+            ETL::Control::Control.parse(control)
+          when ETL::Control::Control
+            control
+          else
+            raise ControlError, "Control must be a String, File or Control object"
+          end
+        end
+      end
+      
+      # Initialize the instance with the given File object
+      def initialize(file)
+        @file = file
+      end
+      
+      # Get the defined source
+      def sources
+        @sources ||= []
+      end
+      
+      # Get the defined destinations
+      def destinations
+        @destinations ||= []
+      end
+      
+      # Get the transforms with the specified name
+      def transform(name)
+        transforms[name] ||= []
+      end
+      
+      # Get an Array of preprocessors
+      def pre_processors
+        @pre_processors ||= []
+      end
+      
+      # Get an Array of post processors
+      def post_processors
+        @post_processors ||= []
+      end
+      
       # Get a map of all transforms for this control
       def transforms
         @transforms ||= {}
@@ -118,17 +197,6 @@ module ETL #:nodoc:
         end
       end
       
-      protected
-      # Get an array of supported source types
-      def source_types
-        [:file, :database]
-      end
-      
-      # Get an array of supported destination types
-      def destination_types
-        [:file, :database]
-      end
-      
     end
   end
 end

data/lib/etl/control/destination.rb

@@ -1,15 +1,41 @@
 module ETL #:nodoc:
   module Control #:nodoc:
+    # Base class for destinations.
     class Destination
-      attr_reader :control, :configuration, :mapping
-      attr_accessor :buffer_size, :current_row, :unique
+      # Read-only accessor for the ETL::Control::Control instance
+      attr_reader :control
+      
+      # Read-only accessor for the configuration Hash
+      attr_reader :configuration
+      
+      # Read-only accessor for the destination mapping Hash
+      attr_reader :mapping
+      
+      # Accessor to the buffer size
+      attr_accessor :buffer_size
+      
+      # Unique flag.
+      attr_accessor :unique
       
       class << self
+        # Get the destination class for the specified name.
+        # 
+        # For example if name is :database or 'database' then the DatabaseDestination class 
+        # is returned
         def class_for_name(name)
           ETL::Control.const_get("#{name.to_s.classify}Destination")
         end
       end
       
+      # Initialize the destination
+      #
+      # Arguments:
+      # * <tt>control</tt>: The ETL::Control::Control instance
+      # * <tt>configuration</tt>: The configuration Hash
+      # * <tt>mapping</tt>: The mapping Hash
+      #
+      # Options:
+      # * <tt>:buffer_size</tt>: The output buffer size (default 1000 records)
       def initialize(control, configuration, mapping)
         @control = control
         @configuration = configuration
@@ -17,10 +43,12 @@ module ETL #:nodoc:
         @buffer_size = configuration[:buffer_size] ||= 1000
       end
       
+      # Get the current row number
       def current_row
         @current_row ||= 1
       end
       
+      # Write the given row
       def write(row)
         buffer << row
         flush if buffer.length >= buffer_size
@@ -36,11 +64,17 @@ module ETL #:nodoc:
         raise NotImplementedError, "close method must be implemented by subclasses"
       end
       
+      def errors
+        @errors ||= []
+      end
+      
       protected
+      # Access the buffer
       def buffer
         @buffer ||= []
       end
       
+      # Access the generators map
       def generators
         @generators ||= {}
       end
@@ -76,15 +110,22 @@ module ETL #:nodoc:
         @compound_key_constraints ||= {}
       end
       
-      # Add any virtual fields to the row
+      # Add any virtual fields to the row. Virtual rows will get their value from one of the following:
+      # * If the mapping is a Class, then an object which implements the next method
+      # * If the mapping is a Symbol, then the XGenerator where X is the classified symbol
+      # * If the mapping is a Proc, then it will be called with the row
+      # * Otherwise the value itself will be assigned to the field
       def add_virtuals!(row)
         if mapping[:virtual]
           mapping[:virtual].each do |key,value|
             # Engine.logger.debug "Mapping virtual #{key}/#{value} for row #{row}"
             case value
+            when Class
+              generator = generators[key] ||= value.new
+              row[key] = generator.next
             when Symbol
-              generators[key] ||= ETL::Generator::Generator.class_for_name(value).new
-              row[key] = generators[key].next
+              generator = generators[key] ||= ETL::Generator::Generator.class_for_name(value).new
+              row[key] = generator.next
             when Proc
               row[key] = value.call(row)
             else