activewarehouse-etl 0.5.2 → 0.6.0

data/CHANGELOG

@@ -18,21 +18,49 @@
 * Added support for :unique => [] option and virtual fields for the database destination
 
 0.5.0 - Feb 17, 2007
-* Changed require_gem to gem and added alias to allow for older versions of rubygems
+* Changed require_gem to gem and added alias to allow for older versions of rubygems.
 * Added support for Hash in the source configuration where :name => :parser_name defines the parser to use and
-  :options => {} defines options to pass to the parser
-* Added support for passing a custom Parser class in the source configuration
-* Removed the need to include Enumerable in each parser implementation
-* Added new date_to_string and string_to_date transformers
-* Implemented foreign_key_lookup transform including an ActiveRecordResolver
-* Added real time activity logging which is called when the etl bin script is invoked
-* Improved error handling
-* Default logger level is now WARN
+  :options => {} defines options to pass to the parser.
+* Added support for passing a custom Parser class in the source configuration.
+* Removed the need to include Enumerable in each parser implementation.
+* Added new date_to_string and string_to_date transformers.
+* Implemented foreign_key_lookup transform including an ActiveRecordResolver.
+* Added real time activity logging which is called when the etl bin script is invoked.
+* Improved error handling.
+* Default logger level is now WARN.
 
 0.5.1 - Feb 18, 2007
-* Fixed up truncate processor
-* Updated HOW_TO_RELEASE doc
+* Fixed up truncate processor.
+* Updated HOW_TO_RELEASE doc.
 
 0.5.2 - Feb 19, 2007
-* Added error threshold
-* Fixed problem with transform error handling
+* Added error threshold.
+* Fixed problem with transform error handling.
+
+0.6.0 - Mar 8, 2007
+* Fixed missing method problem in validate in Control class.
+* Removed control validation for now (source could be code in the control file).
+* Transform interface now defined as taking 3 arguments, the field name, field value and the row. This 
+  is not backwards compatible.
+* Added HierarchyLookupTransform.
+* Added DefaultTransform which will return a specified value if the initial value is blank.
+* Added row-level processing.
+* Added HierarchyExploderProcessor which takes a single hierarchy row and explodes it to multiple rows
+  as used in a hierarchy bridge.
+* Added ApacheCombinedLogParser which parses Apache Combined Log format, including parsing of the 
+  user agent string and the URI, returning a Hash.
+* Fixed bug in SAX parser so that attributes are now set when the start_element event is received.
+* Added an HttpTools module which provides some parsing methods (for user agent and URI).
+* Database source now uses its own class for establishing an ActiveRecord connection.
+* Log files are now timestamped.
+* Source files are now archived automatically during the extraction process
+* Added a :condition option to the destination configuration Hash that accepts a Proc with a single
+  argument passed to it (the row).
+* Added an :append_rows option to the destination configuration Hash that accepts either a Hash (to
+  append a single row) or an Array of Hashes (to append multiple rows).
+* Only print the read and written row counts if there is at least one source and one destination
+  respectively.
+* Added a depends_on directive that accepts a list of arguments of either strings or symbols. Each
+  symbol is converted to a string and .ctl is appended; strings are passed through directly. The
+  dependencies are executed in the order they are specified.
+* The default field separator in the bulk loader is now a comma (was a tab).

data/README

@@ -1,4 +1,4 @@
-Ruby ETL parser.
+Ruby ETL tool.
 
 == Features
 

data/Rakefile

@@ -56,10 +56,11 @@ spec = Gem::Specification.new do |s|
     ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
   EOF
 
-  s.add_dependency('rake', '>= 0.7.1')
-  s.add_dependency('activesupport',    '>= 1.3.1.5618')
-  s.add_dependency('activerecord',     '>= 1.14.4.5618')
-  s.add_dependency('fastercsv',        '>= 1.0.0')
+  s.add_dependency('rake',                '>= 0.7.1')
+  s.add_dependency('activesupport',       '>= 1.3.1')
+  s.add_dependency('activerecord',        '>= 1.14.4')
+  s.add_dependency('fastercsv',           '>= 1.0.0')
+  s.add_dependency('adapter_extensions',  '>= 0.1.0')
 
   s.rdoc_options << '--exclude' << '.'
   s.has_rdoc = false
@@ -122,4 +123,13 @@ end
 desc "Publish the API documentation"
 task :pdoc => [:rdoc] do 
   Rake::SshDirPublisher.new("aeden@rubyforge.org", "/var/www/gforge-projects/activewarehouse/etl/rdoc", "rdoc").upload
+end
+
+desc "Reinstall the gem from a local package copy"
+task :reinstall => [:package] do
+  windows = RUBY_PLATFORM =~ /mswin/
+  sudo = windows ? '' : 'sudo'
+  gem = windows ? 'gem.bat' : 'gem'
+  `#{sudo} #{gem} uninstall -x -i #{PKG_NAME}`
+  `#{sudo} #{gem} install pkg/#{PKG_NAME}-#{PKG_VERSION}`
 end

data/TODO

@@ -8,4 +8,20 @@ TODO
 ** Allow mismatch row length error in delimited parser to be ignored
 * Improve error messages throughout, but especially in problems with the control files
 * Add support for determining if a row should be added, updated or removed vs. just blindly inserting
-* Add support for paritioned views during the insert process. Use specifiable columns as the trigger columns for determining the data output destination.
+* Add support for paritioned views during the insert process. Use specifiable columns as the trigger columns for determining the data output destination.
+
+Audit Record
+  
+Process-Level
+  * Start Time
+  * End Time
+  * (Duration)
+  * Rows Read
+  * Rows Written
+  * Rows Rejected
+  * Errors
+  * Destination
+Record-Level
+  * Source
+  * Timestamp
+  * Transformation Log

data/bin/etl

@@ -23,4 +23,6 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-require File.dirname(__FILE__) + "/../lib/etl/commands/etl"
+$:.unshift(File.dirname(__FILE__) + '/../lib/')
+require 'etl'
+require 'etl/commands/etl'

data/lib/etl.rb

@@ -25,9 +25,7 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-unless defined?(Logger)
-  require 'logger'
-end
+require 'logger'
 
 require 'rubygems'
 
@@ -45,11 +43,19 @@ unless defined?(ActiveRecord)
   require 'active_record'
 end
 
-gem 'fastercsv'
-require 'faster_csv'
+unless defined?(AdapterExtensions)
+  gem 'adapter_extensions'
+  require 'adapter_extensions'
+end
+
+unless defined?(FasterCSV)
+  gem 'fastercsv'
+  require 'faster_csv'
+end
 
 $:.unshift(File.dirname(__FILE__))
 
+require 'etl/http_tools'
 require 'etl/version'
 require 'etl/engine'
 require 'etl/control'
@@ -58,8 +64,6 @@ require 'etl/transform'
 require 'etl/processor'
 require 'etl/generator'
 
-require 'etl/active_record_ext'
-
 module ETL #:nodoc:
   class ETLError < StandardError #:nodoc:
   end

data/lib/etl/commands/etl.rb

@@ -22,7 +22,6 @@
 #++
 
 require 'benchmark'
-require File.dirname(__FILE__) + '/../../etl'
 
 # Print a usage statement
 def usage #:nodoc:

data/lib/etl/control/control.rb

@@ -2,6 +2,8 @@ module ETL #:nodoc:
   module Control #:nodoc:
     # The Context is passed to eval. 
     class Context
+      attr_reader :control
+      
       class << self
         # Create a Context instance
         def create(control)
@@ -16,16 +18,29 @@ module ETL #:nodoc:
       
       # Get the control file
       def file
-        @control.file
+        control.file
       end
       
+      # Set the allowed error threshold
       def set_error_threshold(error_threshold)
-        @control.error_threshold = error_threshold
+        control.error_threshold = error_threshold
+      end
+      
+      # Define a list of control files that this file depends on. Those control files
+      # will be executed prior to this control file. The list may contain symbols that will
+      # be converted to file names by calling to_s + '.ctl', or they may be strings in which
+      # case they will be used as is
+      def depends_on(*args)
+        dependencies << args
+      end
+      
+      # Get the defined dependencies
+      def dependencies
+        control.dependencies
       end
       
       # Define a source.
       def source(name, configuration={}, definition={})
-        source_types = [:file, :db]
         source_types.each do |source_type|
           if configuration[source_type]
             source_class = ETL::Control::Source.class_for_name(source_type)
@@ -36,7 +51,7 @@ module ETL #:nodoc:
       
       # Get the defined source
       def sources
-        @control.sources
+        control.sources
       end
       
       # Define a destination
@@ -51,19 +66,30 @@ module ETL #:nodoc:
       
       # Get the defined destinations
       def destinations
-        @control.destinations
+        control.destinations
       end
       
       # Define a transform
       def transform(name, transformer=nil, configuration={}, &block)
         transforms[name] ||= []
         if transformer
-          class_name = "#{transformer.to_s.classify}Transform"
-          begin
-            transform_class = ETL::Transform.const_get(class_name)
-            transforms[name] << transform_class.new(self, configuration)
-          rescue NameError => e
-            raise ControlError, "Unable to find transformer #{class_name}: #{e}"
+          case transformer
+          when String, Symbol
+            class_name = "#{transformer.to_s.classify}Transform"
+            begin
+              transform_class = ETL::Transform.const_get(class_name)
+              transforms[name] << transform_class.new(self, configuration)
+            rescue NameError => e
+              raise ControlError, "Unable to find transformer #{class_name}: #{e}"
+            end
+          else
+            #transformer.class.inspect
+            if transformer.is_a?(ETL::Transform::Transform)
+              Engine.logger.debug "Adding transformer #{transformer.inspect} for field #{name}"
+              transforms[name]  << transformer
+            else
+              raise ControlError, "Transformer must be a String, Symbol or Transform instance"
+            end
           end
         elsif block_given?
           transforms[name] << block
@@ -74,39 +100,68 @@ module ETL #:nodoc:
       
       # Get the defined transforms
       def transforms
-        @control.transforms
+        control.transforms
+      end
+      
+      protected
+      # This method is used to define a processor and insert into the specified processor
+      # collection.
+      def define_processor(name, processor_collection, configuration)
+        case name
+        when String, Symbol
+          class_name = "#{name.to_s.classify}Processor"
+          begin
+            processor_class = ETL::Processor.const_get(class_name)
+            processor_collection << processor_class.new(self, configuration)
+          rescue NameError => e
+            raise ControlError, "Unable to find processor #{class_name}: #{e}"
+          end
+        when Class
+          processor_collection << name.new(self, configuration)
+        else
+          raise ControlError, "The process declaration requires a String, Symbol or Class"
+        end
+      end
+      
+      public
+      # Define an "after read" processor. This must be a row-level processor.
+      def after_read(name, configuration={})
+        define_processor(name, after_read_processors, configuration)
+      end
+      
+      # Get the defined "after read" processors
+      def after_read_processors
+        control.after_read_processors
+      end
+      
+      # Define a "before write" processor. This must be a row-level processor.
+      def before_write(name, configuration={})
+        define_processor(name, before_write_processors, configuration)
+      end
+      
+      # Get the defined "before write" processors
+      def before_write_processors
+        control.before_write_processors
       end
       
       # Define a pre-processor
       def pre_process(name, configuration={})
-        class_name = "#{name.to_s.classify}Processor"
-        begin
-          processor_class = ETL::Processor.const_get(class_name)
-          pre_processors << processor_class.new(self, configuration)
-        rescue NameError => e
-          raise ControlError, "Unable to find preprocessor #{class_name}: #{e}"
-        end
+        define_processor(name, pre_processors, configuration)
       end
       
       # Get the defined pre-processors
       def pre_processors
-        @control.pre_processors
+        control.pre_processors
       end
       
       # Define a post-processor
       def post_process(name, configuration={})
-        class_name = "#{name.to_s.classify}Processor"
-        begin
-          processor_class = ETL::Processor.const_get(class_name)
-          post_processors << processor_class.new(self, configuration)
-        rescue NameError
-          raise ControlError, "Unable to find postprocessor #{class_name}"
-        end
+        define_processor(name, post_processors, configuration)
       end
       
       # Get the defined post-processors
       def post_processors
-        @control.post_processors
+        control.post_processors
       end
       
       # Get the binding object
@@ -117,12 +172,12 @@ module ETL #:nodoc:
       protected
       # Get an array of supported source types
       def source_types
-        [:file, :database]
+        control.source_types
       end
       
       # Get an array of supported destination types
       def destination_types
-        [:file, :database]
+        control.destination_types
       end
       
     end
@@ -173,6 +228,11 @@ module ETL #:nodoc:
         @file = file
       end
       
+      # Get a list of dependencies
+      def dependencies
+        @dependencies ||= []
+      end
+      
       # Get the defined source
       def sources
         @sources ||= []
@@ -188,6 +248,15 @@ module ETL #:nodoc:
         transforms[name] ||= []
       end
       
+      def after_read_processors
+        @after_read_processors ||= []
+      end
+      
+      # Get all of the "before write" processors
+      def before_write_processors
+        @before_write_processors ||= []
+      end
+      
       # Get an Array of preprocessors
       def pre_processors
         @pre_processors ||= []
@@ -210,12 +279,20 @@ module ETL #:nodoc:
       
       # Validate the control file
       def validate
-        unless sources.length > 0
-          raise ControlError, "Configuration must include one of the following for the source: #{source_types.join(',')}"
-        end
-        unless destinations.length > 0
-          raise ControlError, "Configuration must include one of the following for the destination: #{destination_types.join(',')}"
-        end
+        #unless sources.length > 0
+        #  raise ControlError, "Configuration must include one of the following for the source: #{source_types.join(',')}"
+        #end
+        #unless destinations.length > 0
+        #  raise ControlError, "Configuration must include one of the following for the destination: #{destination_types.join(',')}"
+        #end
+      end
+      
+      def source_types
+        [:file, :database]
+      end
+      
+      def destination_types
+        [:file, :database]
       end
       
     end

data/lib/etl/control/destination.rb

@@ -17,6 +17,12 @@ module ETL #:nodoc:
       # Unique flag.
       attr_accessor :unique
       
+      # A condition for writing
+      attr_accessor :condition
+      
+      # An array of rows to append to the destination
+      attr_accessor :append_rows
+      
       class << self
         # Get the destination class for the specified name.
         # 
@@ -36,11 +42,15 @@ module ETL #:nodoc:
       #
       # Options:
       # * <tt>:buffer_size</tt>: The output buffer size (default 1000 records)
+      # * <tt>:condition</tt>: A conditional proc that must return true for the row to be written
+      # * <tt>:append_rows</tt>: An array of rows to append
       def initialize(control, configuration, mapping)
         @control = control
         @configuration = configuration
         @mapping = mapping
         @buffer_size = configuration[:buffer_size] ||= 1000
+        @condition = configuration[:condition]
+        @append_rows = configuration[:append_rows]
       end
       
       # Get the current row number
@@ -50,7 +60,9 @@ module ETL #:nodoc:
       
       # Write the given row
       def write(row)
-        buffer << row
+        if @condition.nil? || @condition.call(row)
+          buffer << row
+        end
         flush if buffer.length >= buffer_size
       end