activewarehouse-etl 0.3.0 → 0.4.0

data/CHANGELOG

@@ -8,5 +8,11 @@
  * Added support for generators in destinations
  * Added a SurrogateKeyGenerator for cases where the database doesn't support auto generated surrogate keys
  
-0.3.0
- * Added support for calculated values in virtual fields with Proc
+0.3.0 - Dec 19, 2006
+ * Added support for calculated values in virtual fields with Proc
+ 
+0.4.0 - Jan 11, 2006
+ * Added :skip_lines option to file source configurations, which can be used to skip the first n lines in the source data file
+ * Added better error handling in delimited parser - an error is now raised if the expected and actual field lengths do not match
+ * Added :truncate option for database destination. Set to true to truncate before importing data.
+ * Added support for :unique => [] option and virtual fields for the database destination

data/Rakefile

@@ -41,6 +41,7 @@ end
 PKG_FILES = FileList[
   'CHANGELOG',
   'README',
+  'TODO',
   'Rakefile',
   'bin/**/*',
   'doc/**/*',
@@ -116,4 +117,9 @@ task :release => [ :package ] do
     puts release_command
     system(release_command)
   end
+end
+
+desc "Publish the API documentation"
+task :pdoc => [:rdoc] do 
+  Rake::SshDirPublisher.new("aeden@rubyforge.org", "/var/www/gforge-projects/activewarehouse/etl/rdoc", "rdoc").upload
 end

data/TODO

@@ -0,0 +1,9 @@
+TODO
+
+* Add build-in support for audit_dimension
+* Do not rerun the processing if it isn't needed, i.e. the source and control files have not been modified (allow forced override)
+* Add built-in FK lookup
+* Provide greater control in error handling
+** Allow a error threshold
+** Don't die completely if a parse error, just stop processing that specific file if error threshold is reached
+** Allow mismatch row length error in delimited parser to be ignored

data/lib/etl.rb

@@ -1,5 +1,9 @@
+# This source file requires all of the necessary gems and source files for ActiveWarehouse ETL. If you
+# load this source file all of the other required files and gems will also be brought into the 
+# runtime.
+
 #--
-# Copyright (c) 2006 Anthony Eden
+# Copyright (c) 2006-2007 Anthony Eden
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -49,6 +53,8 @@ require 'etl/transform'
 require 'etl/processor'
 require 'etl/generator'
 
+require 'etl/active_record_ext'
+
 module ETL #:nodoc:
   class ETLError < StandardError #:nodoc:
   end
@@ -56,4 +62,6 @@ module ETL #:nodoc:
   end
   class DefinitionError < ControlError #:nodoc:
   end
+  class MismatchError < ETLError #:nodoc:
+  end
 end

data/lib/etl/active_record_ext.rb

@@ -0,0 +1 @@
+require 'etl/active_record_ext/connection_adapters/mysql_adapter'

data/lib/etl/active_record_ext/connection_adapters/mysql_adapter.rb

@@ -0,0 +1,34 @@
+require 'active_record/connection_adapters/abstract_adapter'
+
+module ActiveRecord
+  module ConnectionAdapters
+    class MysqlAdapter < AbstractAdapter
+      # Execute a truncate statement on the table. Note that in MySQL a truncate will *NOT* reset
+      # the auto_increment
+      def truncate(table_name)
+        execute("TRUNCATE #{table_name}")
+      end
+      
+      # Bulk load the data in the specified file. This implementation always uses the LOCAL keyword
+      # so the file must be found locally, not on the remote server, to be loaded.
+      #
+      # Options:
+      # * <tt>:ignore</tt> -- Ignore the specified number of lines from the source file
+      # * <tt>:columns</tt> -- Array of column names defining the source file column order
+      # * <tt>:fields</tt> -- Hash of options for fields:
+      # ** <tt>:delimited_by</tt> -- The field delimiter
+      # ** <tt>:enclosed_by</tt> -- The field enclosure
+      def bulk_load(file, table_name, options={})
+        q = "LOAD DATA LOCAL INFILE '#{file}' INTO TABLE #{table_name}"
+        if options[:fields]
+          q << " FIELDS"
+          q << " TERMINATED BY '#{options[:fields][:delimited_by]}'" if options[:fields][:delimited_by]
+          q << " ENCLOSED BY '#{options[:fields][:enclosed_by]}'" if options[:fields][:enclosed_by]
+        end
+        q << " IGNORE #{options[:ignore]} LINES" if options[:ignore]
+        q << " (#{options[:columns].join(',')})" if options[:columns]
+        execute(q)
+      end
+    end
+  end
+end

data/lib/etl/control/destination.rb

@@ -2,7 +2,7 @@ module ETL #:nodoc:
   module Control #:nodoc:
     class Destination
       attr_reader :control, :configuration, :mapping
-      attr_accessor :buffer_size, :current_row
+      attr_accessor :buffer_size, :current_row, :unique
       
       class << self
         def class_for_name(name)
@@ -58,6 +58,41 @@ module ETL #:nodoc:
         end
         order
       end
+      
+      # Return true if the row is allowed. The row will not be allowed if the :unique option is specified
+      # in the configuration and the compound key already exists
+      def row_allowed?(row)
+        if unique
+          key = (unique.collect { |k| row[k] }).join('|')
+          return false if compound_key_constraints[key]
+          compound_key_constraints[key] = 1
+        end
+        return true
+      end
+      
+      # Get a hash of compound key contraints. This is used to determine if a row can be written when the
+      # unique option is specified
+      def compound_key_constraints
+        @compound_key_constraints ||= {}
+      end
+      
+      # Add any virtual fields to the row
+      def add_virtuals!(row)
+        if mapping[:virtual]
+          mapping[:virtual].each do |key,value|
+            # Engine.logger.debug "Mapping virtual #{key}/#{value} for row #{row}"
+            case value
+            when Symbol
+              generators[key] ||= ETL::Generator::Generator.class_for_name(value).new
+              row[key] = generators[key].next
+            when Proc
+              row[key] = value.call(row)
+            else
+              row[key] = value
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/etl/control/destination/database_destination.rb

@@ -1,19 +1,29 @@
 module ETL
   module Control
     class DatabaseDestination < Destination
-      attr_reader :order
+      attr_reader :order, :truncate
       def initialize(control, configuration, mapping)
         super
+        @truncate = configuration[:truncate] ||= false
+        @unique = configuration[:unique]
         @order = mapping[:order] || order_from_source
         raise ControlError, "Order required in mapping" unless @order
         connect
       end
       
       def flush
-        # TODO: add virtual fields and compound key constraint support like in the FileDestination
         conn = ActiveRecord::Base.connection
         conn.transaction do
+          conn.truncate(configuration[:table]) if truncate
+          
           buffer.each do |row|
+            # check to see if this row's compound key constraint already exists
+            # note that the compound key constraint may not utilize virtual fields
+            next unless row_allowed?(row)
+
+            # add any virtual fields
+            add_virtuals!(row)
+            
             names = []
             values = []
             order.each do |name|
@@ -28,6 +38,8 @@ module ETL
           buffer.clear
         end
       end
+      
+      # Close the connection
       def close
         flush
         ActiveRecord::Base.connection.disconnect!

data/lib/etl/control/destination/file_destination.rb

@@ -3,7 +3,7 @@ module ETL #:nodoc:
     # File as the final destination.
     class FileDestination < Destination
       attr_reader :file, :order
-      attr_accessor :append, :separator, :eol, :enclose, :unique
+      attr_accessor :append, :separator, :eol, :enclose
       
       # Initialize the object.
       # * <tt>control</tt>: The Control object
@@ -57,20 +57,6 @@ module ETL #:nodoc:
       end
       
       private
-      # Return true if the row is allowed. The row will not be allowed if the :unique option is specified
-      # in the configuration and the compound key already exists
-      def row_allowed?(row)
-        if unique
-          key = (unique.collect { |k| row[k] }).join('|')
-          return false if compound_key_constraints[key]
-          compound_key_constraints[key] = 1
-        end
-        return true
-      end
-      
-      def compound_key_constraints
-        @compound_key_constraints ||= {}
-      end
       
       # Get the open file stream
       def f
@@ -82,22 +68,6 @@ module ETL #:nodoc:
         append ? 'a' : 'w'
       end
       
-      def add_virtuals!(row)
-        if mapping[:virtual]
-          mapping[:virtual].each do |key,value|
-            # Engine.logger.debug "Mapping virtual #{key}/#{value} for row #{row}"
-            case value
-            when Symbol
-              generators[key] ||= ETL::Generator::Generator.class_for_name(value).new
-              row[key] = generators[key].next
-            when Proc
-              row[key] = value.call(row)
-            else
-              row[key] = value
-            end
-          end
-        end
-      end
     end
   end
 end

data/lib/etl/control/source/file_source.rb

@@ -1,6 +1,7 @@
 module ETL
   module Control
     class FileSource < Source
+      attr_accessor :skip_lines
       def initialize(control, configuration, definition)
         super
         configure
@@ -13,6 +14,7 @@ module ETL
       private
       def configure
         @parser = ETL::Parser::Parser.class_for_name(@configuration[:parser]).new(self)
+        @skip_lines = @configuration[:skip_lines] ||= 0
       end
     end
   end

data/lib/etl/parser/delimited_parser.rb

@@ -10,12 +10,22 @@ module ETL #:nodoc:
         configure
       end
       
-      # Returns each row
+      # Returns each row.
       def each
         options = {}
         Dir.glob(file).each do |file|
+          ETL::Engine.logger.debug "parsing #{file}"
+          line = 0
+          lines_skipped = 0
           FasterCSV.foreach(file, options) do |raw_row|
+            if lines_skipped < source.skip_lines
+              ETL::Engine.logger.debug "skipping line"
+              lines_skipped += 1
+              next
+            end
+            line += 1
             row = {}
+            validate_row(raw_row, line, file)
             raw_row.each_with_index do |record, index|
               f = fields[index]
               row[f.name] = convert(f.name, record, f.type)
@@ -31,6 +41,16 @@ module ETL #:nodoc:
       end
       
       private
+      def validate_row(row, line, file)
+        ETL::Engine.logger.debug "validing line #{line} in file #{file}"
+        if row.length != fields.length
+          raise_with_info( MismatchError, 
+            "The number of rows from the source (#{row.length}) does not match the number of rows in the definition (#{fields.length})", 
+            line, file
+          )
+        end
+      end
+      
       def configure
         source.definition.each do |options|
           case options

data/lib/etl/parser/fixed_width_parser.rb

@@ -16,7 +16,12 @@ module ETL #:nodoc:
         Dir.glob(file).each do |file|
           open(file).each do |line|
             row = {}
+            lines_skipped = 0
             fields.each do |name, f|
+              if lines_skipped < source.skip_lines
+                lines_skipped += 1
+                next
+              end
               # TODO make strip optional?
               row[name] = convert(name, line[f.field_start, f.field_length].strip, f.type)
             end

data/lib/etl/parser/parser.rb

@@ -38,6 +38,10 @@ module ETL
       def file
         File.join(File.dirname(source.control.file), source.configuration[:file])
       end
+      
+      def raise_with_info(error, message, file, line)
+        raise error, "#{message} (line #{line} in #{file})"
+      end
     end
   end
 end

data/lib/etl/processor/bulk_import_processor.rb

@@ -1,5 +1,6 @@
 module ETL
   module Processor
+    # Processor which is used to bulk import data into a target database
     class BulkImportProcessor < ETL::Processor::Processor
       attr_reader :file, :target, :truncate, :columns
       def initialize(control, configuration)
@@ -16,21 +17,14 @@ module ETL
         conn.transaction do
           # TODO: Support all database types
           # Since LOCAL is used this must be allowed by both the client and server
-          conn.execute("TRUNCATE #{target[:table]}") if truncate
-          q = "LOAD DATA LOCAL INFILE '#{file}' INTO TABLE #{target[:table]}"
-          q << " (#{columns.join(',')})" if columns
-          conn.execute(q)
+          conn.truncate(target[:table]) if truncate
+          options = {}
+          options[:columns] = columns
+          conn.bulk_load(file, target[:table], options)
         end
       end
-      private
-      def log
-        unless @log
-          @log = Logger.new(STDOUT)
-          @log.level = Logger::DEBUG
-        end
-        @log
-      end
       
+      private
       # Connect to the database
       def connect
         ActiveRecord::Base.establish_connection(

data/lib/etl/processor/processor.rb

@@ -13,6 +13,13 @@ module ETL #:nodoc:
       def configuration
         @configuration
       end
+      def log
+        unless @log
+          @log = Logger.new(STDOUT)
+          @log.level = Logger::DEBUG
+        end
+        @log
+      end
     end
   end
 end

data/lib/etl/processor/truncate_processor.rb

@@ -0,0 +1,30 @@
+module ETL
+  module Processor
+    # A processor which will truncate a table. Use as a pre-processor for cleaning out a table
+    # prior to loading
+    class TruncateProcessor < ETL::Processor::Processor
+      attr_reader :file, :table
+      def initialize(control, configuration)
+        super
+        @file = File.join(File.dirname(control.file), configuration[:file])
+        @table = configuration[:table]
+        connect
+      end
+      def process
+        conn = ActiveRecord::Base.connection
+        conn.truncate
+      end
+      
+      # Connect to the database
+      def connect
+        ActiveRecord::Base.establish_connection(
+            :adapter  => (target[:adapter] || :mysql),
+            :username => (target[:username] || 'root'),
+            :host     => (target[:host] || 'localhost'),
+            :password => target[:password],
+            :database => target[:database] 
+        )
+      end
+    end
+  end
+end

data/lib/etl/transform/sha1_transform.rb

@@ -7,6 +7,7 @@ module ETL #:nodoc:
       def initialize(control, configuration={})
         super
       end
+      # Transform the value with a SHA1 digest algorithm.
       def transform(value)
         Digest::SHA1.hexdigest(value)
       end

data/lib/etl/transform/transform.rb

@@ -2,6 +2,9 @@ module ETL
   module Transform
     class Transform
       class << self
+        # Transform the specified value using the given transforms. The transforms can either be
+        # Proc objects or objects which extend from Transform and implement the method <tt>transform(value)</tt>.
+        # Any other object will result in a ControlError being raised.
         def transform(name, value, transforms)
           # logger.debug "Transforming field #{name}" if transforms.length > 0
           transforms.each do |transform|
@@ -20,6 +23,7 @@ module ETL
       
       attr_reader :control, :configuration
       
+      # Initialize the transform object
       def initialize(control, configuration={})
         @control = control
         @configuration = configuration

data/lib/etl/version.rb

@@ -1,7 +1,7 @@
 module ETL
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 3
+    MINOR = 4
     TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
-date: 2006-12-08 00:00:00 -05:00
+  version: 0.4.0
+date: 2007-01-11 00:00:00 -05:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib
@@ -31,10 +31,13 @@ authors:
 files: 
 - CHANGELOG
 - README
+- TODO
 - Rakefile
 - bin/etl
 - lib/etl
 - lib/etl.rb
+- lib/etl/active_record_ext
+- lib/etl/active_record_ext.rb
 - lib/etl/commands
 - lib/etl/control
 - lib/etl/control.rb
@@ -48,6 +51,8 @@ files:
 - lib/etl/transform
 - lib/etl/transform.rb
 - lib/etl/version.rb
+- lib/etl/active_record_ext/connection_adapters
+- lib/etl/active_record_ext/connection_adapters/mysql_adapter.rb
 - lib/etl/commands/etl.rb
 - lib/etl/control/control.rb
 - lib/etl/control/destination
@@ -66,6 +71,7 @@ files:
 - lib/etl/parser/xml_parser.rb
 - lib/etl/processor/bulk_import_processor.rb
 - lib/etl/processor/processor.rb
+- lib/etl/processor/truncate_processor.rb
 - lib/etl/transform/decode_transform.rb
 - lib/etl/transform/sha1_transform.rb
 - lib/etl/transform/transform.rb