activewarehouse-etl 0.6.1 → 0.7.0

data/lib/etl/processor/rename_processor.rb

@@ -0,0 +1,24 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to rename a field in the row.
+    #
+    # Configuration options:
+    # * <tt>:source</tt>: the source field name
+    # * <tt>:dest</tt>: The destination field name
+    class RenameProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        source_value = row[configuration[:source]]
+        case source_value
+        when Numeric
+          row[configuration[:dest]] = source_value
+        when nil
+          row[configuration[:dest]] = nil
+        else
+          row[configuration[:dest]] = source_value.dup
+        end
+        row.delete(configuration[:source])
+        row
+      end
+    end
+  end
+end

data/lib/etl/processor/row_processor.rb

@@ -8,7 +8,7 @@ module ETL #:nodoc:
       def initialize(control, configuration)
         super
       end
-      # Process the specified row
+      # Process the specified row. This method must return the row.
       def process(row)
         raise "process_row is an abstract method"
       end

data/lib/etl/processor/sequence_processor.rb

@@ -0,0 +1,23 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Row level processor to generate a sequence.
+    #
+    # Configuration options:
+    # * <tt>:context</tt>: A context name, if none is specified then the context will be
+    #   the current ETL run
+    # * <tt>:dest</tt>: The destination field name
+    class SequenceProcessor < ETL::Processor::RowProcessor
+      def process(row)
+        sequences[configuration[:context]] ||= 0
+        row[configuration[:dest]] = sequences[configuration[:context]] += 1
+        row
+      end
+      
+      protected
+      # Get a Hash of sequences
+      def sequences
+        @sequences ||= {}
+      end
+    end
+  end
+end

data/lib/etl/processor/surrogate_key_processor.rb

@@ -0,0 +1,31 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # A row level processor that provides surrogate keys
+    class SurrogateKeyProcessor < ETL::Processor::RowProcessor
+      attr_accessor :query
+      attr_accessor :destination
+      
+      # Initialize the surrogate key generator
+      def initialize(control, configuration)
+        super
+        @surrogate_key = ActiveRecord::Base.connection.select_value(configuration[:query])
+        #puts "initial surrogate key: #{@surrogate_key}"
+        @surrogate_key = 0 if @surrogate_key.blank?
+        @surrogate_key = @surrogate_key.to_i
+        #puts "surrogate key: #{@surrogate_key}"
+        @destination = configuration[:destination] || :id
+      end
+      
+      # Add a surrogate key to the row
+      def process(row)
+        if row
+          #puts "processing row #{row.inspect}"
+          @surrogate_key += 1
+          #puts "adding surrogate key to row: #{@surrogate_key}"
+          row[destination] = @surrogate_key
+          row
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor/truncate_processor.rb

@@ -3,8 +3,6 @@ module ETL #:nodoc:
     # A processor which will truncate a table. Use as a pre-processor for cleaning out a table
     # prior to loading
     class TruncateProcessor < ETL::Processor::Processor
-      #attr_reader :file
-      
       # Defines the table to truncate
       attr_reader :table
       

data/lib/etl/row.rb

@@ -0,0 +1,17 @@
+# This source file contains the ETL::Row class.
+
+module ETL #:nodoc:
+  # This class represents a single row currently passing through the ETL pipeline
+  class Row < Hash
+    # All change types
+    CHANGE_TYPES = [:insert, :update, :delete]
+    
+    # Accessor for the row's change type
+    attr_accessor :change_type
+    
+    # Get the change type, defaults to :insert
+    def change_type
+      @change_type ||= :insert
+    end
+  end
+end

data/lib/etl/screen/row_count_screen.rb

@@ -0,0 +1,15 @@
+module ETL
+  module Screen
+    # This screen validates the number of rows which will be bulk loaded
+    # against the results from some sort of a row count query. If there 
+    # is a difference then the screen will not pass
+    class RowCountScreen
+      def initialize(control, configuration={})
+        
+      end
+      def execute
+        
+      end
+    end
+  end
+end

data/lib/etl/transform/block_transform.rb

@@ -0,0 +1,13 @@
+module ETL
+  module Transform
+    class BlockTransform < ETL::Transform::Transform
+      def initialize(control, name, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def transform(name, value, row)
+        @block.call(name, value, row)
+      end
+    end
+  end
+end

data/lib/etl/transform/date_to_string_transform.rb

@@ -6,7 +6,7 @@ module ETL #:nodoc:
       #
       # Configuration options:
       # * <tt>:format</tt>: A format passed to strftime. Defaults to %Y-%m-%d
-      def initialize(control, configuration={})
+      def initialize(control, name, configuration={})
         super
         @format = configuration[:format] || "%Y-%m-%d"
       end

data/lib/etl/transform/decode_transform.rb

@@ -14,7 +14,7 @@ module ETL #:nodoc:
       # * <tt>:decode_table_path</tt>: The path to the decode table (defaults to 'decode.txt')
       # * <tt>:decode_table_delimiter</tt>: The decode table delimiter (defaults to ':')
       # * <tt>:default_value</tt>: The default value to use (defaults to 'No Value')
-      def initialize(control, configuration={})
+      def initialize(control, name, configuration={})
         super
         
         if configuration[:decode_table_path]

data/lib/etl/transform/default_transform.rb

@@ -3,10 +3,15 @@ module ETL #:nodoc:
     # Transform which will replace nil or empty values with a specified value.
     class DefaultTransform < Transform
       attr_accessor :default_value
-      def initialize(control, configuration)
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:default_value</tt>: The default value to use if the incoming value is blank
+      def initialize(control, name, configuration)
         super
         @default_value = configuration[:default_value]
       end
+      # Transform the value
       def transform(name, value, row)
         value.blank? ? default_value : value
       end

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -12,7 +12,7 @@ module ETL #:nodoc:
       #  an empty Hash will be used. This Hash will be used to cache values that have been resolved already
       #  for future use.
       # *<tt>:resolver</tt>: Object or Class which implements the method resolve(value)
-      def initialize(control, configuration={})
+      def initialize(control, name, configuration={})
         super
         
         @collection = (configuration[:collection] || {})

data/lib/etl/transform/hierarchy_lookup_transform.rb

@@ -20,7 +20,7 @@ module ETL #:nodoc:
       # * <tt>:table</tt>: The table to search (required)
       # * <tt>:connection</tt>: The ActiveRecord adapter (required)
       # * <tt>:parent_id_field</tt>: The name of the field to use for the parent ID (defaults to :parent_id)
-      def initialize(control, configuration={})
+      def initialize(control, name, configuration={})
         super
         @parent_id_field = configuration[:parent_id_field] || :parent_id
       end

data/lib/etl/transform/ordinalize_transform.rb

@@ -0,0 +1,12 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a number to an ordinalized version using the ActiveSupport ordinalize
+    # core extension
+    class OrdinalizeTransform < ETL::Transform::Transform
+      # Transform the value from a number to an ordinalized number
+      def transform(name, value, row)
+        value.ordinalize
+      end
+    end
+  end
+end

data/lib/etl/transform/sha1_transform.rb

@@ -4,9 +4,6 @@ module ETL #:nodoc:
   module Transform #:nodoc:
     # Transform which hashes the original value with a SHA-1 hash algorithm
     class Sha1Transform < ETL::Transform::Transform
-      def initialize(control, configuration={})
-        super
-      end
       # Transform the value with a SHA1 digest algorithm.
       def transform(name, value, row)
         Digest::SHA1.hexdigest(value)

data/lib/etl/transform/string_to_date_transform.rb

@@ -2,9 +2,6 @@ module ETL #:nodoc:
   module Transform #:nodoc:
     # Transform a String representation of a date to a Date instance
     class StringToDateTransform < ETL::Transform::Transform
-      def initialize(control, configuration={})
-        super
-      end
       # Transform the value using Date.parse
       def transform(name, value, row)
         Date.parse(value)

data/lib/etl/transform/string_to_datetime_transform.rb

@@ -2,9 +2,6 @@ module ETL #:nodoc:
   module Transform #:nodoc:
     # Transform a String representation of a date to a DateTime instance
     class StringToDateTimeTransform < ETL::Transform::Transform
-      def initialize(control, configuration={})
-        super
-      end
       # Transform the value using DateTime.parse.
       #
       # WARNING: This transform is slow (due to the Ruby implementation), but if you need to 

data/lib/etl/transform/string_to_time_transform.rb

@@ -2,9 +2,6 @@ module ETL #:nodoc:
   module Transform #:nodoc:
     # Transform a String representation of a date to a Time instance
     class StringToTimeTransform < ETL::Transform::Transform
-      def initialize(control, configuration={})
-        super
-      end
       # Transform the value using Time.parse
       def transform(name, value, row)
         Time.parse(value)

data/lib/etl/transform/transform.rb

@@ -22,25 +22,34 @@ module ETL#:nodoc:
         # Any other object will result in a ControlError being raised.
         def transform(name, value, row, transforms)
           transforms.each do |transform|
-            Engine.logger.debug "Transforming field #{name} with #{transform.inspect}"
-            case transform
-            when Proc
-              value = transform.call([name, value, row])
-            when Transform
-              value = transform.transform(name, value, row)
-            else
-              raise ControlError, "Unsupported transform configuration type: #{transform}"
+            benchmarks[transform.class] ||= 0
+            benchmarks[transform.class] += Benchmark.realtime do
+              Engine.logger.debug "Transforming field #{name} with #{transform.inspect}"
+              case transform
+              when Proc
+                value = transform.call([name, value, row])
+              when Transform
+                value = transform.transform(name, value, row)
+              else
+                raise ControlError, "Unsupported transform configuration type: #{transform}"
+              end
             end
           end
           value
         end
+        
+        def benchmarks
+          @benchmarks ||= {}
+        end
       end
       
-      attr_reader :control, :configuration
+      attr_reader :control, :name, :configuration
       
-      # Initialize the transform object
-      def initialize(control, configuration={})
+      # Initialize the transform object with the given control object, field name and 
+      # configuration hash
+      def initialize(control, name, configuration={})
         @control = control
+        @name = name
         @configuration = configuration
       end
       

data/lib/etl/transform/trim_transform.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform to trim string
+    class TrimTransform < ETL::Transform::Transform
+      # Configuration options:
+      # * <tt>:type</tt>: :left, :right or :both. Default is :both
+      def initialize(control, name, configuration={})
+        super
+        @type = (configuration[:type] || :both).to_sym
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :left
+          value.lstrip
+        when :right
+          value.rstrip
+        when :both
+          value.strip
+        else
+          raise "Trim type, if specified, must be :left, :right or :both"
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/type_transform.rb

@@ -2,7 +2,15 @@ module ETL #:nodoc:
   module Transform #:nodoc:
     # Transform from one type to another
     class TypeTransform < ETL::Transform::Transform
-      def initialize(control, configuration={})
+      # Initialize the transformer.
+      # 
+      # Configuration options:
+      # * <tt>:type</tt>: The type to convert to. Supported types:
+      # ** :string
+      # ** :number,:integer
+      # ** :float
+      # ** :decimal
+      def initialize(control, name, configuration={})
         super
         @type = configuration[:type]
         @significant = configuration[:significant] ||= 0

data/lib/etl/version.rb

@@ -1,8 +1,8 @@
 module ETL#:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 6
-    TINY  = 1
+    MINOR = 7
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0.10
 specification_version: 1
 name: activewarehouse-etl
 version: !ruby/object:Gem::Version 
-  version: 0.6.1
-date: 2007-03-22 00:00:00 -04:00
+  version: 0.7.0
+date: 2007-04-08 00:00:00 -04:00
 summary: Pure Ruby ETL package.
 require_paths: 
 - lib
@@ -30,6 +30,7 @@ authors:
 - Anthony Eden
 files: 
 - CHANGELOG
+- LICENSE
 - README
 - TODO
 - Rakefile
@@ -40,6 +41,8 @@ files:
 - lib/etl/control
 - lib/etl/control.rb
 - lib/etl/engine.rb
+- lib/etl/execution
+- lib/etl/execution.rb
 - lib/etl/generator
 - lib/etl/generator.rb
 - lib/etl/http_tools.rb
@@ -47,6 +50,8 @@ files:
 - lib/etl/parser.rb
 - lib/etl/processor
 - lib/etl/processor.rb
+- lib/etl/row.rb
+- lib/etl/screen
 - lib/etl/transform
 - lib/etl/transform.rb
 - lib/etl/version.rb
@@ -60,6 +65,10 @@ files:
 - lib/etl/control/destination/file_destination.rb
 - lib/etl/control/source/database_source.rb
 - lib/etl/control/source/file_source.rb
+- lib/etl/execution/base.rb
+- lib/etl/execution/job.rb
+- lib/etl/execution/migration.rb
+- lib/etl/execution/record.rb
 - lib/etl/generator/generator.rb
 - lib/etl/generator/surrogate_key_generator.rb
 - lib/etl/parser/apache_combined_log_parser.rb
@@ -69,21 +78,30 @@ files:
 - lib/etl/parser/sax_parser.rb
 - lib/etl/parser/xml_parser.rb
 - lib/etl/processor/bulk_import_processor.rb
+- lib/etl/processor/check_exist_processor.rb
+- lib/etl/processor/check_unique_processor.rb
 - lib/etl/processor/copy_field_processor.rb
 - lib/etl/processor/hierarchy_exploder_processor.rb
 - lib/etl/processor/processor.rb
+- lib/etl/processor/rename_processor.rb
 - lib/etl/processor/row_processor.rb
+- lib/etl/processor/sequence_processor.rb
+- lib/etl/processor/surrogate_key_processor.rb
 - lib/etl/processor/truncate_processor.rb
+- lib/etl/screen/row_count_screen.rb
+- lib/etl/transform/block_transform.rb
 - lib/etl/transform/date_to_string_transform.rb
 - lib/etl/transform/decode_transform.rb
 - lib/etl/transform/default_transform.rb
 - lib/etl/transform/foreign_key_lookup_transform.rb
 - lib/etl/transform/hierarchy_lookup_transform.rb
+- lib/etl/transform/ordinalize_transform.rb
 - lib/etl/transform/sha1_transform.rb
 - lib/etl/transform/string_to_date_transform.rb
 - lib/etl/transform/string_to_datetime_transform.rb
 - lib/etl/transform/string_to_time_transform.rb
 - lib/etl/transform/transform.rb
+- lib/etl/transform/trim_transform.rb
 - lib/etl/transform/type_transform.rb
 test_files: []
 
@@ -133,7 +151,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.0.0
+        version: 1.2.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: adapter_extensions