activewarehouse-etl-sgonyea 0.9.6

data/lib/etl/transform/ordinalize_transform.rb

@@ -0,0 +1,14 @@
+require 'active_support/core_ext/integer/inflections.rb'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a number to an ordinalized version using the ActiveSupport ordinalize
+    # core extension
+    class OrdinalizeTransform < ETL::Transform::Transform
+      # Transform the value from a number to an ordinalized number
+      def transform(name, value, row)
+        value.ordinalize
+      end
+    end
+  end
+end

data/lib/etl/transform/sha1_transform.rb

@@ -0,0 +1,13 @@
+require 'digest/sha1'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a SHA-1 hash algorithm
+    class Sha1Transform < ETL::Transform::Transform
+      # Transform the value with a SHA1 digest algorithm.
+      def transform(name, value, row)
+        Digest::SHA1.hexdigest(value)
+      end
+    end
+  end
+end

data/lib/etl/transform/split_fields_transform.rb

@@ -0,0 +1,27 @@
+module ETL
+  module Transform
+    class SplitFieldsTransform < ETL::Transform::Transform
+      attr_reader :delimiter
+      attr_reader :new_fields
+
+      def initialize(control, name, configuration)
+        @delimiter = configuration[:delimiter] || ','
+        @new_fields = configuration[:new_fields]
+        super
+      end
+      
+      def transform(name, value, row)
+        return nil if row.nil?
+        return nil if row[name].nil?
+
+        fields = row[name].split(@delimiter)
+        @new_fields.each_with_index do |new, index|
+          row[new] = fields[index]
+        end
+
+        row[name]
+      end
+
+    end
+  end
+end

data/lib/etl/transform/string_to_date_time_transform.rb

@@ -0,0 +1,14 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a DateTime instance
+    class StringToDateTimeTransform < ETL::Transform::Transform
+      # Transform the value using DateTime.parse.
+      #
+      # WARNING: This transform is slow (due to the Ruby implementation), but if you need to 
+      # parse timestamps before or after the values supported by the Time.parse.
+      def transform(name, value, row)
+        DateTime.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_date_transform.rb

@@ -0,0 +1,16 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Date instance
+    class StringToDateTransform < ETL::Transform::Transform
+      # Transform the value using Date.parse
+      def transform(name, value, row)
+        return value if value.nil?
+        begin
+          Date.parse(value)
+        rescue => e
+          return value
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_time_transform.rb

@@ -0,0 +1,11 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Time instance
+    class StringToTimeTransform < ETL::Transform::Transform
+      # Transform the value using Time.parse
+      def transform(name, value, row)
+        Time.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/lib/etl/transform/transform.rb

@@ -0,0 +1,61 @@
+module ETL#:nodoc:
+  module Transform#:nodoc:
+    # Base class for transforms.
+    #
+    # A transform converts one value to another value using some sort of algorithm.
+    #
+    # A simple transform has two arguments, the field to transform and the name of the transform:
+    #
+    #   transform :ssn, :sha1
+    # 
+    # Transforms can also be blocks:
+    #
+    #   transform(:ssn){ |v| v[0,24] }
+    #
+    # Finally, a transform can include a configuration hash:
+    #
+    #   transform :sex, :decode, {:decode_table_path => 'delimited_decode.txt'}
+    class Transform
+      class << self
+        # Transform the specified value using the given transforms. The transforms can either be
+        # Proc objects or objects which extend from Transform and implement the method <tt>transform(value)</tt>.
+        # Any other object will result in a ControlError being raised.
+        def transform(name, value, row, transforms)
+          transforms.each do |transform|
+            benchmarks[transform.class] ||= 0
+            benchmarks[transform.class] += Benchmark.realtime do
+              Engine.logger.debug "Transforming field #{name} with #{transform.inspect}"
+              case transform
+              when Proc
+                value = transform.call([name, value, row])
+              when Transform
+                value = transform.transform(name, value, row)
+              else
+                raise ControlError, "Unsupported transform configuration type: #{transform}"
+              end
+            end
+          end
+          value
+        end
+        
+        def benchmarks
+          @benchmarks ||= {}
+        end
+      end
+      
+      attr_reader :control, :name, :configuration
+      
+      # Initialize the transform object with the given control object, field name and 
+      # configuration hash
+      def initialize(control, name, configuration={})
+        @control = control
+        @name = name
+        @configuration = configuration
+      end
+      
+      def transform(name, value, row)
+        raise "transform is an abstract method"
+      end
+    end
+  end
+end

data/lib/etl/transform/trim_transform.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform to trim string
+    class TrimTransform < ETL::Transform::Transform
+      # Configuration options:
+      # * <tt>:type</tt>: :left, :right or :both. Default is :both
+      def initialize(control, name, configuration={})
+        super
+        @type = (configuration[:type] || :both).to_sym
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :left
+          value.lstrip
+        when :right
+          value.rstrip
+        when :both
+          value.strip
+        else
+          raise "Trim type, if specified, must be :left, :right or :both"
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/type_transform.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform from one type to another
+    class TypeTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      # 
+      # Configuration options:
+      # * <tt>:type</tt>: The type to convert to. Supported types:
+      # ** :string
+      # ** :number,:integer
+      # ** :float
+      # ** :decimal
+      def initialize(control, name, configuration={})
+        super
+        @type = configuration[:type]
+        @significant = configuration[:significant] ||= 0
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :string
+          value.to_s
+        when :number, :integer
+          value.to_i
+        when :float
+          value.to_f
+        when :decimal
+          BigDecimal.new(value.to_s, @significant)
+        else
+          raise "Unsupported type: #{@type}"
+        end
+      end
+    end
+  end
+end

data/lib/etl/util.rb

@@ -0,0 +1,59 @@
+module ETL
+  module Util
+    # Return the distance of time in words from the given from_time to the specified to_time. If to_time
+    # is not specified then Time.now is used. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def distance_of_time_in_words(from_time, to_time=Time.now)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      seconds = (to_time - from_time).round
+      distance_in_days = (seconds/(60*60*24)).round
+      seconds = seconds % (60*60*24)
+      distance_in_hours = (seconds/(60*60)).round
+      seconds = seconds % (60*60)
+      distance_in_minutes = (seconds/60).round
+      seconds = seconds % 60
+      distance_in_seconds = seconds
+    
+      s = ''
+      s << "#{distance_in_days} days," if distance_in_days > 0
+      s << "#{distance_in_hours} hours, " if distance_in_hours > 0
+      s << "#{distance_in_minutes} minutes, " if distance_in_minutes > 0
+      s << "#{distance_in_seconds} seconds"
+      s
+    end
+  
+    # Get the approximate disntance of time in words from the given from_time
+    # to the the given to_time. If to_time is not specified then it is set
+    # to Time.now. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def approximate_distance_of_time_in_words(from_time, to_time=Time.now, include_seconds=true)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      distance_in_minutes = (((to_time - from_time).abs)/60).round
+      distance_in_seconds = ((to_time - from_time).abs).round
+    
+      case distance_in_minutes
+        when 0..1
+          return (distance_in_minutes == 0) ? 'less than a minute' : '1 minute' unless include_seconds
+        case distance_in_seconds
+          when 0..4   then 'less than 5 seconds'
+          when 5..9   then 'less than 10 seconds'
+          when 10..19 then 'less than 20 seconds'
+          when 20..39 then 'half a minute'
+          when 40..59 then 'less than a minute'
+          else             '1 minute'
+        end
+        when 2..44           then "#{distance_in_minutes} minutes"
+        when 45..89          then 'about 1 hour'
+        when 90..1439        then "about #{(distance_in_minutes.to_f / 60.0).round} hours"
+        when 1440..2879      then '1 day'
+        when 2880..43199     then "#{(distance_in_minutes / 1440).round} days"
+        when 43200..86399    then 'about 1 month'
+        when 86400..525959   then "#{(distance_in_minutes / 43200).round} months"
+        when 525960..1051919 then 'about 1 year'
+        else                      "over #{(distance_in_minutes / 525960).round} years"
+      end
+    end
+  end
+end

data/lib/etl/version.rb

@@ -0,0 +1,3 @@
+module ETL#:nodoc:
+  VERSION = "0.9.5.rc1"
+end

data/test-matrix.yml

@@ -0,0 +1,10 @@
+rvm:
+  - 1.8.7
+  - 1.9.2
+#  - jruby-1.6.2
+rails:
+  - 2.3.x
+  - 3.0.x
+database:
+  - mysql
+  - postgresql

data/test/.gitignore

@@ -0,0 +1 @@
+database*.yml

data/test/.ignore

@@ -0,0 +1,2 @@
+database.yml
+*.txt

data/test/all.ebf

@@ -0,0 +1,6 @@
+# This is an ETL Batch File and defines a means for executing
+# a collection of ETL scripts as a single process.
+
+use_temp_tables
+run 'batched1.ctl'
+run 'batched2.ctl'

data/test/apache_combined_log.ctl

@@ -0,0 +1,11 @@
+source :in, {
+  :file => 'data/apache_combined_log.txt',
+  :parser => :apache_combined_log
+}
+
+destination :out, {
+  :file => 'output/apache_combined_log.txt'
+}, 
+{
+  :order => []
+}

data/test/batch_test.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class BatchTest < Test::Unit::TestCase
+  attr_reader :file, :db_yaml, :engine
+  def setup
+    @file =  File.dirname(__FILE__) + '/all.ebf'
+    @db_yaml = File.dirname(__FILE__) + '/database.yml'
+    @engine = ETL::Engine.new
+  end
+  def teardown
+    
+  end
+  def test_etl_batch_file
+    #`etl #{file} -c #{db_yaml}`
+  end
+  def test_batch
+    assert_nothing_raised do
+      batch = ETL::Batch::Batch.resolve(file, engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_file
+    assert_nothing_raised do
+      batch = ETL::Batch::Batch.resolve(File.new(file), engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_batch_object
+    assert_nothing_raised do
+      batch_instance = ETL::Batch::Batch.new(File.new(file))
+      batch_instance.engine = engine
+      batch = ETL::Batch::Batch.resolve(batch_instance, engine)
+      batch.execute
+    end
+  end
+  def test_batch_with_object_should_fail
+    assert_raise(RuntimeError) do
+      batch = ETL::Batch::Batch.resolve(0, engine)
+    end
+  end
+end

data/test/batch_with_error.ebf

@@ -0,0 +1,6 @@
+# This is an ETL Batch File and defines a means for executing
+# a collection of ETL scripts as a single process.
+
+use_temp_tables
+run 'delimited_with_bulk_load.ctl'
+run 'screen_test_fatal.ctl'

data/test/block_processor.ctl

@@ -0,0 +1,6 @@
+source :in, { :type => :mock, :name => :block_processed_input } 
+
+after_read { |row| row[:added_by_after_read] = "after-" +row[:first_name]; row }
+before_write { |row| row[:added_by_before_write] = "Row #{Engine.current_source_row}"; [row,{:new_row => 'added by post_processor'}] }
+
+destination :out, { :type => :mock, :name => :block_processed_output  }

data/test/block_processor_error.ctl

@@ -0,0 +1 @@
+pre_process { raise ControlError.new( "Cough!") }

data/test/block_processor_pre_post_process.ctl

@@ -0,0 +1,4 @@
+source :in, { :type => :mock, :name => :another_input } 
+pre_process { TestWitness.call("I'm called from pre_process") }
+post_process { TestWitness.call("I'm called from post_process") }
+destination :out, { :type => :mock, :name => :another_output  }

data/test/block_processor_remove_rows.ctl

@@ -0,0 +1,5 @@
+source :in, { :type => :mock, :name => :block_input } 
+
+before_write { |row| row[:obsolete] == true ? nil : row }
+
+destination :out, { :type => :mock, :name => :block_output  }

data/test/block_processor_test.rb

@@ -0,0 +1,38 @@
+require File.dirname(__FILE__) + '/test_helper'
+include ETL
+include ETL::Control
+
+class TestWitness
+end
+
+class BlockProcessorTest < Test::Unit::TestCase
+  
+  def test_block_processor_should_work_as_both_after_read_and_before_write_row_processor
+    MockSource[:block_processed_input] = [{ :first_name => 'John'},{:first_name => 'Gary'}]
+    process 'block_processor.ctl'
+    assert_equal 4, MockDestination[:block_processed_output].size
+    assert_equal({ :first_name => 'John', :added_by_after_read => 'after-John', :added_by_before_write => "Row 1" }, MockDestination[:block_processed_output][0])
+    assert_equal({ :new_row => 'added by post_processor' }, MockDestination[:block_processed_output][1])
+    assert_equal({ :first_name => 'Gary', :added_by_after_read => 'after-Gary', :added_by_before_write => "Row 2" }, MockDestination[:block_processed_output][2])
+    assert_equal({ :new_row => 'added by post_processor' }, MockDestination[:block_processed_output][3])
+  end
+  
+  def test_block_processor_should_let_rows_be_removed_by_setting_it_to_nil
+    MockSource[:block_input] = [{ :obsolete => true, :name => 'John'},{ :obsolete => false, :name => 'Gary'}]
+    process 'block_processor_remove_rows.ctl'
+    assert_equal([{ :obsolete => false, :name => 'Gary' }], MockDestination[:block_output]) # only one record should be kept
+  end
+  
+  def test_block_processor_should_work_as_pre_or_post_processor
+    flexmock(TestWitness).should_receive(:call).with("I'm called from pre_process")
+    flexmock(TestWitness).should_receive(:call).with("I'm called from post_process")
+    MockSource[:another_input] = [{ :obsolete => true, :name => 'John'},{ :obsolete => false, :name => 'Gary'}]
+    process 'block_processor_pre_post_process.ctl'
+    assert_equal(MockSource[:another_input], MockDestination[:another_output])
+  end
+  
+  def test_block_error_should_be_propagated
+    assert_raise(ControlError) { process 'block_processor_error.ctl' }
+  end
+  
+end