etl 0.9.5.rc1

data/lib/etl/transform/md5_transform.rb

@@ -0,0 +1,13 @@
+require 'digest/md5'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a MD5 hash algorithm
+    class Md5Transform < ETL::Transform::Transform
+      # Transform the value with a MD5 digest algorithm.
+      def transform(name, value, row)
+        Digest::MD5.hexdigest(value)
+      end
+    end
+  end
+end

data/lib/etl/transform/ordinalize_transform.rb

@@ -0,0 +1,14 @@
+require 'active_support/core_ext/integer/inflections.rb'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a number to an ordinalized version using the ActiveSupport ordinalize
+    # core extension
+    class OrdinalizeTransform < ETL::Transform::Transform
+      # Transform the value from a number to an ordinalized number
+      def transform(name, value, row)
+        value.ordinalize
+      end
+    end
+  end
+end

data/lib/etl/transform/sha1_transform.rb

@@ -0,0 +1,13 @@
+require 'digest/sha1'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a SHA-1 hash algorithm
+    class Sha1Transform < ETL::Transform::Transform
+      # Transform the value with a SHA1 digest algorithm.
+      def transform(name, value, row)
+        Digest::SHA1.hexdigest(value)
+      end
+    end
+  end
+end

data/lib/etl/transform/split_fields_transform.rb

@@ -0,0 +1,27 @@
+module ETL
+  module Transform
+    class SplitFieldsTransform < ETL::Transform::Transform
+      attr_reader :delimiter
+      attr_reader :new_fields
+
+      def initialize(control, name, configuration)
+        @delimiter = configuration[:delimiter] || ','
+        @new_fields = configuration[:new_fields]
+        super
+      end
+      
+      def transform(name, value, row)
+        return nil if row.nil?
+        return nil if row[name].nil?
+
+        fields = row[name].split(@delimiter)
+        @new_fields.each_with_index do |new, index|
+          row[new] = fields[index]
+        end
+
+        row[name]
+      end
+
+    end
+  end
+end

data/lib/etl/transform/string_to_date_time_transform.rb

@@ -0,0 +1,14 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a DateTime instance
+    class StringToDateTimeTransform < ETL::Transform::Transform
+      # Transform the value using DateTime.parse.
+      #
+      # WARNING: This transform is slow (due to the Ruby implementation), but if you need to 
+      # parse timestamps before or after the values supported by the Time.parse.
+      def transform(name, value, row)
+        DateTime.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_date_transform.rb

@@ -0,0 +1,16 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Date instance
+    class StringToDateTransform < ETL::Transform::Transform
+      # Transform the value using Date.parse
+      def transform(name, value, row)
+        return value if value.nil?
+        begin
+          Date.parse(value)
+        rescue => e
+          return value
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_time_transform.rb

@@ -0,0 +1,11 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Time instance
+    class StringToTimeTransform < ETL::Transform::Transform
+      # Transform the value using Time.parse
+      def transform(name, value, row)
+        Time.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/lib/etl/transform/transform.rb

@@ -0,0 +1,61 @@
+module ETL#:nodoc:
+  module Transform#:nodoc:
+    # Base class for transforms.
+    #
+    # A transform converts one value to another value using some sort of algorithm.
+    #
+    # A simple transform has two arguments, the field to transform and the name of the transform:
+    #
+    #   transform :ssn, :sha1
+    # 
+    # Transforms can also be blocks:
+    #
+    #   transform(:ssn){ |v| v[0,24] }
+    #
+    # Finally, a transform can include a configuration hash:
+    #
+    #   transform :sex, :decode, {:decode_table_path => 'delimited_decode.txt'}
+    class Transform
+      class << self
+        # Transform the specified value using the given transforms. The transforms can either be
+        # Proc objects or objects which extend from Transform and implement the method <tt>transform(value)</tt>.
+        # Any other object will result in a ControlError being raised.
+        def transform(name, value, row, transforms)
+          transforms.each do |transform|
+            benchmarks[transform.class] ||= 0
+            benchmarks[transform.class] += Benchmark.realtime do
+              Engine.logger.debug "Transforming field #{name} with #{transform.inspect}"
+              case transform
+              when Proc
+                value = transform.call([name, value, row])
+              when Transform
+                value = transform.transform(name, value, row)
+              else
+                raise ControlError, "Unsupported transform configuration type: #{transform}"
+              end
+            end
+          end
+          value
+        end
+        
+        def benchmarks
+          @benchmarks ||= {}
+        end
+      end
+      
+      attr_reader :control, :name, :configuration
+      
+      # Initialize the transform object with the given control object, field name and 
+      # configuration hash
+      def initialize(control, name, configuration={})
+        @control = control
+        @name = name
+        @configuration = configuration
+      end
+      
+      def transform(name, value, row)
+        raise "transform is an abstract method"
+      end
+    end
+  end
+end

data/lib/etl/transform/trim_transform.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform to trim string
+    class TrimTransform < ETL::Transform::Transform
+      # Configuration options:
+      # * <tt>:type</tt>: :left, :right or :both. Default is :both
+      def initialize(control, name, configuration={})
+        super
+        @type = (configuration[:type] || :both).to_sym
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :left
+          value.lstrip
+        when :right
+          value.rstrip
+        when :both
+          value.strip
+        else
+          raise "Trim type, if specified, must be :left, :right or :both"
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/type_transform.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform from one type to another
+    class TypeTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      # 
+      # Configuration options:
+      # * <tt>:type</tt>: The type to convert to. Supported types:
+      # ** :string
+      # ** :number,:integer
+      # ** :float
+      # ** :decimal
+      def initialize(control, name, configuration={})
+        super
+        @type = configuration[:type]
+        @significant = configuration[:significant] ||= 0
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :string
+          value.to_s
+        when :number, :integer
+          value.to_i
+        when :float
+          value.to_f
+        when :decimal
+          BigDecimal.new(value.to_s, @significant)
+        else
+          raise "Unsupported type: #{@type}"
+        end
+      end
+    end
+  end
+end

data/lib/etl/util.rb

@@ -0,0 +1,59 @@
+module ETL
+  module Util
+    # Return the distance of time in words from the given from_time to the specified to_time. If to_time
+    # is not specified then Time.now is used. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def distance_of_time_in_words(from_time, to_time=Time.now)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      seconds = (to_time - from_time).round
+      distance_in_days = (seconds/(60*60*24)).round
+      seconds = seconds % (60*60*24)
+      distance_in_hours = (seconds/(60*60)).round
+      seconds = seconds % (60*60)
+      distance_in_minutes = (seconds/60).round
+      seconds = seconds % 60
+      distance_in_seconds = seconds
+    
+      s = ''
+      s << "#{distance_in_days} days," if distance_in_days > 0
+      s << "#{distance_in_hours} hours, " if distance_in_hours > 0
+      s << "#{distance_in_minutes} minutes, " if distance_in_minutes > 0
+      s << "#{distance_in_seconds} seconds"
+      s
+    end
+  
+    # Get the approximate disntance of time in words from the given from_time
+    # to the the given to_time. If to_time is not specified then it is set
+    # to Time.now. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def approximate_distance_of_time_in_words(from_time, to_time=Time.now, include_seconds=true)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      distance_in_minutes = (((to_time - from_time).abs)/60).round
+      distance_in_seconds = ((to_time - from_time).abs).round
+    
+      case distance_in_minutes
+        when 0..1
+          return (distance_in_minutes == 0) ? 'less than a minute' : '1 minute' unless include_seconds
+        case distance_in_seconds
+          when 0..4   then 'less than 5 seconds'
+          when 5..9   then 'less than 10 seconds'
+          when 10..19 then 'less than 20 seconds'
+          when 20..39 then 'half a minute'
+          when 40..59 then 'less than a minute'
+          else             '1 minute'
+        end
+        when 2..44           then "#{distance_in_minutes} minutes"
+        when 45..89          then 'about 1 hour'
+        when 90..1439        then "about #{(distance_in_minutes.to_f / 60.0).round} hours"
+        when 1440..2879      then '1 day'
+        when 2880..43199     then "#{(distance_in_minutes / 1440).round} days"
+        when 43200..86399    then 'about 1 month'
+        when 86400..525959   then "#{(distance_in_minutes / 43200).round} months"
+        when 525960..1051919 then 'about 1 year'
+        else                      "over #{(distance_in_minutes / 525960).round} years"
+      end
+    end
+  end
+end

data/lib/etl/version.rb

@@ -0,0 +1,3 @@
+module ETL#:nodoc:
+  VERSION = "0.9.5.rc1"
+end

data/spec/fixtures/all.ebf

@@ -0,0 +1,6 @@
+# This is an ETL Batch File and defines a means for executing
+# a collection of ETL scripts as a single process.
+
+use_temp_tables
+run 'batched1.ctl'
+run 'batched2.ctl'

data/spec/fixtures/apache_combined_log.ctl

@@ -0,0 +1,11 @@
+source :in, {
+  :file => 'data/apache_combined_log.txt',
+  :parser => :apache_combined_log
+}
+
+destination :out, {
+  :file => 'output/apache_combined_log.txt'
+}, 
+{
+  :order => []
+}

data/spec/fixtures/batch_with_error.ebf

@@ -0,0 +1,6 @@
+# This is an ETL Batch File and defines a means for executing
+# a collection of ETL scripts as a single process.
+
+use_temp_tables
+run 'delimited_with_bulk_load.ctl'
+run 'screen_test_fatal.ctl'

data/spec/fixtures/block_processor.ctl

@@ -0,0 +1,6 @@
+source :in, { :type => :mock, :name => :block_processed_input } 
+
+after_read { |row| row[:added_by_after_read] = "after-" +row[:first_name]; row }
+before_write { |row| row[:added_by_before_write] = "Row #{Engine.current_source_row}"; [row,{:new_row => 'added by post_processor'}] }
+
+destination :out, { :type => :mock, :name => :block_processed_output  }

data/spec/fixtures/block_processor_error.ctl

@@ -0,0 +1 @@
+pre_process { raise ControlError.new( "Cough!") }

data/spec/fixtures/block_processor_pre_post_process.ctl

@@ -0,0 +1,4 @@
+source :in, { :type => :mock, :name => :another_input } 
+pre_process { TestWitness.call("I'm called from pre_process") }
+post_process { TestWitness.call("I'm called from post_process") }
+destination :out, { :type => :mock, :name => :another_output  }

data/spec/fixtures/block_processor_remove_rows.ctl

@@ -0,0 +1,5 @@
+source :in, { :type => :mock, :name => :block_input } 
+
+before_write { |row| row[:obsolete] == true ? nil : row }
+
+destination :out, { :type => :mock, :name => :block_output  }

data/spec/fixtures/data/apache_combined_log.txt

@@ -0,0 +1,3 @@
+127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+127.0.0.1 - bob [11/Oct/2000:05:22:02 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "http://www.foo.com/" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+127.0.0.1 - bob [11/Oct/2000:05:52:31 -0700] "GET /apache_pb.gif HTTP/1.1" 200 2326 "-" "Mozilla/4.08 [en] (Win98; I ;Nav)"

data/spec/fixtures/data/bulk_import.txt

@@ -0,0 +1,3 @@
+1,Chris,Smith,111223333
+2,Jim,Foxworthy,444332222
+3,Brian,Collingsworth,123443435

data/spec/fixtures/data/bulk_import_with_empties.txt

@@ -0,0 +1,3 @@
+1,Chris,Smith,111223333
+2,Jim,,444332222
+3,Brian,Collingsworth,123443435

data/spec/fixtures/data/decode.txt

@@ -0,0 +1,3 @@
+M:Male
+F:Female
+:Unknown

data/spec/fixtures/data/delimited.txt

@@ -0,0 +1,3 @@
+Chris,Smith,111223333,24,M
+Jim,Foxworthy,444332222,51,M
+Brian,Collingsworth,123443435,10,M

data/spec/fixtures/data/encode_source_latin1.txt

@@ -0,0 +1,2 @@
+�ph�m�re has accents.
+let's encode them.

data/spec/fixtures/data/fixed_width.txt

@@ -0,0 +1,3 @@
+Bob      Smith       12344555523
+Jane     Doe         98766211145
+AbcdefghiJklmnopqrstu12345678920

data/spec/fixtures/data/multiple_delimited_1.txt

@@ -0,0 +1,3 @@
+Chris,Smith,111223333,24
+Jim,Foxworthy,444332222,51
+Brian,Collingsworth,123443435,10

data/spec/fixtures/data/multiple_delimited_2.txt

@@ -0,0 +1,3 @@
+Bob,Jones,444223333,28
+Tom,Allen,324001232,33
+Jesse,Baker,555443333,21

data/spec/fixtures/data/nokogiri.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="US-ASCII" standalone="yes"?>
+<!DOCTYPE people >
+<people>
+  <person ssn="123456789" type="employee">
+    <first_name>Bob</first_name>
+    <last_name>Smith</last_name>
+    <email>bsmith@foo.com</email>
+    <colours>
+      <eyes>brown</eyes>
+      <hair>black</hair>
+      <skin>fair</skin>
+    </colours>
+    <age>24</age>
+  </person>
+  <person ssn="111223333" type="client">
+    <first_name>Jane</first_name>
+    <last_name>Doe</last_name>
+    <email>jdoe@bar.com</email>
+    <colours>
+      <eyes>blue</eyes>
+      <hair>blond</hair>
+      <skin>medium</skin>
+    </colours>
+    <age>45</age>
+  </person>
+  <person ssn="133244566" type="client">
+    <first_name>Jake</first_name>
+    <last_name>Smithsonian</last_name>
+    <email>jake@example.com</email>
+    <colours>
+      <eyes>brown</eyes>
+      <hair>black</hair>
+      <skin>dark</skin>
+    </colours>
+    <age>37</age>
+  </person>
+</people>
+