colincasey-activewarehouse-etl 0.9.4

data/lib/etl/transform/hierarchy_lookup_transform.rb

@@ -0,0 +1,49 @@
+module ETL #:nodoc: 
+  module Transform #:nodoc:
+    # Transform which walks up the hierarchy tree to find a value of the current level's value
+    # is nil.
+    #
+    # TODO: Let the resolver be implemented in a class so different resolution methods are
+    # possible.
+    class HierarchyLookupTransform < ETL::Transform::Transform
+      # The name of the field to use for the parent ID
+      attr_accessor :parent_id_field
+      
+      # The target connection name
+      attr_accessor :target
+      
+      # Initialize the transform
+      #
+      # Configuration options:
+      # * <tt>:target</tt>: The target connection name (required)
+      # * <tt>:parent_id_field</tt>: The name of the field to use for the parent ID (defaults to :parent_id)
+      def initialize(control, name, configuration={})
+        super
+        @parent_id_field = configuration[:parent_id_field] || :parent_id
+        @target = configuration[:target]
+      end
+      
+      # Transform the value.
+      def transform(name, value, row)
+        if parent_id = row[parent_id_field]
+          # TODO: should use more than just the first source out of the control
+          parent_id, value = lookup(name, 
+            control.sources.first.configuration[:table], parent_id, parent_id_field)        
+          until value || parent_id.nil?
+            # TODO: should use more than just the first source out of the control
+            parent_id, value = lookup(name, 
+              control.sources.first.configuration[:table], parent_id, parent_id_field)
+          end
+        end
+        value
+      end
+      
+      # Lookup the parent value.
+      def lookup(field, table, parent_id, parent_id_field)
+        q = "SELECT #{parent_id_field}, #{field} FROM #{table} WHERE id = #{parent_id}"
+        row = ETL::Engine.connection(target).select_one(q)
+        return row[parent_id_field.to_s], row[field.to_s]
+      end
+    end
+  end
+end

data/lib/etl/transform/ordinalize_transform.rb

@@ -0,0 +1,12 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a number to an ordinalized version using the ActiveSupport ordinalize
+    # core extension
+    class OrdinalizeTransform < ETL::Transform::Transform
+      # Transform the value from a number to an ordinalized number
+      def transform(name, value, row)
+        value.ordinalize
+      end
+    end
+  end
+end

data/lib/etl/transform/sha1_transform.rb

@@ -0,0 +1,13 @@
+require 'digest/sha1'
+
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which hashes the original value with a SHA-1 hash algorithm
+    class Sha1Transform < ETL::Transform::Transform
+      # Transform the value with a SHA1 digest algorithm.
+      def transform(name, value, row)
+        Digest::SHA1.hexdigest(value)
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_date_transform.rb

@@ -0,0 +1,16 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Date instance
+    class StringToDateTransform < ETL::Transform::Transform
+      # Transform the value using Date.parse
+      def transform(name, value, row)
+        return value if value.nil?
+        begin
+          Date.parse(value)
+        rescue => e
+          return value
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_datetime_transform.rb

@@ -0,0 +1,14 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a DateTime instance
+    class StringToDateTimeTransform < ETL::Transform::Transform
+      # Transform the value using DateTime.parse.
+      #
+      # WARNING: This transform is slow (due to the Ruby implementation), but if you need to 
+      # parse timestamps before or after the values supported by the Time.parse.
+      def transform(name, value, row)
+        DateTime.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/lib/etl/transform/string_to_time_transform.rb

@@ -0,0 +1,11 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a String representation of a date to a Time instance
+    class StringToTimeTransform < ETL::Transform::Transform
+      # Transform the value using Time.parse
+      def transform(name, value, row)
+        Time.parse(value) unless value.nil?
+      end
+    end
+  end
+end

data/lib/etl/transform/transform.rb

@@ -0,0 +1,61 @@
+module ETL#:nodoc:
+  module Transform#:nodoc:
+    # Base class for transforms.
+    #
+    # A transform converts one value to another value using some sort of algorithm.
+    #
+    # A simple transform has two arguments, the field to transform and the name of the transform:
+    #
+    #   transform :ssn, :sha1
+    # 
+    # Transforms can also be blocks:
+    #
+    #   transform(:ssn){ |v| v[0,24] }
+    #
+    # Finally, a transform can include a configuration hash:
+    #
+    #   transform :sex, :decode, {:decode_table_path => 'delimited_decode.txt'}
+    class Transform
+      class << self
+        # Transform the specified value using the given transforms. The transforms can either be
+        # Proc objects or objects which extend from Transform and implement the method <tt>transform(value)</tt>.
+        # Any other object will result in a ControlError being raised.
+        def transform(name, value, row, transforms)
+          transforms.each do |transform|
+            benchmarks[transform.class] ||= 0
+            benchmarks[transform.class] += Benchmark.realtime do
+              Engine.logger.debug "Transforming field #{name} with #{transform.inspect}"
+              case transform
+              when Proc
+                value = transform.call([name, value, row])
+              when Transform
+                value = transform.transform(name, value, row)
+              else
+                raise ControlError, "Unsupported transform configuration type: #{transform}"
+              end
+            end
+          end
+          value
+        end
+        
+        def benchmarks
+          @benchmarks ||= {}
+        end
+      end
+      
+      attr_reader :control, :name, :configuration
+      
+      # Initialize the transform object with the given control object, field name and 
+      # configuration hash
+      def initialize(control, name, configuration={})
+        @control = control
+        @name = name
+        @configuration = configuration
+      end
+      
+      def transform(name, value, row)
+        raise "transform is an abstract method"
+      end
+    end
+  end
+end

data/lib/etl/transform/trim_transform.rb

@@ -0,0 +1,26 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform to trim string
+    class TrimTransform < ETL::Transform::Transform
+      # Configuration options:
+      # * <tt>:type</tt>: :left, :right or :both. Default is :both
+      def initialize(control, name, configuration={})
+        super
+        @type = (configuration[:type] || :both).to_sym
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :left
+          value.lstrip
+        when :right
+          value.rstrip
+        when :both
+          value.strip
+        else
+          raise "Trim type, if specified, must be :left, :right or :both"
+        end
+      end
+    end
+  end
+end

data/lib/etl/transform/type_transform.rb

@@ -0,0 +1,35 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform from one type to another
+    class TypeTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      # 
+      # Configuration options:
+      # * <tt>:type</tt>: The type to convert to. Supported types:
+      # ** :string
+      # ** :number,:integer
+      # ** :float
+      # ** :decimal
+      def initialize(control, name, configuration={})
+        super
+        @type = configuration[:type]
+        @significant = configuration[:significant] ||= 0
+      end
+      # Transform the value
+      def transform(name, value, row)
+        case @type
+        when :string
+          value.to_s
+        when :number, :integer
+          value.to_i
+        when :float
+          value.to_f
+        when :decimal
+          BigDecimal.new(value.to_s, @significant)
+        else
+          raise "Unsupported type: #{@type}"
+        end
+      end
+    end
+  end
+end

data/lib/etl/util.rb

@@ -0,0 +1,59 @@
+module ETL
+  module Util
+    # Return the distance of time in words from the given from_time to the specified to_time. If to_time
+    # is not specified then Time.now is used. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def distance_of_time_in_words(from_time, to_time=Time.now)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      seconds = (to_time - from_time).round
+      distance_in_days = (seconds/(60*60*24)).round
+      seconds = seconds % (60*60*24)
+      distance_in_hours = (seconds/(60*60)).round
+      seconds = seconds % (60*60)
+      distance_in_minutes = (seconds/60).round
+      seconds = seconds % 60
+      distance_in_seconds = seconds
+    
+      s = ''
+      s << "#{distance_in_days} days," if distance_in_days > 0
+      s << "#{distance_in_hours} hours, " if distance_in_hours > 0
+      s << "#{distance_in_minutes} minutes, " if distance_in_minutes > 0
+      s << "#{distance_in_seconds} seconds"
+      s
+    end
+  
+    # Get the approximate disntance of time in words from the given from_time
+    # to the the given to_time. If to_time is not specified then it is set
+    # to Time.now. By default seconds are included...set the include_seconds
+    # argument to false to disable the seconds.
+    def approximate_distance_of_time_in_words(from_time, to_time=Time.now, include_seconds=true)
+      from_time = from_time.to_time if from_time.respond_to?(:to_time)
+      to_time = to_time.to_time if to_time.respond_to?(:to_time)
+      distance_in_minutes = (((to_time - from_time).abs)/60).round
+      distance_in_seconds = ((to_time - from_time).abs).round
+    
+      case distance_in_minutes
+        when 0..1
+          return (distance_in_minutes == 0) ? 'less than a minute' : '1 minute' unless include_seconds
+        case distance_in_seconds
+          when 0..4   then 'less than 5 seconds'
+          when 5..9   then 'less than 10 seconds'
+          when 10..19 then 'less than 20 seconds'
+          when 20..39 then 'half a minute'
+          when 40..59 then 'less than a minute'
+          else             '1 minute'
+        end
+        when 2..44           then "#{distance_in_minutes} minutes"
+        when 45..89          then 'about 1 hour'
+        when 90..1439        then "about #{(distance_in_minutes.to_f / 60.0).round} hours"
+        when 1440..2879      then '1 day'
+        when 2880..43199     then "#{(distance_in_minutes / 1440).round} days"
+        when 43200..86399    then 'about 1 month'
+        when 86400..525959   then "#{(distance_in_minutes / 43200).round} months"
+        when 525960..1051919 then 'about 1 year'
+        else                      "over #{(distance_in_minutes / 525960).round} years"
+      end
+    end
+  end
+end

data/lib/etl/version.rb

@@ -0,0 +1,10 @@
+module ETL#:nodoc:
+  def self.version
+    @version ||= load_from_yaml
+  end
+
+  def self.load_from_yaml
+    yaml = YAML.load_file(File.dirname(__FILE__) + "/../../VERSION.yml")
+    "#{yaml[:major]}.#{yaml[:minor]}.#{yaml[:patch]}"
+  end
+end

metadata

@@ -0,0 +1,224 @@
+--- !ruby/object:Gem::Specification 
+name: colincasey-activewarehouse-etl
+version: !ruby/object:Gem::Version 
+  version: 0.9.4
+platform: ruby
+authors: 
+- Anthony Eden
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-24 00:00:00 -07:00
+default_executable: etl
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.8.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.1.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.1.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: adapter_extensions
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.5.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: roo
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: colincasey-sequel
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.10.4
+    version: 
+description: ActiveWarehouse ETL is a pure Ruby Extract-Transform-Load application for loading data into a database.
+email: casey.colin@gmail.com
+executables: 
+- etl
+extensions: []
+
+extra_rdoc_files: 
+- README
+- CHANGELOG
+- LICENSE
+files: 
+- CHANGELOG
+- LICENSE
+- README
+- TODO
+- Rakefile
+- VERSION.yml
+- bin/etl
+- bin/etl.cmd
+- lib/etl
+- lib/etl/batch
+- lib/etl/batch/batch.rb
+- lib/etl/batch/directives.rb
+- lib/etl/batch.rb
+- lib/etl/builder
+- lib/etl/builder/date_dimension_builder.rb
+- lib/etl/builder/time_dimension_builder.rb
+- lib/etl/builder.rb
+- lib/etl/commands
+- lib/etl/commands/etl.rb
+- lib/etl/control
+- lib/etl/control/control.rb
+- lib/etl/control/destination
+- lib/etl/control/destination/csv_destination.rb
+- lib/etl/control/destination/database_destination.rb
+- lib/etl/control/destination/file_destination.rb
+- lib/etl/control/destination/yaml_destination.rb
+- lib/etl/control/destination.rb
+- lib/etl/control/source
+- lib/etl/control/source/database_source.rb
+- lib/etl/control/source/enumerable_source.rb
+- lib/etl/control/source/file_source.rb
+- lib/etl/control/source/model_source.rb
+- lib/etl/control/source.rb
+- lib/etl/control.rb
+- lib/etl/core_ext
+- lib/etl/core_ext/time
+- lib/etl/core_ext/time/calculations.rb
+- lib/etl/core_ext/time.rb
+- lib/etl/core_ext.rb
+- lib/etl/engine.rb
+- lib/etl/execution
+- lib/etl/execution/base.rb
+- lib/etl/execution/batch.rb
+- lib/etl/execution/job.rb
+- lib/etl/execution/migration.rb
+- lib/etl/execution.rb
+- lib/etl/generator
+- lib/etl/generator/generator.rb
+- lib/etl/generator/surrogate_key_generator.rb
+- lib/etl/generator.rb
+- lib/etl/http_tools.rb
+- lib/etl/parser
+- lib/etl/parser/apache_combined_log_parser.rb
+- lib/etl/parser/delimited_parser.rb
+- lib/etl/parser/fixed_width_parser.rb
+- lib/etl/parser/parser.rb
+- lib/etl/parser/sax_parser.rb
+- lib/etl/parser/spreadsheet_parser.rb
+- lib/etl/parser/xml_parser.rb
+- lib/etl/parser.rb
+- lib/etl/processor
+- lib/etl/processor/block_processor.rb
+- lib/etl/processor/bulk_import_processor.rb
+- lib/etl/processor/check_exist_processor.rb
+- lib/etl/processor/check_unique_processor.rb
+- lib/etl/processor/copy_field_processor.rb
+- lib/etl/processor/encode_processor.rb
+- lib/etl/processor/hierarchy_exploder_processor.rb
+- lib/etl/processor/print_row_processor.rb
+- lib/etl/processor/processor.rb
+- lib/etl/processor/rename_processor.rb
+- lib/etl/processor/require_non_blank_processor.rb
+- lib/etl/processor/row_processor.rb
+- lib/etl/processor/sequence_processor.rb
+- lib/etl/processor/surrogate_key_processor.rb
+- lib/etl/processor/truncate_processor.rb
+- lib/etl/processor.rb
+- lib/etl/row.rb
+- lib/etl/screen
+- lib/etl/screen/row_count_screen.rb
+- lib/etl/screen.rb
+- lib/etl/transform
+- lib/etl/transform/block_transform.rb
+- lib/etl/transform/date_to_string_transform.rb
+- lib/etl/transform/decode_transform.rb
+- lib/etl/transform/default_transform.rb
+- lib/etl/transform/foreign_key_lookup_transform.rb
+- lib/etl/transform/hierarchy_lookup_transform.rb
+- lib/etl/transform/ordinalize_transform.rb
+- lib/etl/transform/sha1_transform.rb
+- lib/etl/transform/string_to_datetime_transform.rb
+- lib/etl/transform/string_to_date_transform.rb
+- lib/etl/transform/string_to_time_transform.rb
+- lib/etl/transform/transform.rb
+- lib/etl/transform/trim_transform.rb
+- lib/etl/transform/type_transform.rb
+- lib/etl/transform.rb
+- lib/etl/util.rb
+- lib/etl/version.rb
+- lib/etl.rb
+has_rdoc: true
+homepage: http://github.com/colincasey/activewarehouse-etl
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Pure Ruby ETL package.
+test_files: []
+