activewarehouse-etl 0.4.0 → 0.5.0

data/lib/etl/generator/generator.rb

@@ -1,11 +1,20 @@
-module ETL
-  module Generator
+module ETL #:nodoc:
+  module Generator #:nodoc:
+    # Base class for generators.
     class Generator
       class << self
+        # Get the Class for the specified name.
+        #
+        # For example, if name is :surrogate_key then a SurrogateKeyGenerator class is returned
         def class_for_name(name)
           ETL::Generator.const_get("#{name.to_s.classify}Generator")
         end
       end
+      
+      # Generate the next value. This method must be implemented by subclasses
+      def next
+        raise "Must be implemented by a subclass"
+      end
     end
   end
 end

data/lib/etl/generator/surrogate_key_generator.rb

@@ -1,5 +1,6 @@
-module ETL
-  module Generator
+module ETL #:nodoc:
+  module Generator #:nodoc:
+    # Surrogate key generator.
     class SurrogateKeyGenerator < Generator
       def next
         @surrogate_key ||= 0

data/lib/etl/parser/delimited_parser.rb

@@ -2,17 +2,16 @@ module ETL #:nodoc:
   module Parser #:nodoc:
     # Parses delimited files
     class DelimitedParser < ETL::Parser::Parser
-      include Enumerable
       # Initialize the parser
       # * <tt>source</tt>: The Source object
-      def initialize(source)
+      # * <tt>options</tt>: Hash of options for the parser, defaults to an empty hash
+      def initialize(source, options={})
         super
         configure
       end
       
       # Returns each row.
       def each
-        options = {}
         Dir.glob(file).each do |file|
           ETL::Engine.logger.debug "parsing #{file}"
           line = 0
@@ -64,7 +63,7 @@ module ETL #:nodoc:
         end
       end
       
-      class Field
+      class Field #:nodoc:
         attr_reader :name, :type
         def initialize(name, type=:string)
           @name = name

data/lib/etl/parser/fixed_width_parser.rb

@@ -2,11 +2,10 @@ module ETL #:nodoc:
   module Parser #:nodoc:
     # Parser for fixed with files
     class FixedWidthParser < ETL::Parser::Parser
-      include Enumerable
-      
       # Initialize the parser
       # * <tt>source</tt>: The source object
-      def initialize(source)
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
         super
         configure
       end
@@ -43,7 +42,7 @@ module ETL #:nodoc:
       end
     end
     
-    class FixedWidthField
+    class FixedWidthField #:nodoc:
       attr_reader :name, :field_start, :field_end, :field_length, :type
       def initialize(name, field_start, field_end=nil, field_length=nil, type=nil)
         @name = name

data/lib/etl/parser/parser.rb

@@ -1,6 +1,7 @@
 module ETL
   module Parser
     class Parser
+      include Enumerable
       class << self
         # Convert the name (string or symbol) to a parser class.
         #
@@ -11,10 +12,15 @@ module ETL
         end
       end
       
+      # The Source object for the data
       attr_reader :source
       
-      def initialize(source)
+      # Options Hash for the parser
+      attr_reader :options
+      
+      def initialize(source, options={})
         @source = source
+        @options = options || {}
       end
       
       # Convert the value to the specified type.

data/lib/etl/parser/sax_parser.rb

@@ -0,0 +1,190 @@
+require 'rexml/parsers/sax2parser'
+require 'rexml/sax2listener'
+
+module ETL
+  module Parser
+    class SaxParser < ETL::Parser::Parser
+      
+      # The write trigger causes whatever values are currently specified for the row to be returned.
+      # After returning the values will not be cleared, thus allowing for values which are assigned
+      # higher in the XML tree to remain in memory.
+      attr_accessor :write_trigger
+      
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each(&block)
+        Dir.glob(file).each do |file|
+          parser = REXML::Parsers::SAX2Parser.new(File.new(file))
+          listener = Listener.new(self, &block)
+          parser.listen(listener)
+          parser.parse
+        end
+      end
+      
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        #puts "write trigger in source.definition: #{source.definition[:write_trigger]}"
+        self.write_trigger = source.definition[:write_trigger]
+        # map paths to field names
+        source.definition[:fields].each do |name, path|
+          #puts "defined field #{name}, path: #{path}"
+          fields << Field.new(name, XPath::Path.parse(path))
+        end
+      end
+      
+      class Field
+        attr_reader :name, :path
+        def initialize(name, path)
+          @name = name
+          @path = path
+        end
+      end
+    end
+    
+    class Listener
+      include REXML::SAX2Listener
+      def initialize(parser, &block)
+        @parser = parser
+        @row = {}
+        @value = nil
+        @proc = Proc.new(&block)
+      end
+      def cdata(text)    
+        @value << text
+      end
+      def characters(text)
+        text = text.strip
+        if (!text.nil? && text != '')
+          @value ||= ''
+          @value << text
+        end
+      end
+      def start_document
+        @path = XPath::Path.new
+      end
+      def end_document
+        
+      end
+      def start_element(uri, localname, qname, attributes)
+        @path.elements << XPath::Element.new(localname, attributes)
+      end
+      def end_element(uri, localname, qname)
+        element = @path.elements.last
+        
+        @parser.fields.each do |field|
+          #puts "#{@path} match? #{field.path}"
+          if @path.match?(field.path)
+            #puts "field.path: #{field.path}"
+            if field.path.is_attribute?
+              @row[field.name] = element.attributes[field.path.attribute]
+            else
+              @row[field.name] = @value
+            end
+          end
+        end
+        #puts @path.to_s
+        if @path.match?(@parser.write_trigger)
+          #puts "matched: #{@path} =~ #{@parser.write_trigger}"
+          #puts "calling proc with #{@row.inspect}"
+          @proc.call(@row.clone)
+        end
+        
+        @value = nil
+        @path.elements.pop
+      end
+      def progress(position)
+        @position = position
+      end
+    end
+
+    module XPath
+      class Path
+        attr_accessor :elements
+        def initialize
+          @elements = []
+        end
+        def to_s
+          @elements.map{ |e| e.to_s }.join("/")
+        end
+        # Returns true if the last part of the path refers to an attribute
+        def is_attribute?
+          elements.last.attributes.length > 0
+        end
+        # Return the name of the attribute referenced by the last element in this path. Returns nil if the last element
+        # does not reference an attribute.
+        #
+        # Warning: the path must only reference a single attribute, otherwise the result of this method will be random, 
+        # since attributes are stored in a Hash.
+        def attribute
+          return nil unless is_attribute?
+          elements.last.attributes.keys.first
+        end
+        # Return true if this XPath::Path matches the given path string. This is a fail-fast match, so the first mismatch
+        # will cause the method to return false.
+        def match?(s)
+          path = Path.parse(s)
+          return false unless path.elements.length == elements.length
+          elements.each_with_index do |element, index|
+            path_element = path.elements[index]
+            return false if path_element.nil?
+            return false if element.name != path_element.name
+            path_element.attributes.each do |key, value|
+              return false unless element.attributes[key] =~ value
+            end
+          end
+          return true
+        end
+        
+        # Parse the string into an XPath::Path object
+        def self.parse(s)
+          return s if s.is_a?(Path)
+          path = Path.new
+          parts = s.split('/')
+          parts.each_with_index do |part, i|
+            attributes = {}
+            part.gsub!(/(.*)\[(.*)\]/, '\1')
+            if !$2.nil?
+              $2.split(",").each do |pair|
+                key, value = pair.split("=")
+                value = ".*" if value.nil?
+                attributes[key] = Regexp.new(value)
+              end
+            end
+            path.elements << Element.new(part, attributes)
+          end
+          path
+        end
+      end
+      class Element
+        attr_reader :name
+        attr_reader :attributes
+        def initialize(name, attributes={})
+          @name = name
+          @attributes = attributes
+        end
+        def to_s
+          s = "#{name}"
+          if !@attributes.empty?
+            attr_str = @attributes.collect do |key,value|
+              value = value.source if value.is_a?(Regexp)
+              "#{key}=#{value}" 
+            end.join(",")
+            s << "[" + attr_str + "]"
+          end
+          s
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/xml_parser.rb

@@ -3,10 +3,10 @@ require 'rexml/document'
 module ETL
   module Parser
     class XmlParser < ETL::Parser::Parser
-      include Enumerable
       # Initialize the parser
       # * <tt>source</tt>: The Source object
-      def initialize(source)
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
         super
         configure
       end

data/lib/etl/processor/bulk_import_processor.rb

@@ -1,5 +1,5 @@
-module ETL
-  module Processor
+module ETL #:nodoc:
+  module Processor #:nodoc:
     # Processor which is used to bulk import data into a target database
     class BulkImportProcessor < ETL::Processor::Processor
       attr_reader :file, :target, :truncate, :columns
@@ -13,7 +13,7 @@ module ETL
       end
       def process
         # columns = control.destinations.first.order.join(',') # TODO: support multiple destinations?
-        conn = ActiveRecord::Base.connection
+        conn = ETL::ActiveRecord::Base.connection
         conn.transaction do
           # TODO: Support all database types
           # Since LOCAL is used this must be allowed by both the client and server
@@ -27,7 +27,7 @@ module ETL
       private
       # Connect to the database
       def connect
-        ActiveRecord::Base.establish_connection(
+        ETL::ActiveRecord::Base.establish_connection(
             :adapter  => (target[:adapter] || :mysql),
             :username => (target[:username] || 'root'),
             :host     => (target[:host] || 'localhost'),

data/lib/etl/processor/processor.rb

@@ -1,6 +1,6 @@
 module ETL #:nodoc:
   module Processor #:nodoc:
-    # Base class for pre and post processors
+    # Base class for pre and post processors. Subclasses must implement the +process+ method.
     class Processor
       def initialize(control, configuration)
         @control = control

data/lib/etl/processor/truncate_processor.rb

@@ -1,5 +1,5 @@
-module ETL
-  module Processor
+module ETL #:nodoc:
+  module Processor #:nodoc:
     # A processor which will truncate a table. Use as a pre-processor for cleaning out a table
     # prior to loading
     class TruncateProcessor < ETL::Processor::Processor
@@ -11,13 +11,13 @@ module ETL
         connect
       end
       def process
-        conn = ActiveRecord::Base.connection
+        conn = ETL::ActiveRecord::Base.connection
         conn.truncate
       end
       
       # Connect to the database
       def connect
-        ActiveRecord::Base.establish_connection(
+        ETL::ActiveRecord::Base.establish_connection(
             :adapter  => (target[:adapter] || :mysql),
             :username => (target[:username] || 'root'),
             :host     => (target[:host] || 'localhost'),

data/lib/etl/transform/date_to_string_transform.rb

@@ -0,0 +1,19 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform a Date or Time to a formatted string instance
+    class DateToStringTransform < ETL::Transform::Transform
+      # Initialize the transformer.
+      #
+      # Configuration options:
+      # * <tt>:format</tt>: A format passed to strftime. Defaults to %Y-%m-%d
+      def initialize(control, configuration={})
+        super
+        @format = configuration[:format] || "%Y-%m-%d"
+      end
+      # Transform the value using strftime
+      def transform(value)
+        value.strftime(@format)
+      end
+    end
+  end
+end

data/lib/etl/transform/decode_transform.rb

@@ -2,7 +2,18 @@ module ETL #:nodoc:
   module Transform #:nodoc:
     # Transform which decodes coded values
     class DecodeTransform < ETL::Transform::Transform
-      attr_accessor :decode_table_path, :decode_table_delimiter, :default_value
+      attr_accessor :decode_table_path
+      
+      attr_accessor :decode_table_delimiter
+      
+      attr_accessor :default_value
+      
+      # Initialize the transformer
+      #
+      # Configuration options:
+      # * <tt>:decode_table_path</tt>: The path to the decode table (defaults to 'decode.txt')
+      # * <tt>:decode_table_delimiter</tt>: The decode table delimiter (defaults to ':')
+      # * <tt>:default_value</tt>: The default value to use (defaults to 'No Value')
       def initialize(control, configuration={})
         super
         
@@ -14,10 +25,13 @@ module ETL #:nodoc:
         @decode_table_delimiter = (configuration[:decode_table_delimiter] || ':')
         @default_value = (configuration[:default_value] || 'No Value')
       end
+      
+      # Transform the value
       def transform(value)
         decode_table[value] || default_value
       end
       
+      # Get the decode table
       def decode_table
         unless @decode_table
           @decode_table = {}

data/lib/etl/transform/foreign_key_lookup_transform.rb

@@ -0,0 +1,53 @@
+module ETL #:nodoc:
+  module Transform #:nodoc:
+    # Transform which looks up the value and replaces it with a foriegn key reference
+    class ForeignKeyLookupTransform < ETL::Transform::Transform
+      # Initialize the foreign key lookup transform.
+      #
+      # Configuration options:
+      # *<tt>:collection</tt>: A Hash of natural keys mapped to surrogate keys. If this is not specified then
+      #  an empty Hash will be used. This Hash will be used to cache values that have been resolved already
+      #  for future use.
+      # *<tt>:resolver</tt>: Object or Class which implements the method resolve(value)
+      def initialize(control, configuration={})
+        super
+        
+        @collection = (configuration[:collection] || {})
+        @resolver = configuration[:resolver]
+        @resolver = @resolver.new if @resolver.is_a?(Class)
+      end
+      
+      # Transform the value by resolving it to a foriegn key
+      def transform(value)
+        fk = @collection[value]
+        unless fk
+          raise ResolverError, "Foreign key for #{value} not found and no resolver specified" unless @resolver
+          raise ResolverError, "Resolver does not appear to respond to resolve method" unless @resolver.respond_to?(:resolve)
+          fk = @resolver.resolve(value)
+          raise ResolverError, "Unable to resolve #{value} to foreign key" unless fk
+          @collection[value] = fk
+        end
+        fk
+      end
+    end
+  end
+end
+
+# Resolver which resolves using ActiveRecord.
+class ActiveRecordResolver
+  # Initialize the resolver. The ar_class argument should extend from ActiveRecord::Base. The find_method argument
+  # must be a symbol for the finder method used. For example:
+  # 
+  # ActiveRecordResolver.new(Person, :find_by_name)
+  #
+  # Note that the find method defined must only take a single argument.
+  def initialize(ar_class, find_method)
+    @ar_class = ar_class
+    @find_method = find_method
+  end
+  # Resolve the value
+  def resolve(value)
+    rec = @ar_class.__send__(@find_method, value)
+    rec.nil? ? nil : rec.id
+  end
+end