factorylabs-activewarehouse-etl 0.9.1.2

data/lib/etl/parser/fixed_width_parser.rb

@@ -0,0 +1,65 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parser for fixed with files
+    class FixedWidthParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Return each row
+      def each
+        Dir.glob(file).each do |file|
+          open(file).each do |line|
+            row = {}
+            lines_skipped = 0
+            fields.each do |name, f|
+              if lines_skipped < source.skip_lines
+                lines_skipped += 1
+                next
+              end
+              # TODO make strip optional?
+              row[name] = line[f.field_start, f.field_length].strip
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Return a map of defined fields
+      def fields
+        @fields ||= {}
+      end
+      
+      private
+      def configure
+        source.definition.each do |field, options|
+          fields[field] = FixedWidthField.new(
+            options[:name], options[:start], options[:end], options[:length]
+          )
+        end
+      end
+    end
+    
+    class FixedWidthField #:nodoc:
+      attr_reader :name, :field_start, :field_end, :field_length
+      # Initialize the field.
+      def initialize(name, field_start, field_end=nil, field_length=nil)
+        @name = name
+        @field_start = field_start - 1
+        if field_end
+          @field_end = field_end
+          @field_length = @field_end - @field_start
+        elsif field_length
+          @field_length = field_length
+          @field_end = @field_start + @field_length
+        else
+          raise DefinitionError, "Either field_end or field_length required"
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/parser.rb

@@ -0,0 +1,41 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Base parser class. Implementation classes must extend this class and implement
+    # the each method. The each method should return each row of the source data as 
+    # a Hash.
+    class Parser
+      include Enumerable
+      class << self
+        # Convert the name (string or symbol) to a parser class.
+        #
+        # Example:
+        #   <tt>class_for_name(:fixed_width)</tt> returns a FixedWidthParser class
+        def class_for_name(name)
+          ETL::Parser.const_get("#{name.to_s.camelize}Parser")
+        end
+      end
+      
+      # The Source object for the data
+      attr_reader :source
+      
+      # Options Hash for the parser
+      attr_reader :options
+      
+      def initialize(source, options={})
+        @source = source
+        @options = options || {}
+      end
+      
+      protected
+      def file
+        path = Pathname.new(source.configuration[:file])
+        path = path.absolute? ? path : Pathname.new(File.dirname(source.control.file)) + path
+        path
+      end
+      
+      def raise_with_info(error, message, file, line)
+        raise error, "#{message} (line #{line} in #{file})"
+      end
+    end
+  end
+end

data/lib/etl/parser/sax_parser.rb

@@ -0,0 +1,218 @@
+require 'rexml/parsers/sax2parser'
+require 'rexml/sax2listener'
+
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # ETL parser implementation which uses SAX to parse XML files.
+    class SaxParser < ETL::Parser::Parser
+      
+      # The write trigger causes whatever values are currently specified for the row to be returned.
+      # After returning the values will not be cleared, thus allowing for values which are assigned
+      # higher in the XML tree to remain in memory.
+      attr_accessor :write_trigger
+      
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each(&block)
+        Dir.glob(file).each do |file|
+          parser = REXML::Parsers::SAX2Parser.new(File.new(file))
+          listener = Listener.new(self, &block)
+          parser.listen(listener)
+          parser.parse
+        end
+      end
+      
+      # Get an array of Field objects
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        #puts "write trigger in source.definition: #{source.definition[:write_trigger]}"
+        self.write_trigger = source.definition[:write_trigger]
+        # map paths to field names
+        source.definition[:fields].each do |name, path|
+          #puts "defined field #{name}, path: #{path}"
+          fields << Field.new(name, XPath::Path.parse(path))
+        end
+      end
+      
+      # Class representing a field to be loaded from the source
+      class Field
+        # The name of the field
+        attr_reader :name
+        # The XPath-like path to the field in the XML document
+        attr_reader :path
+        
+        def initialize(name, path) #:nodoc
+          @name = name
+          @path = path
+        end
+      end
+    end
+    
+    class Listener #:nodoc:
+      include REXML::SAX2Listener
+      def initialize(parser, &block)
+        @parser = parser
+        @row = {}
+        @value = nil
+        @proc = Proc.new(&block)
+      end
+      def cdata(text)    
+        @value << text
+      end
+      def characters(text)
+        text = text.strip
+        if (!text.nil? && text != '')
+          @value ||= ''
+          @value << text
+        end
+      end
+      def start_document
+        @path = XPath::Path.new
+      end
+      def end_document
+        
+      end
+      def start_element(uri, localname, qname, attributes)
+        element = XPath::Element.new(localname, attributes)
+        @path.elements << element
+        
+        @parser.fields.each do |field|
+          #puts "#{@path} match? #{field.path}"
+          if @path.match?(field.path)
+            #puts "field.path: #{field.path}"
+            if field.path.is_attribute?
+              #puts "setting @row[#{field.name}] to #{element.attributes[field.path.attribute]}"
+              @row[field.name] = element.attributes[field.path.attribute]
+            end
+          end
+        end
+      end
+      def end_element(uri, localname, qname)
+        element = @path.elements.last
+        
+        @parser.fields.each do |field|
+          #puts "#{@path} match? #{field.path}"
+          if @path.match?(field.path)
+            #puts "field.path: #{field.path}"
+            if !field.path.is_attribute?
+              @row[field.name] = @value
+            end
+          end
+        end
+        
+        #puts @path.to_s
+        if @path.match?(@parser.write_trigger)
+          #puts "matched: #{@path} =~ #{@parser.write_trigger}"
+          #puts "calling proc with #{@row.inspect}"
+          @proc.call(@row.clone)
+        end
+        
+        @value = nil
+        @path.elements.pop
+      end
+      def progress(position)
+        @position = position
+      end
+    end
+
+    # Module which contains classes that are used for XPath-like filtering
+    # on the SAX parser
+    module XPath #:nodoc:
+      class Path #:nodoc:
+        # Get the elements in the path
+        attr_accessor :elements
+        
+        # Initialize
+        def initialize
+          @elements = []
+        end
+        
+        # Convert to a string representation
+        def to_s
+          @elements.map{ |e| e.to_s }.join("/")
+        end
+        
+        # Returns true if the last part of the path refers to an attribute
+        def is_attribute?
+          elements.last.attributes.length > 0
+        end
+        
+        # Return the name of the attribute referenced by the last element in this path. Returns nil if the last element
+        # does not reference an attribute.
+        #
+        # Warning: the path must only reference a single attribute, otherwise the result of this method will be random, 
+        # since attributes are stored in a Hash.
+        def attribute
+          return nil unless is_attribute?
+          elements.last.attributes.keys.first
+        end
+        
+        # Return true if this XPath::Path matches the given path string. This is a fail-fast match, so the first mismatch
+        # will cause the method to return false.
+        def match?(s)
+          path = Path.parse(s)
+          return false unless path.elements.length == elements.length
+          elements.each_with_index do |element, index|
+            path_element = path.elements[index]
+            return false if path_element.nil?
+            return false if element.name != path_element.name
+            path_element.attributes.each do |key, value|
+              return false unless element.attributes[key] =~ value
+            end
+          end
+          return true
+        end
+        
+        # Parse the string into an XPath::Path object
+        def self.parse(s)
+          return s if s.is_a?(Path)
+          path = Path.new
+          parts = s.split('/')
+          parts.each_with_index do |part, i|
+            attributes = {}
+            part.gsub!(/(.*)\[(.*)\]/, '\1')
+            if !$2.nil?
+              $2.split(",").each do |pair|
+                key, value = pair.split("=")
+                value = ".*" if value.nil?
+                attributes[key] = Regexp.new(value)
+              end
+            end
+            path.elements << Element.new(part, attributes)
+          end
+          path
+        end
+      end
+      class Element #:nodoc
+        attr_reader :name
+        attr_reader :attributes
+        def initialize(name, attributes={})
+          @name = name
+          @attributes = attributes
+        end
+        def to_s
+          s = "#{name}"
+          if !@attributes.empty?
+            attr_str = @attributes.collect do |key,value|
+              value = value.source if value.is_a?(Regexp)
+              "#{key}=#{value}" 
+            end.join(",")
+            s << "[" + attr_str + "]"
+          end
+          s
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/xml_parser.rb

@@ -0,0 +1,65 @@
+require 'rexml/document'
+
+module ETL
+  module Parser
+    class XmlParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each
+        Dir.glob(file).each do |file|
+          doc = nil
+          t = Benchmark.realtime do
+            doc = REXML::Document.new(File.new(file))
+          end
+          Engine.logger.info "XML #{file} parsed in #{t}s"
+          doc.elements.each(@collection_xpath) do |element|
+            row = {}
+            fields.each do |f|
+              value = element.text(f.xpath)
+              row[f.name] = value
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        @collection_xpath = source.definition[:collection]
+        raise "Collection XPath is required" if @collection_xpath.nil?
+        
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options, options.to_s)
+          when Hash
+            options[:xpath] ||= options[:name]
+            fields << Field.new(options[:name], options[:xpath].to_s)
+          else
+            raise DefinitionError, "Each field definition must either be an symbol or a hash of options for the field"
+          end
+        end
+      end
+      
+      class Field
+        attr_reader :name, :xpath
+        def initialize(name, xpath)
+          @name = name
+          @xpath = xpath
+        end
+      end
+    end
+  end
+end

data/lib/etl/processor.rb

@@ -0,0 +1,11 @@
+# This source file contains the ETL::Processor module and requires all of the processors
+
+module ETL #:nodoc:
+  # The ETL::Processor module contains row-level and bulk processors
+  module Processor
+  end
+end
+
+require 'etl/processor/processor'
+require 'etl/processor/row_processor'
+Dir[File.dirname(__FILE__) + "/processor/*.rb"].each { |file| require(file) }

data/lib/etl/processor/block_processor.rb

@@ -0,0 +1,14 @@
+module ETL
+  module Processor
+    # This processor is both a valid RowProcessor (called on each row with after_read) or a Processor (called once on pre_process or post_process)
+    class BlockProcessor < ETL::Processor::RowProcessor
+      def initialize(control, configuration)
+        super
+        @block = configuration[:block]
+      end
+      def process(row=nil)
+        @block.call(row)
+      end
+    end
+  end
+end

data/lib/etl/processor/bulk_import_processor.rb

@@ -0,0 +1,81 @@
+module ETL #:nodoc:
+  module Processor #:nodoc:
+    # Processor which is used to bulk import data into a target database. The
+    # underlying database driver from ActiveRecord must support the methods
+    # +bulk_load+ method.
+    class BulkImportProcessor < ETL::Processor::Processor
+      
+      # The file to load from
+      attr_reader :file
+      # The target database
+      attr_reader :target
+      # The table name
+      attr_reader :table
+      # Set to true to truncate
+      attr_reader :truncate
+      # Array of symbols representing the column load order
+      attr_reader :columns
+      # The field separator (defaults to a comma)
+      attr_accessor :field_separator
+      # The field enclosure (defaults to nil)
+      attr_accessor :field_enclosure
+      # The line separator (defaults to a newline)
+      attr_accessor :line_separator
+      # The string that indicates a NULL (defaults to an empty string)
+      attr_accessor :null_string
+       
+      # Initialize the processor.
+      #
+      # Configuration options:
+      # * <tt>:file</tt>: The file to load data from
+      # * <tt>:target</tt>: The target database
+      # * <tt>:table</tt>: The table name
+      # * <tt>:truncate</tt>: Set to true to truncate before loading
+      # * <tt>:columns</tt>: The columns to load in the order they appear in
+      #   the bulk data file
+      # * <tt>:field_separator</tt>: The field separator. Defaults to a comma
+      # * <tt>:line_separator</tt>: The line separator. Defaults to a newline
+      # * <tt>:field_enclosure</tt>: The field enclosure charcaters
+      def initialize(control, configuration)
+        super
+        @file = File.join(File.dirname(control.file), configuration[:file])
+        @target = configuration[:target]
+        @table = configuration[:table]
+        @truncate = configuration[:truncate] ||= false
+        @columns = configuration[:columns]
+        @field_separator = (configuration[:field_separator] || ',')
+        @line_separator = (configuration[:line_separator] || "\n")
+        @null_string = (configuration[:null_string] || "")
+        @field_enclosure = configuration[:field_enclosure]
+        
+        raise ControlError, "Target must be specified" unless @target
+        raise ControlError, "Table must be specified" unless @table
+      end
+      
+      # Execute the processor
+      def process
+        return if ETL::Engine.skip_bulk_import
+        return if File.size(file) == 0
+        
+        conn = ETL::Engine.connection(target)
+        conn.transaction do
+          conn.truncate(table_name) if truncate
+          options = {}
+          options[:columns] = columns
+          if field_separator || field_enclosure || line_separator || null_string
+            options[:fields] = {}
+            options[:fields][:null_string] = null_string if null_string
+            options[:fields][:delimited_by] = field_separator if field_separator
+            options[:fields][:enclosed_by] = field_enclosure if field_enclosure
+            options[:fields][:terminated_by] = line_separator if line_separator
+          end
+          conn.bulk_load(file, table_name, options)
+        end
+      end
+      
+      def table_name
+        ETL::Engine.table(table, ETL::Engine.connection(target))
+      end
+    end
+  end
+end