activewarehouse-etl-sgonyea 0.9.6

data/lib/etl/parser/excel_parser.rb

@@ -0,0 +1,112 @@
+optional_require 'spreadsheet'
+
+module ETL
+  module Parser
+    class ExcelParser < ETL::Parser::Parser
+
+      attr_accessor :ignore_blank_line
+
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+
+      # Returns each row
+      def each
+        Dir.glob(file).each do |file|
+          ETL::Engine.logger.debug "parsing #{file}"
+          line = 0
+          lines_skipped = 0
+          book = Spreadsheet.open file
+          loopworksheets = []
+
+          if worksheets.empty?
+            loopworksheets = book.worksheets
+          else
+            worksheets.each do |index|
+              loopworksheets << book.worksheet(index)
+            end
+          end
+
+          loopworksheets.each do |sheet|
+            sheet.each do |raw_row|
+              if lines_skipped < source.skip_lines
+                ETL::Engine.logger.debug "skipping line"
+                lines_skipped += 1
+                next
+              end
+              line += 1
+              row = {}
+              if self.ignore_blank_line and raw_row.empty?
+                lines_skipped += 1
+                next
+              end
+              validate_row(raw_row, line, file)
+              raw_row.each_with_index do |value, index|
+                f = fields[index]
+                row[f.name] = value
+              end
+              yield row
+            end
+          end
+        end
+      end
+
+      # Get an array of defined worksheets
+      def worksheets
+        @worksheets ||= []
+      end
+
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+
+      private
+      def validate_row(row, line, file)
+        ETL::Engine.logger.debug "validating line #{line} in file #{file}"
+        if row.length != fields.length
+          raise_with_info( MismatchError, 
+            "The number of columns from the source (#{row.length}) does not match the number of columns in the definition (#{fields.length})", 
+            line, file
+          )
+        end
+      end
+      
+      private
+      def configure
+        source.definition[:worksheets].each do |worksheet|
+          if Integer(worksheet)
+            worksheets << worksheet.to_i
+          else
+            raise DefinitionError, "Each worksheet definition must be an integer"
+          end
+        end unless source.definition[:worksheets].nil?
+
+        self.ignore_blank_line = source.definition[:ignore_blank_line]
+
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options)
+          when Hash
+            fields << Field.new(options[:name])
+          else
+            raise DefinitionError, "Each field definition must either be a symbol or a hash"
+          end
+        end
+      end
+      
+      class Field #:nodoc:
+        attr_reader :name
+        def initialize(name)
+          @name = name
+        end
+      end
+
+    end
+  end
+end

data/lib/etl/parser/fixed_width_parser.rb

@@ -0,0 +1,65 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parser for fixed with files
+    class FixedWidthParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Return each row
+      def each
+        Dir.glob(file).each do |file|
+          open(file).each do |line|
+            row = {}
+            lines_skipped = 0
+            fields.each do |name, f|
+              if lines_skipped < source.skip_lines
+                lines_skipped += 1
+                next
+              end
+              # TODO make strip optional?
+              row[name] = line[f.field_start, f.field_length].strip
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Return a map of defined fields
+      def fields
+        @fields ||= {}
+      end
+      
+      private
+      def configure
+        source.definition.each do |field, options|
+          fields[field] = FixedWidthField.new(
+            options[:name], options[:start], options[:end], options[:length]
+          )
+        end
+      end
+    end
+    
+    class FixedWidthField #:nodoc:
+      attr_reader :name, :field_start, :field_end, :field_length
+      # Initialize the field.
+      def initialize(name, field_start, field_end=nil, field_length=nil)
+        @name = name
+        @field_start = field_start - 1
+        if field_end
+          @field_end = field_end
+          @field_length = @field_end - @field_start
+        elsif field_length
+          @field_length = field_length
+          @field_end = @field_start + @field_length
+        else
+          raise DefinitionError, "Either field_end or field_length required"
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/nokogiri_xml_parser.rb

@@ -0,0 +1,83 @@
+optional_require 'nokogiri'
+require 'open-uri'
+optional_require 'zlib'
+
+module ETL
+  module Parser
+    class NokogiriXmlParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each
+        Dir.glob(file).each do |source|
+
+          doc = nil
+
+          gzip = false
+          magic = "1F8B".to_i(base=16)  # Check for gzip archives
+          if File.exist?(source)
+            gzip = true if magic == (
+              File.open(source).read(2).unpack("H2H2").to_s.to_i(base=16))
+          end
+
+          if gzip
+            doc = Nokogiri::XML(Zlib::GzipReader.open(source))
+          else
+            doc = Nokogiri::XML(open(source))
+          end
+          
+          doc.xpath(@collection_xpath).each do |nodeset|
+            row = {}
+
+            fields.each do |f|
+              value = nodeset.xpath(f.xpath).text
+              row[f.name] = value
+            end
+            yield row
+          end
+
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        @collection_xpath = source.definition[:collection]
+        if @collection_xpath.nil?
+          raise ":collection => 'XPath' argument required"
+        end
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options, options.to_s)
+          when Hash
+            options[:xpath] ||= options[:name]
+            fields << Field.new(options[:name], options[:xpath].to_s)
+          else
+            raise DefinitionError, 
+              "Each field definition must either be an symbol " +
+              "or a hash of options for the field"
+          end
+        end
+      end
+      
+      class Field
+        attr_reader :name, :xpath
+        def initialize(name, xpath)
+          @name = name
+          @xpath = xpath
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/parser.rb

@@ -0,0 +1,41 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Base parser class. Implementation classes must extend this class and implement
+    # the each method. The each method should return each row of the source data as 
+    # a Hash.
+    class Parser
+      include Enumerable
+      class << self
+        # Convert the name (string or symbol) to a parser class.
+        #
+        # Example:
+        #   <tt>class_for_name(:fixed_width)</tt> returns a FixedWidthParser class
+        def class_for_name(name)
+          ETL::Parser.const_get("#{name.to_s.camelize}Parser")
+        end
+      end
+      
+      # The Source object for the data
+      attr_reader :source
+      
+      # Options Hash for the parser
+      attr_reader :options
+      
+      def initialize(source, options={})
+        @source = source
+        @options = options || {}
+      end
+      
+      protected
+      def file
+        path = Pathname.new(source.configuration[:file])
+        path = path.absolute? ? path : Pathname.new(File.dirname(source.control.file)) + path
+        path
+      end
+      
+      def raise_with_info(error, message, file, line)
+        raise error, "#{message} (line #{line} in #{file})"
+      end
+    end
+  end
+end

data/lib/etl/parser/sax_parser.rb

@@ -0,0 +1,218 @@
+require 'rexml/parsers/sax2parser'
+require 'rexml/sax2listener'
+
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # ETL parser implementation which uses SAX to parse XML files.
+    class SaxParser < ETL::Parser::Parser
+      
+      # The write trigger causes whatever values are currently specified for the row to be returned.
+      # After returning the values will not be cleared, thus allowing for values which are assigned
+      # higher in the XML tree to remain in memory.
+      attr_accessor :write_trigger
+      
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each(&block)
+        Dir.glob(file).each do |file|
+          parser = REXML::Parsers::SAX2Parser.new(File.new(file))
+          listener = Listener.new(self, &block)
+          parser.listen(listener)
+          parser.parse
+        end
+      end
+      
+      # Get an array of Field objects
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        #puts "write trigger in source.definition: #{source.definition[:write_trigger]}"
+        self.write_trigger = source.definition[:write_trigger]
+        # map paths to field names
+        source.definition[:fields].each do |name, path|
+          #puts "defined field #{name}, path: #{path}"
+          fields << Field.new(name, XPath::Path.parse(path))
+        end
+      end
+      
+      # Class representing a field to be loaded from the source
+      class Field
+        # The name of the field
+        attr_reader :name
+        # The XPath-like path to the field in the XML document
+        attr_reader :path
+        
+        def initialize(name, path) #:nodoc
+          @name = name
+          @path = path
+        end
+      end
+    end
+    
+    class Listener #:nodoc:
+      include REXML::SAX2Listener
+      def initialize(parser, &block)
+        @parser = parser
+        @row = {}
+        @value = nil
+        @proc = Proc.new(&block)
+      end
+      def cdata(text)    
+        @value << text
+      end
+      def characters(text)
+        text = text.strip
+        if (!text.nil? && text != '')
+          @value ||= ''
+          @value << text
+        end
+      end
+      def start_document
+        @path = XPath::Path.new
+      end
+      def end_document
+        
+      end
+      def start_element(uri, localname, qname, attributes)
+        element = XPath::Element.new(localname, attributes)
+        @path.elements << element
+        
+        @parser.fields.each do |field|
+          #puts "#{@path} match? #{field.path}"
+          if @path.match?(field.path)
+            #puts "field.path: #{field.path}"
+            if field.path.is_attribute?
+              #puts "setting @row[#{field.name}] to #{element.attributes[field.path.attribute]}"
+              @row[field.name] = element.attributes[field.path.attribute]
+            end
+          end
+        end
+      end
+      def end_element(uri, localname, qname)
+        element = @path.elements.last
+        
+        @parser.fields.each do |field|
+          #puts "#{@path} match? #{field.path}"
+          if @path.match?(field.path)
+            #puts "field.path: #{field.path}"
+            if !field.path.is_attribute?
+              @row[field.name] = @value
+            end
+          end
+        end
+        
+        #puts @path.to_s
+        if @path.match?(@parser.write_trigger)
+          #puts "matched: #{@path} =~ #{@parser.write_trigger}"
+          #puts "calling proc with #{@row.inspect}"
+          @proc.call(@row.clone)
+        end
+        
+        @value = nil
+        @path.elements.pop
+      end
+      def progress(position)
+        @position = position
+      end
+    end
+
+    # Module which contains classes that are used for XPath-like filtering
+    # on the SAX parser
+    module XPath #:nodoc:
+      class Path #:nodoc:
+        # Get the elements in the path
+        attr_accessor :elements
+        
+        # Initialize
+        def initialize
+          @elements = []
+        end
+        
+        # Convert to a string representation
+        def to_s
+          @elements.map{ |e| e.to_s }.join("/")
+        end
+        
+        # Returns true if the last part of the path refers to an attribute
+        def is_attribute?
+          elements.last.attributes.length > 0
+        end
+        
+        # Return the name of the attribute referenced by the last element in this path. Returns nil if the last element
+        # does not reference an attribute.
+        #
+        # Warning: the path must only reference a single attribute, otherwise the result of this method will be random, 
+        # since attributes are stored in a Hash.
+        def attribute
+          return nil unless is_attribute?
+          elements.last.attributes.keys.first
+        end
+        
+        # Return true if this XPath::Path matches the given path string. This is a fail-fast match, so the first mismatch
+        # will cause the method to return false.
+        def match?(s)
+          path = Path.parse(s)
+          return false unless path.elements.length == elements.length
+          elements.each_with_index do |element, index|
+            path_element = path.elements[index]
+            return false if path_element.nil?
+            return false if element.name != path_element.name
+            path_element.attributes.each do |key, value|
+              return false unless element.attributes[key] =~ value
+            end
+          end
+          return true
+        end
+        
+        # Parse the string into an XPath::Path object
+        def self.parse(s)
+          return s if s.is_a?(Path)
+          path = Path.new
+          parts = s.split('/')
+          parts.each_with_index do |part, i|
+            attributes = {}
+            part.gsub!(/(.*)\[(.*)\]/, '\1')
+            if !$2.nil?
+              $2.split(",").each do |pair|
+                key, value = pair.split("=")
+                value = ".*" if value.nil?
+                attributes[key] = Regexp.new(value)
+              end
+            end
+            path.elements << Element.new(part, attributes)
+          end
+          path
+        end
+      end
+      class Element #:nodoc
+        attr_reader :name
+        attr_reader :attributes
+        def initialize(name, attributes={})
+          @name = name
+          @attributes = attributes
+        end
+        def to_s
+          s = "#{name}"
+          if !@attributes.empty?
+            attr_str = @attributes.collect do |key,value|
+              value = value.source if value.is_a?(Regexp)
+              "#{key}=#{value}" 
+            end.join(",")
+            s << "[" + attr_str + "]"
+          end
+          s
+        end
+      end
+    end
+  end
+end