colincasey-activewarehouse-etl 0.9.4

data/lib/etl/parser/fixed_width_parser.rb

@@ -0,0 +1,65 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parser for fixed with files
+    class FixedWidthParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Return each row
+      def each
+        Dir.glob(file).each do |file|
+          open(file).each do |line|
+            row = {}
+            lines_skipped = 0
+            fields.each do |name, f|
+              if lines_skipped < source.skip_lines
+                lines_skipped += 1
+                next
+              end
+              # TODO make strip optional?
+              row[name] = line[f.field_start, f.field_length].strip
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Return a map of defined fields
+      def fields
+        @fields ||= {}
+      end
+      
+      private
+      def configure
+        source.definition.each do |field, options|
+          fields[field] = FixedWidthField.new(
+            options[:name], options[:start], options[:end], options[:length]
+          )
+        end
+      end
+    end
+    
+    class FixedWidthField #:nodoc:
+      attr_reader :name, :field_start, :field_end, :field_length
+      # Initialize the field.
+      def initialize(name, field_start, field_end=nil, field_length=nil)
+        @name = name
+        @field_start = field_start - 1
+        if field_end
+          @field_end = field_end
+          @field_length = @field_end - @field_start
+        elsif field_length
+          @field_length = field_length
+          @field_end = @field_start + @field_length
+        else
+          raise DefinitionError, "Either field_end or field_length required"
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/parser.rb

@@ -0,0 +1,41 @@
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Base parser class. Implementation classes must extend this class and implement
+    # the each method. The each method should return each row of the source data as 
+    # a Hash.
+    class Parser
+      include Enumerable
+      class << self
+        # Convert the name (string or symbol) to a parser class.
+        #
+        # Example:
+        #   <tt>class_for_name(:fixed_width)</tt> returns a FixedWidthParser class
+        def class_for_name(name)
+          ETL::Parser.const_get("#{name.to_s.camelize}Parser")
+        end
+      end
+      
+      # The Source object for the data
+      attr_reader :source
+      
+      # Options Hash for the parser
+      attr_reader :options
+      
+      def initialize(source, options={})
+        @source = source
+        @options = options || {}
+      end
+      
+      protected
+      def file
+        path = Pathname.new(source.configuration[:file])
+        path = path.absolute? ? path : Pathname.new(File.dirname(source.control.file)) + path
+        path
+      end
+      
+      def raise_with_info(error, message, file, line)
+        raise error, "#{message} (line #{line} in #{file})"
+      end
+    end
+  end
+end

data/lib/etl/parser/sax_parser.rb

@@ -0,0 +1,218 @@
+require 'rexml/parsers/sax2parser'
+require 'rexml/sax2listener'
+
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # ETL parser implementation which uses SAX to parse XML files.
+    class SaxParser < ETL::Parser::Parser
+      
+      # The write trigger causes whatever values are currently specified for the row to be returned.
+      # After returning the values will not be cleared, thus allowing for values which are assigned
+      # higher in the XML tree to remain in memory.
+      attr_accessor :write_trigger
+      
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each(&block)
+        Dir.glob(file).each do |file|
+          parser = REXML::Parsers::SAX2Parser.new(File.new(file))
+          listener = Listener.new(self, &block)
+          parser.listen(listener)
+          parser.parse
+        end
+      end
+      
+      # Get an array of Field objects
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        #puts "write trigger in source.definition: #{source.definition[:write_trigger]}"
+        self.write_trigger = source.definition[:write_trigger]
+        # map paths to field names
+        source.definition[:fields].each do |name, path|
+          #puts "defined field #{name}, path: #{path}"
+          fields << Field.new(name, XPath::Path.parse(path))
+        end
+      end
+      
+      # Class representing a field to be loaded from the source
+      class Field
+        # The name of the field
+        attr_reader :name
+        # The XPath-like path to the field in the XML document
+        attr_reader :path
+        
+        def initialize(name, path) #:nodoc
+          @name = name
+          @path = path
+        end
+      end
+    end
+    
+    class Listener #:nodoc:
+      include REXML::SAX2Listener
+      def initialize(parser, &block)
+        @parser = parser
+        @row = {}
+        @value = nil
+        @proc = Proc.new(&block)
+      end
+      def cdata(text)    
+        @value << text
+      end
+      def characters(text)
+        text = text.strip
+        if (!text.nil? && text != '')
+          @value ||= ''
+          @value << text
+        end
+      end
+      def start_document
+        @path = XPath::Path.new
+      end
+      def end_document
+        
+      end
+      def start_element(uri, localname, qname, attributes)
+        element = XPath::Element.new(localname, attributes)
+        @path.elements << element
+        
+        @parser.fields.each do |field|
+          #puts "#{@path} match? #{field.path}"
+          if @path.match?(field.path)
+            #puts "field.path: #{field.path}"
+            if field.path.is_attribute?
+              #puts "setting @row[#{field.name}] to #{element.attributes[field.path.attribute]}"
+              @row[field.name] = element.attributes[field.path.attribute]
+            end
+          end
+        end
+      end
+      def end_element(uri, localname, qname)
+        element = @path.elements.last
+        
+        @parser.fields.each do |field|
+          #puts "#{@path} match? #{field.path}"
+          if @path.match?(field.path)
+            #puts "field.path: #{field.path}"
+            if !field.path.is_attribute?
+              @row[field.name] = @value
+            end
+          end
+        end
+        
+        #puts @path.to_s
+        if @path.match?(@parser.write_trigger)
+          #puts "matched: #{@path} =~ #{@parser.write_trigger}"
+          #puts "calling proc with #{@row.inspect}"
+          @proc.call(@row.clone)
+        end
+        
+        @value = nil
+        @path.elements.pop
+      end
+      def progress(position)
+        @position = position
+      end
+    end
+
+    # Module which contains classes that are used for XPath-like filtering
+    # on the SAX parser
+    module XPath #:nodoc:
+      class Path #:nodoc:
+        # Get the elements in the path
+        attr_accessor :elements
+        
+        # Initialize
+        def initialize
+          @elements = []
+        end
+        
+        # Convert to a string representation
+        def to_s
+          @elements.map{ |e| e.to_s }.join("/")
+        end
+        
+        # Returns true if the last part of the path refers to an attribute
+        def is_attribute?
+          elements.last.attributes.length > 0
+        end
+        
+        # Return the name of the attribute referenced by the last element in this path. Returns nil if the last element
+        # does not reference an attribute.
+        #
+        # Warning: the path must only reference a single attribute, otherwise the result of this method will be random, 
+        # since attributes are stored in a Hash.
+        def attribute
+          return nil unless is_attribute?
+          elements.last.attributes.keys.first
+        end
+        
+        # Return true if this XPath::Path matches the given path string. This is a fail-fast match, so the first mismatch
+        # will cause the method to return false.
+        def match?(s)
+          path = Path.parse(s)
+          return false unless path.elements.length == elements.length
+          elements.each_with_index do |element, index|
+            path_element = path.elements[index]
+            return false if path_element.nil?
+            return false if element.name != path_element.name
+            path_element.attributes.each do |key, value|
+              return false unless element.attributes[key] =~ value
+            end
+          end
+          return true
+        end
+        
+        # Parse the string into an XPath::Path object
+        def self.parse(s)
+          return s if s.is_a?(Path)
+          path = Path.new
+          parts = s.split('/')
+          parts.each_with_index do |part, i|
+            attributes = {}
+            part.gsub!(/(.*)\[(.*)\]/, '\1')
+            if !$2.nil?
+              $2.split(",").each do |pair|
+                key, value = pair.split("=")
+                value = ".*" if value.nil?
+                attributes[key] = Regexp.new(value)
+              end
+            end
+            path.elements << Element.new(part, attributes)
+          end
+          path
+        end
+      end
+      class Element #:nodoc
+        attr_reader :name
+        attr_reader :attributes
+        def initialize(name, attributes={})
+          @name = name
+          @attributes = attributes
+        end
+        def to_s
+          s = "#{name}"
+          if !@attributes.empty?
+            attr_str = @attributes.collect do |key,value|
+              value = value.source if value.is_a?(Regexp)
+              "#{key}=#{value}" 
+            end.join(",")
+            s << "[" + attr_str + "]"
+          end
+          s
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/spreadsheet_parser.rb

@@ -0,0 +1,114 @@
+gem 'roo', ">= 1.2.3"
+require 'roo'
+
+module ETL #:nodoc:
+  module Parser #:nodoc:
+    # Parses spreadsheet files
+    class SpreadsheetParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      #
+      # Configuration options:
+      # * <tt>:type</tt>: Either :excel, :excel_x, :open_office, or :google      
+      # * <tt>:sheet</tt>: The name or index of the sheet to read from
+      # * <tt>:first_row</tt>:
+      # * <tt>:last_row</tt>:
+      # * <tt>:first_column</tt>:
+      # * <tt>:last_column</tt>: 
+      def initialize(source, options={})
+        super
+        raise ':sheet is a required option' unless options[:sheet]
+        @sheet = options[:sheet]
+        @first_row = options[:first_row]
+        @last_row = options[:last_row]
+        @first_column = options[:first_column]
+        @last_column = options[:last_column]
+        configure
+      end
+      
+      # Returns each row.
+      def each
+        Dir.glob(file).each do |file|
+          ETL::Engine.logger.debug "parsing #{file}"
+          spreadsheet = create_spreadsheet(file)
+          spreadsheet.default_sheet = @sheet
+          first_row = @first_row || spreadsheet.first_row
+          last_row  = @last_row  || spreadsheet.last_row
+          first_column = @first_column || spreadsheet.first_column
+          last_column = @last_column || spreadsheet.last_column
+          #puts "#({first_row},#{first_column}):(#{last_row},#{last_column})"
+
+          current_row = first_row
+          while(current_row <= last_row) do
+            raw_row = []
+            current_column = first_column
+            while(current_column <= last_column) do
+              raw_row << spreadsheet.cell(current_row, current_column)
+              current_column += 1
+            end
+            current_row += 1
+            #puts "#{raw_row.inspect}"
+            row = {}
+            validate_row(raw_row, current_row, file)
+            raw_row.each_with_index do |value, index|
+              f = fields[index]
+              row[f.name] = value
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def validate_row(row, line, file)
+        ETL::Engine.logger.debug "validating line #{line} in file #{file}"
+        if row.length != fields.length
+          raise_with_info( MismatchError, 
+            "The number of columns from the source (#{row.length}) does not match the number of columns in the definition (#{fields.length})", 
+            line, file
+          )
+        end
+      end
+      
+      def configure
+        source.definition.each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options)
+          when Hash
+            fields << Field.new(options[:name])
+          else
+            raise DefinitionError, "Each field definition must either be a symbol or a hash"
+          end
+        end
+      end
+
+      def create_spreadsheet(file)
+        case options[:type]
+        when :excel
+          Excel.new(file)
+        when :excel_x
+          Excelx.new(file)
+        when :open_office
+          OpenOffice.new(file)
+        when :google
+          Google.new(file)
+        else
+          raise 'Spreadsheet parser must be of type :excel, :excel_x, :open_office, or :google'
+        end
+      end
+      
+      class Field #:nodoc:
+        attr_reader :name
+        def initialize(name)
+          @name = name
+        end
+      end
+    end
+  end
+end

data/lib/etl/parser/xml_parser.rb

@@ -0,0 +1,65 @@
+require 'rexml/document'
+
+module ETL
+  module Parser
+    class XmlParser < ETL::Parser::Parser
+      # Initialize the parser
+      # * <tt>source</tt>: The Source object
+      # * <tt>options</tt>: Parser options Hash
+      def initialize(source, options={})
+        super
+        configure
+      end
+      
+      # Returns each row
+      def each
+        Dir.glob(file).each do |file|
+          doc = nil
+          t = Benchmark.realtime do
+            doc = REXML::Document.new(File.new(file))
+          end
+          Engine.logger.info "XML #{file} parsed in #{t}s"
+          doc.elements.each(@collection_xpath) do |element|
+            row = {}
+            fields.each do |f|
+              value = element.text(f.xpath)
+              row[f.name] = value
+            end
+            yield row
+          end
+        end
+      end
+      
+      # Get an array of defined fields
+      def fields
+        @fields ||= []
+      end
+      
+      private
+      def configure
+        @collection_xpath = source.definition[:collection]
+        raise "Collection XPath is required" if @collection_xpath.nil?
+        
+        source.definition[:fields].each do |options|
+          case options
+          when Symbol
+            fields << Field.new(options, options.to_s)
+          when Hash
+            options[:xpath] ||= options[:name]
+            fields << Field.new(options[:name], options[:xpath].to_s)
+          else
+            raise DefinitionError, "Each field definition must either be an symbol or a hash of options for the field"
+          end
+        end
+      end
+      
+      class Field
+        attr_reader :name, :xpath
+        def initialize(name, xpath)
+          @name = name
+          @xpath = xpath
+        end
+      end
+    end
+  end
+end