arboretum 0.0.3 → 0.0.4

data/lib/arboretum/scandent.rb

@@ -0,0 +1,882 @@
+module Arboretum
+  module Scandent
+
+    class TokenizationException < StandardError
+      def initialize(msg="An error occurred while tokenizing input")
+        super(msg)
+      end
+    end
+
+    class ParseException < StandardError
+      def initialize(msg="An error occurred while parsing input")
+        super(msg)
+      end
+    end
+
+    class InvalidExpressionException < StandardError
+      def initialize(msg="Invalid Scandent expression")
+        super(msg)
+      end
+    end
+
+    class ScandentRule
+      attr_accessor :paths
+
+      def initialize(rule_paths)
+        @paths = rule_paths
+      end
+
+      def valid_on?(element)
+        @paths.each {|path| return true if path.valid_on?(element)}
+        return false
+      end
+      alias_method :selects?, :valid_on?
+      alias_method :matches?, :valid_on?
+
+      def to_s
+        rule_str = ''
+        @paths.each do |path|
+          rule_str << ', ' if !rule_str.empty?
+          rule_str << path.to_s
+        end
+        rule_str
+      end
+    end
+
+    class ListenerRule < ScandentRule
+    end
+
+    class LocatorRule < ScandentRule
+      def locate(element)
+        all_located = []
+        @paths.each do |path|
+          path.locate(element) do |located_element|
+            unless all_located.include?(located_element)
+              yield located_element if block_given?
+              all_located << located_element
+            end
+          end
+        end
+        all_located
+      end
+    end
+
+    class ListenerPath
+      attr_accessor :steps
+
+      def initialize(steps=[])
+        # Make sure that the listener stems from the root element
+        if !steps[0].nil? and steps[0].element_ref != :ELEMENT_ROOT
+          if steps[0].element_ref.nil?
+            implied_step = [
+              [:T_TILDE, '~', :STATE_ROOT_PATH],
+              [:T_SLASH2, '//', :STATE_ROOT_PATH]
+            ]
+            steps.unshift(Parser.parse_step_tokens(implied_step, :PATH_LISTENER))
+          else steps[0].element_ref == :ELEMENT_SELF
+            steps[0].element_ref = :ELEMENT_ROOT
+          end
+        end
+        # Make sure there are no misplaced :ELEMENT_SELF references
+        steps.each {|step| raise InvalidExpressionException.new if step.element_ref == :ELEMENT_SELF and !step.eql?(steps.first)}
+
+        @steps = steps
+      end
+
+      def valid_on?(candidate)
+        steps_valid_on?(@steps, candidate)
+      end
+
+      def steps_valid_on?(steps, candidate)
+        return true if steps.empty?
+        steps.last.match(candidate) do |element|
+          return true if steps_valid_on?(steps[0..-2], element)
+        end
+        return false
+      end
+
+      def to_s
+        path_string = ''
+        @steps.each{|step| path_string << step.to_s}
+        path_string
+      end
+
+    end
+
+    class LocatorPath
+      attr_accessor :steps
+
+      def initialize(steps=[])
+        # Make sure that the locators stems from the root or current context element
+        if steps.first.element_ref.nil?
+          steps.first.action = :ACTION_CHILD if steps.first.action == :ACTION_SELF
+          implied_step = [
+            [:T_DOT, '.', :STATE_ROOT_PATH],
+          ]
+          steps.unshift(Parser.parse_step_tokens(implied_step, :PATH_LOCATOR))
+        end
+        # Make sure there are no misplaced :ELEMENT_SELF references
+        steps.each {|step| raise InvalidExpressionException.new if step.element_ref == :ELEMENT_SELF and !step.eql?(steps.first)}
+
+        @steps = steps
+      end
+
+      def locate(element)
+        matches = [element]
+        next_round = []
+        @steps.each do |step|
+          next_round = []
+          matches.each do |matched_element|
+            next_round += step.match(matched_element)
+          end
+          return next_round if next_round.empty?
+          matches = next_round
+        end
+        matches.each {|match| yield match if block_given?}
+        return matches
+      end
+
+      def valid_on?(candidate)
+        steps_valid_on?(@steps, candidate)
+      end
+
+      def steps_valid_on?(steps, candidate)
+        return true if steps.empty?
+        steps.first.match(candidate) do |element|
+          return true if steps_valid_on?(steps[1..-1], element)
+        end
+        return false
+      end
+
+      def to_s
+        path_string = ''
+        @steps.each{|step| path_string << step.to_s}
+        path_string
+      end
+    end
+
+    class PathStep
+      @@inverse_actions = {
+        :ACTION_CHILD             => :ACTION_PARENT,
+        :ACTION_DESCENDANT        => :ACTION_ANCESTOR,
+        :ACTION_PARENT            => :ACTION_CHILD,
+        :ACTION_ANCESTOR          => :ACTION_DESCENDANT,
+        :ACTION_FOLLOWING_SIBLING => :ACTION_PRECEDING_SIBLING,
+        :ACTION_FOLLOWING         => :ACTION_PRECEDING,
+        :ACTION_PRECEDING_SIBLING => :ACTION_FOLLOWING_SIBLING,
+        :ACTION_PRECEDING         => :ACTION_FOLLOWING,
+        :ACTION_SELF              => :ACTION_SELF
+      }
+      @@action_group = {
+        :ACTION_CHILD             => :content,
+        :ACTION_DESCENDANT        => :descendants,
+        :ACTION_PARENT            => :parent,
+        :ACTION_ANCESTOR          => :ancestors,
+        :ACTION_FOLLOWING_SIBLING => :sibling_next,
+        :ACTION_FOLLOWING         => :following_siblings,
+        :ACTION_PRECEDING_SIBLING => :sibling_prev,
+        :ACTION_PRECEDING         => :preceding_siblings,
+        :ACTION_SELF              => :itself
+      }
+      @@action_str = {
+        :ACTION_CHILD             =>    '/'.freeze,
+        :ACTION_DESCENDANT        =>   '//'.freeze,
+        :ACTION_PARENT            =>  '/..'.freeze,
+        :ACTION_ANCESTOR          => '/...'.freeze,
+        :ACTION_FOLLOWING_SIBLING =>   '/>'.freeze,
+        :ACTION_FOLLOWING         =>  '/>>'.freeze,
+        :ACTION_PRECEDING_SIBLING =>   '/<'.freeze,
+        :ACTION_PRECEDING         =>  '/<<'.freeze,
+        :ACTION_SELF              =>     ''.freeze
+      }
+
+      attr_accessor :action, :element_ref, :tag, :namespace, :id, :attrs, :pseudo_exps, :valid_rules
+
+      def initialize(action, element_ref, tag, namespace, id, attrs, pseudo_exps, valid_rules)
+        @action = action
+        @element_ref = element_ref
+        @tag = tag.nil? ? nil : tag.to_sym
+        @namespace = namespace.nil? ? nil : namespace.to_sym
+        @id = id
+        @attrs = attrs
+        @pseudo_exps = pseudo_exps
+        @valid_rules = valid_rules
+      end
+
+      # Check if all fields match that of the given element, with no attention paid to the action
+      # FIXME: Search time improves by 25% when a tag is given. Reject non-tagged Elements more quickly
+      #   to get a similar performance boost across the board
+      def describes?(element)
+        if element.kind_of?(Arboretum::DocTree::Elements::TaggedElement)
+          return false if !@tag.nil? and element.tag != @tag
+          return false if !@namespace.nil? and element.namespace != @namespace
+          return false if !@id.nil? and !element.equals_attr_val?(:id, [@id])
+          @attrs[:has].each {|attr_name| return false if !element.has_attr?(attr_name)}
+          @attrs[:contains].each {|name,values| values.each {|value| return false if !element.contains_attr_val?(name,value)}}
+          @attrs[:equals].each {|name,values| values.each {|value| return false if !element.equals_attr_val?(name,value)}}
+          @attrs[:matches].each {|name,values| values.each {|value| return false if !element.matches_attr_val?(name,value)}}
+          return false if element_ref == :ELEMENT_ROOT and !element.parent.nil?
+        else
+          return false if !@tag.nil? or
+                          !@namespace.nil? or
+                          !@id.nil? or
+                          !@attrs[:has].empty? or
+                          !@attrs[:contains].empty? or
+                          !@attrs[:equals].empty? or
+                          !@attrs[:matches].empty? or
+                          (element_ref == :ELEMENT_ROOT and !element.parent.nil?)
+        end
+        @pseudo_exps.each do |psuedo_name, pseudo_arg|
+          return false if !PseudoElements.match(element, pseudo_name, pseudo_arg)
+        end
+        @valid_rules.each do |rule|
+          return false if !rule.valid_on?(element)
+        end
+        return true
+      end
+
+      def to_s_sans_action
+        step_str = ''
+        if !@element_ref.nil?
+          if @element_ref == :ELEMENT_ROOT
+            step_str << '~'
+          elsif @element_ref == :ELEMENT_SELF
+            step_str << '.'
+          end
+        end
+        step_str << "%#{@tag.to_s}" if !tag.nil?
+        step_str << "@#{@namespace.to_s}" if !namespace.nil?
+        step_str << "##{@id}" if !@id.nil?
+        @attrs[:has].each {|attr_name| step_str << "[#{attr_name}]"}
+        @attrs[:contains].each {|name,values| values.each {|value| step_str << "[#{name}=\"#{value}\"]"}}
+        @attrs[:equals].each {|name,values| values.each {|value| step_str << "[#{name}==\"#{value.join(' ')}\"]"}}
+        @attrs[:matches].each {|name,values| values.each {|value| step_str << "[#{name}~=|/#{value}/|]"}}
+        @valid_rules.each {|rule| step_str << "{#{rule.to_s}}"}
+        @pseudo_exps.each {|pseudo, arg| step_str << ":#{pseudo}(#{arg.to_s})"}
+        step_str
+      end
+    end
+
+    class ListenerStep < PathStep
+      # Take step action and check for elements matching description
+      def match(element)
+        result = []
+        inverse_action = @@inverse_actions[@action]
+        search_group = element.public_send(@@action_group[inverse_action])
+        search_group.listing.each do |searched_element|
+          yield searched_element if self.describes?(searched_element) and block_given?
+          result << searched_element if self.describes?(searched_element)
+        end
+        result
+      end
+
+      def to_s
+        step_str = self.to_s_sans_action
+        step_str << @@action_str[@action]
+      end
+    end
+
+    class LocatorStep < PathStep
+      # Take step action and check for elements matching description
+      def match(element)
+        result = []
+        search_group = element.public_send(@@action_group[@action])
+        search_group.listing.each do |searched_element|
+          yield searched_element if self.describes?(searched_element) and block_given?
+          result << searched_element if self.describes?(searched_element)
+        end
+        result
+      end
+
+      def to_s
+        step_str = '' # Don't copy reference to @@action_str[@action]
+        step_str << @@action_str[@action]
+        step_str << self.to_s_sans_action
+      end
+    end
+
+    class Formula
+      def initialize(form_tokens)
+        @coefficient = 0
+        @intercept = 0
+        form_token_types = form_tokens.map {|token| token[0]}
+        if form_token_types[0] == :T_KEY_EVEN
+          @coefficient = 2
+          @intercept = 0
+        elsif form_token_types[0] == :T_KEY_ODD
+          @coefficient = 2
+          @intercept = 1
+        else
+          term_negative = false
+          term_coef = false
+          value = 0
+
+          index = 0
+          while index < form_tokens.length
+            case form_token_types[index]
+            when :T_FORM_PLUS
+              # Resolve Term
+              if term_coef
+                @coefficient += value
+              else
+                @intercept += value
+              end
+              # Reset with new sign
+              term_negative = false
+              term_coef = false
+              value = 0
+            when :T_FORM_MINUS
+              # Resolve Term
+              if term_coef
+                @coefficient += value
+              else
+                @intercept += value
+              end
+              # Reset with new sign
+              term_negative = true
+              term_coef = false
+              value = 0
+            when :LITERAL_INT
+              value = form_tokens[index][1].to_i
+              value *= -1 if term_negative
+            when :T_FORM_N
+              term_coef = true
+              value = 1 if value.zero?
+            else
+              raise InvalidExpressionException.new
+            end
+            index += 1
+          end
+          # Resolve one last time
+          if term_coef
+            @coefficient += value
+          else
+            @intercept += value
+          end
+        end
+      end
+
+      def to_s
+        "#{@coefficient}n#{'+' if @intercept >= 0}#{@intercept}"
+      end
+    end
+
+    class PseudoElements
+      def self.match(element, pseudo_name, pseudo_arg)
+        PseudoElements.public_send(:pseudo_name, element, pseudo_arg)
+      end
+    end
+
+    # Parser interprets tokens formed by a Scandent string and formed ScandentRules
+    # that represent the interpreted form of the string
+    # ScandentRules can then be matched to Elements in a DocTree
+    class Parser
+      @@actions = {
+        :T_SLASH => :ACTION_CHILD,
+        :T_SLASH2 => :ACTION_DESCENDANT,
+        :T_SLASHDOT2 => :ACTION_PARENT,
+        :T_SLASHDOT3 => :ACTION_ANCESTOR,
+        :T_SLASHGT => :ACTION_FOLLOWING_SIBLING,
+        :T_SLASHGT2 => :ACTION_FOLLOWING,
+        :T_SLASHLT => :ACTION_PRECEDING_SIBLING,
+        :T_SLASHLT2 => :ACTION_PRECEDING
+      }
+
+      # Parse a Scandent string by giving it to the Tokenizer and then parsing the results
+      def self.parse_rule_string(rule_string, type)
+        Parser.parse_rule_tokens(Tokenizer.tokenize(rule_string), type)
+      end
+
+      # Directly parse Scandent string tokens
+      def self.parse_rule_tokens(rule_tokens, type)
+        # Separate rules into its comma-delimited paths and remove the delimiter
+        rule_paths_tokens = rule_tokens.slice_after {|token| token[0] == :T_COMMA and token[2] == :STATE_ROOT_PATH}.to_a
+        rule_paths_tokens.each {|path_tokens| path_tokens.pop if path_tokens.last[0] == :T_COMMA}
+
+        # Parse each path individually
+        rule_paths = rule_paths_tokens.map{|path| Parser.parse_path_tokens(path, type)}
+
+        if type == :PATH_LISTENER
+          ListenerRule.new(rule_paths)
+        elsif type == :PATH_LOCATOR
+          LocatorRule.new(rule_paths)
+        else
+          raise ParseException.new("Unknown step type")
+        end
+      end
+
+      # Parse an individual path of a rule
+      def self.parse_path_tokens(path_tokens, type)
+        # Double check that the end state of the path is valid
+        raise InvalidExpressionException.new("End state of path is '#{path_tokens.last[2]}' instead of :STATE_ROOT_PATH") if path_tokens.last[2] != :STATE_ROOT_PATH
+
+        step_delimiters = [:T_SLASH, :T_SLASH2, :T_SLASHDOT2, :T_SLASHDOT3, :T_SLASHGT, :T_SLASHGT2, :T_SLASHLT, :T_SLASHLT2]
+
+        # Seperate paths into its steps
+        if type == :PATH_LISTENER
+          path_steps_tokens = path_tokens.slice_after {|token| step_delimiters.include? token[0] and token[2] == :STATE_ROOT_PATH}.to_a
+        elsif type == :PATH_LOCATOR
+          path_steps_tokens = path_tokens.slice_before {|token| step_delimiters.include? token[0] and token[2] == :STATE_ROOT_PATH}.to_a
+        else
+          raise ParseException.new("Unknown step type")
+        end
+
+        # Parse each step individually
+        path_steps = path_steps_tokens.map{|step| Parser.parse_step_tokens(step, type)}
+
+        if type == :PATH_LISTENER
+          ListenerPath.new(path_steps)
+        else
+          LocatorPath.new(path_steps)
+        end
+      end
+
+      # Parse an individual step of a path
+      def self.parse_step_tokens(step_tokens, type)
+        # Remove whitespace tokens
+        step_tokens.delete_if {|token| token[0] == :T_WHITESPACE}
+
+        if type == :PATH_LISTENER
+          action = @@actions.has_key?(step_tokens.last[0]) ? @@actions[step_tokens.last[0]] : :ACTION_SELF
+        elsif type == :PATH_LOCATOR
+          action = @@actions.has_key?(step_tokens.first[0]) ? @@actions[step_tokens.first[0]] : :ACTION_SELF
+        else
+          raise ParseException.new("Unknown step type")
+        end
+
+        element_ref = []
+        tag = []
+        namespace = []
+        id = []
+        attrs = {
+          :has => [],
+          :contains => Hash.new{|k, v| k[v] = Array.new},
+          :equals => Hash.new{|k, v| k[v] = Array.new},
+          :matches => Hash.new{|k, v| k[v] = Array.new}
+        }
+        pseudo_exps = []
+        valid_rules = []
+
+        index = 0
+        state = :STATE_ROOT_PATH
+        while index < step_tokens.length
+          # Consume current token and increment
+          index_token = step_tokens[index]
+          index += 1
+
+          case index_token[0]
+          when :T_PCT
+            following_token = step_tokens[index]
+            index += 1
+            raise InvalidExpressionException.new if following_token[0] != :LITERAL_IDENT
+            tag << following_token[1]
+          when :T_AT
+            following_token = step_tokens[index]
+            index += 1
+            raise InvalidExpressionException.new if following_token[0] != :LITERAL_IDENT
+            namespace << following_token[1]
+          when :T_PND
+            following_token = step_tokens[index]
+            index += 1
+            raise InvalidExpressionException.new if following_token[0] != :LITERAL_IDENT
+            id << following_token[1]
+          when :T_COLON
+            following_token = step_tokens[index]
+            index += 1
+            raise InvalidExpressionException.new if following_token[0] != :LITERAL_IDENT
+            pseudo_name = following_token[1].to_sym
+            arg_tokens = []
+
+            following_token = step_tokens[index]
+            if following_token[0] == :T_LPAREN
+              index += 1 # To consume the LPAREN
+              following_token = step_tokens[index]
+              index += 1
+              until following_token[0] == :T_RPAREN or index > step_tokens.length
+                arg_tokens << following_token
+                following_token = step_tokens[index]
+                index += 1
+              end
+              raise InvalidExpressionException.new if index > step_tokens.length # Undesirable exit condition to above loop
+              index += 1 # To consume the RPAREN
+            end
+            pseudo_exps << [pseudo_name, Parser.parse_arg(arg_tokens)]
+          when :T_ASTERISK # Adds no restrictions, so do nothing
+          when :T_TILDE
+            element_ref << :ELEMENT_ROOT
+          when :T_DOT
+            element_ref << :ELEMENT_SELF
+          when :T_LBRAK
+            following_token = step_tokens[index]
+            index += 1
+            raise InvalidExpressionException.new if following_token[0] != :LITERAL_IDENT
+            attr_name = following_token[1].to_sym
+            attr_value = nil
+            operation = nil
+
+            following_token = step_tokens[index]
+            index += 1
+            case following_token[0]
+            when :T_EQL
+              operation = :contains
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new("Expected a string after '='") if ![:T_DQUOTE, :T_SQUOTE].include?(following_token[0])
+              string_limiter = following_token[0]
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if not [:LITERAL_IDENT, :LITERAL_STRING, :LITERAL_INT, :LITERAL_FLOAT].include?(following_token[0])
+              attr_value = following_token[1]
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if following_token[0] != string_limiter
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if following_token[0] != :T_RBRAK
+            when :T_EQL2
+              operation = :equals
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if ![:T_DQUOTE, :T_SQUOTE].include?(following_token[0])
+              string_limiter = following_token[0]
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if not [:LITERAL_IDENT, :LITERAL_STRING, :LITERAL_INT, :LITERAL_FLOAT].include?(following_token[0])
+              attr_value = following_token[1].split
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if following_token[0] != string_limiter
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if following_token[0] != :T_RBRAK
+            when :T_TILDE_EQL
+              operation = :matches
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if following_token[0] != :T_VBARSLASH
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if not [:LITERAL_IDENT, :LITERAL_STRING, :LITERAL_INT, :LITERAL_FLOAT].include?(following_token[0])
+              attr_value = Regexp.new(following_token[1])
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if following_token[0] != :T_SLASHVBAR
+
+              following_token = step_tokens[index]
+              index += 1
+              raise InvalidExpressionException.new if following_token[0] != :T_RBRAK
+            when :T_RBRAK
+              operation = nil
+            else
+              raise InvalidExpressionException.new
+            end
+            if operation.nil?
+              attrs[:has] << attr_name
+            else
+              attrs[operation][attr_name] << attr_value
+            end
+          when :T_LBRACE
+            equilibrium = 1
+            reformed_path_string = ''
+            following_token = step_tokens[index]
+            index += 1
+            until (following_token[0] == :T_RBRACE and equilibrium.zero?) or index > step_tokens.length
+              reformed_path_string << following_token[1]
+              following_token = step_tokens[index]
+              index += 1
+              equilibrium += 1 if following_token[0] == :T_LBRACE
+              equilibrium -= 1 if following_token[0] == :T_RBRACE
+            end
+            raise InvalidExpressionException.new("Could not find matching R_BRACE in #{reformed_path_string}") if index > step_tokens.length # Undesirable exit condition to above loop
+            valid_rules << Parser.parse_rule_string(reformed_path_string, :PATH_LOCATOR)
+          when :T_SLASH, :T_SLASH2, :T_SLASHDOT2, :T_SLASHDOT3, :T_SLASHGT, :T_SLASHGT2, :T_SLASHLT, :T_SLASHLT2
+          else
+            raise ParseException.new("Consumed unexpected token: #{index_token}")
+          end
+        end # All tokens consumed
+
+        # Validate results
+        raise InvalidExpressionException.new if tag.length > 1
+        raise InvalidExpressionException.new if namespace.length > 1
+        raise InvalidExpressionException.new if element_ref.length > 1
+
+        if type == :PATH_LISTENER
+          ListenerStep.new(action, element_ref[0], tag[0], namespace[0], id[0], attrs, pseudo_exps, valid_rules)
+        else
+          LocatorStep.new(action, element_ref[0], tag[0], namespace[0], id[0], attrs, pseudo_exps, valid_rules)
+        end
+      end
+
+      # Parse an argument given to a pseudo-class
+      def self.parse_arg(arg_tokens)
+        arg_token_types = arg_tokens.map {|token| token[0]}
+        if ((arg_token_types[0] == :T_SQUOTE and arg_token_types[2] == :T_SQUOTE) or (arg_token_types[0] == :T_DQUOTE and arg_token_types[2] == :T_DQUOTE)) and arg_tokens.length == 3
+          arg_tokens[1][1]
+        elsif (arg_token_types[0] == :T_VBARSLASH and arg_token_types[2] == :T_SLASHVBAR) and arg_tokens.length == 3
+          Regexp.new(arg_tokens[1][1])
+        elsif (!(arg_token_types & [:T_KEY_EVEN, :T_KEY_ODD]).empty? and arg_tokens.length == 1) or arg_token_types.include? :T_FORM_N
+          Formula.new(arg_tokens)
+        elsif arg_token_types[0] == :LITERAL_INT and arg_tokens.length == 1
+          arg_tokens[0][1].to_i
+        elsif arg_token_types[0] == :LITERAL_FLOAT and arg_tokens.length == 1
+          arg_tokens[0][1].to_f
+        elsif arg_tokens.length.zero?
+          nil
+        else
+          arg_str = ''
+          arg_tokens.each {|token| arg_str << token[1]}
+          raise InvalidExpressionException.new("Invalid argument '#{arg_str}'")
+          nil
+        end
+      end
+
+    end # Parser
+
+    # A class with class methods used to tokenize a Scandent string
+    # Has information regarding which character patterns match which tokens
+    # Has information regarding which tokens trigger which state in the tokenizer
+    class Tokenizer
+      # A Hash with keys cooresponding to states, and value Hashes that match patterns in that state to tokens
+      @@tokens = {
+        :STATE_ROOT_PATH => {
+          ','   => :T_COMMA,
+          '~'   => :T_TILDE,
+          '//'  => :T_SLASH2,
+          '/'   => :T_SLASH,
+          '.'   => :T_DOT,
+          '/..'  => :T_SLASHDOT2,
+          '/...' => :T_SLASHDOT3,
+          '/>'  => :T_SLASHGT,
+          '/>>' => :T_SLASHGT2,
+          '/<'  => :T_SLASHLT,
+          '/<<' => :T_SLASHLT2,
+          '*'   => :T_ASTERISK,
+          '{'   => :T_LBRACE,
+          '['   => :T_LBRAK,
+          '('   => :T_LPAREN,
+          ':'   => :T_COLON,
+          '#'   => :T_PND,
+          '%'   => :T_PCT,
+          '@'   => :T_AT,
+          # Whitespace
+          "\n"  => :T_WHITESPACE,
+          "\t"  => :T_WHITESPACE,
+          ' '   => :T_WHITESPACE
+        },
+        :STATE_SSTRING => {
+          '\''  => :T_SQUOTE # Closing Token
+        },
+        :STATE_DSTRING => {
+          '"'  => :T_DQUOTE, # Closing Token
+        },
+        :STATE_REGEX => {
+          '/|'  => :T_SLASHVBAR # Closing Token
+        },
+        :STATE_ATTR_EXP => {
+          ']'   => :T_RBRAK, # Closing Token
+          '='   => :T_EQL,
+          '=='  => :T_EQL2,
+          '~='  => :T_TILDE_EQL,
+          '"'  => :T_DQUOTE,
+          '\''  => :T_SQUOTE,
+          '|/'  => :T_VBARSLASH,
+          # Whitespace
+          "\n"  => :T_WHITESPACE,
+          "\t"  => :T_WHITESPACE,
+          ' '   => :T_WHITESPACE
+        },
+        :STATE_PATH_EXP => {
+          '}'   => :T_RBRACE, # Closing Token
+          # From ROOT_PATH
+          ','   => :T_COMMA,
+          '~'   => :T_TILDE,
+          '//'  => :T_SLASH2,
+          '/'   => :T_SLASH,
+          '.'   => :T_DOT,
+          '/..'  => :T_SLASHDOT2,
+          '/...' => :T_SLASHDOT3,
+          '/>'  => :T_SLASHGT,
+          '/>>' => :T_SLASHGT2,
+          '/<'  => :T_SLASHLT,
+          '/<<' => :T_SLASHLT2,
+          '*'   => :T_ASTERISK,
+          '{'   => :T_LBRACE,
+          '['   => :T_LBRAK,
+          '('   => :T_LPAREN,
+          ':'   => :T_COLON,
+          '#'   => :T_PND,
+          '%'   => :T_PCT,
+          '@'   => :T_AT,
+          # Whitespace
+          "\n"  => :T_WHITESPACE,
+          "\t"  => :T_WHITESPACE,
+          ' '   => :T_WHITESPACE
+        },
+        :STATE_ARGS => {
+          ')'     => :T_RPAREN, # Closing Token
+          '|/'    => :T_VBARSLASH,
+          'n'     => :T_FORM_N,
+          '+'     => :T_FORM_PLUS,
+          '-'     => :T_FORM_MINUS,
+          'even'  => :T_KEY_EVEN,
+          'odd'   => :T_KEY_ODD,
+          '"'  => :T_DQUOTE,
+          '\''  => :T_SQUOTE,
+          # Whitespace
+          "\n"  => :T_WHITESPACE,
+          "\t"  => :T_WHITESPACE,
+          ' '   => :T_WHITESPACE
+        },
+      }
+      # A Hash with keys cooresponding to states, and value Hashes that describe what tokens trigger the closing
+      # of the *current* state or the opening of a *different* state
+      @@triggers = {
+        :STATE_ROOT_PATH => {
+          :open => {
+            :T_LBRACE => :STATE_PATH_EXP,
+            :T_LBRAK => :STATE_ATTR_EXP,
+            :T_LPAREN => :STATE_ARGS
+          },
+          :close => []
+        },
+        :STATE_SSTRING => {
+          :open => {},
+          :close => [:T_SQUOTE]
+        },
+        :STATE_DSTRING => {
+          :open => {},
+          :close => [:T_DQUOTE]
+        },
+        :STATE_REGEX => {
+          :open => {},
+          :close => [:T_SLASHVBAR]
+        },
+        :STATE_ATTR_EXP => {
+          :open => {
+            :T_SQUOTE => :STATE_SSTRING,
+            :T_DQUOTE => :STATE_DSTRING,
+            :T_VBARSLASH => :STATE_REGEX
+          },
+          :close => [:T_RBRAK]
+        },
+        :STATE_PATH_EXP => {
+          :open => {
+            :T_LBRACE => :STATE_PATH_EXP,
+            :T_LBRAK => :STATE_ATTR_EXP,
+            :T_LPAREN => :STATE_ARGS
+          },
+          :close => [:T_RBRACE]
+        },
+        :STATE_ARGS => {
+          :open => {
+            :T_SQUOTE => :STATE_SSTRING,
+            :T_DQUOTE => :STATE_DSTRING,
+            :T_VBARSLASH => :STATE_REGEX
+          },
+          :close => [:T_RPAREN]
+        },
+      }
+
+      # Returns the type of a string of characters that does not match a pattern
+      def self.literal_type(literal)
+        return :LITERAL_IDENT if literal =~ /^[[:alpha:]][[:alnum:]-_]*$/
+        return :LITERAL_INT if literal =~ /^\d+$/
+        return :LITERAL_FLOAT if literal =~ /^\d*\.?\d+$/
+        return :LITERAL_STRING if literal =~ /^.+$/
+        return :LITERAL_UNKNOWN
+      end
+
+      # If no matches found for a letter, move to next letter, even if the possible patterns would match more
+      def self.tokenize(input)
+        state = [:STATE_ROOT_PATH] # State stack for the tokenizer, state.last will return the current state
+        match_start = 0
+        match_end = 0
+        unmatched_buffer = ''
+        largest_full_match = nil
+        token_list = [] # The list of tokens in the input, each item is an Array in the form: 
+                        # [Token type, Pattern that matched, State of the tokenizer after the token]
+
+        # Until we have checked and matched at or on every single character
+        while match_start < input.length
+          # Start building a substring from a single character
+          match_end = match_start
+          # Start will all patterns for current state as candidates
+          candidates = @@tokens[state.last].keys
+          # Start will no full match detected
+          largest_full_match = nil
+
+          # Until nothing can match substring or end of input has been reached
+          until candidates.empty? or match_end >= input.length
+            # String that candidate patterns will have to match
+            matched_string = input[match_start..match_end]
+            # Check to see if each remaining candidate pattern matches
+            #  - If a full match, set as largest full match
+            #  - Delete if the pattern does not match the string
+            candidates.delete_if do |pattern|
+              largest_full_match = [pattern, match_start, match_end] if pattern.eql?(matched_string)
+              !pattern.start_with?(matched_string) # Element deleted if true is the last statement in block
+            end
+            # Increase size of match by one if further matching is to be done
+            match_end += 1 if not candidates.empty?
+          end # Substring is now one character too large to be matched to
+
+          # If no full match found, add the last checked character as unmatched
+          # Otherwise:
+          #  - Parse the unmatched_buffer as a literal and store
+          #  - Activate state triggers associated with the largest fully matched token
+          #  - Store the largest fully matched token
+          #  - Start again where the matched token completes
+          if largest_full_match.nil?
+            # Add last checked character to the unmatched buffer
+            unmatched_buffer << input[match_start]
+            # Start matching again on the next letter
+            match_start += 1
+          else
+
+            # Do not activate state triggers associate with the parsed literal
+            # Is there a use case?
+
+            # Parse the unmatched_buffer as a literal and store (if it exists), then clear the buffer
+            token_list << [Tokenizer.literal_type(unmatched_buffer), unmatched_buffer, state.last] if unmatched_buffer.length > 0
+            unmatched_buffer = ''
+
+            # Info from the largest fully matched token
+            matched_pattern, pattern_start, pattern_end = largest_full_match
+            matched_token_type = @@tokens[state.last][matched_pattern]
+
+            # Activate state triggers associated with the largest fully matched token
+            current_state_triggers = @@triggers[state.last]
+            if current_state_triggers[:open].has_key?(matched_token_type)
+              state.push(current_state_triggers[:open][matched_token_type])
+            elsif current_state_triggers[:close].include?(matched_token_type)
+              state.pop
+            end
+
+            # Store the largest fully matched token
+            token_list << [matched_token_type, matched_pattern, state.last]
+
+            # Start again where the matched token completes
+            match_start = pattern_end + 1
+
+          end # Next token has been added to list
+        end # Input has been fully tokenized
+
+        # Parse and store the unmatched_buffer one last time
+        token_list << [Tokenizer.literal_type(unmatched_buffer), unmatched_buffer, state.last] if unmatched_buffer.length > 0
+
+        token_list
+      end # tokenize_path
+
+    end # Tokenizer
+  end # Scandent
+end # Arboretum