rouge 3.5.1 → 3.10.0

data/lib/rouge/lexers/scala.rb

@@ -78,7 +78,7 @@ module Rouge
           (#{keywords.join("|")})\b|
           (<[%:-]|=>|>:|[#=@_\u21D2\u2190])(\b|(?=\s)|$)
         )x, Keyword
-        rule %r/:(?!#{op})/, Keyword, :type        
+        rule %r/:(?!#{op})/, Keyword, :type
         rule %r/(true|false|null)\b/, Keyword::Constant
         rule %r/(import|package)(\s+)/ do
           groups Keyword, Text

data/lib/rouge/lexers/sed.rb

@@ -141,7 +141,7 @@ module Rouge
         end
 
         # alternate regex rage delimiters
-        rule %r((\\)(.)(\\.|.)*?(\2)) do |m|
+        rule %r((\\)(.)((?:\\.|.)*?)(\2)) do |m|
           token addr_tok, m[1] + m[2]
           delegate regex, m[3]
           token addr_tok, m[4]

data/lib/rouge/lexers/shell.rb

@@ -52,7 +52,7 @@ module Rouge
         rule %r/\b(#{BUILTINS})\s*\b(?!(\.|-))/, Name::Builtin
         rule %r/[.](?=\s)/, Name::Builtin
 
-        rule %r/(\b\w+)(=)/ do |m|
+        rule %r/(\b\w+)(=)/ do
           groups Name::Variable, Operator
         end
 

data/lib/rouge/lexers/smarty.rb

@@ -64,18 +64,18 @@ module Rouge
         rule %r/#[a-zA-Z_]\w*#/, Name::Variable
         rule %r/\$[a-zA-Z_]\w*(\.\w+)*/, Name::Variable
         rule %r/(true|false|null)\b/, Keyword::Constant
-	rule %r/[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?/, Num
-	rule %r/"(\\.|.)*?"/, Str::Double
+        rule %r/[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|0[xX][0-9a-fA-F]+[Ll]?/, Num
+        rule %r/"(\\.|.)*?"/, Str::Double
         rule %r/'(\\.|.)*?'/, Str::Single
-	rule %r/([a-zA-Z_]\w*)/ do |m|
-	  if self.class.builtins.include? m[0]
-	    token Name::Builtin
-	  else
-	    token Name::Attribute
-	  end
-	end
-      end
 
+        rule %r/([a-zA-Z_]\w*)/ do |m|
+          if self.class.builtins.include? m[0]
+            token Name::Builtin
+          else
+            token Name::Attribute
+          end
+        end
+      end
     end
   end
 end

data/lib/rouge/lexers/sparql.rb

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class SPARQL < RegexLexer
+      title "SPARQL"
+      desc "Semantic Query Language, for RDF data"
+      tag 'sparql'
+      filenames '*.rq'
+      mimetypes 'application/sparql-query'
+
+      def self.builtins
+        @builtins = Set.new %w[
+          ABS AVG BNODE BOUND CEIL COALESCE CONCAT CONTAINS COUNT DATATYPE DAY
+          ENCODE_FOR_URI FLOOR GROUP_CONCAT HOURS IF IRI isBLANK isIRI
+          isLITERAL isNUMERIC isURI LANG LANGMATCHES LCASE MAX MD5 MIN MINUTES
+          MONTH NOW RAND REGEX REPLACE ROUND SAMETERM SAMPLE SECONDS SEPARATOR
+          SHA1 SHA256 SHA384 SHA512 STR STRAFTER STRBEFORE STRDT STRENDS
+          STRLANG STRLEN STRSTARTS STRUUID SUBSTR SUM TIMEZONE TZ UCASE URI
+          UUID YEAR
+        ]
+      end
+
+      def self.keywords
+        @keywords = Set.new %w[
+          ADD ALL AS ASC ASK BASE BIND BINDINGS BY CLEAR CONSTRUCT COPY CREATE
+          DATA DEFAULT DELETE DESC DESCRIBE DISTINCT DROP EXISTS FILTER FROM
+          GRAPH GROUP BY HAVING IN INSERT LIMIT LOAD MINUS MOVE NAMED NOT
+          OFFSET OPTIONAL ORDER PREFIX SELECT REDUCED SERVICE SILENT TO UNDEF
+          UNION USING VALUES WHERE WITH
+        ]
+      end
+
+      state :root do
+        rule %r(\s+)m, Text::Whitespace
+        rule %r(#.*), Comment::Single
+        
+        rule %r("""), Str::Double, :string_double_literal
+        rule %r("), Str::Double, :string_double
+        rule %r('''), Str::Single, :string_single_literal
+        rule %r('), Str::Single, :string_single
+       
+        rule %r([$?]\w+), Name::Variable
+        rule %r((\w*:)(\w+)?) do |m|
+          token Name::Namespace, m[1]
+          token Str::Symbol, m[2]
+        end
+        rule %r(<[^>]*>), Name::Namespace
+        rule %r(true|false)i, Keyword::Constant
+        
+        rule %r([A-Z]\w+\b)i do |m|
+          if self.class.builtins.include? m[0].upcase
+            token Name::Builtin
+          elsif self.class.keywords.include? m[0].upcase
+            token Keyword
+          else
+            token Error
+          end
+        end
+
+        rule %r([+\-]?(?:\d+\.\d*|\.\d+)(?:[e][+\-]?[0-9]+)?)i, Num::Float
+        rule %r([+\-]?\d+), Num::Integer
+        rule %r([\]\[(){}.,;=]), Punctuation
+        rule %r([/?*+=!<>]|&&|\|\||\^\^), Operator
+      end
+
+      state :string_double_common do
+        mixin :string_escapes
+        rule %r(\\), Str::Double
+        rule %r([^"\\]+), Str::Double
+      end
+
+      state :string_double do
+        rule %r(") do
+          token Str::Double
+          goto :string_end
+        end
+        mixin :string_double_common
+      end
+
+      state :string_double_literal do
+        rule %r(""") do
+          token Str::Double
+          goto :string_end
+        end
+        rule %r("), Str::Double
+        mixin :string_double_common
+      end
+
+      state :string_single_common do
+        mixin :string_escapes
+        rule %r(\\), Str::Single
+        rule %r([^'\\]+), Str::Single
+      end
+
+      state :string_single do
+        rule %r(') do
+          token Str::Single
+          goto :string_end
+        end
+        mixin :string_single_common
+      end
+      
+      state :string_single_literal do
+        rule %r(''') do
+          token Str::Single
+          goto :string_end
+        end
+        rule %r('), Str::Single
+        mixin :string_single_common
+      end
+
+      state :string_escapes do
+        rule %r(\\[tbnrf"'\\]), Str::Escape
+        rule %r(\\u\h{4}), Str::Escape
+        rule %r(\\U\h{8}), Str::Escape
+      end
+
+      state :string_end do
+        rule %r((@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)) do
+          groups Operator, Name::Property
+        end
+        rule %r(\^\^), Operator
+        rule(//) { pop! }
+      end
+    end
+  end
+end

data/lib/rouge/lexers/sql.rb

@@ -51,7 +51,7 @@ module Rouge
           LOCALTIME LOCALTIMESTAMP LOCATION LOCATOR LOCK LOWER MAP MATCH
           MAX MAXVALUE MESSAGE_LENGTH MESSAGE_OCTET_LENGTH MESSAGE_TEXT
           METHOD MIN MINUTE MINVALUE MOD MODE MODIFIES MODIFY MONTH
-          MORE MOVE MUMPS NAMES NATIONAL NATURAL NCHAR NCLOB NEW NEXT
+          MORE MOVE MUMPS NAMES NATURAL NCLOB NEW NEXT
           NO NOCREATEDB NOCREATEUSER NONE NOT NOTHING NOTIFY NOTNULL
           NULL NULLABLE NULLIF OBJECT OCTET_LENGTH OF OFF OFFSET OIDS
           OLD ON ONLY OPEN OPERATION OPERATOR OPTION OPTIONS OR ORDER
@@ -59,7 +59,7 @@ module Rouge
           OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME
           PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG
           PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PASCAL
-          PENDANT PLACING PLI POSITION POSTFIX PRECISION PREFIX PREORDER
+          PENDANT PLACING PLI POSITION POSTFIX PREFIX PREORDER
           PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE
           PUBLIC READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING
           REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART
@@ -75,7 +75,7 @@ module Rouge
           STDIN STDOUT STORAGE STRICT STRUCTURE STYPE SUBCLASS_ORIGIN
           SUBLIST SUBSTRING SUM SYMMETRIC SYSID SYSTEM SYSTEM_USER
           TABLE TABLE_NAME  TEMP TEMPLATE TEMPORARY TERMINATE THAN THEN
-          TIMESTAMP TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING
+          TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING
           TRANSATION TRANSACTIONS_COMMITTED TRANSACTIONS_ROLLED_BACK
           TRANSATION_ACTIVE TRANSFORM TRANSFORMS TRANSLATE TRANSLATION
           TREAT TRIGGER TRIGGER_CATALOG TRIGGER_NAME TRIGGER_SCHEMA TRIM
@@ -84,21 +84,41 @@ module Rouge
           USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME
           USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES
           VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE
-          WITH WITHOUT WORK WRITE YEAR ZONE
+          WITH WITHOUT WORK WRITE ZONE
         )
       end
 
+      def self.keywords_type
+        # sources:
+        # https://dev.mysql.com/doc/refman/5.7/en/numeric-type-overview.html
+        # https://dev.mysql.com/doc/refman/5.7/en/date-and-time-type-overview.html
+        # https://dev.mysql.com/doc/refman/5.7/en/string-type-overview.html
+        @keywords_type ||= Set.new(%w(
+            ZEROFILL UNSIGNED SIGNED SERIAL BIT TINYINT BOOL BOOLEAN SMALLINT
+            MEDIUMINT INT INTEGER BIGINT DECIMAL DEC NUMERIC FIXED FLOAT DOUBLE
+            PRECISION REAL
+            DATE DATETIME TIMESTAMP TIME YEAR
+            NATIONAL CHAR CHARACTER NCHAR BYTE
+            VARCHAR VARYING BINARY VARBINARY TINYBLOB TINYTEXT BLOB TEXT
+            MEDIUMBLOB MEDIUMTEXT LONGBLOB LONGTEXT ENUM
+        ))
+      end
+
       state :root do
         rule %r/\s+/m, Text
         rule %r/--.*/, Comment::Single
         rule %r(/\*), Comment::Multiline, :multiline_comments
         rule %r/\d+/, Num::Integer
         rule %r/'/, Str::Single, :single_string
+        # A double-quoted string refers to a database object in our default SQL
+        # dialect, which is apropriate for e.g. MS SQL and PostgreSQL.
         rule %r/"/, Name::Variable, :double_string
         rule %r/`/, Name::Variable, :backtick
 
-        rule %r/\w\w*/ do |m|
-          if self.class.keywords.include? m[0].upcase
+        rule %r/\w[\w\d]*/ do |m|
+          if self.class.keywords_type.include? m[0].upcase
+            token Name::Builtin
+          elsif self.class.keywords.include? m[0].upcase
             token Keyword
           else
             token Name

data/lib/rouge/lexers/swift.rb

@@ -97,13 +97,13 @@ module Rouge
             groups Keyword::Declaration, Error, Keyword::Declaration
           end
         end
-        
+
         rule %r/#available\([^)]+\)/, Keyword::Declaration
-        
+
         rule %r/(#(?:selector|keyPath)\()([^)]+?(?:[(].*?[)])?)(\))/ do
           groups Keyword::Declaration, Name::Function, Keyword::Declaration
         end
-        
+
         rule %r/#(line|file|column|function|dsohandle)/, Keyword::Declaration
 
         rule %r/(let|var)\b(\s*)(#{id})/ do
@@ -122,7 +122,7 @@ module Rouge
             token Name::Function
           end
         end
-        
+
         rule %r/as[?!]?(?=\s)/, Keyword
         rule %r/try[!]?(?=\s)/, Keyword
 

data/lib/rouge/lexers/terraform.rb

@@ -37,6 +37,14 @@ module Rouge
         @builtins ||= %w()
       end
 
+      prepend :hash do
+        rule %r/[.,()*]/, Punctuation
+      end
+
+      prepend :array do
+        rule %r/[.,()*]/, Punctuation
+      end
+
       state :strings do
         rule %r/\\./, Str::Escape
         rule %r/\$\{/ do

data/lib/rouge/lexers/tex.rb

@@ -30,7 +30,7 @@ module Rouge
         rule %r/\$/, Punctuation, :inlinemath
         rule %r/\\(begin|end)\{.*?\}/, Name::Tag
 
-        rule %r/(\\verb)\b(\S)(.*?)(\2)/ do |m|
+        rule %r/(\\verb)\b(\S)(.*?)(\2)/ do
           groups Name::Builtin, Keyword::Pseudo, Str::Other, Keyword::Pseudo
         end
 

data/lib/rouge/lexers/toml.rb

@@ -8,7 +8,7 @@ module Rouge
       desc 'the TOML configuration format (https://github.com/mojombo/toml)'
       tag 'toml'
 
-      filenames '*.toml'
+      filenames '*.toml', 'Pipfile'
       mimetypes 'text/x-toml'
 
       identifier = /\S+/

data/lib/rouge/lexers/vala.rb

@@ -24,7 +24,7 @@ module Rouge
       )
 
       keywords_type = %w(
-        bool char double float int int8 int16 int32 int64 long short size_t 
+        bool char double float int int8 int16 int32 int64 long short size_t
         ssize_t string unichar uint uint8 uint16 uint32 uint64 ulong ushort
       )
 

data/lib/rouge/lexers/vhdl.rb

@@ -59,7 +59,7 @@ module Rouge
 
         # Find Attributes
         rule %r/'#{id}/i, Name::Attribute
-        
+
         # Punctuations
         rule %r/[(),:;]/, Punctuation
 

data/lib/rouge/lexers/wollok.rb

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*- # 
+# -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
 module Rouge

data/lib/rouge/lexers/xml.rb

@@ -9,7 +9,7 @@ module Rouge
       tag 'xml'
       filenames '*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd', '*.wsdl', '*.svg',
                 '*.plist'
-      mimetypes 'text/xml', 'application/xml', 'image/svg+xml', 
+      mimetypes 'text/xml', 'application/xml', 'image/svg+xml',
                 'application/rss+xml', 'application/atom+xml'
 
       def self.detect?(text)

data/lib/rouge/lexers/xojo.rb

@@ -8,7 +8,7 @@ module Rouge
       desc "Xojo"
       tag 'xojo'
       aliases 'realbasic'
-      filenames '*.xojo_code', '*.xojo_window', '*.xojo_toolbar', '*.xojo_menu'
+      filenames '*.xojo_code', '*.xojo_window', '*.xojo_toolbar', '*.xojo_menu', '*.xojo_image', '*.rbbas', '*.rbfrm', '*.rbmnu', '*.rbres', '*.rbtbar'
 
       keywords = %w(
           addhandler aggregates array asc assigns attributes begin break
@@ -30,15 +30,15 @@ module Rouge
         )
 
       operator_words = %w(
-          addressof and as in is isa mod not or xor
+          addressof weakaddressof and as in is isa mod not or xor
         )
 
       state :root do
         rule %r/\s+/, Text::Whitespace
 
         rule %r/rem\b.*?$/i, Comment::Single
-        rule %r(//.*$), Comment::Single
-        rule %r/\#tag Note.*\#tag EndNote/m, Comment::Preproc
+        rule %r([//'].*$), Comment::Single
+        rule %r/\#tag Note.*?\#tag EndNote/mi, Comment::Preproc
         rule %r/\s*[#].*$/x, Comment::Preproc
 
         rule %r/".*?"/, Literal::String::Double

data/lib/rouge/lexers/xpath.rb

@@ -0,0 +1,332 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class XPath < RegexLexer
+      title 'XPath'
+      desc 'XML Path Language (XPath) 3.1'
+      tag 'xpath'
+      filenames '*.xpath'
+
+      # Terminal literals:
+      # https://www.w3.org/TR/xpath-31/#terminal-symbols
+      def self.digits
+        @digits ||= %r/[0-9]+/
+      end
+
+      def self.decimalLiteral
+        @decimalLiteral ||= %r/\.#{digits}|#{digits}\.[0-9]*/
+      end
+
+      def self.doubleLiteral
+        @doubleLiteral ||= %r/(\.#{digits})|#{digits}(\.[0-9]*)?[eE][+-]?#{digits}/
+      end
+
+      def self.stringLiteral
+        @stringLiteral ||= %r/("(("")|[^"])*")|('(('')|[^'])*')/
+      end
+
+      def self.ncName
+        @ncName ||= %r/[a-z_][a-z_\-.0-9]*/i
+      end
+
+      def self.qName
+        @qName ||= %r/(?:#{ncName})(?::#{ncName})?/
+      end
+
+      def self.uriQName
+        @uriQName ||= %r/Q\{[^{}]*\}#{ncName}/
+      end
+
+      def self.eqName
+        @eqName ||= %r/(?:#{uriQName}|#{qName})/
+      end
+
+      def self.commentStart
+        @commentStart ||= %r/\(:/
+      end
+
+      def self.openParen
+        @openParen ||= %r/\((?!:)/
+      end
+
+      # Terminal symbols:
+      # https://www.w3.org/TR/xpath-30/#id-terminal-delimitation
+      def self.kindTest
+        @kindTest ||= Regexp.union %w(
+          element attribute schema-element schema-attribute
+          comment text node document-node namespace-node
+        )
+      end
+
+      def self.kindTestForPI
+        @kindTestForPI ||= Regexp.union %w(processing-instruction)
+      end
+
+      def self.axes
+        @axes ||= Regexp.union %w(
+          child descendant attribute self descendant-or-self
+          following-sibling following namespace
+          parent ancestor preceding-sibling preceding ancestor-or-self
+        )
+      end
+
+      def self.operators
+        @operators ||= Regexp.union %w(, => = := : >= >> > <= << < - * != + // / || |)
+      end
+
+      def self.keywords
+        @keywords ||= Regexp.union %w(let for some every if then else return in satisfies)
+      end
+
+      def self.word_operators
+        @word_operators ||= Regexp.union %w(
+          and or eq ge gt le lt ne is
+          div mod idiv
+          intersect except union
+          to
+        )
+      end
+
+      def self.constructorTypes
+        @constructorTypes ||= Regexp.union %w(function array map empty-sequence)
+      end
+
+      # Mixin states:
+
+      state :commentsAndWhitespace do
+        rule XPath.commentStart, Comment, :comment
+        rule %r/\s+/m, Text::Whitespace
+      end
+
+      # Lexical states:
+      # https://www.w3.org/TR/xquery-xpath-parsing/#XPath-lexical-states
+      # https://lists.w3.org/Archives/Public/public-qt-comments/2004Aug/0127.html
+      # https://www.w3.org/TR/xpath-30/#id-revision-log
+      # https://www.w3.org/TR/xpath-31/#id-revision-log
+
+      state :root do
+        mixin :commentsAndWhitespace
+
+        # Literals
+        rule XPath.doubleLiteral, Num::Float
+        rule XPath.decimalLiteral, Num::Float
+        rule XPath.digits, Num
+        rule XPath.stringLiteral, Literal::String
+
+        # Variables
+        rule %r/\$/, Name::Variable, :varname
+
+        # Operators
+        rule XPath.operators, Operator
+        rule %r/#{XPath.word_operators}\b/, Operator::Word
+        rule %r/#{XPath.keywords}\b/, Keyword
+        rule %r/[?,{}()\[\]]/, Punctuation
+
+        # Functions
+        rule %r/(function)(\s*)(#{XPath.openParen})/ do # function declaration
+          groups Keyword, Text::Whitespace, Punctuation
+        end
+        rule %r/(map|array|empty-sequence)/, Keyword # constructors
+        rule %r/(#{XPath.kindTest})(\s*)(#{XPath.openParen})/ do  # kindtest
+          groups Keyword, Text::Whitespace, Punctuation
+          push :kindtest
+        end
+        rule %r/(#{XPath.kindTestForPI})(\s*)(#{XPath.openParen})/ do # processing instruction kindtest
+          groups Keyword, Text::Whitespace, Punctuation
+          push :kindtestforpi
+        end
+        rule %r/(#{XPath.eqName})(\s*)(#{XPath.openParen})/ do # function call
+          groups Name::Function, Text::Whitespace, Punctuation
+        end
+        rule %r/(#{XPath.eqName})(\s*)(#)(\s*)(\d+)/ do # namedFunctionRef
+          groups Name::Function, Text::Whitespace, Name::Function, Text::Whitespace, Name::Function
+        end
+
+        # Type commands
+        rule %r/(cast|castable)(\s+)(as)/ do
+          groups Keyword, Text::Whitespace, Keyword
+          push :singletype
+        end
+        rule %r/(treat)(\s+)(as)/ do
+          groups Keyword, Text::Whitespace, Keyword
+          push :itemtype
+        end
+        rule %r/(instance)(\s+)(of)/ do
+          groups Keyword, Text::Whitespace, Keyword
+          push :itemtype
+        end
+        rule %r/(as)\b/ do
+          token Keyword
+          push :itemtype
+        end
+
+        # Paths
+        rule %r/(#{XPath.ncName})(\s*)(:)(\s*)(\*)/ do
+          groups Name::Tag, Text::Whitespace, Punctuation, Text::Whitespace, Operator
+        end
+        rule %r/(\*)(\s*)(:)(\s*)(#{XPath.ncName})/ do
+          groups Operator, Text::Whitespace, Punctuation, Text::Whitespace, Name::Tag
+        end
+        rule %r/(#{XPath.axes})(\s*)(::)/ do
+          groups Keyword, Text::Whitespace, Operator
+        end
+        rule %r/\.\.|\.|\*/, Operator
+        rule %r/@/, Name::Attribute, :attrname
+        rule XPath.eqName, Name::Tag
+      end
+
+      state :singletype do
+        mixin :commentsAndWhitespace
+
+        # Type name
+        rule XPath.eqName do
+          token Keyword::Type
+          pop!
+        end
+      end
+
+      state :itemtype do
+        mixin :commentsAndWhitespace
+
+        # Type tests
+        rule %r/(#{XPath.kindTest})(\s*)(#{XPath.openParen})/ do
+          groups Keyword::Type, Text::Whitespace, Punctuation
+          # go to kindtest then occurrenceindicator
+          goto :occurrenceindicator
+          push :kindtest
+        end
+        rule %r/(#{XPath.kindTestForPI})(\s*)(#{XPath.openParen})/ do
+          groups Keyword::Type, Text::Whitespace, Punctuation
+          # go to kindtestforpi then occurrenceindicator
+          goto :occurrenceindicator
+          push :kindtestforpi
+        end
+        rule %r/(item)(\s*)(#{XPath.openParen})(\s*)(\))/ do
+          groups Keyword::Type, Text::Whitespace, Punctuation, Text::Whitespace, Punctuation
+          goto :occurrenceindicator
+        end
+        rule %r/(#{XPath.constructorTypes})(\s*)(#{XPath.openParen})/ do
+          groups Keyword::Type, Text::Whitespace, Punctuation
+        end
+
+        # Type commands
+        rule %r/(cast|castable)(\s+)(as)/ do
+          groups Keyword, Text::Whitespace, Keyword
+          goto :singletype
+        end
+        rule %r/(treat)(\s+)(as)/ do
+          groups Keyword, Text::Whitespace, Keyword
+          goto :itemtype
+        end
+        rule %r/(instance)(\s+)(of)/ do
+          groups Keyword, Text::Whitespace, Keyword
+          goto :itemtype
+        end
+        rule %r/(as)\b/, Keyword
+
+        # Operators
+        rule XPath.operators do
+          token Operator
+          pop!
+        end
+        rule %r/#{XPath.word_operators}\b/ do
+          token Operator::Word
+          pop!
+        end
+        rule %r/#{XPath.keywords}\b/ do
+          token Keyword
+          pop!
+        end
+        rule %r/[\[),]/ do
+          token Punctuation
+          pop!
+        end
+
+        # Other types (e.g. xs:double)
+        rule XPath.eqName do
+          token Keyword::Type
+          goto :occurrenceindicator
+        end
+      end
+
+      # For pseudo-parameters for the KindTest productions
+      state :kindtest do
+        mixin :commentsAndWhitespace
+
+        # Pseudo-parameters:
+        rule %r/[?*]/, Operator
+        rule %r/,/, Punctuation
+        rule %r/(element|schema-element)(\s*)(#{XPath.openParen})/ do
+          groups Keyword::Type, Text::Whitespace, Punctuation
+          push :kindtest
+        end
+        rule XPath.eqName, Name::Tag
+
+        # End of pseudo-parameters
+        rule %r/\)/, Punctuation, :pop!
+      end
+
+      # Similar to :kindtest, but recognizes NCNames instead of EQNames
+      state :kindtestforpi do
+        mixin :commentsAndWhitespace
+
+        # Pseudo-parameters
+        rule XPath.ncName, Name
+        rule XPath.stringLiteral, Literal::String
+
+        # End of pseudo-parameters
+        rule %r/\)/, Punctuation, :pop!
+      end
+
+      state :occurrenceindicator do
+        mixin :commentsAndWhitespace
+
+        # Occurrence indicator
+        rule %r/[?*+]/ do
+          token Operator
+          pop!
+        end
+
+        # Otherwise, lex it in root state:
+        rule %r/(?![?*+])/ do
+          pop!
+        end
+      end
+
+      state :varname do
+        mixin :commentsAndWhitespace
+
+        # Function call
+        rule %r/(#{XPath.eqName})(\s*)(#{XPath.openParen})/ do
+          groups Name::Variable, Text::Whitespace, Punctuation
+          pop!
+        end
+
+        # Variable name
+        rule XPath.eqName, Name::Variable, :pop!
+      end
+
+      state :attrname do
+        mixin :commentsAndWhitespace
+
+        # Attribute name
+        rule XPath.eqName, Name::Attribute, :pop!
+        rule %r/\*/, Operator, :pop!
+      end
+
+      state :comment do
+        # Comment end
+        rule %r/:\)/, Comment, :pop!
+
+        # Nested comment
+        rule XPath.commentStart, Comment, :comment
+
+        # Comment contents
+        rule %r/[^:(]+/m, Comment
+        rule %r/[:(]/, Comment
+      end
+    end
+  end
+end