rouge 3.3.0 → 3.8.0

data/lib/rouge/lexers/xquery.rb

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'xpath.rb'
+    class XQuery < XPath
+      title 'XQuery'
+      desc 'XQuery 3.1: An XML Query Language'
+      tag 'xquery'
+      filenames '*.xquery', '*.xq'
+      mimetypes 'application/xquery'
+
+      def self.keywords
+        @keywords ||= Regexp.union super, Regexp.union(%w(
+          xquery encoding version declare module
+          namespace copy-namespaces boundary-space construction
+          default collation base-uri preserve strip
+          ordering ordered unordered order empty greatest least
+          preserve no-preserve inherit no-inherit
+          decimal-format decimal-separator grouping-separator
+          infinity minus-sign NaN percent per-mille
+          zero-digit digit pattern-separator exponent-separator
+          import schema at element option
+          function external context item
+          typeswitch switch case
+          try catch
+          validate lax strict type
+          document element attribute text comment processing-instruction
+          for let where order group by return
+          allowing tumbling stable sliding window
+          start end only when previous next count collation
+          ascending descending
+        ))
+      end
+
+      # Mixin states:
+
+      state :tags do
+        rule %r/<#{XPath.qName}/, Name::Tag, :start_tag
+        rule %r/<!--/, Comment, :xml_comment
+        rule %r/<\?.*?\?>/, Comment::Preproc
+        rule %r/<!\[CDATA\[.*?\]\]>/, Comment::Preproc
+        rule %r/&\S*?;/, Name::Entity
+      end
+
+      # Lexical states:
+
+      prepend :root do
+        mixin :tags
+
+        rule %r/\{/, Punctuation
+        rule %r/\}`?/ do
+          token Punctuation
+          if stack.length > 1
+            pop!
+          end
+        end
+
+        rule %r/(namespace)(\s+)(#{XPath.ncName})/ do
+          groups Keyword, Text::Whitespace, Name::Namespace
+        end
+
+        rule %r/(#{XQuery.keywords})\b/, Keyword
+        rule %r/;/, Punctuation
+        rule %r/%/, Keyword::Declaration, :annotation
+
+        rule %r/(\(#)(\s*)(#{XPath.eqName})/ do
+          groups Comment::Preproc, Text::Whitespace, Name::Tag
+          push :pragma
+        end
+
+        rule %r/``\[/, Str, :str_constructor
+      end
+
+      state :annotation do
+        mixin :commentsAndWhitespace
+        rule XPath.eqName, Keyword::Declaration, :pop!
+      end
+
+      state :pragma do
+        mixin :commentsAndWhitespace
+        rule %r/#\)/, Comment::Preproc, :pop!
+        rule %r/./, Comment::Preproc
+      end
+
+      # https://www.w3.org/TR/xquery-31/#id-string-constructors
+      state :str_constructor do
+        rule %r/`\{/, Punctuation, :root
+        rule %r/\]``/, Str, :pop!
+        rule %r/[^`\]]+/m, Str
+        rule %r/[`\]]/, Str
+      end
+
+      state :xml_comment do
+        rule %r/[^-]+/m, Comment
+        rule %r/-->/, Comment, :pop!
+        rule %r/-/, Comment
+      end
+
+      state :start_tag do
+        rule %r/\s+/m, Text::Whitespace
+        rule %r/([\w.:-]+\s*=)(")/m do
+          groups Name::Attribute, Str
+          push :quot_attr
+        end
+        rule %r/([\w.:-]+\s*=)(')/m do
+          groups Name::Attribute, Str
+          push :apos_attr
+        end
+        rule %r/>/, Name::Tag, :tag_content
+        rule %r(/>), Name::Tag, :pop!
+      end
+
+      state :quot_attr do
+        rule %r/"/, Str, :pop!
+        rule %r/\{\{/, Str
+        rule %r/\{/, Punctuation, :root
+        rule %r/[^"{>]+/m, Str
+      end
+
+      state :apos_attr do
+        rule %r/'/, Str, :pop!
+        rule %r/\{\{/, Str
+        rule %r/\{/, Punctuation, :root
+        rule %r/[^'{>]+/m, Str
+      end
+
+      state :tag_content do
+        rule %r/\s+/m, Text::Whitespace
+        mixin :tags
+
+        rule %r/(\{\{|\}\})/, Text
+        rule %r/\{/, Punctuation, :root
+
+        rule %r/[^{}<&]/, Text
+
+        rule %r(</#{XPath.qName}(\s*)>) do
+          token Name::Tag
+          pop! 2 # pop self and tag_start
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/yaml.rb

@@ -78,38 +78,38 @@ module Rouge
       start { reset_indent }
 
       state :basic do
-        rule /#.*$/, Comment::Single
+        rule %r/#.*$/, Comment::Single
       end
 
       state :root do
         mixin :basic
 
-        rule /\n+/, Text
+        rule %r/\n+/, Text
 
         # trailing or pre-comment whitespace
-        rule /[ ]+(?=#|$)/, Text
+        rule %r/[ ]+(?=#|$)/, Text
 
-        rule /^%YAML\b/ do
+        rule %r/^%YAML\b/ do
           token Name::Tag
           reset_indent
           push :yaml_directive
         end
 
-        rule /^%TAG\b/ do
+        rule %r/^%TAG\b/ do
           token Name::Tag
           reset_indent
           push :tag_directive
         end
 
         # doc-start and doc-end indicators
-        rule /^(?:---|\.\.\.)(?= |$)/ do
+        rule %r/^(?:---|\.\.\.)(?= |$)/ do
           token Name::Namespace
           reset_indent
           push :block_line
         end
 
         # indentation spaces
-        rule /[ ]*(?!\s|$)/ do |m|
+        rule %r/[ ]*(?!\s|$)/ do |m|
           text, err = save_indent(m[0])
           token Text, text
           token Error, err
@@ -120,7 +120,7 @@ module Rouge
       state :indentation do
         rule(/\s*?\n/) { token Text; pop! 2 }
         # whitespace preceding block collection indicators
-        rule /[ ]+(?=[-:?](?:[ ]|$))/ do |m|
+        rule %r/[ ]+(?=[-:?](?:[ ]|$))/ do |m|
           token Text
           continue_indent(m[0])
         end
@@ -138,8 +138,8 @@ module Rouge
       # indented line in the block context
       state :block_line do
         # line end
-        rule /[ ]*(?=#|$)/, Text, :pop!
-        rule /[ ]+/, Text
+        rule %r/[ ]*(?=#|$)/, Text, :pop!
+        rule %r/[ ]+/, Text
         # tags, anchors, and aliases
         mixin :descriptors
         # block collections and scalars
@@ -148,7 +148,7 @@ module Rouge
         mixin :flow_nodes
 
         # a plain scalar
-        rule /(?=#{plain_scalar_start}|[?:-][^ \t\n\r\f\v])/ do
+        rule %r/(?=#{plain_scalar_start}|[?:-][^ \t\n\r\f\v])/ do
           token Name::Variable
           push :plain_scalar_in_block_context
         end
@@ -156,7 +156,7 @@ module Rouge
 
       state :descriptors do
         # a full-form tag
-        rule /!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, Keyword::Type
+        rule %r/!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, Keyword::Type
 
         # a tag in the form '!', '!suffix' or '!handle!suffix'
         rule %r(
@@ -165,21 +165,21 @@ module Rouge
         )x, Keyword::Type
 
         # an anchor
-        rule /&[\w-]+/, Name::Label
+        rule %r/&[\w-]+/, Name::Label
 
         # an alias
-        rule /\*[\w-]+/, Name::Variable
+        rule %r/\*[\w-]+/, Name::Variable
       end
 
       state :block_nodes do
         # implicit key
-        rule /((?:\w[\w -]*)?)(:)(?=\s|$)/ do |m|
+        rule %r/((?:\w[\w -]*)?)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end
 
         # literal and folded scalars
-        rule /[\|>]/ do
+        rule %r/[\|>][+-]?/ do
           token Punctuation::Indicator
           push :block_scalar_content
           push :block_scalar_header
@@ -187,39 +187,39 @@ module Rouge
       end
 
       state :flow_nodes do
-        rule /\[/, Punctuation::Indicator, :flow_sequence
-        rule /\{/, Punctuation::Indicator, :flow_mapping
-        rule /'/, Str::Single, :single_quoted_scalar
-        rule /"/, Str::Double, :double_quoted_scalar
+        rule %r/\[/, Punctuation::Indicator, :flow_sequence
+        rule %r/\{/, Punctuation::Indicator, :flow_mapping
+        rule %r/'/, Str::Single, :single_quoted_scalar
+        rule %r/"/, Str::Double, :double_quoted_scalar
       end
 
       state :flow_collection do
-        rule /\s+/m, Text
+        rule %r/\s+/m, Text
         mixin :basic
-        rule /[?:,]/, Punctuation::Indicator
+        rule %r/[?:,]/, Punctuation::Indicator
         mixin :descriptors
         mixin :flow_nodes
 
-        rule /(?=#{plain_scalar_start})/ do
+        rule %r/(?=#{plain_scalar_start})/ do
           push :plain_scalar_in_flow_context
         end
       end
 
       state :flow_sequence do
-        rule /\]/, Punctuation::Indicator, :pop!
+        rule %r/\]/, Punctuation::Indicator, :pop!
         mixin :flow_collection
       end
 
       state :flow_mapping do
-        rule /\}/, Punctuation::Indicator, :pop!
+        rule %r/\}/, Punctuation::Indicator, :pop!
         mixin :flow_collection
       end
 
       state :block_scalar_content do
-        rule /\n+/, Text
+        rule %r/\n+/, Text
 
         # empty lines never dedent, but they might be part of the scalar.
-        rule /^[ ]+$/ do |m|
+        rule %r/^[ ]+$/ do |m|
           text = m[0]
           indent_size = text.size
 
@@ -231,7 +231,7 @@ module Rouge
 
         # TODO: ^ doesn't actually seem to affect the match at all.
         # Find a way to work around this limitation.
-        rule /^[ ]*/ do |m|
+        rule %r/^[ ]*/ do |m|
           token Text
 
           indent_size = m[0].size
@@ -246,7 +246,7 @@ module Rouge
           end
         end
 
-        rule /[^\n\r\f\v]+/, Name::Constant
+        rule %r/[^\n\r\f\v]+/, Str
       end
 
       state :block_scalar_header do
@@ -271,46 +271,46 @@ module Rouge
 
       state :ignored_line do
         mixin :basic
-        rule /[ ]+/, Text
-        rule /\n/, Text, :pop!
+        rule %r/[ ]+/, Text
+        rule %r/\n/, Text, :pop!
       end
 
       state :quoted_scalar_whitespaces do
         # leading and trailing whitespace is ignored
-        rule /^[ ]+/, Text
-        rule /[ ]+$/, Text
+        rule %r/^[ ]+/, Text
+        rule %r/[ ]+$/, Text
 
-        rule /\n+/m, Text
+        rule %r/\n+/m, Text
 
-        rule /[ ]+/, Name::Variable
+        rule %r/[ ]+/, Name::Variable
       end
 
       state :single_quoted_scalar do
         mixin :quoted_scalar_whitespaces
-        rule /\\'/, Str::Escape
-        rule /'/, Str, :pop!
-        rule /[^\s']+/, Str
+        rule %r/\\'/, Str::Escape
+        rule %r/'/, Str, :pop!
+        rule %r/[^\s']+/, Str
       end
 
       state :double_quoted_scalar do
-        rule /"/, Str, :pop!
+        rule %r/"/, Str, :pop!
         mixin :quoted_scalar_whitespaces
         # escapes
-        rule /\\[0abt\tn\nvfre "\\N_LP]/, Str::Escape
-        rule /\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+        rule %r/\\[0abt\tn\nvfre "\\N_LP]/, Str::Escape
+        rule %r/\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
           Str::Escape
-        rule /[^ \t\n\r\f\v"\\]+/, Str
+        rule %r/[^ \t\n\r\f\v"\\]+/, Str
       end
 
       state :plain_scalar_in_block_context_new_line do
-        rule /^[ ]+\n/, Text
-        rule /\n+/m, Text
-        rule /^(?=---|\.\.\.)/ do
+        rule %r/^[ ]+\n/, Text
+        rule %r/\n+/m, Text
+        rule %r/^(?=---|\.\.\.)/ do
           pop! 3
         end
 
         # dedent detection
-        rule /^[ ]*/ do |m|
+        rule %r/^[ ]*/ do |m|
           token Text
           pop!
 
@@ -327,34 +327,36 @@ module Rouge
 
       state :plain_scalar_in_block_context do
         # the : indicator ends a scalar
-        rule /[ ]*(?=:[ \n]|:$)/, Text, :pop!
-        rule /[ ]*:/, Str
-        rule /[ ]+(?=#)/, Text, :pop!
-        rule /[ ]+$/, Text
+        rule %r/[ ]*(?=:[ \n]|:$)/, Text, :pop!
+        rule %r/[ ]*:\S+/, Str
+        rule %r/[ ]+(?=#)/, Text, :pop!
+        rule %r/[ ]+$/, Text
         # check for new documents or dedents at the new line
-        rule /\n+/ do
+        rule %r/\n+/ do
           token Text
           push :plain_scalar_in_block_context_new_line
         end
 
-        rule /[ ]+/, Str
+        rule %r/[ ]+/, Str
         rule SPECIAL_VALUES, Name::Constant
+        rule %r/\d+(?:\.\d+)?(?=(\r?\n)| +#)/, Literal::Number, :pop!
+
         # regular non-whitespace characters
-        rule /[^\s:]+/, Str
+        rule %r/[^\s:]+/, Str
       end
 
       state :plain_scalar_in_flow_context do
-        rule /[ ]*(?=[,:?\[\]{}])/, Text, :pop!
-        rule /[ ]+(?=#)/, Text, :pop!
-        rule /^[ ]+/, Text
-        rule /[ ]+$/, Text
-        rule /\n+/, Text
-        rule /[ ]+/, Name::Variable
-        rule /[^\s,:?\[\]{}]+/, Name::Variable
+        rule %r/[ ]*(?=[,:?\[\]{}])/, Text, :pop!
+        rule %r/[ ]+(?=#)/, Text, :pop!
+        rule %r/^[ ]+/, Text
+        rule %r/[ ]+$/, Text
+        rule %r/\n+/, Text
+        rule %r/[ ]+/, Name::Variable
+        rule %r/[^\s,:?\[\]{}]+/, Name::Variable
       end
 
       state :yaml_directive do
-        rule /([ ]+)(\d+\.\d+)/ do
+        rule %r/([ ]+)(\d+\.\d+)/ do
           groups Text, Num
           goto :ignored_line
         end

data/lib/rouge/plugins/redcarpet.rb

@@ -15,7 +15,7 @@ module Rouge
         # so we assume you're not using leading spaces that aren't tabs,
         # and just replace them here.
         if lexer.tag == 'make'
-          code.gsub! /^    /, "\t"
+          code.gsub! %r/^    /, "\t"
         end
 
         formatter = rouge_formatter(lexer)

data/lib/rouge/regex_lexer.rb

@@ -102,21 +102,21 @@ module Rouge
         callback ||= case next_state
         when :pop!
           proc do |stream|
-            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
             puts "    popping stack: 1" if @debug
             @stack.pop or raise 'empty stack!'
           end
         when :push
           proc do |stream|
-            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
             puts "    pushing :#{@stack.last.name}" if @debug
             @stack.push(@stack.last)
           end
         when Symbol
           proc do |stream|
-            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
             state = @states[next_state] || self.class.get_state(next_state)
             puts "    pushing :#{state.name}" if @debug
@@ -124,7 +124,7 @@ module Rouge
           end
         when nil
           proc do |stream|
-            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
           end
         else
@@ -183,18 +183,18 @@ module Rouge
     # Define a new state for this lexer with the given name.
     # The block will be evaluated in the context of a {StateDSL}.
     def self.state(name, &b)
-      name = name.to_s
+      name = name.to_sym
       state_definitions[name] = StateDSL.new(name, &b)
     end
 
     def self.prepend(name, &b)
-      name = name.to_s
+      name = name.to_sym
       dsl = state_definitions[name] or raise "no such state #{name.inspect}"
       replace_state(name, dsl.prepended(&b))
     end
 
     def self.append(name, &b)
-      name = name.to_s
+      name = name.to_sym
       dsl = state_definitions[name] or raise "no such state #{name.inspect}"
       replace_state(name, dsl.appended(&b))
     end
@@ -204,7 +204,7 @@ module Rouge
       return name if name.is_a? State
 
       states[name.to_sym] ||= begin
-        defn = state_definitions[name.to_s] or raise "unknown state: #{name.inspect}"
+        defn = state_definitions[name.to_sym] or raise "unknown state: #{name.inspect}"
         defn.to_state(self)
       end
     end
@@ -263,6 +263,7 @@ module Rouge
 
       until stream.eos?
         if @debug
+          puts
           puts "lexer: #{self.class.tag}"
           puts "stack: #{stack.map(&:name).map(&:to_sym).inspect}"
           puts "stream: #{stream.peek(20).inspect}"
@@ -289,11 +290,11 @@ module Rouge
     def step(state, stream)
       state.rules.each do |rule|
         if rule.is_a?(State)
-          puts "  entering mixin #{rule.name}" if @debug
+          puts "  entering: mixin :#{rule.name}" if @debug
           return true if step(rule, stream)
-          puts "  exiting  mixin #{rule.name}" if @debug
+          puts "  exiting: mixin :#{rule.name}" if @debug
         else
-          puts "  trying #{rule.inspect}" if @debug
+          puts "  trying: #{rule.inspect}" if @debug
 
           # XXX HACK XXX
           # StringScanner's implementation of ^ is b0rken.
@@ -303,14 +304,14 @@ module Rouge
           next if rule.beginning_of_line && !stream.beginning_of_line?
 
           if (size = stream.skip(rule.re))
-            puts "    got #{stream[0].inspect}" if @debug
+            puts "    got: #{stream[0].inspect}" if @debug
 
             instance_exec(stream, &rule.callback)
 
             if size.zero?
               @null_steps += 1
               if @null_steps > MAX_NULL_SCANS
-                puts "    too many scans without consuming the string!" if @debug
+                puts "    warning: too many scans without consuming the string!" if @debug
                 return false
               end
             else
@@ -352,20 +353,20 @@ module Rouge
       end
     end
 
-    # Delegate the lex to another lexer.  The #lex method will be called
-    # with `:continue` set to true, so that #reset! will not be called.
-    # In this way, a single lexer can be repeatedly delegated to while
-    # maintaining its own internal state stack.
+    # Delegate the lex to another lexer. We use the `continue_lex` method
+    # so that #reset! will not be called.  In this way, a single lexer
+    # can be repeatedly delegated to while maintaining its own internal
+    # state stack.
     #
     # @param [#lex] lexer
     #   The lexer or lexer class to delegate to
     # @param [String] text
     #   The text to delegate.  This defaults to the last matched string.
     def delegate(lexer, text=nil)
-      puts "    delegating to #{lexer.inspect}" if @debug
+      puts "    delegating to: #{lexer.inspect}" if @debug
       text ||= @current_stream[0]
 
-      lexer.lex(text, :continue => true) do |tok, val|
+      lexer.continue_lex(text) do |tok, val|
         puts "    delegated token: #{tok.inspect}, #{val.inspect}" if @debug
         yield_token(tok, val)
       end
@@ -388,7 +389,7 @@ module Rouge
         self.state
       end
 
-      puts "    pushing :#{push_state.name}" if @debug
+      puts "    pushing: :#{push_state.name}" if @debug
       stack.push(push_state)
     end
 
@@ -408,7 +409,7 @@ module Rouge
     def goto(state_name)
       raise 'empty stack!' if stack.empty?
 
-      puts "    going to state :#{state_name} " if @debug
+      puts "    going to: state :#{state_name} " if @debug
       stack[-1] = get_state(state_name)
     end
 
@@ -421,21 +422,21 @@ module Rouge
 
     # Check if `state_name` is in the state stack.
     def in_state?(state_name)
-      state_name = state_name.to_s
+      state_name = state_name.to_sym
       stack.any? do |state|
-        state.name == state_name.to_s
+        state.name == state_name.to_sym
       end
     end
 
     # Check if `state_name` is the state on top of the state stack.
     def state?(state_name)
-      state_name.to_s == state.name
+      state_name.to_sym == state.name
     end
 
   private
     def yield_token(tok, val)
       return if val.nil? || val.empty?
-      puts "    yielding #{tok.qualname}, #{val.inspect}" if @debug
+      puts "    yielding: #{tok.qualname}, #{val.inspect}" if @debug
       @output_stream.yield(tok, val)
     end
   end