rouge 3.1.0 → 3.18.0

data/lib/rouge/lexers/ruby.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -9,7 +10,8 @@ module Rouge
       aliases 'rb'
       filenames '*.rb', '*.ruby', '*.rbw', '*.rake', '*.gemspec', '*.podspec',
                 'Rakefile', 'Guardfile', 'Gemfile', 'Capfile', 'Podfile',
-                'Vagrantfile', '*.ru', '*.prawn', 'Berksfile', '*.arb'
+                'Vagrantfile', '*.ru', '*.prawn', 'Berksfile', '*.arb',
+                'Dangerfile', 'Fastfile', 'Deliverfile', 'Appfile'
 
       mimetypes 'text/x-ruby', 'application/x-ruby'
 
@@ -29,15 +31,15 @@ module Rouge
         rule %r(:(?:\*\*|[-+]@|[/\%&\|^`~]|\[\]=?|<<|>>|<=?>|<=?|===?)),
           Str::Symbol
 
-        rule /:'(\\\\|\\'|[^'])*'/, Str::Symbol
-        rule /:"/, Str::Symbol, :simple_sym
+        rule %r/:'(\\\\|\\'|[^'])*'/, Str::Symbol
+        rule %r/:"/, Str::Symbol, :simple_sym
       end
 
       state :sigil_strings do
         # %-sigiled strings
         # %(abc), %[abc], %<abc>, %.abc., %r.abc., etc
         delimiter_map = { '{' => '}', '[' => ']', '(' => ')', '<' => '>' }
-        rule /%([rqswQWxiI])?([^\w\s])/ do |m|
+        rule %r/%([rqswQWxiI])?([^\w\s])/ do |m|
           open = Regexp.escape(m[2])
           close = Regexp.escape(delimiter_map[m[2]] || m[2])
           interp = /[rQWxI]/ === m[1]
@@ -55,38 +57,40 @@ module Rouge
           token toktype
 
           push do
-            rule /\\[##{open}#{close}\\]/, Str::Escape
+            uniq_chars = "#{open}#{close}".squeeze
+            uniq_chars = '' if open == close && open == "\\#"
+            rule %r/\\[##{uniq_chars}\\]/, Str::Escape
             # nesting rules only with asymmetric delimiters
             if open != close
-              rule /#{open}/ do
+              rule %r/#{open}/ do
                 token toktype
                 push
               end
             end
-            rule /#{close}/, toktype, :pop!
+            rule %r/#{close}/, toktype, :pop!
 
             if interp
               mixin :string_intp_escaped
-              rule /#/, toktype
+              rule %r/#/, toktype
             else
-              rule /[\\#]/, toktype
+              rule %r/[\\#]/, toktype
             end
 
-            rule /[^##{open}#{close}\\]+/m, toktype
+            rule %r/[^##{uniq_chars}\\]+/m, toktype
           end
         end
       end
 
       state :strings do
         mixin :symbols
-        rule /\b[a-z_]\w*?[?!]?:\s+/, Str::Symbol, :expr_start
-        rule /'(\\\\|\\'|[^'])*'/, Str::Single
-        rule /"/, Str::Double, :simple_string
-        rule /(?<!\.)`/, Str::Backtick, :simple_backtick
+        rule %r/\b[a-z_]\w*?[?!]?:\s+/, Str::Symbol, :expr_start
+        rule %r/'(\\\\|\\'|[^'])*'/, Str::Single
+        rule %r/"/, Str::Double, :simple_string
+        rule %r/(?<!\.)`/, Str::Backtick, :simple_backtick
       end
 
       state :regex_flags do
-        rule /[mixounse]*/, Str::Regex, :pop!
+        rule %r/[mixounse]*/, Str::Regex, :pop!
       end
 
       # double-quoted string and symbol
@@ -95,9 +99,9 @@ module Rouge
        [:backtick, Str::Backtick, '`']].each do |name, tok, fin|
         state :"simple_#{name}" do
           mixin :string_intp_escaped
-          rule /[^\\#{fin}#]+/m, tok
-          rule /[\\#]/, tok
-          rule /#{fin}/, tok, :pop!
+          rule %r/[^\\#{fin}#]+/m, tok
+          rule %r/[\\#]/, tok
+          rule %r/#{fin}/, tok, :pop!
         end
       end
 
@@ -149,38 +153,41 @@ module Rouge
 
       state :whitespace do
         mixin :inline_whitespace
-        rule /\n\s*/m, Text, :expr_start
-        rule /#.*$/, Comment::Single
+        rule %r/\n\s*/m, Text, :expr_start
+        rule %r/#.*$/, Comment::Single
 
         rule %r(=begin\b.*?\n=end\b)m, Comment::Multiline
       end
 
       state :inline_whitespace do
-        rule /[ \t\r]+/, Text
+        rule %r/[ \t\r]+/, Text
       end
 
       state :root do
         mixin :whitespace
-        rule /__END__/, Comment::Preproc, :end_part
+        rule %r/__END__/, Comment::Preproc, :end_part
 
-        rule /0_?[0-7]+(?:_[0-7]+)*/, Num::Oct
-        rule /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Hex
-        rule /0b[01]+(?:_[01]+)*/, Num::Bin
-        rule /\d+\.\d+(e[\+\-]?\d+)?/, Num::Float
-        rule /[\d]+(?:_\d+)*/, Num::Integer
+        rule %r/0_?[0-7]+(?:_[0-7]+)*/, Num::Oct
+        rule %r/0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Hex
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
+
+        decimal = %r/[\d]+(?:_\d+)*/
+        exp = %r/e[+-]?\d+/i
+        rule %r/#{decimal}(?:\.#{decimal}#{exp}?|#{exp})/, Num::Float
+        rule decimal, Num::Integer
 
         # names
-        rule /@@[a-z_]\w*/i, Name::Variable::Class
-        rule /@[a-z_]\w*/i, Name::Variable::Instance
-        rule /\$\w+/, Name::Variable::Global
+        rule %r/@@[a-z_]\w*/i, Name::Variable::Class
+        rule %r/@[a-z_]\w*/i, Name::Variable::Instance
+        rule %r/\$\w+/, Name::Variable::Global
         rule %r(\$[!@&`'+~=/\\,;.<>_*\$?:"]), Name::Variable::Global
-        rule /\$-[0adFiIlpvw]/, Name::Variable::Global
-        rule /::/, Operator
+        rule %r/\$-[0adFiIlpvw]/, Name::Variable::Global
+        rule %r/::/, Operator
 
         mixin :strings
 
-        rule /(?:#{keywords.join('|')})\b/, Keyword, :expr_start
-        rule /(?:#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo, :expr_start
+        rule %r/(?:#{keywords.join('|')})(?=\W|$)/, Keyword, :expr_start
+        rule %r/(?:#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo, :expr_start
 
         rule %r(
           (module)
@@ -190,52 +197,52 @@ module Rouge
           groups Keyword, Text, Name::Namespace
         end
 
-        rule /(def\b)(\s*)/ do
+        rule %r/(def\b)(\s*)/ do
           groups Keyword, Text
           push :funcname
         end
 
-        rule /(class\b)(\s*)/ do
+        rule %r/(class\b)(\s*)/ do
           groups Keyword, Text
           push :classname
         end
 
-        rule /(?:#{builtins_q.join('|')})[?]/, Name::Builtin, :expr_start
-        rule /(?:#{builtins_b.join('|')})!/,  Name::Builtin, :expr_start
-        rule /(?<!\.)(?:#{builtins_g.join('|')})\b/,
+        rule %r/(?:#{builtins_q.join('|')})[?]/, Name::Builtin, :expr_start
+        rule %r/(?:#{builtins_b.join('|')})!/,  Name::Builtin, :expr_start
+        rule %r/(?<!\.)(?:#{builtins_g.join('|')})\b/,
           Name::Builtin, :method_call
 
         mixin :has_heredocs
 
         # `..` and `...` for ranges must have higher priority than `.`
         # Otherwise, they will be parsed as :method_call
-        rule /\.{2,3}/, Operator, :expr_start
+        rule %r/\.{2,3}/, Operator, :expr_start
 
-        rule /[A-Z][a-zA-Z0-9_]*/, Name::Constant, :method_call
-        rule /(\.|::)(\s*)([a-z_]\w*[!?]?|[*%&^`~+-\/\[<>=])/ do
+        rule %r/[A-Z][a-zA-Z0-9_]*/, Name::Constant, :method_call
+        rule %r/(\.|::)(\s*)([a-z_]\w*[!?]?|[*%&^`~+-\/\[<>=])/ do
           groups Punctuation, Text, Name::Function
           push :method_call
         end
 
-        rule /[a-zA-Z_]\w*[?!]/, Name, :expr_start
-        rule /[a-zA-Z_]\w*/, Name, :method_call
-        rule /\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|!~|&&?|\|\||\./,
+        rule %r/[a-zA-Z_]\w*[?!]/, Name, :expr_start
+        rule %r/[a-zA-Z_]\w*/, Name, :method_call
+        rule %r/\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|!~|&&?|\|\||\./,
           Operator, :expr_start
-        rule /[-+\/*%=<>&!^|~]=?/, Operator, :expr_start
+        rule %r/[-+\/*%=<>&!^|~]=?/, Operator, :expr_start
         rule(/[?]/) { token Punctuation; push :ternary; push :expr_start }
         rule %r<[\[({,:\\;/]>, Punctuation, :expr_start
         rule %r<[\])}]>, Punctuation
       end
 
       state :has_heredocs do
-        rule /(?<!\w)(<<[-~]?)(["`']?)([a-zA-Z_]\w*)(\2)/ do |m|
+        rule %r/(?<!\w)(<<[-~]?)(["`']?)([a-zA-Z_]\w*)(\2)/ do |m|
           token Operator, m[1]
           token Name::Constant, "#{m[2]}#{m[3]}#{m[4]}"
           @heredoc_queue << [['<<-', '<<~'].include?(m[1]), m[3]]
           push :heredoc_queue unless state? :heredoc_queue
         end
 
-        rule /(<<[-~]?)(["'])(\2)/ do |m|
+        rule %r/(<<[-~]?)(["'])(\2)/ do |m|
           token Operator, m[1]
           token Name::Constant, "#{m[2]}#{m[3]}#{m[4]}"
           @heredoc_queue << [['<<-', '<<~'].include?(m[1]), '']
@@ -244,7 +251,7 @@ module Rouge
       end
 
       state :heredoc_queue do
-        rule /(?=\n)/ do
+        rule %r/(?=\n)/ do
           goto :resolve_heredocs
         end
 
@@ -254,13 +261,13 @@ module Rouge
       state :resolve_heredocs do
         mixin :string_intp_escaped
 
-        rule /\n/, Str::Heredoc, :test_heredoc
-        rule /[#\\\n]/, Str::Heredoc
-        rule /[^#\\\n]+/, Str::Heredoc
+        rule %r/\n/, Str::Heredoc, :test_heredoc
+        rule %r/[#\\\n]/, Str::Heredoc
+        rule %r/[^#\\\n]+/, Str::Heredoc
       end
 
       state :test_heredoc do
-        rule /[^#\\\n]*$/ do |m|
+        rule %r/[^#\\\n]*$/ do |m|
           tolerant, heredoc_name = @heredoc_queue.first
           check = tolerant ? m[0].strip : m[0].rstrip
 
@@ -282,12 +289,12 @@ module Rouge
       end
 
       state :funcname do
-        rule /\s+/, Text
-        rule /\(/, Punctuation, :defexpr
+        rule %r/\s+/, Text
+        rule %r/\(/, Punctuation, :defexpr
         rule %r(
-          (?:([a-zA-Z_][\w_]*)(\.))?
+          (?:([a-zA-Z_]\w*)(\.))?
           (
-            [a-zA-Z_][\w_]*[!?]? |
+            [a-zA-Z_]\w*[!?]? |
             \*\*? | [-+]@? | [/%&\|^`~] | \[\]=? |
             <<? | >>? | <=>? | >= | ===?
           )
@@ -301,36 +308,46 @@ module Rouge
       end
 
       state :classname do
-        rule /\s+/, Text
-        rule /\(/ do
+        rule %r/\s+/, Text
+        rule %r/\w+(::\w+)+/, Name::Class
+
+        rule %r/\(/ do
           token Punctuation
           push :defexpr
           push :expr_start
         end
 
         # class << expr
-        rule /<</ do
+        rule %r/<</ do
           token Operator
           goto :expr_start
         end
 
-        rule /[A-Z_]\w*/, Name::Class, :pop!
+        rule %r/[A-Z_]\w*/, Name::Class, :pop!
 
         rule(//) { pop! }
       end
 
       state :ternary do
-        rule(/:(?!:)/) { token Punctuation; goto :expr_start }
+        rule %r/(:)(\s+)/ do
+          groups Punctuation, Text
+          goto :expr_start
+        end
+
+        rule %r/:(?![^#\n]*?[:\\])/ do
+          token Punctuation
+          goto :expr_start
+        end
 
         mixin :root
       end
 
       state :defexpr do
-        rule /(\))(\.|::)?/ do
+        rule %r/(\))(\.|::)?/ do
           groups Punctuation, Operator
           pop!
         end
-        rule /\(/ do
+        rule %r/\(/ do
           token Punctuation
           push :defexpr
           push :expr_start
@@ -340,20 +357,20 @@ module Rouge
       end
 
       state :in_interp do
-        rule /}/, Str::Interpol, :pop!
+        rule %r/}/, Str::Interpol, :pop!
         mixin :root
       end
 
       state :string_intp do
-        rule /[#][{]/, Str::Interpol, :in_interp
-        rule /#(@@?|\$)[a-z_]\w*/i, Str::Interpol
+        rule %r/[#][{]/, Str::Interpol, :in_interp
+        rule %r/#(@@?|\$)[a-z_]\w*/i, Str::Interpol
       end
 
       state :string_intp_escaped do
         mixin :string_intp
-        rule /\\([\\abefnrstv#"']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})/,
+        rule %r/\\([\\abefnrstv#"']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})/,
           Str::Escape
-        rule /\\./, Str::Escape
+        rule %r/\\./, Str::Escape
       end
 
       state :method_call do
@@ -406,7 +423,7 @@ module Rouge
 
         # special case for using a single space.  Ruby demands that
         # these be in a single line, otherwise it would make no sense.
-        rule /(\s*)(%[rqswQWxiI]? \S* )/ do
+        rule %r/(\s*)(%[rqswQWxiI]? \S* )/ do
           groups Text, Str::Other
           pop!
         end
@@ -430,7 +447,7 @@ module Rouge
 
       state :end_part do
         # eat up the rest of the stream as Comment::Preproc
-        rule /.+/m, Comment::Preproc, :pop!
+        rule %r/.+/m, Comment::Preproc, :pop!
       end
     end
   end

data/lib/rouge/lexers/rust.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -6,7 +7,12 @@ module Rouge
       title "Rust"
       desc 'The Rust programming language (rust-lang.org)'
       tag 'rust'
-      aliases 'rs'
+      aliases 'rs',
+        # So that directives from https://github.com/budziq/rust-skeptic
+        # do not prevent highlighting.
+        'rust,no_run', 'rs,no_run',
+        'rust,ignore', 'rs,ignore',
+        'rust,should_panic', 'rs,should_panic'
       filenames '*.rs'
       mimetypes 'text/x-rust'
 
@@ -16,7 +22,7 @@ module Rouge
 
       def self.keywords
         @keywords ||= %w(
-          as assert break const copy do drop else enum extern fail false
+          as assert async await break const copy do drop else enum extern fail false
           fn for if impl let log loop match mod move mut priv pub pure
           ref return self static struct true trait type unsafe use where
           while box
@@ -43,6 +49,7 @@ module Rouge
 
       start {
         @macro_delims = { ']' => 0, ')' => 0, '}' => 0 }
+        push :bol
       }
 
       delim_map = { '[' => ']', '(' => ')', '{' => '}' }
@@ -50,57 +57,59 @@ module Rouge
       id = /[a-z_]\w*/i
       hex = /[0-9a-f]/i
       escapes = %r(
-        \\ ([nrt'\\] | x#{hex}{2} | u#{hex}{4} | U#{hex}{8})
+        \\ ([nrt'"\\0] | x#{hex}{2} | u#{hex}{4} | U#{hex}{8})
       )x
       size = /8|16|32|64/
 
-      state :start_line do
+      # Although not officially part of Rust, the rustdoc tool allows code in
+      # comments to begin with `#`. Code like this will be evaluated but not
+      # included in the HTML output produced by rustdoc. So that code intended
+      # for these comments can be higlighted with Rouge, the  Rust lexer needs
+      # to check if the beginning of the line begins with a `# `.
+      state :bol do
         mixin :whitespace
-        rule /\s+/, Text
-        rule /#\[/ do
-          token Name::Decorator; push :attribute
-        end
+        rule %r/#\s[^\n]*/, Comment::Special
         rule(//) { pop! }
-        rule /#\s[^\n]*/, Comment::Preproc
       end
 
       state :attribute do
         mixin :whitespace
         mixin :has_literals
-        rule /[(,)=]/, Name::Decorator
-        rule /\]/, Name::Decorator, :pop!
+        rule %r/[(,)=:]/, Name::Decorator
+        rule %r/\]/, Name::Decorator, :pop!
         rule id, Name::Decorator
       end
 
       state :whitespace do
-        rule /\s+/, Text
+        rule %r/\s+/, Text
         rule %r(//[^\n]*), Comment
         rule %r(/[*].*?[*]/)m, Comment::Multiline
       end
 
       state :root do
-        rule /\n/, Text, :start_line
+        rule %r/\n/, Text, :bol
         mixin :whitespace
-        rule /\b(?:#{Rust.keywords.join('|')})\b/, Keyword
+        rule %r/#!?\[/, Name::Decorator, :attribute
+        rule %r/\b(?:#{Rust.keywords.join('|')})\b/, Keyword
         mixin :has_literals
 
         rule %r([=-]>), Keyword
         rule %r(<->), Keyword
-        rule /[()\[\]{}|,:;]/, Punctuation
-        rule /[*!@~&+%^<>=-\?]|\.{2,3}/, Operator
+        rule %r/[()\[\]{}|,:;]/, Punctuation
+        rule %r/[*\/!@~&+%^<>=\?-]|\.{2,3}/, Operator
 
-        rule /([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
-        rule /[.]\s*#{id}/, Name::Property
-        rule /(#{id})(::)/m do
+        rule %r/([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
+        rule %r/[.]\s*#{id}/, Name::Property
+        rule %r/(#{id})(::)/m do
           groups Name::Namespace, Punctuation
         end
 
         # macros
-        rule /\bmacro_rules!/, Name::Decorator, :macro_rules
-        rule /#{id}!/, Name::Decorator, :macro
+        rule %r/\bmacro_rules!/, Name::Decorator, :macro_rules
+        rule %r/#{id}!/, Name::Decorator, :macro
 
-        rule /'#{id}/, Name::Variable
-        rule /#{id}/ do |m|
+        rule %r/'#{id}/, Name::Variable
+        rule %r/#{id}/ do |m|
           name = m[0]
           if self.class.builtins.include? name
             token Name::Builtin
@@ -113,13 +122,13 @@ module Rouge
       state :macro do
         mixin :has_literals
 
-        rule /[\[{(]/ do |m|
+        rule %r/[\[{(]/ do |m|
           @macro_delims[delim_map[m[0]]] += 1
           puts "    macro_delims: #{@macro_delims.inspect}" if @debug
           token Punctuation
         end
 
-        rule /[\]})]/ do |m|
+        rule %r/[\]})]/ do |m|
           @macro_delims[m[0]] -= 1
           puts "    macro_delims: #{@macro_delims.inspect}" if @debug
           pop! if macro_closed?
@@ -127,29 +136,30 @@ module Rouge
         end
 
         # same as the rule in root, but don't push another macro state
-        rule /#{id}!/, Name::Decorator
+        rule %r/#{id}!/, Name::Decorator
         mixin :root
 
         # No syntax errors in macros
-        rule /./, Text
+        rule %r/./, Text
       end
 
       state :macro_rules do
-        rule /[$]#{id}(:#{id})?/, Name::Variable
-        rule /[$]/, Name::Variable
+        rule %r/[$]#{id}(:#{id})?/, Name::Variable
+        rule %r/[$]/, Name::Variable
 
         mixin :macro
       end
 
       state :has_literals do
         # constants
-        rule /\b(?:true|false|nil)\b/, Keyword::Constant
+        rule %r/\b(?:true|false|nil)\b/, Keyword::Constant
         # characters
         rule %r(
           ' (?: #{escapes} | [^\\] ) '
         )x, Str::Char
 
-        rule /"/, Str, :string
+        rule %r/"/, Str, :string
+        rule %r/r(#*)".*?"\1/m, Str
 
         # numbers
         dot = /[.][0-9_]+/
@@ -166,26 +176,17 @@ module Rouge
 
         rule %r(
           ( 0b[10_]+
-          | 0x[0-9a-fA-F-]+
-          | [0-9]+
+          | 0x[0-9a-fA-F_]+
+          | [0-9_]+
           ) (u#{size}?|i#{size})?
         )x, Num::Integer
 
       end
 
       state :string do
-        rule /"/, Str, :pop!
+        rule %r/"/, Str, :pop!
         rule escapes, Str::Escape
-        rule /%%/, Str::Interpol
-        rule %r(
-          %
-          ( [0-9]+ [$] )?  # Parameter
-          [0#+-]*          # Flag
-          ( [0-9]+ [$]? )? # Width
-          ( [.] [0-9]+ )?  # Precision
-          [bcdfiostuxX?]   # Type
-        )x, Str::Interpol
-        rule /[^%"\\]+/m, Str
+        rule %r/[^"\\]+/m, Str
       end
     end
   end