rouge 3.0.0 → 3.18.0

data/lib/rouge/lexers/ruby.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -9,7 +10,8 @@ module Rouge
       aliases 'rb'
       filenames '*.rb', '*.ruby', '*.rbw', '*.rake', '*.gemspec', '*.podspec',
                 'Rakefile', 'Guardfile', 'Gemfile', 'Capfile', 'Podfile',
-                'Vagrantfile', '*.ru', '*.prawn', 'Berksfile'
+                'Vagrantfile', '*.ru', '*.prawn', 'Berksfile', '*.arb',
+                'Dangerfile', 'Fastfile', 'Deliverfile', 'Appfile'
 
       mimetypes 'text/x-ruby', 'application/x-ruby'
 
@@ -29,15 +31,15 @@ module Rouge
         rule %r(:(?:\*\*|[-+]@|[/\%&\|^`~]|\[\]=?|<<|>>|<=?>|<=?|===?)),
           Str::Symbol
 
-        rule /:'(\\\\|\\'|[^'])*'/, Str::Symbol
-        rule /:"/, Str::Symbol, :simple_sym
+        rule %r/:'(\\\\|\\'|[^'])*'/, Str::Symbol
+        rule %r/:"/, Str::Symbol, :simple_sym
       end
 
       state :sigil_strings do
         # %-sigiled strings
         # %(abc), %[abc], %<abc>, %.abc., %r.abc., etc
         delimiter_map = { '{' => '}', '[' => ']', '(' => ')', '<' => '>' }
-        rule /%([rqswQWxiI])?([^\w\s])/ do |m|
+        rule %r/%([rqswQWxiI])?([^\w\s])/ do |m|
           open = Regexp.escape(m[2])
           close = Regexp.escape(delimiter_map[m[2]] || m[2])
           interp = /[rQWxI]/ === m[1]
@@ -55,38 +57,40 @@ module Rouge
           token toktype
 
           push do
-            rule /\\[##{open}#{close}\\]/, Str::Escape
+            uniq_chars = "#{open}#{close}".squeeze
+            uniq_chars = '' if open == close && open == "\\#"
+            rule %r/\\[##{uniq_chars}\\]/, Str::Escape
             # nesting rules only with asymmetric delimiters
             if open != close
-              rule /#{open}/ do
+              rule %r/#{open}/ do
                 token toktype
                 push
               end
             end
-            rule /#{close}/, toktype, :pop!
+            rule %r/#{close}/, toktype, :pop!
 
             if interp
               mixin :string_intp_escaped
-              rule /#/, toktype
+              rule %r/#/, toktype
             else
-              rule /[\\#]/, toktype
+              rule %r/[\\#]/, toktype
             end
 
-            rule /[^##{open}#{close}\\]+/m, toktype
+            rule %r/[^##{uniq_chars}\\]+/m, toktype
           end
         end
       end
 
       state :strings do
         mixin :symbols
-        rule /\b[a-z_]\w*?[?!]?:\s+/, Str::Symbol, :expr_start
-        rule /'(\\\\|\\'|[^'])*'/, Str::Single
-        rule /"/, Str::Double, :simple_string
-        rule /(?<!\.)`/, Str::Backtick, :simple_backtick
+        rule %r/\b[a-z_]\w*?[?!]?:\s+/, Str::Symbol, :expr_start
+        rule %r/'(\\\\|\\'|[^'])*'/, Str::Single
+        rule %r/"/, Str::Double, :simple_string
+        rule %r/(?<!\.)`/, Str::Backtick, :simple_backtick
       end
 
       state :regex_flags do
-        rule /[mixounse]*/, Str::Regex, :pop!
+        rule %r/[mixounse]*/, Str::Regex, :pop!
       end
 
       # double-quoted string and symbol
@@ -95,9 +99,9 @@ module Rouge
        [:backtick, Str::Backtick, '`']].each do |name, tok, fin|
         state :"simple_#{name}" do
           mixin :string_intp_escaped
-          rule /[^\\#{fin}#]+/m, tok
-          rule /[\\#]/, tok
-          rule /#{fin}/, tok, :pop!
+          rule %r/[^\\#{fin}#]+/m, tok
+          rule %r/[\\#]/, tok
+          rule %r/#{fin}/, tok, :pop!
         end
       end
 
@@ -149,38 +153,41 @@ module Rouge
 
       state :whitespace do
         mixin :inline_whitespace
-        rule /\n\s*/m, Text, :expr_start
-        rule /#.*$/, Comment::Single
+        rule %r/\n\s*/m, Text, :expr_start
+        rule %r/#.*$/, Comment::Single
 
         rule %r(=begin\b.*?\n=end\b)m, Comment::Multiline
       end
 
       state :inline_whitespace do
-        rule /[ \t\r]+/, Text
+        rule %r/[ \t\r]+/, Text
       end
 
       state :root do
         mixin :whitespace
-        rule /__END__/, Comment::Preproc, :end_part
+        rule %r/__END__/, Comment::Preproc, :end_part
 
-        rule /0_?[0-7]+(?:_[0-7]+)*/, Num::Oct
-        rule /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Hex
-        rule /0b[01]+(?:_[01]+)*/, Num::Bin
-        rule /\d+\.\d+(e[\+\-]?\d+)?/, Num::Float
-        rule /[\d]+(?:_\d+)*/, Num::Integer
+        rule %r/0_?[0-7]+(?:_[0-7]+)*/, Num::Oct
+        rule %r/0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Hex
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
+
+        decimal = %r/[\d]+(?:_\d+)*/
+        exp = %r/e[+-]?\d+/i
+        rule %r/#{decimal}(?:\.#{decimal}#{exp}?|#{exp})/, Num::Float
+        rule decimal, Num::Integer
 
         # names
-        rule /@@[a-z_]\w*/i, Name::Variable::Class
-        rule /@[a-z_]\w*/i, Name::Variable::Instance
-        rule /\$\w+/, Name::Variable::Global
+        rule %r/@@[a-z_]\w*/i, Name::Variable::Class
+        rule %r/@[a-z_]\w*/i, Name::Variable::Instance
+        rule %r/\$\w+/, Name::Variable::Global
         rule %r(\$[!@&`'+~=/\\,;.<>_*\$?:"]), Name::Variable::Global
-        rule /\$-[0adFiIlpvw]/, Name::Variable::Global
-        rule /::/, Operator
+        rule %r/\$-[0adFiIlpvw]/, Name::Variable::Global
+        rule %r/::/, Operator
 
         mixin :strings
 
-        rule /(?:#{keywords.join('|')})\b/, Keyword, :expr_start
-        rule /(?:#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo, :expr_start
+        rule %r/(?:#{keywords.join('|')})(?=\W|$)/, Keyword, :expr_start
+        rule %r/(?:#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo, :expr_start
 
         rule %r(
           (module)
@@ -190,52 +197,52 @@ module Rouge
           groups Keyword, Text, Name::Namespace
         end
 
-        rule /(def\b)(\s*)/ do
+        rule %r/(def\b)(\s*)/ do
           groups Keyword, Text
           push :funcname
         end
 
-        rule /(class\b)(\s*)/ do
+        rule %r/(class\b)(\s*)/ do
           groups Keyword, Text
           push :classname
         end
 
-        rule /(?:#{builtins_q.join('|')})[?]/, Name::Builtin, :expr_start
-        rule /(?:#{builtins_b.join('|')})!/,  Name::Builtin, :expr_start
-        rule /(?<!\.)(?:#{builtins_g.join('|')})\b/,
+        rule %r/(?:#{builtins_q.join('|')})[?]/, Name::Builtin, :expr_start
+        rule %r/(?:#{builtins_b.join('|')})!/,  Name::Builtin, :expr_start
+        rule %r/(?<!\.)(?:#{builtins_g.join('|')})\b/,
           Name::Builtin, :method_call
 
         mixin :has_heredocs
 
         # `..` and `...` for ranges must have higher priority than `.`
         # Otherwise, they will be parsed as :method_call
-        rule /\.{2,3}/, Operator, :expr_start
+        rule %r/\.{2,3}/, Operator, :expr_start
 
-        rule /[A-Z][a-zA-Z0-9_]*/, Name::Constant, :method_call
-        rule /(\.|::)(\s*)([a-z_]\w*[!?]?|[*%&^`~+-\/\[<>=])/ do
+        rule %r/[A-Z][a-zA-Z0-9_]*/, Name::Constant, :method_call
+        rule %r/(\.|::)(\s*)([a-z_]\w*[!?]?|[*%&^`~+-\/\[<>=])/ do
           groups Punctuation, Text, Name::Function
           push :method_call
         end
 
-        rule /[a-zA-Z_]\w*[?!]/, Name, :expr_start
-        rule /[a-zA-Z_]\w*/, Name, :method_call
-        rule /\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|!~|&&?|\|\||\./,
+        rule %r/[a-zA-Z_]\w*[?!]/, Name, :expr_start
+        rule %r/[a-zA-Z_]\w*/, Name, :method_call
+        rule %r/\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|!~|&&?|\|\||\./,
           Operator, :expr_start
-        rule /[-+\/*%=<>&!^|~]=?/, Operator, :expr_start
+        rule %r/[-+\/*%=<>&!^|~]=?/, Operator, :expr_start
         rule(/[?]/) { token Punctuation; push :ternary; push :expr_start }
         rule %r<[\[({,:\\;/]>, Punctuation, :expr_start
         rule %r<[\])}]>, Punctuation
       end
 
       state :has_heredocs do
-        rule /(?<!\w)(<<[-~]?)(["`']?)([a-zA-Z_]\w*)(\2)/ do |m|
+        rule %r/(?<!\w)(<<[-~]?)(["`']?)([a-zA-Z_]\w*)(\2)/ do |m|
           token Operator, m[1]
           token Name::Constant, "#{m[2]}#{m[3]}#{m[4]}"
           @heredoc_queue << [['<<-', '<<~'].include?(m[1]), m[3]]
           push :heredoc_queue unless state? :heredoc_queue
         end
 
-        rule /(<<[-~]?)(["'])(\2)/ do |m|
+        rule %r/(<<[-~]?)(["'])(\2)/ do |m|
           token Operator, m[1]
           token Name::Constant, "#{m[2]}#{m[3]}#{m[4]}"
           @heredoc_queue << [['<<-', '<<~'].include?(m[1]), '']
@@ -244,7 +251,7 @@ module Rouge
       end
 
       state :heredoc_queue do
-        rule /(?=\n)/ do
+        rule %r/(?=\n)/ do
           goto :resolve_heredocs
         end
 
@@ -254,13 +261,13 @@ module Rouge
       state :resolve_heredocs do
         mixin :string_intp_escaped
 
-        rule /\n/, Str::Heredoc, :test_heredoc
-        rule /[#\\\n]/, Str::Heredoc
-        rule /[^#\\\n]+/, Str::Heredoc
+        rule %r/\n/, Str::Heredoc, :test_heredoc
+        rule %r/[#\\\n]/, Str::Heredoc
+        rule %r/[^#\\\n]+/, Str::Heredoc
       end
 
       state :test_heredoc do
-        rule /[^#\\\n]*$/ do |m|
+        rule %r/[^#\\\n]*$/ do |m|
           tolerant, heredoc_name = @heredoc_queue.first
           check = tolerant ? m[0].strip : m[0].rstrip
 
@@ -282,12 +289,12 @@ module Rouge
       end
 
       state :funcname do
-        rule /\s+/, Text
-        rule /\(/, Punctuation, :defexpr
+        rule %r/\s+/, Text
+        rule %r/\(/, Punctuation, :defexpr
         rule %r(
-          (?:([a-zA-Z_][\w_]*)(\.))?
+          (?:([a-zA-Z_]\w*)(\.))?
           (
-            [a-zA-Z_][\w_]*[!?]? |
+            [a-zA-Z_]\w*[!?]? |
             \*\*? | [-+]@? | [/%&\|^`~] | \[\]=? |
             <<? | >>? | <=>? | >= | ===?
           )
@@ -301,36 +308,46 @@ module Rouge
       end
 
       state :classname do
-        rule /\s+/, Text
-        rule /\(/ do
+        rule %r/\s+/, Text
+        rule %r/\w+(::\w+)+/, Name::Class
+
+        rule %r/\(/ do
           token Punctuation
           push :defexpr
           push :expr_start
         end
 
         # class << expr
-        rule /<</ do
+        rule %r/<</ do
           token Operator
           goto :expr_start
         end
 
-        rule /[A-Z_]\w*/, Name::Class, :pop!
+        rule %r/[A-Z_]\w*/, Name::Class, :pop!
 
         rule(//) { pop! }
       end
 
       state :ternary do
-        rule(/:(?!:)/) { token Punctuation; goto :expr_start }
+        rule %r/(:)(\s+)/ do
+          groups Punctuation, Text
+          goto :expr_start
+        end
+
+        rule %r/:(?![^#\n]*?[:\\])/ do
+          token Punctuation
+          goto :expr_start
+        end
 
         mixin :root
       end
 
       state :defexpr do
-        rule /(\))(\.|::)?/ do
+        rule %r/(\))(\.|::)?/ do
           groups Punctuation, Operator
           pop!
         end
-        rule /\(/ do
+        rule %r/\(/ do
           token Punctuation
           push :defexpr
           push :expr_start
@@ -340,20 +357,20 @@ module Rouge
       end
 
       state :in_interp do
-        rule /}/, Str::Interpol, :pop!
+        rule %r/}/, Str::Interpol, :pop!
         mixin :root
       end
 
       state :string_intp do
-        rule /[#][{]/, Str::Interpol, :in_interp
-        rule /#(@@?|\$)[a-z_]\w*/i, Str::Interpol
+        rule %r/[#][{]/, Str::Interpol, :in_interp
+        rule %r/#(@@?|\$)[a-z_]\w*/i, Str::Interpol
       end
 
       state :string_intp_escaped do
         mixin :string_intp
-        rule /\\([\\abefnrstv#"']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})/,
+        rule %r/\\([\\abefnrstv#"']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})/,
           Str::Escape
-        rule /\\./, Str::Escape
+        rule %r/\\./, Str::Escape
       end
 
       state :method_call do
@@ -406,7 +423,7 @@ module Rouge
 
         # special case for using a single space.  Ruby demands that
         # these be in a single line, otherwise it would make no sense.
-        rule /(\s*)(%[rqswQWxiI]? \S* )/ do
+        rule %r/(\s*)(%[rqswQWxiI]? \S* )/ do
           groups Text, Str::Other
           pop!
         end
@@ -430,7 +447,7 @@ module Rouge
 
       state :end_part do
         # eat up the rest of the stream as Comment::Preproc
-        rule /.+/m, Comment::Preproc, :pop!
+        rule %r/.+/m, Comment::Preproc, :pop!
       end
     end
   end

data/lib/rouge/lexers/rust.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -6,7 +7,12 @@ module Rouge
       title "Rust"
       desc 'The Rust programming language (rust-lang.org)'
       tag 'rust'
-      aliases 'rs'
+      aliases 'rs',
+        # So that directives from https://github.com/budziq/rust-skeptic
+        # do not prevent highlighting.
+        'rust,no_run', 'rs,no_run',
+        'rust,ignore', 'rs,ignore',
+        'rust,should_panic', 'rs,should_panic'
       filenames '*.rs'
       mimetypes 'text/x-rust'
 
@@ -16,10 +22,10 @@ module Rouge
 
       def self.keywords
         @keywords ||= %w(
-          as assert break const copy do drop else enum extern fail false
+          as assert async await break const copy do drop else enum extern fail false
           fn for if impl let log loop match mod move mut priv pub pure
-          ref return self static struct true trait type unsafe use while
-          box
+          ref return self static struct true trait type unsafe use where
+          while box
         )
       end
 
@@ -43,6 +49,7 @@ module Rouge
 
       start {
         @macro_delims = { ']' => 0, ')' => 0, '}' => 0 }
+        push :bol
       }
 
       delim_map = { '[' => ']', '(' => ')', '{' => '}' }
@@ -50,57 +57,59 @@ module Rouge
       id = /[a-z_]\w*/i
       hex = /[0-9a-f]/i
       escapes = %r(
-        \\ ([nrt'\\] | x#{hex}{2} | u#{hex}{4} | U#{hex}{8})
+        \\ ([nrt'"\\0] | x#{hex}{2} | u#{hex}{4} | U#{hex}{8})
       )x
       size = /8|16|32|64/
 
-      state :start_line do
+      # Although not officially part of Rust, the rustdoc tool allows code in
+      # comments to begin with `#`. Code like this will be evaluated but not
+      # included in the HTML output produced by rustdoc. So that code intended
+      # for these comments can be higlighted with Rouge, the  Rust lexer needs
+      # to check if the beginning of the line begins with a `# `.
+      state :bol do
         mixin :whitespace
-        rule /\s+/, Text
-        rule /#\[/ do
-          token Name::Decorator; push :attribute
-        end
+        rule %r/#\s[^\n]*/, Comment::Special
         rule(//) { pop! }
-        rule /#\s[^\n]*/, Comment::Preproc
       end
 
       state :attribute do
         mixin :whitespace
         mixin :has_literals
-        rule /[(,)=]/, Name::Decorator
-        rule /\]/, Name::Decorator, :pop!
+        rule %r/[(,)=:]/, Name::Decorator
+        rule %r/\]/, Name::Decorator, :pop!
         rule id, Name::Decorator
       end
 
       state :whitespace do
-        rule /\s+/, Text
+        rule %r/\s+/, Text
         rule %r(//[^\n]*), Comment
         rule %r(/[*].*?[*]/)m, Comment::Multiline
       end
 
       state :root do
-        rule /\n/, Text, :start_line
+        rule %r/\n/, Text, :bol
         mixin :whitespace
-        rule /\b(?:#{Rust.keywords.join('|')})\b/, Keyword
+        rule %r/#!?\[/, Name::Decorator, :attribute
+        rule %r/\b(?:#{Rust.keywords.join('|')})\b/, Keyword
         mixin :has_literals
 
         rule %r([=-]>), Keyword
         rule %r(<->), Keyword
-        rule /[()\[\]{}|,:;]/, Punctuation
-        rule /[*!@~&+%^<>=-\?]|\.{2,3}/, Operator
+        rule %r/[()\[\]{}|,:;]/, Punctuation
+        rule %r/[*\/!@~&+%^<>=\?-]|\.{2,3}/, Operator
 
-        rule /([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
-        rule /[.]\s*#{id}/, Name::Property
-        rule /(#{id})(::)/m do
+        rule %r/([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
+        rule %r/[.]\s*#{id}/, Name::Property
+        rule %r/(#{id})(::)/m do
           groups Name::Namespace, Punctuation
         end
 
         # macros
-        rule /\bmacro_rules!/, Name::Decorator, :macro_rules
-        rule /#{id}!/, Name::Decorator, :macro
+        rule %r/\bmacro_rules!/, Name::Decorator, :macro_rules
+        rule %r/#{id}!/, Name::Decorator, :macro
 
-        rule /'#{id}/, Name::Variable
-        rule /#{id}/ do |m|
+        rule %r/'#{id}/, Name::Variable
+        rule %r/#{id}/ do |m|
           name = m[0]
           if self.class.builtins.include? name
             token Name::Builtin
@@ -113,13 +122,13 @@ module Rouge
       state :macro do
         mixin :has_literals
 
-        rule /[\[{(]/ do |m|
+        rule %r/[\[{(]/ do |m|
           @macro_delims[delim_map[m[0]]] += 1
           puts "    macro_delims: #{@macro_delims.inspect}" if @debug
           token Punctuation
         end
 
-        rule /[\]})]/ do |m|
+        rule %r/[\]})]/ do |m|
           @macro_delims[m[0]] -= 1
           puts "    macro_delims: #{@macro_delims.inspect}" if @debug
           pop! if macro_closed?
@@ -127,29 +136,30 @@ module Rouge
         end
 
         # same as the rule in root, but don't push another macro state
-        rule /#{id}!/, Name::Decorator
+        rule %r/#{id}!/, Name::Decorator
         mixin :root
 
         # No syntax errors in macros
-        rule /./, Text
+        rule %r/./, Text
       end
 
       state :macro_rules do
-        rule /[$]#{id}(:#{id})?/, Name::Variable
-        rule /[$]/, Name::Variable
+        rule %r/[$]#{id}(:#{id})?/, Name::Variable
+        rule %r/[$]/, Name::Variable
 
         mixin :macro
       end
 
       state :has_literals do
         # constants
-        rule /\b(?:true|false|nil)\b/, Keyword::Constant
+        rule %r/\b(?:true|false|nil)\b/, Keyword::Constant
         # characters
         rule %r(
           ' (?: #{escapes} | [^\\] ) '
         )x, Str::Char
 
-        rule /"/, Str, :string
+        rule %r/"/, Str, :string
+        rule %r/r(#*)".*?"\1/m, Str
 
         # numbers
         dot = /[.][0-9_]+/
@@ -166,26 +176,17 @@ module Rouge
 
         rule %r(
           ( 0b[10_]+
-          | 0x[0-9a-fA-F-]+
-          | [0-9]+
+          | 0x[0-9a-fA-F_]+
+          | [0-9_]+
           ) (u#{size}?|i#{size})?
         )x, Num::Integer
 
       end
 
       state :string do
-        rule /"/, Str, :pop!
+        rule %r/"/, Str, :pop!
         rule escapes, Str::Escape
-        rule /%%/, Str::Interpol
-        rule %r(
-          %
-          ( [0-9]+ [$] )?  # Parameter
-          [0#+-]*          # Flag
-          ( [0-9]+ [$]? )? # Width
-          ( [.] [0-9]+ )?  # Precision
-          [bcdfiostuxX?]   # Type
-        )x, Str::Interpol
-        rule /[^%"\\]+/m, Str
+        rule %r/[^"\\]+/m, Str
       end
     end
   end