rouge 3.0.0 → 3.18.0

data/lib/rouge/lexers/properties.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -13,38 +14,38 @@ module Rouge
       identifier = /[\w.-]+/
 
       state :basic do
-        rule /[!#].*?\n/, Comment
-        rule /\s+/, Text
-        rule /\\\n/, Str::Escape
+        rule %r/[!#].*?\n/, Comment
+        rule %r/\s+/, Text
+        rule %r/\\\n/, Str::Escape
       end
 
       state :root do
         mixin :basic
 
-        rule /(#{identifier})(\s*)([=:])/ do
+        rule %r/(#{identifier})(\s*)([=:])/ do
           groups Name::Property, Text, Punctuation
           push :value
         end
       end
 
       state :value do
-        rule /\n/, Text, :pop!
+        rule %r/\n/, Text, :pop!
         mixin :basic
-        rule /"/, Str, :dq
-        rule /'.*?'/, Str
+        rule %r/"/, Str, :dq
+        rule %r/'.*?'/, Str
         mixin :esc_str
-        rule /[^\\\n]+/, Str
+        rule %r/[^\\\n]+/, Str
       end
 
       state :dq do
-        rule /"/, Str, :pop!
+        rule %r/"/, Str, :pop!
         mixin :esc_str
-        rule /[^\\"]+/m, Str
+        rule %r/[^\\"]+/m, Str
       end
 
       state :esc_str do
-        rule /\\u[0-9]{4}/, Str::Escape
-        rule /\\./m, Str::Escape
+        rule %r/\\u[0-9]{4}/, Str::Escape
+        rule %r/\\./m, Str::Escape
       end
     end
   end

data/lib/rouge/lexers/protobuf.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -14,55 +15,55 @@ module Rouge
       datatype = /\b(bool|bytes|double|fixed32|fixed64|float|int32|int64|sfixed32|sfixed64|sint32|sint64|string|uint32|uint64)\b/
 
       state :root do
-        rule /[\s]+/, Text
-        rule /[,;{}\[\]()]/, Punctuation
-        rule /\/(\\\n)?\/(\n|(.|\n)*?[^\\]\n)/, Comment::Single
-        rule /\/(\\\n)?\*(.|\n)*?\*(\\\n)?\//, Comment::Multiline
+        rule %r/[\s]+/, Text
+        rule %r/[,;{}\[\]()]/, Punctuation
+        rule %r/\/(\\\n)?\/($|(.|\n)*?[^\\]$)/, Comment::Single
+        rule %r/\/(\\\n)?\*(.|\n)*?\*(\\\n)?\//, Comment::Multiline
         rule kw, Keyword
         rule datatype, Keyword::Type
-        rule /true|false/, Keyword::Constant
-        rule /(package)(\s+)/ do
+        rule %r/true|false/, Keyword::Constant
+        rule %r/(package)(\s+)/ do
           groups Keyword::Namespace, Text
           push :package
         end
 
-        rule /(message|extend)(\s+)/ do
+        rule %r/(message|extend)(\s+)/ do
           groups Keyword::Declaration, Text
           push :message
         end
 
-        rule /(enum|group|service)(\s+)/ do
+        rule %r/(enum|group|service)(\s+)/ do
           groups Keyword::Declaration, Text
           push :type
         end
 
-        rule /".*?"/, Str
-        rule /'.*?'/, Str
-        rule /(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*/, Num::Float
-        rule /(\d+\.\d*|\.\d+|\d+[fF])[fF]?/, Num::Float
-        rule /(\-?(inf|nan))\b/, Num::Float
-        rule /0x[0-9a-fA-F]+[LlUu]*/, Num::Hex
-        rule /0[0-7]+[LlUu]*/, Num::Oct
-        rule /\d+[LlUu]*/, Num::Integer
-        rule /[+-=]/, Operator
-        rule /([a-zA-Z_][\w.]*)([ \t]*)(=)/ do
+        rule %r/".*?"/, Str
+        rule %r/'.*?'/, Str
+        rule %r/(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*/, Num::Float
+        rule %r/(\d+\.\d*|\.\d+|\d+[fF])[fF]?/, Num::Float
+        rule %r/(\-?(inf|nan))\b/, Num::Float
+        rule %r/0x[0-9a-fA-F]+[LlUu]*/, Num::Hex
+        rule %r/0[0-7]+[LlUu]*/, Num::Oct
+        rule %r/\d+[LlUu]*/, Num::Integer
+        rule %r/[+-=]/, Operator
+        rule %r/([a-zA-Z_][\w.]*)([ \t]*)(=)/ do
           groups Name::Attribute, Text, Operator
         end
-        rule /[a-zA-Z_][\w.]*/, Name
+        rule %r/[a-zA-Z_][\w.]*/, Name
       end
 
       state :package do
-        rule /[a-zA-Z_]\w*/, Name::Namespace, :pop!
+        rule %r/[a-zA-Z_]\w*/, Name::Namespace, :pop!
         rule(//) { pop! }
       end
 
       state :message do
-        rule /[a-zA-Z_]\w*/, Name::Class, :pop!
+        rule %r/[a-zA-Z_]\w*/, Name::Class, :pop!
         rule(//) { pop! }
       end
 
       state :type do
-        rule /[a-zA-Z_]\w*/, Name, :pop!
+        rule %r/[a-zA-Z_]\w*/, Name, :pop!
         rule(//) { pop! }
       end
     end

data/lib/rouge/lexers/puppet.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -38,15 +39,15 @@ module Rouge
       qualname = /(::)?(#{id}::)*\w+/
 
       state :whitespace do
-        rule /\s+/m, Text
-        rule /#.*?\n/, Comment
+        rule %r/\s+/m, Text
+        rule %r/#.*?\n/, Comment
       end
 
       state :root do
         mixin :whitespace
 
-        rule /[$]#{qualname}/, Name::Variable
-        rule /(#{id})(?=\s*[=+]>)/m do |m|
+        rule %r/[$]#{qualname}/, Name::Variable
+        rule %r/(#{id})(?=\s*[=+]>)/m do |m|
           if self.class.metaparameters.include? m[0]
             token Keyword::Pseudo
           else
@@ -54,31 +55,31 @@ module Rouge
           end
         end
 
-        rule /(#{qualname})(?=\s*[(])/m, Name::Function
+        rule %r/(#{qualname})(?=\s*[(])/m, Name::Function
         rule cap_id, Name::Class
 
-        rule /[+=|~-]>|<[|~-]/, Punctuation
-        rule /[:}();\[\]]/, Punctuation
+        rule %r/[+=|~-]>|<[|~-]/, Punctuation
+        rule %r/[|:}();\[\]]/, Punctuation
 
         # HACK for case statements and selectors
-        rule /{/, Punctuation, :regex_allowed
-        rule /,/, Punctuation, :regex_allowed
+        rule %r/{/, Punctuation, :regex_allowed
+        rule %r/,/, Punctuation, :regex_allowed
 
-        rule /(in|and|or)\b/, Operator::Word
-        rule /[=!<>]=/, Operator
-        rule /[=!]~/, Operator, :regex_allowed
-        rule %r([<>!+*/-]), Operator
+        rule %r/(in|and|or)\b/, Operator::Word
+        rule %r/[=!<>]=/, Operator
+        rule %r/[=!]~/, Operator, :regex_allowed
+        rule %r([.=<>!+*/-]), Operator
 
-        rule /(class|include)(\s*)(#{qualname})/ do
+        rule %r/(class|include)(\s*)(#{qualname})/ do
           groups Keyword, Text, Name::Class
         end
 
-        rule /node\b/, Keyword, :regex_allowed
+        rule %r/node\b/, Keyword, :regex_allowed
 
-        rule /'(\\[\\']|[^'])*'/m, Str::Single
-        rule /"/, Str::Double, :dquotes
+        rule %r/'(\\[\\']|[^'])*'/m, Str::Single
+        rule %r/"/, Str::Double, :dquotes
 
-        rule /\d+([.]\d+)?(e[+-]\d+)?/, Num
+        rule %r/\d+([.]\d+)?(e[+-]\d+)?/, Num
 
         # a valid regex.  TODO: regexes are only allowed
         # in certain places in puppet.
@@ -102,26 +103,26 @@ module Rouge
 
       state :regex do
         rule %r(/), Str::Regex, :pop!
-        rule /\\./, Str::Escape
-        rule /[(){}]/, Str::Interpol
-        rule /\[/, Str::Interpol, :regex_class
-        rule /./, Str::Regex
+        rule %r/\\./, Str::Escape
+        rule %r/[(){}]/, Str::Interpol
+        rule %r/\[/, Str::Interpol, :regex_class
+        rule %r/./, Str::Regex
       end
 
       state :regex_class do
-        rule /\]/, Str::Interpol, :pop!
-        rule /(?<!\[)-(?=\])/, Str::Regex
-        rule /-/, Str::Interpol
-        rule /\\./, Str::Escape
-        rule /[^\\\]-]+/, Str::Regex
+        rule %r/\]/, Str::Interpol, :pop!
+        rule %r/(?<!\[)-(?=\])/, Str::Regex
+        rule %r/-/, Str::Interpol
+        rule %r/\\./, Str::Escape
+        rule %r/[^\\\]-]+/, Str::Regex
       end
 
       state :dquotes do
-        rule /"/, Str::Double, :pop!
-        rule /[^$\\"]+/m, Str::Double
-        rule /\\./m, Str::Escape
-        rule /[$]#{qualname}/, Name::Variable
-        rule /[$][{]#{qualname}[}]/, Name::Variable
+        rule %r/"/, Str::Double, :pop!
+        rule %r/[^$\\"]+/m, Str::Double
+        rule %r/\\./m, Str::Escape
+        rule %r/[$]#{qualname}/, Name::Variable
+        rule %r/[$][{]#{qualname}[}]/, Name::Variable
       end
     end
   end

data/lib/rouge/lexers/python.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -7,11 +8,12 @@ module Rouge
       desc "The Python programming language (python.org)"
       tag 'python'
       aliases 'py'
-      filenames '*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac'
+      filenames '*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac',
+                '*.bzl', 'BUCK', 'BUILD', 'BUILD.bazel', 'WORKSPACE'
       mimetypes 'text/x-python', 'application/x-python'
 
       def self.detect?(text)
-        return true if text.shebang?(/pythonw?(3|2(\.\d)?)?/)
+        return true if text.shebang?(/pythonw?(?:[23](?:\.\d+)?)?/)
       end
 
       def self.keywords
@@ -19,17 +21,18 @@ module Rouge
           assert break continue del elif else except exec
           finally for global if lambda pass print raise
           return try while yield as with from import yield
+          async await nonlocal
         )
       end
 
       def self.builtins
         @builtins ||= %w(
-          __import__ abs all any apply basestring bin bool buffer
+          __import__ abs all any apply ascii basestring bin bool buffer
           bytearray bytes callable chr classmethod cmp coerce compile
           complex delattr dict dir divmod enumerate eval execfile exit
-          file filter float frozenset getattr globals hasattr hash hex id
+          file filter float format frozenset getattr globals hasattr hash hex id
           input int intern isinstance issubclass iter len list locals
-          long map max min next object oct open ord pow property range
+          long map max memoryview min next object oct open ord pow property range
           raw_input reduce reload repr reversed round set setattr slice
           sorted staticmethod str sum super tuple type unichr unicode
           vars xrange zip
@@ -43,15 +46,21 @@ module Rouge
       def self.exceptions
         @exceptions ||= %w(
           ArithmeticError AssertionError AttributeError
-          BaseException DeprecationWarning EOFError EnvironmentError
-          Exception FloatingPointError FutureWarning GeneratorExit IOError
-          ImportError ImportWarning IndentationError IndexError KeyError
-          KeyboardInterrupt LookupError MemoryError NameError
-          NotImplemented NotImplementedError OSError OverflowError
-          OverflowWarning PendingDeprecationWarning ReferenceError
-          RuntimeError RuntimeWarning StandardError StopIteration
-          SyntaxError SyntaxWarning SystemError SystemExit TabError
-          TypeError UnboundLocalError UnicodeDecodeError
+          BaseException BlockingIOError BrokenPipeError BufferError
+          BytesWarning ChildProcessError ConnectionAbortedError
+          ConnectionError ConnectionRefusedError ConnectionResetError
+          DeprecationWarning EOFError EnvironmentError
+          Exception FileExistsError FileNotFoundError
+          FloatingPointError FutureWarning GeneratorExit IOError
+          ImportError ImportWarning IndentationError IndexError
+          InterruptedError IsADirectoryError KeyError KeyboardInterrupt
+          LookupError MemoryError ModuleNotFoundError NameError
+          NotADirectoryError NotImplemented NotImplementedError OSError
+          OverflowError OverflowWarning PendingDeprecationWarning
+          ProcessLookupError RecursionError ReferenceError ResourceWarning
+          RuntimeError RuntimeWarning StandardError StopAsyncIteration
+          StopIteration SyntaxError SyntaxWarning SystemError SystemExit
+          TabError TimeoutError TypeError UnboundLocalError UnicodeDecodeError
           UnicodeEncodeError UnicodeError UnicodeTranslateError
           UnicodeWarning UserWarning ValueError VMSError Warning
           WindowsError ZeroDivisionError
@@ -60,22 +69,28 @@ module Rouge
 
       identifier =        /[a-z_][a-z0-9_]*/i
       dotted_identifier = /[a-z_.][a-z0-9_.]*/i
+
+      def current_string
+        @string_register ||= StringRegister.new
+      end
+
       state :root do
-        rule /\n+/m, Text
-        rule /^(:)(\s*)([ru]{,2}""".*?""")/mi do
+        rule %r/\n+/m, Text
+        rule %r/^(:)(\s*)([ru]{,2}""".*?""")/mi do
           groups Punctuation, Text, Str::Doc
         end
 
-        rule /[^\S\n]+/, Text
-        rule /#.*$/, Comment
-        rule /[\[\]{}:(),;]/, Punctuation
-        rule /\\\n/, Text
-        rule /\\/, Text
+        rule %r/[^\S\n]+/, Text
+        rule %r(#(.*)?\n?), Comment::Single
+        rule %r/[\[\]{}:(),;.]/, Punctuation
+        rule %r/\\\n/, Text
+        rule %r/\\/, Text
 
-        rule /(in|is|and|or|not)\b/, Operator::Word
-        rule /!=|==|<<|>>|[-~+\/*%=<>&^|.]/, Operator
+        rule %r/(in|is|and|or|not)\b/, Operator::Word
+        rule %r/(<<|>>|\/\/|\*\*)=?/, Operator
+        rule %r/[-~+\/*%=<>&^|@]=?|!=/, Operator
 
-        rule /(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(import)/ do
+        rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(import)/ do
           groups Keyword::Namespace,
                  Text,
                  Name::Namespace,
@@ -83,35 +98,32 @@ module Rouge
                  Keyword::Namespace
         end
 
-        rule /(import)(\s+)(#{dotted_identifier})/ do
+        rule %r/(import)(\s+)(#{dotted_identifier})/ do
           groups Keyword::Namespace, Text, Name::Namespace
         end
 
-        rule /(def)((?:\s|\\\s)+)/ do
+        rule %r/(def)((?:\s|\\\s)+)/ do
           groups Keyword, Text
           push :funcname
         end
 
-        rule /(class)((?:\s|\\\s)+)/ do
+        rule %r/(class)((?:\s|\\\s)+)/ do
           groups Keyword, Text
           push :classname
         end
 
         # TODO: not in python 3
-        rule /`.*?`/, Str::Backtick
-        rule /(?:r|ur|ru)"""/i, Str, :raw_tdqs
-        rule /(?:r|ur|ru)'''/i, Str, :raw_tsqs
-        rule /(?:r|ur|ru)"/i,   Str, :raw_dqs
-        rule /(?:r|ur|ru)'/i,   Str, :raw_sqs
-        rule /u?"""/i,          Str, :tdqs
-        rule /u?'''/i,          Str, :tsqs
-        rule /u?"/i,            Str, :dqs
-        rule /u?'/i,            Str, :sqs
-
-        rule /@#{dotted_identifier}/i, Name::Decorator
+        rule %r/`.*?`/, Str::Backtick
+        rule %r/([rfbu]{0,2})('''|"""|['"])/i do |m|
+          token Str
+          current_string.register type: m[1].downcase, delim: m[2]
+          push :generic_string
+        end
+
+        rule %r/@#{dotted_identifier}/i, Name::Decorator
 
         # using negative lookbehind so we don't match property names
-        rule /(?<!\.)#{identifier}/ do |m|
+        rule %r/(?<!\.)#{identifier}/ do |m|
           if self.class.keywords.include? m[0]
             token Keyword
           elsif self.class.exceptions.include? m[0]
@@ -127,12 +139,18 @@ module Rouge
 
         rule identifier, Name
 
-        rule /(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?/i, Num::Float
-        rule /\d+e[+-]?[0-9]+/i, Num::Float
-        rule /0[0-7]+/, Num::Oct
-        rule /0x[a-f0-9]+/i, Num::Hex
-        rule /\d+L/, Num::Integer::Long
-        rule /\d+/, Num::Integer
+        digits = /[0-9](_?[0-9])*/
+        decimal = /((#{digits})?\.#{digits}|#{digits}\.)/
+        exponent = /e[+-]?#{digits}/i
+        rule %r/#{decimal}(#{exponent})?j?/i, Num::Float
+        rule %r/#{digits}#{exponent}j?/i, Num::Float
+        rule %r/#{digits}j/i, Num::Float
+
+        rule %r/0b(_?[0-1])+/i, Num::Bin
+        rule %r/0o(_?[0-7])+/i, Num::Oct
+        rule %r/0x(_?[a-f0-9])+/i, Num::Hex
+        rule %r/\d+L/, Num::Integer::Long
+        rule %r/([1-9](_?[0-9])*|0(_?0)*)/, Num::Integer
       end
 
       state :funcname do
@@ -144,11 +162,11 @@ module Rouge
       end
 
       state :raise do
-        rule /from\b/, Keyword
-        rule /raise\b/, Keyword
-        rule /yield\b/, Keyword
-        rule /\n/, Text, :pop!
-        rule /;/, Punctuation, :pop!
+        rule %r/from\b/, Keyword
+        rule %r/raise\b/, Keyword
+        rule %r/yield\b/, Keyword
+        rule %r/\n/, Text, :pop!
+        rule %r/;/, Punctuation, :pop!
         mixin :root
       end
 
@@ -156,76 +174,84 @@ module Rouge
         mixin :raise
       end
 
-      state :strings do
-        rule /%(\([a-z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/i, Str::Interpol
-      end
+      state :generic_string do
+        rule %r/[^'"\\{]+/, Str
+        rule %r/{{/, Str
 
-      state :strings_double do
-        rule /[^\\"%\n]+/, Str
-        mixin :strings
-      end
+        rule %r/'''|"""|['"]/ do |m|
+          token Str
+          if current_string.delim? m[0]
+            current_string.remove
+            pop!
+          end
+        end
 
-      state :strings_single do
-        rule /[^\\'%\n]+/, Str
-        mixin :strings
-      end
+        rule %r/\\/ do |m|
+          if current_string.type? "r"
+            token Str
+          else
+            token Str::Interpol
+          end
+          push :generic_escape
+        end
 
-      state :nl do
-        rule /\n/, Str
+        rule %r/{/ do |m|
+          if current_string.type? "f"
+            token Str::Interpol
+            push :generic_interpol
+          else
+            token Str
+          end
+        end
       end
 
-      state :escape do
-        rule %r(\\
+      state :generic_escape do
+        rule %r(
           ( [\\abfnrtv"']
           | \n
-          | N{[a-zA-z][a-zA-Z ]+[a-zA-Z]}
+          | N{[a-zA-Z][a-zA-Z ]+[a-zA-Z]}
           | u[a-fA-F0-9]{4}
           | U[a-fA-F0-9]{8}
           | x[a-fA-F0-9]{2}
           | [0-7]{1,3}
           )
-        )x, Str::Escape
-      end
-
-      state :raw_escape do
-        rule /\\./, Str
+        )x do
+          if current_string.type? "r"
+            token Str
+          else
+            token Str::Escape
+          end
+          pop!
+        end
       end
 
-      state :dqs do
-        rule /"/, Str, :pop!
-        mixin :escape
-        mixin :strings_double
+      state :generic_interpol do
+        rule %r/[^{}]+/ do |m|
+          recurse m[0]
+        end
+        rule %r/{/, Str::Interpol, :generic_interpol
+        rule %r/}/, Str::Interpol, :pop!
       end
 
-      state :sqs do
-        rule /'/, Str, :pop!
-        mixin :escape
-        mixin :strings_single
-      end
+      class StringRegister < Array
+        def delim?(delim)
+          self.last[1] == delim
+        end
 
-      state :tdqs do
-        rule /"""/, Str, :pop!
-        rule /"/, Str
-        mixin :escape
-        mixin :strings_double
-        mixin :nl
-      end
+        def register(type: "u", delim: "'")
+          self.push [type, delim]
+        end
 
-      state :tsqs do
-        rule /'''/, Str, :pop!
-        rule /'/, Str
-        mixin :escape
-        mixin :strings_single
-        mixin :nl
-      end
+        def remove
+          self.pop
+        end
 
-      %w(tdqs tsqs dqs sqs).each do |qtype|
-        state :"raw_#{qtype}" do
-          mixin :raw_escape
-          mixin :"#{qtype}"
+        def type?(type)
+          self.last[0].include? type
         end
       end
 
+      private_constant :StringRegister
     end
   end
 end