rouge 3.13.0 → 3.18.0

data/lib/rouge/lexers/objective_c/common.rb

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    module ObjectiveCCommon
+      id = /[a-z$_][a-z0-9$_]*/i
+
+      def at_keywords
+        @at_keywords ||= %w(
+          selector private protected public encode synchronized try
+          throw catch finally end property synthesize dynamic optional
+          interface implementation import autoreleasepool
+        )
+      end
+
+      def at_builtins
+        @at_builtins ||= %w(true false YES NO)
+      end
+
+      def builtins
+        @builtins ||= %w(YES NO nil)
+      end
+
+      def self.extended(base)
+        base.prepend :statements do
+          rule %r/@"/, base::Str, :string
+          rule %r/@'(\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|\\.|[^\\'\n]')/,
+            base::Str::Char
+          rule %r/@(\d+[.]\d*|[.]\d+|\d+)e[+-]?\d+l?/i,
+            base::Num::Float
+          rule %r/@(\d+[.]\d*|[.]\d+|\d+f)f?/i, base::Num::Float
+          rule %r/@0x\h+[lL]?/, base::Num::Hex
+          rule %r/@0[0-7]+l?/i, base::Num::Oct
+          rule %r/@\d+l?/, base::Num::Integer
+          rule %r/\bin\b/, base::Keyword
+
+          rule %r/@(?:interface|implementation)\b/, base::Keyword, :objc_classname
+          rule %r/@(?:class|protocol)\b/, base::Keyword, :forward_classname
+
+          rule %r/@([[:alnum:]]+)/ do |m|
+            if base.at_keywords.include? m[1]
+              token base::Keyword
+            elsif base.at_builtins.include? m[1]
+              token base::Name::Builtin
+            else
+              token base::Error
+            end
+          end
+
+          rule %r/[?]/, base::Punctuation, :ternary
+          rule %r/\[/,  base::Punctuation, :message
+          rule %r/@\[/, base::Punctuation, :array_literal
+          rule %r/@\{/, base::Punctuation, :dictionary_literal
+        end
+
+        id = /[a-z$_][a-z0-9$_]*/i
+
+        base.state :ternary do
+          rule %r/:/, base::Punctuation, :pop!
+          mixin :statements
+        end
+
+        base.state :message_shared do
+          rule %r/\]/, base::Punctuation, :pop!
+          rule %r/\{/, base::Punctuation, :pop!
+          rule %r/;/, base::Error
+
+          mixin :statements
+        end
+
+        base.state :message do
+          rule %r/(#{id})(\s*)(:)/ do
+            groups(base::Name::Function, base::Text, base::Punctuation)
+            goto :message_with_args
+          end
+
+          rule %r/(#{id})(\s*)(\])/ do
+            groups(base::Name::Function, base::Text, base::Punctuation)
+            pop!
+          end
+
+          mixin :message_shared
+        end
+
+        base.state :message_with_args do
+          rule %r/\{/, base::Punctuation, :function
+          rule %r/(#{id})(\s*)(:)/ do
+            groups(base::Name::Function, base::Text, base::Punctuation)
+            pop!
+          end
+
+          mixin :message_shared
+        end
+
+        base.state :array_literal do
+          rule %r/]/, base::Punctuation, :pop!
+          rule %r/,/, base::Punctuation
+          mixin :statements
+        end
+
+        base.state :dictionary_literal do
+          rule %r/}/, base::Punctuation, :pop!
+          rule %r/,/, base::Punctuation
+          mixin :statements
+        end
+
+        base.state :objc_classname do
+          mixin :whitespace
+
+          rule %r/(#{id})(\s*)(:)(\s*)(#{id})/ do
+            groups(base::Name::Class, base::Text,
+                   base::Punctuation, base::Text,
+                   base::Name::Class)
+            pop!
+          end
+
+          rule %r/(#{id})(\s*)([(])(\s*)(#{id})(\s*)([)])/ do
+            groups(base::Name::Class, base::Text,
+                   base::Punctuation, base::Text,
+                   base::Name::Label, base::Text,
+                   base::Punctuation)
+            pop!
+          end
+
+          rule id, base::Name::Class, :pop!
+        end
+
+        base.state :forward_classname do
+          mixin :whitespace
+
+          rule %r/(#{id})(\s*)(,)(\s*)/ do
+            groups(base::Name::Class, base::Text, base::Punctuation, base::Text)
+            push
+          end
+
+          rule %r/(#{id})(\s*)(;?)/ do
+            groups(base::Name::Class, base::Text, base::Punctuation)
+            pop!
+          end
+        end
+
+        base.prepend :root do
+          rule %r(
+            ([-+])(\s*)
+            ([(].*?[)])?(\s*)
+            (?=#{id}:?)
+          )ix do |m|
+            token base::Keyword, m[1]
+            token base::Text, m[2]
+            recurse(m[3]) if m[3]
+            token base::Text, m[4]
+            push :method_definition
+          end
+        end
+
+        base.state :method_definition do
+          rule %r/,/, base::Punctuation
+          rule %r/[.][.][.]/, base::Punctuation
+          rule %r/([(].*?[)])(#{id})/ do |m|
+            recurse m[1]; token base::Name::Variable, m[2]
+          end
+
+          rule %r/(#{id})(\s*)(:)/m do
+            groups(base::Name::Function, base::Text, base::Punctuation)
+          end
+
+          rule %r/;/, base::Punctuation, :pop!
+
+          rule %r/{/ do
+            token base::Punctuation
+            goto :function
+          end
+
+          mixin :inline_whitespace
+          rule %r(//.*?\n), base::Comment::Single
+          rule %r/\s+/m, base::Text
+
+          rule(//) { pop! }
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/objective_cpp.rb

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'cpp.rb'
+    load_lexer 'objective_c/common.rb'
+
+    class ObjectiveCpp < Cpp
+      extend ObjectiveCCommon
+
+      tag 'objective_cpp'
+      title "Objective-C++"
+      desc 'an extension of C++ uncommonly used to write Apple software'
+      aliases 'objcpp', 'obj-cpp', 'obj_cpp', 'objectivecpp',
+              'objc++', 'obj-c++', 'obj_c++', 'objectivec++'
+      filenames '*.mm', '*.h'
+
+      mimetypes 'text/x-objective-c++', 'application/x-objective-c++'
+
+      prepend :statements do
+        rule %r/(\.)(class)/ do
+          groups(Operator, Name::Builtin::Pseudo)
+        end
+        rule %r/(@selector)(\()(class)(\))/ do
+          groups(Keyword, Punctuation, Name::Builtin::Pseudo, Punctuation)
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/pascal.rb

@@ -7,7 +7,7 @@ module Rouge
       tag 'pascal'
       title "Pascal"
       desc 'a procedural programming language commonly used as a teaching language.'
-      filenames '*.pas'
+      filenames '*.pas', '*.lpr'
 
       mimetypes 'text/x-pascal'
 

data/lib/rouge/lexers/php.rb

@@ -29,7 +29,7 @@ module Rouge
       end
 
       def self.builtins
-        load File.join(__dir__, 'php/builtins.rb')
+        load File.join(Lexers::BASE_DIR, 'php/builtins.rb')
         self.builtins
       end
 
@@ -63,16 +63,23 @@ module Rouge
       end
 
       def self.keywords
+        # (echo parent ; echo self ; sed -nE 's/<ST_IN_SCRIPTING>"((__)?[[:alpha:]_]+(__)?)".*/\1/p' zend_language_scanner.l | tr '[A-Z]' '[a-z]') | sort -u | grep -Fwv -e isset -e unset -e empty -e const -e use -e function -e namespace
+        # - isset, unset and empty are actually keywords (directly handled by PHP's lexer but let's pretend these are functions, you use them like so)
+        # - self and parent are kind of keywords, they are not handled by PHP's lexer
+        # - use, const, namespace and function are handled by specific rules to highlight what's next to the keyword
+        # - class is also listed here, in addition to the rule below, to handle anonymous classes
         @keywords ||= Set.new %w(
-          and E_PARSE old_function E_ERROR or as E_WARNING parent eval
-          PHP_OS break exit case extends PHP_VERSION cfunction FALSE
-          print for require continue foreach require_once declare return
-          default static do switch die stdClass echo else TRUE elseif
-          var empty if xor enddeclare include virtual endfor include_once
-          while endforeach global __FILE__ endif list __LINE__ endswitch
-          new __sleep endwhile not array __wakeup E_ALL NULL final
-          php_user_filter interface implements public private protected
-          abstract clone try catch finally throw this use namespace yield
+          old_function cfunction
+          __class__ __dir__ __file__ __function__ __halt_compiler
+          __line__ __method__ __namespace__ __trait__ abstract and
+          array as break callable case catch class clone continue
+          declare default die do echo else elseif enddeclare
+          endfor endforeach endif endswitch endwhile eval exit
+          extends final finally fn for foreach global goto if
+          implements include include_once instanceof insteadof
+          interface list new or parent print private protected
+          public require require_once return self static switch
+          throw trait try var while xor yield
         )
       end
 
@@ -85,49 +92,51 @@ module Rouge
       state :root do
         # some extremely rough heuristics to decide whether to start inline or not
         rule(/\s*(?=<)/m) { delegate parent; push :template }
-        rule(/[^$]+(?=<\?(php|=))/) { delegate parent; push :template }
+        rule(/[^$]+(?=<\?(php|=))/i) { delegate parent; push :template }
 
         rule(//) { push :template; push :php }
       end
 
       state :template do
-        rule %r/<\?(php|=)?/, Comment::Preproc, :php
+        rule %r/<\?(php|=)?/i, Comment::Preproc, :php
         rule(/.*?(?=<\?)|.*/m) { delegate parent }
       end
 
       state :php do
         rule %r/\?>/, Comment::Preproc, :pop!
         # heredocs
-        rule %r/<<<('?)(#{id})\1\n.*?\n\s*\2;?/im, Str::Heredoc
+        rule %r/<<<(["']?)(#{id})\1\n.*?\n\s*\2;?/im, Str::Heredoc
         rule %r/\s+/, Text
         rule %r/#.*?$/, Comment::Single
         rule %r(//.*?$), Comment::Single
         rule %r(/\*\*(?!/).*?\*/)m, Comment::Doc
         rule %r(/\*.*?\*/)m, Comment::Multiline
-        
+
         rule %r/(->|::)(\s*)(#{id})/ do
           groups Operator, Text, Name::Attribute
         end
 
-        rule %r/[~!%^&*+=\|:.<>\/?@-]+/, Operator
+        rule %r/(void|\??(int|float|bool|string|iterable|self|callable))\b/i, Keyword::Type
+        rule %r/[~!%^&*+=\|:.<>\/@-]+/, Operator
+        rule %r/\?/, Operator
         rule %r/[\[\]{}();,]/, Punctuation
-        rule %r/(class|interface|trait)(\s+)(#{nsid})/ do
+        rule %r/(class|interface|trait)(\s+)(#{nsid})/i do
           groups Keyword::Declaration, Text, Name::Class
         end
-        rule %r/(use)(\s+)(function|const|)(\s*)(#{nsid})/ do
+        rule %r/(use)(\s+)(function|const|)(\s*)(#{nsid})/i do
           groups Keyword::Namespace, Text, Keyword::Namespace, Text, Name::Namespace
           push :use
         end
-        rule %r/(namespace)(\s+)(#{nsid})/ do
+        rule %r/(namespace)(\s+)(#{nsid})/i do
           groups Keyword::Namespace, Text, Name::Namespace
         end
         # anonymous functions
-        rule %r/(function)(\s*)(?=\()/ do
+        rule %r/(function)(\s*)(?=\()/i do
           groups Keyword, Text
         end
 
         # named functions
-        rule %r/(function)(\s+)(&?)(\s*)/ do
+        rule %r/(function)(\s+)(&?)(\s*)/i do
           groups Keyword, Text, Operator, Text
           push :funcname
         end
@@ -136,13 +145,18 @@ module Rouge
           groups Keyword, Text, Name::Constant
         end
 
-        rule %r/(true|false|null)\b/, Keyword::Constant
+        rule %r/stdClass\b/i, Name::Class
+        rule %r/(true|false|null)\b/i, Keyword::Constant
+        rule %r/(E|PHP)(_[[:upper:]]+)+\b/, Keyword::Constant
         rule %r/\$\{\$+#{id}\}/i, Name::Variable
         rule %r/\$+#{id}/i, Name::Variable
+        rule %r/(yield)([ \n\r\t]+)(from)/i do
+          groups Keyword, Text, Keyword
+        end
 
         # may be intercepted for builtin highlighting
         rule %r/\\?#{nsid}/i do |m|
-          name = m[0]
+          name = m[0].downcase
 
           if self.class.keywords.include? name
             token Keyword
@@ -153,30 +167,30 @@ module Rouge
           end
         end
 
-        rule %r/(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?/i, Num::Float
-        rule %r/\d+e[+-]?\d+/i, Num::Float
-        rule %r/0[0-7]+/, Num::Oct
-        rule %r/0x[a-f0-9]+/i, Num::Hex
-        rule %r/\d+/, Num::Integer
+        rule %r/(\d[_\d]*)?\.(\d[_\d]*)?(e[+-]?\d[_\d]*)?/i, Num::Float
+        rule %r/0[0-7][0-7_]*/, Num::Oct
+        rule %r/0b[01][01_]*/i, Num::Bin
+        rule %r/0x[a-f0-9][a-f0-9_]*/i, Num::Hex
+        rule %r/\d[_\d]*/, Num::Integer
         rule %r/'([^'\\]*(?:\\.[^'\\]*)*)'/, Str::Single
         rule %r/`([^`\\]*(?:\\.[^`\\]*)*)`/, Str::Backtick
         rule %r/"/, Str::Double, :string
       end
-      
+
       state :use do
-        rule %r/(\s+)(as)(\s+)(#{id})/ do
+        rule %r/(\s+)(as)(\s+)(#{id})/i do
           groups Text, Keyword, Text, Name
           :pop!
         end
         rule %r/\\\{/, Operator, :uselist
         rule %r/;/, Punctuation, :pop!
       end
-      
+
       state :uselist do
         rule %r/\s+/, Text
         rule %r/,/, Operator
         rule %r/\}/, Operator, :pop!
-        rule %r/(as)(\s+)(#{id})/ do
+        rule %r/(as)(\s+)(#{id})/i do
           groups Keyword, Text, Name
         end
         rule %r/#{id}/, Name::Namespace
@@ -189,7 +203,8 @@ module Rouge
       state :string do
         rule %r/"/, Str::Double, :pop!
         rule %r/[^\\{$"]+/, Str::Double
-        rule %r/\\([nrt\"$\\]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2})/,
+        rule %r/\\u\{[0-9a-fA-F]+\}/, Str::Escape
+        rule %r/\\([efrntv\"$\\]|[0-7]{1,3}|[xX][0-9a-fA-F]{1,2})/,
           Str::Escape
         rule %r/\$#{id}(\[\S+\]|->#{id})?/, Name::Variable
 

data/lib/rouge/lexers/pony.rb

@@ -46,7 +46,7 @@ module Rouge
       state :root do
         mixin :whitespace
         rule %r/"""/, Str::Doc, :docstring
-        rule %r{//(.*?)\n}, Comment::Single
+        rule %r{//.*}, Comment::Single
         rule %r{/(\\\n)?[*](.|\n)*?[*](\\\n)?/}, Comment::Multiline
         rule %r/"/, Str, :string
         rule %r([~!%^&*+=\|?:<>/-]), Operator

data/lib/rouge/lexers/powershell.rb

@@ -201,6 +201,11 @@ module Rouge
         rule %r/(?:#{KEYWORDS})\b(?![-.])/i, Keyword::Reserved
 
         rule %r/-{1,2}\w+/, Name::Tag
+        
+        rule %r/(\.)?([-\w]+)(\[)/ do |m|
+          groups Operator, Name::Function, Punctuation
+          push :bracket
+        end
 
         rule %r/([\/\\~\w][-.:\/\\~\w]*)(\n)?/ do |m|
           groups Name::Function, Text::Whitespace

data/lib/rouge/lexers/protobuf.rb

@@ -17,7 +17,7 @@ module Rouge
       state :root do
         rule %r/[\s]+/, Text
         rule %r/[,;{}\[\]()]/, Punctuation
-        rule %r/\/(\\\n)?\/(\n|(.|\n)*?[^\\]\n)/, Comment::Single
+        rule %r/\/(\\\n)?\/($|(.|\n)*?[^\\]$)/, Comment::Single
         rule %r/\/(\\\n)?\*(.|\n)*?\*(\\\n)?\//, Comment::Multiline
         rule kw, Keyword
         rule datatype, Keyword::Type

data/lib/rouge/lexers/python.rb

@@ -8,7 +8,8 @@ module Rouge
       desc "The Python programming language (python.org)"
       tag 'python'
       aliases 'py'
-      filenames '*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac'
+      filenames '*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac',
+                '*.bzl', 'BUCK', 'BUILD', 'BUILD.bazel', 'WORKSPACE'
       mimetypes 'text/x-python', 'application/x-python'
 
       def self.detect?(text)
@@ -68,6 +69,11 @@ module Rouge
 
       identifier =        /[a-z_][a-z0-9_]*/i
       dotted_identifier = /[a-z_.][a-z0-9_.]*/i
+
+      def current_string
+        @string_register ||= StringRegister.new
+      end
+
       state :root do
         rule %r/\n+/m, Text
         rule %r/^(:)(\s*)([ru]{,2}""".*?""")/mi do
@@ -76,14 +82,13 @@ module Rouge
 
         rule %r/[^\S\n]+/, Text
         rule %r(#(.*)?\n?), Comment::Single
-        rule %r/[\[\]{}:(),;]/, Punctuation
+        rule %r/[\[\]{}:(),;.]/, Punctuation
         rule %r/\\\n/, Text
         rule %r/\\/, Text
 
         rule %r/(in|is|and|or|not)\b/, Operator::Word
         rule %r/(<<|>>|\/\/|\*\*)=?/, Operator
         rule %r/[-~+\/*%=<>&^|@]=?|!=/, Operator
-        rule %r/\.(?![0-9])/, Operator  # so it doesn't match float literals
 
         rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(import)/ do
           groups Keyword::Namespace,
@@ -109,14 +114,11 @@ module Rouge
 
         # TODO: not in python 3
         rule %r/`.*?`/, Str::Backtick
-        rule %r/(?:r|ur|ru)"""/i, Str, :raw_tdqs
-        rule %r/(?:r|ur|ru)'''/i, Str, :raw_tsqs
-        rule %r/(?:r|ur|ru)"/i,   Str, :raw_dqs
-        rule %r/(?:r|ur|ru)'/i,   Str, :raw_sqs
-        rule %r/u?"""/i,          Str, :tdqs
-        rule %r/u?'''/i,          Str, :tsqs
-        rule %r/u?"/i,            Str, :dqs
-        rule %r/u?'/i,            Str, :sqs
+        rule %r/([rfbu]{0,2})('''|"""|['"])/i do |m|
+          token Str
+          current_string.register type: m[1].downcase, delim: m[2]
+          push :generic_string
+        end
 
         rule %r/@#{dotted_identifier}/i, Name::Decorator
 
@@ -172,26 +174,39 @@ module Rouge
         mixin :raise
       end
 
-      state :strings do
-        rule %r/%(\([a-z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/i, Str::Interpol
-      end
+      state :generic_string do
+        rule %r/[^'"\\{]+/, Str
+        rule %r/{{/, Str
 
-      state :strings_double do
-        rule %r/[^\\"%\n]+/, Str
-        mixin :strings
-      end
+        rule %r/'''|"""|['"]/ do |m|
+          token Str
+          if current_string.delim? m[0]
+            current_string.remove
+            pop!
+          end
+        end
 
-      state :strings_single do
-        rule %r/[^\\'%\n]+/, Str
-        mixin :strings
-      end
+        rule %r/\\/ do |m|
+          if current_string.type? "r"
+            token Str
+          else
+            token Str::Interpol
+          end
+          push :generic_escape
+        end
 
-      state :nl do
-        rule %r/\n/, Str
+        rule %r/{/ do |m|
+          if current_string.type? "f"
+            token Str::Interpol
+            push :generic_interpol
+          else
+            token Str
+          end
+        end
       end
 
-      state :escape do
-        rule %r(\\
+      state :generic_escape do
+        rule %r(
           ( [\\abfnrtv"']
           | \n
           | N{[a-zA-Z][a-zA-Z ]+[a-zA-Z]}
@@ -200,48 +215,43 @@ module Rouge
           | x[a-fA-F0-9]{2}
           | [0-7]{1,3}
           )
-        )x, Str::Escape
-      end
-
-      state :raw_escape do
-        rule %r/\\./, Str
+        )x do
+          if current_string.type? "r"
+            token Str
+          else
+            token Str::Escape
+          end
+          pop!
+        end
       end
 
-      state :dqs do
-        rule %r/"/, Str, :pop!
-        mixin :escape
-        mixin :strings_double
+      state :generic_interpol do
+        rule %r/[^{}]+/ do |m|
+          recurse m[0]
+        end
+        rule %r/{/, Str::Interpol, :generic_interpol
+        rule %r/}/, Str::Interpol, :pop!
       end
 
-      state :sqs do
-        rule %r/'/, Str, :pop!
-        mixin :escape
-        mixin :strings_single
-      end
+      class StringRegister < Array
+        def delim?(delim)
+          self.last[1] == delim
+        end
 
-      state :tdqs do
-        rule %r/"""/, Str, :pop!
-        rule %r/"/, Str
-        mixin :escape
-        mixin :strings_double
-        mixin :nl
-      end
+        def register(type: "u", delim: "'")
+          self.push [type, delim]
+        end
 
-      state :tsqs do
-        rule %r/'''/, Str, :pop!
-        rule %r/'/, Str
-        mixin :escape
-        mixin :strings_single
-        mixin :nl
-      end
+        def remove
+          self.pop
+        end
 
-      %w(tdqs tsqs dqs sqs).each do |qtype|
-        state :"raw_#{qtype}" do
-          mixin :raw_escape
-          mixin :"#{qtype}"
+        def type?(type)
+          self.last[0].include? type
         end
       end
 
+      private_constant :StringRegister
     end
   end
 end