rouge 4.7.0 → 5.0.0

data/lib/rouge/lexers/php.rb

@@ -18,6 +18,10 @@ module Rouge
       option :funcnamehighlighting, 'Whether to highlight builtin functions (default: true)'
       option :disabledmodules, 'Disable certain modules from being highlighted as builtins (default: empty)'
 
+      lazy auto: false do
+        require_relative "php/keywords"
+      end
+
       def initialize(*)
         super
 
@@ -26,6 +30,8 @@ module Rouge
         @start_inline = bool_option(:start_inline) { :guess }
         @funcnamehighlighting = bool_option(:funcnamehighlighting) { true }
         @disabledmodules = list_option(:disabledmodules)
+
+        eager_load! if @funcnamehighlighting
       end
 
       def self.detect?(text)
@@ -48,16 +54,11 @@ module Rouge
         )
       end
 
-      def self.builtins
-        Kernel::load File.join(Lexers::BASE_DIR, 'php/keywords.rb')
-        builtins
-      end
-
       def builtins
         return [] unless @funcnamehighlighting
 
         @builtins ||= Set.new.tap do |builtins|
-          self.class.builtins.each do |mod, fns|
+          BUILTINS.each do |mod, fns|
             next if @disabledmodules.include? mod
             builtins.merge(fns)
           end
@@ -109,7 +110,7 @@ module Rouge
           name = m[0].downcase
           if self.class.keywords.include? name
             token Keyword
-          elsif self.builtins.include? name
+          elsif builtins.include? name
             token Name::Builtin
           else
             token Name::Function
@@ -324,7 +325,7 @@ module Rouge
         rule %r/=/, Operator, :in_assign
         rule %r/\b(?:public|protected|private|readonly)(?:\(set\)|\b)/i, Keyword
         rule %r/\breadonly\b/i, Keyword
-        rule %r/\??#{id}/, Keyword::Type, :in_assign
+        rule %r/\??#{id}/, Keyword::Type, :in_property
         mixin :escape
         mixin :whitespace
         mixin :variables
@@ -388,7 +389,10 @@ module Rouge
         rule %r/\b(?:public|protected|private)(?:\(set\)|\b)/i, Keyword
         rule %r/\b(?:readonly|static)\b/i, Keyword
         rule %r/(?=(abstract|const|function)\b)/i, Keyword, :pop!
-        rule %r/\??#{id}/, Keyword::Type, :pop!
+        rule %r/\??#{id}/ do
+          token Keyword::Type
+          goto :in_property
+        end
         mixin :escape
         mixin :whitespace
         mixin :return
@@ -416,6 +420,51 @@ module Rouge
         end
         mixin :php
       end
+
+      state :in_property do
+        rule %r/\$+#{id}/, Name::Variable
+        rule %r/\{/ do
+          token Punctuation
+          goto :in_property_hooks
+        end
+        rule %r/[;,]/, Punctuation, :pop!
+        rule %r/(?==)/ do
+          pop!
+        end
+        rule %r/[|&]/, Operator
+        rule %r/\??#{id}/, Keyword::Type
+        mixin :escape
+        mixin :whitespace
+        mixin :return
+      end
+
+      state :in_property_hooks do
+        rule %r/\}/, Punctuation, :pop!
+        rule %r/\bfinal\b/i, Keyword
+        rule %r/&(?=get\b)/, Operator
+        rule %r/(\bset\b)(\s*)(\()/ do
+          groups Keyword, Text, Punctuation
+          push :in_property_hook_params
+        end
+        rule %r/\b(?:get|set)\b/, Keyword
+        rule %r/\{/, Punctuation, :in_function_body
+        rule %r/[;,\(\)\[\]]/, Punctuation
+        mixin :escape
+        mixin :whitespace
+        mixin :variables
+        mixin :values
+        mixin :names
+        mixin :operators
+        rule %r/[=?]/, Operator
+      end
+
+      state :in_property_hook_params do
+        rule %r/\)/, Punctuation, :pop!
+        rule %r/\??#{id}/, Keyword::Type
+        mixin :escape
+        mixin :whitespace
+        mixin :variables
+      end
     end
   end
 end

data/lib/rouge/lexers/plsql.rb

@@ -22,7 +22,7 @@ module Rouge
         OCIDURATION OCIINTERVAL OCILOBLOCATOR OCINUMBER OCIRAW OCIREF OCIREFCURSOR
         OCIROWID OCISTRING OCITYPE OF ON OPTION OR ORACLE ORADATA ORDER ORLANY ORLVARY
         OUT OVERRIDING PARALLEL_ENABLE PARAMETER PASCAL PCTFREE PIPE PIPELINED POLYMORPHIC
-        PRAGMA PRIOR PUBLIC RAISE RECORD RELIES_ON REM RENAME RESOURCE RESULT REVOKE ROWID 
+        PRAGMA PRIOR PUBLIC RAISE RECORD RELIES_ON REM RENAME RESOURCE RESULT REVOKE ROWID
         SB1 SB2 SELECT SEPARATE SET SHARE SHORT SIZE SIZE_T SPARSE SQLCODE SQLDATA
         SQLNAME SQLSTATE STANDARD START STORED STRUCT STYLE SYNONYM TABLE TDO THEN
         TRANSACTIONAL TRIGGER UB1 UB4 UNION UNIQUE UNSIGNED UNTRUSTED UPDATE VALIST
@@ -414,7 +414,7 @@ module Rouge
         PREDICTION PREDICTION_BOUNDS PREDICTION_COST PREDICTION_DETAILS PREDICTION_PROBABILITY
         PREDICTION_SET PRESENTNNV PRESENTV PREVIOUS RANK RATIO_TO_REPORT RAWTOHEX
         RAWTONHEX REFTOHEX REGEXP_COUNT REGEXP_INSTR REGEXP_REPLACE REGEXP_SUBSTR
-        REMAINDER REPLACE ROUND ROUND ROWIDTOCHAR ROWIDTONCHAR ROW_NUMBER RPAD RTRIM
+        REMAINDER REPLACE ROUND ROWIDTOCHAR ROWIDTONCHAR ROW_NUMBER RPAD RTRIM
         SCN_TO_TIMESTAMP SESSIONTIMEZONE SET SIGN SIN SINH SOUNDEX SQRT STANDARD_HASH
         STATS_BINOMIAL_TEST STATS_CROSSTAB STATS_F_TEST STATS_KS_TEST STATS_MODE STATS_MW_TEST
         STATS_ONE_WAY_ANOVA STATS_WSR_TEST STDDEV STDDEV_POP STDDEV_SAMP SUBSTR SUM
@@ -439,9 +439,8 @@ module Rouge
         DATE TIMESTAMP INTERVAL LOCAL TIME ZONE TO MONTH SECOND YEAR DAY
         BLOB CLOB NCLOB BFILE
         UROWID
-        CHARACTER VARYING VARCHAR NATIONAL CHARACTER
-        NUMERIC DECIMAL DEC INTEGER INT SMALLINT
-        FLOAT DOUBLE PRECISION REAL
+        CHARACTER VARYING VARCHAR NATIONAL
+        NUMERIC DECIMAL DEC INTEGER INT SMALLINT DOUBLE PRECISION REAL
         SDO_GEOMETRY SDO_TOPO_GEOMETRY SDO_GEORASTER
         REF ANYTYPE ANYDATA ANYDATASET XMLTYPE HTTPURITYPE XDBURITYPE DUBRITYPE
         BOOLEAN PLS_INTEGER BINARY_INTEGER SIMPLE_FLOAT SIMPLE_INTEGER SIMPLE_DOUBLE SYS_REFCURSOR
@@ -479,7 +478,7 @@ module Rouge
         # Numbers
         rule %r/[+-]?(?:(?:\.\d+(?:[eE][+-]?\d+)?)|\d+\.(?:\d+(?:[eE][+-]?\d+)?)?)[fFdD]?/, Num::Float
         rule %r/[+-]?\d+/, Num::Integer
-        
+
         # Operators
         # Special semi-operator, but this seems an appropriate classification
         rule %r/%(?:TYPE|ROWTYPE|FOUND|ISOPEN|NOTFOUND|ROWCOUNT)\b/i, Name::Attribute
@@ -517,7 +516,7 @@ module Rouge
         #
         rule %r/([a-zA-Z][\w$]*)(\.(?=\w))?/ do |m|
           if self.class.keywords_type.include? m[1].upcase
-            tok = Keyword::Type 
+            tok = Keyword::Type
           elsif self.class.keywords_func.include? m[1].upcase
             tok = Name::Function
           elsif self.class.keywords_reserved.include? m[1].upcase
@@ -562,12 +561,12 @@ module Rouge
         # this rule WILL be true if something pushed into our state. That is our state contract
         rule %r/[a-zA-Z][\w\$]*/ do |m|
           if self.class.keywords_func.include? m[0].upcase
-            # The Function lookup allows collection methods like COUNT, FIRST, LAST, etc.. to be 
+            # The Function lookup allows collection methods like COUNT, FIRST, LAST, etc.. to be
             # classified correctly. Occasionally misidentifies ordinary names as builtin functions,
             # but seems to be as correct as we can get without becoming a full blown parser
             token Name::Function
           else
-            token Name 
+            token Name
           end
           pop!
         end

data/lib/rouge/lexers/postscript.rb

@@ -16,7 +16,7 @@ module Rouge
         return true if /^%!/ =~ text
       end
 
-      delimiter = %s"()<>\[\]{}/%\s"
+      delimiter = :"()<>\\[\\]{}/%\\s"
       delimiter_end = Regexp.new("(?=[#{delimiter}])")
       valid_name_chars = Regexp.new("[^#{delimiter}]")
       valid_name = /#{valid_name_chars}+#{delimiter_end}/

data/lib/rouge/lexers/powershell.rb

@@ -8,7 +8,7 @@ module Rouge
       title 'powershell'
       desc 'powershell'
       tag 'powershell'
-      aliases 'posh', 'microsoftshell', 'msshell'
+      aliases 'posh', 'microsoftshell', 'msshell', 'pwsh'
       filenames '*.ps1', '*.psm1', '*.psd1', '*.psrc', '*.pssc'
       mimetypes 'text/x-powershell'
 

data/lib/rouge/lexers/python.rb

@@ -17,7 +17,7 @@ module Rouge
       end
 
       def self.keywords
-        @keywords ||= %w(
+        @keywords ||= Set.new %w(
           assert break continue del elif else except exec
           finally for global if lambda pass print raise
           return try while yield as with from import
@@ -26,7 +26,7 @@ module Rouge
       end
 
       def self.builtins
-        @builtins ||= %w(
+        @builtins ||= Set.new %w(
           __import__ abs aiter all anext any apply ascii
           basestring bin bool buffer breakpoint bytearray bytes
           callable chr classmethod cmp coerce compile complex
@@ -42,11 +42,11 @@ module Rouge
       end
 
       def self.builtins_pseudo
-        @builtins_pseudo ||= %w(None Ellipsis NotImplemented False True)
+        @builtins_pseudo ||= Set.new %w(None Ellipsis NotImplemented False True)
       end
 
       def self.exceptions
-        @exceptions ||= %w(
+        @exceptions ||= Set.new %w(
           ArithmeticError AssertionError AttributeError BaseException
           BaseExceptionGroup BlockingIOError BrokenPipeError BufferError
           BytesWarning ChildProcessError ConnectionAbortedError ConnectionError
@@ -72,57 +72,49 @@ module Rouge
 
       identifier =        /[[:alpha:]_][[:alnum:]_]*/
       dotted_identifier = /[[:alpha:]_.][[:alnum:]_.]*/
+      inline_ws = /(?:[ \t]|\\\n)*?/
+      inline_content = /(?:[^\\\n]|\\[\n.])*?/
 
       def current_string
-        @string_register ||= StringRegister.new
+        @current_string ||= StringRegister.new
+      end
+
+      operator_words = %r/(in|is|and|or|not)\b/
+      operators = %r{(<<|>>|//|[*][*])=?|!=|[-~+\/*%=<>&^|@]=?|!=}
+
+      start do
+        push :newline
+      end
+
+      state :inline_whitespace do
+        rule %r/[ \t]+/, Text
+        rule %r/\\\n/, Str::Escape
       end
 
       state :root do
-        rule %r/\n+/m, Text
+        rule %r/\n+/m, Text, :newline
         rule %r/^(:)(\s*)([ru]{,2}""".*?""")/mi do
           groups Punctuation, Text, Str::Doc
         end
 
         rule %r/\.\.\.\B$/, Name::Builtin::Pseudo
 
-        rule %r/[^\S\n]+/, Text
-        rule %r(#(.*)?\n?), Comment::Single
-        rule %r/[\[\]{}:(),;.]/, Punctuation
-        rule %r/\\\n/, Text
-        rule %r/\\/, Text
-
-        rule %r/@#{dotted_identifier}/i, Name::Decorator
+        mixin :inline_whitespace
 
-        rule %r/(in|is|and|or|not)\b/, Operator::Word
-        rule %r/(<<|>>|\/\/|\*\*)=?/, Operator
-        rule %r/[-~+\/*%=<>&^|@]=?|!=/, Operator
+        rule %r(#(.*)?\n?), Comment::Single, :newline
+        rule %r/[\[\]{}:(),;]/, Punctuation
+        rule %r/[.]/, Punctuation, :post_dot
+        rule %r/\\/, Str::Escape
 
-        rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(import)/ do
-          groups Keyword::Namespace,
-                 Text,
-                 Name,
-                 Text,
-                 Keyword::Namespace
-        end
-
-        rule %r/(import)(\s+)(#{dotted_identifier})/ do
-          groups Keyword::Namespace, Text, Name
-        end
+        rule %r/@#{dotted_identifier}/i, Name::Decorator
 
-        rule %r/(def)((?:\s|\\\s)+)/ do
-          groups Keyword, Text
-          push :funcname
-        end
+        rule operator_words, Operator::Word
+        rule operators, Operator
 
-        rule %r/(class)((?:\s|\\\s)+)/ do
-          groups Keyword, Text
-          push :classname
-        end
+        rule %r/def\b/, Keyword, :funcname
 
-        rule %r/([a-z_]\w*)[ \t]*(?=(\(.*\)))/m, Name::Function
-        rule %r/([A-Z_]\w*)[ \t]*(?=(\(.*\)))/m, Name::Class
+        rule %r/class\b/, Keyword, :classname
 
-        # TODO: not in python 3
         rule %r/`.*?`/, Str::Backtick
         rule %r/([rtfbu]{0,2})('''|"""|['"])/i do |m|
           groups Str::Affix, Str::Heredoc
@@ -130,17 +122,15 @@ module Rouge
           push :generic_string
         end
 
-        mixin :soft_keywords
-
         # using negative lookbehind so we don't match property names
         rule %r/(?<!\.)#{identifier}/ do |m|
-          if self.class.keywords.include? m[0]
+          if self.class.keywords.include?(m[0])
             token Keyword
-          elsif self.class.exceptions.include? m[0]
-            token Name::Builtin
-          elsif self.class.builtins.include? m[0]
+          elsif self.class.exceptions.include?(m[0])
+            token Name::Exception
+          elsif self.class.builtins.include?(m[0])
             token Name::Builtin
-          elsif self.class.builtins_pseudo.include? m[0]
+          elsif self.class.builtins_pseudo.include?(m[0])
             token Name::Builtin::Pseudo
           else
             token Name
@@ -163,34 +153,80 @@ module Rouge
         rule %r/([1-9](_?[0-9])*|0(_?0)*)/, Num::Integer
       end
 
+      state :import do
+        mixin :inline_whitespace
+        rule dotted_identifier, Name::Namespace, :pop!
+        rule(//) { pop! }
+      end
+
+      state :from do
+        mixin :inline_whitespace
+
+        rule dotted_identifier do
+          token Name::Namespace
+          goto :from_import
+        end
+
+        rule(//) { pop! }
+      end
+
+      # import after from, meaning we don't push the :import state
+      state :from_import do
+        mixin :inline_whitespace
+        rule %r/import\b/, Keyword::Namespace, :pop!
+        rule(//) { pop! }
+      end
+
+      state :post_dot do
+        mixin :inline_whitespace
+        rule %r/([A-Z]\w*)(?=#{inline_ws}[(])/m, Name::Class
+        rule %r/(#{identifier})(?=#{inline_ws}[(])/m, Name::Function
+        rule(//) { pop! }
+      end
+
+      state :newline do
+        mixin :inline_whitespace
+
+        rule %r/from\b/, Keyword::Namespace, :from
+        rule %r/import\b/, Keyword::Namespace, :import
+
+        # [jneen] This lookahead is a best-effort hack, since soft keywords are
+        # technically not possible to detect in the lexing stage. If we see an
+        # operator like `and`, `or`, inline `if`, etc which would expect an
+        # expression beforehand, we know that it is almost certainly not a keyword.
+        inline_ops = /#{operator_words}|if\b|#{operators}/
+        rule %r/(?:case|match)(?=#{inline_ws}#{inline_ops})/, Name::Other, :pop!
+
+        rule %r/(?:case|match)(?=#{inline_content}:#{inline_ws}[#\n])/ do |m|
+          token Keyword
+          if m[0] == 'case'
+            goto :case_pattern
+          else
+            pop!
+          end
+        end
+
+        rule(//) { pop! }
+      end
+
       state :funcname do
+        mixin :inline_whitespace
         rule identifier, Name::Function, :pop!
       end
 
       state :classname do
+        mixin :inline_whitespace
         rule identifier, Name::Class, :pop!
       end
 
-      state :soft_keywords do
-        rule %r/
-          (^[ \t]*)
-          (match|case)\b
-          (?![ \t]*
-            (?:[:,;=^&|@~)\]}] |
-              (?:#{Python.keywords.join('|')})\b))
-        /x do |m|
-          token Text::Whitespace, m[1]
-          token Keyword, m[2]
-          push :soft_keywords_inner
-        end
-      end
-
-      state :soft_keywords_inner do
-        rule %r((\s+)([^\n_]*)(_\b)) do |m|
-          groups Text::Whitespace, Text, Keyword
+      state :case_pattern do
+        rule %r/\n/ do
+          token Text
+          goto :newline
         end
 
-        rule(//) { pop! }
+        rule %r/_\b/, Keyword
+        mixin :root
       end
 
       state :raise do
@@ -207,8 +243,8 @@ module Rouge
       end
 
       state :generic_string do
-        rule %r/^\s*(>>>|\.\.\.)\B/, Generic::Prompt, :doctest
-        rule %r/[^'"\\{]+?/, Str
+        rule %r/\n/, Str, :generic_string_newline
+        rule %r/[^'"\\{\n]+/, Str
         rule %r/{{/, Str
 
         rule %r/'''|"""|['"]/ do |m|
@@ -231,6 +267,15 @@ module Rouge
         end
       end
 
+      state :generic_string_newline do
+        rule %r/[ \t]+/, Str
+        rule %r/(>>>|\.\.\.)\B/ do
+          token Generic::Prompt
+          goto :doctest
+        end
+        rule(//) { pop! }
+      end
+
       state :generic_escape do
         rule %r(\\
           ( [\\abfnrtv"']

data/lib/rouge/lexers/qml.rb

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'javascript'
+
 module Rouge
   module Lexers
-    load_lexer 'javascript.rb'
-
     class Qml < Javascript
       title "QML"
       desc 'QML, a UI markup language'

data/lib/rouge/lexers/reasonml.rb

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'ocaml/common'
+
 module Rouge
   module Lexers
-    load_lexer 'ocaml/common.rb'
-
     class ReasonML < OCamlCommon
       title "ReasonML"
       desc 'New syntax on top of OCaml ecosystem (reasonml.github.io)'

data/lib/rouge/lexers/rego.rb

@@ -2,59 +2,59 @@
 # frozen_string_literal: true
 
 module Rouge
-    module Lexers
-        class Rego < RegexLexer
-            title "Rego"
-            desc "The Rego open-policy-agent (OPA) policy language (openpolicyagent.org)"
-            tag 'rego'
-            filenames '*.rego'
-
-            def self.constants
-              @constants ||= Set.new %w(
-                true false null
-              )
-            end
-
-            def self.operators
-              @operators ||= Set.new %w(
-                as default else import not package some with
-              )
-            end
-
-            state :basic do
-              rule %r/\s+/, Text
-              rule %r/#.*/, Comment::Single
-
-              rule %r/[\[\](){}|.,;!]/, Punctuation
-
-              rule %r/"[^"]*"/, Str::Double
-
-              rule %r/-?\d+\.\d+([eE][+-]?\d+)?/, Num::Float
-              rule %r/-?\d+([eE][+-]?\d+)?/, Num
-
-              rule %r/\\u[0-9a-fA-F]{4}/, Num::Hex
-              rule %r/\\["\/bfnrt]/, Str::Escape
-            end
-
-            state :operators do
-              rule %r/(=|!=|>=|<=|>|<|\+|-|\*|%|\/|\||&|:=)/, Operator
-              rule %r/[\/:?@^~]+/, Operator
-            end
-
-            state :root do
-              mixin :basic
-              mixin :operators
-
-              rule %r/[[:word:]]+/ do |m|
-                if self.class.constants.include? m[0]
-                  token Keyword::Constant
-                elsif self.class.operators.include? m[0]
-                  token Operator::Word
-                else
-                  token Name
-                end
-              end
-            end
+  module Lexers
+    class Rego < RegexLexer
+      title "Rego"
+      desc "The Rego open-policy-agent (OPA) policy language (openpolicyagent.org)"
+      tag 'rego'
+      filenames '*.rego'
+
+      def self.constants
+        @constants ||= Set.new %w(
+          true false null
+        )
+      end
+
+      def self.operators
+        @operators ||= Set.new %w(
+          as default else import not package some with
+        )
+      end
+
+      state :basic do
+        rule %r/\s+/, Text
+        rule %r/#.*/, Comment::Single
+
+        rule %r/[\[\](){}|.,;!]/, Punctuation
+
+        rule %r/"[^"]*"/, Str::Double
+
+        rule %r/-?\d+\.\d+([eE][+-]?\d+)?/, Num::Float
+        rule %r/-?\d+([eE][+-]?\d+)?/, Num
+
+        rule %r/\\u[0-9a-fA-F]{4}/, Num::Hex
+        rule %r/\\["\/bfnrt]/, Str::Escape
+      end
+
+      state :operators do
+        rule %r/(=|!=|>=|<=|>|<|\+|-|\*|%|\/|\||&|:=)/, Operator
+        rule %r/[\/:?@^~]+/, Operator
+      end
+
+      state :root do
+        mixin :basic
+        mixin :operators
+
+        rule %r/[[:word:]]+/ do |m|
+          if self.class.constants.include? m[0]
+            token Keyword::Constant
+          elsif self.class.operators.include? m[0]
+            token Operator::Word
+          else
+            token Name
+          end
         end
+      end
     end
+  end
 end

data/lib/rouge/lexers/rescript.rb

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'ocaml/common'
+
 module Rouge
   module Lexers
-    load_lexer 'ocaml/common.rb'
-
     class ReScript < OCamlCommon
       title "ReScript"
       desc "The ReScript programming language (rescript-lang.org)"