rouge 3.16.0 → 3.21.0

data/lib/rouge/lexers/powershell.rb

@@ -165,6 +165,7 @@ module Rouge
       end
 
       state :parameters do
+        rule %r/`./m, Str::Escape
         rule %r/\s*?\n/, Text::Whitespace, :pop!
         rule %r/[;(){}\]]/, Punctuation, :pop!
         rule %r/[|=]/, Operator, :pop!
@@ -201,7 +202,7 @@ module Rouge
         rule %r/(?:#{KEYWORDS})\b(?![-.])/i, Keyword::Reserved
 
         rule %r/-{1,2}\w+/, Name::Tag
-        
+
         rule %r/(\.)?([-\w]+)(\[)/ do |m|
           groups Operator, Name::Function, Punctuation
           push :bracket
@@ -209,14 +210,15 @@ module Rouge
 
         rule %r/([\/\\~\w][-.:\/\\~\w]*)(\n)?/ do |m|
           groups Name::Function, Text::Whitespace
-          push :parameters unless m[2]
+          push :parameters
         end
 
         rule %r/(\.)?([-\w]+)(?:(\()|(\n))?/ do |m|
           groups Operator, Name::Function, Punctuation, Text::Whitespace
-          push :parameters unless m[3]
+          push :parameters unless m[3].nil?
         end
 
+        rule %r/\?/, Name::Function, :parameters
         rule %r/[-+*\/%=!.&|]/, Operator
         rule %r/@\{/, Punctuation, :hasht
         rule %r/@\(/, Punctuation, :array

data/lib/rouge/lexers/python.rb

@@ -67,8 +67,13 @@ module Rouge
         )
       end
 
-      identifier =        /[a-z_][a-z0-9_]*/i
-      dotted_identifier = /[a-z_.][a-z0-9_.]*/i
+      identifier =        /[[:alpha:]_][[:alnum:]_]*/
+      dotted_identifier = /[[:alpha:]_.][[:alnum:]_.]*/
+
+      def current_string
+        @string_register ||= StringRegister.new
+      end
+
       state :root do
         rule %r/\n+/m, Text
         rule %r/^(:)(\s*)([ru]{,2}""".*?""")/mi do
@@ -77,14 +82,13 @@ module Rouge
 
         rule %r/[^\S\n]+/, Text
         rule %r(#(.*)?\n?), Comment::Single
-        rule %r/[\[\]{}:(),;]/, Punctuation
+        rule %r/[\[\]{}:(),;.]/, Punctuation
         rule %r/\\\n/, Text
         rule %r/\\/, Text
 
         rule %r/(in|is|and|or|not)\b/, Operator::Word
         rule %r/(<<|>>|\/\/|\*\*)=?/, Operator
         rule %r/[-~+\/*%=<>&^|@]=?|!=/, Operator
-        rule %r/\.(?![0-9])/, Operator  # so it doesn't match float literals
 
         rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(import)/ do
           groups Keyword::Namespace,
@@ -110,14 +114,11 @@ module Rouge
 
         # TODO: not in python 3
         rule %r/`.*?`/, Str::Backtick
-        rule %r/(?:r|ur|ru)"""/i, Str, :raw_tdqs
-        rule %r/(?:r|ur|ru)'''/i, Str, :raw_tsqs
-        rule %r/(?:r|ur|ru)"/i,   Str, :raw_dqs
-        rule %r/(?:r|ur|ru)'/i,   Str, :raw_sqs
-        rule %r/u?"""/i,          Str, :tdqs
-        rule %r/u?'''/i,          Str, :tsqs
-        rule %r/u?"/i,            Str, :dqs
-        rule %r/u?'/i,            Str, :sqs
+        rule %r/([rfbu]{0,2})('''|"""|['"])/i do |m|
+          token Str
+          current_string.register type: m[1].downcase, delim: m[2]
+          push :generic_string
+        end
 
         rule %r/@#{dotted_identifier}/i, Name::Decorator
 
@@ -173,76 +174,76 @@ module Rouge
         mixin :raise
       end
 
-      state :strings do
-        rule %r/%(\([a-z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/i, Str::Interpol
-      end
+      state :generic_string do
+        rule %r/[^'"\\{]+/, Str
+        rule %r/{{/, Str
 
-      state :strings_double do
-        rule %r/[^\\"%\n]+/, Str
-        mixin :strings
-      end
+        rule %r/'''|"""|['"]/ do |m|
+          token Str
+          if current_string.delim? m[0]
+            current_string.remove
+            pop!
+          end
+        end
 
-      state :strings_single do
-        rule %r/[^\\'%\n]+/, Str
-        mixin :strings
-      end
+        rule %r/(?=\\)/, Str, :generic_escape
 
-      state :nl do
-        rule %r/\n/, Str
+        rule %r/{/ do |m|
+          if current_string.type? "f"
+            token Str::Interpol
+            push :generic_interpol
+          else
+            token Str
+          end
+        end
       end
 
-      state :escape do
+      state :generic_escape do
         rule %r(\\
           ( [\\abfnrtv"']
           | \n
+          | newline
           | N{[a-zA-Z][a-zA-Z ]+[a-zA-Z]}
           | u[a-fA-F0-9]{4}
           | U[a-fA-F0-9]{8}
           | x[a-fA-F0-9]{2}
           | [0-7]{1,3}
           )
-        )x, Str::Escape
-      end
+        )x do
+          current_string.type?("r") ? token(Str) : token(Str::Escape)
+          pop!
+        end
 
-      state :raw_escape do
-        rule %r/\\./, Str
+        rule %r/\\./, Str, :pop!
       end
 
-      state :dqs do
-        rule %r/"/, Str, :pop!
-        mixin :escape
-        mixin :strings_double
+      state :generic_interpol do
+        rule %r/[^{}]+/ do |m|
+          recurse m[0]
+        end
+        rule %r/{/, Str::Interpol, :generic_interpol
+        rule %r/}/, Str::Interpol, :pop!
       end
 
-      state :sqs do
-        rule %r/'/, Str, :pop!
-        mixin :escape
-        mixin :strings_single
-      end
+      class StringRegister < Array
+        def delim?(delim)
+          self.last[1] == delim
+        end
 
-      state :tdqs do
-        rule %r/"""/, Str, :pop!
-        rule %r/"/, Str
-        mixin :escape
-        mixin :strings_double
-        mixin :nl
-      end
+        def register(type: "u", delim: "'")
+          self.push [type, delim]
+        end
 
-      state :tsqs do
-        rule %r/'''/, Str, :pop!
-        rule %r/'/, Str
-        mixin :escape
-        mixin :strings_single
-        mixin :nl
-      end
+        def remove
+          self.pop
+        end
 
-      %w(tdqs tsqs dqs sqs).each do |qtype|
-        state :"raw_#{qtype}" do
-          mixin :raw_escape
-          mixin :"#{qtype}"
+        def type?(type)
+          self.last[0].include? type
         end
       end
 
+      private_constant :StringRegister
     end
   end
 end

data/lib/rouge/lexers/racket.rb

@@ -488,6 +488,9 @@ module Rouge
       state :root do
         # comments
         rule %r/;.*$/, Comment::Single
+        rule %r/#!.*/, Comment::Single
+        rule %r/#\|/, Comment::Multiline, :block_comment
+        rule %r/#;/, Comment::Multiline, :sexp_comment
         rule %r/\s+/m, Text
 
         rule %r/[+-]inf[.][f0]/, Num::Float
@@ -510,7 +513,7 @@ module Rouge
         rule %r/['`]#{id}/i, Str::Symbol
         rule %r/#\\([()\/'"._!\$%& ?=+-]{1}|[a-z0-9]+)/i,
           Str::Char
-        rule %r/#t|#f/, Name::Constant
+        rule %r/#t(rue)?|#f(alse)?/i, Name::Constant
         rule %r/(?:'|#|`|,@|,|\.)/, Operator
 
         rule %r/(['#])(\s*)(\()/m do
@@ -524,6 +527,26 @@ module Rouge
         rule id, Name::Variable
       end
 
+      state :block_comment do
+        rule %r/[^|#]+/, Comment::Multiline
+        rule %r/\|#/, Comment::Multiline, :pop!
+        rule %r/#\|/, Comment::Multiline, :block_comment
+        rule %r/[|#]/, Comment::Multiline
+      end
+
+      state :sexp_comment do
+        rule %r/[({\[]/, Comment::Multiline, :sexp_comment_inner
+        rule %r/"(?:\\"|[^"])*?"/, Comment::Multiline, :pop!
+        rule %r/[^\s]+/, Comment::Multiline, :pop!
+        rule(//) { pop! }
+      end
+
+      state :sexp_comment_inner do
+        rule %r/[^(){}\[\]]+/, Comment::Multiline
+        rule %r/[)}\]]/, Comment::Multiline, :pop!
+        rule %r/[({\[]/, Comment::Multiline, :sexp_comment_inner
+      end
+
       state :command do
         rule id, Name::Function do |m|
           if self.class.keywords.include? m[0]

data/lib/rouge/lexers/rego.rb

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+    module Lexers
+        class Rego < RegexLexer
+            title "Rego"
+            desc "The Rego open-policy-agent (OPA) policy language (openpolicyagent.org)"
+            tag 'rego'
+            filenames '*.rego'
+
+            def self.constants
+              @constants ||= Set.new %w(
+                true false null
+              )
+            end
+
+            def self.operators
+              @operators ||= Set.new %w(
+                as default else import not package some with
+              )
+            end
+
+            state :basic do
+              rule %r/\s+/, Text
+              rule %r/#.*/, Comment::Single
+
+              rule %r/[\[\](){}|.,;!]/, Punctuation
+
+              rule %r/"[^"]*"/, Str::Double
+
+              rule %r/-?\d+\.\d+([eE][+-]?\d+)?/, Num::Float
+              rule %r/-?\d+([eE][+-]?\d+)?/, Num
+
+              rule %r/\\u[0-9a-fA-F]{4}/, Num::Hex
+              rule %r/\\["\/bfnrt]/, Str::Escape
+            end
+
+            state :operators do
+              rule %r/(=|!=|>=|<=|>|<|\+|-|\*|%|\/|\||&|:=)/, Operator
+              rule %r/[\/:?@^~]+/, Operator
+            end
+
+            state :root do
+              mixin :basic
+              mixin :operators
+
+              rule %r/[[:word:]]+/ do |m|
+                if self.class.constants.include? m[0]
+                  token Keyword::Constant
+                elsif self.class.operators.include? m[0]
+                  token Operator::Word
+                else
+                  token Name
+                end
+              end
+            end
+        end
+    end
+end

data/lib/rouge/lexers/ruby.rb

@@ -170,8 +170,11 @@ module Rouge
         rule %r/0_?[0-7]+(?:_[0-7]+)*/, Num::Oct
         rule %r/0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Hex
         rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
-        rule %r/\d+\.\d+(e[\+\-]?\d+)?/, Num::Float
-        rule %r/[\d]+(?:_\d+)*/, Num::Integer
+
+        decimal = %r/[\d]+(?:_\d+)*/
+        exp = %r/e[+-]?\d+/i
+        rule %r/#{decimal}(?:\.#{decimal}#{exp}?|#{exp})/, Num::Float
+        rule decimal, Num::Integer
 
         # names
         rule %r/@@[a-z_]\w*/i, Name::Variable::Class
@@ -306,6 +309,8 @@ module Rouge
 
       state :classname do
         rule %r/\s+/, Text
+        rule %r/\w+(::\w+)+/, Name::Class
+
         rule %r/\(/ do
           token Punctuation
           push :defexpr
@@ -324,7 +329,15 @@ module Rouge
       end
 
       state :ternary do
-        rule(/:(?!:)/) { token Punctuation; goto :expr_start }
+        rule %r/(:)(\s+)/ do
+          groups Punctuation, Text
+          goto :expr_start
+        end
+
+        rule %r/:(?![^#\n]*?[:\\])/ do
+          token Punctuation
+          goto :expr_start
+        end
 
         mixin :root
       end

data/lib/rouge/lexers/sass/common.rb

@@ -29,6 +29,7 @@ module Rouge
         end
 
         rule %r/@#{id}/, Keyword, :selector
+        rule %r/&/, Keyword, :selector
 
         # $variable: assignment
         rule %r/([$]#{id})([ \t]*)(:)/ do

data/lib/rouge/lexers/scala.rb

@@ -40,7 +40,7 @@ module Rouge
           groups Keyword, Text
           push :class
         end
-        rule %r/'#{idrest}[^']/, Str::Symbol
+        rule %r/'#{idrest}(?!')/, Str::Symbol
         rule %r/[^\S\n]+/, Text
 
         rule %r(//.*), Comment::Single

data/lib/rouge/lexers/solidity.rb

@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class Solidity < RegexLexer
+      title "Solidity"
+      desc "Solidity, an Ethereum smart contract programming language"
+      tag 'solidity'
+      filenames '*.sol', '*.solidity'
+      mimetypes 'text/x-solidity'
+
+      # optional comment or whitespace
+      ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+)
+      id = /[a-zA-Z$_][\w$_]*/
+
+      def self.detect?(text)
+        return true if text.start_with? 'pragma solidity'
+      end
+
+      # TODO: seperate by "type"
+      def self.keywords
+        @keywords ||= Set.new %w(
+          abstract anonymous as assembly break catch calldata constant
+          constructor continue contract do delete else emit enum event
+          external fallback for function hex if indexed interface
+          internal import is library mapping memory modifier new
+          override payable public pure pragma private receive return
+          returns storage struct throw try type using var view virtual
+          while
+        )
+      end
+
+      def self.builtins
+        return @builtins if @builtins
+
+        @builtins = Set.new %w(
+          now
+          false true
+          balance now selector super this
+          blockhash gasleft
+          assert require revert
+          selfdestruct suicide
+          call callcode delegatecall
+          send transfer
+          addmod ecrecover keccak256 mulmod sha256 sha3 ripemd160
+        )
+
+        # TODO: use (currently shadowed by catch-all in :statements)
+        abi = %w(decode encode encodePacked encodeWithSelector encodeWithSignature)
+        @builtins.merge( abi.map { |i| "abi.#{i}" } )
+        block = %w(coinbase difficulty gaslimit hash number timestamp)
+        @builtins.merge( block.map { |i| "block.#{i}" } )
+        msg = %w(data gas sender sig value)
+        @builtins.merge( msg.map { |i| "msg.#{i}" } )
+        tx = %w(gasprice origin)
+        @builtins.merge( tx.map { |i| "tx.#{i}" } )
+      end
+
+      def self.constants
+        @constants ||= Set.new %w(
+          wei finney szabo ether
+          seconds minutes hours days weeks years
+        )
+      end
+
+      def self.keywords_type
+        @keywords_type ||= Set.new %w(
+          address bool byte bytes int string uint
+        )
+      end
+
+      def self.reserved
+        @reserved ||= Set.new %w(
+          alias after apply auto case copyof default define final fixed
+          immutable implements in inline let macro match mutable null of
+          partial promise reference relocatable sealed sizeof static
+          supports switch typedef typeof ufixed unchecked
+        )
+      end
+
+      start { push :bol }
+
+      state :expr_bol do
+        mixin :inline_whitespace
+
+        rule(//) { pop! }
+      end
+
+      # :expr_bol is the same as :bol but without labels, since
+      # labels can only appear at the beginning of a statement.
+      state :bol do
+        mixin :expr_bol
+      end
+
+      # TODO: natspec in comments
+      state :inline_whitespace do
+        rule %r/[ \t\r]+/, Text
+        rule %r/\\\n/, Text # line continuation
+        rule %r(/\*), Comment::Multiline, :comment_multi
+      end
+
+      state :whitespace do
+        rule %r/\n+/m, Text, :bol
+        rule %r(//(\\.|.)*?\n), Comment::Single, :bol
+        mixin :inline_whitespace
+      end
+
+      state :expr_whitespace do
+        rule %r/\n+/m, Text, :expr_bol
+        mixin :whitespace
+      end
+
+      state :statements do
+        mixin :whitespace
+        rule %r/(hex)?\"/, Str, :string_double
+        rule %r/(hex)?\'/, Str, :string_single
+        rule %r('(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\'\n])')i, Str::Char
+        rule %r/\d\d*\.\d+([eE]\d+)?/i, Num::Float
+        rule %r/0x[0-9a-f]+/i, Num::Hex
+        rule %r/\d+([eE]\d+)?/i, Num::Integer
+        rule %r(\*/), Error
+        rule %r([~!%^&*+=\|?:<>/-]), Operator
+        rule %r/[()\[\],.]/, Punctuation
+        rule %r/u?fixed\d+x\d+/, Keyword::Reserved
+        rule %r/bytes\d+/, Keyword::Type
+        rule %r/u?int\d+/, Keyword::Type
+        rule id do |m|
+          name = m[0]
+
+          if self.class.keywords.include? name
+            token Keyword
+          elsif self.class.builtins.include? name
+            token Name::Builtin
+          elsif self.class.constants.include? name
+            token Keyword::Constant
+          elsif self.class.keywords_type.include? name
+            token Keyword::Type
+          elsif self.class.reserved.include? name
+            token Keyword::Reserved
+          else
+            token Name
+          end
+        end
+      end
+
+      state :root do
+        mixin :expr_whitespace
+        rule(//) { push :statement }
+        # TODO: function declarations
+      end
+
+      state :statement do
+        rule %r/;/, Punctuation, :pop!
+        mixin :expr_whitespace
+        mixin :statements
+        rule %r/[{}]/, Punctuation
+      end
+
+      state :string_common do
+        rule %r/\\(u[a-fA-F0-9]{4}|x..|[^x])/, Str::Escape
+        rule %r/[^\\\"\'\n]+/, Str
+        rule %r/\\\n/, Str # line continuation
+        rule %r/\\/, Str # stray backslash
+      end
+
+      state :string_double do
+        mixin :string_common
+        rule %r/\"/, Str, :pop!
+        rule %r/\'/, Str
+      end
+
+      state :string_single do
+        mixin :string_common
+        rule %r/\'/, Str, :pop!
+        rule %r/\"/, Str
+      end
+
+       state :comment_multi do
+         rule %r(\*/), Comment::Multiline, :pop!
+         rule %r([^*/]+), Comment::Multiline
+         rule %r([*/]), Comment::Multiline
+       end
+    end
+  end
+end