rouge 3.3.0 → 3.8.0

data/lib/rouge/lexers/csharp.rb

@@ -43,59 +43,58 @@ module Rouge
       )
 
       state :whitespace do
-        rule /\s+/m, Text
+        rule %r/\s+/m, Text
         rule %r(//.*?$), Comment::Single
         rule %r(/[*].*?[*]/)m, Comment::Multiline
       end
 
       state :nest do
-        rule /{/, Punctuation, :nest
-        rule /}/, Punctuation, :pop!
+        rule %r/{/, Punctuation, :nest
+        rule %r/}/, Punctuation, :pop!
         mixin :root
       end
 
       state :splice_string do
-        rule /\\./, Str
-        rule /{/, Punctuation, :nest
-        rule /"|\n/, Str, :pop!
-        rule /./, Str
+        rule %r/\\./, Str
+        rule %r/{/, Punctuation, :nest
+        rule %r/"|\n/, Str, :pop!
+        rule %r/./, Str
       end
 
       state :splice_literal do
-        rule /""/, Str
-        rule /{/, Punctuation, :nest
-        rule /"/, Str, :pop!
-        rule /./, Str
+        rule %r/""/, Str
+        rule %r/{/, Punctuation, :nest
+        rule %r/"/, Str, :pop!
+        rule %r/./, Str
       end
 
       state :root do
         mixin :whitespace
 
-        rule /^\s*\[.*?\]/, Name::Attribute
-        rule /[$]\s*"/, Str, :splice_string
-        rule /[$]@\s*"/, Str, :splice_literal
+        rule %r/[$]\s*"/, Str, :splice_string
+        rule %r/[$]@\s*"/, Str, :splice_literal
 
-        rule /(<\[)\s*(#{id}:)?/, Keyword
-        rule /\]>/, Keyword
+        rule %r/(<\[)\s*(#{id}:)?/, Keyword
+        rule %r/\]>/, Keyword
 
-        rule /[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation
-        rule /@"(""|[^"])*"/m, Str
-        rule /"(\\.|.)*?["\n]/, Str
-        rule /'(\\.|.)'/, Str::Char
-        rule /0x[0-9a-f]+[lu]?/i, Num
+        rule %r/[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation
+        rule %r/@"(""|[^"])*"/m, Str
+        rule %r/"(\\.|.)*?["\n]/, Str
+        rule %r/'(\\.|.)'/, Str::Char
+        rule %r/0x[0-9a-f]+[lu]?/i, Num
         rule %r(
           [0-9]
           ([.][0-9]*)? # decimal
           (e[+-][0-9]+)? # exponent
           [fldu]? # type
         )ix, Num
-        rule /\b(?:class|struct|interface)\b/, Keyword, :class
-        rule /\b(?:namespace|using)\b/, Keyword, :namespace
-        rule /^#[ \t]*(#{cpp_keywords.join('|')})\b.*?\n/,
+        rule %r/\b(?:class|struct|interface)\b/, Keyword, :class
+        rule %r/\b(?:namespace|using)\b/, Keyword, :namespace
+        rule %r/^#[ \t]*(#{cpp_keywords.join('|')})\b.*?\n/,
           Comment::Preproc
-        rule /\b(#{keywords.join('|')})\b/, Keyword
-        rule /\b(#{keywords_type.join('|')})\b/, Keyword::Type
-        rule /#{id}(?=\s*[(])/, Name::Function
+        rule %r/\b(#{keywords.join('|')})\b/, Keyword
+        rule %r/\b(#{keywords_type.join('|')})\b/, Keyword::Type
+        rule %r/#{id}(?=\s*[(])/, Name::Function
         rule id, Name
       end
 
@@ -106,8 +105,8 @@ module Rouge
 
       state :namespace do
         mixin :whitespace
-        rule /(?=[(])/, Text, :pop!
-        rule /(#{id}|[.])+/, Name::Namespace, :pop!
+        rule %r/(?=[(])/, Text, :pop!
+        rule %r/(#{id}|[.])+/, Name::Namespace, :pop!
       end
 
     end

data/lib/rouge/lexers/css.rb

@@ -185,25 +185,25 @@ module Rouge
 
       state :root do
         mixin :basics
-        rule /{/, Punctuation, :stanza
-        rule /:[:]?#{identifier}/, Name::Decorator
-        rule /\.#{identifier}/, Name::Class
-        rule /##{identifier}/, Name::Function
-        rule /@#{identifier}/, Keyword, :at_rule
+        rule %r/{/, Punctuation, :stanza
+        rule %r/:[:]?#{identifier}/, Name::Decorator
+        rule %r/\.#{identifier}/, Name::Class
+        rule %r/##{identifier}/, Name::Function
+        rule %r/@#{identifier}/, Keyword, :at_rule
         rule identifier, Name::Tag
         rule %r([~^*!%&\[\]()<>|+=@:;,./?-]), Operator
-        rule /"(\\\\|\\"|[^"])*"/, Str::Single
-        rule /'(\\\\|\\'|[^'])*'/, Str::Double
+        rule %r/"(\\\\|\\"|[^"])*"/, Str::Single
+        rule %r/'(\\\\|\\'|[^'])*'/, Str::Double
       end
 
       state :value do
         mixin :basics
-        rule /url\(.*?\)/, Str::Other
-        rule /#[0-9a-f]{1,6}/i, Num # colors
-        rule /#{number}(?:%|(?:em|px|pt|pc|in|mm|cm|ex|rem|ch|vw|vh|vmin|vmax|dpi|dpcm|dppx|deg|grad|rad|turn|s|ms|Hz|kHz)\b)?/, Num
-        rule /[\[\]():\/.,]/, Punctuation
-        rule /"(\\\\|\\"|[^"])*"/, Str::Single
-        rule /'(\\\\|\\'|[^'])*'/, Str::Double
+        rule %r/url\(.*?\)/, Str::Other
+        rule %r/#[0-9a-f]{1,6}/i, Num # colors
+        rule %r/#{number}(?:%|(?:em|px|pt|pc|in|mm|cm|ex|rem|ch|vw|vh|vmin|vmax|dpi|dpcm|dppx|deg|grad|rad|turn|s|ms|Hz|kHz)\b)?/, Num
+        rule %r/[\[\]():\/.,]/, Punctuation
+        rule %r/"(\\\\|\\"|[^"])*"/, Str::Single
+        rule %r/'(\\\\|\\'|[^'])*'/, Str::Double
         rule(identifier) do |m|
           if self.class.constants.include? m[0]
             token Name::Constant
@@ -216,9 +216,9 @@ module Rouge
       end
 
       state :at_rule do
-        rule /{(?=\s*#{identifier}\s*:)/m, Punctuation, :at_stanza
-        rule /{/, Punctuation, :at_body
-        rule /;/, Punctuation, :pop!
+        rule %r/{(?=\s*#{identifier}\s*:)/m, Punctuation, :at_stanza
+        rule %r/{/, Punctuation, :at_body
+        rule %r/;/, Punctuation, :pop!
         mixin :value
       end
 
@@ -233,21 +233,21 @@ module Rouge
       end
 
       state :at_content do
-        rule /}/ do
+        rule %r/}/ do
           token Punctuation
           pop! 2
         end
       end
 
       state :basics do
-        rule /\s+/m, Text
+        rule %r/\s+/m, Text
         rule %r(/\*(?:.*?)\*/)m, Comment
       end
 
       state :stanza do
         mixin :basics
-        rule /}/, Punctuation, :pop!
-        rule /(#{identifier})(\s*)(:)/m do |m|
+        rule %r/}/, Punctuation, :pop!
+        rule %r/(#{identifier})(\s*)(:)/m do |m|
           name_tok = if self.class.attributes.include? m[1]
             Name::Label
           elsif self.class.vendor_prefixes.any? { |p| m[1].start_with?(p) }
@@ -263,10 +263,10 @@ module Rouge
       end
 
       state :stanza_value do
-        rule /;/, Punctuation, :pop!
+        rule %r/;/, Punctuation, :pop!
         rule(/(?=})/) { pop! }
-        rule /!\s*important\b/, Comment::Preproc
-        rule /^@.*?$/, Comment::Preproc
+        rule %r/!\s*important\b/, Comment::Preproc
+        rule %r/^@.*?$/, Comment::Preproc
         mixin :value
       end
     end

data/lib/rouge/lexers/cuda.rb

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    load_lexer 'cpp.rb'
+
+    class CUDA < Cpp
+      title "CUDA"
+      desc "Compute Unified Device Architecture, used for programming with NVIDIA GPU"
+
+      tag 'cuda'
+      filenames '*.cu', '*.cuh'
+
+      def self.keywords
+        @keywords ||= super + Set.new(%w(
+          __global__ __device__ __host__ __noinline__ __forceinline__
+          __constant__ __shared__ __managed__ __restrict__
+        ))
+      end
+
+      def self.keywords_type
+        @keywords_type ||= super + Set.new(%w(
+          char1 char2 char3 char4 uchar1 uchar2 uchar3 uchar4
+          short1 short2 short3 short4 ushort1 ushort2 ushort3 ushort4
+          int1 int2 int3 int4 uint1 uint2 uint3 uint4
+          long1 long2 long3 long4 ulong1 ulong2 ulong3 ulong4
+          longlong1 longlong2 longlong3 longlong4 
+          ulonglong1 ulonglong2 ulonglong3 ulonglong4 
+          float1 float2 float3 float4 double1 double2 double3 double4
+          dim3
+        ))
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cython.rb

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'python.rb'
+
+    class Cython < Python
+      title "Cython"
+      desc "Cython and Pyrex source code (cython.org)"
+      tag 'cython'
+      aliases 'pyx', 'pyrex'
+      filenames '*.pyx', '*.pxd', '*.pxi'
+      mimetypes 'text/x-cython', 'application/x-cython'
+
+      def initialize(opts = {})
+        super opts
+        @indentation = nil
+      end
+
+      def self.keywords
+        @keywords ||= super + %w(
+          by except? fused gil nogil
+        )
+      end
+
+      def self.c_keywords
+        @ckeywords ||= %w(
+          public readonly extern api inline enum union
+        )
+      end
+
+      identifier = /[a-z_]\w*/i
+      dotted_identifier = /[a-z_.][\w.]*/i
+
+      prepend :root do
+        rule %r/cp?def|ctypedef/ do
+          token Keyword
+          push :c_definitions
+          push :c_start
+        end
+
+        rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(cimport)/ do
+          groups Keyword::Namespace,
+                 Text,
+                 Name::Namespace,
+                 Text,
+                 Keyword::Namespace
+        end
+
+        rule %r/(cimport)(\s+)(#{dotted_identifier})/ do
+          groups Keyword::Namespace, Text, Name::Namespace
+        end
+
+        rule %r/(struct)((?:\\\s|\s)+)/ do
+          groups Keyword, Text
+          push :classname
+        end
+
+        mixin :func_call_fix
+
+        rule %r/[(,]/, Punctuation, :c_start
+      end
+
+      prepend :classname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+
+      prepend :funcname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+      # This is a fix for the way that function calls are lexed in the Python
+      # lexer. This should be moved to the Python lexer once confirmed that it
+      # does not cause any regressions.
+      state :func_call_fix do
+        rule %r/#{identifier}(?=\()/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+          elsif self.class.exceptions.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins_pseudo.include? m[0]
+            token Name::Builtin::Pseudo
+          else
+            token Name::Function
+          end
+        end
+      end
+
+      # The Cython lexer adds three states to those already in the Python lexer.
+      # Calls to `cdef`, `cpdef` and `ctypedef` move the lexer into the :c_start
+      # state. The primary purpose of this state is to highlight datatypes. Once
+      # this has been done, the lexer moves to the :c_definitions state where
+      # the majority of text in a definition is lexed. Finally, newlines cause
+      # the lexer to move to :c_indent. This state is used to check whether we
+      # have moved out of a C block.
+
+      state :c_start do
+        rule %r/[^\S\n]+/, Text
+
+        rule %r/cp?def|ctypedef/, Keyword
+
+        rule %r/(?:un)?signed/, Keyword::Type
+
+        # This rule matches identifiers that could be type declarations. The
+        # lookahead matches (1) pointers, (2) arrays and (3) variable names.
+        rule %r/#{identifier}(?=(?:\*+)|(?:[ \t]*\[)|(?:[ \t]+\w))/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+            pop!
+          elsif %w(def).include? m[0]
+            token Keyword
+            goto :funcname
+          elsif %w(struct class).include? m[0]
+            token Keyword::Reserved
+            goto :classname
+          elsif self.class.c_keywords.include? m[0]
+            token Keyword::Reserved
+          else
+            token Keyword::Type
+            pop!
+          end
+        end
+
+        rule(//) { pop! }
+      end
+
+      state :c_definitions do
+        rule %r/\n/, Text, :c_indent
+        mixin :root
+      end
+
+      state :c_indent do
+        rule %r/[ \t]+/ do |m|
+          token Text
+          goto :c_start
+
+          if @indentation.nil?
+            @indentation = m[0]
+          elsif @indentation.length > m[0].length
+            @indentation = nil
+            pop! 2 # Pop :c_start and :c_definitions
+          end
+        end
+
+        rule(//) { @indentation = nil; reset_stack }
+      end
+    end
+  end
+end

data/lib/rouge/lexers/d.rb

@@ -41,8 +41,8 @@ module Rouge
       )
 
       state :whitespace do
-        rule /\n/m, Text
-        rule /\s+/m, Text
+        rule %r/\n/m, Text
+        rule %r/\s+/m, Text
       end
 
       state :root do
@@ -52,58 +52,58 @@ module Rouge
         rule %r(/(\\\n)?[*](.|\n)*?[*](\\\n)?/), Comment::Multiline
         rule %r(/\+), Comment::Multiline, :nested_comment
         # Keywords
-        rule /(#{keywords.join('|')})\b/, Keyword
-        rule /(#{keywords_type.join('|')})\b/, Keyword::Type
-        rule /(false|true|null)\b/, Keyword::Constant
-        rule /(#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo
-        rule /macro\b/, Keyword::Reserved
-        rule /(string|wstring|dstring|size_t|ptrdiff_t)\b/, Name::Builtin
+        rule %r/(#{keywords.join('|')})\b/, Keyword
+        rule %r/(#{keywords_type.join('|')})\b/, Keyword::Type
+        rule %r/(false|true|null)\b/, Keyword::Constant
+        rule %r/(#{keywords_pseudo.join('|')})\b/, Keyword::Pseudo
+        rule %r/macro\b/, Keyword::Reserved
+        rule %r/(string|wstring|dstring|size_t|ptrdiff_t)\b/, Name::Builtin
         # Literals
         # HexFloat
-        rule /0[xX]([0-9a-fA-F_]*\.[0-9a-fA-F_]+|[0-9a-fA-F_]+)[pP][+\-]?[0-9_]+[fFL]?[i]?/, Num::Float
+        rule %r/0[xX]([0-9a-fA-F_]*\.[0-9a-fA-F_]+|[0-9a-fA-F_]+)[pP][+\-]?[0-9_]+[fFL]?[i]?/, Num::Float
         # DecimalFloat
-        rule /[0-9_]+(\.[0-9_]+[eE][+\-]?[0-9_]+|\.[0-9_]*|[eE][+\-]?[0-9_]+)[fFL]?[i]?/, Num::Float
-        rule /\.(0|[1-9][0-9_]*)([eE][+\-]?[0-9_]+)?[fFL]?[i]?/, Num::Float
+        rule %r/[0-9_]+(\.[0-9_]+[eE][+\-]?[0-9_]+|\.[0-9_]*|[eE][+\-]?[0-9_]+)[fFL]?[i]?/, Num::Float
+        rule %r/\.(0|[1-9][0-9_]*)([eE][+\-]?[0-9_]+)?[fFL]?[i]?/, Num::Float
         # IntegerLiteral
         # Binary
-        rule /0[Bb][01_]+/, Num::Bin
+        rule %r/0[Bb][01_]+/, Num::Bin
         # Octal
         # TODO: 0[0-7] isn't supported use octal![0-7] instead
-        rule /0[0-7_]+/, Num::Oct
+        rule %r/0[0-7_]+/, Num::Oct
         # Hexadecimal
-        rule /0[xX][0-9a-fA-F_]+/, Num::Hex
+        rule %r/0[xX][0-9a-fA-F_]+/, Num::Hex
         # Decimal
-        rule /(0|[1-9][0-9_]*)([LUu]|Lu|LU|uL|UL)?/, Num::Integer
+        rule %r/(0|[1-9][0-9_]*)([LUu]|Lu|LU|uL|UL)?/, Num::Integer
         # CharacterLiteral
-        rule /'(\\['"?\\abfnrtv]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\&\w+;|.)'/, Str::Char
+        rule %r/'(\\['"?\\abfnrtv]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\&\w+;|.)'/, Str::Char
         # StringLiteral
         # WysiwygString
-        rule /r"[^"]*"[cwd]?/, Str
+        rule %r/r"[^"]*"[cwd]?/, Str
         # Alternate WysiwygString
-        rule /`[^`]*`[cwd]?/, Str
+        rule %r/`[^`]*`[cwd]?/, Str
         # DoubleQuotedString
-        rule /"(\\\\|\\"|[^"])*"[cwd]?/, Str
+        rule %r/"(\\\\|\\"|[^"])*"[cwd]?/, Str
         # EscapeSequence
-        rule /\\(['\"?\\abfnrtv]|x[0-9a-fA-F]{2}|[0-7]{1,3}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8}|&\w+;)/, Str
+        rule %r/\\(['\"?\\abfnrtv]|x[0-9a-fA-F]{2}|[0-7]{1,3}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8}|&\w+;)/, Str
         # HexString
-        rule /x"[0-9a-fA-F_\s]*"[cwd]?/, Str
+        rule %r/x"[0-9a-fA-F_\s]*"[cwd]?/, Str
         # DelimitedString
-        rule /q"\[/, Str, :delimited_bracket
-        rule /q"\(/, Str, :delimited_parenthesis
-        rule /q"</, Str, :delimited_angle
-        rule /q"\{/, Str, :delimited_curly
-        rule /q"([a-zA-Z_]\w*)\n.*?\n\1"/, Str
-        rule /q"(.).*?\1"/, Str
+        rule %r/q"\[/, Str, :delimited_bracket
+        rule %r/q"\(/, Str, :delimited_parenthesis
+        rule %r/q"</, Str, :delimited_angle
+        rule %r/q"\{/, Str, :delimited_curly
+        rule %r/q"([a-zA-Z_]\w*)\n.*?\n\1"/, Str
+        rule %r/q"(.).*?\1"/, Str
         # TokenString
-        rule /q\{/, Str, :token_string
+        rule %r/q\{/, Str, :token_string
         # Attributes
-        rule /@([a-zA-Z_]\w*)?/, Name::Decorator
+        rule %r/@([a-zA-Z_]\w*)?/, Name::Decorator
         # Tokens
         rule %r`(~=|\^=|%=|\*=|==|!>=|!<=|!<>=|!<>|!<|!>|!=|>>>=|>>>|>>=|>>|>=|<>=|<>|<<=|<<|<=|\+\+|\+=|--|-=|\|\||\|=|&&|&=|\.\.\.|\.\.|/=)|[/.&|\-+<>!()\[\]{}?,;:$=*%^~]`, Punctuation
         # Identifier
-        rule /[a-zA-Z_]\w*/, Name
+        rule %r/[a-zA-Z_]\w*/, Name
         # Line
-        rule /#line\s.*\n/, Comment::Special
+        rule %r/#line\s.*\n/, Comment::Special
       end
 
       state :nested_comment do
@@ -114,63 +114,63 @@ module Rouge
       end
 
       state :token_string do
-        rule /\{/, Punctuation, :token_string_nest
-        rule /\}/, Str, :pop!
+        rule %r/\{/, Punctuation, :token_string_nest
+        rule %r/\}/, Str, :pop!
         mixin :root
       end
 
       state :token_string_nest do
-        rule /\{/, Punctuation, :push
-        rule /\}/, Punctuation, :pop!
+        rule %r/\{/, Punctuation, :push
+        rule %r/\}/, Punctuation, :pop!
         mixin :root
       end
 
       state :delimited_bracket do
-        rule /[^\[\]]+/, Str
-        rule /\[/, Str, :delimited_inside_bracket
-        rule /\]"/, Str, :pop!
+        rule %r/[^\[\]]+/, Str
+        rule %r/\[/, Str, :delimited_inside_bracket
+        rule %r/\]"/, Str, :pop!
       end
 
       state :delimited_inside_bracket do
-        rule /[^\[\]]+/, Str
-        rule /\[/, Str, :push
-        rule /\]/, Str, :pop!
+        rule %r/[^\[\]]+/, Str
+        rule %r/\[/, Str, :push
+        rule %r/\]/, Str, :pop!
       end
 
       state :delimited_parenthesis do
-        rule /[^()]+/, Str
-        rule /\(/, Str, :delimited_inside_parenthesis
-        rule /\)"/, Str, :pop!
+        rule %r/[^()]+/, Str
+        rule %r/\(/, Str, :delimited_inside_parenthesis
+        rule %r/\)"/, Str, :pop!
       end
 
       state :delimited_inside_parenthesis do
-        rule /[^()]+/, Str
-        rule /\(/, Str, :push
-        rule /\)/, Str, :pop!
+        rule %r/[^()]+/, Str
+        rule %r/\(/, Str, :push
+        rule %r/\)/, Str, :pop!
       end
 
       state :delimited_angle do
-        rule /[^<>]+/, Str
-        rule /</, Str, :delimited_inside_angle
-        rule />"/, Str, :pop!
+        rule %r/[^<>]+/, Str
+        rule %r/</, Str, :delimited_inside_angle
+        rule %r/>"/, Str, :pop!
       end
 
       state :delimited_inside_angle do
-        rule /[^<>]+/, Str
-        rule /</, Str, :push
-        rule />/, Str, :pop!
+        rule %r/[^<>]+/, Str
+        rule %r/</, Str, :push
+        rule %r/>/, Str, :pop!
       end
 
       state :delimited_curly do
-        rule /[^{}]+/, Str
-        rule /\{/, Str, :delimited_inside_curly
-        rule /\}"/, Str, :pop!
+        rule %r/[^{}]+/, Str
+        rule %r/\{/, Str, :delimited_inside_curly
+        rule %r/\}"/, Str, :pop!
       end
 
       state :delimited_inside_curly do
-        rule /[^{}]+/, Str
-        rule /\{/, Str, :push
-        rule /\}/, Str, :pop!
+        rule %r/[^{}]+/, Str
+        rule %r/\{/, Str, :push
+        rule %r/\}/, Str, :pop!
       end
     end
   end