rouge 3.6.0 → 3.8.0

data/lib/rouge/lexers/bbcbasic.rb

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class BBCBASIC < RegexLexer
+      title "BBCBASIC"
+      desc "BBC BASIC syntax"
+      tag 'bbcbasic'
+      filenames '*,fd1'
+
+      def self.punctuation
+        @punctuation ||= %w(
+          [,;'~] SPC TAB
+        )
+      end
+
+      def self.function
+        @function ||= %w(
+          ABS ACS ADVAL ASC ASN ATN BEATS BEAT BGET# CHR\$ COS COUNT DEG DIM
+          EOF# ERL ERR EVAL EXP EXT# FN GET\$# GET\$ GET HIMEM INKEY\$ INKEY
+          INSTR INT LEFT\$ LEN LN LOG LOMEM MID\$ OPENIN OPENOUT OPENUP PAGE
+          POINT POS PTR# RAD REPORT\$ RIGHT\$ RND SGN SIN SQR STR\$ STRING\$ SUM
+          SUMLEN TAN TEMPO TIME\$ TIME TOP USR VAL VPOS
+        )
+      end
+
+      def self.statement
+        @statement ||= %w(
+          BEATS BPUT# CALL CASE CHAIN CLEAR CLG CLOSE# CLS COLOR COLOUR DATA
+          ELSE ENDCASE ENDIF ENDPROC ENDWHILE END ENVELOPE FOR GCOL GOSUB GOTO
+          IF INSTALL LET LIBRARY MODE NEXT OFF OF ON ORIGIN OSCI OTHERWISE
+          OVERLAY PLOT PRINT# PRINT PROC QUIT READ REPEAT REPORT RETURN SOUND
+          STEP STEREO STOP SWAP SYS THEN TINT TO VDU VOICES VOICE UNTIL WAIT
+          WHEN WHILE WIDTH
+        )
+      end
+
+      def self.operator
+        @operator ||= %w(
+          << <= <> < >= >>> >> > [-!$()*+/=?^|] AND DIV EOR MOD NOT OR
+        )
+      end
+
+      def self.constant
+        @constant ||= %w(
+          FALSE TRUE
+        )
+      end
+
+      state :expression do
+        rule %r/#{BBCBASIC.function.join('|')}/o, Name::Builtin  # function or pseudo-variable
+        rule %r/#{BBCBASIC.operator.join('|')}/o, Operator
+        rule %r/#{BBCBASIC.constant.join('|')}/o, Name::Constant
+        rule %r/"[^"]*"/o, Literal::String
+        rule %r/[a-z_`][\w`]*[$%]?/io, Name::Variable
+        rule %r/@%/o, Name::Variable
+        rule %r/[\d.]+/o, Literal::Number
+        rule %r/%[01]+/o, Literal::Number::Bin
+        rule %r/&[\h]+/o, Literal::Number::Hex
+      end
+
+      state :root do
+        rule %r/(:+)( *)(\*)(.*)/ do
+          groups Punctuation, Text, Keyword, Text # CLI command
+        end
+        rule %r/(\n+ *)(\*)(.*)/ do
+          groups Text, Keyword, Text # CLI command
+        end
+        rule %r/(ELSE|OTHERWISE|REPEAT|THEN)( *)(\*)(.*)/ do
+          groups Keyword, Text, Keyword, Text # CLI command
+        end
+        rule %r/[ \n]+/o, Text
+        rule %r/:+/o, Punctuation
+        rule %r/[\[]/o, Keyword, :assembly1
+        rule %r/REM *>.*/o, Comment::Special
+        rule %r/REM.*/o, Comment
+        rule %r/(?:#{BBCBASIC.statement.join('|')}|CIRCLE(?: *FILL)?|DEF *(?:FN|PROC)|DRAW(?: *BY)?|DIM(?!\()|ELLIPSE(?: *FILL)?|ERROR(?: *EXT)?|FILL(?: *BY)?|INPUT(?:#| *LINE)?|LINE(?: *INPUT)?|LOCAL(?: *DATA| *ERROR)?|MOUSE(?: *COLOUR| *OFF| *ON| *RECTANGLE| *STEP| *TO)?|MOVE(?: *BY)?|ON(?! *ERROR)|ON *ERROR *(?:LOCAL|OFF)?|POINT(?: *BY)?(?!\()|RECTANGE(?: *FILL)?|RESTORE(?: *DATA| *ERROR)?|TRACE(?: *CLOSE| *ENDPROC| *OFF| *STEP(?: *FN| *ON| *PROC)?| *TO)?)/o, Keyword
+        mixin :expression
+        rule %r/#{BBCBASIC.punctuation.join('|')}/o, Punctuation
+      end
+
+      # Assembly statements are parsed as
+      # {label} {directive|opcode |']' {expressions}} {comment}
+      # Technically, you don't need whitespace between opcodes and arguments,
+      # but this is rare in uncrunched source and trying to enumerate all
+      # possible opcodes here is impractical so we colour it as though
+      # the whitespace is required. Opcodes and directives can only easily be
+      # distinguished from the symbols that make up expressions by looking at
+      # their position within the statement. Similarly, ']' is treated as a
+      # keyword at the start of a statement or as punctuation elsewhere. This
+      # requires a two-state state machine.
+
+      state :assembly1 do
+        rule %r/ +/o, Text
+        rule %r/]/o, Keyword, :pop!
+        rule %r/[:\n]/o, Punctuation
+        rule %r/\.[a-z_`][\w`]*%? */io, Name::Label
+        rule %r/(?:REM|;)[^:\n]*/o, Comment
+        rule %r/[^ :\n]+/o, Keyword, :assembly2
+      end
+
+      state :assembly2 do
+        rule %r/ +/o, Text
+        rule %r/[:\n]/o, Punctuation, :pop!
+        rule %r/(?:REM|;)[^:\n]*/o, Comment, :pop!
+        mixin :expression
+        rule %r/[!#,@\[\]^{}]/, Punctuation
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cmhg.rb

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class CMHG < RegexLexer
+      title "CMHG"
+      desc "RISC OS C module header generator source file"
+      tag 'cmhg'
+      filenames '*.cmhg'
+
+      def self.preproc_keyword
+        @preproc_keyword ||= %w(
+          define elif else endif error if ifdef ifndef include line pragma undef warning
+        )
+      end
+
+      state :root do
+        rule %r/;[^\n]*/, Comment
+        rule %r/^([ \t]*)(#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})(?:[ \t].*)?)?)(?=\n)/ do
+          groups Text, Comment::Preproc
+        end
+        rule %r/[-a-z]+:/, Keyword::Declaration
+        rule %r/[a-z_]\w+/i, Name::Entity
+        rule %r/"[^"]*"/, Literal::String
+        rule %r/(?:&|0x)\h+/, Literal::Number::Hex
+        rule %r/\d+/, Literal::Number
+        rule %r/[,\/()]/, Punctuation
+        rule %r/[ \t]+/, Text
+        rule %r/\n+/, Text
+      end
+    end
+  end
+end

data/lib/rouge/lexers/console.rb

@@ -116,7 +116,7 @@ module Rouge
           # before we pass to the lang lexer so it can determine where
           # the "real" beginning of the line is
           $' =~ /\A\s*/
-          yield Text, $& unless $&.empty?
+          yield Text::Whitespace, $& unless $&.empty?
 
           lang_lexer.continue_lex($', &output)
         elsif comment_regex =~ input[0].strip

data/lib/rouge/lexers/cpp.rb

@@ -17,7 +17,7 @@ module Rouge
                 '*.cc',  '*.hh',
                 '*.cxx', '*.hxx',
                 '*.pde', '*.ino',
-                '*.tpp'
+                '*.tpp', '*.h'
       mimetypes 'text/x-c++hdr', 'text/x-c++src'
 
       def self.keywords
@@ -58,9 +58,11 @@ module Rouge
 
       prepend :statements do
         rule %r/class\b/, Keyword, :classname
+        rule %r/\d+(\.\d+)?(?:h|(?:min)|s|(?:ms)|(?:us)|(?:ns))/, Num::Other
         rule %r((#{dq}[.]#{dq}?|[.]#{dq})(e[+-]?#{dq}[lu]*)?)i, Num::Float
         rule %r(#{dq}e[+-]?#{dq}[lu]*)i, Num::Float
         rule %r/0x\h('?\h)*[lu]*/i, Num::Hex
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
         rule %r/0[0-7]('?[0-7])*[lu]*/i, Num::Oct
         rule %r/#{dq}[lu]*/i, Num::Integer
         rule %r/\bnullptr\b/, Name::Builtin
@@ -72,6 +74,7 @@ module Rouge
 
         # template specification
         rule %r/\s*(?=>)/m, Text, :pop!
+        rule %r/[.]{3}/, Operator
         mixin :whitespace
       end
     end

data/lib/rouge/lexers/cuda.rb

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    load_lexer 'cpp.rb'
+
+    class CUDA < Cpp
+      title "CUDA"
+      desc "Compute Unified Device Architecture, used for programming with NVIDIA GPU"
+
+      tag 'cuda'
+      filenames '*.cu', '*.cuh'
+
+      def self.keywords
+        @keywords ||= super + Set.new(%w(
+          __global__ __device__ __host__ __noinline__ __forceinline__
+          __constant__ __shared__ __managed__ __restrict__
+        ))
+      end
+
+      def self.keywords_type
+        @keywords_type ||= super + Set.new(%w(
+          char1 char2 char3 char4 uchar1 uchar2 uchar3 uchar4
+          short1 short2 short3 short4 ushort1 ushort2 ushort3 ushort4
+          int1 int2 int3 int4 uint1 uint2 uint3 uint4
+          long1 long2 long3 long4 ulong1 ulong2 ulong3 ulong4
+          longlong1 longlong2 longlong3 longlong4 
+          ulonglong1 ulonglong2 ulonglong3 ulonglong4 
+          float1 float2 float3 float4 double1 double2 double3 double4
+          dim3
+        ))
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cython.rb

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'python.rb'
+
+    class Cython < Python
+      title "Cython"
+      desc "Cython and Pyrex source code (cython.org)"
+      tag 'cython'
+      aliases 'pyx', 'pyrex'
+      filenames '*.pyx', '*.pxd', '*.pxi'
+      mimetypes 'text/x-cython', 'application/x-cython'
+
+      def initialize(opts = {})
+        super opts
+        @indentation = nil
+      end
+
+      def self.keywords
+        @keywords ||= super + %w(
+          by except? fused gil nogil
+        )
+      end
+
+      def self.c_keywords
+        @ckeywords ||= %w(
+          public readonly extern api inline enum union
+        )
+      end
+
+      identifier = /[a-z_]\w*/i
+      dotted_identifier = /[a-z_.][\w.]*/i
+
+      prepend :root do
+        rule %r/cp?def|ctypedef/ do
+          token Keyword
+          push :c_definitions
+          push :c_start
+        end
+
+        rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(cimport)/ do
+          groups Keyword::Namespace,
+                 Text,
+                 Name::Namespace,
+                 Text,
+                 Keyword::Namespace
+        end
+
+        rule %r/(cimport)(\s+)(#{dotted_identifier})/ do
+          groups Keyword::Namespace, Text, Name::Namespace
+        end
+
+        rule %r/(struct)((?:\\\s|\s)+)/ do
+          groups Keyword, Text
+          push :classname
+        end
+
+        mixin :func_call_fix
+
+        rule %r/[(,]/, Punctuation, :c_start
+      end
+
+      prepend :classname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+
+      prepend :funcname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+      # This is a fix for the way that function calls are lexed in the Python
+      # lexer. This should be moved to the Python lexer once confirmed that it
+      # does not cause any regressions.
+      state :func_call_fix do
+        rule %r/#{identifier}(?=\()/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+          elsif self.class.exceptions.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins_pseudo.include? m[0]
+            token Name::Builtin::Pseudo
+          else
+            token Name::Function
+          end
+        end
+      end
+
+      # The Cython lexer adds three states to those already in the Python lexer.
+      # Calls to `cdef`, `cpdef` and `ctypedef` move the lexer into the :c_start
+      # state. The primary purpose of this state is to highlight datatypes. Once
+      # this has been done, the lexer moves to the :c_definitions state where
+      # the majority of text in a definition is lexed. Finally, newlines cause
+      # the lexer to move to :c_indent. This state is used to check whether we
+      # have moved out of a C block.
+
+      state :c_start do
+        rule %r/[^\S\n]+/, Text
+
+        rule %r/cp?def|ctypedef/, Keyword
+
+        rule %r/(?:un)?signed/, Keyword::Type
+
+        # This rule matches identifiers that could be type declarations. The
+        # lookahead matches (1) pointers, (2) arrays and (3) variable names.
+        rule %r/#{identifier}(?=(?:\*+)|(?:[ \t]*\[)|(?:[ \t]+\w))/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+            pop!
+          elsif %w(def).include? m[0]
+            token Keyword
+            goto :funcname
+          elsif %w(struct class).include? m[0]
+            token Keyword::Reserved
+            goto :classname
+          elsif self.class.c_keywords.include? m[0]
+            token Keyword::Reserved
+          else
+            token Keyword::Type
+            pop!
+          end
+        end
+
+        rule(//) { pop! }
+      end
+
+      state :c_definitions do
+        rule %r/\n/, Text, :c_indent
+        mixin :root
+      end
+
+      state :c_indent do
+        rule %r/[ \t]+/ do |m|
+          token Text
+          goto :c_start
+
+          if @indentation.nil?
+            @indentation = m[0]
+          elsif @indentation.length > m[0].length
+            @indentation = nil
+            pop! 2 # Pop :c_start and :c_definitions
+          end
+        end
+
+        rule(//) { @indentation = nil; reset_stack }
+      end
+    end
+  end
+end

data/lib/rouge/lexers/epp.rb

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class EPP < TemplateLexer
+      title "EPP"
+      desc "Embedded Puppet template files"
+
+      tag 'epp'
+
+      filenames '*.epp'
+
+      def initialize(opts={})
+        super(opts)
+        @parent = lexer_option(:parent) { PlainText.new(opts) }
+        @puppet_lexer = Puppet.new(opts)
+      end
+
+      start do
+        parent.reset!
+        @puppet_lexer.reset!
+      end
+
+      open  = /<%%|<%=|<%#|(<%-|<%)(\s*\|)?/
+      close = /%%>|(\|\s*)?(-%>|%>)/
+
+      state :root do
+        rule %r/<%#/, Comment, :comment
+
+        rule open, Comment::Preproc, :puppet
+
+        rule %r/.+?(?=#{open})|.+/m do
+          delegate parent
+        end
+      end
+
+      state :comment do
+        rule close, Comment, :pop!
+        rule %r/.+?(?=#{close})|.+/m, Comment
+      end
+
+      state :puppet do
+        rule close, Comment::Preproc, :pop!
+
+        rule %r/.+?(?=#{close})|.+/m do
+          delegate @puppet_lexer
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/escape.rb

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
 module Rouge
   module Lexers
     class Escape < Lexer

data/lib/rouge/lexers/gdscript.rb

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class GDScript < RegexLexer
+      title "GDScript"
+      desc "The Godot Engine programming language (https://godotengine.org/)"
+      tag 'gdscript'
+      aliases 'gd', 'gdscript'
+      filenames '*.gd'
+      mimetypes 'text/x-gdscript', 'application/x-gdscript'
+
+      def self.keywords
+        @keywords = %w(
+          and in not or as breakpoint class class_name extends is func setget
+          signal tool const enum export onready static var break continue
+          if elif else for pass return match while remote master puppet
+          remotesync mastersync puppetsync
+        ).join('|')
+      end
+
+      # Reserved for future implementation
+      def self.keywords_reserved
+        @keywords_reserved = %w(
+          do switch case
+        ).join('|')
+      end
+
+      def self.builtins
+        builtins = %w(
+          Color8 ColorN abs acos asin assert atan atan2 bytes2var ceil char
+          clamp convert cos cosh db2linear decimals dectime deg2rad dict2inst
+          ease exp floor fmod fposmod funcref hash inst2dict instance_from_id
+          is_inf is_nan lerp linear2db load log max min nearest_po2 pow
+          preload print print_stack printerr printraw prints printt rad2deg
+          rand_range rand_seed randf randi randomize range round seed sign
+          sin sinh sqrt stepify str str2var tan tan tanh type_exist typeof
+          var2bytes var2str weakref yield
+        ).join('|')
+      end
+
+      def self.builtins_type
+        @builtins_type = %w(
+          bool int float String Vector2 Rect2 Transform2D Vector3 AABB
+          Plane Quat Basis Transform Color RID Object NodePath Dictionary
+          Array PoolByteArray PoolIntArray PoolRealArray PoolStringArray
+          PoolVector2Array PoolVector3Array PoolColorArray null
+        ).join('|')
+      end
+
+      state :root do
+        rule %r/\n/, Text
+        rule %r/[^\S\n]+/, Text
+        rule %r/#.*/, Comment::Single
+        rule %r/[\[\]{}:(),;]/, Punctuation
+        rule %r/\\\n/, Text
+        rule %r/(in|and|or|not)\b/, Operator::Word
+        rule %r/!=|==|<<|>>|&&|\+=|-=|\*=|\/=|%=|&=|\|=|\|\||[-~+\/*%=<>&^.!|$]/, Operator
+        rule %r/(func)((?:\s|\\)+)/ do
+          groups Keyword, Text
+          push :funcname
+        end
+        rule %r/(class)((?:\s|\\)+)/ do
+          groups Keyword, Text
+          push :classname
+        end
+        mixin :keywords
+        mixin :builtins
+        rule %r/"""/, Str::Double, :escape_tdqs
+        rule %r/'''/, Str::Double, :escape_tsqs
+        rule %r/"/, Str::Double, :escape_dqs
+        rule %r/'/, Str::Double, :escape_sqs
+        mixin :name
+        mixin :numbers
+      end
+
+      state :keywords do
+        rule %r/\b(#{GDScript.keywords})\b/, Keyword
+        rule %r/\b(#{GDScript.keywords_reserved})\b/, Keyword::Reserved
+      end
+
+      state :builtins do
+        rule %r/\b(#{GDScript.builtins})\b/, Name::Builtin
+        rule %r/\b((self|false|true)|(PI|TAU|NAN|INF))\b/, Name::Builtin::Pseudo
+        rule %r/\b(#{GDScript.builtins_type})\b/, Keyword::Type
+      end
+
+      state :numbers do
+        rule %r/(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?/, Num::Float
+        rule %r/\d+[eE][+-]?[0-9]+j?/, Num::Float
+        rule %r/0[xX][a-fA-F0-9]+/, Num::Hex
+        rule %r/\d+j?/, Num::Integer
+      end
+
+      state :name do
+        rule %r/[a-zA-Z_]\w*/, Name
+      end
+
+      state :funcname do
+        rule %r/[a-zA-Z_]\w*/, Name::Function, :pop!
+      end
+
+      state :classname do
+        rule %r/[a-zA-Z_]\w*/, Name::Class, :pop!
+      end
+
+      state :string_escape do
+        rule %r/\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})/, Str::Escape
+      end
+
+      state :strings_single do
+        rule %r/%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]/, Str::Interpol
+        rule %r/[^\\'%\n]+/, Str::Single
+        rule %r/["\\]/, Str::Single
+        rule %r/%/, Str::Single
+      end
+
+      state :strings_double do
+        rule %r/%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]/, Str::Interpol
+        rule %r/[^\\"%\n]+/, Str::Double
+        rule %r/['\\]/, Str::Double
+        rule %r/%/, Str::Double
+      end
+
+      state :dqs do
+        rule %r/"/, Str::Double, :pop!
+        rule %r/\\\\|\\"|\\\n/, Str::Escape
+        mixin :strings_double
+      end
+
+      state :escape_dqs do
+        mixin :string_escape
+        mixin :dqs
+      end
+
+      state :sqs do
+        rule %r/'/, Str::Single, :pop!
+        rule %r/\\\\|\\'|\\\n/, Str::Escape
+        mixin :strings_single
+      end
+
+      state :escape_sqs do
+        mixin :string_escape
+        mixin :sqs
+      end
+
+      state :tdqs do
+        rule %r/"""/, Str::Double, :pop!
+        mixin :strings_double
+        rule %r/\n/, Str::Double
+      end
+
+      state :escape_tdqs do
+        mixin :string_escape
+        mixin :tdqs
+      end
+
+      state :tsqs do
+        rule %r/'''/, Str::Single, :pop!
+        mixin :strings_single
+        rule %r/\n/, Str::Single
+      end
+
+      state :escape_tsqs do
+        mixin :string_escape
+        mixin :tsqs
+      end
+    end
+  end
+end