rouge 3.7.0 → 3.9.0

data/lib/rouge/lexers/batchfile.rb

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Batchfile < RegexLexer
+      title "Batchfile"
+      desc "Windows Batch File"
+
+      tag 'batchfile'
+      aliases 'bat', 'batch', 'dosbatch', 'winbatch'
+      filenames '*.bat', '*.cmd'
+
+      mimetypes 'application/bat', 'application/x-bat', 'application/x-msdos-program'
+
+      def self.keywords
+        @keywords ||= %w(
+          if else for in do goto call exit
+        )
+      end
+
+      def self.operator_words
+        @operator_words ||= %w(
+          exist defined errorlevel cmdextversion not equ neq lss leq gtr geq
+        )
+      end
+
+      def self.devices
+        @devices ||= %w(
+          con prn aux nul com1 com2 com3 com4 com5 com6 com7 com8 com9 lpt1 lpt2
+          lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9
+        )
+      end
+
+      def self.builtin_commands
+        @builtin_commands ||= %w(
+          assoc attrib break bcdedit cacls cd chcp chdir chkdsk chkntfs choice
+          cls cmd color comp compact convert copy date del dir diskpart doskey
+          dpath driverquery echo endlocal erase fc find findstr format fsutil
+          ftype gpresult graftabl help icacls label md mkdir mklink mode more
+          move openfiles path pause popd print prompt pushd rd recover ren
+          rename replace rmdir robocopy setlocal sc schtasks shift shutdown sort
+          start subst systeminfo takeown tasklist taskkill time timeout title
+          tree type ver verify vol xcopy waitfor wmic
+        )
+      end
+
+      def self.other_commands
+        @other_commands ||= %w(
+          addusers admodcmd ansicon arp at bcdboot bitsadmin browstat certreq
+          certutil change cidiag cipher cleanmgr clip cmdkey compress convertcp
+          coreinfo csccmd csvde cscript curl debug defrag delprof deltree devcon
+          diamond dirquota diruse diskshadow diskuse dism dnscmd dsacls dsadd
+          dsget dsquery dsmod dsmove dsrm dsmgmt dsregcmd edlin eventcreate
+          expand extract fdisk fltmc forfiles freedisk ftp getmac gpupdate
+          hostname ifmember inuse ipconfig kill lgpo lodctr logman logoff
+          logtime makecab mapisend mbsacli mem mountvol moveuser msg mshta
+          msiexec msinfo32 mstsc nbtstat net net1 netdom netsh netstat nlsinfo
+          nltest now nslookup ntbackup ntdsutil ntoskrnl ntrights nvspbind
+          pathping perms ping portqry powercfg pngout pnputil printbrm prncnfg
+          prnmngr procdump psexec psfile psgetsid psinfo pskill pslist
+          psloggedon psloglist pspasswd psping psservice psshutdown pssuspend
+          qbasic qgrep qprocess query quser qwinsta rasdial reg reg1 regdump
+          regedt32 regsvr32 regini reset restore rundll32 rmtshare route rpcping
+          run runas scandisk setspn setx sfc share shellrunas shortcut sigcheck
+          sleep slmgr strings subinacl sysmon telnet tftp tlist touch tracerpt
+          tracert tscon tsdiscon tskill tttracer typeperf tzutil undelete
+          unformat verifier vmconnect vssadmin w32tm wbadmin wecutil wevtutil
+          wget where whoami windiff winrm winrs wpeutil wpr wusa wuauclt wscript
+        )
+      end
+
+      def self.attributes
+        @attributes ||= %w(
+          on off disable enableextensions enabledelayedexpansion
+        )
+      end
+
+      state :basic do
+        # Comments
+        rule %r/\brem\b.*$/i, Comment
+        # Empty Labels
+        rule %r/^::.*$/, Comment
+
+        # Labels
+        rule %r/:[a-z]+/i, Name::Label
+
+        rule %r/([a-z]\w*)(\.exe|com|bat|cmd|msi)?/i do |m|
+          if self.class.devices.include? m[1]
+            groups Keyword::Reserved, Error
+          elsif self.class.keywords.include? m[1]
+            groups Keyword, Error
+          elsif self.class.operator_words.include? m[1]
+            groups Operator::Word, Error
+          elsif self.class.builtin_commands.include? m[1]
+            token Name::Builtin
+          elsif self.class.other_commands.include? m[1]
+            token Name::Builtin
+          elsif self.class.attributes.include? m[1]
+            groups Name::Attribute, Error
+          elsif "set".casecmp m[1]
+            groups Keyword::Declaration, Error
+          else
+            token Text
+          end
+        end
+
+        rule %r/([\/\-+][a-z]+)\s*/i, Name::Attribute
+
+        mixin :expansions
+
+        rule %r/[<>&|(){}\[\]\-+=;,~?*]/, Operator
+      end
+
+      state :escape do
+        rule %r/\^./m, Str::Escape
+      end
+
+      state :expansions do
+        # Normal and Delayed expansion
+        rule %r/[%!]+([a-z_$@#]+)[%!]+/i, Name::Variable
+        # For Variables
+        rule %r/(\%+~?[a-z]+\d?)/i, Name::Variable::Magic
+      end
+
+      state :double_quotes do
+        mixin :escape
+        rule %r/["]/, Str::Double, :pop!
+        mixin :expansions
+        rule %r/[^\^"%!]+/, Str::Double
+      end
+
+      state :single_quotes do
+        mixin :escape
+        rule %r/[']/, Str::Single, :pop!
+        mixin :expansions
+        rule %r/[^\^'%!]+/, Str::Single
+      end
+
+      state :backtick do
+        mixin :escape
+        rule %r/[`]/, Str::Backtick, :pop!
+        mixin :expansions
+        rule %r/[^\^`%!]+/, Str::Backtick
+      end
+
+      state :data do
+        rule %r/\s+/, Text
+        rule %r/0x[0-9a-f]+/i, Literal::Number::Hex
+        rule %r/[0-9]/, Literal::Number
+        rule %r/["]/, Str::Double, :double_quotes
+        rule %r/[']/, Str::Single, :single_quotes
+        rule %r/[`]/, Str::Backtick, :backtick
+        rule %r/[^\s&|()\[\]{}\^=;!%+\-,"'`~?*]+/, Text
+        mixin :escape
+      end
+
+      state :root do
+        mixin :basic
+        mixin :data
+      end
+    end
+  end
+end

data/lib/rouge/lexers/bbcbasic.rb

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class BBCBASIC < RegexLexer
+      title "BBCBASIC"
+      desc "BBC BASIC syntax"
+      tag 'bbcbasic'
+      filenames '*,fd1'
+
+      def self.punctuation
+        @punctuation ||= %w(
+          [,;'~] SPC TAB
+        )
+      end
+
+      def self.function
+        @function ||= %w(
+          ABS ACS ADVAL ASC ASN ATN BEATS BEAT BGET# CHR\$ COS COUNT DEG DIM
+          EOF# ERL ERR EVAL EXP EXT# FN GET\$# GET\$ GET HIMEM INKEY\$ INKEY
+          INSTR INT LEFT\$ LEN LN LOG LOMEM MID\$ OPENIN OPENOUT OPENUP PAGE
+          POINT POS PTR# RAD REPORT\$ RIGHT\$ RND SGN SIN SQR STR\$ STRING\$ SUM
+          SUMLEN TAN TEMPO TIME\$ TIME TOP USR VAL VPOS
+        )
+      end
+
+      def self.statement
+        @statement ||= %w(
+          BEATS BPUT# CALL CASE CHAIN CLEAR CLG CLOSE# CLS COLOR COLOUR DATA
+          ELSE ENDCASE ENDIF ENDPROC ENDWHILE END ENVELOPE FOR GCOL GOSUB GOTO
+          IF INSTALL LET LIBRARY MODE NEXT OFF OF ON ORIGIN OSCI OTHERWISE
+          OVERLAY PLOT PRINT# PRINT PROC QUIT READ REPEAT REPORT RETURN SOUND
+          STEP STEREO STOP SWAP SYS THEN TINT TO VDU VOICES VOICE UNTIL WAIT
+          WHEN WHILE WIDTH
+        )
+      end
+
+      def self.operator
+        @operator ||= %w(
+          << <= <> < >= >>> >> > [-!$()*+/=?^|] AND DIV EOR MOD NOT OR
+        )
+      end
+
+      def self.constant
+        @constant ||= %w(
+          FALSE TRUE
+        )
+      end
+
+      state :expression do
+        rule %r/#{BBCBASIC.function.join('|')}/o, Name::Builtin  # function or pseudo-variable
+        rule %r/#{BBCBASIC.operator.join('|')}/o, Operator
+        rule %r/#{BBCBASIC.constant.join('|')}/o, Name::Constant
+        rule %r/"[^"]*"/o, Literal::String
+        rule %r/[a-z_`][\w`]*[$%]?/io, Name::Variable
+        rule %r/@%/o, Name::Variable
+        rule %r/[\d.]+/o, Literal::Number
+        rule %r/%[01]+/o, Literal::Number::Bin
+        rule %r/&[\h]+/o, Literal::Number::Hex
+      end
+
+      state :root do
+        rule %r/(:+)( *)(\*)(.*)/ do
+          groups Punctuation, Text, Keyword, Text # CLI command
+        end
+        rule %r/(\n+ *)(\*)(.*)/ do
+          groups Text, Keyword, Text # CLI command
+        end
+        rule %r/(ELSE|OTHERWISE|REPEAT|THEN)( *)(\*)(.*)/ do
+          groups Keyword, Text, Keyword, Text # CLI command
+        end
+        rule %r/[ \n]+/o, Text
+        rule %r/:+/o, Punctuation
+        rule %r/[\[]/o, Keyword, :assembly1
+        rule %r/REM *>.*/o, Comment::Special
+        rule %r/REM.*/o, Comment
+        rule %r/(?:#{BBCBASIC.statement.join('|')}|CIRCLE(?: *FILL)?|DEF *(?:FN|PROC)|DRAW(?: *BY)?|DIM(?!\()|ELLIPSE(?: *FILL)?|ERROR(?: *EXT)?|FILL(?: *BY)?|INPUT(?:#| *LINE)?|LINE(?: *INPUT)?|LOCAL(?: *DATA| *ERROR)?|MOUSE(?: *COLOUR| *OFF| *ON| *RECTANGLE| *STEP| *TO)?|MOVE(?: *BY)?|ON(?! *ERROR)|ON *ERROR *(?:LOCAL|OFF)?|POINT(?: *BY)?(?!\()|RECTANGE(?: *FILL)?|RESTORE(?: *DATA| *ERROR)?|TRACE(?: *CLOSE| *ENDPROC| *OFF| *STEP(?: *FN| *ON| *PROC)?| *TO)?)/o, Keyword
+        mixin :expression
+        rule %r/#{BBCBASIC.punctuation.join('|')}/o, Punctuation
+      end
+
+      # Assembly statements are parsed as
+      # {label} {directive|opcode |']' {expressions}} {comment}
+      # Technically, you don't need whitespace between opcodes and arguments,
+      # but this is rare in uncrunched source and trying to enumerate all
+      # possible opcodes here is impractical so we colour it as though
+      # the whitespace is required. Opcodes and directives can only easily be
+      # distinguished from the symbols that make up expressions by looking at
+      # their position within the statement. Similarly, ']' is treated as a
+      # keyword at the start of a statement or as punctuation elsewhere. This
+      # requires a two-state state machine.
+
+      state :assembly1 do
+        rule %r/ +/o, Text
+        rule %r/]/o, Keyword, :pop!
+        rule %r/[:\n]/o, Punctuation
+        rule %r/\.[a-z_`][\w`]*%? */io, Name::Label
+        rule %r/(?:REM|;)[^:\n]*/o, Comment
+        rule %r/[^ :\n]+/o, Keyword, :assembly2
+      end
+
+      state :assembly2 do
+        rule %r/ +/o, Text
+        rule %r/[:\n]/o, Punctuation, :pop!
+        rule %r/(?:REM|;)[^:\n]*/o, Comment, :pop!
+        mixin :expression
+        rule %r/[!#,@\[\]^{}]/, Punctuation
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cmhg.rb

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class CMHG < RegexLexer
+      title "CMHG"
+      desc "RISC OS C module header generator source file"
+      tag 'cmhg'
+      filenames '*.cmhg'
+
+      def self.preproc_keyword
+        @preproc_keyword ||= %w(
+          define elif else endif error if ifdef ifndef include line pragma undef warning
+        )
+      end
+
+      state :root do
+        rule %r/;[^\n]*/, Comment
+        rule %r/^([ \t]*)(#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})(?:[ \t].*)?)?)(?=\n)/ do
+          groups Text, Comment::Preproc
+        end
+        rule %r/[-a-z]+:/, Keyword::Declaration
+        rule %r/[a-z_]\w+/i, Name::Entity
+        rule %r/"[^"]*"/, Literal::String
+        rule %r/(?:&|0x)\h+/, Literal::Number::Hex
+        rule %r/\d+/, Literal::Number
+        rule %r/[,\/()]/, Punctuation
+        rule %r/[ \t]+/, Text
+        rule %r/\n+/, Text
+      end
+    end
+  end
+end

data/lib/rouge/lexers/console.rb

@@ -116,7 +116,7 @@ module Rouge
           # before we pass to the lang lexer so it can determine where
           # the "real" beginning of the line is
           $' =~ /\A\s*/
-          yield Text, $& unless $&.empty?
+          yield Text::Whitespace, $& unless $&.empty?
 
           lang_lexer.continue_lex($', &output)
         elsif comment_regex =~ input[0].strip

data/lib/rouge/lexers/cpp.rb

@@ -17,7 +17,7 @@ module Rouge
                 '*.cc',  '*.hh',
                 '*.cxx', '*.hxx',
                 '*.pde', '*.ino',
-                '*.tpp'
+                '*.tpp', '*.h'
       mimetypes 'text/x-c++hdr', 'text/x-c++src'
 
       def self.keywords
@@ -58,9 +58,11 @@ module Rouge
 
       prepend :statements do
         rule %r/class\b/, Keyword, :classname
+        rule %r/\d+(\.\d+)?(?:h|(?:min)|s|(?:ms)|(?:us)|(?:ns))/, Num::Other
         rule %r((#{dq}[.]#{dq}?|[.]#{dq})(e[+-]?#{dq}[lu]*)?)i, Num::Float
         rule %r(#{dq}e[+-]?#{dq}[lu]*)i, Num::Float
         rule %r/0x\h('?\h)*[lu]*/i, Num::Hex
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
         rule %r/0[0-7]('?[0-7])*[lu]*/i, Num::Oct
         rule %r/#{dq}[lu]*/i, Num::Integer
         rule %r/\bnullptr\b/, Name::Builtin
@@ -72,6 +74,7 @@ module Rouge
 
         # template specification
         rule %r/\s*(?=>)/m, Text, :pop!
+        rule %r/[.]{3}/, Operator
         mixin :whitespace
       end
     end

data/lib/rouge/lexers/cython.rb

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'python.rb'
+
+    class Cython < Python
+      title "Cython"
+      desc "Cython and Pyrex source code (cython.org)"
+      tag 'cython'
+      aliases 'pyx', 'pyrex'
+      filenames '*.pyx', '*.pxd', '*.pxi'
+      mimetypes 'text/x-cython', 'application/x-cython'
+
+      def initialize(opts = {})
+        super opts
+        @indentation = nil
+      end
+
+      def self.keywords
+        @keywords ||= super + %w(
+          by except? fused gil nogil
+        )
+      end
+
+      def self.c_keywords
+        @ckeywords ||= %w(
+          public readonly extern api inline enum union
+        )
+      end
+
+      identifier = /[a-z_]\w*/i
+      dotted_identifier = /[a-z_.][\w.]*/i
+
+      prepend :root do
+        rule %r/cp?def|ctypedef/ do
+          token Keyword
+          push :c_definitions
+          push :c_start
+        end
+
+        rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(cimport)/ do
+          groups Keyword::Namespace,
+                 Text,
+                 Name::Namespace,
+                 Text,
+                 Keyword::Namespace
+        end
+
+        rule %r/(cimport)(\s+)(#{dotted_identifier})/ do
+          groups Keyword::Namespace, Text, Name::Namespace
+        end
+
+        rule %r/(struct)((?:\\\s|\s)+)/ do
+          groups Keyword, Text
+          push :classname
+        end
+
+        mixin :func_call_fix
+
+        rule %r/[(,]/, Punctuation, :c_start
+      end
+
+      prepend :classname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+
+      prepend :funcname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+      # This is a fix for the way that function calls are lexed in the Python
+      # lexer. This should be moved to the Python lexer once confirmed that it
+      # does not cause any regressions.
+      state :func_call_fix do
+        rule %r/#{identifier}(?=\()/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+          elsif self.class.exceptions.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins_pseudo.include? m[0]
+            token Name::Builtin::Pseudo
+          else
+            token Name::Function
+          end
+        end
+      end
+
+      # The Cython lexer adds three states to those already in the Python lexer.
+      # Calls to `cdef`, `cpdef` and `ctypedef` move the lexer into the :c_start
+      # state. The primary purpose of this state is to highlight datatypes. Once
+      # this has been done, the lexer moves to the :c_definitions state where
+      # the majority of text in a definition is lexed. Finally, newlines cause
+      # the lexer to move to :c_indent. This state is used to check whether we
+      # have moved out of a C block.
+
+      state :c_start do
+        rule %r/[^\S\n]+/, Text
+
+        rule %r/cp?def|ctypedef/, Keyword
+
+        rule %r/(?:un)?signed/, Keyword::Type
+
+        # This rule matches identifiers that could be type declarations. The
+        # lookahead matches (1) pointers, (2) arrays and (3) variable names.
+        rule %r/#{identifier}(?=(?:\*+)|(?:[ \t]*\[)|(?:[ \t]+\w))/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+            pop!
+          elsif %w(def).include? m[0]
+            token Keyword
+            goto :funcname
+          elsif %w(struct class).include? m[0]
+            token Keyword::Reserved
+            goto :classname
+          elsif self.class.c_keywords.include? m[0]
+            token Keyword::Reserved
+          else
+            token Keyword::Type
+            pop!
+          end
+        end
+
+        rule(//) { pop! }
+      end
+
+      state :c_definitions do
+        rule %r/\n/, Text, :c_indent
+        mixin :root
+      end
+
+      state :c_indent do
+        rule %r/[ \t]+/ do |m|
+          token Text
+          goto :c_start
+
+          if @indentation.nil?
+            @indentation = m[0]
+          elsif @indentation.length > m[0].length
+            @indentation = nil
+            pop! 2 # Pop :c_start and :c_definitions
+          end
+        end
+
+        rule(//) { @indentation = nil; reset_stack }
+      end
+    end
+  end
+end