rouge 3.7.0 → 3.8.0

data/lib/rouge/lexers/bbcbasic.rb

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class BBCBASIC < RegexLexer
+      title "BBCBASIC"
+      desc "BBC BASIC syntax"
+      tag 'bbcbasic'
+      filenames '*,fd1'
+
+      def self.punctuation
+        @punctuation ||= %w(
+          [,;'~] SPC TAB
+        )
+      end
+
+      def self.function
+        @function ||= %w(
+          ABS ACS ADVAL ASC ASN ATN BEATS BEAT BGET# CHR\$ COS COUNT DEG DIM
+          EOF# ERL ERR EVAL EXP EXT# FN GET\$# GET\$ GET HIMEM INKEY\$ INKEY
+          INSTR INT LEFT\$ LEN LN LOG LOMEM MID\$ OPENIN OPENOUT OPENUP PAGE
+          POINT POS PTR# RAD REPORT\$ RIGHT\$ RND SGN SIN SQR STR\$ STRING\$ SUM
+          SUMLEN TAN TEMPO TIME\$ TIME TOP USR VAL VPOS
+        )
+      end
+
+      def self.statement
+        @statement ||= %w(
+          BEATS BPUT# CALL CASE CHAIN CLEAR CLG CLOSE# CLS COLOR COLOUR DATA
+          ELSE ENDCASE ENDIF ENDPROC ENDWHILE END ENVELOPE FOR GCOL GOSUB GOTO
+          IF INSTALL LET LIBRARY MODE NEXT OFF OF ON ORIGIN OSCI OTHERWISE
+          OVERLAY PLOT PRINT# PRINT PROC QUIT READ REPEAT REPORT RETURN SOUND
+          STEP STEREO STOP SWAP SYS THEN TINT TO VDU VOICES VOICE UNTIL WAIT
+          WHEN WHILE WIDTH
+        )
+      end
+
+      def self.operator
+        @operator ||= %w(
+          << <= <> < >= >>> >> > [-!$()*+/=?^|] AND DIV EOR MOD NOT OR
+        )
+      end
+
+      def self.constant
+        @constant ||= %w(
+          FALSE TRUE
+        )
+      end
+
+      state :expression do
+        rule %r/#{BBCBASIC.function.join('|')}/o, Name::Builtin  # function or pseudo-variable
+        rule %r/#{BBCBASIC.operator.join('|')}/o, Operator
+        rule %r/#{BBCBASIC.constant.join('|')}/o, Name::Constant
+        rule %r/"[^"]*"/o, Literal::String
+        rule %r/[a-z_`][\w`]*[$%]?/io, Name::Variable
+        rule %r/@%/o, Name::Variable
+        rule %r/[\d.]+/o, Literal::Number
+        rule %r/%[01]+/o, Literal::Number::Bin
+        rule %r/&[\h]+/o, Literal::Number::Hex
+      end
+
+      state :root do
+        rule %r/(:+)( *)(\*)(.*)/ do
+          groups Punctuation, Text, Keyword, Text # CLI command
+        end
+        rule %r/(\n+ *)(\*)(.*)/ do
+          groups Text, Keyword, Text # CLI command
+        end
+        rule %r/(ELSE|OTHERWISE|REPEAT|THEN)( *)(\*)(.*)/ do
+          groups Keyword, Text, Keyword, Text # CLI command
+        end
+        rule %r/[ \n]+/o, Text
+        rule %r/:+/o, Punctuation
+        rule %r/[\[]/o, Keyword, :assembly1
+        rule %r/REM *>.*/o, Comment::Special
+        rule %r/REM.*/o, Comment
+        rule %r/(?:#{BBCBASIC.statement.join('|')}|CIRCLE(?: *FILL)?|DEF *(?:FN|PROC)|DRAW(?: *BY)?|DIM(?!\()|ELLIPSE(?: *FILL)?|ERROR(?: *EXT)?|FILL(?: *BY)?|INPUT(?:#| *LINE)?|LINE(?: *INPUT)?|LOCAL(?: *DATA| *ERROR)?|MOUSE(?: *COLOUR| *OFF| *ON| *RECTANGLE| *STEP| *TO)?|MOVE(?: *BY)?|ON(?! *ERROR)|ON *ERROR *(?:LOCAL|OFF)?|POINT(?: *BY)?(?!\()|RECTANGE(?: *FILL)?|RESTORE(?: *DATA| *ERROR)?|TRACE(?: *CLOSE| *ENDPROC| *OFF| *STEP(?: *FN| *ON| *PROC)?| *TO)?)/o, Keyword
+        mixin :expression
+        rule %r/#{BBCBASIC.punctuation.join('|')}/o, Punctuation
+      end
+
+      # Assembly statements are parsed as
+      # {label} {directive|opcode |']' {expressions}} {comment}
+      # Technically, you don't need whitespace between opcodes and arguments,
+      # but this is rare in uncrunched source and trying to enumerate all
+      # possible opcodes here is impractical so we colour it as though
+      # the whitespace is required. Opcodes and directives can only easily be
+      # distinguished from the symbols that make up expressions by looking at
+      # their position within the statement. Similarly, ']' is treated as a
+      # keyword at the start of a statement or as punctuation elsewhere. This
+      # requires a two-state state machine.
+
+      state :assembly1 do
+        rule %r/ +/o, Text
+        rule %r/]/o, Keyword, :pop!
+        rule %r/[:\n]/o, Punctuation
+        rule %r/\.[a-z_`][\w`]*%? */io, Name::Label
+        rule %r/(?:REM|;)[^:\n]*/o, Comment
+        rule %r/[^ :\n]+/o, Keyword, :assembly2
+      end
+
+      state :assembly2 do
+        rule %r/ +/o, Text
+        rule %r/[:\n]/o, Punctuation, :pop!
+        rule %r/(?:REM|;)[^:\n]*/o, Comment, :pop!
+        mixin :expression
+        rule %r/[!#,@\[\]^{}]/, Punctuation
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cmhg.rb

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class CMHG < RegexLexer
+      title "CMHG"
+      desc "RISC OS C module header generator source file"
+      tag 'cmhg'
+      filenames '*.cmhg'
+
+      def self.preproc_keyword
+        @preproc_keyword ||= %w(
+          define elif else endif error if ifdef ifndef include line pragma undef warning
+        )
+      end
+
+      state :root do
+        rule %r/;[^\n]*/, Comment
+        rule %r/^([ \t]*)(#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})(?:[ \t].*)?)?)(?=\n)/ do
+          groups Text, Comment::Preproc
+        end
+        rule %r/[-a-z]+:/, Keyword::Declaration
+        rule %r/[a-z_]\w+/i, Name::Entity
+        rule %r/"[^"]*"/, Literal::String
+        rule %r/(?:&|0x)\h+/, Literal::Number::Hex
+        rule %r/\d+/, Literal::Number
+        rule %r/[,\/()]/, Punctuation
+        rule %r/[ \t]+/, Text
+        rule %r/\n+/, Text
+      end
+    end
+  end
+end

data/lib/rouge/lexers/console.rb

@@ -116,7 +116,7 @@ module Rouge
           # before we pass to the lang lexer so it can determine where
           # the "real" beginning of the line is
           $' =~ /\A\s*/
-          yield Text, $& unless $&.empty?
+          yield Text::Whitespace, $& unless $&.empty?
 
           lang_lexer.continue_lex($', &output)
         elsif comment_regex =~ input[0].strip

data/lib/rouge/lexers/cpp.rb

@@ -17,7 +17,7 @@ module Rouge
                 '*.cc',  '*.hh',
                 '*.cxx', '*.hxx',
                 '*.pde', '*.ino',
-                '*.tpp'
+                '*.tpp', '*.h'
       mimetypes 'text/x-c++hdr', 'text/x-c++src'
 
       def self.keywords
@@ -58,9 +58,11 @@ module Rouge
 
       prepend :statements do
         rule %r/class\b/, Keyword, :classname
+        rule %r/\d+(\.\d+)?(?:h|(?:min)|s|(?:ms)|(?:us)|(?:ns))/, Num::Other
         rule %r((#{dq}[.]#{dq}?|[.]#{dq})(e[+-]?#{dq}[lu]*)?)i, Num::Float
         rule %r(#{dq}e[+-]?#{dq}[lu]*)i, Num::Float
         rule %r/0x\h('?\h)*[lu]*/i, Num::Hex
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
         rule %r/0[0-7]('?[0-7])*[lu]*/i, Num::Oct
         rule %r/#{dq}[lu]*/i, Num::Integer
         rule %r/\bnullptr\b/, Name::Builtin
@@ -72,6 +74,7 @@ module Rouge
 
         # template specification
         rule %r/\s*(?=>)/m, Text, :pop!
+        rule %r/[.]{3}/, Operator
         mixin :whitespace
       end
     end

data/lib/rouge/lexers/cython.rb

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'python.rb'
+
+    class Cython < Python
+      title "Cython"
+      desc "Cython and Pyrex source code (cython.org)"
+      tag 'cython'
+      aliases 'pyx', 'pyrex'
+      filenames '*.pyx', '*.pxd', '*.pxi'
+      mimetypes 'text/x-cython', 'application/x-cython'
+
+      def initialize(opts = {})
+        super opts
+        @indentation = nil
+      end
+
+      def self.keywords
+        @keywords ||= super + %w(
+          by except? fused gil nogil
+        )
+      end
+
+      def self.c_keywords
+        @ckeywords ||= %w(
+          public readonly extern api inline enum union
+        )
+      end
+
+      identifier = /[a-z_]\w*/i
+      dotted_identifier = /[a-z_.][\w.]*/i
+
+      prepend :root do
+        rule %r/cp?def|ctypedef/ do
+          token Keyword
+          push :c_definitions
+          push :c_start
+        end
+
+        rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(cimport)/ do
+          groups Keyword::Namespace,
+                 Text,
+                 Name::Namespace,
+                 Text,
+                 Keyword::Namespace
+        end
+
+        rule %r/(cimport)(\s+)(#{dotted_identifier})/ do
+          groups Keyword::Namespace, Text, Name::Namespace
+        end
+
+        rule %r/(struct)((?:\\\s|\s)+)/ do
+          groups Keyword, Text
+          push :classname
+        end
+
+        mixin :func_call_fix
+
+        rule %r/[(,]/, Punctuation, :c_start
+      end
+
+      prepend :classname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+
+      prepend :funcname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+      # This is a fix for the way that function calls are lexed in the Python
+      # lexer. This should be moved to the Python lexer once confirmed that it
+      # does not cause any regressions.
+      state :func_call_fix do
+        rule %r/#{identifier}(?=\()/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+          elsif self.class.exceptions.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins_pseudo.include? m[0]
+            token Name::Builtin::Pseudo
+          else
+            token Name::Function
+          end
+        end
+      end
+
+      # The Cython lexer adds three states to those already in the Python lexer.
+      # Calls to `cdef`, `cpdef` and `ctypedef` move the lexer into the :c_start
+      # state. The primary purpose of this state is to highlight datatypes. Once
+      # this has been done, the lexer moves to the :c_definitions state where
+      # the majority of text in a definition is lexed. Finally, newlines cause
+      # the lexer to move to :c_indent. This state is used to check whether we
+      # have moved out of a C block.
+
+      state :c_start do
+        rule %r/[^\S\n]+/, Text
+
+        rule %r/cp?def|ctypedef/, Keyword
+
+        rule %r/(?:un)?signed/, Keyword::Type
+
+        # This rule matches identifiers that could be type declarations. The
+        # lookahead matches (1) pointers, (2) arrays and (3) variable names.
+        rule %r/#{identifier}(?=(?:\*+)|(?:[ \t]*\[)|(?:[ \t]+\w))/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+            pop!
+          elsif %w(def).include? m[0]
+            token Keyword
+            goto :funcname
+          elsif %w(struct class).include? m[0]
+            token Keyword::Reserved
+            goto :classname
+          elsif self.class.c_keywords.include? m[0]
+            token Keyword::Reserved
+          else
+            token Keyword::Type
+            pop!
+          end
+        end
+
+        rule(//) { pop! }
+      end
+
+      state :c_definitions do
+        rule %r/\n/, Text, :c_indent
+        mixin :root
+      end
+
+      state :c_indent do
+        rule %r/[ \t]+/ do |m|
+          token Text
+          goto :c_start
+
+          if @indentation.nil?
+            @indentation = m[0]
+          elsif @indentation.length > m[0].length
+            @indentation = nil
+            pop! 2 # Pop :c_start and :c_definitions
+          end
+        end
+
+        rule(//) { @indentation = nil; reset_stack }
+      end
+    end
+  end
+end

data/lib/rouge/lexers/epp.rb

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class EPP < TemplateLexer
+      title "EPP"
+      desc "Embedded Puppet template files"
+
+      tag 'epp'
+
+      filenames '*.epp'
+
+      def initialize(opts={})
+        super(opts)
+        @parent = lexer_option(:parent) { PlainText.new(opts) }
+        @puppet_lexer = Puppet.new(opts)
+      end
+
+      start do
+        parent.reset!
+        @puppet_lexer.reset!
+      end
+
+      open  = /<%%|<%=|<%#|(<%-|<%)(\s*\|)?/
+      close = /%%>|(\|\s*)?(-%>|%>)/
+
+      state :root do
+        rule %r/<%#/, Comment, :comment
+
+        rule open, Comment::Preproc, :puppet
+
+        rule %r/.+?(?=#{open})|.+/m do
+          delegate parent
+        end
+      end
+
+      state :comment do
+        rule close, Comment, :pop!
+        rule %r/.+?(?=#{close})|.+/m, Comment
+      end
+
+      state :puppet do
+        rule close, Comment::Preproc, :pop!
+
+        rule %r/.+?(?=#{close})|.+/m do
+          delegate @puppet_lexer
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/json.rb

@@ -34,7 +34,7 @@ module Rouge
       end
 
       state :name do
-        rule %r/("(?:\"|[^"\n])*?")(\s*)(:)/ do
+        rule %r/("(?:\\.|[^"\n])*?")(\s*)(:)/ do
           groups Name::Label, Text::Whitespace, Punctuation
         end
       end

data/lib/rouge/lexers/julia.rb

@@ -199,7 +199,7 @@ module Rouge
         end
 
         # types
-        rule %r/((mutable )?struct|(abstract|primitive) type)((?:\s|\\\s)+)/ do
+        rule %r/((?:mutable )?struct|(?:abstract|primitive) type)((?:\s|\\\s)+)/ do
           groups Keyword, Name::Class
           push :typename
         end

data/lib/rouge/lexers/make.rb

@@ -8,9 +8,18 @@ module Rouge
       desc "Makefile syntax"
       tag 'make'
       aliases 'makefile', 'mf', 'gnumake', 'bsdmake'
-      filenames '*.make', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile'
+      filenames '*.make', '*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile', '*,fe1'
       mimetypes 'text/x-makefile'
 
+      def self.functions
+        @functions ||= %w(
+          abspath addprefix addsuffix and basename call dir error eval file
+          filter filter-out findstring firstword flavor foreach if join lastword
+          notdir or origin patsubst realpath shell sort strip subst suffix value
+          warning wildcard word wordlist words
+        )
+      end
+
       # TODO: Add support for special keywords
       # bsd_special = %w(
       #   include undef error warning if else elif endif for endfor
@@ -28,15 +37,25 @@ module Rouge
 
         rule %r/#.*?\n/, Comment
 
-        rule %r/(export)(\s+)(?=[a-zA-Z0-9_\${}\t -]+\n)/ do
+        rule %r/([-s]?include)((?:[\t ]+[^\t\n #]+)+)/ do
+          groups Keyword, Literal::String::Other
+        end
+
+        rule %r/(ifn?def|ifn?eq)([\t ]+)([^#\n]+)/ do
+          groups Keyword, Text, Name::Variable
+        end
+
+        rule %r/(?:else|endif)[\t ]*(?=[#\n])/, Keyword
+
+        rule %r/(export)([\t ]+)(?=[\w\${}()\t -]+\n)/ do
           groups Keyword, Text
           push :export
         end
 
-        rule %r/export\s+/, Keyword
+        rule %r/export[\t ]+/, Keyword
 
         # assignment
-        rule %r/([a-zA-Z0-9_${}.-]+)(\s*)([!?:+]?=)/m do |m|
+        rule %r/([\w${}().-]+)([\t ]*)([!?:+]?=)/m do |m|
           token Name::Variable, m[1]
           token Text, m[2]
           token Operator, m[3]
@@ -52,9 +71,9 @@ module Rouge
       end
 
       state :export do
-        rule %r/[\w\${}-]/, Name::Variable
+        rule %r/[\w\${}()-]/, Name::Variable
         rule %r/\n/, Text, :pop!
-        rule %r/\s+/, Text
+        rule %r/[\t ]+/, Text
       end
 
       state :block_header do
@@ -70,6 +89,14 @@ module Rouge
       end
 
       state :block_body do
+        rule %r/(ifn?def|ifn?eq)([\t ]+)([^#\n]+)(#.*)?(\n)/ do
+          groups Keyword, Text, Name::Variable, Comment, Text
+        end
+
+        rule %r/(else|endif)([\t ]*)(#.*)?(\n)/ do
+          groups Keyword, Text, Comment, Text
+        end
+
         rule %r/(\t[\t ]*)([@-]?)/ do
           groups Text, Punctuation
           push :shell_line
@@ -80,22 +107,22 @@ module Rouge
 
       state :shell do
         # macro interpolation
-        rule %r/\$\(\s*[a-z_]\w*\s*\)/i, Name::Variable
-        # $(shell ...)
-        rule %r/(\$\()(\s*)(shell)(\s+)/m do
+        rule %r/\$[({][\t ]*\w[\w:=%.]*[\t ]*[)}]/i, Name::Variable
+        # function invocation
+        rule %r/(\$[({])([\t ]*)(#{Make.functions.join('|')})([\t ]+)/m do
           groups Name::Function, Text, Name::Builtin, Text
           push :shell_expr
         end
 
         rule(/\\./m) { delegate @shell }
-        stop = /\$\(|\(|\)|\\|$/
+        stop = /\$\(|\$\{|\(|\)|\}|\\|$/
         rule(/.+?(?=#{stop})/m) { delegate @shell }
         rule(stop) { delegate @shell }
       end
 
       state :shell_expr do
-        rule(/\(/) { delegate @shell; push }
-        rule %r/\)/, Name::Variable, :pop!
+        rule(/[({]/) { delegate @shell; push }
+        rule %r/[)}]/, Name::Function, :pop!
         mixin :shell
       end