rouge 3.5.1 → 3.10.0

data/lib/rouge/lexers/ceylon.rb

@@ -31,14 +31,14 @@ module Rouge
         rule %r((abstracts|extends|satisfies|super|given|of|out|assign)\b), Keyword::Declaration
 
         rule %r((function|value|void|new)\b), Keyword::Type
-        
+
         rule %r((assembly|module|package)(\s+)) do
           groups Keyword::Namespace, Text
           push :import
         end
-        
+
         rule %r((true|false|null)\b), Keyword::Constant
-        
+
         rule %r((class|interface|object|alias)(\s+)) do
           groups Keyword::Declaration, Text
           push :class
@@ -48,7 +48,7 @@ module Rouge
           groups Keyword::Namespace, Text
           push :import
         end
-        
+
         rule %r("(\\\\|\\"|[^"])*"), Literal::String
         rule %r('\\.'|'[^\\]'|'\\\{#[0-9a-fA-F]{4}\}'), Literal::String::Char
         rule %r(".*``.*``.*"', String::Interpol
@@ -84,7 +84,7 @@ module Rouge
         rule %r([a-z][\w.]*), Name::Namespace, :pop!
         rule %r("(\\\\|\\"|[^"])*"), Literal::String, :pop!
       end
-      
+
       state :comment do
         rule %r([^*/]), Comment.Multiline
         rule %r(/\*), Comment::Multiline, :push!

data/lib/rouge/lexers/clean.rb

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Clean < RegexLexer
+      title "Clean"
+      desc "The Clean programming language (clean.cs.ru.nl)"
+
+      tag 'clean'
+      filenames '*.dcl', '*.icl'
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          if otherwise
+          let in
+          with where
+          case of
+          infix infixl infixr
+          class instance
+          generic derive
+          special
+          implementation definition system module
+          from import qualified as
+          dynamic
+          code inline foreign export ccall stdcall
+        )
+      end
+
+      # These are literal patterns common to the ABC intermediate language and
+      # Clean. Clean has more extensive literal patterns (see :basic below).
+      state :common_literals do
+        rule %r/'(?:[^'\\]|\\(?:x[0-9a-fA-F]+|\d+|.))'/, Str::Char
+
+        rule %r/[+~-]?\d+\.\d+(?:E[+-]?\d+)?\b/, Num::Float
+        rule %r/[+~-]?\d+E[+-]?\d+\b/, Num::Float
+        rule %r/[+~-]?\d+/, Num::Integer
+
+        rule %r/"/, Str::Double, :string
+      end
+
+      state :basic do
+        rule %r/\s+/m, Text::Whitespace
+
+        rule %r/\/\/\*.*/, Comment::Doc
+        rule %r/\/\/.*/, Comment::Single
+        rule %r/\/\*\*/, Comment::Doc, :comment_doc
+        rule %r/\/\*/, Comment::Multiline, :comment
+
+        rule %r/[+~-]?0[0-7]+/, Num::Oct
+        rule %r/[+~-]?0x[0-9a-fA-F]+/, Num::Hex
+        mixin :common_literals
+        rule %r/(\[)(\s*)(')(?=.*?'\])/ do
+          groups Punctuation, Text::Whitespace, Str::Single, Punctuation
+          push :charlist
+        end
+      end
+
+      # nested commenting
+      state :comment_doc do
+        rule %r/\*\//, Comment::Doc, :pop!
+        rule %r/\/\/.*/, Comment::Doc # Singleline comments in multiline comments are skipped
+        rule %r/\/\*/, Comment::Doc, :comment
+        rule %r/[^*\/]+/, Comment::Doc
+        rule %r/[*\/]/, Comment::Doc
+      end
+
+      # This is the same as the above, but with Multiline instead of Doc
+      state :comment do
+        rule %r/\*\//, Comment::Multiline, :pop!
+        rule %r/\/\/.*/, Comment::Multiline # Singleline comments in multiline comments are skipped
+        rule %r/\/\*/, Comment::Multiline, :comment
+        rule %r/[^*\/]+/, Comment::Multiline
+        rule %r/[*\/]/, Comment::Multiline
+      end
+
+      state :root do
+        mixin :basic
+
+        rule %r/code(\s+inline)?\s*{/, Comment::Preproc, :abc
+
+        rule %r/_*[a-z][\w_`]*/ do |m|
+          if self.class.keywords.include?(m[0])
+            token Keyword
+          else
+            token Name
+          end
+        end
+
+        rule %r/_*[A-Z][\w_`]*/ do |m|
+          if m[0]=='True' || m[0]=='False'
+            token Keyword::Constant
+          else
+            token Keyword::Type
+          end
+        end
+
+        rule %r/[^\w_\s`]/, Punctuation
+        rule %r/_\b/, Punctuation
+      end
+
+      state :escapes do
+        rule %r/\\x[0-9a-fA-F]{1,2}/i, Str::Escape
+        rule %r/\\d\d{0,3}/i, Str::Escape
+        rule %r/\\0[0-7]{0,3}/, Str::Escape
+        rule %r/\\[0-7]{1,3}/, Str::Escape
+        rule %r/\\[nrfbtv\\"']/, Str::Escape
+      end
+
+      state :string do
+        rule %r/"/, Str::Double, :pop!
+        mixin :escapes
+        rule %r/[^\\"]+/, Str::Double
+      end
+
+      state :charlist do
+        rule %r/(')(\])/ do
+          groups Str::Single, Punctuation
+          pop!
+        end
+        mixin :escapes
+        rule %r/[^\\']/, Str::Single
+      end
+
+      state :abc_basic do
+        rule %r/\s+/, Text::Whitespace
+        rule %r/\|.*/, Comment::Single
+        mixin :common_literals
+      end
+
+      # The ABC intermediate language can be included, similar to C's inline
+      # assembly. For some information about ABC, see:
+      # https://en.wikipedia.org/wiki/Clean_(programming_language)#The_ABC-Machine
+      state :abc do
+        mixin :abc_basic
+
+        rule %r/}/, Comment::Preproc, :pop!
+        rule %r/\.\w*/, Keyword, :abc_rest_of_line
+        rule %r/[\w_]+/, Name::Builtin, :abc_rest_of_line
+      end
+
+      state :abc_rest_of_line do
+        rule %r/\n/, Text::Whitespace, :pop!
+        rule %r/}/ do
+          token Comment::Preproc
+          pop!
+          pop!
+        end
+
+        mixin :abc_basic
+
+        rule %r/\S+/, Name
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cmhg.rb

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class CMHG < RegexLexer
+      title "CMHG"
+      desc "RISC OS C module header generator source file"
+      tag 'cmhg'
+      filenames '*.cmhg'
+
+      def self.preproc_keyword
+        @preproc_keyword ||= %w(
+          define elif else endif error if ifdef ifndef include line pragma undef warning
+        )
+      end
+
+      state :root do
+        rule %r/;[^\n]*/, Comment
+        rule %r/^([ \t]*)(#[ \t]*(?:(?:#{CMHG.preproc_keyword.join('|')})(?:[ \t].*)?)?)(?=\n)/ do
+          groups Text, Comment::Preproc
+        end
+        rule %r/[-a-z]+:/, Keyword::Declaration
+        rule %r/[a-z_]\w+/i, Name::Entity
+        rule %r/"[^"]*"/, Literal::String
+        rule %r/(?:&|0x)\h+/, Literal::Number::Hex
+        rule %r/\d+/, Literal::Number
+        rule %r/[,\/()]/, Punctuation
+        rule %r/[ \t]+/, Text
+        rule %r/\n+/, Text
+      end
+    end
+  end
+end

data/lib/rouge/lexers/common_lisp.rb

@@ -7,7 +7,7 @@ module Rouge
       title "Common Lisp"
       desc "The Common Lisp variant of Lisp (common-lisp.net)"
       tag 'common_lisp'
-      aliases 'cl', 'common-lisp', 'elisp', 'emacs-lisp'
+      aliases 'cl', 'common-lisp', 'elisp', 'emacs-lisp', 'lisp'
 
       filenames '*.cl', '*.lisp', '*.asd', '*.el' # used for Elisp too
       mimetypes 'text/x-common-lisp'

data/lib/rouge/lexers/console.rb

@@ -116,7 +116,7 @@ module Rouge
           # before we pass to the lang lexer so it can determine where
           # the "real" beginning of the line is
           $' =~ /\A\s*/
-          yield Text, $& unless $&.empty?
+          yield Text::Whitespace, $& unless $&.empty?
 
           lang_lexer.continue_lex($', &output)
         elsif comment_regex =~ input[0].strip

data/lib/rouge/lexers/cpp.rb

@@ -17,7 +17,7 @@ module Rouge
                 '*.cc',  '*.hh',
                 '*.cxx', '*.hxx',
                 '*.pde', '*.ino',
-                '*.tpp'
+                '*.tpp', '*.h'
       mimetypes 'text/x-c++hdr', 'text/x-c++src'
 
       def self.keywords
@@ -58,9 +58,11 @@ module Rouge
 
       prepend :statements do
         rule %r/class\b/, Keyword, :classname
+        rule %r/\d+(\.\d+)?(?:h|(?:min)|s|(?:ms)|(?:us)|(?:ns))/, Num::Other
         rule %r((#{dq}[.]#{dq}?|[.]#{dq})(e[+-]?#{dq}[lu]*)?)i, Num::Float
         rule %r(#{dq}e[+-]?#{dq}[lu]*)i, Num::Float
         rule %r/0x\h('?\h)*[lu]*/i, Num::Hex
+        rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
         rule %r/0[0-7]('?[0-7])*[lu]*/i, Num::Oct
         rule %r/#{dq}[lu]*/i, Num::Integer
         rule %r/\bnullptr\b/, Name::Builtin
@@ -72,6 +74,7 @@ module Rouge
 
         # template specification
         rule %r/\s*(?=>)/m, Text, :pop!
+        rule %r/[.]{3}/, Operator
         mixin :whitespace
       end
     end

data/lib/rouge/lexers/cuda.rb

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    load_lexer 'cpp.rb'
+
+    class CUDA < Cpp
+      title "CUDA"
+      desc "Compute Unified Device Architecture, used for programming with NVIDIA GPU"
+
+      tag 'cuda'
+      filenames '*.cu', '*.cuh'
+
+      def self.keywords
+        @keywords ||= super + Set.new(%w(
+          __global__ __device__ __host__ __noinline__ __forceinline__
+          __constant__ __shared__ __managed__ __restrict__
+        ))
+      end
+
+      def self.keywords_type
+        @keywords_type ||= super + Set.new(%w(
+          char1 char2 char3 char4 uchar1 uchar2 uchar3 uchar4
+          short1 short2 short3 short4 ushort1 ushort2 ushort3 ushort4
+          int1 int2 int3 int4 uint1 uint2 uint3 uint4
+          long1 long2 long3 long4 ulong1 ulong2 ulong3 ulong4
+          longlong1 longlong2 longlong3 longlong4 
+          ulonglong1 ulonglong2 ulonglong3 ulonglong4 
+          float1 float2 float3 float4 double1 double2 double3 double4
+          dim3
+        ))
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cython.rb

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'python.rb'
+
+    class Cython < Python
+      title "Cython"
+      desc "Cython and Pyrex source code (cython.org)"
+      tag 'cython'
+      aliases 'pyx', 'pyrex'
+      filenames '*.pyx', '*.pxd', '*.pxi'
+      mimetypes 'text/x-cython', 'application/x-cython'
+
+      def initialize(opts = {})
+        super opts
+        @indentation = nil
+      end
+
+      def self.keywords
+        @keywords ||= super + %w(
+          by except? fused gil nogil
+        )
+      end
+
+      def self.c_keywords
+        @ckeywords ||= %w(
+          public readonly extern api inline enum union
+        )
+      end
+
+      identifier = /[a-z_]\w*/i
+      dotted_identifier = /[a-z_.][\w.]*/i
+
+      prepend :root do
+        rule %r/cp?def|ctypedef/ do
+          token Keyword
+          push :c_definitions
+          push :c_start
+        end
+
+        rule %r/(from)((?:\\\s|\s)+)(#{dotted_identifier})((?:\\\s|\s)+)(cimport)/ do
+          groups Keyword::Namespace,
+                 Text,
+                 Name::Namespace,
+                 Text,
+                 Keyword::Namespace
+        end
+
+        rule %r/(cimport)(\s+)(#{dotted_identifier})/ do
+          groups Keyword::Namespace, Text, Name::Namespace
+        end
+
+        rule %r/(struct)((?:\\\s|\s)+)/ do
+          groups Keyword, Text
+          push :classname
+        end
+
+        mixin :func_call_fix
+
+        rule %r/[(,]/, Punctuation, :c_start
+      end
+
+      prepend :classname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+
+      prepend :funcname do
+        rule %r/(?:\\\s|\s)+/, Text
+      end
+      # This is a fix for the way that function calls are lexed in the Python
+      # lexer. This should be moved to the Python lexer once confirmed that it
+      # does not cause any regressions.
+      state :func_call_fix do
+        rule %r/#{identifier}(?=\()/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+          elsif self.class.exceptions.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins.include? m[0]
+            token Name::Builtin
+          elsif self.class.builtins_pseudo.include? m[0]
+            token Name::Builtin::Pseudo
+          else
+            token Name::Function
+          end
+        end
+      end
+
+      # The Cython lexer adds three states to those already in the Python lexer.
+      # Calls to `cdef`, `cpdef` and `ctypedef` move the lexer into the :c_start
+      # state. The primary purpose of this state is to highlight datatypes. Once
+      # this has been done, the lexer moves to the :c_definitions state where
+      # the majority of text in a definition is lexed. Finally, newlines cause
+      # the lexer to move to :c_indent. This state is used to check whether we
+      # have moved out of a C block.
+
+      state :c_start do
+        rule %r/[^\S\n]+/, Text
+
+        rule %r/cp?def|ctypedef/, Keyword
+
+        rule %r/(?:un)?signed/, Keyword::Type
+
+        # This rule matches identifiers that could be type declarations. The
+        # lookahead matches (1) pointers, (2) arrays and (3) variable names.
+        rule %r/#{identifier}(?=(?:\*+)|(?:[ \t]*\[)|(?:[ \t]+\w))/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+            pop!
+          elsif %w(def).include? m[0]
+            token Keyword
+            goto :funcname
+          elsif %w(struct class).include? m[0]
+            token Keyword::Reserved
+            goto :classname
+          elsif self.class.c_keywords.include? m[0]
+            token Keyword::Reserved
+          else
+            token Keyword::Type
+            pop!
+          end
+        end
+
+        rule(//) { pop! }
+      end
+
+      state :c_definitions do
+        rule %r/\n/, Text, :c_indent
+        mixin :root
+      end
+
+      state :c_indent do
+        rule %r/[ \t]+/ do |m|
+          token Text
+          goto :c_start
+
+          if @indentation.nil?
+            @indentation = m[0]
+          elsif @indentation.length > m[0].length
+            @indentation = nil
+            pop! 2 # Pop :c_start and :c_definitions
+          end
+        end
+
+        rule(//) { @indentation = nil; reset_stack }
+      end
+    end
+  end
+end