rouge 3.21.0 → 3.30.0

data/lib/rouge/lexers/isbl.rb

@@ -41,7 +41,7 @@ module Rouge
 
       state :dotted do
         mixin :whitespace
-        rule %r/[a-zа-яё_0-9]*/i do |m|
+        rule %r/[a-zа-яё_0-9]+/i do |m|
           name = m[0]
           if self.class.constants.include? name.downcase
             token Name::Builtin
@@ -56,7 +56,7 @@ module Rouge
 
       state :type do
         mixin :whitespace
-        rule %r/[a-zа-яё_0-9]*/i do |m|
+        rule %r/[a-zа-яё_0-9]+/i do |m|
           name = m[0]
           if self.class.interfaces.include? name.downcase
             token Keyword::Type

data/lib/rouge/lexers/j.rb

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class J < RegexLexer
+      title 'J'
+      desc "The J programming language (jsoftware.com)"
+      tag 'j'
+      filenames '*.ijs', '*.ijt'
+
+      # For J-specific terms we use, see:
+      #   https://code.jsoftware.com/wiki/Vocabulary/AET
+      #   https://code.jsoftware.com/wiki/Vocabulary/Glossary
+
+      # https://code.jsoftware.com/wiki/Vocabulary/PartsOfSpeech
+      def self.token_map
+        @token_map ||= {
+          noun: Keyword::Constant,
+          verb: Name::Function,
+          modifier: Operator,
+          name: Name,
+          param: Name::Builtin::Pseudo,
+          other: Punctuation,
+          nil => Error,
+        }
+      end
+
+      # https://code.jsoftware.com/wiki/NuVoc
+      def self.inflection_list
+        @inflection_list ||= ['', '.', ':', '..', '.:', ':.', '::']
+      end
+
+      def self.primitive_table
+        @primitive_table ||= Hash.new([:name]).tap do |h|
+          {
+            '()' => [:other],
+            '=' => [:verb, :other, :other],
+            '<>+-*%$|,#' => [:verb, :verb, :verb],
+            '^' => [:verb, :verb, :modifier],
+            '~"' => [:modifier, :verb, :verb],
+            '.:@' => [:modifier, :modifier, :modifier],
+            ';' => [:verb, :modifier, :verb],
+            '!' => [:verb, :modifier, :modifier],
+            '/\\' => [:modifier, :modifier, :verb],
+            '[' => [:verb, nil, :verb],
+            ']' => [:verb],
+            '{' => [:verb, :verb, :verb, nil, nil, nil, :verb],
+            '}' => [:modifier, :verb, :verb, nil, nil, nil, :modifier],
+            '`' => [:modifier, nil, :modifier],
+            '&' => [:modifier, :modifier, :modifier, nil, :modifier],
+            '?' => [:verb, :verb],
+            'a' => [:name, :noun, :noun],
+            'ACeEIjorv' => [:name, :verb],
+            'bdfHMT' => [:name, :modifier],
+            'Dt' => [:name, :modifier, :modifier],
+            'F' => [:name, :modifier, :modifier, :modifier, :modifier,
+                    :modifier, :modifier],
+            'iu' => [:name, :verb, :verb],
+            'L' => [:name, :verb, :modifier],
+            'mny' => [:param],
+            'p' => [:name, :verb, :verb, :verb],
+            'qsZ' => [:name, nil, :verb],
+            'S' => [:name, nil, :modifier],
+            'u' => [:param, :verb, :verb],
+            'v' => [:param, :verb],
+            'x' => [:param, nil, :verb],
+          }.each {|k, v| k.each_char {|c| h[c] = v } }
+        end
+      end
+
+      def self.primitive(char, inflection)
+        i = inflection_list.index(inflection) or return Error
+        token_map[primitive_table[char][i]]
+      end
+
+      def self.control_words
+        @control_words ||= Set.new %w(
+          assert break case catch catchd catcht continue do else elseif end
+          fcase for if return select throw try while whilst
+        )
+      end
+
+      def self.control_words_id
+        @control_words_id ||= Set.new %w(for goto label)
+      end
+
+      state :expr do
+        rule %r/\s+/, Text
+
+        rule %r'([!-&(-/:-@\[-^`{-~]|[A-Za-z]\b)([.:]*)' do |m|
+          token J.primitive(m[1], m[2])
+        end
+
+        rule %r/(?:\d|_\d?):([.:]*)/ do |m|
+          token m[1].empty? ? J.token_map[:verb] : Error
+        end
+
+        rule %r/[\d_][\w.]*([.:]*)/ do |m|
+          token m[1].empty? ? Num : Error
+        end
+
+        rule %r/'/, Str::Single, :str
+
+        rule %r/NB\.(?![.:]).*/, Comment::Single
+
+        rule %r/([A-Za-z]\w*)([.:]*)/ do |m|
+          if m[2] == '.'
+            word, sep, id = m[1].partition '_'
+            list = if sep.empty?
+              J.control_words
+            elsif not id.empty?
+              J.control_words_id
+            end
+            if list and list.include? word
+              token Keyword, word + sep
+              token((word == 'for' ? Name : Name::Label), id)
+              token Keyword, m[2]
+            else
+              token Error
+            end
+          else
+            token m[2].empty? ? Name : Error
+          end
+        end
+      end
+
+      state :str do
+        rule %r/''/, Str::Escape
+        rule %r/[^'\n]+/, Str::Single
+        rule %r/'|$/, Str::Single, :pop!
+      end
+
+      start do
+        @note_next = false
+      end
+
+      state :root do
+        rule %r/\n/ do
+          token Text
+          if @note_next
+            push :note
+            @note_next = false
+          end
+        end
+
+        # https://code.jsoftware.com/wiki/Vocabulary/com
+        # https://code.jsoftware.com/wiki/Vocabulary/NounExplicitDefinition
+        rule %r/
+          ([0-4]|13|adverb|conjunction|dyad|monad|noun|verb)([\ \t]+)
+          (def(?:ine)?\b|:)(?![.:])([\ \t]*)
+        /x do |m|
+          groups Keyword::Pseudo, Text, Keyword::Pseudo, Text
+          @def_body = (m[1] == '0' || m[1] == 'noun') ? :noun : :code
+          if m[3] == 'define'
+            # stack: [:root]
+            #    or  [:root, ..., :def_next]
+            pop! if stack.size > 1
+            push @def_body
+            push :def_next  # [:root, ..., @def_body, :def_next]
+          else
+            push :expl_def
+          end
+        end
+
+        rule %r/^([ \t]*)(Note\b(?![.:]))([ \t\r]*)(?!=[.:]|$)/ do
+          groups Text, Name, Text
+          @note_next = true
+        end
+
+        rule %r/[mnuvxy]\b(?![.:])/, Name
+        mixin :expr
+      end
+
+      state :def_next do
+        rule %r/\n/, Text, :pop!
+        mixin :root
+      end
+
+      state :expl_def do
+        rule %r/0\b(?![.:])/ do
+          token Keyword::Pseudo
+          # stack: [:root, :expl_def]
+          #    or  [:root, ..., :def_next, :expl_def]
+          pop! if stack.size > 2
+          goto @def_body
+          push :def_next  # [:root, ..., @def_body, :def_next]
+        end
+        rule %r/'/ do
+          if @def_body == :noun
+            token Str::Single
+            goto :str
+          else
+            token Punctuation
+            goto :q_expr
+          end
+        end
+        rule(//) { pop! }
+      end
+
+      # `q_expr` lexes the content of a string literal which is a part of an
+      # explicit definition.
+      # e.g. dyad def 'x + y'
+      state :q_expr do
+        rule %r/''/, Str::Single, :q_str
+        rule %r/'|$/, Punctuation, :pop!
+        rule %r/NB\.(?![.:])([^'\n]|'')*/, Comment::Single
+        mixin :expr
+      end
+
+      state :q_str do
+        rule %r/''''/, Str::Escape
+        rule %r/[^'\n]+/, Str::Single
+        rule %r/''/, Str::Single, :pop!
+        rule(/'|$/) { token Punctuation; pop! 2 }
+      end
+
+      state :note do
+        mixin :delimiter
+        rule %r/.+\n?/, Comment::Multiline
+      end
+
+      state :noun do
+        mixin :delimiter
+        rule %r/.+\n?/, Str::Heredoc
+      end
+
+      state :code do
+        mixin :delimiter
+        rule %r/^([ \t]*)(:)([ \t\r]*)$/ do
+          groups Text, Punctuation, Text
+        end
+        mixin :expr
+      end
+
+      state :delimiter do
+        rule %r/^([ \t]*)(\))([ \t\r]*$\n?)/ do
+          groups Text, Punctuation, Text
+          pop!
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/janet.rb

@@ -114,7 +114,7 @@ module Rouge
         end
       end
 
-      punctuation = %r/[_!@$%^&*+=~<>.?\/-]/o
+      punctuation = %r/[_!$%^&*+=~<>.?\/-]/o
       symbol = %r/([[:alpha:]]|#{punctuation})([[:word:]]|#{punctuation}|:)*/o
 
       state :root do
@@ -143,20 +143,19 @@ module Rouge
 
         rule %r/\(/, Punctuation, :function
 
-        rule %r/(')([\(\[])/ do
+        rule %r/(')(@?[(\[{])/ do
           groups Operator, Punctuation
           push :quote
         end
 
-        rule %r/(~)([\(\[])/ do
+        rule %r/(~)(@?[(\[{])/ do
           groups Operator, Punctuation
           push :quasiquote
         end
 
         rule %r/[\#~,';\|]/, Operator
 
-        rule %r/@?[({\[]/, Punctuation, :push
-        rule %r/[)}\]]/, Punctuation, :pop!
+        rule %r/@?[(){}\[\]]/, Punctuation
 
         rule symbol, Name
       end
@@ -169,6 +168,8 @@ module Rouge
       end
 
       state :function do
+        rule %r/[\)]/, Punctuation, :pop!
+
         rule symbol do |m|
           case m[0]
           when "quote"
@@ -187,8 +188,9 @@ module Rouge
       end
 
       state :quote do
-        rule %r/[\(\[]/, Punctuation, :push
-        rule symbol, Str::Symbol
+        rule %r/[(\[{]/, Punctuation, :push
+        rule %r/[)\]}]/, Punctuation, :pop!
+        rule symbol, Str::Escape
         mixin :root
       end
 
@@ -199,7 +201,6 @@ module Rouge
         end
         rule %r/(\()(\s*)(unquote)(\s+)(\()/ do
           groups Punctuation, Text, Keyword, Text, Punctuation
-          push :root
           push :function
         end
 

data/lib/rouge/lexers/javascript.rb

@@ -149,7 +149,7 @@ module Rouge
         rule %r/\A\s*#!.*?\n/m, Comment::Preproc, :statement
         rule %r((?<=\n)(?=\s|/|<!--)), Text, :expr_start
         mixin :comments_and_whitespace
-        rule %r(\+\+ | -- | ~ | && | \|\| | \\(?=\n) | << | >>>? | ===
+        rule %r(\+\+ | -- | ~ | \?\?=? | && | \|\| | \\(?=\n) | << | >>>? | ===
                | !== )x,
           Operator, :expr_start
         rule %r([-<>+*%&|\^/!=]=?), Operator, :expr_start
@@ -162,6 +162,10 @@ module Rouge
           push :template_string
         end
 
+        # special case for the safe navigation operator ?.
+        # so that we don't start detecting a ternary expr
+        rule %r/[?][.]/, Punctuation
+
         rule %r/[?]/ do
           token Punctuation
           push :ternary
@@ -265,9 +269,11 @@ module Rouge
 
       # template strings
       state :template_string do
-        rule %r/\${/, Punctuation, :template_string_expr
+        rule %r/[$]{/, Punctuation, :template_string_expr
         rule %r/`/, Str::Double, :pop!
-        rule %r/(\\\\|\\[\$`]|[^\$`]|\$(?!{))*/, Str::Double
+        rule %r/\\[$`]/, Str::Escape
+        rule %r/[^$`\\]+/, Str::Double
+        rule %r/[\\$]/, Str::Double
       end
 
       state :template_string_expr do

data/lib/rouge/lexers/jsl.rb

@@ -14,41 +14,52 @@ module Rouge
         rule %r/\s+/m, Text::Whitespace
 
         rule %r(//.*?$), Comment::Single
-        rule %r(/\*.*?\*/)m, Comment::Multiline
+        rule %r'/[*].*?', Comment::Multiline, :comment # multiline block comment
 
         # messages
-        rule %r/(<<)(.*?)(\(|;)/ do |m|
-          groups Operator, Name::Function, Punctuation
-        end
+        rule %r/<</, Operator, :message
 
         # covers built-in and custom functions
-        rule %r/([a-z_][\w\s'%.\\]*)(\()/i do |m|
-          groups Keyword, Punctuation
+        rule %r/(::|:)?([a-z_][\w\s'%.\\]*)(\()/i do |m|
+          groups Punctuation, Keyword, Punctuation
         end
 
-        rule %r/\b[+-]?(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+|\.)(?:e[+-]?[0-9]+)?i?\b/i, Num
-
         rule %r/\d{2}(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d{2}(\d{2})?(:\d{2}:\d{2}(:\d{2}(\.\d*)?)?)?/i, Literal::Date
 
-        rule %r/::[a-z_][\w\s'%.\\]*/i, Name::Variable
-        rule %r/:\w+/, Name
-        rule %r/[a-z_][\w\s'%.\\]*/i, Name::Variable
-        rule %r/"(?:\\!"|[^"])*?"n/m, Name::Variable
+        rule %r/-?(?:[0-9]+(?:[.][0-9]+)?|[.][0-9]*)(?:e[+-]?[0-9]+)?i?/i, Num
+
+        rule %r/(::|:)?([a-z_][\w\s'%.\\]*|"(?:\\!"|[^"])*?"n)/i do |m|
+          groups Punctuation, Name::Variable
+        end
 
         rule %r/(")(\\\[)(.*?)(\]\\)(")/m do
           groups Str::Double, Str::Escape, Str::Double, Str::Escape, Str::Double  # escaped string
         end
         rule %r/"/, Str::Double, :dq
 
-        rule %r/[-+*\/!%&<>\|=:]/, Operator
+        rule %r/[-+*\/!%&<>\|=:`^]/, Operator
         rule %r/[\[\](){},;]/, Punctuation
       end
 
+      state :message do
+        rule %r/\s+/m, Text::Whitespace
+        rule %r/[a-z_][\w\s'%.\\]*/i, Name::Function
+        rule %r/[(),;]/, Punctuation, :pop!
+        rule %r/[&|!=<>]/, Operator, :pop!
+      end
+
       state :dq do
         rule %r/\\![btrnNf0\\"]/, Str::Escape
         rule %r/\\/, Str::Double
         rule %r/"/, Str::Double, :pop!
-        rule %r/[^\\"]*/m, Str::Double
+        rule %r/[^\\"]+/m, Str::Double
+      end
+
+      state :comment do
+        rule %r'/[*]', Comment::Multiline, :comment
+        rule %r'[*]/', Comment::Multiline, :pop!
+        rule %r'[^/*]+', Comment::Multiline
+        rule %r'[/*]', Comment::Multiline
       end
     end
   end

data/lib/rouge/lexers/jsonnet.rb

@@ -126,6 +126,7 @@ module Rouge
 
       state :string do
         rule %r/\\([\\\/bfnrt]|(u[0-9a-fA-F]{4}))/, Str::Escape
+        rule %r/\\./, Str::Escape
       end
 
       state :string_double do
@@ -137,15 +138,15 @@ module Rouge
 
       state :string_single do
         mixin :string
-        rule %r/\\'/, Str::Escape
         rule %r/'/, Str, :pop!
         rule %r/[^\\']+/, Str
       end
 
       state :string_block do
         mixin :string
-        rule %r/\|\|\|/, Str, :pop!
-        rule %r/.*/, Str
+        rule %r/[|][|][|]/, Str, :pop!
+        rule %r/[^|\\]+/, Str
+        rule %r/[|]/, Str
       end
     end
   end

data/lib/rouge/lexers/jsp.rb

@@ -112,9 +112,8 @@ module Rouge
 
       state :jsp_interp_literal_start do
         rule %r/'/, Literal, :pop!
-        rule %r/[^']*/, Literal
+        rule %r/[^']+/, Literal
       end
-
     end
   end
-end
+end

data/lib/rouge/lexers/jsx.rb

@@ -57,7 +57,7 @@ module Rouge
           push :interpol
           push :expr_start
         end
-        rule %r/\w+/, Name::Attribute
+        rule %r/\w[\w-]*/, Name::Attribute
         rule %r/=/, Punctuation
         rule %r/(["']).*?(\1)/, Str
       end
@@ -89,4 +89,3 @@ module Rouge
     end
   end
 end
-

data/lib/rouge/lexers/kotlin.rb

@@ -53,7 +53,7 @@ module Rouge
           groups Keyword::Declaration, Text
           push :property
         end
-        rule %r'(return|continue|break|this|super)(@#{name})?' do
+        rule %r'(return|continue|break|this|super)(@#{name})?\b' do
           groups Keyword, Name::Decorator
         end
         rule %r'\bfun\b', Keyword
@@ -65,6 +65,9 @@ module Rouge
         rule %r'/[*].*[*]/', Comment::Multiline # single line block comment
         rule %r'/[*].*', Comment::Multiline, :comment # multiline block comment
         rule %r'\n', Text
+        rule %r'(::)(class)' do
+          groups Operator, Keyword
+        end
         rule %r'::|!!|\?[:.]', Operator
         rule %r"(\.\.)", Operator
         # Number literals
@@ -113,12 +116,14 @@ module Rouge
         rule class_name, Name::Class
         rule %r'(<)', Punctuation, :generic_parameters
         rule %r'(reified|out|in)', Keyword
-        rule %r'([,:])', Punctuation
+        rule %r'([,:.?])', Punctuation
         rule %r'(\s+)', Text
         rule %r'(>)', Punctuation, :pop!
       end
 
       state :property do
+        rule %r'(<)', Punctuation, :generic_parameters
+        rule %r'(\s+)', Text
         rule name, Name::Property, :pop!
       end
 
@@ -130,9 +135,10 @@ module Rouge
       end
 
       state :comment do
-        rule %r'\s*/[*].*', Comment::Multiline, :comment
-        rule %r'.*[*]/', Comment::Multiline, :pop!
-        rule %r'.*', Comment::Multiline
+        rule %r'/[*]', Comment::Multiline, :comment
+        rule %r'[*]/', Comment::Multiline, :pop!
+        rule %r'[^/*]+', Comment::Multiline
+        rule %r'[/*]', Comment::Multiline
       end
     end
   end

data/lib/rouge/lexers/lasso/keywords.rb

@@ -17,4 +17,4 @@ module Rouge
       end
     end
   end
-end
+end

data/lib/rouge/lexers/lean.rb

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+# NOTE: This is for Lean 3 (community fork).
+module Rouge
+  module Lexers
+    class Lean < RegexLexer
+      title 'Lean'
+      desc 'The Lean programming language (leanprover.github.io)'
+      tag 'lean'
+      aliases 'lean'
+      filenames '*.lean'
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          abbreviation
+          add_rewrite
+          alias
+          assume
+          axiom
+          begin
+          by
+          calc
+          calc_refl
+          calc_subst
+          calc_trans
+          #check
+          coercion
+          conjecture
+          constant
+          constants
+          context
+          corollary
+          def
+          definition
+          end
+          #eval
+          example
+          export
+          expose
+          exposing
+          exit
+          extends
+          from
+          fun
+          have
+          help
+          hiding
+          hott
+          hypothesis
+          import
+          include
+          including
+          inductive
+          infix
+          infixl
+          infixr
+          inline
+          instance
+          irreducible
+          lemma
+          match
+          namespace
+          notation
+          opaque
+          opaque_hint
+          open
+          options
+          parameter
+          parameters
+          postfix
+          precedence
+          prefix
+          #print
+          private
+          protected
+          #reduce
+          reducible
+          renaming
+          repeat
+          section
+          set_option
+          show
+          tactic_hint
+          theorem
+          universe
+          universes
+          using
+          variable
+          variables
+          with
+        )
+      end
+
+      def self.types
+        @types ||= %w(
+          Sort
+          Prop
+          Type
+        )
+      end
+
+      def self.operators
+        @operators ||= %w(
+          != # & && \* \+ - / @ ! ` -\. ->
+          \. \.\. \.\.\. :: :> ; ;; <
+          <- = == > _ \| \|\| ~ => <= >=
+          /\ \/ ∀ Π λ ↔ ∧ ∨ ≠ ≤ ≥ ⊎
+          ¬ ⁻¹ ⬝ ▸ → ∃ ℕ ℤ ≈ × ⌞ ⌟ ≡ ⟨ ⟩
+        )
+      end
+
+      state :root do
+        # comments starting after some space
+        rule %r/\s*--+\s+.*?$/, Comment::Doc
+
+        rule %r/"/, Str, :string
+        rule %r/\d+/, Num::Integer
+
+        # special commands or keywords
+        rule(/#?\w+/) do |m|
+          match = m[0]
+          if self.class.keywords.include?(match)
+            token Keyword
+          elsif self.class.types.include?(match)
+            token Keyword::Type
+          else
+            token Name
+          end
+        end
+
+        # special unicode keywords
+        rule %r/[λ]/, Keyword
+
+        # ----------------
+        # operators rules
+        # ----------------
+
+        rule %r/\:=?/, Text
+        rule %r/\.[0-9]*/, Operator
+
+        rule %r(#{Lean.operators.join('|')}), Operator
+
+        # unmatched symbols
+        rule %r/[\s\(\),\[\]αβ‹›]+/, Text
+
+        # show missing matches
+        rule %r/./, Error
+
+      end
+
+      state :string do
+        rule %r/"/, Str, :pop!
+        rule %r/\\/, Str::Escape, :escape
+        rule %r/[^\\"]+/, Str
+      end
+
+      state :escape do
+        rule %r/[nrt"'\\]/, Str::Escape, :pop!
+        rule %r/x[\da-f]+/i, Str::Escape, :pop!
+      end
+    end
+  end
+end